mirror of
https://github.com/FliegendeWurst/KIT-ILIAS-downloader.git
synced 2024-08-28 04:04:18 +00:00
Display a progress bar if possible
(experimental)
This commit is contained in:
parent
f291bcbb4b
commit
5c17ec0326
58
Cargo.lock
generated
58
Cargo.lock
generated
@ -7,14 +7,17 @@ name = "KIT-ILIAS-downloader"
|
|||||||
version = "0.2.16"
|
version = "0.2.16"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
"atty",
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"colored",
|
"colored",
|
||||||
"futures",
|
"futures",
|
||||||
"futures-channel",
|
"futures-channel",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"ignore",
|
"ignore",
|
||||||
|
"indicatif",
|
||||||
"keyring",
|
"keyring",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
|
"once_cell",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
@ -237,10 +240,25 @@ dependencies = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "const_fn"
|
name = "console"
|
||||||
version = "0.4.6"
|
version = "0.14.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "076a6803b0dacd6a88cfe64deba628b01533ff5ef265687e6938280c1afd0a28"
|
checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45"
|
||||||
|
dependencies = [
|
||||||
|
"encode_unicode",
|
||||||
|
"lazy_static",
|
||||||
|
"libc",
|
||||||
|
"regex",
|
||||||
|
"terminal_size",
|
||||||
|
"unicode-width",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "const_fn"
|
||||||
|
version = "0.4.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "convert_case"
|
name = "convert_case"
|
||||||
@ -417,6 +435,12 @@ version = "1.6.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
|
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "encode_unicode"
|
||||||
|
version = "0.3.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "encoding_rs"
|
name = "encoding_rs"
|
||||||
version = "0.8.28"
|
version = "0.8.28"
|
||||||
@ -797,6 +821,18 @@ dependencies = [
|
|||||||
"hashbrown",
|
"hashbrown",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "indicatif"
|
||||||
|
version = "0.15.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
|
||||||
|
dependencies = [
|
||||||
|
"console",
|
||||||
|
"lazy_static",
|
||||||
|
"number_prefix",
|
||||||
|
"regex",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "instant"
|
name = "instant"
|
||||||
version = "0.1.9"
|
version = "0.1.9"
|
||||||
@ -1048,6 +1084,12 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "number_prefix"
|
||||||
|
version = "0.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "once_cell"
|
name = "once_cell"
|
||||||
version = "1.7.2"
|
version = "1.7.2"
|
||||||
@ -1799,6 +1841,16 @@ dependencies = [
|
|||||||
"utf-8",
|
"utf-8",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "terminal_size"
|
||||||
|
version = "0.1.16"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "86ca8ced750734db02076f44132d802af0b33b09942331f4459dde8636fd2406"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "textwrap"
|
name = "textwrap"
|
||||||
version = "0.11.0"
|
version = "0.11.0"
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "KIT-ILIAS-downloader"
|
name = "KIT-ILIAS-downloader"
|
||||||
version = "0.2.16"
|
version = "0.2.16"
|
||||||
authors = ["FliegendeWurst <2012gdwu@web.de>"]
|
authors = ["FliegendeWurst <2012gdwu@posteo.de>"]
|
||||||
license = "GPL-2.0"
|
license = "GPL-2.0"
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
|
|
||||||
@ -28,6 +28,9 @@ anyhow = "1.0.28"
|
|||||||
colored = "2.0.0"
|
colored = "2.0.0"
|
||||||
keyring = { git = "https://github.com/FliegendeWurst/keyring-rs", branch = "application-name", optional = true }
|
keyring = { git = "https://github.com/FliegendeWurst/keyring-rs", branch = "application-name", optional = true }
|
||||||
cfg-if = "1.0.0"
|
cfg-if = "1.0.0"
|
||||||
|
indicatif = "0.15.0"
|
||||||
|
once_cell = "1.7.2"
|
||||||
|
atty = "0.2.14"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
|
31
src/main.rs
31
src/main.rs
@ -6,7 +6,9 @@ use futures::future::{self, Either};
|
|||||||
use futures_channel::mpsc::UnboundedSender;
|
use futures_channel::mpsc::UnboundedSender;
|
||||||
use futures_util::{stream::TryStreamExt, StreamExt};
|
use futures_util::{stream::TryStreamExt, StreamExt};
|
||||||
use ignore::gitignore::Gitignore;
|
use ignore::gitignore::Gitignore;
|
||||||
|
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
|
||||||
use lazy_static::lazy_static;
|
use lazy_static::lazy_static;
|
||||||
|
use once_cell::sync::Lazy;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use reqwest::{Client, Proxy};
|
use reqwest::{Client, Proxy};
|
||||||
use scraper::{ElementRef, Html, Selector};
|
use scraper::{ElementRef, Html, Selector};
|
||||||
@ -17,7 +19,7 @@ use tokio::task::{self, JoinHandle};
|
|||||||
use tokio_util::io::StreamReader;
|
use tokio_util::io::StreamReader;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
use std::future::Future;
|
use std::{future::Future, sync::atomic::AtomicBool};
|
||||||
use std::io;
|
use std::io;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
@ -30,12 +32,18 @@ use util::*;
|
|||||||
const ILIAS_URL: &str = "https://ilias.studium.kit.edu/";
|
const ILIAS_URL: &str = "https://ilias.studium.kit.edu/";
|
||||||
|
|
||||||
static LOG_LEVEL: AtomicUsize = AtomicUsize::new(0);
|
static LOG_LEVEL: AtomicUsize = AtomicUsize::new(0);
|
||||||
|
static PROGRESS_BAR_ENABLED: AtomicBool = AtomicBool::new(false);
|
||||||
|
static PROGRESS_BAR: Lazy<ProgressBar> = Lazy::new(|| ProgressBar::new(0));
|
||||||
|
|
||||||
macro_rules! log {
|
macro_rules! log {
|
||||||
($lvl:expr, $($t:expr),+) => {
|
($lvl:expr, $($t:expr),+) => {
|
||||||
#[allow(unused_comparisons)] // 0 <= 0
|
#[allow(unused_comparisons)] // 0 <= 0
|
||||||
if $lvl <= LOG_LEVEL.load(Ordering::SeqCst) {
|
if $lvl <= LOG_LEVEL.load(Ordering::SeqCst) {
|
||||||
println!($($t),+);
|
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
|
||||||
|
PROGRESS_BAR.println(format!($($t),+));
|
||||||
|
} else {
|
||||||
|
println!($($t),+);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -125,6 +133,13 @@ async fn main() {
|
|||||||
let (tx, mut rx) = futures_channel::mpsc::unbounded::<JoinHandle<()>>();
|
let (tx, mut rx) = futures_channel::mpsc::unbounded::<JoinHandle<()>>();
|
||||||
*TASKS.lock() = Some(tx.clone());
|
*TASKS.lock() = Some(tx.clone());
|
||||||
TASKS_RUNNING.add_permits(ilias.opt.jobs);
|
TASKS_RUNNING.add_permits(ilias.opt.jobs);
|
||||||
|
PROGRESS_BAR_ENABLED.store(atty::is(atty::Stream::Stdout), Ordering::SeqCst);
|
||||||
|
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
|
||||||
|
PROGRESS_BAR.set_draw_target(ProgressDrawTarget::stderr_nohz());
|
||||||
|
PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}+] {msg}"));
|
||||||
|
PROGRESS_BAR.inc_length(1);
|
||||||
|
PROGRESS_BAR.set_message("initializing..");
|
||||||
|
}
|
||||||
if let Some(url) = ilias.opt.sync_url.as_ref() {
|
if let Some(url) = ilias.opt.sync_url.as_ref() {
|
||||||
for item in ilias.get_course_content(&URL::from_href(url).expect("invalid URL")).await.expect("invalid response") {
|
for item in ilias.get_course_content(&URL::from_href(url).expect("invalid URL")).await.expect("invalid response") {
|
||||||
let item = item.expect("invalid item");
|
let item = item.expect("invalid item");
|
||||||
@ -161,6 +176,11 @@ async fn main() {
|
|||||||
warning!("could not disable content tree:", e);
|
warning!("could not disable content tree:", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
|
||||||
|
PROGRESS_BAR.inc(1);
|
||||||
|
PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}] {msg}"));
|
||||||
|
PROGRESS_BAR.finish_with_message("done");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
lazy_static! {
|
lazy_static! {
|
||||||
@ -225,6 +245,9 @@ fn process_gracefully(
|
|||||||
path: PathBuf,
|
path: PathBuf,
|
||||||
obj: Object,
|
obj: Object,
|
||||||
) -> impl Future<Output = ()> + Send { async move {
|
) -> impl Future<Output = ()> + Send { async move {
|
||||||
|
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
|
||||||
|
PROGRESS_BAR.inc_length(1);
|
||||||
|
}
|
||||||
let permit = TASKS_RUNNING.acquire().await.unwrap();
|
let permit = TASKS_RUNNING.acquire().await.unwrap();
|
||||||
let path_text = path.to_string_lossy().into_owned();
|
let path_text = path.to_string_lossy().into_owned();
|
||||||
if let Err(e) = process(ilias, path, obj).await.context("failed to process URL") {
|
if let Err(e) = process(ilias, path, obj).await.context("failed to process URL") {
|
||||||
@ -276,6 +299,10 @@ const NO_ENTRIES: &str = "Keine Einträge";
|
|||||||
|
|
||||||
async fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> Result<()> {
|
async fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> Result<()> {
|
||||||
let relative_path = path.strip_prefix(&ilias.opt.output).unwrap();
|
let relative_path = path.strip_prefix(&ilias.opt.output).unwrap();
|
||||||
|
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
|
||||||
|
PROGRESS_BAR.inc(1);
|
||||||
|
PROGRESS_BAR.set_message(&relative_path.display().to_string());
|
||||||
|
}
|
||||||
if ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() {
|
if ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() {
|
||||||
log!(1, "Ignored {}", relative_path.to_string_lossy());
|
log!(1, "Ignored {}", relative_path.to_string_lossy());
|
||||||
return Ok(());
|
return Ok(());
|
||||||
|
Loading…
Reference in New Issue
Block a user