Display a progress bar if possible

(experimental)
This commit is contained in:
FliegendeWurst 2021-04-21 20:18:54 +02:00
parent f291bcbb4b
commit 5c17ec0326
3 changed files with 88 additions and 6 deletions

58
Cargo.lock generated
View File

@ -7,14 +7,17 @@ name = "KIT-ILIAS-downloader"
version = "0.2.16" version = "0.2.16"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"atty",
"cfg-if", "cfg-if",
"colored", "colored",
"futures", "futures",
"futures-channel", "futures-channel",
"futures-util", "futures-util",
"ignore", "ignore",
"indicatif",
"keyring", "keyring",
"lazy_static", "lazy_static",
"once_cell",
"parking_lot", "parking_lot",
"regex", "regex",
"reqwest", "reqwest",
@ -237,10 +240,25 @@ dependencies = [
] ]
[[package]] [[package]]
name = "const_fn" name = "console"
version = "0.4.6" version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "076a6803b0dacd6a88cfe64deba628b01533ff5ef265687e6938280c1afd0a28" checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45"
dependencies = [
"encode_unicode",
"lazy_static",
"libc",
"regex",
"terminal_size",
"unicode-width",
"winapi",
]
[[package]]
name = "const_fn"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "402da840495de3f976eaefc3485b7f5eb5b0bf9761f9a47be27fe975b3b8c2ec"
[[package]] [[package]]
name = "convert_case" name = "convert_case"
@ -417,6 +435,12 @@ version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]] [[package]]
name = "encoding_rs" name = "encoding_rs"
version = "0.8.28" version = "0.8.28"
@ -797,6 +821,18 @@ dependencies = [
"hashbrown", "hashbrown",
] ]
[[package]]
name = "indicatif"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
dependencies = [
"console",
"lazy_static",
"number_prefix",
"regex",
]
[[package]] [[package]]
name = "instant" name = "instant"
version = "0.1.9" version = "0.1.9"
@ -1048,6 +1084,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "number_prefix"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
[[package]] [[package]]
name = "once_cell" name = "once_cell"
version = "1.7.2" version = "1.7.2"
@ -1799,6 +1841,16 @@ dependencies = [
"utf-8", "utf-8",
] ]
[[package]]
name = "terminal_size"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86ca8ced750734db02076f44132d802af0b33b09942331f4459dde8636fd2406"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "textwrap" name = "textwrap"
version = "0.11.0" version = "0.11.0"

View File

@ -1,7 +1,7 @@
[package] [package]
name = "KIT-ILIAS-downloader" name = "KIT-ILIAS-downloader"
version = "0.2.16" version = "0.2.16"
authors = ["FliegendeWurst <2012gdwu@web.de>"] authors = ["FliegendeWurst <2012gdwu@posteo.de>"]
license = "GPL-2.0" license = "GPL-2.0"
edition = "2018" edition = "2018"
@ -28,6 +28,9 @@ anyhow = "1.0.28"
colored = "2.0.0" colored = "2.0.0"
keyring = { git = "https://github.com/FliegendeWurst/keyring-rs", branch = "application-name", optional = true } keyring = { git = "https://github.com/FliegendeWurst/keyring-rs", branch = "application-name", optional = true }
cfg-if = "1.0.0" cfg-if = "1.0.0"
indicatif = "0.15.0"
once_cell = "1.7.2"
atty = "0.2.14"
[features] [features]
default = [] default = []

View File

@ -6,7 +6,9 @@ use futures::future::{self, Either};
use futures_channel::mpsc::UnboundedSender; use futures_channel::mpsc::UnboundedSender;
use futures_util::{stream::TryStreamExt, StreamExt}; use futures_util::{stream::TryStreamExt, StreamExt};
use ignore::gitignore::Gitignore; use ignore::gitignore::Gitignore;
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use once_cell::sync::Lazy;
use parking_lot::Mutex; use parking_lot::Mutex;
use reqwest::{Client, Proxy}; use reqwest::{Client, Proxy};
use scraper::{ElementRef, Html, Selector}; use scraper::{ElementRef, Html, Selector};
@ -17,7 +19,7 @@ use tokio::task::{self, JoinHandle};
use tokio_util::io::StreamReader; use tokio_util::io::StreamReader;
use url::Url; use url::Url;
use std::future::Future; use std::{future::Future, sync::atomic::AtomicBool};
use std::io; use std::io;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::atomic::Ordering; use std::sync::atomic::Ordering;
@ -30,15 +32,21 @@ use util::*;
const ILIAS_URL: &str = "https://ilias.studium.kit.edu/"; const ILIAS_URL: &str = "https://ilias.studium.kit.edu/";
static LOG_LEVEL: AtomicUsize = AtomicUsize::new(0); static LOG_LEVEL: AtomicUsize = AtomicUsize::new(0);
static PROGRESS_BAR_ENABLED: AtomicBool = AtomicBool::new(false);
static PROGRESS_BAR: Lazy<ProgressBar> = Lazy::new(|| ProgressBar::new(0));
macro_rules! log { macro_rules! log {
($lvl:expr, $($t:expr),+) => { ($lvl:expr, $($t:expr),+) => {
#[allow(unused_comparisons)] // 0 <= 0 #[allow(unused_comparisons)] // 0 <= 0
if $lvl <= LOG_LEVEL.load(Ordering::SeqCst) { if $lvl <= LOG_LEVEL.load(Ordering::SeqCst) {
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
PROGRESS_BAR.println(format!($($t),+));
} else {
println!($($t),+); println!($($t),+);
} }
} }
} }
}
macro_rules! info { macro_rules! info {
($t:tt) => { ($t:tt) => {
@ -125,6 +133,13 @@ async fn main() {
let (tx, mut rx) = futures_channel::mpsc::unbounded::<JoinHandle<()>>(); let (tx, mut rx) = futures_channel::mpsc::unbounded::<JoinHandle<()>>();
*TASKS.lock() = Some(tx.clone()); *TASKS.lock() = Some(tx.clone());
TASKS_RUNNING.add_permits(ilias.opt.jobs); TASKS_RUNNING.add_permits(ilias.opt.jobs);
PROGRESS_BAR_ENABLED.store(atty::is(atty::Stream::Stdout), Ordering::SeqCst);
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
PROGRESS_BAR.set_draw_target(ProgressDrawTarget::stderr_nohz());
PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}+] {msg}"));
PROGRESS_BAR.inc_length(1);
PROGRESS_BAR.set_message("initializing..");
}
if let Some(url) = ilias.opt.sync_url.as_ref() { if let Some(url) = ilias.opt.sync_url.as_ref() {
for item in ilias.get_course_content(&URL::from_href(url).expect("invalid URL")).await.expect("invalid response") { for item in ilias.get_course_content(&URL::from_href(url).expect("invalid URL")).await.expect("invalid response") {
let item = item.expect("invalid item"); let item = item.expect("invalid item");
@ -161,6 +176,11 @@ async fn main() {
warning!("could not disable content tree:", e); warning!("could not disable content tree:", e);
} }
} }
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
PROGRESS_BAR.inc(1);
PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}] {msg}"));
PROGRESS_BAR.finish_with_message("done");
}
} }
lazy_static! { lazy_static! {
@ -225,6 +245,9 @@ fn process_gracefully(
path: PathBuf, path: PathBuf,
obj: Object, obj: Object,
) -> impl Future<Output = ()> + Send { async move { ) -> impl Future<Output = ()> + Send { async move {
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
PROGRESS_BAR.inc_length(1);
}
let permit = TASKS_RUNNING.acquire().await.unwrap(); let permit = TASKS_RUNNING.acquire().await.unwrap();
let path_text = path.to_string_lossy().into_owned(); let path_text = path.to_string_lossy().into_owned();
if let Err(e) = process(ilias, path, obj).await.context("failed to process URL") { if let Err(e) = process(ilias, path, obj).await.context("failed to process URL") {
@ -276,6 +299,10 @@ const NO_ENTRIES: &str = "Keine Einträge";
async fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> Result<()> { async fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> Result<()> {
let relative_path = path.strip_prefix(&ilias.opt.output).unwrap(); let relative_path = path.strip_prefix(&ilias.opt.output).unwrap();
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
PROGRESS_BAR.inc(1);
PROGRESS_BAR.set_message(&relative_path.display().to_string());
}
if ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() { if ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() {
log!(1, "Ignored {}", relative_path.to_string_lossy()); log!(1, "Ignored {}", relative_path.to_string_lossy());
return Ok(()); return Ok(());