diff --git a/Cargo.lock b/Cargo.lock index b8e1569..7132b9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,6 +12,7 @@ dependencies = [ "reqwest", "scraper", "serde_json", + "structopt", "tokio", "url 2.1.1", ] @@ -25,12 +26,32 @@ dependencies = [ "memchr", ] +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "arc-swap" version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.8", +] + [[package]] name = "autocfg" version = "0.1.7" @@ -107,6 +128,21 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +[[package]] +name = "clap" +version = "2.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + [[package]] name = "cloudabi" version = "0.0.3" @@ -385,6 +421,15 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "heck" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.11" @@ -896,6 +941,32 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "proc-macro-error" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "syn-mid", + "version_check", +] + [[package]] name = "proc-macro-hack" version = "0.5.15" @@ -1400,6 +1471,36 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6da2e8d107dfd7b74df5ef4d205c6aebee0706c647f6bc6a2d5789905c00fb" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a489c87c08fbaf12e386665109dd13470dcc9c4583ea3e10dd2b4523e5ebd9ac" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "syn" version = "1.0.17" @@ -1411,6 +1512,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "syn-mid" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tempfile" version = "3.1.0" @@ -1436,6 +1548,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "thin-slice" version = "0.1.1" @@ -1559,6 +1680,12 @@ dependencies = [ "smallvec 1.3.0", ] +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" + [[package]] name = "unicode-width" version = "0.1.7" @@ -1605,6 +1732,12 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" + [[package]] name = "version_check" version = "0.9.1" diff --git a/Cargo.toml b/Cargo.toml index 3d172c5..e75542f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,3 +18,4 @@ futures-util = "0.3.4" regex = "1.3.7" lazy_static = "1.4.0" parking_lot = "0.10.2" +structopt = "0.3.13" diff --git a/README.md b/README.md index f8af1d6..db0ed5f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Download content from ILIAS. That includes: -* files +* files (latest version) * Opencast lectures ## Installation @@ -19,7 +19,24 @@ $ cp target/release/KIT-ILIAS-downloader [directory in $PATH] ## Usage -TBD +```sh +$ KIT-ILIAS-downloader --help +KIT-ILIAS-downloader 0.1.0 + +USAGE: + KIT-ILIAS-downloader [FLAGS] --output + +FLAGS: + -f, Re-download already present files + -h, --help Prints help information + -n, --no-videos Do not download Opencast videos + -s, --skip-files Do not download files + -V, --version Prints version information + -v Verbosity (>=1 prints progress) + +OPTIONS: + -o, --output Output directory +``` ## Credits diff --git a/src/main.rs b/src/main.rs index 299dde3..0dc5916 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,7 @@ use regex::Regex; use reqwest::Client; use scraper::{ElementRef, Html, Selector}; use serde_json::json; +use structopt::StructOpt; use tokio::fs::File as AsyncFile; use tokio::io::{stream_reader, BufWriter}; use tokio::task; @@ -24,6 +25,7 @@ use errors::*; const ILIAS_URL: &'static str = "https://ilias.studium.kit.edu/"; struct ILIAS { + opt: Opt, user: String, pass: String, path_prefix: PathBuf, @@ -186,7 +188,7 @@ impl URL { } impl ILIAS { - async fn login(user: S1, pass: S1) -> Result where S1: Into, S2: Into { + async fn login(opt: Opt, user: S1, pass: S1) -> Result where S1: Into, S2: Into { let user = user.into(); let pass = pass.into(); let client = Client::builder() @@ -238,7 +240,7 @@ impl ILIAS { println!("Logged in!"); let path_prefix = PathBuf::from(env!("ILIAS_DIR")); Ok(ILIAS { - client, user, pass, path_prefix + opt, client, user, pass, path_prefix }) } @@ -270,31 +272,22 @@ impl ILIAS { } async fn download(&self, url: &str) -> Result { - //let url = format!("{}{}", ILIAS_URL, url.url); - if VERBOSITY > 0 { + if self.opt.verbose > 0 { println!("Downloading {}", url); } Ok(self.client.get(url).send().await?) } } -const DOWNLOAD_FILES: bool = true; -const DOWNLOAD_VIDEOS: bool = true; - -const SKIP_EXISTING: bool = true; - -const VERBOSITY: usize = 1; - #[tokio::main] async fn main() { + let opt = Opt::from_args(); *PANIC_HOOK.lock() = panic::take_hook(); panic::set_hook(Box::new(|info| { *TASKS_RUNNING.lock() -= 1; PANIC_HOOK.lock()(info); })); - // TODO: config at runtime.. - // it's literally in the executable currently - let mut ilias = match ILIAS::login::<&str, &str>(env!("ILIAS_USER"), env!("ILIAS_PASS")).await { + let mut ilias = match ILIAS::login::<&str, &str>(opt, env!("ILIAS_USER"), env!("ILIAS_PASS")).await { Ok(ilias) => ilias, Err(e) => panic!("error: {:?}", e) }; @@ -328,7 +321,7 @@ lazy_static!{ // see https://github.com/rust-lang/rust/issues/53690#issuecomment-418911229 //async fn process(ilias: Arc, path: PathBuf, obj: Object) { fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::Future + Send { async move { - if VERBOSITY > 0 { + if ilias.opt.verbose > 0 { println!("Syncing {} {}..", obj.kind(), path.strip_prefix(&ilias.path_prefix).unwrap().to_string_lossy()); } match &obj { @@ -369,11 +362,11 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F } }, File { name, url } => { - if !DOWNLOAD_FILES { + if ilias.opt.skip_files { return; } - if SKIP_EXISTING && fs::metadata(&path).is_ok() { - if VERBOSITY > 1 { + if !ilias.opt.force && fs::metadata(&path).is_ok() { + if ilias.opt.verbose > 1 { println!("Skipping download, file exists already"); } return; @@ -392,7 +385,7 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F } }, PluginDispatch { name, url } => { - if !DOWNLOAD_VIDEOS { + if ilias.opt.no_videos { return; } if let Err(e) = fs::create_dir(&path) { @@ -422,7 +415,7 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F } let mut path = path.clone(); path.push(format!("{}.mp4", title)); - if VERBOSITY > 0 { + if ilias.opt.verbose > 0 { println!("Found video: {}", title); } let video = Video { @@ -442,11 +435,11 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F lazy_static!{ static ref XOCT_REGEX: Regex = Regex::new(r#"(?m)"#).unwrap(); } - if !DOWNLOAD_VIDEOS { + if ilias.opt.no_videos { return; } - if SKIP_EXISTING && fs::metadata(&path).is_ok() { - if VERBOSITY > 1 { + if !ilias.opt.force && fs::metadata(&path).is_ok() { + if ilias.opt.verbose > 1 { println!("Skipping download, file exists already"); } return; @@ -468,7 +461,7 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F let mut reader = stream_reader(resp.bytes_stream().map_err(|x| { io::Error::new(io::ErrorKind::Other, x) })); - if VERBOSITY > 0 { + if ilias.opt.verbose > 0 { println!("Saving video to {:?}", path); } let file = AsyncFile::create(&path).await.unwrap(); @@ -476,9 +469,33 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F tokio::io::copy(&mut reader, &mut file).await.unwrap(); }, o => { - if VERBOSITY > 0 { + if ilias.opt.verbose > 0 { println!("ignoring {:#?}", o) } } } }} + +#[derive(Debug, StructOpt)] +#[structopt(name = "KIT-ILIAS-downloader")] +struct Opt { + /// Do not download files + #[structopt(short, long)] + skip_files: bool, + + /// Do not download Opencast videos + #[structopt(short, long)] + no_videos: bool, + + /// Re-download already present files + #[structopt(short)] + force: bool, + + /// Verbosity (>=1 prints progress) + #[structopt(short, multiple = true, parse(from_occurrences))] + verbose: usize, + + /// Output directory + #[structopt(short, long, parse(from_os_str))] + output: PathBuf, +}