From 7faf14e37f8a8afb2f3e1a49f355b8ee0de7d4ad Mon Sep 17 00:00:00 2001 From: FliegendeWurst <2012gdwu@web.de> Date: Fri, 24 Apr 2020 10:14:53 +0200 Subject: [PATCH] Ignore courses specified in .iliasignore file --- Cargo.lock | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/main.rs | 24 ++++++++++++-- 3 files changed, 114 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5680a8..73379d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,6 +6,7 @@ version = "0.2.1" dependencies = [ "error-chain", "futures-util", + "ignore", "lazy_static", "parking_lot", "regex", @@ -119,6 +120,15 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bstr" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2889e6d50f394968c8bf4240dc3f2a7eb4680844d27308f798229ac9d4725f41" +dependencies = [ + "memchr", +] + [[package]] name = "bumpalo" version = "3.2.1" @@ -224,6 +234,27 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061" +dependencies = [ + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg 1.0.0", + "cfg-if", + "lazy_static", +] + [[package]] name = "cssparser" version = "0.25.9" @@ -427,6 +458,19 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "globset" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ad1da430bd7281dde2576f44c84cc3f0f7b475e7202cd503042dff01a8c8120" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + [[package]] name = "h2" version = "0.2.4" @@ -569,6 +613,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "ignore" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf60d063dbe6b75388eec66cfc07781167ae3d34a09e0c433e6c5de0511f7fb" +dependencies = [ + "crossbeam-channel", + "crossbeam-utils", + "globset", + "lazy_static", + "log", + "memchr", + "regex", + "same-file", + "thread_local", + "walkdir", + "winapi-util", +] + [[package]] name = "indexmap" version = "1.3.2" @@ -1245,6 +1308,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.18" @@ -1738,6 +1810,17 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce" +[[package]] +name = "walkdir" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +dependencies = [ + "same-file", + "winapi 0.3.8", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" @@ -1873,6 +1956,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index ddb04e5..55029b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,3 +21,4 @@ parking_lot = "0.10.2" structopt = "0.3.13" rpassword = "4.0.5" rprompt = "1.0.5" +ignore = "0.4.14" diff --git a/src/main.rs b/src/main.rs index 3609ec5..8bba91a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ use error_chain::ChainedError; use futures_util::stream::TryStreamExt; +use ignore::gitignore::Gitignore; use lazy_static::lazy_static; use parking_lot::Mutex; use regex::Regex; @@ -27,6 +28,7 @@ const ILIAS_URL: &'static str = "https://ilias.studium.kit.edu/"; struct ILIAS { opt: Opt, + ignore: Gitignore, // TODO: use these for re-authentication in case of session timeout/invalidation user: String, pass: String, @@ -128,6 +130,13 @@ impl Object { } } + fn is_dir(&self) -> bool { + match self { + Course { .. } | Folder { .. } | Forum { .. } | Thread { .. } | Wiki { .. } | ExerciseHandler { .. } | PluginDispatch { .. } => true, + File { .. } | Video { .. } | Generic { .. } => false + } + } + fn from_link(item: ElementRef, link: ElementRef) -> Self { let mut name = link.text().collect::().replace('/', "-").trim().to_owned(); let mut url = URL::from_href(link.value().attr("href").unwrap()); @@ -301,7 +310,7 @@ impl URL { } impl ILIAS { - async fn login, S2: Into>(opt: Opt, user: S1, pass: S2) -> Result { + async fn login, S2: Into>(mut opt: Opt, user: S1, pass: S2) -> Result { let user = user.into(); let pass = pass.into(); let client = Client::builder() @@ -309,8 +318,12 @@ impl ILIAS { .user_agent(concat!("KIT-ILIAS-downloader/", env!("CARGO_PKG_VERSION"))) .timeout(Duration::from_secs(20)) .build()?; + // load .iliasignore file + opt.output.push(".iliasignore"); + let ignore = Gitignore::new(&opt.output).0; // ignore errors + opt.output.pop(); let this = ILIAS { - opt, client, user, pass + opt, client, user, pass, ignore }; println!("Logging into ILIAS using KIT account.."); let session_establishment = this.client @@ -536,8 +549,13 @@ use crate::selectors::*; // see https://github.com/rust-lang/rust/issues/53690#issuecomment-418911229 //async fn process(ilias: Arc, path: PathBuf, obj: Object) { fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::Future> + Send { async move { + let relative_path = path.strip_prefix(&ilias.opt.output).unwrap(); + // TODO: match more than root directories (causes issues with whitelist patterns) + if relative_path.components().count() == 1 && ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() { + return Ok(()); + } if ilias.opt.verbose > 0 { - println!("Syncing {} {}.. {}", obj.kind(), path.strip_prefix(&ilias.opt.output).unwrap().to_string_lossy(), obj.url().url); + println!("Syncing {} {}.. {}", obj.kind(), relative_path.to_string_lossy(), obj.url().url); } match &obj { Course { url, name } => {