Ignore courses specified in .iliasignore file

Cargo.lock

@@ -6,6 +6,7 @@ version = "0.2.1"
 dependencies = [
  "error-chain",
  "futures-util",
+ "ignore",
  "lazy_static",
  "parking_lot",
  "regex",
@@ -119,6 +120,15 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 
+[[package]]
+name = "bstr"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2889e6d50f394968c8bf4240dc3f2a7eb4680844d27308f798229ac9d4725f41"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.2.1"
@@ -224,6 +234,27 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061"
+dependencies = [
+ "crossbeam-utils",
+ "maybe-uninit",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
+dependencies = [
+ "autocfg 1.0.0",
+ "cfg-if",
+ "lazy_static",
+]
+
 [[package]]
 name = "cssparser"
 version = "0.25.9"
@@ -427,6 +458,19 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "globset"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ad1da430bd7281dde2576f44c84cc3f0f7b475e7202cd503042dff01a8c8120"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "fnv",
+ "log",
+ "regex",
+]
+
 [[package]]
 name = "h2"
 version = "0.2.4"
@@ -569,6 +613,25 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "ignore"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddf60d063dbe6b75388eec66cfc07781167ae3d34a09e0c433e6c5de0511f7fb"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-utils",
+ "globset",
+ "lazy_static",
+ "log",
+ "memchr",
+ "regex",
+ "same-file",
+ "thread_local",
+ "walkdir",
+ "winapi-util",
+]
+
 [[package]]
 name = "indexmap"
 version = "1.3.2"
@@ -1245,6 +1308,15 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "schannel"
 version = "0.1.18"
@@ -1738,6 +1810,17 @@ version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
 
+[[package]]
+name = "walkdir"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
+dependencies = [
+ "same-file",
+ "winapi 0.3.8",
+ "winapi-util",
+]
+
 [[package]]
 name = "want"
 version = "0.3.0"
@@ -1873,6 +1956,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi 0.3.8",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"

Cargo.toml

@@ -21,3 +21,4 @@ parking_lot = "0.10.2"
 structopt = "0.3.13"
 rpassword = "4.0.5"
 rprompt = "1.0.5"
+ignore = "0.4.14"

src/main.rs

@@ -1,5 +1,6 @@
 use error_chain::ChainedError;
 use futures_util::stream::TryStreamExt;
+use ignore::gitignore::Gitignore;
 use lazy_static::lazy_static;
 use parking_lot::Mutex;
 use regex::Regex;
@@ -27,6 +28,7 @@ const ILIAS_URL: &'static str = "https://ilias.studium.kit.edu/";
 
 struct ILIAS {
 	opt: Opt,
+	ignore: Gitignore,
 	// TODO: use these for re-authentication in case of session timeout/invalidation
 	user: String,
 	pass: String,
@@ -128,6 +130,13 @@ impl Object {
 		}
 	}
 
+	fn is_dir(&self) -> bool {
+		match self {
+			Course { .. } | Folder { .. } | Forum { .. } | Thread { .. } | Wiki { .. } | ExerciseHandler { .. } | PluginDispatch { .. } => true,
+			File { .. } | Video { .. } | Generic { .. } => false
+		}
+	}
+
 	fn from_link(item: ElementRef, link: ElementRef) -> Self {
 		let mut name = link.text().collect::<String>().replace('/', "-").trim().to_owned();
 		let mut url = URL::from_href(link.value().attr("href").unwrap());
@@ -301,7 +310,7 @@ impl URL {
 }
 
 impl ILIAS {
-	async fn login<S1: Into<String>, S2: Into<String>>(opt: Opt, user: S1, pass: S2) -> Result<Self> {
+	async fn login<S1: Into<String>, S2: Into<String>>(mut opt: Opt, user: S1, pass: S2) -> Result<Self> {
 		let user = user.into();
 		let pass = pass.into();
 		let client = Client::builder()
@@ -309,8 +318,12 @@ impl ILIAS {
 			.user_agent(concat!("KIT-ILIAS-downloader/", env!("CARGO_PKG_VERSION")))
 			.timeout(Duration::from_secs(20))
 			.build()?;
+		// load .iliasignore file
+		opt.output.push(".iliasignore");
+		let ignore = Gitignore::new(&opt.output).0; // ignore errors
+		opt.output.pop();
 		let this = ILIAS {
-			opt, client, user, pass
+			opt, client, user, pass, ignore
 		};
 		println!("Logging into ILIAS using KIT account..");
 		let session_establishment = this.client
@@ -536,8 +549,13 @@ use crate::selectors::*;
 // see https://github.com/rust-lang/rust/issues/53690#issuecomment-418911229
 //async fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) {
 fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::Future<Output = Result<()>> + Send { async move {
+	let relative_path = path.strip_prefix(&ilias.opt.output).unwrap();
+	// TODO: match more than root directories (causes issues with whitelist patterns)
+	if relative_path.components().count() == 1 && ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() {
+		return Ok(());
+	}
 	if ilias.opt.verbose > 0 {
-		println!("Syncing {} {}.. {}", obj.kind(), path.strip_prefix(&ilias.opt.output).unwrap().to_string_lossy(), obj.url().url);
+		println!("Syncing {} {}.. {}", obj.kind(), relative_path.to_string_lossy(), obj.url().url);
 	}
 	match &obj {
 		Course { url, name } => {