Ignore courses specified in .iliasignore file

This commit is contained in:
FliegendeWurst 2020-04-24 10:14:53 +02:00
parent 12876a3e8c
commit 7faf14e37f
3 changed files with 114 additions and 3 deletions

92
Cargo.lock generated
View File

@ -6,6 +6,7 @@ version = "0.2.1"
dependencies = [
"error-chain",
"futures-util",
"ignore",
"lazy_static",
"parking_lot",
"regex",
@ -119,6 +120,15 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "bstr"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2889e6d50f394968c8bf4240dc3f2a7eb4680844d27308f798229ac9d4725f41"
dependencies = [
"memchr",
]
[[package]]
name = "bumpalo"
version = "3.2.1"
@ -224,6 +234,27 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-channel"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061"
dependencies = [
"crossbeam-utils",
"maybe-uninit",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg 1.0.0",
"cfg-if",
"lazy_static",
]
[[package]]
name = "cssparser"
version = "0.25.9"
@ -427,6 +458,19 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "globset"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ad1da430bd7281dde2576f44c84cc3f0f7b475e7202cd503042dff01a8c8120"
dependencies = [
"aho-corasick",
"bstr",
"fnv",
"log",
"regex",
]
[[package]]
name = "h2"
version = "0.2.4"
@ -569,6 +613,25 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "ignore"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddf60d063dbe6b75388eec66cfc07781167ae3d34a09e0c433e6c5de0511f7fb"
dependencies = [
"crossbeam-channel",
"crossbeam-utils",
"globset",
"lazy_static",
"log",
"memchr",
"regex",
"same-file",
"thread_local",
"walkdir",
"winapi-util",
]
[[package]]
name = "indexmap"
version = "1.3.2"
@ -1245,6 +1308,15 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "schannel"
version = "0.1.18"
@ -1738,6 +1810,17 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
[[package]]
name = "walkdir"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
dependencies = [
"same-file",
"winapi 0.3.8",
"winapi-util",
]
[[package]]
name = "want"
version = "0.3.0"
@ -1873,6 +1956,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi 0.3.8",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"

View File

@ -21,3 +21,4 @@ parking_lot = "0.10.2"
structopt = "0.3.13"
rpassword = "4.0.5"
rprompt = "1.0.5"
ignore = "0.4.14"

View File

@ -1,5 +1,6 @@
use error_chain::ChainedError;
use futures_util::stream::TryStreamExt;
use ignore::gitignore::Gitignore;
use lazy_static::lazy_static;
use parking_lot::Mutex;
use regex::Regex;
@ -27,6 +28,7 @@ const ILIAS_URL: &'static str = "https://ilias.studium.kit.edu/";
struct ILIAS {
opt: Opt,
ignore: Gitignore,
// TODO: use these for re-authentication in case of session timeout/invalidation
user: String,
pass: String,
@ -128,6 +130,13 @@ impl Object {
}
}
fn is_dir(&self) -> bool {
match self {
Course { .. } | Folder { .. } | Forum { .. } | Thread { .. } | Wiki { .. } | ExerciseHandler { .. } | PluginDispatch { .. } => true,
File { .. } | Video { .. } | Generic { .. } => false
}
}
fn from_link(item: ElementRef, link: ElementRef) -> Self {
let mut name = link.text().collect::<String>().replace('/', "-").trim().to_owned();
let mut url = URL::from_href(link.value().attr("href").unwrap());
@ -301,7 +310,7 @@ impl URL {
}
impl ILIAS {
async fn login<S1: Into<String>, S2: Into<String>>(opt: Opt, user: S1, pass: S2) -> Result<Self> {
async fn login<S1: Into<String>, S2: Into<String>>(mut opt: Opt, user: S1, pass: S2) -> Result<Self> {
let user = user.into();
let pass = pass.into();
let client = Client::builder()
@ -309,8 +318,12 @@ impl ILIAS {
.user_agent(concat!("KIT-ILIAS-downloader/", env!("CARGO_PKG_VERSION")))
.timeout(Duration::from_secs(20))
.build()?;
// load .iliasignore file
opt.output.push(".iliasignore");
let ignore = Gitignore::new(&opt.output).0; // ignore errors
opt.output.pop();
let this = ILIAS {
opt, client, user, pass
opt, client, user, pass, ignore
};
println!("Logging into ILIAS using KIT account..");
let session_establishment = this.client
@ -536,8 +549,13 @@ use crate::selectors::*;
// see https://github.com/rust-lang/rust/issues/53690#issuecomment-418911229
//async fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) {
fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::Future<Output = Result<()>> + Send { async move {
let relative_path = path.strip_prefix(&ilias.opt.output).unwrap();
// TODO: match more than root directories (causes issues with whitelist patterns)
if relative_path.components().count() == 1 && ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() {
return Ok(());
}
if ilias.opt.verbose > 0 {
println!("Syncing {} {}.. {}", obj.kind(), path.strip_prefix(&ilias.opt.output).unwrap().to_string_lossy(), obj.url().url);
println!("Syncing {} {}.. {}", obj.kind(), relative_path.to_string_lossy(), obj.url().url);
}
match &obj {
Course { url, name } => {