From 8ea2cae769c208fc05e51172b323f3d264a164e9 Mon Sep 17 00:00:00 2001 From: FliegendeWurst <2012gdwu+github@posteo.de> Date: Sun, 30 May 2021 13:38:52 +0200 Subject: [PATCH] Session re-use --- CHANGELOG.md | 3 +- Cargo.lock | 88 +++++++++++++++++++++++++++++++++++++++++++++------- Cargo.toml | 2 ++ src/cli.rs | 6 ++-- src/ilias.rs | 64 +++++++++++++++++++++++++++++--------- src/main.rs | 79 ++++++++++++++++++++++++++++++++++------------ 6 files changed, 193 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac05da6..5852800 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ and this project loosely adheres to [Semantic Versioning](https://semver.org/spe ## [Unreleased] ### Added -- `--sync-url https://ilias.studium.kit.edu/ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToMemberships` to download all courses (unless specified otherwise in the `.iliasignore`) +- `--sync-url` can now download all courses (unless specified otherwise in the `.iliasignore`) +- `--keep-session` flag to save and restore session cookies ## [0.2.21] - 2021-05-18 ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 3de0393..2ee73f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,6 +10,7 @@ dependencies = [ "atty", "cfg-if", "colored", + "cookie_store 0.14.1", "futures", "futures-channel", "futures-util", @@ -20,6 +21,7 @@ dependencies = [ "once_cell", "regex", "reqwest", + "reqwest_cookie_store", "rpassword", "rprompt", "scraper", @@ -175,9 +177,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" +checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" [[package]] name = "byteorder" @@ -274,16 +276,43 @@ dependencies = [ "version_check", ] +[[package]] +name = "cookie" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf8865bac3d9a3bde5bde9088ca431b11f5d37c7a578b8086af77248b76627" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + [[package]] name = "cookie_store" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3" dependencies = [ - "cookie", + "cookie 0.14.4", "idna", "log", - "publicsuffix", + "publicsuffix 1.5.6", + "serde", + "serde_json", + "time", + "url", +] + +[[package]] +name = "cookie_store" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ba4e410d33a326cee69c778c761b6a822469ed7c6b82aa7802bef6e12716060" +dependencies = [ + "cookie 0.15.0", + "idna", + "log", + "publicsuffix 2.1.0", "serde", "serde_json", "time", @@ -659,6 +688,12 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + [[package]] name = "heck" version = "0.3.2" @@ -820,7 +855,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.9.1", ] [[package]] @@ -1232,6 +1267,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "psl-types" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b398073e7cdd6f05934389a8f5961e3aabfa66675b6f440df4e2c793d51a4f" + [[package]] name = "publicsuffix" version = "1.5.6" @@ -1242,6 +1283,18 @@ dependencies = [ "url", ] +[[package]] +name = "publicsuffix" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ac055aef7cc7a1caefbc65144be879e862467dcd9b8a8d57b64a13e7dce15d" +dependencies = [ + "byteorder", + "hashbrown 0.11.2", + "idna", + "psl-types", +] + [[package]] name = "quote" version = "1.0.9" @@ -1328,8 +1381,8 @@ dependencies = [ "async-compression", "base64", "bytes", - "cookie", - "cookie_store", + "cookie 0.14.4", + "cookie_store 0.12.0", "encoding_rs", "futures-core", "futures-util", @@ -1361,6 +1414,19 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest_cookie_store" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a81253b4f49eba884e5b555d08617d2b59be9478fcb53875ded9825d957af4a" +dependencies = [ + "bytes", + "cookie 0.15.0", + "cookie_store 0.14.1", + "reqwest", + "url", +] + [[package]] name = "ring" version = "0.16.20" @@ -1891,9 +1957,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37" +checksum = "0a38d31d7831c6ed7aad00aa4c12d9375fd225a6dd77da1d25b707346319a975" dependencies = [ "autocfg", "bytes", @@ -2002,9 +2068,9 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +checksum = "33717dca7ac877f497014e10d73f3acf948c342bee31b5ca7892faf94ccc6b49" dependencies = [ "tinyvec", ] diff --git a/Cargo.toml b/Cargo.toml index 6e6b02c..3c81f60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,8 @@ indicatif = "0.16.0" once_cell = "1.7.2" atty = "0.2.14" h2 = "0.3.3" +cookie_store = "0.14.0" +reqwest_cookie_store = "0.1.5" [features] default = [] diff --git a/src/cli.rs b/src/cli.rs index ef827c3..d6f952d 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -12,7 +12,7 @@ use indicatif::ProgressBar; use once_cell::sync::Lazy; use structopt::StructOpt; -#[derive(Debug, StructOpt)] +#[derive(Debug, Clone, StructOpt)] #[structopt(name = env!("CARGO_PKG_NAME"))] pub struct Opt { /// Do not download files @@ -86,7 +86,7 @@ pub static PROGRESS_BAR_ENABLED: AtomicBool = AtomicBool::new(false); pub static PROGRESS_BAR: Lazy = Lazy::new(|| ProgressBar::new(0)); macro_rules! log { - ($lvl:expr, $($t:expr),+) => { + ($lvl:expr, $($t:expr),+) => {{ #[allow(unused_comparisons)] // 0 <= 0 if $lvl <= crate::cli::LOG_LEVEL.load(std::sync::atomic::Ordering::SeqCst) { if crate::cli::PROGRESS_BAR_ENABLED.load(std::sync::atomic::Ordering::SeqCst) { @@ -95,7 +95,7 @@ macro_rules! log { println!($($t),+); } } - } + }} } macro_rules! info { diff --git a/src/ilias.rs b/src/ilias.rs index 0de3893..f5243bc 100644 --- a/src/ilias.rs +++ b/src/ilias.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: GPL-3.0-or-later -use std::error::Error as _; +use std::{error::Error as _, sync::Arc}; use anyhow::{anyhow, Context, Result}; use colored::Colorize; +use cookie_store::CookieStore; use ignore::gitignore::Gitignore; use reqwest::{Client, IntoUrl, Proxy, Url}; +use reqwest_cookie_store::CookieStoreMutex; use scraper::{ElementRef, Html, Selector}; use serde_json::json; @@ -14,10 +16,8 @@ use crate::{cli::Opt, get_request_ticket, selectors::*, ILIAS_URL}; pub struct ILIAS { pub opt: Opt, pub ignore: Gitignore, - // TODO: use these for re-authentication in case of session timeout/invalidation - user: String, - pass: String, client: Client, + cookies: Arc, } /// Returns true if the error is caused by: @@ -36,12 +36,37 @@ fn error_is_http2(error: &reqwest::Error) -> bool { } impl ILIAS { - pub async fn login(opt: Opt, user: impl Into, pass: impl Into, ignore: Gitignore) -> Result { - let user = user.into(); - let pass = pass.into(); - let mut builder = Client::builder() - .cookie_store(true) - .user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"))); + // TODO: de-duplicate the logic below + pub async fn with_session(opt: Opt, session: Arc, ignore: Gitignore) -> Result { + let mut builder = + Client::builder() + .cookie_provider(Arc::clone(&session)) + .user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"))); + if let Some(proxy) = opt.proxy.as_ref() { + let proxy = Proxy::all(proxy)?; + builder = builder.proxy(proxy); + } + let client = builder + // timeout is infinite by default + .build()?; + info!("Re-using previous session cookies.."); + Ok(ILIAS { + opt, + ignore, + client, + cookies: session, + }) + } + + pub async fn login(opt: Opt, user: &str, pass: &str, ignore: Gitignore) -> Result { + let cookie_store = CookieStore::default(); + let cookie_store = reqwest_cookie_store::CookieStoreMutex::new(cookie_store); + let cookie_store = std::sync::Arc::new(cookie_store); + let mut builder = Client::builder().cookie_provider(Arc::clone(&cookie_store)).user_agent(concat!( + env!("CARGO_PKG_NAME"), + "/", + env!("CARGO_PKG_VERSION") + )); if let Some(proxy) = opt.proxy.as_ref() { let proxy = Proxy::all(proxy)?; builder = builder.proxy(proxy); @@ -52,9 +77,8 @@ impl ILIAS { let this = ILIAS { opt, ignore, - user, - pass, client, + cookies: cookie_store, }; info!("Logging into ILIAS using KIT account.."); let session_establishment = this @@ -75,14 +99,16 @@ impl ILIAS { .select(&Selector::parse(r#"input[name="csrf_token"]"#).unwrap()) .next() .context("no CSRF token found")? - .value().attr("value").context("no CSRF token value")?; + .value() + .attr("value") + .context("no CSRF token value")?; info!("Logging into Shibboleth.."); let login_response = this .client .post(url) .form(&json!({ - "j_username": &this.user, - "j_password": &this.pass, + "j_username": user, + "j_password": pass, "_eventId_proceed": "", "csrf_token": csrf_token, })) @@ -108,6 +134,14 @@ impl ILIAS { Ok(this) } + pub async fn save_session(&self) -> Result<()> { + let session_path = self.opt.output.join(".iliassession"); + let mut writer = std::fs::File::create(session_path).map(std::io::BufWriter::new).unwrap(); + let store = self.cookies.lock().map_err(|x| anyhow!("{}", x))?; + store.save_json(&mut writer).map_err(|x| anyhow!(x))?; + Ok(()) + } + pub async fn download(&self, url: &str) -> Result { get_request_ticket().await; log!(2, "Downloading {}", url); diff --git a/src/main.rs b/src/main.rs index 58d3773..55f751b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,9 +21,11 @@ use url::Url; use std::collections::HashSet; use std::future::Future; use std::io; +use std::io::BufReader; use std::path::PathBuf; use std::sync::atomic::Ordering; use std::sync::Arc; +use std::time::SystemTime; pub const ILIAS_URL: &str = "https://ilias.studium.kit.edu/"; @@ -67,26 +69,41 @@ async fn main() { } } -async fn real_main(mut opt: Opt) -> Result<()> { - LOG_LEVEL.store(opt.verbose, Ordering::SeqCst); - #[cfg(windows)] - let _ = colored::control::set_virtual_terminal(true); - - create_dir(&opt.output).await.context("failed to create output directory")?; - // use UNC paths on Windows (#6) - opt.output = fs::canonicalize(opt.output).await.context("failed to canonicalize output directory")?; - - // load .iliasignore file - opt.output.push(".iliasignore"); - let (ignore, error) = Gitignore::new(&opt.output); - if let Some(err) = error { - warning!(err); +async fn try_to_load_session(opt: Opt, ignore: Gitignore) -> Result { + let session_path = opt.output.join(".iliassession"); + let meta = tokio::fs::metadata(&session_path).await?; + let modified = meta.modified()?; + let now = SystemTime::now(); + // the previous session is only useful if it isn't older than ~1 hour + let duration = now.duration_since(modified)?; + if duration.as_secs() <= 60 * 60 { + let file = std::fs::File::open(session_path)?; + let cookies = cookie_store::CookieStore::load_json(BufReader::new(file)) + .map_err(|err| anyhow!(err)) + .context("failed to load session cookies")?; + let cookie_store = reqwest_cookie_store::CookieStoreMutex::new(cookies); + let cookie_store = std::sync::Arc::new(cookie_store); + Ok(ILIAS::with_session(opt, cookie_store, ignore).await?) + } else { + Err(anyhow!("session data too old")) + } +} + +async fn login(opt: Opt, ignore: Gitignore) -> Result { + // load .iliassession file + if opt.keep_session { + match try_to_load_session(opt.clone(), ignore.clone()) + .await + .context("failed to load previous session") + { + Ok(ilias) => return Ok(ilias), + Err(e) => warning!(e), + } } - opt.output.pop(); // loac .iliaslogin file - opt.output.push(".iliaslogin"); - let login = std::fs::read_to_string(&opt.output); + let iliaslogin = opt.output.join(".iliaslogin"); + let login = std::fs::read_to_string(&iliaslogin); let (user, pass) = if let Ok(login) = login { let mut lines = login.split('\n'); let user = lines.next().context("missing user in .iliaslogin")?; @@ -97,15 +114,34 @@ async fn real_main(mut opt: Opt) -> Result<()> { } else { ask_user_pass(&opt).context("credentials input failed")? }; - opt.output.pop(); - let ilias = match ILIAS::login(opt, user, pass, ignore).await { + let ilias = match ILIAS::login(opt, &user, &pass, ignore).await { Ok(ilias) => ilias, Err(e) => { error!(e); std::process::exit(77); }, }; + Ok(ilias) +} + +async fn real_main(mut opt: Opt) -> Result<()> { + LOG_LEVEL.store(opt.verbose, Ordering::SeqCst); + #[cfg(windows)] + let _ = colored::control::set_virtual_terminal(true); + + create_dir(&opt.output).await.context("failed to create output directory")?; + // use UNC paths on Windows (to avoid the default max. path length of 255) + opt.output = fs::canonicalize(opt.output).await.context("failed to canonicalize output directory")?; + + // load .iliasignore file + let (ignore, error) = Gitignore::new(opt.output.join(".iliasignore")); + if let Some(err) = error { + warning!(err); + } + + let ilias = login(opt, ignore).await?; + if ilias.opt.content_tree { if let Err(e) = ilias .download("ilias.php?baseClass=ilRepositoryGUI&cmd=frameset&set_mode=tree&ref_id=1") @@ -149,6 +185,11 @@ async fn real_main(mut opt: Opt) -> Result<()> { warning!("could not disable content tree:", e); } } + if ilias.opt.keep_session { + if let Err(e) = ilias.save_session().await.context("failed to save session cookies") { + warning!(e) + } + } if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) { PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}] {wide_msg}")); PROGRESS_BAR.finish_with_message("done");