Session re-use

CHANGELOG.md

@@ -6,7 +6,8 @@ and this project loosely adheres to [Semantic Versioning](https://semver.org/spe
 
 ## [Unreleased]
 ### Added
-- `--sync-url https://ilias.studium.kit.edu/ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToMemberships` to download all courses (unless specified otherwise in the `.iliasignore`)
+- `--sync-url` can now download all courses (unless specified otherwise in the `.iliasignore`)
+- `--keep-session` flag to save and restore session cookies
 
 ## [0.2.21] - 2021-05-18
 ### Fixed

Cargo.lock

@@ -10,6 +10,7 @@ dependencies = [
  "atty",
  "cfg-if",
  "colored",
+ "cookie_store 0.14.1",
  "futures",
  "futures-channel",
  "futures-util",
@@ -20,6 +21,7 @@ dependencies = [
  "once_cell",
  "regex",
  "reqwest",
+ "reqwest_cookie_store",
  "rpassword",
  "rprompt",
  "scraper",
@@ -175,9 +177,9 @@ dependencies = [
 
 [[package]]
 name = "bumpalo"
-version = "3.6.1"
+version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe"
+checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
 
 [[package]]
 name = "byteorder"
@@ -274,16 +276,43 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "cookie"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffdf8865bac3d9a3bde5bde9088ca431b11f5d37c7a578b8086af77248b76627"
+dependencies = [
+ "percent-encoding",
+ "time",
+ "version_check",
+]
+
 [[package]]
 name = "cookie_store"
 version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3"
 dependencies = [
- "cookie",
+ "cookie 0.14.4",
  "idna",
  "log",
- "publicsuffix",
+ "publicsuffix 1.5.6",
+ "serde",
+ "serde_json",
+ "time",
+ "url",
+]
+
+[[package]]
+name = "cookie_store"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ba4e410d33a326cee69c778c761b6a822469ed7c6b82aa7802bef6e12716060"
+dependencies = [
+ "cookie 0.15.0",
+ "idna",
+ "log",
+ "publicsuffix 2.1.0",
  "serde",
  "serde_json",
  "time",
@@ -659,6 +688,12 @@ version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
 
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+
 [[package]]
 name = "heck"
 version = "0.3.2"
@@ -820,7 +855,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
 dependencies = [
  "autocfg",
- "hashbrown",
+ "hashbrown 0.9.1",
 ]
 
 [[package]]
@@ -1232,6 +1267,12 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "psl-types"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66b398073e7cdd6f05934389a8f5961e3aabfa66675b6f440df4e2c793d51a4f"
+
 [[package]]
 name = "publicsuffix"
 version = "1.5.6"
@@ -1242,6 +1283,18 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "publicsuffix"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3ac055aef7cc7a1caefbc65144be879e862467dcd9b8a8d57b64a13e7dce15d"
+dependencies = [
+ "byteorder",
+ "hashbrown 0.11.2",
+ "idna",
+ "psl-types",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.9"
@@ -1328,8 +1381,8 @@ dependencies = [
  "async-compression",
  "base64",
  "bytes",
- "cookie",
+ "cookie 0.14.4",
- "cookie_store",
+ "cookie_store 0.12.0",
  "encoding_rs",
  "futures-core",
  "futures-util",
@@ -1361,6 +1414,19 @@ dependencies = [
  "winreg",
 ]
 
+[[package]]
+name = "reqwest_cookie_store"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a81253b4f49eba884e5b555d08617d2b59be9478fcb53875ded9825d957af4a"
+dependencies = [
+ "bytes",
+ "cookie 0.15.0",
+ "cookie_store 0.14.1",
+ "reqwest",
+ "url",
+]
+
 [[package]]
 name = "ring"
 version = "0.16.20"
@@ -1891,9 +1957,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.6.0"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37"
+checksum = "0a38d31d7831c6ed7aad00aa4c12d9375fd225a6dd77da1d25b707346319a975"
 dependencies = [
  "autocfg",
  "bytes",
@@ -2002,9 +2068,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.17"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef"
+checksum = "33717dca7ac877f497014e10d73f3acf948c342bee31b5ca7892faf94ccc6b49"
 dependencies = [
  "tinyvec",
 ]

Cargo.toml

@@ -30,6 +30,8 @@ indicatif = "0.16.0"
 once_cell = "1.7.2"
 atty = "0.2.14"
 h2 = "0.3.3"
+cookie_store = "0.14.0"
+reqwest_cookie_store = "0.1.5"
 
 [features]
 default = []

src/cli.rs

@@ -12,7 +12,7 @@ use indicatif::ProgressBar;
 use once_cell::sync::Lazy;
 use structopt::StructOpt;
 
-#[derive(Debug, StructOpt)]
+#[derive(Debug, Clone, StructOpt)]
 #[structopt(name = env!("CARGO_PKG_NAME"))]
 pub struct Opt {
 	/// Do not download files
@@ -86,7 +86,7 @@ pub static PROGRESS_BAR_ENABLED: AtomicBool = AtomicBool::new(false);
 pub static PROGRESS_BAR: Lazy<ProgressBar> = Lazy::new(|| ProgressBar::new(0));
 
 macro_rules! log {
-	($lvl:expr, $($t:expr),+) => {
+	($lvl:expr, $($t:expr),+) => {{
 		#[allow(unused_comparisons)] // 0 <= 0
 		if $lvl <= crate::cli::LOG_LEVEL.load(std::sync::atomic::Ordering::SeqCst) {
 			if crate::cli::PROGRESS_BAR_ENABLED.load(std::sync::atomic::Ordering::SeqCst) {
@@ -95,7 +95,7 @@ macro_rules! log {
 				println!($($t),+);
 			}
 		}
-	}
+	}}
 }
 
 macro_rules! info {

src/ilias.rs

@@ -1,11 +1,13 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
-use std::error::Error as _;
+use std::{error::Error as _, sync::Arc};
 
 use anyhow::{anyhow, Context, Result};
 use colored::Colorize;
+use cookie_store::CookieStore;
 use ignore::gitignore::Gitignore;
 use reqwest::{Client, IntoUrl, Proxy, Url};
+use reqwest_cookie_store::CookieStoreMutex;
 use scraper::{ElementRef, Html, Selector};
 use serde_json::json;
 
@@ -14,10 +16,8 @@ use crate::{cli::Opt, get_request_ticket, selectors::*, ILIAS_URL};
 pub struct ILIAS {
 	pub opt: Opt,
 	pub ignore: Gitignore,
-	// TODO: use these for re-authentication in case of session timeout/invalidation
-	user: String,
-	pass: String,
 	client: Client,
+	cookies: Arc<CookieStoreMutex>,
 }
 
 /// Returns true if the error is caused by:
@@ -36,12 +36,37 @@ fn error_is_http2(error: &reqwest::Error) -> bool {
 }
 
 impl ILIAS {
-	pub async fn login(opt: Opt, user: impl Into<String>, pass: impl Into<String>, ignore: Gitignore) -> Result<Self> {
+	// TODO: de-duplicate the logic below
-		let user = user.into();
+	pub async fn with_session(opt: Opt, session: Arc<CookieStoreMutex>, ignore: Gitignore) -> Result<Self> {
-		let pass = pass.into();
+		let mut builder =
-		let mut builder = Client::builder()
+			Client::builder()
-			.cookie_store(true)
+				.cookie_provider(Arc::clone(&session))
-			.user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")));
+				.user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")));
+		if let Some(proxy) = opt.proxy.as_ref() {
+			let proxy = Proxy::all(proxy)?;
+			builder = builder.proxy(proxy);
+		}
+		let client = builder
+			// timeout is infinite by default
+			.build()?;
+		info!("Re-using previous session cookies..");
+		Ok(ILIAS {
+			opt,
+			ignore,
+			client,
+			cookies: session,
+		})
+	}
+
+	pub async fn login(opt: Opt, user: &str, pass: &str, ignore: Gitignore) -> Result<Self> {
+		let cookie_store = CookieStore::default();
+		let cookie_store = reqwest_cookie_store::CookieStoreMutex::new(cookie_store);
+		let cookie_store = std::sync::Arc::new(cookie_store);
+		let mut builder = Client::builder().cookie_provider(Arc::clone(&cookie_store)).user_agent(concat!(
+			env!("CARGO_PKG_NAME"),
+			"/",
+			env!("CARGO_PKG_VERSION")
+		));
 		if let Some(proxy) = opt.proxy.as_ref() {
 			let proxy = Proxy::all(proxy)?;
 			builder = builder.proxy(proxy);
@@ -52,9 +77,8 @@ impl ILIAS {
 		let this = ILIAS {
 			opt,
 			ignore,
-			user,
-			pass,
 			client,
+			cookies: cookie_store,
 		};
 		info!("Logging into ILIAS using KIT account..");
 		let session_establishment = this
@@ -75,14 +99,16 @@ impl ILIAS {
 			.select(&Selector::parse(r#"input[name="csrf_token"]"#).unwrap())
 			.next()
 			.context("no CSRF token found")?
-			.value().attr("value").context("no CSRF token value")?;
+			.value()
+			.attr("value")
+			.context("no CSRF token value")?;
 		info!("Logging into Shibboleth..");
 		let login_response = this
 			.client
 			.post(url)
 			.form(&json!({
-				"j_username": &this.user,
+				"j_username": user,
-				"j_password": &this.pass,
+				"j_password": pass,
 				"_eventId_proceed": "",
 				"csrf_token": csrf_token,
 			}))
@@ -108,6 +134,14 @@ impl ILIAS {
 		Ok(this)
 	}
 
+	pub async fn save_session(&self) -> Result<()> {
+		let session_path = self.opt.output.join(".iliassession");
+		let mut writer = std::fs::File::create(session_path).map(std::io::BufWriter::new).unwrap();
+		let store = self.cookies.lock().map_err(|x| anyhow!("{}", x))?;
+		store.save_json(&mut writer).map_err(|x| anyhow!(x))?;
+		Ok(())
+	}
+
 	pub async fn download(&self, url: &str) -> Result<reqwest::Response> {
 		get_request_ticket().await;
 		log!(2, "Downloading {}", url);

src/main.rs

@@ -21,9 +21,11 @@ use url::Url;
 use std::collections::HashSet;
 use std::future::Future;
 use std::io;
+use std::io::BufReader;
 use std::path::PathBuf;
 use std::sync::atomic::Ordering;
 use std::sync::Arc;
+use std::time::SystemTime;
 
 pub const ILIAS_URL: &str = "https://ilias.studium.kit.edu/";
 
@@ -67,26 +69,41 @@ async fn main() {
 	}
 }
 
-async fn real_main(mut opt: Opt) -> Result<()> {
+async fn try_to_load_session(opt: Opt, ignore: Gitignore) -> Result<ILIAS> {
-	LOG_LEVEL.store(opt.verbose, Ordering::SeqCst);
+	let session_path = opt.output.join(".iliassession");
-	#[cfg(windows)]
+	let meta = tokio::fs::metadata(&session_path).await?;
-	let _ = colored::control::set_virtual_terminal(true);
+	let modified = meta.modified()?;
-
+	let now = SystemTime::now();
-	create_dir(&opt.output).await.context("failed to create output directory")?;
+	// the previous session is only useful if it isn't older than ~1 hour
-	// use UNC paths on Windows (#6)
+	let duration = now.duration_since(modified)?;
-	opt.output = fs::canonicalize(opt.output).await.context("failed to canonicalize output directory")?;
+	if duration.as_secs() <= 60 * 60 {
-
+		let file = std::fs::File::open(session_path)?;
-	// load .iliasignore file
+		let cookies = cookie_store::CookieStore::load_json(BufReader::new(file))
-	opt.output.push(".iliasignore");
+			.map_err(|err| anyhow!(err))
-	let (ignore, error) = Gitignore::new(&opt.output);
+			.context("failed to load session cookies")?;
-	if let Some(err) = error {
+		let cookie_store = reqwest_cookie_store::CookieStoreMutex::new(cookies);
-		warning!(err);
+		let cookie_store = std::sync::Arc::new(cookie_store);
+		Ok(ILIAS::with_session(opt, cookie_store, ignore).await?)
+	} else {
+		Err(anyhow!("session data too old"))
+	}
+}
+
+async fn login(opt: Opt, ignore: Gitignore) -> Result<ILIAS> {
+	// load .iliassession file
+	if opt.keep_session {
+		match try_to_load_session(opt.clone(), ignore.clone())
+			.await
+			.context("failed to load previous session")
+		{
+			Ok(ilias) => return Ok(ilias),
+			Err(e) => warning!(e),
+		}
 	}
-	opt.output.pop();
 
 	// loac .iliaslogin file
-	opt.output.push(".iliaslogin");
+	let iliaslogin = opt.output.join(".iliaslogin");
-	let login = std::fs::read_to_string(&opt.output);
+	let login = std::fs::read_to_string(&iliaslogin);
 	let (user, pass) = if let Ok(login) = login {
 		let mut lines = login.split('\n');
 		let user = lines.next().context("missing user in .iliaslogin")?;
@@ -97,15 +114,34 @@ async fn real_main(mut opt: Opt) -> Result<()> {
 	} else {
 		ask_user_pass(&opt).context("credentials input failed")?
 	};
-	opt.output.pop();
 
-	let ilias = match ILIAS::login(opt, user, pass, ignore).await {
+	let ilias = match ILIAS::login(opt, &user, &pass, ignore).await {
 		Ok(ilias) => ilias,
 		Err(e) => {
 			error!(e);
 			std::process::exit(77);
 		},
 	};
+	Ok(ilias)
+}
+
+async fn real_main(mut opt: Opt) -> Result<()> {
+	LOG_LEVEL.store(opt.verbose, Ordering::SeqCst);
+	#[cfg(windows)]
+	let _ = colored::control::set_virtual_terminal(true);
+
+	create_dir(&opt.output).await.context("failed to create output directory")?;
+	// use UNC paths on Windows (to avoid the default max. path length of 255)
+	opt.output = fs::canonicalize(opt.output).await.context("failed to canonicalize output directory")?;
+
+	// load .iliasignore file
+	let (ignore, error) = Gitignore::new(opt.output.join(".iliasignore"));
+	if let Some(err) = error {
+		warning!(err);
+	}
+
+	let ilias = login(opt, ignore).await?;
+
 	if ilias.opt.content_tree {
 		if let Err(e) = ilias
 			.download("ilias.php?baseClass=ilRepositoryGUI&cmd=frameset&set_mode=tree&ref_id=1")
@@ -149,6 +185,11 @@ async fn real_main(mut opt: Opt) -> Result<()> {
 			warning!("could not disable content tree:", e);
 		}
 	}
+	if ilias.opt.keep_session {
+		if let Err(e) = ilias.save_session().await.context("failed to save session cookies") {
+			warning!(e)
+		}
+	}
 	if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
 		PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}] {wide_msg}"));
 		PROGRESS_BAR.finish_with_message("done");