Session re-use

This commit is contained in:
FliegendeWurst 2021-05-30 13:38:52 +02:00
parent 5fb2faabfd
commit 8ea2cae769
6 changed files with 193 additions and 49 deletions

View File

@ -6,7 +6,8 @@ and this project loosely adheres to [Semantic Versioning](https://semver.org/spe
## [Unreleased]
### Added
- `--sync-url https://ilias.studium.kit.edu/ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToMemberships` to download all courses (unless specified otherwise in the `.iliasignore`)
- `--sync-url` can now download all courses (unless specified otherwise in the `.iliasignore`)
- `--keep-session` flag to save and restore session cookies
## [0.2.21] - 2021-05-18
### Fixed

88
Cargo.lock generated
View File

@ -10,6 +10,7 @@ dependencies = [
"atty",
"cfg-if",
"colored",
"cookie_store 0.14.1",
"futures",
"futures-channel",
"futures-util",
@ -20,6 +21,7 @@ dependencies = [
"once_cell",
"regex",
"reqwest",
"reqwest_cookie_store",
"rpassword",
"rprompt",
"scraper",
@ -175,9 +177,9 @@ dependencies = [
[[package]]
name = "bumpalo"
version = "3.6.1"
version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe"
checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
[[package]]
name = "byteorder"
@ -274,16 +276,43 @@ dependencies = [
"version_check",
]
[[package]]
name = "cookie"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffdf8865bac3d9a3bde5bde9088ca431b11f5d37c7a578b8086af77248b76627"
dependencies = [
"percent-encoding",
"time",
"version_check",
]
[[package]]
name = "cookie_store"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3"
dependencies = [
"cookie",
"cookie 0.14.4",
"idna",
"log",
"publicsuffix",
"publicsuffix 1.5.6",
"serde",
"serde_json",
"time",
"url",
]
[[package]]
name = "cookie_store"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ba4e410d33a326cee69c778c761b6a822469ed7c6b82aa7802bef6e12716060"
dependencies = [
"cookie 0.15.0",
"idna",
"log",
"publicsuffix 2.1.0",
"serde",
"serde_json",
"time",
@ -659,6 +688,12 @@ version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
[[package]]
name = "hashbrown"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
[[package]]
name = "heck"
version = "0.3.2"
@ -820,7 +855,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
dependencies = [
"autocfg",
"hashbrown",
"hashbrown 0.9.1",
]
[[package]]
@ -1232,6 +1267,12 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "psl-types"
version = "2.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66b398073e7cdd6f05934389a8f5961e3aabfa66675b6f440df4e2c793d51a4f"
[[package]]
name = "publicsuffix"
version = "1.5.6"
@ -1242,6 +1283,18 @@ dependencies = [
"url",
]
[[package]]
name = "publicsuffix"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3ac055aef7cc7a1caefbc65144be879e862467dcd9b8a8d57b64a13e7dce15d"
dependencies = [
"byteorder",
"hashbrown 0.11.2",
"idna",
"psl-types",
]
[[package]]
name = "quote"
version = "1.0.9"
@ -1328,8 +1381,8 @@ dependencies = [
"async-compression",
"base64",
"bytes",
"cookie",
"cookie_store",
"cookie 0.14.4",
"cookie_store 0.12.0",
"encoding_rs",
"futures-core",
"futures-util",
@ -1361,6 +1414,19 @@ dependencies = [
"winreg",
]
[[package]]
name = "reqwest_cookie_store"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a81253b4f49eba884e5b555d08617d2b59be9478fcb53875ded9825d957af4a"
dependencies = [
"bytes",
"cookie 0.15.0",
"cookie_store 0.14.1",
"reqwest",
"url",
]
[[package]]
name = "ring"
version = "0.16.20"
@ -1891,9 +1957,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.6.0"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37"
checksum = "0a38d31d7831c6ed7aad00aa4c12d9375fd225a6dd77da1d25b707346319a975"
dependencies = [
"autocfg",
"bytes",
@ -2002,9 +2068,9 @@ dependencies = [
[[package]]
name = "unicode-normalization"
version = "0.1.17"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef"
checksum = "33717dca7ac877f497014e10d73f3acf948c342bee31b5ca7892faf94ccc6b49"
dependencies = [
"tinyvec",
]

View File

@ -30,6 +30,8 @@ indicatif = "0.16.0"
once_cell = "1.7.2"
atty = "0.2.14"
h2 = "0.3.3"
cookie_store = "0.14.0"
reqwest_cookie_store = "0.1.5"
[features]
default = []

View File

@ -12,7 +12,7 @@ use indicatif::ProgressBar;
use once_cell::sync::Lazy;
use structopt::StructOpt;
#[derive(Debug, StructOpt)]
#[derive(Debug, Clone, StructOpt)]
#[structopt(name = env!("CARGO_PKG_NAME"))]
pub struct Opt {
/// Do not download files
@ -86,7 +86,7 @@ pub static PROGRESS_BAR_ENABLED: AtomicBool = AtomicBool::new(false);
pub static PROGRESS_BAR: Lazy<ProgressBar> = Lazy::new(|| ProgressBar::new(0));
macro_rules! log {
($lvl:expr, $($t:expr),+) => {
($lvl:expr, $($t:expr),+) => {{
#[allow(unused_comparisons)] // 0 <= 0
if $lvl <= crate::cli::LOG_LEVEL.load(std::sync::atomic::Ordering::SeqCst) {
if crate::cli::PROGRESS_BAR_ENABLED.load(std::sync::atomic::Ordering::SeqCst) {
@ -95,7 +95,7 @@ macro_rules! log {
println!($($t),+);
}
}
}
}}
}
macro_rules! info {

View File

@ -1,11 +1,13 @@
// SPDX-License-Identifier: GPL-3.0-or-later
use std::error::Error as _;
use std::{error::Error as _, sync::Arc};
use anyhow::{anyhow, Context, Result};
use colored::Colorize;
use cookie_store::CookieStore;
use ignore::gitignore::Gitignore;
use reqwest::{Client, IntoUrl, Proxy, Url};
use reqwest_cookie_store::CookieStoreMutex;
use scraper::{ElementRef, Html, Selector};
use serde_json::json;
@ -14,10 +16,8 @@ use crate::{cli::Opt, get_request_ticket, selectors::*, ILIAS_URL};
pub struct ILIAS {
pub opt: Opt,
pub ignore: Gitignore,
// TODO: use these for re-authentication in case of session timeout/invalidation
user: String,
pass: String,
client: Client,
cookies: Arc<CookieStoreMutex>,
}
/// Returns true if the error is caused by:
@ -36,25 +36,49 @@ fn error_is_http2(error: &reqwest::Error) -> bool {
}
impl ILIAS {
pub async fn login(opt: Opt, user: impl Into<String>, pass: impl Into<String>, ignore: Gitignore) -> Result<Self> {
let user = user.into();
let pass = pass.into();
let mut builder = Client::builder()
.cookie_store(true)
// TODO: de-duplicate the logic below
pub async fn with_session(opt: Opt, session: Arc<CookieStoreMutex>, ignore: Gitignore) -> Result<Self> {
let mut builder =
Client::builder()
.cookie_provider(Arc::clone(&session))
.user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")));
if let Some(proxy) = opt.proxy.as_ref() {
let proxy = Proxy::all(proxy)?;
builder = builder.proxy(proxy);
}
let client = builder
// timeout is infinite by default
.build()?;
info!("Re-using previous session cookies..");
Ok(ILIAS {
opt,
ignore,
client,
cookies: session,
})
}
pub async fn login(opt: Opt, user: &str, pass: &str, ignore: Gitignore) -> Result<Self> {
let cookie_store = CookieStore::default();
let cookie_store = reqwest_cookie_store::CookieStoreMutex::new(cookie_store);
let cookie_store = std::sync::Arc::new(cookie_store);
let mut builder = Client::builder().cookie_provider(Arc::clone(&cookie_store)).user_agent(concat!(
env!("CARGO_PKG_NAME"),
"/",
env!("CARGO_PKG_VERSION")
));
if let Some(proxy) = opt.proxy.as_ref() {
let proxy = Proxy::all(proxy)?;
builder = builder.proxy(proxy);
}
let client = builder
// timeout is infinite by default
.build()?;
let this = ILIAS {
opt,
ignore,
user,
pass,
client,
cookies: cookie_store,
};
info!("Logging into ILIAS using KIT account..");
let session_establishment = this
@ -75,14 +99,16 @@ impl ILIAS {
.select(&Selector::parse(r#"input[name="csrf_token"]"#).unwrap())
.next()
.context("no CSRF token found")?
.value().attr("value").context("no CSRF token value")?;
.value()
.attr("value")
.context("no CSRF token value")?;
info!("Logging into Shibboleth..");
let login_response = this
.client
.post(url)
.form(&json!({
"j_username": &this.user,
"j_password": &this.pass,
"j_username": user,
"j_password": pass,
"_eventId_proceed": "",
"csrf_token": csrf_token,
}))
@ -108,6 +134,14 @@ impl ILIAS {
Ok(this)
}
pub async fn save_session(&self) -> Result<()> {
let session_path = self.opt.output.join(".iliassession");
let mut writer = std::fs::File::create(session_path).map(std::io::BufWriter::new).unwrap();
let store = self.cookies.lock().map_err(|x| anyhow!("{}", x))?;
store.save_json(&mut writer).map_err(|x| anyhow!(x))?;
Ok(())
}
pub async fn download(&self, url: &str) -> Result<reqwest::Response> {
get_request_ticket().await;
log!(2, "Downloading {}", url);

View File

@ -21,9 +21,11 @@ use url::Url;
use std::collections::HashSet;
use std::future::Future;
use std::io;
use std::io::BufReader;
use std::path::PathBuf;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::SystemTime;
pub const ILIAS_URL: &str = "https://ilias.studium.kit.edu/";
@ -67,26 +69,41 @@ async fn main() {
}
}
async fn real_main(mut opt: Opt) -> Result<()> {
LOG_LEVEL.store(opt.verbose, Ordering::SeqCst);
#[cfg(windows)]
let _ = colored::control::set_virtual_terminal(true);
create_dir(&opt.output).await.context("failed to create output directory")?;
// use UNC paths on Windows (#6)
opt.output = fs::canonicalize(opt.output).await.context("failed to canonicalize output directory")?;
// load .iliasignore file
opt.output.push(".iliasignore");
let (ignore, error) = Gitignore::new(&opt.output);
if let Some(err) = error {
warning!(err);
async fn try_to_load_session(opt: Opt, ignore: Gitignore) -> Result<ILIAS> {
let session_path = opt.output.join(".iliassession");
let meta = tokio::fs::metadata(&session_path).await?;
let modified = meta.modified()?;
let now = SystemTime::now();
// the previous session is only useful if it isn't older than ~1 hour
let duration = now.duration_since(modified)?;
if duration.as_secs() <= 60 * 60 {
let file = std::fs::File::open(session_path)?;
let cookies = cookie_store::CookieStore::load_json(BufReader::new(file))
.map_err(|err| anyhow!(err))
.context("failed to load session cookies")?;
let cookie_store = reqwest_cookie_store::CookieStoreMutex::new(cookies);
let cookie_store = std::sync::Arc::new(cookie_store);
Ok(ILIAS::with_session(opt, cookie_store, ignore).await?)
} else {
Err(anyhow!("session data too old"))
}
}
async fn login(opt: Opt, ignore: Gitignore) -> Result<ILIAS> {
// load .iliassession file
if opt.keep_session {
match try_to_load_session(opt.clone(), ignore.clone())
.await
.context("failed to load previous session")
{
Ok(ilias) => return Ok(ilias),
Err(e) => warning!(e),
}
}
opt.output.pop();
// loac .iliaslogin file
opt.output.push(".iliaslogin");
let login = std::fs::read_to_string(&opt.output);
let iliaslogin = opt.output.join(".iliaslogin");
let login = std::fs::read_to_string(&iliaslogin);
let (user, pass) = if let Ok(login) = login {
let mut lines = login.split('\n');
let user = lines.next().context("missing user in .iliaslogin")?;
@ -97,15 +114,34 @@ async fn real_main(mut opt: Opt) -> Result<()> {
} else {
ask_user_pass(&opt).context("credentials input failed")?
};
opt.output.pop();
let ilias = match ILIAS::login(opt, user, pass, ignore).await {
let ilias = match ILIAS::login(opt, &user, &pass, ignore).await {
Ok(ilias) => ilias,
Err(e) => {
error!(e);
std::process::exit(77);
},
};
Ok(ilias)
}
async fn real_main(mut opt: Opt) -> Result<()> {
LOG_LEVEL.store(opt.verbose, Ordering::SeqCst);
#[cfg(windows)]
let _ = colored::control::set_virtual_terminal(true);
create_dir(&opt.output).await.context("failed to create output directory")?;
// use UNC paths on Windows (to avoid the default max. path length of 255)
opt.output = fs::canonicalize(opt.output).await.context("failed to canonicalize output directory")?;
// load .iliasignore file
let (ignore, error) = Gitignore::new(opt.output.join(".iliasignore"));
if let Some(err) = error {
warning!(err);
}
let ilias = login(opt, ignore).await?;
if ilias.opt.content_tree {
if let Err(e) = ilias
.download("ilias.php?baseClass=ilRepositoryGUI&cmd=frameset&set_mode=tree&ref_id=1")
@ -149,6 +185,11 @@ async fn real_main(mut opt: Opt) -> Result<()> {
warning!("could not disable content tree:", e);
}
}
if ilias.opt.keep_session {
if let Err(e) = ilias.save_session().await.context("failed to save session cookies") {
warning!(e)
}
}
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}] {wide_msg}"));
PROGRESS_BAR.finish_with_message("done");