mirror of
https://github.com/FliegendeWurst/KIT-ILIAS-downloader.git
synced 2024-08-28 04:04:18 +00:00
Session re-use
This commit is contained in:
FliegendeWurst
parent
5fb2faabfd
commit
8ea2cae769
@ -6,7 +6,8 @@ and this project loosely adheres to [Semantic Versioning](https://semver.org/spe
|
||||
|
||||
## [Unreleased]
|
||||
### Added
|
||||
- `--sync-url https://ilias.studium.kit.edu/ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToMemberships` to download all courses (unless specified otherwise in the `.iliasignore`)
|
||||
- `--sync-url` can now download all courses (unless specified otherwise in the `.iliasignore`)
|
||||
- `--keep-session` flag to save and restore session cookies
|
||||
|
||||
## [0.2.21] - 2021-05-18
|
||||
### Fixed
|
||||
|
||||
88
Cargo.lock
generated
88
Cargo.lock
generated
@ -10,6 +10,7 @@ dependencies = [
|
||||
"atty",
|
||||
"cfg-if",
|
||||
"colored",
|
||||
"cookie_store 0.14.1",
|
||||
"futures",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
@ -20,6 +21,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"reqwest_cookie_store",
|
||||
"rpassword",
|
||||
"rprompt",
|
||||
"scraper",
|
||||
@ -175,9 +177,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.6.1"
|
||||
version = "3.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe"
|
||||
checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
@ -274,16 +276,43 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cookie"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ffdf8865bac3d9a3bde5bde9088ca431b11f5d37c7a578b8086af77248b76627"
|
||||
dependencies = [
|
||||
"percent-encoding",
|
||||
"time",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cookie_store"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3818dfca4b0cb5211a659bbcbb94225b7127407b2b135e650d717bfb78ab10d3"
|
||||
dependencies = [
|
||||
"cookie",
|
||||
"cookie 0.14.4",
|
||||
"idna",
|
||||
"log",
|
||||
"publicsuffix",
|
||||
"publicsuffix 1.5.6",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"time",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cookie_store"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ba4e410d33a326cee69c778c761b6a822469ed7c6b82aa7802bef6e12716060"
|
||||
dependencies = [
|
||||
"cookie 0.15.0",
|
||||
"idna",
|
||||
"log",
|
||||
"publicsuffix 2.1.0",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"time",
|
||||
@ -659,6 +688,12 @@ version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.3.2"
|
||||
@ -820,7 +855,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"hashbrown",
|
||||
"hashbrown 0.9.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1232,6 +1267,12 @@ dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psl-types"
|
||||
version = "2.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66b398073e7cdd6f05934389a8f5961e3aabfa66675b6f440df4e2c793d51a4f"
|
||||
|
||||
[[package]]
|
||||
name = "publicsuffix"
|
||||
version = "1.5.6"
|
||||
@ -1242,6 +1283,18 @@ dependencies = [
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "publicsuffix"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3ac055aef7cc7a1caefbc65144be879e862467dcd9b8a8d57b64a13e7dce15d"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"hashbrown 0.11.2",
|
||||
"idna",
|
||||
"psl-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.9"
|
||||
@ -1328,8 +1381,8 @@ dependencies = [
|
||||
"async-compression",
|
||||
"base64",
|
||||
"bytes",
|
||||
"cookie",
|
||||
"cookie_store",
|
||||
"cookie 0.14.4",
|
||||
"cookie_store 0.12.0",
|
||||
"encoding_rs",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
@ -1361,6 +1414,19 @@ dependencies = [
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest_cookie_store"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a81253b4f49eba884e5b555d08617d2b59be9478fcb53875ded9825d957af4a"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"cookie 0.15.0",
|
||||
"cookie_store 0.14.1",
|
||||
"reqwest",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ring"
|
||||
version = "0.16.20"
|
||||
@ -1891,9 +1957,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.6.0"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd3076b5c8cc18138b8f8814895c11eb4de37114a5d127bafdc5e55798ceef37"
|
||||
checksum = "0a38d31d7831c6ed7aad00aa4c12d9375fd225a6dd77da1d25b707346319a975"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"bytes",
|
||||
@ -2002,9 +2068,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.17"
|
||||
version = "0.1.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef"
|
||||
checksum = "33717dca7ac877f497014e10d73f3acf948c342bee31b5ca7892faf94ccc6b49"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
@ -30,6 +30,8 @@ indicatif = "0.16.0"
|
||||
once_cell = "1.7.2"
|
||||
atty = "0.2.14"
|
||||
h2 = "0.3.3"
|
||||
cookie_store = "0.14.0"
|
||||
reqwest_cookie_store = "0.1.5"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
|
||||
@ -12,7 +12,7 @@ use indicatif::ProgressBar;
|
||||
use once_cell::sync::Lazy;
|
||||
use structopt::StructOpt;
|
||||
|
||||
#[derive(Debug, StructOpt)]
|
||||
#[derive(Debug, Clone, StructOpt)]
|
||||
#[structopt(name = env!("CARGO_PKG_NAME"))]
|
||||
pub struct Opt {
|
||||
/// Do not download files
|
||||
@ -86,7 +86,7 @@ pub static PROGRESS_BAR_ENABLED: AtomicBool = AtomicBool::new(false);
|
||||
pub static PROGRESS_BAR: Lazy<ProgressBar> = Lazy::new(|| ProgressBar::new(0));
|
||||
|
||||
macro_rules! log {
|
||||
($lvl:expr, $($t:expr),+) => {
|
||||
($lvl:expr, $($t:expr),+) => {{
|
||||
#[allow(unused_comparisons)] // 0 <= 0
|
||||
if $lvl <= crate::cli::LOG_LEVEL.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
if crate::cli::PROGRESS_BAR_ENABLED.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
@ -95,7 +95,7 @@ macro_rules! log {
|
||||
println!($($t),+);
|
||||
}
|
||||
}
|
||||
}
|
||||
}}
|
||||
}
|
||||
|
||||
macro_rules! info {
|
||||
|
||||
64
src/ilias.rs
64
src/ilias.rs
@ -1,11 +1,13 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
use std::error::Error as _;
|
||||
use std::{error::Error as _, sync::Arc};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use colored::Colorize;
|
||||
use cookie_store::CookieStore;
|
||||
use ignore::gitignore::Gitignore;
|
||||
use reqwest::{Client, IntoUrl, Proxy, Url};
|
||||
use reqwest_cookie_store::CookieStoreMutex;
|
||||
use scraper::{ElementRef, Html, Selector};
|
||||
use serde_json::json;
|
||||
|
||||
@ -14,10 +16,8 @@ use crate::{cli::Opt, get_request_ticket, selectors::*, ILIAS_URL};
|
||||
pub struct ILIAS {
|
||||
pub opt: Opt,
|
||||
pub ignore: Gitignore,
|
||||
// TODO: use these for re-authentication in case of session timeout/invalidation
|
||||
user: String,
|
||||
pass: String,
|
||||
client: Client,
|
||||
cookies: Arc<CookieStoreMutex>,
|
||||
}
|
||||
|
||||
/// Returns true if the error is caused by:
|
||||
@ -36,12 +36,37 @@ fn error_is_http2(error: &reqwest::Error) -> bool {
|
||||
}
|
||||
|
||||
impl ILIAS {
|
||||
pub async fn login(opt: Opt, user: impl Into<String>, pass: impl Into<String>, ignore: Gitignore) -> Result<Self> {
|
||||
let user = user.into();
|
||||
let pass = pass.into();
|
||||
let mut builder = Client::builder()
|
||||
.cookie_store(true)
|
||||
.user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")));
|
||||
// TODO: de-duplicate the logic below
|
||||
pub async fn with_session(opt: Opt, session: Arc<CookieStoreMutex>, ignore: Gitignore) -> Result<Self> {
|
||||
let mut builder =
|
||||
Client::builder()
|
||||
.cookie_provider(Arc::clone(&session))
|
||||
.user_agent(concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")));
|
||||
if let Some(proxy) = opt.proxy.as_ref() {
|
||||
let proxy = Proxy::all(proxy)?;
|
||||
builder = builder.proxy(proxy);
|
||||
}
|
||||
let client = builder
|
||||
// timeout is infinite by default
|
||||
.build()?;
|
||||
info!("Re-using previous session cookies..");
|
||||
Ok(ILIAS {
|
||||
opt,
|
||||
ignore,
|
||||
client,
|
||||
cookies: session,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn login(opt: Opt, user: &str, pass: &str, ignore: Gitignore) -> Result<Self> {
|
||||
let cookie_store = CookieStore::default();
|
||||
let cookie_store = reqwest_cookie_store::CookieStoreMutex::new(cookie_store);
|
||||
let cookie_store = std::sync::Arc::new(cookie_store);
|
||||
let mut builder = Client::builder().cookie_provider(Arc::clone(&cookie_store)).user_agent(concat!(
|
||||
env!("CARGO_PKG_NAME"),
|
||||
"/",
|
||||
env!("CARGO_PKG_VERSION")
|
||||
));
|
||||
if let Some(proxy) = opt.proxy.as_ref() {
|
||||
let proxy = Proxy::all(proxy)?;
|
||||
builder = builder.proxy(proxy);
|
||||
@ -52,9 +77,8 @@ impl ILIAS {
|
||||
let this = ILIAS {
|
||||
opt,
|
||||
ignore,
|
||||
user,
|
||||
pass,
|
||||
client,
|
||||
cookies: cookie_store,
|
||||
};
|
||||
info!("Logging into ILIAS using KIT account..");
|
||||
let session_establishment = this
|
||||
@ -75,14 +99,16 @@ impl ILIAS {
|
||||
.select(&Selector::parse(r#"input[name="csrf_token"]"#).unwrap())
|
||||
.next()
|
||||
.context("no CSRF token found")?
|
||||
.value().attr("value").context("no CSRF token value")?;
|
||||
.value()
|
||||
.attr("value")
|
||||
.context("no CSRF token value")?;
|
||||
info!("Logging into Shibboleth..");
|
||||
let login_response = this
|
||||
.client
|
||||
.post(url)
|
||||
.form(&json!({
|
||||
"j_username": &this.user,
|
||||
"j_password": &this.pass,
|
||||
"j_username": user,
|
||||
"j_password": pass,
|
||||
"_eventId_proceed": "",
|
||||
"csrf_token": csrf_token,
|
||||
}))
|
||||
@ -108,6 +134,14 @@ impl ILIAS {
|
||||
Ok(this)
|
||||
}
|
||||
|
||||
pub async fn save_session(&self) -> Result<()> {
|
||||
let session_path = self.opt.output.join(".iliassession");
|
||||
let mut writer = std::fs::File::create(session_path).map(std::io::BufWriter::new).unwrap();
|
||||
let store = self.cookies.lock().map_err(|x| anyhow!("{}", x))?;
|
||||
store.save_json(&mut writer).map_err(|x| anyhow!(x))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn download(&self, url: &str) -> Result<reqwest::Response> {
|
||||
get_request_ticket().await;
|
||||
log!(2, "Downloading {}", url);
|
||||
|
||||
79
src/main.rs
79
src/main.rs
@ -21,9 +21,11 @@ use url::Url;
|
||||
use std::collections::HashSet;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::io::BufReader;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
pub const ILIAS_URL: &str = "https://ilias.studium.kit.edu/";
|
||||
|
||||
@ -67,26 +69,41 @@ async fn main() {
|
||||
}
|
||||
}
|
||||
|
||||
async fn real_main(mut opt: Opt) -> Result<()> {
|
||||
LOG_LEVEL.store(opt.verbose, Ordering::SeqCst);
|
||||
#[cfg(windows)]
|
||||
let _ = colored::control::set_virtual_terminal(true);
|
||||
|
||||
create_dir(&opt.output).await.context("failed to create output directory")?;
|
||||
// use UNC paths on Windows (#6)
|
||||
opt.output = fs::canonicalize(opt.output).await.context("failed to canonicalize output directory")?;
|
||||
|
||||
// load .iliasignore file
|
||||
opt.output.push(".iliasignore");
|
||||
let (ignore, error) = Gitignore::new(&opt.output);
|
||||
if let Some(err) = error {
|
||||
warning!(err);
|
||||
async fn try_to_load_session(opt: Opt, ignore: Gitignore) -> Result<ILIAS> {
|
||||
let session_path = opt.output.join(".iliassession");
|
||||
let meta = tokio::fs::metadata(&session_path).await?;
|
||||
let modified = meta.modified()?;
|
||||
let now = SystemTime::now();
|
||||
// the previous session is only useful if it isn't older than ~1 hour
|
||||
let duration = now.duration_since(modified)?;
|
||||
if duration.as_secs() <= 60 * 60 {
|
||||
let file = std::fs::File::open(session_path)?;
|
||||
let cookies = cookie_store::CookieStore::load_json(BufReader::new(file))
|
||||
.map_err(|err| anyhow!(err))
|
||||
.context("failed to load session cookies")?;
|
||||
let cookie_store = reqwest_cookie_store::CookieStoreMutex::new(cookies);
|
||||
let cookie_store = std::sync::Arc::new(cookie_store);
|
||||
Ok(ILIAS::with_session(opt, cookie_store, ignore).await?)
|
||||
} else {
|
||||
Err(anyhow!("session data too old"))
|
||||
}
|
||||
}
|
||||
|
||||
async fn login(opt: Opt, ignore: Gitignore) -> Result<ILIAS> {
|
||||
// load .iliassession file
|
||||
if opt.keep_session {
|
||||
match try_to_load_session(opt.clone(), ignore.clone())
|
||||
.await
|
||||
.context("failed to load previous session")
|
||||
{
|
||||
Ok(ilias) => return Ok(ilias),
|
||||
Err(e) => warning!(e),
|
||||
}
|
||||
}
|
||||
opt.output.pop();
|
||||
|
||||
// loac .iliaslogin file
|
||||
opt.output.push(".iliaslogin");
|
||||
let login = std::fs::read_to_string(&opt.output);
|
||||
let iliaslogin = opt.output.join(".iliaslogin");
|
||||
let login = std::fs::read_to_string(&iliaslogin);
|
||||
let (user, pass) = if let Ok(login) = login {
|
||||
let mut lines = login.split('\n');
|
||||
let user = lines.next().context("missing user in .iliaslogin")?;
|
||||
@ -97,15 +114,34 @@ async fn real_main(mut opt: Opt) -> Result<()> {
|
||||
} else {
|
||||
ask_user_pass(&opt).context("credentials input failed")?
|
||||
};
|
||||
opt.output.pop();
|
||||
|
||||
let ilias = match ILIAS::login(opt, user, pass, ignore).await {
|
||||
let ilias = match ILIAS::login(opt, &user, &pass, ignore).await {
|
||||
Ok(ilias) => ilias,
|
||||
Err(e) => {
|
||||
error!(e);
|
||||
std::process::exit(77);
|
||||
},
|
||||
};
|
||||
Ok(ilias)
|
||||
}
|
||||
|
||||
async fn real_main(mut opt: Opt) -> Result<()> {
|
||||
LOG_LEVEL.store(opt.verbose, Ordering::SeqCst);
|
||||
#[cfg(windows)]
|
||||
let _ = colored::control::set_virtual_terminal(true);
|
||||
|
||||
create_dir(&opt.output).await.context("failed to create output directory")?;
|
||||
// use UNC paths on Windows (to avoid the default max. path length of 255)
|
||||
opt.output = fs::canonicalize(opt.output).await.context("failed to canonicalize output directory")?;
|
||||
|
||||
// load .iliasignore file
|
||||
let (ignore, error) = Gitignore::new(opt.output.join(".iliasignore"));
|
||||
if let Some(err) = error {
|
||||
warning!(err);
|
||||
}
|
||||
|
||||
let ilias = login(opt, ignore).await?;
|
||||
|
||||
if ilias.opt.content_tree {
|
||||
if let Err(e) = ilias
|
||||
.download("ilias.php?baseClass=ilRepositoryGUI&cmd=frameset&set_mode=tree&ref_id=1")
|
||||
@ -149,6 +185,11 @@ async fn real_main(mut opt: Opt) -> Result<()> {
|
||||
warning!("could not disable content tree:", e);
|
||||
}
|
||||
}
|
||||
if ilias.opt.keep_session {
|
||||
if let Err(e) = ilias.save_session().await.context("failed to save session cookies") {
|
||||
warning!(e)
|
||||
}
|
||||
}
|
||||
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
|
||||
PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}] {wide_msg}"));
|
||||
PROGRESS_BAR.finish_with_message("done");
|
||||
|
||||
Loading…
Reference in New Issue
Block a user