mirror of
https://github.com/FliegendeWurst/KIT-ILIAS-downloader.git
synced 2024-08-28 04:04:18 +00:00
Use anyhow instead of error-chain for errors
This commit is contained in:
FliegendeWurst
parent
fc9afdd66a
commit
a3f84aa6fb
57
Cargo.lock
generated
57
Cargo.lock
generated
@ -4,7 +4,7 @@
|
||||
name = "KIT-ILIAS-downloader"
|
||||
version = "0.2.4"
|
||||
dependencies = [
|
||||
"error-chain",
|
||||
"anyhow",
|
||||
"futures-util",
|
||||
"ignore",
|
||||
"lazy_static",
|
||||
@ -44,6 +44,12 @@ dependencies = [
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9a60d744a80c30fcb657dfe2c1b22bcb3e814c1a1e3674f32bf5820b570fbff"
|
||||
|
||||
[[package]]
|
||||
name = "arc-swap"
|
||||
version = "0.4.6"
|
||||
@ -86,28 +92,6 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1e692897359247cc6bb902933361652380af0f1b7651ae5c5013407f30e109e"
|
||||
dependencies = [
|
||||
"backtrace-sys",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"rustc-demangle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backtrace-sys"
|
||||
version = "0.1.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "18fbebbe1c9d1f383a9cc7e8ccdb471b91c8d024ee9c2ca5b5346121fe8b4399"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.11.0"
|
||||
@ -332,7 +316,6 @@ version = "0.12.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d371106cc88ffdfb1eabd7111e432da544f16f3e2d7bf1dfe8bf575f1df045cd"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
@ -956,18 +939,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pin-project"
|
||||
version = "0.4.10"
|
||||
version = "0.4.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36e3dcd42688c05a66f841d22c5d8390d9a5d4c9aaf57b9285eae4900a080063"
|
||||
checksum = "82c3bfbfb5bb42f99498c7234bbd768c220eb0cea6818259d0d18a1aa3d2595d"
|
||||
dependencies = [
|
||||
"pin-project-internal",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-internal"
|
||||
version = "0.4.10"
|
||||
version = "0.4.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4d7346ac577ff1296e06a418e7618e22655bae834d4970cb6e39d6da8119969"
|
||||
checksum = "ccbf6449dcfb18562c015526b085b8df1aa3cdab180af8ec2ebd300a3bd28f63"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@ -976,9 +959,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.1.4"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae"
|
||||
checksum = "f7505eeebd78492e0f6108f7171c4948dbb120ee8119d9d77d0afa5469bef67f"
|
||||
|
||||
[[package]]
|
||||
name = "pin-utils"
|
||||
@ -1277,12 +1260,6 @@ version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b386f4748bdae2aefc96857f5fda07647f851d089420e577831e2a14b45230f8"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.17.0"
|
||||
@ -1367,9 +1344,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "0.4.3"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f331b9025654145cd425b9ded0caf8f5ae0df80d418b326e2dc1c3dc5eb0620"
|
||||
checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"core-foundation",
|
||||
@ -1587,9 +1564,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.18"
|
||||
version = "1.0.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "410a7488c0a728c7ceb4ad59b9567eb4053d02e8cc7f5c0e0eeeb39518369213"
|
||||
checksum = "e8e5aa70697bb26ee62214ae3288465ecec0000f05182f039b477001f08f5ae7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
@ -9,7 +9,6 @@ edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
reqwest = { version = "0.10.4", default-features = false, features = ["cookies", "gzip", "json", "rustls-tls", "stream"] }
|
||||
error-chain = "0.12.2"
|
||||
tokio = { version = "0.2", features = ["full"] }
|
||||
serde_json = "1.0.51"
|
||||
scraper = "0.11.0"
|
||||
@ -22,3 +21,4 @@ structopt = "0.3.13"
|
||||
rpassword = "4.0.5"
|
||||
rprompt = "1.0.5"
|
||||
ignore = "0.4.14"
|
||||
anyhow = "1.0.28"
|
||||
|
||||
@ -1,25 +0,0 @@
|
||||
use error_chain::error_chain;
|
||||
|
||||
use super::*;
|
||||
|
||||
error_chain! {
|
||||
types {
|
||||
Error, ErrorKind, ResultExt, Result;
|
||||
}
|
||||
|
||||
links {
|
||||
|
||||
}
|
||||
|
||||
foreign_links {
|
||||
Io(std::io::Error);
|
||||
Json(serde_json::Error);
|
||||
Reqwest(reqwest::Error);
|
||||
}
|
||||
|
||||
errors {
|
||||
|
||||
}
|
||||
|
||||
//skip_msg_variant
|
||||
}
|
||||
81
src/main.rs
81
src/main.rs
@ -1,4 +1,4 @@
|
||||
use error_chain::ChainedError;
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use futures_util::stream::TryStreamExt;
|
||||
use ignore::gitignore::Gitignore;
|
||||
use lazy_static::lazy_static;
|
||||
@ -21,9 +21,6 @@ use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
mod errors;
|
||||
use errors::*;
|
||||
|
||||
const ILIAS_URL: &'static str = "https://ilias.studium.kit.edu/";
|
||||
|
||||
struct ILIAS {
|
||||
@ -139,7 +136,7 @@ impl Object {
|
||||
|
||||
fn from_link(item: ElementRef, link: ElementRef) -> Result<Self> {
|
||||
let mut name = link.text().collect::<String>().replace('/', "-").trim().to_owned();
|
||||
let mut url = URL::from_href(link.value().attr("href").ok_or::<Error>("link missing href".into())?);
|
||||
let mut url = URL::from_href(link.value().attr("href").context("link missing href")?);
|
||||
|
||||
if url.thr_pk.is_some() {
|
||||
return Ok(Thread {
|
||||
@ -194,7 +191,7 @@ impl Object {
|
||||
});
|
||||
}
|
||||
if url.target.as_ref().map(|x| x.starts_with("file_")).unwrap_or(false) {
|
||||
let target = url.target.as_ref().ok_or("no download target")?;
|
||||
let target = url.target.as_ref().ok_or(anyhow!("no download target"))?;
|
||||
if !target.ends_with("download") {
|
||||
// download page containing metadata
|
||||
return Ok(Generic {
|
||||
@ -204,8 +201,8 @@ impl Object {
|
||||
} else {
|
||||
let item_prop = Selector::parse("span.il_ItemProperty").unwrap();
|
||||
let mut item_props = item.select(&item_prop);
|
||||
let ext = item_props.next().ok_or("cannot find file extension")?;
|
||||
let version = item_props.nth(1).ok_or("cannot find 3rd file metadata")?.text().collect::<String>();
|
||||
let ext = item_props.next().ok_or(anyhow!("cannot find file extension"))?;
|
||||
let version = item_props.nth(1).ok_or(anyhow!("cannot find 3rd file metadata"))?.text().collect::<String>();
|
||||
let version = version.trim();
|
||||
if version.starts_with("Version: ") {
|
||||
name.push_str("_v");
|
||||
@ -362,15 +359,15 @@ impl ILIAS {
|
||||
login_soup = BeautifulSoup(otp_response.text, 'lxml')
|
||||
*/
|
||||
let saml = Selector::parse(r#"input[name="SAMLResponse"]"#).unwrap();
|
||||
let saml = dom.select(&saml).next().ok_or::<Error>("no SAML response, incorrect password?".into())?;
|
||||
let saml = dom.select(&saml).next().context("no SAML response, incorrect password?")?;
|
||||
let relay_state = Selector::parse(r#"input[name="RelayState"]"#).unwrap();
|
||||
let relay_state = dom.select(&relay_state).next().ok_or::<Error>("no relay state".into())?;
|
||||
let relay_state = dom.select(&relay_state).next().context("no relay state")?;
|
||||
println!("Logging into ILIAS..");
|
||||
this.client
|
||||
.post("https://ilias.studium.kit.edu/Shibboleth.sso/SAML2/POST")
|
||||
.form(&json!({
|
||||
"SAMLResponse": saml.value().attr("value").ok_or("no SAML value")?,
|
||||
"RelayState": relay_state.value().attr("value").ok_or("no RelayState value")?
|
||||
"SAMLResponse": saml.value().attr("value").ok_or(anyhow!("no SAML value"))?,
|
||||
"RelayState": relay_state.value().attr("value").ok_or(anyhow!("no RelayState value"))?
|
||||
}))
|
||||
.send().await?;
|
||||
println!("Logged in!");
|
||||
@ -392,7 +389,7 @@ impl ILIAS {
|
||||
let text = self.download(url).await?.text().await?;
|
||||
let html = Html::parse_document(&text);
|
||||
if html.select(&alert_danger).next().is_some() {
|
||||
Err("ILIAS error".into())
|
||||
Err(anyhow!("ILIAS error"))
|
||||
} else {
|
||||
Ok(html)
|
||||
}
|
||||
@ -402,7 +399,7 @@ impl ILIAS {
|
||||
let text = self.download(url).await?.text().await?;
|
||||
let html = Html::parse_fragment(&text);
|
||||
if html.select(&alert_danger).next().is_some() {
|
||||
Err("ILIAS error".into())
|
||||
Err(anyhow!("ILIAS error"))
|
||||
} else {
|
||||
Ok(html)
|
||||
}
|
||||
@ -416,7 +413,7 @@ impl ILIAS {
|
||||
.select(&container_item_title)
|
||||
.next()
|
||||
.map(|link| Object::from_link(item, link))
|
||||
.unwrap_or_else(|| Err("can't find link".into()))
|
||||
.unwrap_or_else(|| Err(anyhow!("can't find link")))
|
||||
}).collect()
|
||||
}
|
||||
|
||||
@ -471,18 +468,18 @@ async fn main() {
|
||||
let ilias = match ILIAS::login(opt, user, pass).await {
|
||||
Ok(ilias) => ilias,
|
||||
Err(e) => {
|
||||
print!("{}", e.display_chain());
|
||||
print!("{:?}", e);
|
||||
std::process::exit(77);
|
||||
}
|
||||
};
|
||||
if ilias.opt.content_tree {
|
||||
// need this to get the content tree
|
||||
if let Err(e) = ilias.client.get("https://ilias.studium.kit.edu/ilias.php?baseClass=ilRepositoryGUI&cmd=frameset&set_mode=tree&ref_id=1").send().await {
|
||||
println!("Warning: could not enable content tree: {}", e);
|
||||
println!("Warning: could not enable content tree: {:?}", e);
|
||||
}
|
||||
}
|
||||
let ilias = Arc::new(ilias);
|
||||
let desktop = ilias.personal_desktop().await;
|
||||
let desktop = ilias.personal_desktop().await.context("Failed to load personal desktop");
|
||||
match desktop {
|
||||
Ok(desktop) => {
|
||||
for item in desktop.items {
|
||||
@ -494,7 +491,7 @@ async fn main() {
|
||||
});
|
||||
}
|
||||
},
|
||||
Err(e) => println!("Error getting personal desktop: {}", e.display_chain())
|
||||
Err(e) => println!("{:?}", e)
|
||||
}
|
||||
// TODO: do this with tokio
|
||||
// https://github.com/tokio-rs/tokio/issues/2039
|
||||
@ -504,7 +501,7 @@ async fn main() {
|
||||
if ilias.opt.content_tree {
|
||||
// restore fast page loading times
|
||||
if let Err(e) = ilias.client.get("https://ilias.studium.kit.edu/ilias.php?baseClass=ilRepositoryGUI&cmd=frameset&set_mode=flat&ref_id=1").send().await {
|
||||
println!("Warning: could not disable content tree: {}", e);
|
||||
println!("Warning: could not disable content tree: {:?}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -523,8 +520,8 @@ fn process_gracefully(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std
|
||||
}
|
||||
*TASKS_RUNNING.lock() += 1;
|
||||
let path_text = format!("{:?}", path);
|
||||
if let Err(e) = process(ilias, path, obj).await {
|
||||
print!("Error syncing {}: {}", path_text, e.display_chain());
|
||||
if let Err(e) = process(ilias, path, obj).await.context("Failed to process URL") {
|
||||
print!("Syncing {:?}: {:?}", path_text, e);
|
||||
}
|
||||
*TASKS_RUNNING.lock() -= 1;
|
||||
*TASKS_QUEUED.lock() -= 1;
|
||||
@ -580,7 +577,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
}
|
||||
let content = if ilias.opt.content_tree {
|
||||
let html = ilias.download(&url.url).await?.text().await?;
|
||||
let cmd_node = cmd_node_regex.find(&html).ok_or::<Error>("can't find cmdNode".into())?.as_str()[8..].to_owned();
|
||||
let cmd_node = cmd_node_regex.find(&html).context("can't find cmdNode")?.as_str()[8..].to_owned();
|
||||
let content_tree = ilias.get_course_content_tree(&url.ref_id, &cmd_node).await;
|
||||
match content_tree {
|
||||
Ok(tree) => tree,
|
||||
@ -590,7 +587,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
if html.contains(r#"input[name="cmd[join]""#) {
|
||||
return Ok(()); // ignore groups we are not in
|
||||
}
|
||||
println!("Warning: {:?} falling back to incomplete course content extractor! {}", name, e.display_chain());
|
||||
println!("Warning: {:?} falling back to incomplete course content extractor! {:?}", name, e);
|
||||
ilias.get_course_content(&url).await?.into_iter().flat_map(Result::ok).collect() // TODO: perhaps don't download almost the same content 3x
|
||||
}
|
||||
}
|
||||
@ -616,7 +613,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
for item in content {
|
||||
if item.is_err() {
|
||||
if ilias.opt.verbose > 0 {
|
||||
println!("Ignoring: {}", item.err().unwrap().display_chain());
|
||||
println!("Ignoring: {:?}", item.err().unwrap());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@ -681,7 +678,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
println!("Found video: {}", title);
|
||||
}
|
||||
let video = Video {
|
||||
url: URL::raw(link.value().attr("href").ok_or("video link without href")?.to_owned())
|
||||
url: URL::raw(link.value().attr("href").ok_or(anyhow!("video link without href"))?.to_owned())
|
||||
};
|
||||
let ilias = Arc::clone(&ilias);
|
||||
task::spawn(async {
|
||||
@ -712,11 +709,11 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
}
|
||||
let json: serde_json::Value = {
|
||||
let mut json_capture = XOCT_REGEX.captures_iter(&html);
|
||||
let json = &json_capture.next().ok_or::<Error>("xoct player json not found".into())?[1];
|
||||
let json = &json_capture.next().context("xoct player json not found")?[1];
|
||||
if ilias.opt.verbose > 1 {
|
||||
println!("{}", json);
|
||||
}
|
||||
let json = json.split(",\n").nth(0).ok_or::<Error>("invalid xoct player json".into())?;
|
||||
let json = json.split(",\n").nth(0).context("invalid xoct player json")?;
|
||||
serde_json::from_str(&json.trim())?
|
||||
};
|
||||
if ilias.opt.verbose > 1 {
|
||||
@ -725,8 +722,8 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
let url = json
|
||||
.pointer("/streams/0/sources/mp4/0/src")
|
||||
.map(|x| x.as_str())
|
||||
.ok_or("video src not found")?
|
||||
.ok_or("video src not string")?;
|
||||
.ok_or(anyhow!("video src not found"))?
|
||||
.ok_or(anyhow!("video src not string"))?;
|
||||
let resp = ilias.download(&url).await?;
|
||||
let mut reader = stream_reader(resp.bytes_stream().map_err(|x| {
|
||||
io::Error::new(io::ErrorKind::Other, x)
|
||||
@ -757,7 +754,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
.flat_map(|x| x.value().attr("href"))
|
||||
.filter(|x| x.contains("trows=800"))
|
||||
.next()
|
||||
.ok_or::<Error>("can't find forum thread count selector (empty forum?)".into())?.to_owned()
|
||||
.context("can't find forum thread count selector (empty forum?)")?.to_owned()
|
||||
};
|
||||
let data = ilias.download(&url);
|
||||
let html = data.await?.text().await?;
|
||||
@ -769,11 +766,11 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
println!("Warning: unusual table row ({} cells) in {}", cells.len(), url);
|
||||
continue;
|
||||
}
|
||||
let link = cells[1].select(&a).next().ok_or::<Error>("thread link not found".into())?;
|
||||
let link = cells[1].select(&a).next().context("thread link not found")?;
|
||||
let object = Object::from_link(link, link)?;
|
||||
let mut path = path.clone();
|
||||
let name = format!("{}_{}",
|
||||
object.url().thr_pk.as_ref().ok_or::<Error>("thr_pk not found for thread".into())?,
|
||||
object.url().thr_pk.as_ref().context("thr_pk not found for thread")?,
|
||||
link.text().collect::<String>().replace('/', "-").trim()
|
||||
);
|
||||
path.push(name);
|
||||
@ -784,7 +781,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
Err(_) => 0
|
||||
}
|
||||
};
|
||||
let available_posts = cells[3].text().next().unwrap_or_default().trim().parse::<usize>().chain_err(|| "parsing post count failed")?;
|
||||
let available_posts = cells[3].text().next().unwrap_or_default().trim().parse::<usize>().context("parsing post count failed")?;
|
||||
if available_posts <= saved_posts && !ilias.opt.force {
|
||||
continue;
|
||||
}
|
||||
@ -809,14 +806,14 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
}
|
||||
let html = ilias.get_html(&url.url).await?;
|
||||
for post in html.select(&post_row) {
|
||||
let title = post.select(&post_title).next().ok_or("post title not found")?.text().collect::<String>().replace('/', "-");
|
||||
let author = post.select(&span_small).next().ok_or("post author not found")?;
|
||||
let title = post.select(&post_title).next().ok_or(anyhow!("post title not found"))?.text().collect::<String>().replace('/', "-");
|
||||
let author = post.select(&span_small).next().ok_or(anyhow!("post author not found"))?;
|
||||
let author = author.text().collect::<String>();
|
||||
let author = author.trim().split('|').nth(1).ok_or("author data in unknown format")?.trim();
|
||||
let container = post.select(&post_container).next().ok_or("post container not found")?;
|
||||
let link = container.select(&a).next().ok_or("post link not found")?;
|
||||
let name = format!("{}_{}_{}.html", link.value().attr("name").ok_or("post name in link not found")?, author, title.trim());
|
||||
let data = post.select(&post_content).next().ok_or("post content not found")?;
|
||||
let author = author.trim().split('|').nth(1).ok_or(anyhow!("author data in unknown format"))?.trim();
|
||||
let container = post.select(&post_container).next().ok_or(anyhow!("post container not found"))?;
|
||||
let link = container.select(&a).next().ok_or(anyhow!("post link not found"))?;
|
||||
let name = format!("{}_{}_{}.html", link.value().attr("name").ok_or(anyhow!("post name in link not found"))?, author, title.trim());
|
||||
let data = post.select(&post_content).next().ok_or(anyhow!("post content not found"))?;
|
||||
let data = data.inner_html();
|
||||
let mut path = path.clone();
|
||||
path.push(name);
|
||||
@ -851,7 +848,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
// not last page yet
|
||||
let ilias = Arc::clone(&ilias);
|
||||
let next_page = Thread {
|
||||
url: URL::from_href(last.value().attr("href").ok_or("page link not found")?)
|
||||
url: URL::from_href(last.value().attr("href").ok_or(anyhow!("page link not found"))?)
|
||||
};
|
||||
task::spawn(async move {
|
||||
process_gracefully(ilias, path, next_page).await;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user