From 7f688f33c500ed06dabea1b7bfb79b8b7dbaf716 Mon Sep 17 00:00:00 2001 From: FliegendeWurst <2012gdwu@web.de> Date: Thu, 23 Apr 2020 11:21:05 +0200 Subject: [PATCH] Handle errors properly --- src/errors.rs | 1 + src/main.rs | 255 +++++++++++++++++++++++--------------------------- 2 files changed, 119 insertions(+), 137 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index e8191af..fce64c1 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -13,6 +13,7 @@ error_chain! { foreign_links { Io(std::io::Error); + Json(serde_json::Error); Reqwest(reqwest::Error); } diff --git a/src/main.rs b/src/main.rs index a13a284..1c23470 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ -use futures_util::stream::{StreamExt, TryStreamExt}; +use error_chain::ChainedError; +use futures_util::stream::TryStreamExt; use lazy_static::lazy_static; use parking_lot::Mutex; use regex::Regex; @@ -11,7 +12,6 @@ use tokio::io::{stream_reader, BufWriter}; use tokio::task; use url::Url; -use std::collections::VecDeque; use std::default::Default; use std::fs; use std::io; @@ -228,12 +228,12 @@ impl ILIAS { let pass = pass.into(); let client = Client::builder() .cookie_store(true) - .user_agent("KIT-ILIAS-downloader/0.2.0") + .user_agent(concat!("KIT-ILIAS-downloader/", env!("CARGO_PKG_VERSION"))) .build()?; let this = ILIAS { opt, client, user, pass }; - println!("Logging into Shibboleth.."); + println!("Logging into ILIAS using KIT account.."); let session_establishment = this.client .post("https://ilias.studium.kit.edu/Shibboleth.sso/Login") .form(&json!({ @@ -243,7 +243,7 @@ impl ILIAS { "home_organization_selection": "Mit KIT-Account anmelden" })) .send().await?; - println!("Logging into identity provider.."); + println!("Logging into Shibboleth.."); let login_response = this.client .post(session_establishment.url().clone()) .form(&json!({ @@ -264,9 +264,9 @@ impl ILIAS { login_soup = BeautifulSoup(otp_response.text, 'lxml') */ let saml = Selector::parse(r#"input[name="SAMLResponse"]"#).unwrap(); - let saml = dom.select(&saml).next().expect("no SAML response, incorrect password?"); + let saml = dom.select(&saml).next().ok_or::("no SAML response, incorrect password?".into())?; let relay_state = Selector::parse(r#"input[name="RelayState"]"#).unwrap(); - let relay_state = dom.select(&relay_state).next().expect("no relay state"); + let relay_state = dom.select(&relay_state).next().ok_or::("no relay state".into())?; println!("Logging into ILIAS.."); this.client .post("https://ilias.studium.kit.edu/Shibboleth.sso/SAML2/POST") @@ -279,20 +279,23 @@ impl ILIAS { Ok(this) } - async fn personal_desktop(&mut self) -> Result { + async fn personal_desktop(&self) -> Result { let html = self.get_html("https://ilias.studium.kit.edu/ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToSelectedItems").await?; - let items = ILIAS::get_items(&html); + let items = ILIAS::get_items(&html)?; Ok(Dashboard { items }) } - fn get_items(html: &Html) -> Vec { + fn get_items(html: &Html) -> Result> { let container_items = Selector::parse("div.il_ContainerListItem").unwrap(); let container_item_title = Selector::parse("a.il_ContainerItemTitle").unwrap(); html.select(&container_items).map(|item| { - let link = item.select(&container_item_title).next().unwrap(); - Object::from_link(item, link) + item + .select(&container_item_title) + .next() + .map(|link| Object::from_link(item, link)) + .ok_or::("can't find link".into()) }).collect() } @@ -303,7 +306,7 @@ impl ILIAS { async fn get_course_content(&self, url: &URL) -> Result> { let html = self.get_html(&format!("{}{}", ILIAS_URL, url.url)).await?; - Ok(ILIAS::get_items(&html)) + Ok(ILIAS::get_items(&html)?) } async fn download(&self, url: &str) -> Result { @@ -325,31 +328,24 @@ async fn main() { })); let user = rprompt::prompt_reply_stdout("Username: ").unwrap(); let pass = rpassword::read_password_from_tty(Some("Password: ")).unwrap(); - let mut ilias = match ILIAS::login::<_, String>(opt, user, pass).await { + let ilias = match ILIAS::login::<_, String>(opt, user, pass).await { Ok(ilias) => ilias, - Err(e) => panic!("error: {:?}", e) + Err(e) => { + print!("{}", e.display_chain()); + std::process::exit(77); + } }; + let ilias = Arc::new(ilias); let desktop = ilias.personal_desktop().await.unwrap(); - let mut queue = VecDeque::new(); for item in desktop.items { let mut path = ilias.opt.output.clone(); path.push(item.name()); - queue.push_back((path, item)); - } - let ilias = Arc::new(ilias); - while let Some((path, obj)) = queue.pop_front() { let ilias = Arc::clone(&ilias); task::spawn(async { - *TASKS_QUEUED.lock() += 1; - while *TASKS_RUNNING.lock() >= ilias.opt.jobs { - tokio::time::delay_for(Duration::from_millis(100)).await; - } - *TASKS_RUNNING.lock() += 1; - process(ilias, path, obj).await; - *TASKS_RUNNING.lock() -= 1; - *TASKS_QUEUED.lock() -= 1; + process_gracefully(ilias, path, item).await; }); } + // TODO: do this with tokio while *TASKS_QUEUED.lock() > 0 { tokio::time::delay_for(Duration::from_millis(500)).await; } @@ -362,9 +358,39 @@ lazy_static!{ static ref PANIC_HOOK: Mutex> = Mutex::new(Box::new(|_| {})); } +fn process_gracefully(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::Future + Send { async move { + *TASKS_QUEUED.lock() += 1; + while *TASKS_RUNNING.lock() >= ilias.opt.jobs { + tokio::time::delay_for(Duration::from_millis(100)).await; + } + *TASKS_RUNNING.lock() += 1; + let path_text = format!("{:?}", path); + if let Err(e) = process(ilias, path, obj).await { + print!("Error syncing {}: {}", path_text, e.display_chain()); + } + *TASKS_RUNNING.lock() -= 1; + *TASKS_QUEUED.lock() -= 1; +}} + // see https://github.com/rust-lang/rust/issues/53690#issuecomment-418911229 //async fn process(ilias: Arc, path: PathBuf, obj: Object) { -fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::Future + Send { async move { +#[allow(non_upper_case_globals)] +fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::Future> + Send { async move { + // construct CSS selectors once + lazy_static!{ + static ref a: Selector = Selector::parse("a").unwrap(); + static ref a_target_blank: Selector = Selector::parse(r#"a[target="_blank"]"#).unwrap(); + static ref table: Selector = Selector::parse("table").unwrap(); + static ref links_in_table: Selector = Selector::parse("tbody tr td a").unwrap(); + static ref td: Selector = Selector::parse("td").unwrap(); + static ref tr: Selector = Selector::parse("tr").unwrap(); + static ref post_row: Selector = Selector::parse(".ilFrmPostRow").unwrap(); + static ref post_title: Selector = Selector::parse(".ilFrmPostTitle").unwrap(); + static ref post_container: Selector = Selector::parse(".ilFrmPostContentContainer").unwrap(); + static ref post_content: Selector = Selector::parse(".ilFrmPostContent").unwrap(); + static ref span_small: Selector = Selector::parse("span.small").unwrap(); + static ref forum_pages: Selector = Selector::parse("div.ilTableNav > table > tbody > tr > td > a").unwrap(); + } if ilias.opt.verbose > 0 { println!("Syncing {} {}..", obj.kind(), path.strip_prefix(&ilias.opt.output).unwrap().to_string_lossy()); } @@ -372,91 +398,69 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F Course { url, .. } => { if let Err(e) = fs::create_dir(&path) { if e.kind() != io::ErrorKind::AlreadyExists { - println!("error: {:?}", e); + Err(e)?; } } - let content = ilias.get_course_content(&url).await.unwrap(); + let content = ilias.get_course_content(&url).await?; for item in content { let mut path = path.clone(); path.push(item.name()); let ilias = Arc::clone(&ilias); task::spawn(async { - *TASKS_QUEUED.lock() += 1; - while *TASKS_RUNNING.lock() >= ilias.opt.jobs { - tokio::time::delay_for(Duration::from_millis(100)).await; - } - *TASKS_RUNNING.lock() += 1; - process(ilias, path, item).await; - *TASKS_RUNNING.lock() -= 1; - *TASKS_QUEUED.lock() -= 1; + process_gracefully(ilias, path, item).await; }); } }, Folder { url, .. } => { if let Err(e) = fs::create_dir(&path) { if e.kind() != io::ErrorKind::AlreadyExists { - println!("error: {:?}", e); + Err(e)?; } } - let content = ilias.get_course_content(&url).await.unwrap(); + let content = ilias.get_course_content(&url).await?; for item in content { let mut path = path.clone(); path.push(item.name()); let ilias = Arc::clone(&ilias); task::spawn(async { - *TASKS_QUEUED.lock() += 1; - while *TASKS_RUNNING.lock() >= ilias.opt.jobs { - tokio::time::delay_for(Duration::from_millis(100)).await; - } - *TASKS_RUNNING.lock() += 1; - process(ilias, path, item).await; - *TASKS_RUNNING.lock() -= 1; - *TASKS_QUEUED.lock() -= 1; + process_gracefully(ilias, path, item).await; }); } }, File { url, .. } => { if ilias.opt.skip_files { - return; + return Ok(()); } if !ilias.opt.force && fs::metadata(&path).is_ok() { if ilias.opt.verbose > 1 { println!("Skipping download, file exists already"); } - return; - } - let data = ilias.download(&url.url).await; - match data { - Ok(resp) => { - let mut reader = stream_reader(resp.bytes_stream().map_err(|x| { - io::Error::new(io::ErrorKind::Other, x) - })); - println!("Writing to {:?}..", path); - let file = AsyncFile::create(&path).await.unwrap(); - let mut file = BufWriter::new(file); - tokio::io::copy(&mut reader, &mut file).await.unwrap(); - }, - Err(e) => println!("error: {:?}", e) + return Ok(()); } + let data = ilias.download(&url.url).await?; + let mut reader = stream_reader(data.bytes_stream().map_err(|x| { + io::Error::new(io::ErrorKind::Other, x) + })); + println!("Writing to {:?}..", path); + let file = AsyncFile::create(&path).await?; + let mut file = BufWriter::new(file); + tokio::io::copy(&mut reader, &mut file).await?; }, PluginDispatch { url, .. } => { if ilias.opt.no_videos { - return; + return Ok(()); } if let Err(e) = fs::create_dir(&path) { if e.kind() != io::ErrorKind::AlreadyExists { - println!("error: {:?}", e); + Err(e)?; } } let list_url = format!("{}ilias.php?ref_id={}&cmdClass=xocteventgui&cmdNode=n7:mz:14p&baseClass=ilObjPluginDispatchGUI&lang=de&limit=20&cmd=asyncGetTableGUI&cmdMode=asynch", ILIAS_URL, url.ref_id); let data = ilias.download(&list_url); - let html = data.await.unwrap().text().await.unwrap(); + let html = data.await?.text().await?; let html = Html::parse_fragment(&html); - let tr = Selector::parse("tr").unwrap(); - let td = Selector::parse("td").unwrap(); - let a = Selector::parse(r#"a[target="_blank"]"#).unwrap(); for row in html.select(&tr) { - let link = row.select(&a).next(); + let link = row.select(&a_target_blank).next(); if link.is_none() { continue; } @@ -478,14 +482,7 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F }; let ilias = Arc::clone(&ilias); task::spawn(async { - *TASKS_QUEUED.lock() += 1; - while *TASKS_RUNNING.lock() >= ilias.opt.jobs { - tokio::time::delay_for(Duration::from_millis(100)).await; - } - *TASKS_RUNNING.lock() += 1; - process(ilias, path, video).await; - *TASKS_RUNNING.lock() -= 1; - *TASKS_QUEUED.lock() -= 1; + process_gracefully(ilias, path, video).await; }); } @@ -496,81 +493,82 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F static ref XOCT_REGEX: Regex = Regex::new(r#"(?m)"#).unwrap(); } if ilias.opt.no_videos { - return; + return Ok(()); } if !ilias.opt.force && fs::metadata(&path).is_ok() { if ilias.opt.verbose > 1 { println!("Skipping download, file exists already"); } - return; + return Ok(()); } let url = format!("{}{}", ILIAS_URL, url); let data = ilias.download(&url); - let html = data.await.unwrap().text().await.unwrap(); + let html = data.await?.text().await?; if ilias.opt.verbose > 1 { println!("{}", html); } let json: serde_json::Value = { let mut json_capture = XOCT_REGEX.captures_iter(&html); - let json = &json_capture.next().unwrap()[1]; + let json = &json_capture.next().ok_or::("xoct player json not found".into())?[1]; if ilias.opt.verbose > 1 { println!("{}", json); } - let json = json.split(",\n").nth(0).unwrap(); - serde_json::from_str(&json.trim()).unwrap() + let json = json.split(",\n").nth(0).ok_or::("invalid xoct player json".into())?; + serde_json::from_str(&json.trim())? }; if ilias.opt.verbose > 1 { println!("{}", json); } let url = json["streams"][0]["sources"]["mp4"][0]["src"].as_str().unwrap(); - let resp = ilias.download(&url).await.unwrap(); + let resp = ilias.download(&url).await?; let mut reader = stream_reader(resp.bytes_stream().map_err(|x| { io::Error::new(io::ErrorKind::Other, x) })); println!("Saving video to {:?}", path); - let file = AsyncFile::create(&path).await.unwrap(); + let file = AsyncFile::create(&path).await?; let mut file = BufWriter::new(file); - tokio::io::copy(&mut reader, &mut file).await.unwrap(); + tokio::io::copy(&mut reader, &mut file).await?; }, Forum { url, .. } => { if !ilias.opt.forum { - return; + return Ok(()); } if let Err(e) = fs::create_dir(&path) { if e.kind() != io::ErrorKind::AlreadyExists { - println!("error: {:?}", e); + Err(e)?; } } let url = format!("{}ilias.php?ref_id={}&cmd=showThreads&cmdClass=ilrepositorygui&cmdNode=uf&baseClass=ilrepositorygui", ILIAS_URL, url.ref_id); let html = { - let a = Selector::parse("a").unwrap(); let data = ilias.download(&url); - let html_text = data.await.unwrap().text().await.unwrap(); + let html_text = data.await?.text().await?; let url = { let html = Html::parse_document(&html_text); //https://ilias.studium.kit.edu/ilias.php?ref_id=122&cmdClass=ilobjforumgui&frm_tt_e39_122_trows=800&cmd=showThreads&cmdNode=uf:lg&baseClass=ilrepositorygui - let url = { - let t800 = html.select(&a).filter(|x| x.value().attr("href").unwrap_or("").contains("trows=800")).next().unwrap_or_else(|| panic!("can't find forum thread count selector in {:?}", path)); - t800.value().attr("href").unwrap() - }; + let url = html + .select(&a) + .flat_map(|x| x.value().attr("href")) + .filter(|x| x.contains("trows=800")) + .next() + .ok_or::("can't find forum thread count selector (empty forum?)".into())?; format!("{}{}", ILIAS_URL, url) }; let data = ilias.download(&url); - let html = data.await.unwrap().text().await.unwrap(); + let html = data.await?.text().await?; Html::parse_document(&html) }; - let a = Selector::parse("a").unwrap(); - let tr = Selector::parse("tr").unwrap(); - let td = Selector::parse("td").unwrap(); for row in html.select(&tr) { let cells = row.select(&td).collect::>(); if cells.len() != 6 { continue; } - let link = cells[1].select(&a).next().unwrap(); + let link = cells[1].select(&a).next().ok_or::("thread link not found".into())?; let object = Object::from_link(link, link); let mut path = path.clone(); - let name = format!("{}_{}", object.url().thr_pk.as_ref().expect("thr_pk not found for thread"), link.text().collect::().replace('/', "-").trim()); + let name = format!("{}_{}", + object.url().thr_pk.as_ref().ok_or::("thr_pk not found for thread".into())?, + link.text().collect::().replace('/', "-").trim() + ); path.push(name); // TODO: set modification date? let saved_posts = { @@ -579,49 +577,34 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F Err(_) => 0 } }; - let available_posts = cells[3].text().next().unwrap().trim().parse::().unwrap(); + let available_posts = cells[3].text().next().unwrap().trim().parse::().chain_err(|| "parsing post count failed")?; if available_posts <= saved_posts && !ilias.opt.force { continue; } println!("New posts in {:?}..", path); let ilias = Arc::clone(&ilias); task::spawn(async { - *TASKS_QUEUED.lock() += 1; - while *TASKS_RUNNING.lock() >= ilias.opt.jobs { - tokio::time::delay_for(Duration::from_millis(100)).await; - } - *TASKS_RUNNING.lock() += 1; - process(ilias, path, object).await; - *TASKS_RUNNING.lock() -= 1; - *TASKS_QUEUED.lock() -= 1; + process_gracefully(ilias, path, object).await; }); } - let pages = Selector::parse("div.ilTableNav > table > tbody > tr > td > a").unwrap(); - if html.select(&pages).count() > 0 { + if html.select(&forum_pages).count() > 0 { println!("Ignoring older threads (801st+) in {:?}..", path); } }, Thread { url } => { if !ilias.opt.forum { - return; + return Ok(()); } if let Err(e) = fs::create_dir(&path) { if e.kind() != io::ErrorKind::AlreadyExists { - println!("error: {:?}", e); - return; + Err(e)?; } } let url = format!("{}{}", ILIAS_URL, url.url); let data = ilias.download(&url); - let html = data.await.unwrap().text().await.unwrap(); + let html = data.await?.text().await?; let html = Html::parse_document(&html); - let post = Selector::parse(".ilFrmPostRow").unwrap(); - let post_container = Selector::parse(".ilFrmPostContentContainer").unwrap(); - let post_title = Selector::parse(".ilFrmPostTitle").unwrap(); - let post_content = Selector::parse(".ilFrmPostContent").unwrap(); - let span_small = Selector::parse("span.small").unwrap(); - let a = Selector::parse("a").unwrap(); - for post in html.select(&post) { + for post in html.select(&post_row) { let title = post.select(&post_title).next().unwrap().text().collect::().replace('/', "-"); let author = post.select(&span_small).next().unwrap(); let author = author.text().collect::(); @@ -643,17 +626,21 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F if ilias.opt.verbose > 1 { println!("Writing to {:?}..", path); } - let file = AsyncFile::create(&path).await.unwrap(); - let mut file = BufWriter::new(file); - tokio::io::copy(&mut data.as_bytes(), &mut file).await.unwrap(); + let file = AsyncFile::create(&path).await; + if file.is_err() { + println!("Error creating file {:?}: {:?}", path, file.err().unwrap()); + return; + } + let mut file = BufWriter::new(file.unwrap()); + if let Err(e) = tokio::io::copy(&mut data.as_bytes(), &mut file).await { + println!("Error writing to {:?}: {:?}", path, e); + } *TASKS_RUNNING.lock() -= 1; *TASKS_QUEUED.lock() -= 1; }); } // pagination - let table = Selector::parse("table").unwrap(); if let Some(pages) = html.select(&table).next() { - let links_in_table = Selector::parse("tbody tr td a").unwrap(); if let Some(last) = pages.select(&links_in_table).last() { let text = last.text().collect::(); if text.trim() == ">>" { @@ -663,14 +650,7 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F url: URL::from_href(last.value().attr("href").unwrap()) }; task::spawn(async move { - *TASKS_QUEUED.lock() += 1; - while *TASKS_RUNNING.lock() >= ilias.opt.jobs { - tokio::time::delay_for(Duration::from_millis(100)).await; - } - *TASKS_RUNNING.lock() += 1; - process(ilias, path, next_page).await; - *TASKS_RUNNING.lock() -= 1; - *TASKS_QUEUED.lock() -= 1; + process_gracefully(ilias, path, next_page).await; }); } } else { @@ -684,6 +664,7 @@ fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::F } } } + Ok(()) }} #[derive(Debug, StructOpt)]