From a003d274427f8c951bce31fbe15254be6ab989ee Mon Sep 17 00:00:00 2001 From: FliegendeWurst <2012gdwu+github@posteo.de> Date: Fri, 28 May 2021 15:32:24 +0200 Subject: [PATCH] Support download of other personal desktop pages --- CHANGELOG.md | 2 ++ src/ilias.rs | 36 +++++++++++++++--------------------- src/main.rs | 33 ++++++++++++++++----------------- 3 files changed, 33 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00b6f69..ac05da6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project loosely adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- `--sync-url https://ilias.studium.kit.edu/ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToMemberships` to download all courses (unless specified otherwise in the `.iliasignore`) ## [0.2.21] - 2021-05-18 ### Fixed diff --git a/src/ilias.rs b/src/ilias.rs index fe6bbd9..eaecf55 100644 --- a/src/ilias.rs +++ b/src/ilias.rs @@ -167,12 +167,15 @@ impl ILIAS { .collect() } - /// Returns subfolders and the main text on the course page. + /// Returns subfolders and the main text in a course/folder/personal desktop. pub async fn get_course_content(&self, url: &URL) -> Result<(Vec>, Option)> { let html = self.get_html(&url.url).await?; + let main_text = if let Some(el) = html.select(&il_content_container).next() { - if !el.children().flat_map(|x| x.value().as_element()).next().map(|x| - x.attr("class").unwrap_or_default().contains("ilContainerBlock")).unwrap_or(false) + + if !el.children().flat_map(|x| x.value().as_element()).next() + .map(|x| x.attr("class").unwrap_or_default() + .contains("ilContainerBlock")).unwrap_or(false) && el.inner_html().len() > 40 { // ^ minimum length of useful content? Some(el.inner_html()) @@ -186,15 +189,6 @@ impl ILIAS { Ok((ILIAS::get_items(&html), main_text)) } - pub async fn personal_desktop(&self) -> Result { - let html = self.get_html("https://ilias.studium.kit.edu/ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToSelectedItems").await?; - let items = ILIAS::get_items(&html) - .into_iter() - .flat_map(Result::ok) - .collect(); - Ok(Dashboard { items }) - } - pub async fn get_course_content_tree(&self, ref_id: &str, cmd_node: &str) -> Result> { // TODO: this magically does not return sub-folders // opening the same url in browser does show sub-folders?! @@ -213,15 +207,11 @@ impl ILIAS { } } -#[derive(Debug)] -pub struct Dashboard { - pub items: Vec, -} - #[derive(Debug)] pub enum Object { Course { name: String, url: URL }, Folder { name: String, url: URL }, + PersonalDesktop { url: URL }, File { name: String, url: URL }, Forum { name: String, url: URL }, Thread { url: URL }, @@ -253,6 +243,7 @@ impl Object { | Generic { name, .. } => &name, Thread { url } => &url.thr_pk.as_ref().unwrap(), Video { url } => &url.url, + PersonalDesktop { url } => url.cmd.as_ref().unwrap() } } @@ -260,6 +251,7 @@ impl Object { match self { Course { url, .. } | Folder { url, .. } + | PersonalDesktop { url } | File { url, .. } | Forum { url, .. } | Thread { url } @@ -278,6 +270,7 @@ impl Object { match self { Course { .. } => "course", Folder { .. } => "folder", + PersonalDesktop { .. } => "personal desktop", File { .. } => "file", Forum { .. } => "forum", Thread { .. } => "thread", @@ -293,16 +286,16 @@ impl Object { } pub fn is_dir(&self) -> bool { - match self { + matches!(self, Course { .. } | Folder { .. } + | PersonalDesktop { .. } | Forum { .. } | Thread { .. } | Wiki { .. } | ExerciseHandler { .. } - | PluginDispatch { .. } => true, - _ => false, - } + | PluginDispatch { .. } + ) } pub fn from_link(item: ElementRef, link: ElementRef) -> Result { @@ -399,6 +392,7 @@ impl Object { None => Course { name, url }, }, "ilobjplugindispatchgui" => PluginDispatch { name, url }, + "ilpersonaldesktopgui" => PersonalDesktop { url }, _ => Generic { name, url }, }) } diff --git a/src/main.rs b/src/main.rs index 097e96b..143eec6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -120,23 +120,19 @@ async fn real_main(mut opt: Opt) -> Result<()> { if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) { PROGRESS_BAR.set_draw_target(ProgressDrawTarget::stderr_nohz()); PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}+] {wide_msg}")); - PROGRESS_BAR.inc_length(1); PROGRESS_BAR.set_message("initializing.."); } - if let Some(url) = ilias.opt.sync_url.as_ref() { - // TODO: this should be unified with the download logic below - let obj = Object::from_url(URL::from_href(url).context("invalid sync URL")?, "Sync URL".to_owned(), None).context("invalid sync object")?; // name can be empty for first element - spawn!(process_gracefully(ilias.clone(), ilias.opt.output.clone(), obj)); - } else { - let desktop = ilias.personal_desktop().await.context("Failed to load personal desktop")?; - for item in desktop.items { - let path = ilias.opt.output.join(file_escape(item.name())); - tx.unbounded_send(task::spawn(process_gracefully(ilias.clone(), path, item))).unwrap(); - } - } + + // default sync URL: main personal desktop + let sync_url = ilias.opt.sync_url.clone().unwrap_or_else(|| format!("{}ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToSelectedItems", ILIAS_URL)); + let obj = Object::from_url(URL::from_href(&sync_url).context("invalid sync URL")?, String::new(), None).context("invalid sync object")?; // name can be empty for first element + spawn!(process_gracefully(ilias.clone(), ilias.opt.output.clone(), obj)); + while let Either::Left((task, _)) = future::select(rx.next(), future::ready(())).await { if let Some(task) = task { - let _ = task.await; + if let Err(e) = task.await { + error!(e) + } } else { break; } @@ -149,7 +145,6 @@ async fn real_main(mut opt: Opt) -> Result<()> { } } if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) { - PROGRESS_BAR.inc(1); PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}] {wide_msg}")); PROGRESS_BAR.finish_with_message("done"); } @@ -220,9 +215,13 @@ async fn process(ilias: Arc, path: PathBuf, obj: Object) -> Result<()> { let relative_path = path.strip_prefix(&ilias.opt.output).unwrap(); if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) { PROGRESS_BAR.inc(1); - PROGRESS_BAR.set_message(relative_path.display().to_string()); + let path = relative_path.display().to_string(); + if !path.is_empty() { + PROGRESS_BAR.set_message(path); + } } - if ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() { + // root path should not be matched + if relative_path.parent().is_some() && ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() { log!(1, "Ignored {}", relative_path.to_string_lossy()); return Ok(()); } @@ -263,7 +262,7 @@ async fn process(ilias: Arc, path: PathBuf, obj: Object) -> Result<()> { spawn!(process_gracefully(ilias, path, item)); } }, - Folder { url, .. } => { + Folder { url, .. } | PersonalDesktop { url } => { let content = ilias.get_course_content(&url).await?; if let Some(s) = content.1.as_ref() { let path = path.join("folder.html");