Support download of other personal desktop pages

CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project loosely adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- `--sync-url https://ilias.studium.kit.edu/ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToMemberships` to download all courses (unless specified otherwise in the `.iliasignore`)
 
 ## [0.2.21] - 2021-05-18
 ### Fixed

src/ilias.rs

@@ -167,12 +167,15 @@ impl ILIAS {
 			.collect()
 	}
 
-	/// Returns subfolders and the main text on the course page.
+	/// Returns subfolders and the main text in a course/folder/personal desktop.
 	pub async fn get_course_content(&self, url: &URL) -> Result<(Vec<Result<Object>>, Option<String>)> {
 		let html = self.get_html(&url.url).await?;
+
 		let main_text = if let Some(el) = html.select(&il_content_container).next() {
-			if !el.children().flat_map(|x| x.value().as_element()).next().map(|x|
+
-				x.attr("class").unwrap_or_default().contains("ilContainerBlock")).unwrap_or(false)
+			if !el.children().flat_map(|x| x.value().as_element()).next()
+				.map(|x| x.attr("class").unwrap_or_default()
+					.contains("ilContainerBlock")).unwrap_or(false)
 				&& el.inner_html().len() > 40 {
 				// ^ minimum length of useful content?
 				Some(el.inner_html())
@@ -186,15 +189,6 @@ impl ILIAS {
 		Ok((ILIAS::get_items(&html), main_text))
 	}
 
-	pub async fn personal_desktop(&self) -> Result<Dashboard> {
-		let html = self.get_html("https://ilias.studium.kit.edu/ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToSelectedItems").await?;
-		let items = ILIAS::get_items(&html)
-			.into_iter()
-			.flat_map(Result::ok)
-			.collect();
-		Ok(Dashboard { items })
-	}
-
 	pub async fn get_course_content_tree(&self, ref_id: &str, cmd_node: &str) -> Result<Vec<Object>> {
 		// TODO: this magically does not return sub-folders
 		// opening the same url in browser does show sub-folders?!
@@ -213,15 +207,11 @@ impl ILIAS {
 	}
 }
 
-#[derive(Debug)]
-pub struct Dashboard {
-	pub items: Vec<Object>,
-}
-
 #[derive(Debug)]
 pub enum Object {
 	Course { name: String, url: URL },
 	Folder { name: String, url: URL },
+	PersonalDesktop { url: URL },
 	File { name: String, url: URL },
 	Forum { name: String, url: URL },
 	Thread { url: URL },
@@ -253,6 +243,7 @@ impl Object {
 			| Generic { name, .. } => &name,
 			Thread { url } => &url.thr_pk.as_ref().unwrap(),
 			Video { url } => &url.url,
+			PersonalDesktop { url } => url.cmd.as_ref().unwrap()
 		}
 	}
 
@@ -260,6 +251,7 @@ impl Object {
 		match self {
 			Course { url, .. }
 			| Folder { url, .. }
+			| PersonalDesktop { url }
 			| File { url, .. }
 			| Forum { url, .. }
 			| Thread { url }
@@ -278,6 +270,7 @@ impl Object {
 		match self {
 			Course { .. } => "course",
 			Folder { .. } => "folder",
+			PersonalDesktop { .. } => "personal desktop",
 			File { .. } => "file",
 			Forum { .. } => "forum",
 			Thread { .. } => "thread",
@@ -293,16 +286,16 @@ impl Object {
 	}
 
 	pub fn is_dir(&self) -> bool {
-		match self {
+		matches!(self,
 			Course { .. }
 			| Folder { .. }
+			| PersonalDesktop { .. }
 			| Forum { .. }
 			| Thread { .. }
 			| Wiki { .. }
 			| ExerciseHandler { .. }
-			| PluginDispatch { .. } => true,
+			| PluginDispatch { .. }
-			_ => false,
+		)
-		}
 	}
 
 	pub fn from_link(item: ElementRef, link: ElementRef) -> Result<Self> {
@@ -399,6 +392,7 @@ impl Object {
 				None => Course { name, url },
 			},
 			"ilobjplugindispatchgui" => PluginDispatch { name, url },
+			"ilpersonaldesktopgui" => PersonalDesktop { url },
 			_ => Generic { name, url },
 		})
 	}

src/main.rs

@@ -120,23 +120,19 @@ async fn real_main(mut opt: Opt) -> Result<()> {
 	if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
 		PROGRESS_BAR.set_draw_target(ProgressDrawTarget::stderr_nohz());
 		PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}+] {wide_msg}"));
-		PROGRESS_BAR.inc_length(1);
 		PROGRESS_BAR.set_message("initializing..");
 	}
-	if let Some(url) = ilias.opt.sync_url.as_ref() {
+
-		// TODO: this should be unified with the download logic below
+	// default sync URL: main personal desktop
-		let obj = Object::from_url(URL::from_href(url).context("invalid sync URL")?, "Sync URL".to_owned(), None).context("invalid sync object")?; // name can be empty for first element
+	let sync_url = ilias.opt.sync_url.clone().unwrap_or_else(|| format!("{}ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToSelectedItems", ILIAS_URL));
+	let obj = Object::from_url(URL::from_href(&sync_url).context("invalid sync URL")?, String::new(), None).context("invalid sync object")?; // name can be empty for first element
 	spawn!(process_gracefully(ilias.clone(), ilias.opt.output.clone(), obj));
-	} else {
+
-		let desktop = ilias.personal_desktop().await.context("Failed to load personal desktop")?;
-		for item in desktop.items {
-			let path = ilias.opt.output.join(file_escape(item.name()));
-			tx.unbounded_send(task::spawn(process_gracefully(ilias.clone(), path, item))).unwrap();
-		}
-	}
 	while let Either::Left((task, _)) = future::select(rx.next(), future::ready(())).await {
 		if let Some(task) = task {
-			let _ = task.await;
+			if let Err(e) = task.await {
+				error!(e)
+			}
 		} else {
 			break;
 		}
@@ -149,7 +145,6 @@ async fn real_main(mut opt: Opt) -> Result<()> {
 		}
 	}
 	if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
-		PROGRESS_BAR.inc(1);
 		PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}] {wide_msg}"));
 		PROGRESS_BAR.finish_with_message("done");
 	}
@@ -220,9 +215,13 @@ async fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> Result<()> {
 	let relative_path = path.strip_prefix(&ilias.opt.output).unwrap();
 	if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
 		PROGRESS_BAR.inc(1);
-		PROGRESS_BAR.set_message(relative_path.display().to_string());
+		let path = relative_path.display().to_string();
+		if !path.is_empty() {
+			PROGRESS_BAR.set_message(path);
 		}
-	if ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() {
+	}
+	// root path should not be matched
+	if relative_path.parent().is_some() && ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() {
 		log!(1, "Ignored {}", relative_path.to_string_lossy());
 		return Ok(());
 	}
@@ -263,7 +262,7 @@ async fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> Result<()> {
 				spawn!(process_gracefully(ilias, path, item));
 			}
 		},
-		Folder { url, .. } => {
+		Folder { url, .. } | PersonalDesktop { url } => {
 			let content = ilias.get_course_content(&url).await?;
 			if let Some(s) = content.1.as_ref() {
 				let path = path.join("folder.html");