Merge branch 'master' of https://github.com/FliegendeWurst/KIT-ILIAS-downloader

CHANGELOG.md

@@ -3,7 +3,24 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
-## [Unreleased]
+## [0.3.5]
+### Added
+- `--pass-path` option to get the password from [pass](https://www.passwordstore.org/) (PR [#33] by [@Ma27])
+
+## [0.3.4]
+### Added
+- Display a warning if two or more courses/folders have the same name ([#31])
+
+### Fixed
+- `--keyring` option on Linux now tries to unlock password before using it (previously, this error would occur: `PlatformFailure(Zbus(MethodError("org.freedesktop.Secret.Error.IsLocked", None, Msg { type: Error, sender: ":1.34", reply-serial: 6 })))`)
+ 
+## [0.3.3] - 2022-03-21
+### Addded
+- `--all` flag to download all courses ([#30])
+
+## [0.3.2] - 2022-01-21
+### Fixed
+- Downloading of videos (PR [#28] by [@funnym0nk3y])
 
 ## [0.3.1] - 2022-01-07
 ### Fixed
@@ -145,6 +162,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [0.1.0] - 2020-04-21
 (undocumented)
 
+[#33]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/pull/33
+[#31]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/issues/31
+[#30]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/issues/30
+[#28]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/pull/28
 [#27]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/issues/27
 [#19]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/pull/19
 [#15]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/issues/15
@@ -164,7 +185,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 [@Craeckie]: https://github.com/Craeckie
 [@funnym0nk3y]: https://github.com/funnym0nk3y
 [@Ma27]: https://github.com/Ma27
-[Unreleased]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/compare/v0.3.1...HEAD
+[Unreleased]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/compare/v0.3.5...HEAD
+[0.3.5]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/compare/v0.3.5...v0.3.4
+[0.3.4]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/compare/v0.3.3...v0.3.4
+[0.3.3]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/compare/v0.3.2...v0.3.3
+[0.3.2]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/compare/v0.3.1...v0.3.2
 [0.3.1]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/compare/v0.3.0...v0.3.1
 [0.3.0]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/compare/v0.2.24...v0.3.0
 [0.2.24]: https://github.com/FliegendeWurst/KIT-ILIAS-downloader/compare/v0.2.23...v0.2.24

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "KIT-ILIAS-downloader"
-version = "0.3.1"
+version = "0.3.5"
 authors = ["FliegendeWurst <2012gdwu@posteo.de>"]
 license = "GPL-3.0-or-later"
 edition = "2018"
@@ -10,28 +10,28 @@ edition = "2018"
 [dependencies]
 reqwest = { version = "0.11.0", default-features = false, features = ["cookies", "gzip", "json", "rustls-tls", "stream", "socks"] }
 tokio = { version = "1.0.2", features = ["fs", "macros", "net", "rt-multi-thread", "process"] }
-tokio-util = { version = "0.6.1", features = ["io"] }
+tokio-util = { version = "0.7.0", features = ["io"] }
 serde_json = "1.0.51"
-scraper = "0.12.0"
+scraper = "0.13.0"
 url = "2.1.1"
 futures = "0.3.8"
 futures-util = "0.3.8"
 futures-channel = "0.3.8"
 regex = "1.3.7"
 structopt = "0.3.13"
-rpassword = "5.0.0"
+rpassword = "6.0.1"
 rprompt = "1.0.5"
 ignore = "0.4.14"
 anyhow = "1.0.28"
 colored = "2.0.0"
-keyring = "1.0.0"
+keyring = { git = "https://github.com/FliegendeWurst/keyring-rs" }
 cfg-if = "1.0.0"
 indicatif = "0.16.0"
 once_cell = "1.7.2"
 atty = "0.2.14"
 h2 = "0.3.3"
-cookie_store = "0.15.1"
+cookie_store = "0.16.1"
-reqwest_cookie_store = "0.2.0"
+reqwest_cookie_store = "0.3.0"
 bytes = "1.0.1"
 toml = "0.5.8"
 tempfile = "3.2.0"

README.md

@@ -12,7 +12,7 @@ Download content from ILIAS. That includes:
 **Windows/Linux users**: go to the [releases](../../releases) and download the executable for your operating system.   
 **macOS users**: [Install Rust](https://www.rust-lang.org/tools/install) and compile from source:
 ```
-$ cargo install --all-features --git 'https://github.com/FliegendeWurst/KIT-ILIAS-downloader' --branch stable
+$ cargo install --all-features --git 'https://github.com/FliegendeWurst/KIT-ILIAS-downloader'
 ```
 
 ## Usage
@@ -35,13 +35,13 @@ $ KIT-ILIAS-downloader -o ./ILIAS/WS2021-HM1 --sync-url 'https://ilias.studium.k
 ### Options
 
 ```
-$ KIT-ILIAS-downloader --help
+KIT-ILIAS-downloader 0.3.5
-KIT-ILIAS-downloader 0.2.24
 
 USAGE:
     KIT-ILIAS-downloader [FLAGS] [OPTIONS] --output <output>
 
 FLAGS:
+        --all                 Download all courses
         --check-videos        Re-check OpenCast lectures (slow)
         --combine-videos      Combine videos if there is more than one stream (requires ffmpeg)
         --content-tree        Use content tree (experimental)
@@ -59,6 +59,7 @@ FLAGS:
 OPTIONS:
     -j, --jobs <jobs>              Parallel download jobs [default: 1]
     -o, --output <output>          Output directory
+        --pass-path <pass-path>    Path inside `pass(1)` to the password for your KIT account
     -P, --password <password>      KIT account password
     -p, --proxy <proxy>            Proxy, e.g. socks5h://127.0.0.1:1080
         --rate <rate>              Requests per minute [default: 8]
@@ -68,7 +69,7 @@ OPTIONS:
 
 ### .iliasignore
 
-.gitignore syntax can be used in a `.iliasignore` file: (located in the output folder)
+.gitignore syntax can be used in a `.iliasignore` file: (located in the output directory)
 ```ignore
 # example 1: only download a single course
 /*/
@@ -80,18 +81,25 @@ OPTIONS:
 
 ### Credentials
 
-You can use the `--user` and `--keyring` options to get/store the password using the system password store.  
+You can use the `--user` and `--keyring` options to get/store the password using the system password store:
-If you use Linux, you'll have to compile from source to be able to use this option.
 ```
 $ KIT-ILIAS-downloader -U uabcd --keyring [...]
 ```
 
+If you use [pass](https://www.passwordstore.org/), you can use the `--pass-path` option to specify which password store entry to use:
+```
+$ KIT-ILIAS-downloader -U uabcd --pass-path edu/kit/uskyk [...]
+```
+
 You can also save your username and password in a `.iliaslogin` file: (located in the output folder)
 ```
 username
 password
 ```
 
+When running the downloader multiple times in a short period of time, you may want to use the `--keep-session` flag.
+If specified, the downloader will save and restore session cookies (`.iliassession` file in the output directory).
+
 ### Renaming course names (0.2.24+)
 If you'd like to avoid unwieldy course names (e.g. "24030 – Programmierparadigmen"), you can create a `course_names.toml` file in the output directory. It should contain the desired mapping of course names to folder names, e.g.:
 ```

src/cli.rs

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
+use std::io::{Error, ErrorKind};
 use std::path::PathBuf;
+use std::process::Command;
 use std::sync::atomic::{AtomicBool, AtomicUsize};
 
 use anyhow::anyhow;
@@ -72,6 +74,10 @@ pub struct Opt {
 	#[structopt(short = "P", long)]
 	pub password: Option<String>,
 
+	/// Path inside `pass(1)` to the password for your KIT account
+	#[structopt(long)]
+	pub pass_path: Option<String>,
+
 	/// ILIAS page to download
 	#[structopt(long)]
 	pub sync_url: Option<String>,
@@ -83,6 +89,10 @@ pub struct Opt {
 	/// Attempt to re-use session cookies
 	#[structopt(long)]
 	pub keep_session: bool,
+
+	/// Download all courses
+	#[structopt(long)]
+	pub all: bool,
 }
 
 pub static LOG_LEVEL: AtomicUsize = AtomicUsize::new(0);
@@ -162,12 +172,41 @@ pub fn ask_user_pass(opt: &Opt) -> Result<(String, String)> {
 			},
 			Err(e) => {
 				error!(e);
-				pass = rpassword::read_password_from_tty(Some("Password: ")).context("password prompt")?;
+				pass = rpassword::prompt_password("Password: ").context("password prompt")?;
 				should_store = true;
 			}
 		}
+	} else if let Some(pass_path) = &opt.pass_path {
+		let pw_out = Command::new("pass")
+			.arg("show")
+			.arg(pass_path)
+			.output()
+			.map_err(|x| {
+				if x.kind() == ErrorKind::NotFound {
+					Error::new(ErrorKind::NotFound, "pass not found in $PATH!")
 				} else {
-		pass = rpassword::read_password_from_tty(Some("Password: ")).context("password prompt")?;
+					x
+				}
+			})?;
+		if !pw_out.status.success() {
+			return Err(Error::new(
+				ErrorKind::Other,
+				format!(
+					"`pass` failed with non-zero exit code {}: {}",
+					pw_out.status.code()
+						.expect("Failed retrieving pass exit code!"),
+					String::from_utf8(pw_out.stderr)
+						.expect("Failed decoding stderr of pass!")
+				)
+			))?
+		} else {
+			pass = String::from_utf8(pw_out.stdout).map(|x| {
+				x.lines().next().map(|x| x.to_owned()).ok_or_else(|| anyhow!("empty pass(1) entry!"))
+			})?.expect("utf-8 decode of `pass(1)`-output failed");
+			should_store = false;
+		}
+	} else {
+		pass = rpassword::prompt_password("Password: ").context("password prompt")?;
 		should_store = true;
 	}
 	if should_store && opt.keyring {

src/ilias/folder.rs

@@ -1,4 +1,4 @@
-use std::{path::Path, sync::Arc};
+use std::{path::Path, sync::Arc, collections::HashSet};
 
 use anyhow::{Context, Result};
 
@@ -20,11 +20,17 @@ pub async fn download(path: &Path, ilias: Arc<ILIAS>, url: &URL) -> Result<()> {
 				.context("failed to write folder page html")?;
 		}
 	}
+	let mut names = HashSet::new();
 	for item in content.0 {
 		let item = item?;
-		let path = path.join(file_escape(
+		let item_name = file_escape(
 			ilias.course_names.get(item.name()).map(|x| &**x).unwrap_or(item.name()),
-		));
+		);
+		if names.contains(&item_name) {
+			warning!(format => "folder {} contains duplicated folder {:?}", path.display(), item_name);
+		}
+		names.insert(item_name.clone());
+		let path = path.join(item_name);
 		let ilias = Arc::clone(&ilias);
 		spawn(process_gracefully(ilias, path, item));
 	}

src/main.rs

@@ -160,9 +160,14 @@ async fn real_main(mut opt: Opt) -> Result<()> {
 		PROGRESS_BAR.set_message("initializing..");
 	}
 
-	let sync_url = ilias.opt.sync_url.as_deref().unwrap_or(DEFAULT_SYNC_URL);
+	let sync_url = if ilias.opt.all {
+		// change on ILIAS update
+		format!("{}ilias.php?cmdClass=ilmembershipoverviewgui&cmdNode=iy&baseClass=ilmembershipoverviewgui", ILIAS_URL)
+	} else {
+		ilias.opt.sync_url.as_deref().unwrap_or(DEFAULT_SYNC_URL).to_owned()
+	};
 	let obj = Object::from_url(
-		URL::from_href(sync_url).context("invalid sync URL")?,
+		URL::from_href(&sync_url).context("invalid sync URL")?,
 		String::new(),
 		None,
 	)