mirror of
https://github.com/FliegendeWurst/KIT-ILIAS-downloader.git
synced 2024-08-28 04:04:18 +00:00
Parse command-line arguments
This commit is contained in:
FliegendeWurst
parent
1529a678e0
commit
f10bba3e3e
133
Cargo.lock
generated
133
Cargo.lock
generated
@ -12,6 +12,7 @@ dependencies = [
|
||||
"reqwest",
|
||||
"scraper",
|
||||
"serde_json",
|
||||
"structopt",
|
||||
"tokio",
|
||||
"url 2.1.1",
|
||||
]
|
||||
@ -25,12 +26,32 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ansi_term"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
|
||||
dependencies = [
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arc-swap"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62"
|
||||
|
||||
[[package]]
|
||||
name = "atty"
|
||||
version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "0.1.7"
|
||||
@ -107,6 +128,21 @@ version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "2.33.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9"
|
||||
dependencies = [
|
||||
"ansi_term",
|
||||
"atty",
|
||||
"bitflags",
|
||||
"strsim",
|
||||
"textwrap",
|
||||
"unicode-width",
|
||||
"vec_map",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cloudabi"
|
||||
version = "0.0.3"
|
||||
@ -385,6 +421,15 @@ dependencies = [
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
|
||||
dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.11"
|
||||
@ -896,6 +941,32 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-error"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678"
|
||||
dependencies = [
|
||||
"proc-macro-error-attr",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-error-attr"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"syn-mid",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-hack"
|
||||
version = "0.5.15"
|
||||
@ -1400,6 +1471,36 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
|
||||
|
||||
[[package]]
|
||||
name = "structopt"
|
||||
version = "0.3.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff6da2e8d107dfd7b74df5ef4d205c6aebee0706c647f6bc6a2d5789905c00fb"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"lazy_static",
|
||||
"structopt-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "structopt-derive"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a489c87c08fbaf12e386665109dd13470dcc9c4583ea3e10dd2b4523e5ebd9ac"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro-error",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.17"
|
||||
@ -1411,6 +1512,17 @@ dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn-mid"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.1.0"
|
||||
@ -1436,6 +1548,15 @@ dependencies = [
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "textwrap"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
|
||||
dependencies = [
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thin-slice"
|
||||
version = "0.1.1"
|
||||
@ -1559,6 +1680,12 @@ dependencies = [
|
||||
"smallvec 1.3.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.7"
|
||||
@ -1605,6 +1732,12 @@ version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168"
|
||||
|
||||
[[package]]
|
||||
name = "vec_map"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.1"
|
||||
|
||||
@ -18,3 +18,4 @@ futures-util = "0.3.4"
|
||||
regex = "1.3.7"
|
||||
lazy_static = "1.4.0"
|
||||
parking_lot = "0.10.2"
|
||||
structopt = "0.3.13"
|
||||
|
||||
21
README.md
21
README.md
@ -2,7 +2,7 @@
|
||||
|
||||
Download content from ILIAS. That includes:
|
||||
|
||||
* files
|
||||
* files (latest version)
|
||||
* Opencast lectures
|
||||
|
||||
## Installation
|
||||
@ -19,7 +19,24 @@ $ cp target/release/KIT-ILIAS-downloader [directory in $PATH]
|
||||
|
||||
## Usage
|
||||
|
||||
TBD
|
||||
```sh
|
||||
$ KIT-ILIAS-downloader --help
|
||||
KIT-ILIAS-downloader 0.1.0
|
||||
|
||||
USAGE:
|
||||
KIT-ILIAS-downloader [FLAGS] --output <output>
|
||||
|
||||
FLAGS:
|
||||
-f, Re-download already present files
|
||||
-h, --help Prints help information
|
||||
-n, --no-videos Do not download Opencast videos
|
||||
-s, --skip-files Do not download files
|
||||
-V, --version Prints version information
|
||||
-v Verbosity (>=1 prints progress)
|
||||
|
||||
OPTIONS:
|
||||
-o, --output <output> Output directory
|
||||
```
|
||||
|
||||
## Credits
|
||||
|
||||
|
||||
67
src/main.rs
67
src/main.rs
@ -5,6 +5,7 @@ use regex::Regex;
|
||||
use reqwest::Client;
|
||||
use scraper::{ElementRef, Html, Selector};
|
||||
use serde_json::json;
|
||||
use structopt::StructOpt;
|
||||
use tokio::fs::File as AsyncFile;
|
||||
use tokio::io::{stream_reader, BufWriter};
|
||||
use tokio::task;
|
||||
@ -24,6 +25,7 @@ use errors::*;
|
||||
const ILIAS_URL: &'static str = "https://ilias.studium.kit.edu/";
|
||||
|
||||
struct ILIAS {
|
||||
opt: Opt,
|
||||
user: String,
|
||||
pass: String,
|
||||
path_prefix: PathBuf,
|
||||
@ -186,7 +188,7 @@ impl URL {
|
||||
}
|
||||
|
||||
impl ILIAS {
|
||||
async fn login<S1, S2>(user: S1, pass: S1) -> Result<Self> where S1: Into<String>, S2: Into<String> {
|
||||
async fn login<S1, S2>(opt: Opt, user: S1, pass: S1) -> Result<Self> where S1: Into<String>, S2: Into<String> {
|
||||
let user = user.into();
|
||||
let pass = pass.into();
|
||||
let client = Client::builder()
|
||||
@ -238,7 +240,7 @@ impl ILIAS {
|
||||
println!("Logged in!");
|
||||
let path_prefix = PathBuf::from(env!("ILIAS_DIR"));
|
||||
Ok(ILIAS {
|
||||
client, user, pass, path_prefix
|
||||
opt, client, user, pass, path_prefix
|
||||
})
|
||||
}
|
||||
|
||||
@ -270,31 +272,22 @@ impl ILIAS {
|
||||
}
|
||||
|
||||
async fn download(&self, url: &str) -> Result<reqwest::Response> {
|
||||
//let url = format!("{}{}", ILIAS_URL, url.url);
|
||||
if VERBOSITY > 0 {
|
||||
if self.opt.verbose > 0 {
|
||||
println!("Downloading {}", url);
|
||||
}
|
||||
Ok(self.client.get(url).send().await?)
|
||||
}
|
||||
}
|
||||
|
||||
const DOWNLOAD_FILES: bool = true;
|
||||
const DOWNLOAD_VIDEOS: bool = true;
|
||||
|
||||
const SKIP_EXISTING: bool = true;
|
||||
|
||||
const VERBOSITY: usize = 1;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let opt = Opt::from_args();
|
||||
*PANIC_HOOK.lock() = panic::take_hook();
|
||||
panic::set_hook(Box::new(|info| {
|
||||
*TASKS_RUNNING.lock() -= 1;
|
||||
PANIC_HOOK.lock()(info);
|
||||
}));
|
||||
// TODO: config at runtime..
|
||||
// it's literally in the executable currently
|
||||
let mut ilias = match ILIAS::login::<&str, &str>(env!("ILIAS_USER"), env!("ILIAS_PASS")).await {
|
||||
let mut ilias = match ILIAS::login::<&str, &str>(opt, env!("ILIAS_USER"), env!("ILIAS_PASS")).await {
|
||||
Ok(ilias) => ilias,
|
||||
Err(e) => panic!("error: {:?}", e)
|
||||
};
|
||||
@ -328,7 +321,7 @@ lazy_static!{
|
||||
// see https://github.com/rust-lang/rust/issues/53690#issuecomment-418911229
|
||||
//async fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) {
|
||||
fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::Future<Output = ()> + Send { async move {
|
||||
if VERBOSITY > 0 {
|
||||
if ilias.opt.verbose > 0 {
|
||||
println!("Syncing {} {}..", obj.kind(), path.strip_prefix(&ilias.path_prefix).unwrap().to_string_lossy());
|
||||
}
|
||||
match &obj {
|
||||
@ -369,11 +362,11 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
}
|
||||
},
|
||||
File { name, url } => {
|
||||
if !DOWNLOAD_FILES {
|
||||
if ilias.opt.skip_files {
|
||||
return;
|
||||
}
|
||||
if SKIP_EXISTING && fs::metadata(&path).is_ok() {
|
||||
if VERBOSITY > 1 {
|
||||
if !ilias.opt.force && fs::metadata(&path).is_ok() {
|
||||
if ilias.opt.verbose > 1 {
|
||||
println!("Skipping download, file exists already");
|
||||
}
|
||||
return;
|
||||
@ -392,7 +385,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
}
|
||||
},
|
||||
PluginDispatch { name, url } => {
|
||||
if !DOWNLOAD_VIDEOS {
|
||||
if ilias.opt.no_videos {
|
||||
return;
|
||||
}
|
||||
if let Err(e) = fs::create_dir(&path) {
|
||||
@ -422,7 +415,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
}
|
||||
let mut path = path.clone();
|
||||
path.push(format!("{}.mp4", title));
|
||||
if VERBOSITY > 0 {
|
||||
if ilias.opt.verbose > 0 {
|
||||
println!("Found video: {}", title);
|
||||
}
|
||||
let video = Video {
|
||||
@ -442,11 +435,11 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
lazy_static!{
|
||||
static ref XOCT_REGEX: Regex = Regex::new(r#"(?m)<script>\s+xoctPaellaPlayer\.init\(([\s\S]+)\)\s+</script>"#).unwrap();
|
||||
}
|
||||
if !DOWNLOAD_VIDEOS {
|
||||
if ilias.opt.no_videos {
|
||||
return;
|
||||
}
|
||||
if SKIP_EXISTING && fs::metadata(&path).is_ok() {
|
||||
if VERBOSITY > 1 {
|
||||
if !ilias.opt.force && fs::metadata(&path).is_ok() {
|
||||
if ilias.opt.verbose > 1 {
|
||||
println!("Skipping download, file exists already");
|
||||
}
|
||||
return;
|
||||
@ -468,7 +461,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
let mut reader = stream_reader(resp.bytes_stream().map_err(|x| {
|
||||
io::Error::new(io::ErrorKind::Other, x)
|
||||
}));
|
||||
if VERBOSITY > 0 {
|
||||
if ilias.opt.verbose > 0 {
|
||||
println!("Saving video to {:?}", path);
|
||||
}
|
||||
let file = AsyncFile::create(&path).await.unwrap();
|
||||
@ -476,9 +469,33 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
|
||||
tokio::io::copy(&mut reader, &mut file).await.unwrap();
|
||||
},
|
||||
o => {
|
||||
if VERBOSITY > 0 {
|
||||
if ilias.opt.verbose > 0 {
|
||||
println!("ignoring {:#?}", o)
|
||||
}
|
||||
}
|
||||
}
|
||||
}}
|
||||
|
||||
#[derive(Debug, StructOpt)]
|
||||
#[structopt(name = "KIT-ILIAS-downloader")]
|
||||
struct Opt {
|
||||
/// Do not download files
|
||||
#[structopt(short, long)]
|
||||
skip_files: bool,
|
||||
|
||||
/// Do not download Opencast videos
|
||||
#[structopt(short, long)]
|
||||
no_videos: bool,
|
||||
|
||||
/// Re-download already present files
|
||||
#[structopt(short)]
|
||||
force: bool,
|
||||
|
||||
/// Verbosity (>=1 prints progress)
|
||||
#[structopt(short, multiple = true, parse(from_occurrences))]
|
||||
verbose: usize,
|
||||
|
||||
/// Output directory
|
||||
#[structopt(short, long, parse(from_os_str))]
|
||||
output: PathBuf,
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user