arnekeller/KIT-ILIAS-downloader
1
0
Fork 0
mirror of https://github.com/FliegendeWurst/KIT-ILIAS-downloader.git synced 2024-08-28 04:04:18 +00:00
Code Issues Projects Releases Wiki Activity

Logging macro

Browse Source
This commit is contained in:
FliegendeWurst 2020-05-09 09:02:33 +02:00
parent 474ad43fc8
commit 1cc3e608f9
1 changed files with 33 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
src/main.rs
View File

@ -558,16 +558,23 @@ use crate::selectors::*;
// see https://github.com/rust-lang/rust/issues/53690#issuecomment-418911229 // see https://github.com/rust-lang/rust/issues/53690#issuecomment-418911229
//async fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) { //async fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) {
fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::Future<Output = Result<()>> + Send { async move { fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::Future<Output = Result<()>> + Send { async move {
let log_level = ilias.opt.verbose;
macro_rules! log {
($lvl:expr, $($arg:expr),*) => {
#[allow(unused_comparisons)] // 0 >= 0
if log_level >= $lvl {
println!($($arg),*);
}
}
}
let relative_path = path.strip_prefix(&ilias.opt.output).unwrap(); let relative_path = path.strip_prefix(&ilias.opt.output).unwrap();
if ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() { if ilias.ignore.matched(relative_path, obj.is_dir()).is_ignore() {
if ilias.opt.verbose > 0 { log!(1, "Ignored {}", relative_path.to_string_lossy());
println!("Ignoring {:?}..", relative_path);
}
return Ok(()); return Ok(());
} }
if ilias.opt.verbose > 0 { log!(1, "Syncing {} {}..", obj.kind(), relative_path.to_string_lossy());
println!("Syncing {} {}.. {}", obj.kind(), relative_path.to_string_lossy(), obj.url().url); log!(2, " URL: {}", obj.url().url);
}
match &obj { match &obj {
Course { url, name } => { Course { url, name } => {
if let Err(e) = fs::create_dir(&path) { if let Err(e) = fs::create_dir(&path) {
@ -587,7 +594,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
if html.contains(r#"input[name="cmd[join]""#) { if html.contains(r#"input[name="cmd[join]""#) {
return Ok(()); // ignore groups we are not in return Ok(()); // ignore groups we are not in
} }
println!("Warning: {:?} falling back to incomplete course content extractor! {:?}", name, e); log!(0, "Warning: {:?} falling back to incomplete course content extractor! {:?}", name, e);
ilias.get_course_content(&url).await?.into_iter().flat_map(Result::ok).collect() // TODO: perhaps don't download almost the same content 3x ilias.get_course_content(&url).await?.into_iter().flat_map(Result::ok).collect() // TODO: perhaps don't download almost the same content 3x
} }
} }
@ -612,9 +619,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
let content = ilias.get_course_content(&url).await?; let content = ilias.get_course_content(&url).await?;
for item in content { for item in content {
if item.is_err() { if item.is_err() {
if ilias.opt.verbose > 0 { log!(1, "Ignoring: {:?}", item.err().unwrap());
println!("Ignoring: {:?}", item.err().unwrap());
}
continue; continue;
} }
let item = item.unwrap(); let item = item.unwrap();
@ -631,16 +636,14 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
return Ok(()); return Ok(());
} }
if !ilias.opt.force && fs::metadata(&path).is_ok() { if !ilias.opt.force && fs::metadata(&path).is_ok() {
if ilias.opt.verbose > 1 { log!(2, "Skipping download, file exists already");
println!("Skipping download, file exists already");
}
return Ok(()); return Ok(());
} }
let data = ilias.download(&url.url).await?; let data = ilias.download(&url.url).await?;
let mut reader = stream_reader(data.bytes_stream().map_err(|x| { let mut reader = stream_reader(data.bytes_stream().map_err(|x| {
io::Error::new(io::ErrorKind::Other, x) io::Error::new(io::ErrorKind::Other, x)
})); }));
println!("Writing file to {:?}..", path); log!(0, "Writing {}..", relative_path.to_string_lossy());
let file = AsyncFile::create(&path).await?; let file = AsyncFile::create(&path).await?;
let mut file = BufWriter::new(file); let mut file = BufWriter::new(file);
tokio::io::copy(&mut reader, &mut file).await?; tokio::io::copy(&mut reader, &mut file).await?;
@ -661,7 +664,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
for row in html.select(&video_tr) { for row in html.select(&video_tr) {
let link = row.select(&a_target_blank).next(); let link = row.select(&a_target_blank).next();
if link.is_none() { if link.is_none() {
println!("Warning: table row without link in {}", url.url); log!(0, "Warning: table row without link in {}", url.url);
continue; continue;
} }
let link = link.unwrap(); let link = link.unwrap();
@ -674,9 +677,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
} }
let mut path = path.clone(); let mut path = path.clone();
path.push(format!("{}.mp4", title)); path.push(format!("{}.mp4", title));
if ilias.opt.verbose > 0 { log!(1, "Found video: {}", title);
println!("Found video: {}", title);
}
let video = Video { let video = Video {
url: URL::raw(link.value().attr("href").ok_or(anyhow!("video link without href"))?.to_owned()) url: URL::raw(link.value().attr("href").ok_or(anyhow!("video link without href"))?.to_owned())
}; };
@ -698,21 +699,15 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
let url = format!("{}{}", ILIAS_URL, url.url); let url = format!("{}{}", ILIAS_URL, url.url);
let data = ilias.download(&url); let data = ilias.download(&url);
let html = data.await?.text().await?; let html = data.await?.text().await?;
if ilias.opt.verbose > 1 { log!(2, "{}", html);
println!("{}", html);
}
let json: serde_json::Value = { let json: serde_json::Value = {
let mut json_capture = XOCT_REGEX.captures_iter(&html); let mut json_capture = XOCT_REGEX.captures_iter(&html);
let json = &json_capture.next().context("xoct player json not found")?[1]; let json = &json_capture.next().context("xoct player json not found")?[1];
if ilias.opt.verbose > 1 { log!(2, "{}", json);
println!("{}", json);
}
let json = json.split(",\n").nth(0).context("invalid xoct player json")?; let json = json.split(",\n").nth(0).context("invalid xoct player json")?;
serde_json::from_str(&json.trim())? serde_json::from_str(&json.trim())?
}; };
if ilias.opt.verbose > 1 { log!(2, "{}", json);
println!("{}", json);
}
let url = json let url = json
.pointer("/streams/0/sources/mp4/0/src") .pointer("/streams/0/sources/mp4/0/src")
.map(|x| x.as_str()) .map(|x| x.as_str())
@ -722,12 +717,10 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
let head = ilias.client.head(url).send().await.context("HEAD request failed")?; let head = ilias.client.head(url).send().await.context("HEAD request failed")?;
if let Some(len) = head.headers().get("content-length") { if let Some(len) = head.headers().get("content-length") {
if meta.len() != len.to_str()?.parse::<u64>()? { if meta.len() != len.to_str()?.parse::<u64>()? {
println!("Warning: {} was updated, consider moving the outdated file", relative_path.to_string_lossy()); log!(0, "Warning: {} was updated, consider moving the outdated file", relative_path.to_string_lossy());
} }
} }
if ilias.opt.verbose > 1 { log!(2, "Skipping download, file exists already");
println!("Skipping download, file exists already");
}
if !ilias.opt.force { if !ilias.opt.force {
return Ok(()); return Ok(());
} }
@ -736,7 +729,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
let mut reader = stream_reader(resp.bytes_stream().map_err(|x| { let mut reader = stream_reader(resp.bytes_stream().map_err(|x| {
io::Error::new(io::ErrorKind::Other, x) io::Error::new(io::ErrorKind::Other, x)
})); }));
println!("Saving video to {:?}", path); log!(0, "Writing {}", relative_path.to_string_lossy());
let file = AsyncFile::create(&path).await?; let file = AsyncFile::create(&path).await?;
let mut file = BufWriter::new(file); let mut file = BufWriter::new(file);
tokio::io::copy(&mut reader, &mut file).await?; tokio::io::copy(&mut reader, &mut file).await?;
@ -771,7 +764,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
for row in html.select(&tr) { for row in html.select(&tr) {
let cells = row.select(&td).collect::<Vec<_>>(); let cells = row.select(&td).collect::<Vec<_>>();
if cells.len() != 6 { if cells.len() != 6 {
println!("Warning: unusual table row ({} cells) in {}", cells.len(), url); log!(0, "Warning: unusual table row ({} cells) in {}", cells.len(), url);
continue; continue;
} }
let link = cells[1].select(&a).next().context("thread link not found")?; let link = cells[1].select(&a).next().context("thread link not found")?;
@ -793,14 +786,14 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
if available_posts <= saved_posts && !ilias.opt.force { if available_posts <= saved_posts && !ilias.opt.force {
continue; continue;
} }
println!("New posts in {:?}..", path); log!(0, "New posts in {:?}..", path);
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async { task::spawn(async {
process_gracefully(ilias, path, object).await; process_gracefully(ilias, path, object).await;
}); });
} }
if html.select(&forum_pages).count() > 0 { if html.select(&forum_pages).count() > 0 {
println!("Ignoring older threads in {:?}..", path); log!(0, "Ignoring older threads in {:?}..", path);
} }
}, },
Thread { url } => { Thread { url } => {
@ -832,17 +825,15 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
tokio::time::delay_for(Duration::from_millis(100)).await; tokio::time::delay_for(Duration::from_millis(100)).await;
} }
*TASKS_RUNNING.lock() += 1; *TASKS_RUNNING.lock() += 1;
if ilias.opt.verbose > 1 { log!(2, "Writing to {:?}..", path);
println!("Writing to {:?}..", path);
}
let file = AsyncFile::create(&path).await; let file = AsyncFile::create(&path).await;
if file.is_err() { if file.is_err() {
println!("Error creating file {:?}: {}", path, file.err().unwrap()); log!(0, "Error creating file {:?}: {}", path, file.err().unwrap());
return; return;
} }
let mut file = BufWriter::new(file.unwrap()); let mut file = BufWriter::new(file.unwrap());
if let Err(e) = tokio::io::copy(&mut data.as_bytes(), &mut file).await { if let Err(e) = tokio::io::copy(&mut data.as_bytes(), &mut file).await {
println!("Error writing to {:?}: {}", path, e); log!(0, "Error writing to {:?}: {}", path, e);
} }
*TASKS_RUNNING.lock() -= 1; *TASKS_RUNNING.lock() -= 1;
*TASKS_QUEUED.lock() -= 1; *TASKS_QUEUED.lock() -= 1;
@ -863,14 +854,12 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
}); });
} }
} else { } else {
println!("Warning: unable to find pagination links in {}", url.url); log!(0, "Warning: unable to find pagination links in {}", url.url);
} }
} }
}, },
o => { o => {
if ilias.opt.verbose > 0 { log!(1, "Ignored {:?}", o)
println!("ignoring {:?}", o)
}
} }
} }
Ok(()) Ok(())