arnekeller/KIT-ILIAS-downloader
1
0
Fork 0
mirror of https://github.com/FliegendeWurst/KIT-ILIAS-downloader.git synced 2024-08-28 04:04:18 +00:00
Code Issues Projects Releases Wiki Activity

Clean up code slightly

Browse Source
This commit is contained in:
FliegendeWurst 2020-05-11 17:35:30 +02:00
parent dfdd6829fc
commit 0fd1e6ce5b
2 changed files with 18 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/main.rs
View File

@ -24,7 +24,7 @@ use std::time::Duration;
mod util; mod util;
use util::*; use util::*;
const ILIAS_URL: &'static str = "https://ilias.studium.kit.edu/"; const ILIAS_URL: &str = "https://ilias.studium.kit.edu/";
#[tokio::main] #[tokio::main]
async fn main() { async fn main() {
@ -61,9 +61,7 @@ async fn main() {
let mut path = ilias.opt.output.clone(); let mut path = ilias.opt.output.clone();
path.push(item.name()); path.push(item.name());
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async { task::spawn(process_gracefully(ilias, path, item));
process_gracefully(ilias, path, item).await;
});
} }
}, },
Err(e) => println!("{:?}", e) Err(e) => println!("{:?}", e)
@ -184,9 +182,7 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
let mut path = path.clone(); let mut path = path.clone();
path.push(item.name()); path.push(item.name());
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async { task::spawn(process_gracefully(ilias, path, item));
process_gracefully(ilias, path, item).await;
});
} }
}, },
Folder { url, .. } => { Folder { url, .. } => {
@ -201,9 +197,7 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
let mut path = path.clone(); let mut path = path.clone();
path.push(item.name()); path.push(item.name());
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async { task::spawn(process_gracefully(ilias, path, item));
process_gracefully(ilias, path, item).await;
});
} }
}, },
File { url, .. } => { File { url, .. } => {
@ -227,8 +221,8 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
} }
create_dir(&path).await?; create_dir(&path).await?;
let list_url = format!("{}ilias.php?ref_id={}&cmdClass=xocteventgui&cmdNode=n7:mz:14p&baseClass=ilObjPluginDispatchGUI&lang=de&limit=20&cmd=asyncGetTableGUI&cmdMode=asynch", ILIAS_URL, url.ref_id); let list_url = format!("{}ilias.php?ref_id={}&cmdClass=xocteventgui&cmdNode=n7:mz:14p&baseClass=ilObjPluginDispatchGUI&lang=de&limit=20&cmd=asyncGetTableGUI&cmdMode=asynch", ILIAS_URL, url.ref_id);
let data = ilias.download(&list_url); let data = ilias.download(&list_url).await?;
let html = data.await?.text().await?; let html = data.text().await?;
let html = Html::parse_fragment(&html); let html = Html::parse_fragment(&html);
for row in html.select(&video_tr) { for row in html.select(&video_tr) {
let link = row.select(&a_target_blank).next(); let link = row.select(&a_target_blank).next();
@ -272,7 +266,7 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
let mut json_capture = XOCT_REGEX.captures_iter(&html); let mut json_capture = XOCT_REGEX.captures_iter(&html);
let json = &json_capture.next().context("xoct player json not found")?[1]; let json = &json_capture.next().context("xoct player json not found")?[1];
log!(2, "{}", json); log!(2, "{}", json);
let json = json.split(",\n").nth(0).context("invalid xoct player json")?; let json = json.split(",\n").next().context("invalid xoct player json")?;
serde_json::from_str(&json.trim())? serde_json::from_str(&json.trim())?
}; };
log!(2, "{}", json); log!(2, "{}", json);
@ -315,8 +309,7 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
html html
.select(&a) .select(&a)
.flat_map(|x| x.value().attr("href")) .flat_map(|x| x.value().attr("href"))
.filter(|x| x.contains("trows=800")) .find(|x| x.contains("trows=800"))
.next()
.context("can't find forum thread count selector (empty forum?)")?.to_owned() .context("can't find forum thread count selector (empty forum?)")?.to_owned()
}; };
let data = ilias.download(&url); let data = ilias.download(&url);
@ -350,9 +343,7 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
} }
log!(0, "New posts in {:?}..", path); log!(0, "New posts in {:?}..", path);
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async { task::spawn(process_gracefully(ilias, path, object));
process_gracefully(ilias, path, object).await;
});
} }
if html.select(&forum_pages).count() > 0 { if html.select(&forum_pages).count() > 0 {
log!(0, "Ignoring older threads in {:?}..", path); log!(0, "Ignoring older threads in {:?}..", path);
@ -376,12 +367,10 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
let data = data.inner_html(); let data = data.inner_html();
let mut path = path.clone(); let mut path = path.clone();
path.push(name); path.push(name);
task::spawn(async move { task::spawn(handle_gracefully(async move {
handle_gracefully(async move {
write_file_data(&path, &mut data.as_bytes()).await write_file_data(&path, &mut data.as_bytes()).await
.context("failed to write forum post") .context("failed to write forum post")
}).await; }));
});
} }
// pagination // pagination
if let Some(pages) = html.select(&table).next() { if let Some(pages) = html.select(&table).next() {
@ -393,9 +382,7 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
let next_page = Thread { let next_page = Thread {
url: URL::from_href(last.value().attr("href").ok_or(anyhow!("page link not found"))?)? url: URL::from_href(last.value().attr("href").ok_or(anyhow!("page link not found"))?)?
}; };
task::spawn(async move { task::spawn(process_gracefully(ilias, path, next_page));
process_gracefully(ilias, path, next_page).await;
});
} }
} else { } else {
log!(0, "Warning: unable to find pagination links in {}", url.url); log!(0, "Warning: unable to find pagination links in {}", url.url);
@ -427,9 +414,7 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
let mut path = path.clone(); let mut path = path.clone();
path.push(item.name()); path.push(item.name());
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async { task::spawn(process_gracefully(ilias, path, item));
process_gracefully(ilias, path, item).await;
});
} }
}, },
Weblink { url, .. } => { Weblink { url, .. } => {
@ -441,7 +426,7 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
let url = head.url().as_str(); let url = head.url().as_str();
if url.starts_with(ILIAS_URL) { if url.starts_with(ILIAS_URL) {
// is a link list // is a link list
if !fs::metadata(&path).await.is_ok() { if fs::metadata(&path).await.is_err() {
create_dir(&path).await?; create_dir(&path).await?;
log!(0, "Writing {}", relative_path.to_string_lossy()); log!(0, "Writing {}", relative_path.to_string_lossy());
} }
@ -892,7 +877,7 @@ impl Object {
return Ok(Generic { name, url }); return Ok(Generic { name, url });
} }
if url.cmd.as_ref().map(|x| &**x) == Some("showThreads") { if url.cmd.as_deref() == Some("showThreads") {
return Ok(Forum { name, url }); return Ok(Forum { name, url });
} }

2
src/util.rs
View File

@ -18,7 +18,7 @@ where R: AsyncRead + Unpin {
pub async fn create_dir(path: &Path) -> Result<()> { pub async fn create_dir(path: &Path) -> Result<()> {
if let Err(e) = tokio::fs::create_dir(&path).await { if let Err(e) = tokio::fs::create_dir(&path).await {
if e.kind() != tokio::io::ErrorKind::AlreadyExists { if e.kind() != tokio::io::ErrorKind::AlreadyExists {
Err(e)?; return Err(e.into());
} }
} }
Ok(()) Ok(())