arnekeller/KIT-ILIAS-downloader
1
0
Fork 0
mirror of https://github.com/FliegendeWurst/KIT-ILIAS-downloader.git synced 2024-08-28 04:04:18 +00:00
Code Issues Projects Releases Wiki Activity

Update tokio to 1.0

Browse Source
This commit is contained in:
FliegendeWurst 2021-01-20 10:26:38 +01:00
parent c5b8cc7399
commit 711bfe8d17
3 changed files with 202 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

567
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

5
Cargo.toml
View File

@ -8,8 +8,9 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
reqwest = { version = "0.10.4", default-features = false, features = ["cookies", "gzip", "json", "rustls-tls", "stream"] } reqwest = { version = "0.11.0", default-features = false, features = ["cookies", "gzip", "json", "rustls-tls", "stream"] }
tokio = { version = "0.2", features = ["full"] } tokio = { version = "1.0.2", features = ["full"] }
tokio-util = { version = "0.6.1", features = ["io"] }
serde_json = "1.0.51" serde_json = "1.0.51"
scraper = "0.12.0" scraper = "0.12.0"
url = "2.1.1" url = "2.1.1"

8
src/main.rs
View File

@ -11,8 +11,8 @@ use scraper::{ElementRef, Html, Selector};
use serde_json::json; use serde_json::json;
use structopt::StructOpt; use structopt::StructOpt;
use tokio::fs; use tokio::fs;
use tokio::io::stream_reader;
use tokio::task::{self, JoinHandle}; use tokio::task::{self, JoinHandle};
use tokio_util::io::StreamReader;
use url::Url; use url::Url;
use std::{collections::HashSet, default::Default}; use std::{collections::HashSet, default::Default};
@ -138,7 +138,7 @@ fn process_gracefully(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl Fut
break; break;
} }
} }
tokio::time::delay_for(Duration::from_millis(50)).await; tokio::time::sleep(Duration::from_millis(50)).await;
} }
let path_text = path.to_string_lossy().into_owned(); let path_text = path.to_string_lossy().into_owned();
if let Err(e) = process(ilias, path, obj).await.context("failed to process URL") { if let Err(e) = process(ilias, path, obj).await.context("failed to process URL") {
@ -256,7 +256,7 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
return Ok(()); return Ok(());
} }
let data = ilias.download(&url.url).await?; let data = ilias.download(&url.url).await?;
let mut reader = stream_reader(data.bytes_stream().map_err(|x| { let mut reader = StreamReader::new(data.bytes_stream().map_err(|x| {
io::Error::new(io::ErrorKind::Other, x) io::Error::new(io::ErrorKind::Other, x)
})); }));
log!(0, "Writing {}", relative_path.to_string_lossy()); log!(0, "Writing {}", relative_path.to_string_lossy());
@ -360,7 +360,7 @@ fn process(ilias: Arc<ILIAS>, mut path: PathBuf, obj: Object) -> impl Future<Out
} }
} else { } else {
let resp = ilias.download(&url).await?; let resp = ilias.download(&url).await?;
let mut reader = stream_reader(resp.bytes_stream().map_err(|x| { let mut reader = StreamReader::new(resp.bytes_stream().map_err(|x| {
io::Error::new(io::ErrorKind::Other, x) io::Error::new(io::ErrorKind::Other, x)
})); }));
log!(0, "Writing {}", relative_path.to_string_lossy()); log!(0, "Writing {}", relative_path.to_string_lossy());