mirror of
https://github.com/FliegendeWurst/KIT-ILIAS-downloader.git
synced 2024-08-28 04:04:18 +00:00
Retry requests on HTTP/2 NO_ERROR
This commit is contained in:
FliegendeWurst
parent
d899fcccba
commit
d5037f9ae7
51
Cargo.lock
generated
51
Cargo.lock
generated
@ -13,6 +13,7 @@ dependencies = [
|
||||
"futures",
|
||||
"futures-channel",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"ignore",
|
||||
"indicatif",
|
||||
"keyring",
|
||||
@ -491,9 +492,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253"
|
||||
checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
@ -506,9 +507,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25"
|
||||
checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
@ -516,15 +517,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815"
|
||||
checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d"
|
||||
checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
@ -533,16 +534,17 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04"
|
||||
checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b"
|
||||
checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"proc-macro-hack",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@ -551,22 +553,23 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23"
|
||||
checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc"
|
||||
checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025"
|
||||
checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
@ -1530,18 +1533,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.125"
|
||||
version = "1.0.126"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171"
|
||||
checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.125"
|
||||
version = "1.0.126"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d"
|
||||
checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@ -1589,9 +1592,9 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d"
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.9.4"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8f6b75b17576b792bef0db1bcc4b8b8bcdf9506744cf34b974195487af6cff2"
|
||||
checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"cfg-if",
|
||||
|
||||
@ -29,6 +29,7 @@ cfg-if = "1.0.0"
|
||||
indicatif = "0.16.0"
|
||||
once_cell = "1.7.2"
|
||||
atty = "0.2.14"
|
||||
h2 = "0.3.3"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
|
||||
53
src/main.rs
53
src/main.rs
@ -19,6 +19,7 @@ use tokio::task::{self, JoinHandle};
|
||||
use tokio_util::io::StreamReader;
|
||||
use url::Url;
|
||||
|
||||
use std::error::Error as _;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::path::PathBuf;
|
||||
@ -156,7 +157,7 @@ async fn real_main(mut opt: Opt) -> Result<()> {
|
||||
};
|
||||
if ilias.opt.content_tree {
|
||||
// need this to get the content tree
|
||||
if let Err(e) = ilias.client.get("https://ilias.studium.kit.edu/ilias.php?baseClass=ilRepositoryGUI&cmd=frameset&set_mode=tree&ref_id=1").send().await {
|
||||
if let Err(e) = ilias.download("ilias.php?baseClass=ilRepositoryGUI&cmd=frameset&set_mode=tree&ref_id=1").await {
|
||||
warning!("could not enable content tree:", e);
|
||||
}
|
||||
}
|
||||
@ -167,7 +168,7 @@ async fn real_main(mut opt: Opt) -> Result<()> {
|
||||
PROGRESS_BAR_ENABLED.store(atty::is(atty::Stream::Stdout), Ordering::SeqCst);
|
||||
if PROGRESS_BAR_ENABLED.load(Ordering::SeqCst) {
|
||||
PROGRESS_BAR.set_draw_target(ProgressDrawTarget::stderr_nohz());
|
||||
PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}+] {msg}"));
|
||||
PROGRESS_BAR.set_style(ProgressStyle::default_bar().template("[{pos}/{len}+] {wide_msg}"));
|
||||
PROGRESS_BAR.inc_length(1);
|
||||
PROGRESS_BAR.set_message("initializing..");
|
||||
}
|
||||
@ -192,7 +193,7 @@ async fn real_main(mut opt: Opt) -> Result<()> {
|
||||
// channel is empty => all tasks are completed
|
||||
if ilias.opt.content_tree {
|
||||
// restore fast page loading times
|
||||
if let Err(e) = ilias.client.get("https://ilias.studium.kit.edu/ilias.php?baseClass=ilRepositoryGUI&cmd=frameset&set_mode=flat&ref_id=1").send().await {
|
||||
if let Err(e) = ilias.download("ilias.php?baseClass=ilRepositoryGUI&cmd=frameset&set_mode=flat&ref_id=1").await {
|
||||
warning!("could not disable content tree:", e);
|
||||
}
|
||||
}
|
||||
@ -911,6 +912,18 @@ struct ILIAS {
|
||||
client: Client,
|
||||
}
|
||||
|
||||
/// Returns true if the error is caused by:
|
||||
/// "http2 error: protocol error: not a result of an error"
|
||||
fn error_is_http2(error: &reqwest::Error) -> bool {
|
||||
error.source()
|
||||
.map(|x| x.downcast_ref::<h2::Error>())
|
||||
.flatten()
|
||||
.map(|x| x.reason())
|
||||
.flatten()
|
||||
.map(|x| x == h2::Reason::NO_ERROR)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
impl ILIAS {
|
||||
async fn login(opt: Opt, user: impl Into<String>, pass: impl Into<String>, ignore: Gitignore) -> Result<Self> {
|
||||
let user = user.into();
|
||||
@ -975,16 +988,42 @@ impl ILIAS {
|
||||
async fn download(&self, url: &str) -> Result<reqwest::Response> {
|
||||
get_request_ticket().await;
|
||||
log!(2, "Downloading {}", url);
|
||||
if url.starts_with("http") || url.starts_with("ilias.studium.kit.edu") {
|
||||
Ok(self.client.get(url).send().await?)
|
||||
let url = if url.starts_with("http://") || url.starts_with("https://") {
|
||||
url.to_owned()
|
||||
} else if url.starts_with("ilias.studium.kit.edu") {
|
||||
format!("https://{}", url)
|
||||
} else {
|
||||
Ok(self.client.get(&format!("{}{}", ILIAS_URL, url)).send().await?)
|
||||
format!("{}{}", ILIAS_URL, url)
|
||||
};
|
||||
for attempt in 1..10 {
|
||||
let result = self.client.head(url.clone()).send().await;
|
||||
match result {
|
||||
Ok(x) => return Ok(x),
|
||||
Err(e) if attempt <= 3 && error_is_http2(&e) => {
|
||||
warning!("encountered HTTP/2 NO_ERROR, retrying download..");
|
||||
continue
|
||||
},
|
||||
Err(e) => return Err(e.into())
|
||||
}
|
||||
}
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
async fn head<U: IntoUrl>(&self, url: U) -> Result<reqwest::Response, reqwest::Error> {
|
||||
get_request_ticket().await;
|
||||
self.client.head(url).send().await
|
||||
let url = url.into_url()?;
|
||||
for attempt in 1..10 {
|
||||
let result = self.client.head(url.clone()).send().await;
|
||||
match result {
|
||||
Ok(x) => return Ok(x),
|
||||
Err(e) if attempt <= 3 && error_is_http2(&e) => {
|
||||
warning!("encountered HTTP/2 NO_ERROR, retrying download..");
|
||||
continue
|
||||
},
|
||||
Err(e) => return Err(e)
|
||||
}
|
||||
}
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
async fn get_html(&self, url: &str) -> Result<Html> {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user