arnekeller/KIT-ILIAS-downloader
1
0
Fork 0
mirror of https://github.com/FliegendeWurst/KIT-ILIAS-downloader.git synced 2024-08-28 04:04:18 +00:00
Code Issues Projects Releases Wiki Activity

Limit parallel network requests to CLI parameter

Browse Source
This commit is contained in:
FliegendeWurst 2020-04-22 12:19:12 +02:00
parent 1aca252f8c
commit 84eda01c29
1 changed files with 16 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/main.rs
View File

@ -340,7 +340,7 @@ async fn main() {
while let Some((path, obj)) = queue.pop_front() { while let Some((path, obj)) = queue.pop_front() {
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async { task::spawn(async {
while *TASKS_RUNNING.lock() > 1 { while *TASKS_RUNNING.lock() > ilias.opt.jobs {
tokio::time::delay_for(Duration::from_millis(100)).await; tokio::time::delay_for(Duration::from_millis(100)).await;
} }
*TASKS_RUNNING.lock() += 1; *TASKS_RUNNING.lock() += 1;
@ -378,7 +378,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
path.push(item.name()); path.push(item.name());
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async { task::spawn(async {
while *TASKS_RUNNING.lock() > 1 { while *TASKS_RUNNING.lock() > ilias.opt.jobs {
tokio::time::delay_for(Duration::from_millis(100)).await; tokio::time::delay_for(Duration::from_millis(100)).await;
} }
*TASKS_RUNNING.lock() += 1; *TASKS_RUNNING.lock() += 1;
@ -399,7 +399,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
path.push(item.name()); path.push(item.name());
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async { task::spawn(async {
while *TASKS_RUNNING.lock() > 1 { while *TASKS_RUNNING.lock() > ilias.opt.jobs {
tokio::time::delay_for(Duration::from_millis(100)).await; tokio::time::delay_for(Duration::from_millis(100)).await;
} }
*TASKS_RUNNING.lock() += 1; *TASKS_RUNNING.lock() += 1;
@ -470,7 +470,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
}; };
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async { task::spawn(async {
while *TASKS_RUNNING.lock() > 1 { while *TASKS_RUNNING.lock() > ilias.opt.jobs {
tokio::time::delay_for(Duration::from_millis(100)).await; tokio::time::delay_for(Duration::from_millis(100)).await;
} }
*TASKS_RUNNING.lock() += 1; *TASKS_RUNNING.lock() += 1;
@ -563,7 +563,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
path.push(name); path.push(name);
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async { task::spawn(async {
while *TASKS_RUNNING.lock() > 1 { while *TASKS_RUNNING.lock() > ilias.opt.jobs {
tokio::time::delay_for(Duration::from_millis(100)).await; tokio::time::delay_for(Duration::from_millis(100)).await;
} }
*TASKS_RUNNING.lock() += 1; *TASKS_RUNNING.lock() += 1;
@ -607,7 +607,7 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
path.push(name); path.push(name);
let ilias = Arc::clone(&ilias); let ilias = Arc::clone(&ilias);
task::spawn(async move { task::spawn(async move {
while *TASKS_RUNNING.lock() > 1 { while *TASKS_RUNNING.lock() > ilias.opt.jobs {
tokio::time::delay_for(Duration::from_millis(100)).await; tokio::time::delay_for(Duration::from_millis(100)).await;
} }
*TASKS_RUNNING.lock() += 1; *TASKS_RUNNING.lock() += 1;
@ -632,16 +632,16 @@ fn process(ilias: Arc<ILIAS>, path: PathBuf, obj: Object) -> impl std::future::F
#[derive(Debug, StructOpt)] #[derive(Debug, StructOpt)]
#[structopt(name = "KIT-ILIAS-downloader")] #[structopt(name = "KIT-ILIAS-downloader")]
struct Opt { struct Opt {
/// Do not download files /// Do not download files
#[structopt(short, long)] #[structopt(short, long)]
skip_files: bool, skip_files: bool,
/// Do not download Opencast videos /// Do not download Opencast videos
#[structopt(short, long)] #[structopt(short, long)]
no_videos: bool, no_videos: bool,
/// Re-download already present files /// Re-download already present files
#[structopt(short)] #[structopt(short)]
force: bool, force: bool,
/// Verbose logging (print objects downloaded) /// Verbose logging (print objects downloaded)
@ -649,6 +649,10 @@ struct Opt {
verbose: usize, verbose: usize,
/// Output directory /// Output directory
#[structopt(short, long, parse(from_os_str))] #[structopt(short, long, parse(from_os_str))]
output: PathBuf, output: PathBuf,
/// Parallel download jobs
#[structopt(short, long, default_value = "1")]
jobs: usize,
} }