mirror of
https://github.com/FliegendeWurst/KIT-ILIAS-downloader.git
synced 2024-08-28 04:04:18 +00:00
Download/combine multiple video streams (fix #24)
This commit is contained in:
FliegendeWurst
parent
2d596f8e08
commit
1c41c95773
123
Cargo.lock
generated
123
Cargo.lock
generated
@ -28,6 +28,7 @@ dependencies = [
|
||||
"scraper",
|
||||
"serde_json",
|
||||
"structopt",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"toml",
|
||||
@ -649,7 +650,18 @@ checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
"wasi 0.9.0+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi 0.10.2+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1158,7 +1170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"
|
||||
dependencies = [
|
||||
"phf_shared",
|
||||
"rand",
|
||||
"rand 0.7.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1296,14 +1308,26 @@ version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"getrandom 0.1.16",
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
"rand_hc",
|
||||
"rand_chacha 0.2.2",
|
||||
"rand_core 0.5.1",
|
||||
"rand_hc 0.2.0",
|
||||
"rand_pcg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha 0.3.1",
|
||||
"rand_core 0.6.3",
|
||||
"rand_hc 0.3.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.2.2"
|
||||
@ -1311,7 +1335,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
"rand_core 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.6.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1320,7 +1354,16 @@ version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"getrandom 0.1.16",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
|
||||
dependencies = [
|
||||
"getrandom 0.2.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1329,7 +1372,16 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
|
||||
dependencies = [
|
||||
"rand_core",
|
||||
"rand_core 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_hc"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
|
||||
dependencies = [
|
||||
"rand_core 0.6.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1338,7 +1390,16 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
|
||||
dependencies = [
|
||||
"rand_core",
|
||||
"rand_core 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1358,6 +1419,15 @@ version = "0.6.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.11.4"
|
||||
@ -1528,7 +1598,7 @@ dependencies = [
|
||||
"hkdf",
|
||||
"lazy_static",
|
||||
"num",
|
||||
"rand",
|
||||
"rand 0.7.3",
|
||||
"sha2",
|
||||
]
|
||||
|
||||
@ -1680,6 +1750,15 @@ dependencies = [
|
||||
"opaque-debug",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "0.3.7"
|
||||
@ -1850,6 +1929,20 @@ dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"rand 0.8.4",
|
||||
"redox_syscall",
|
||||
"remove_dir_all",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.4.2"
|
||||
@ -1980,7 +2073,9 @@ dependencies = [
|
||||
"memchr",
|
||||
"mio",
|
||||
"num_cpus",
|
||||
"once_cell",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"tokio-macros",
|
||||
"winapi",
|
||||
]
|
||||
@ -2182,6 +2277,12 @@ version = "0.9.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.10.2+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.78"
|
||||
|
||||
@ -9,7 +9,7 @@ edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
reqwest = { version = "0.11.0", default-features = false, features = ["cookies", "gzip", "json", "rustls-tls", "stream", "socks"] }
|
||||
tokio = { version = "1.0.2", features = ["fs", "macros", "net", "rt-multi-thread"] }
|
||||
tokio = { version = "1.0.2", features = ["fs", "macros", "net", "rt-multi-thread", "process"] }
|
||||
tokio-util = { version = "0.6.1", features = ["io"] }
|
||||
serde_json = "1.0.51"
|
||||
scraper = "0.12.0"
|
||||
@ -34,6 +34,7 @@ cookie_store = "0.14.0"
|
||||
reqwest_cookie_store = "0.1.5"
|
||||
bytes = "1.0.1"
|
||||
toml = "0.5.8"
|
||||
tempfile = "3.2.0"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
|
||||
@ -37,6 +37,10 @@ pub struct Opt {
|
||||
#[structopt(long)]
|
||||
pub check_videos: bool,
|
||||
|
||||
/// Combine videos if there is more than one stream (requires ffmpeg)
|
||||
#[structopt(long)]
|
||||
pub combine_videos: bool,
|
||||
|
||||
/// Verbose logging
|
||||
#[structopt(short, multiple = true, parse(from_occurrences))]
|
||||
pub verbose: usize,
|
||||
|
||||
@ -1,9 +1,10 @@
|
||||
use std::{path::Path, sync::Arc};
|
||||
use std::{path::{Path, PathBuf}, process::{ExitStatus, Stdio}, sync::Arc};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use once_cell::sync::Lazy;
|
||||
use regex::Regex;
|
||||
use tokio::fs;
|
||||
use tempfile::tempdir;
|
||||
use tokio::{fs, process::Command};
|
||||
|
||||
use crate::{util::write_stream_to_file, ILIAS_URL};
|
||||
|
||||
@ -32,11 +33,71 @@ pub async fn download(path: &Path, relative_path: &Path, ilias: Arc<ILIAS>, url:
|
||||
serde_json::from_str(&json.trim())?
|
||||
};
|
||||
log!(2, "{}", json);
|
||||
let url = json
|
||||
.pointer("/streams/0/sources/mp4/0/src")
|
||||
let streams = json
|
||||
.get("streams")
|
||||
.context("video streams not found")?
|
||||
.as_array()
|
||||
.context("video streams not an array")?;
|
||||
if streams.len() == 1 {
|
||||
let url = streams[0]
|
||||
.pointer("/sources/mp4/0/src")
|
||||
.context("video src not found")?
|
||||
.as_str()
|
||||
.context("video src not string")?;
|
||||
download_to_path(&ilias, path, relative_path, url).await?;
|
||||
} else {
|
||||
if !ilias.opt.combine_videos {
|
||||
fs::create_dir(path).await.context("failed to create video directory")?;
|
||||
download_all(path, streams, ilias, relative_path).await?;
|
||||
} else {
|
||||
let dir = tempdir()?;
|
||||
// construct ffmpeg command to combine all files
|
||||
let mut arguments = vec![];
|
||||
for file in download_all(dir.path(), streams, ilias, relative_path).await? {
|
||||
arguments.push("-i".to_owned());
|
||||
arguments.push(file.to_str().context("invalid UTF8")?.into());
|
||||
}
|
||||
arguments.push("-c".into());
|
||||
arguments.push("copy".into());
|
||||
for i in 0..(arguments.len() / 2)-1 {
|
||||
arguments.push("-map".into());
|
||||
arguments.push(format!("{}", i));
|
||||
}
|
||||
arguments.push(path.to_str().context("invalid UTF8 in path")?.into());
|
||||
let status = Command::new("ffmpeg")
|
||||
.args(&arguments)
|
||||
.stderr(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.spawn()
|
||||
.context("failed to start ffmpeg")?
|
||||
.wait().await
|
||||
.context("failed to wait for ffmpeg")?;
|
||||
if !status.success() {
|
||||
error!(format!("ffmpeg failed to merge video files into {}", path.display()));
|
||||
error!(format!("check this directory: {}", dir.into_path().display()));
|
||||
error!(format!("ffmpeg command: {}", arguments.join(" ")));
|
||||
}
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn download_all(path: &Path, streams: &Vec<serde_json::Value>, ilias: Arc<ILIAS>, relative_path: &Path) -> Result<Vec<PathBuf>> {
|
||||
let mut paths = Vec::new();
|
||||
for (i, stream) in streams.into_iter().enumerate() {
|
||||
let url = stream
|
||||
.pointer("/sources/mp4/0/src")
|
||||
.context("video src not found")?
|
||||
.as_str()
|
||||
.context("video src not string")?;
|
||||
let new_path = path.join(format!("Stream{}.mp4", i + 1));
|
||||
download_to_path(&ilias, &new_path, &relative_path.join(format!("Stream{}.mp4", i + 1)), url).await?;
|
||||
paths.push(new_path);
|
||||
}
|
||||
Ok(paths)
|
||||
}
|
||||
|
||||
async fn download_to_path(ilias: &ILIAS, path: &Path, relative_path: &Path, url: &str) -> Result<()> {
|
||||
let meta = fs::metadata(&path).await;
|
||||
if !ilias.opt.force && meta.is_ok() && ilias.opt.check_videos {
|
||||
let head = ilias.head(url).await.context("HEAD request failed")?;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user