From 1c41c957737bb83aad8f8b4507e80299e6fd3567 Mon Sep 17 00:00:00 2001 From: FliegendeWurst <2012gdwu+github@posteo.de> Date: Mon, 1 Nov 2021 17:36:25 +0100 Subject: [PATCH] Download/combine multiple video streams (fix #24) --- Cargo.lock | 123 +++++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 3 +- src/cli.rs | 4 ++ src/ilias/video.rs | 75 ++++++++++++++++++++++++--- 4 files changed, 186 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8f2af5c..21d7e04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,7 @@ dependencies = [ "scraper", "serde_json", "structopt", + "tempfile", "tokio", "tokio-util", "toml", @@ -649,7 +650,18 @@ checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] @@ -1158,7 +1170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" dependencies = [ "phf_shared", - "rand", + "rand 0.7.3", ] [[package]] @@ -1296,14 +1308,26 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom", + "getrandom 0.1.16", "libc", - "rand_chacha", - "rand_core", - "rand_hc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", "rand_pcg", ] +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.3", + "rand_hc 0.3.1", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -1311,7 +1335,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.3", ] [[package]] @@ -1320,7 +1354,16 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom", + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom 0.2.3", ] [[package]] @@ -1329,7 +1372,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" dependencies = [ - "rand_core", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_hc" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" +dependencies = [ + "rand_core 0.6.3", ] [[package]] @@ -1338,7 +1390,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" dependencies = [ - "rand_core", + "rand_core 0.5.1", +] + +[[package]] +name = "redox_syscall" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +dependencies = [ + "bitflags", ] [[package]] @@ -1358,6 +1419,15 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "reqwest" version = "0.11.4" @@ -1528,7 +1598,7 @@ dependencies = [ "hkdf", "lazy_static", "num", - "rand", + "rand 0.7.3", "sha2", ] @@ -1680,6 +1750,15 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + [[package]] name = "siphasher" version = "0.3.7" @@ -1850,6 +1929,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if", + "libc", + "rand 0.8.4", + "redox_syscall", + "remove_dir_all", + "winapi", +] + [[package]] name = "tendril" version = "0.4.2" @@ -1980,7 +2073,9 @@ dependencies = [ "memchr", "mio", "num_cpus", + "once_cell", "pin-project-lite", + "signal-hook-registry", "tokio-macros", "winapi", ] @@ -2182,6 +2277,12 @@ version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + [[package]] name = "wasm-bindgen" version = "0.2.78" diff --git a/Cargo.toml b/Cargo.toml index 9673ffc..6fb97bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ edition = "2018" [dependencies] reqwest = { version = "0.11.0", default-features = false, features = ["cookies", "gzip", "json", "rustls-tls", "stream", "socks"] } -tokio = { version = "1.0.2", features = ["fs", "macros", "net", "rt-multi-thread"] } +tokio = { version = "1.0.2", features = ["fs", "macros", "net", "rt-multi-thread", "process"] } tokio-util = { version = "0.6.1", features = ["io"] } serde_json = "1.0.51" scraper = "0.12.0" @@ -34,6 +34,7 @@ cookie_store = "0.14.0" reqwest_cookie_store = "0.1.5" bytes = "1.0.1" toml = "0.5.8" +tempfile = "3.2.0" [features] default = [] diff --git a/src/cli.rs b/src/cli.rs index faf4f5a..3d43518 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -37,6 +37,10 @@ pub struct Opt { #[structopt(long)] pub check_videos: bool, + /// Combine videos if there is more than one stream (requires ffmpeg) + #[structopt(long)] + pub combine_videos: bool, + /// Verbose logging #[structopt(short, multiple = true, parse(from_occurrences))] pub verbose: usize, diff --git a/src/ilias/video.rs b/src/ilias/video.rs index e322544..12bd4b5 100644 --- a/src/ilias/video.rs +++ b/src/ilias/video.rs @@ -1,9 +1,10 @@ -use std::{path::Path, sync::Arc}; +use std::{path::{Path, PathBuf}, process::{ExitStatus, Stdio}, sync::Arc}; use anyhow::{Context, Result}; use once_cell::sync::Lazy; use regex::Regex; -use tokio::fs; +use tempfile::tempdir; +use tokio::{fs, process::Command}; use crate::{util::write_stream_to_file, ILIAS_URL}; @@ -32,11 +33,71 @@ pub async fn download(path: &Path, relative_path: &Path, ilias: Arc, url: serde_json::from_str(&json.trim())? }; log!(2, "{}", json); - let url = json - .pointer("/streams/0/sources/mp4/0/src") - .context("video src not found")? - .as_str() - .context("video src not string")?; + let streams = json + .get("streams") + .context("video streams not found")? + .as_array() + .context("video streams not an array")?; + if streams.len() == 1 { + let url = streams[0] + .pointer("/sources/mp4/0/src") + .context("video src not found")? + .as_str() + .context("video src not string")?; + download_to_path(&ilias, path, relative_path, url).await?; + } else { + if !ilias.opt.combine_videos { + fs::create_dir(path).await.context("failed to create video directory")?; + download_all(path, streams, ilias, relative_path).await?; + } else { + let dir = tempdir()?; + // construct ffmpeg command to combine all files + let mut arguments = vec![]; + for file in download_all(dir.path(), streams, ilias, relative_path).await? { + arguments.push("-i".to_owned()); + arguments.push(file.to_str().context("invalid UTF8")?.into()); + } + arguments.push("-c".into()); + arguments.push("copy".into()); + for i in 0..(arguments.len() / 2)-1 { + arguments.push("-map".into()); + arguments.push(format!("{}", i)); + } + arguments.push(path.to_str().context("invalid UTF8 in path")?.into()); + let status = Command::new("ffmpeg") + .args(&arguments) + .stderr(Stdio::null()) + .stdout(Stdio::null()) + .spawn() + .context("failed to start ffmpeg")? + .wait().await + .context("failed to wait for ffmpeg")?; + if !status.success() { + error!(format!("ffmpeg failed to merge video files into {}", path.display())); + error!(format!("check this directory: {}", dir.into_path().display())); + error!(format!("ffmpeg command: {}", arguments.join(" "))); + } + }; + } + Ok(()) +} + +async fn download_all(path: &Path, streams: &Vec, ilias: Arc, relative_path: &Path) -> Result> { + let mut paths = Vec::new(); + for (i, stream) in streams.into_iter().enumerate() { + let url = stream + .pointer("/sources/mp4/0/src") + .context("video src not found")? + .as_str() + .context("video src not string")?; + let new_path = path.join(format!("Stream{}.mp4", i + 1)); + download_to_path(&ilias, &new_path, &relative_path.join(format!("Stream{}.mp4", i + 1)), url).await?; + paths.push(new_path); + } + Ok(paths) +} + +async fn download_to_path(ilias: &ILIAS, path: &Path, relative_path: &Path, url: &str) -> Result<()> { let meta = fs::metadata(&path).await; if !ilias.opt.force && meta.is_ok() && ilias.opt.check_videos { let head = ilias.head(url).await.context("HEAD request failed")?;