Download/combine multiple video streams (fix #24)

Cargo.lock

@@ -28,6 +28,7 @@ dependencies = [
  "scraper",
  "serde_json",
  "structopt",
+ "tempfile",
  "tokio",
  "tokio-util",
  "toml",
@@ -649,7 +650,18 @@ checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
 dependencies = [
  "cfg-if",
  "libc",
- "wasi",
+ "wasi 0.9.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.10.2+wasi-snapshot-preview1",
 ]
 
 [[package]]
@@ -1158,7 +1170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"
 dependencies = [
  "phf_shared",
- "rand",
+ "rand 0.7.3",
 ]
 
 [[package]]
@@ -1296,14 +1308,26 @@ version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
 dependencies = [
- "getrandom",
+ "getrandom 0.1.16",
  "libc",
- "rand_chacha",
+ "rand_chacha 0.2.2",
- "rand_core",
+ "rand_core 0.5.1",
- "rand_hc",
+ "rand_hc 0.2.0",
  "rand_pcg",
 ]
 
+[[package]]
+name = "rand"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
+dependencies = [
+ "libc",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.3",
+ "rand_hc 0.3.1",
+]
+
 [[package]]
 name = "rand_chacha"
 version = "0.2.2"
@@ -1311,7 +1335,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.3",
 ]
 
 [[package]]
@@ -1320,7 +1354,16 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
 dependencies = [
- "getrandom",
+ "getrandom 0.1.16",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
+dependencies = [
+ "getrandom 0.2.3",
 ]
 
 [[package]]
@@ -1329,7 +1372,16 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
 dependencies = [
- "rand_core",
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
+dependencies = [
+ "rand_core 0.6.3",
 ]
 
 [[package]]
@@ -1338,7 +1390,16 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
 dependencies = [
- "rand_core",
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
+dependencies = [
+ "bitflags",
 ]
 
 [[package]]
@@ -1358,6 +1419,15 @@ version = "0.6.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
 
+[[package]]
+name = "remove_dir_all"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "reqwest"
 version = "0.11.4"
@@ -1528,7 +1598,7 @@ dependencies = [
  "hkdf",
  "lazy_static",
  "num",
- "rand",
+ "rand 0.7.3",
  "sha2",
 ]
 
@@ -1680,6 +1750,15 @@ dependencies = [
  "opaque-debug",
 ]
 
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "siphasher"
 version = "0.3.7"
@@ -1850,6 +1929,20 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "tempfile"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "rand 0.8.4",
+ "redox_syscall",
+ "remove_dir_all",
+ "winapi",
+]
+
 [[package]]
 name = "tendril"
 version = "0.4.2"
@@ -1980,7 +2073,9 @@ dependencies = [
  "memchr",
  "mio",
  "num_cpus",
+ "once_cell",
  "pin-project-lite",
+ "signal-hook-registry",
  "tokio-macros",
  "winapi",
 ]
@@ -2182,6 +2277,12 @@ version = "0.9.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
 
+[[package]]
+name = "wasi"
+version = "0.10.2+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.78"

Cargo.toml

@@ -9,7 +9,7 @@ edition = "2018"
 
 [dependencies]
 reqwest = { version = "0.11.0", default-features = false, features = ["cookies", "gzip", "json", "rustls-tls", "stream", "socks"] }
-tokio = { version = "1.0.2", features = ["fs", "macros", "net", "rt-multi-thread"] }
+tokio = { version = "1.0.2", features = ["fs", "macros", "net", "rt-multi-thread", "process"] }
 tokio-util = { version = "0.6.1", features = ["io"] }
 serde_json = "1.0.51"
 scraper = "0.12.0"
@@ -34,6 +34,7 @@ cookie_store = "0.14.0"
 reqwest_cookie_store = "0.1.5"
 bytes = "1.0.1"
 toml = "0.5.8"
+tempfile = "3.2.0"
 
 [features]
 default = []

src/cli.rs

@@ -37,6 +37,10 @@ pub struct Opt {
 	#[structopt(long)]
 	pub check_videos: bool,
 
+	/// Combine videos if there is more than one stream (requires ffmpeg)
+	#[structopt(long)]
+	pub combine_videos: bool,
+
 	/// Verbose logging
 	#[structopt(short, multiple = true, parse(from_occurrences))]
 	pub verbose: usize,

src/ilias/video.rs

@@ -1,9 +1,10 @@
-use std::{path::Path, sync::Arc};
+use std::{path::{Path, PathBuf}, process::{ExitStatus, Stdio}, sync::Arc};
 
 use anyhow::{Context, Result};
 use once_cell::sync::Lazy;
 use regex::Regex;
-use tokio::fs;
+use tempfile::tempdir;
+use tokio::{fs, process::Command};
 
 use crate::{util::write_stream_to_file, ILIAS_URL};
 
@@ -32,11 +33,71 @@ pub async fn download(path: &Path, relative_path: &Path, ilias: Arc<ILIAS>, url:
 		serde_json::from_str(&json.trim())?
 	};
 	log!(2, "{}", json);
-	let url = json
+	let streams = json
-		.pointer("/streams/0/sources/mp4/0/src")
+		.get("streams")
+		.context("video streams not found")?
+		.as_array()
+		.context("video streams not an array")?;
+	if streams.len() == 1 {
+		let url = streams[0]
+			.pointer("/sources/mp4/0/src")
 			.context("video src not found")?
 			.as_str()
 			.context("video src not string")?;
+		download_to_path(&ilias, path, relative_path, url).await?;
+	} else {
+		if !ilias.opt.combine_videos {
+			fs::create_dir(path).await.context("failed to create video directory")?;
+			download_all(path, streams, ilias, relative_path).await?;
+		} else {
+			let dir = tempdir()?;
+			// construct ffmpeg command to combine all files
+			let mut arguments = vec![];
+			for file in download_all(dir.path(), streams, ilias, relative_path).await? {
+				arguments.push("-i".to_owned());
+				arguments.push(file.to_str().context("invalid UTF8")?.into());
+			}
+			arguments.push("-c".into());
+			arguments.push("copy".into());
+			for i in 0..(arguments.len() / 2)-1 {
+				arguments.push("-map".into());
+				arguments.push(format!("{}", i));
+			}
+			arguments.push(path.to_str().context("invalid UTF8 in path")?.into());
+			let status = Command::new("ffmpeg")
+				.args(&arguments)
+				.stderr(Stdio::null())
+				.stdout(Stdio::null())
+				.spawn()
+				.context("failed to start ffmpeg")?
+				.wait().await
+				.context("failed to wait for ffmpeg")?;
+			if !status.success() {
+				error!(format!("ffmpeg failed to merge video files into {}", path.display()));
+				error!(format!("check this directory: {}", dir.into_path().display()));
+				error!(format!("ffmpeg command: {}", arguments.join(" ")));
+			}
+		};
+	}
+	Ok(())
+}
+
+async fn download_all(path: &Path, streams: &Vec<serde_json::Value>, ilias: Arc<ILIAS>, relative_path: &Path) -> Result<Vec<PathBuf>> {
+	let mut paths = Vec::new();
+	for (i, stream) in streams.into_iter().enumerate() {
+		let url = stream
+			.pointer("/sources/mp4/0/src")
+			.context("video src not found")?
+			.as_str()
+			.context("video src not string")?;
+		let new_path = path.join(format!("Stream{}.mp4", i + 1));
+		download_to_path(&ilias, &new_path, &relative_path.join(format!("Stream{}.mp4", i + 1)), url).await?;
+		paths.push(new_path);
+	}
+	Ok(paths)
+}
+
+async fn download_to_path(ilias: &ILIAS, path: &Path, relative_path: &Path, url: &str) -> Result<()> {
 	let meta = fs::metadata(&path).await;
 	if !ilias.opt.force && meta.is_ok() && ilias.opt.check_videos {
 		let head = ilias.head(url).await.context("HEAD request failed")?;