From 766211edc43deb1b798e8ed555a53beb61db71d2 Mon Sep 17 00:00:00 2001 From: phiresky Date: Fri, 12 Jun 2020 00:09:37 +0200 Subject: [PATCH] partial fixes --- src/adapters.rs | 4 +-- src/adapters/custom.rs | 53 +++++++++++++++++++++++++++++++++++++--- src/adapters/ffmpeg.rs | 24 +++++++++++------- src/adapters/fns.rs | 4 ++- src/adapters/spawning.rs | 8 ++++-- src/adapters/writing.rs | 1 + src/bin/rga-preproc.rs | 4 +-- 7 files changed, 78 insertions(+), 20 deletions(-) diff --git a/src/adapters.rs b/src/adapters.rs index 8584a91..bf382fb 100644 --- a/src/adapters.rs +++ b/src/adapters.rs @@ -1,6 +1,6 @@ pub mod custom; pub mod decompress; -//pub mod ffmpeg; +pub mod ffmpeg; pub mod fns; //pub mod pdfpages; pub mod poppler; @@ -95,7 +95,7 @@ pub fn get_all_adapters(custom_adapters: Option>) -> Ad } let internal_adapters: Vec> = vec![ - //Rc::new(ffmpeg::FFmpegAdapter::new()), + Rc::new(ffmpeg::FFmpegAdapter::new()), //Rc::new(zip::ZipAdapter::new()), Rc::new(decompress::DecompressAdapter::new()), // Rc::new(tar::TarAdapter::new()), diff --git a/src/adapters/custom.rs b/src/adapters/custom.rs index 71d7c6c..94923e4 100644 --- a/src/adapters/custom.rs +++ b/src/adapters/custom.rs @@ -3,9 +3,12 @@ use super::{ AdapterMeta, GetMetadata, }; use crate::matching::{FastMatcher, SlowMatcher}; +use anyhow::{Context, Result}; use lazy_static::lazy_static; +use regex::{Captures, Regex}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use std::path::Path; // mostly the same as AdapterMeta + SpawningFileAdapter #[derive(Debug, Deserialize, Serialize, JsonSchema, Default, PartialEq, Clone)] @@ -115,17 +118,59 @@ impl GetMetadata for CustomSpawningFileAdapter { &self.meta } } +fn arg_replacer(arg: &str, filepath_hint: &Path) -> Result { + lazy_static::lazy_static! { + static ref ARG_REP: Regex = Regex::new(r"\{([a-z_]+)\}").unwrap(); + } + let mut err = None; + let r = ARG_REP.replace_all(arg, |m: &Captures| -> String { + let idx = m.get(0).unwrap().range(); + if arg.chars().nth(idx.start - 1) == Some('{') { + // skip + return m.get(0).unwrap().as_str().to_string(); + } + if arg.chars().nth(idx.end + 1) == Some('}') { + // skip + return m.get(0).unwrap().as_str().to_string(); + } + let key = m.get(1).unwrap().as_str(); + if key == "file_extension" { + return filepath_hint + .extension() + .map(|e| e.to_string_lossy().to_string()) + .unwrap_or("".to_string()); + } + err = Some(anyhow::anyhow!( + "Unknown arg replacement key '{}' in '{}'", + key, + arg + )); + "".to_string() + //let + }); + if let Some(err) = err { + Err(err) + } else { + Ok(r.to_string()) + } +} impl SpawningFileAdapterTrait for CustomSpawningFileAdapter { fn get_exe(&self) -> &str { &self.binary } fn command( &self, - _filepath_hint: &std::path::Path, + filepath_hint: &std::path::Path, mut command: std::process::Command, - ) -> std::process::Command { - command.args(&self.args); - command + ) -> Result { + command.args( + self.args + .iter() + .map(|arg| arg_replacer(arg, &filepath_hint)) + .collect::>>()?, + ); + log::debug!("running command {:?}", command); + Ok(command) } } impl CustomAdapterConfig { diff --git a/src/adapters/ffmpeg.rs b/src/adapters/ffmpeg.rs index e23751b..e3e09b7 100644 --- a/src/adapters/ffmpeg.rs +++ b/src/adapters/ffmpeg.rs @@ -2,9 +2,11 @@ use super::spawning::map_exe_error; use super::*; use anyhow::*; use lazy_static::lazy_static; +use regex::Regex; use serde::{Deserialize, Serialize}; use std::io::BufReader; use std::process::*; +use writing::{WritingFileAdapter, WritingFileAdapterTrait}; // todo: // maybe todo: read list of extensions from // ffmpeg -demuxers | tail -n+5 | awk '{print $2}' | while read demuxer; do echo MUX=$demuxer; ffmpeg -h demuxer=$demuxer | grep 'Common extensions'; done 2>/dev/null @@ -26,12 +28,12 @@ lazy_static! { }; } -#[derive(Default)] +#[derive(Default, Clone)] pub struct FFmpegAdapter; impl FFmpegAdapter { - pub fn new() -> FFmpegAdapter { - FFmpegAdapter + pub fn new() -> WritingFileAdapter { + WritingFileAdapter::new(Box::new(FFmpegAdapter)) } } impl GetMetadata for FFmpegAdapter { @@ -48,12 +50,16 @@ struct FFprobeOutput { struct FFprobeStream { codec_type: String, // video,audio,subtitle } -impl FileAdapter for FFmpegAdapter { - fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> { +impl WritingFileAdapterTrait for FFmpegAdapter { + fn adapt_write( + &self, + ai: AdaptInfo, + _detection_reason: &SlowMatcher, + oup: &mut dyn Write, + ) -> Result<()> { let AdaptInfo { is_real_file, filepath_hint, - oup, line_prefix, .. } = ai; @@ -80,7 +86,7 @@ impl FileAdapter for FFmpegAdapter { "stream=codec_type", ]) .arg("-i") - .arg(inp_fname) + .arg(&inp_fname) .output() .map_err(spawn_fail)?; if !probe.status.success() { @@ -107,7 +113,7 @@ impl FileAdapter for FFmpegAdapter { //"-count_packets", ]) .arg("-i") - .arg(inp_fname) + .arg(&inp_fname) .stdout(Stdio::piped()) .spawn()?; for line in BufReader::new(probe.stdout.as_mut().unwrap()).lines() { @@ -125,7 +131,7 @@ impl FileAdapter for FFmpegAdapter { .arg("-loglevel") .arg("panic") .arg("-i") - .arg(inp_fname) + .arg(&inp_fname) .arg("-f") .arg("webvtt") .arg("-"); diff --git a/src/adapters/fns.rs b/src/adapters/fns.rs index 4f6e9e8..47cf93c 100644 --- a/src/adapters/fns.rs +++ b/src/adapters/fns.rs @@ -34,6 +34,7 @@ where inner: R, next_read: Vec, replacer: Box Vec>, + haystacker: Box Option>, } impl ByteReplacer @@ -77,7 +78,7 @@ where match read { Ok(u) => { - match memchr::memchr2(b'\n', b'\x0c', &buf[0..u]) { + match (self.haystacker)(&buf[0..u]) { Some(i) => { let data = (self.replacer)(buf[i]); @@ -98,6 +99,7 @@ pub fn postprocB(_line_prefix: &str, inp: impl Read) -> Result { Ok(ByteReplacer { inner: inp, next_read: Vec::new(), + haystacker: Box::new(|buf| memchr::memchr2(b'\n', b'\x0c', buf)), replacer: Box::new(move |b| match b { b'\n' => format!("\nPage {}:", page_count).into_bytes(), b'\x0c' => { diff --git a/src/adapters/spawning.rs b/src/adapters/spawning.rs index 0281963..618a6d2 100644 --- a/src/adapters/spawning.rs +++ b/src/adapters/spawning.rs @@ -2,6 +2,7 @@ use super::*; use anyhow::*; use encoding_rs_io::DecodeReaderBytesBuilder; use log::*; +use regex::Regex; use std::io::prelude::*; use std::io::BufReader; use std::process::Command; @@ -55,7 +56,7 @@ pub fn postproc_line_prefix( } pub trait SpawningFileAdapterTrait: GetMetadata { fn get_exe(&self) -> &str; - fn command(&self, filepath_hint: &Path, command: Command) -> Command; + fn command(&self, filepath_hint: &Path, command: Command) -> Result; /*fn postproc(&self, line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Result<()> { postproc_line_prefix(line_prefix, inp, oup) @@ -146,7 +147,10 @@ impl FileAdapter for SpawningFileAdapter { } = ai; let cmd = Command::new(self.inner.get_exe()); - let cmd = self.inner.command(&filepath_hint, cmd); + let cmd = self + .inner + .command(&filepath_hint, cmd) + .with_context(|| format!("Could not set cmd arguments for {}", self.inner.get_exe()))?; debug!("executing {:?}", cmd); pipe_output(&line_prefix, cmd, &mut inp, self.inner.get_exe(), "") } diff --git a/src/adapters/writing.rs b/src/adapters/writing.rs index d0c95da..a9e9ba1 100644 --- a/src/adapters/writing.rs +++ b/src/adapters/writing.rs @@ -2,6 +2,7 @@ use super::{FileAdapter, GetMetadata, ReadBox}; use anyhow::Result; use std::io::Write; +// this trait / struct split is necessary because of "conflicting trait implementation" otherwise with SpawningFileAdapter #[dyn_clonable::clonable] pub trait WritingFileAdapterTrait: GetMetadata + Send + Clone { fn adapt_write( diff --git a/src/bin/rga-preproc.rs b/src/bin/rga-preproc.rs index 53a4ace..4638da1 100644 --- a/src/bin/rga-preproc.rs +++ b/src/bin/rga-preproc.rs @@ -16,7 +16,7 @@ fn main() -> anyhow::Result<()> { std::env::current_dir()?.join(&filepath) }; - let i = File::open(&path)?; + let i = File::open(&path).context("Specified input file not found")?; let mut o = std::io::stdout(); let cache = if args.no_cache { None @@ -31,7 +31,7 @@ fn main() -> anyhow::Result<()> { archive_recursion_depth: 0, config: PreprocConfig { cache, args }, }; - let mut oup = rga_preproc(ai)?; + let mut oup = rga_preproc(ai).context("during preprocessing")?; std::io::copy(&mut oup, &mut o).context("copying adapter output to stdout")?; Ok(()) }