diff --git a/src/adapters/spawning.rs b/src/adapters/spawning.rs index 657d9f8..171ea21 100644 --- a/src/adapters/spawning.rs +++ b/src/adapters/spawning.rs @@ -15,6 +15,8 @@ pub fn postproc_line_prefix( inp: &mut dyn Read, oup: &mut dyn Write, ) -> Fallible<()> { + //std::io::copy(inp, oup)?; + //return Ok(()); let mut reader = BufReader::with_capacity(1 << 12, inp); let fourk = reader.fill_buf()?; if fourk.contains(&0u8) { diff --git a/src/adapters/zip.rs b/src/adapters/zip.rs index 4fae076..57f8911 100644 --- a/src/adapters/zip.rs +++ b/src/adapters/zip.rs @@ -4,6 +4,7 @@ use ::zip::read::ZipFile; use failure::*; use lazy_static::lazy_static; + // todo: // maybe todo: read list of extensions from //ffmpeg -demuxers | tail -n+5 | awk '{print $2}' | while read demuxer; do echo MUX=$demuxer; ffmpeg -h demuxer=$demuxer | grep 'Common extensions'; done 2>/dev/null diff --git a/src/bin/rga-preproc.rs b/src/bin/rga-preproc.rs index 526fed1..d61477d 100644 --- a/src/bin/rga-preproc.rs +++ b/src/bin/rga-preproc.rs @@ -3,6 +3,7 @@ use rga::adapters::*; use rga::preproc::*; use std::env; use std::fs::File; +use std::io::{BufReader}; fn main() -> Result<(), Error> { let path = { @@ -14,11 +15,13 @@ fn main() -> Result<(), Error> { std::env::current_dir()?.join(&filepath) }; + let i = File::open(&path)?; + let mut o = std::io::stdout(); let ai = AdaptInfo { - inp: &mut File::open(&path)?, + inp: &mut BufReader::new(i), filepath_hint: &path, is_real_file: true, - oup: &mut std::io::stdout(), + oup: &mut o, line_prefix: "", }; diff --git a/src/preproc.rs b/src/preproc.rs index 5e45b69..ca52cc8 100644 --- a/src/preproc.rs +++ b/src/preproc.rs @@ -2,6 +2,7 @@ use crate::adapters::*; use crate::CachingWriter; use failure::{format_err, Error}; use path_clean::PathClean; +use std::io::BufWriter; // longest compressed conversion output to save in cache const MAX_DB_BLOB_LEN: usize = 2_000_000; @@ -102,7 +103,9 @@ pub fn rga_preproc<'a>( Some(_) => Err(format_err!("Integrity: value not blob")), None => { drop(reader); - let mut compbuf = CachingWriter::new(oup, MAX_DB_BLOB_LEN, ZSTD_LEVEL)?; + // wrapping BufWriter here gives ~10% perf boost + let mut compbuf = + BufWriter::new(CachingWriter::new(oup, MAX_DB_BLOB_LEN, ZSTD_LEVEL)?); eprintln!("adapting..."); ad.adapt(AdaptInfo { line_prefix, @@ -111,7 +114,11 @@ pub fn rga_preproc<'a>( inp, oup: &mut compbuf, })?; - let compressed = compbuf.finish()?; + let compressed = compbuf + .into_inner() + .map_err(|_| "could not finish zstd") + .unwrap() + .finish()?; if let Some(cached) = compressed { eprintln!("compressed len: {}", cached.len());