performance

src/adapters/spawning.rs

@@ -15,6 +15,8 @@ pub fn postproc_line_prefix(
     inp: &mut dyn Read,
     oup: &mut dyn Write,
 ) -> Fallible<()> {
+    //std::io::copy(inp, oup)?;
+    //return Ok(());
     let mut reader = BufReader::with_capacity(1 << 12, inp);
     let fourk = reader.fill_buf()?;
     if fourk.contains(&0u8) {

src/adapters/zip.rs

@@ -4,6 +4,7 @@ use ::zip::read::ZipFile;
 use failure::*;
 use lazy_static::lazy_static;
 
+
 // todo:
 // maybe todo: read list of extensions from
 //ffmpeg -demuxers | tail -n+5 | awk '{print $2}' | while read demuxer; do echo MUX=$demuxer; ffmpeg -h demuxer=$demuxer | grep 'Common extensions'; done 2>/dev/null

src/bin/rga-preproc.rs

@@ -3,6 +3,7 @@ use rga::adapters::*;
 use rga::preproc::*;
 use std::env;
 use std::fs::File;
+use std::io::{BufReader};
 
 fn main() -> Result<(), Error> {
     let path = {
@@ -14,11 +15,13 @@ fn main() -> Result<(), Error> {
         std::env::current_dir()?.join(&filepath)
     };
 
+    let i = File::open(&path)?;
+    let mut o = std::io::stdout();
     let ai = AdaptInfo {
-        inp: &mut File::open(&path)?,
+        inp: &mut BufReader::new(i),
         filepath_hint: &path,
         is_real_file: true,
-        oup: &mut std::io::stdout(),
+        oup: &mut o,
         line_prefix: "",
     };
 

src/preproc.rs

@@ -2,6 +2,7 @@ use crate::adapters::*;
 use crate::CachingWriter;
 use failure::{format_err, Error};
 use path_clean::PathClean;
+use std::io::BufWriter;
 
 // longest compressed conversion output to save in cache
 const MAX_DB_BLOB_LEN: usize = 2_000_000;
@@ -102,7 +103,9 @@ pub fn rga_preproc<'a>(
                     Some(_) => Err(format_err!("Integrity: value not blob")),
                     None => {
                         drop(reader);
-                        let mut compbuf = CachingWriter::new(oup, MAX_DB_BLOB_LEN, ZSTD_LEVEL)?;
+                        // wrapping BufWriter here gives ~10% perf boost
+                        let mut compbuf =
+                            BufWriter::new(CachingWriter::new(oup, MAX_DB_BLOB_LEN, ZSTD_LEVEL)?);
                         eprintln!("adapting...");
                         ad.adapt(AdaptInfo {
                             line_prefix,
@@ -111,7 +114,11 @@ pub fn rga_preproc<'a>(
                             inp,
                             oup: &mut compbuf,
                         })?;
-                        let compressed = compbuf.finish()?;
+                        let compressed = compbuf
+                            .into_inner()
+                            .map_err(|_| "could not finish zstd")
+                            .unwrap()
+                            .finish()?;
                         if let Some(cached) = compressed {
                             eprintln!("compressed len: {}", cached.len());