diff --git a/Cargo.toml b/Cargo.toml index d848c45..8764bfa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,3 @@ -cargo-features = ["default-run"] - [package] name = "rga" @@ -9,7 +7,6 @@ version = "0.3.0" repository = "https://github.com/phiresky/rga" authors = ["phiresky "] edition = "2018" -default-run = "rga" exclude = [ "exampledir/*" ] diff --git a/src/adapters.rs b/src/adapters.rs index 7cc94e0..f0e76dd 100644 --- a/src/adapters.rs +++ b/src/adapters.rs @@ -45,6 +45,8 @@ pub struct AdaptInfo<'a> { pub filepath_hint: &'a Path, /// true if filepath_hint is an actual file on the file system pub is_real_file: bool, + /// depth at which this file is in archives. 0 for real filesystem + pub archive_recursion_depth: i32, /// stream to read the file from. can be from a file or from some decoder pub inp: &'a mut dyn Read, /// stream to write to. will be written to from a different thread diff --git a/src/adapters/ffmpeg.rs b/src/adapters/ffmpeg.rs index 8cf40d3..553cafb 100644 --- a/src/adapters/ffmpeg.rs +++ b/src/adapters/ffmpeg.rs @@ -49,10 +49,12 @@ impl FileAdapter for FFmpegAdapter { is_real_file, filepath_hint, oup, + line_prefix, .. } = ai; if !is_real_file { - eprintln!("Skipping video in archive"); + // we *could* probably adapt this to also work based on streams, but really when would you want to search for videos within archives? + writeln!(oup, "{}[rga: skipping video in archive]", line_prefix,)?; return Ok(()); } let inp_fname = filepath_hint; diff --git a/src/adapters/spawning.rs b/src/adapters/spawning.rs index 171ea21..4cacb5a 100644 --- a/src/adapters/spawning.rs +++ b/src/adapters/spawning.rs @@ -20,14 +20,14 @@ pub fn postproc_line_prefix( let mut reader = BufReader::with_capacity(1 << 12, inp); let fourk = reader.fill_buf()?; if fourk.contains(&0u8) { - oup.write_all(format!("{}[binary data]\n", line_prefix).as_bytes())?; + writeln!(oup, "{}[rga: binary data]\n", line_prefix)?; return Ok(()); } // intentionally do not call reader.consume for line in reader.split(b'\n') { let line = line?; if line.contains(&0u8) { - oup.write_all(format!("{}[binary data]\n", line_prefix).as_bytes())?; + writeln!(oup, "{}[rga: binary data]\n", line_prefix)?; return Ok(()); } oup.write_all(line_prefix.as_bytes())?; diff --git a/src/adapters/tar.rs b/src/adapters/tar.rs index a31260c..08938e5 100644 --- a/src/adapters/tar.rs +++ b/src/adapters/tar.rs @@ -57,6 +57,7 @@ impl FileAdapter for TarAdapter { mut inp, oup, line_prefix, + archive_recursion_depth, .. } = ai; @@ -76,6 +77,7 @@ impl FileAdapter for TarAdapter { let ai2: AdaptInfo = AdaptInfo { filepath_hint: &path, is_real_file: false, + archive_recursion_depth: archive_recursion_depth + 1, inp: &mut file, oup, line_prefix, diff --git a/src/adapters/zip.rs b/src/adapters/zip.rs index 57f8911..634826e 100644 --- a/src/adapters/zip.rs +++ b/src/adapters/zip.rs @@ -4,7 +4,6 @@ use ::zip::read::ZipFile; use failure::*; use lazy_static::lazy_static; - // todo: // maybe todo: read list of extensions from //ffmpeg -demuxers | tail -n+5 | awk '{print $2}' | while read demuxer; do echo MUX=$demuxer; ffmpeg -h demuxer=$demuxer | grep 'Common extensions'; done 2>/dev/null @@ -50,6 +49,7 @@ impl FileAdapter for ZipAdapter { mut inp, oup, line_prefix, + archive_recursion_depth, .. } = ai; loop { @@ -74,6 +74,7 @@ impl FileAdapter for ZipAdapter { inp: &mut file, oup, line_prefix, + archive_recursion_depth, }, None, )?; diff --git a/src/preproc.rs b/src/preproc.rs index ca52cc8..397f662 100644 --- a/src/preproc.rs +++ b/src/preproc.rs @@ -18,8 +18,8 @@ pub fn open_cache_db() -> Result>, Er .get_or_create(app_cache.as_path(), |p| { let mut builder = rkv::Rkv::environment_builder(); builder - .set_flags(rkv::EnvironmentFlags::NO_SYNC | rkv::EnvironmentFlags::WRITE_MAP) // not durable - // i'm not sure why this is needed. otherwise LMDB transactions (open readers) will keep piling up until it fails with + .set_flags(rkv::EnvironmentFlags::NO_SYNC | rkv::EnvironmentFlags::WRITE_MAP) // not durable cuz it's a cache + // i'm not sure why NO_TLS is needed. otherwise LMDB transactions (open readers) will keep piling up until it fails with // LmdbError(ReadersFull) // hope it doesn't break integrity .set_flags(rkv::EnvironmentFlags::NO_TLS) @@ -113,6 +113,7 @@ pub fn rga_preproc<'a>( is_real_file, inp, oup: &mut compbuf, + archive_recursion_depth: 0, })?; let compressed = compbuf .into_inner() @@ -144,6 +145,7 @@ pub fn rga_preproc<'a>( is_real_file, inp, oup, + archive_recursion_depth: 0, })?; Ok(()) }