track recursion depth

Cargo.toml

@@ -1,5 +1,3 @@
-cargo-features = ["default-run"]
-
 
 [package]
 name = "rga"
@@ -9,7 +7,6 @@ version = "0.3.0"
 repository = "https://github.com/phiresky/rga"
 authors = ["phiresky <phireskyde+git@gmail.com>"]
 edition = "2018"
-default-run = "rga"
 exclude = [
     "exampledir/*"
 ]

src/adapters.rs

@@ -45,6 +45,8 @@ pub struct AdaptInfo<'a> {
     pub filepath_hint: &'a Path,
     /// true if filepath_hint is an actual file on the file system
     pub is_real_file: bool,
+    /// depth at which this file is in archives. 0 for real filesystem
+    pub archive_recursion_depth: i32,
     /// stream to read the file from. can be from a file or from some decoder
     pub inp: &'a mut dyn Read,
     /// stream to write to. will be written to from a different thread

src/adapters/ffmpeg.rs

@@ -49,10 +49,12 @@ impl FileAdapter for FFmpegAdapter {
             is_real_file,
             filepath_hint,
             oup,
+            line_prefix,
             ..
         } = ai;
         if !is_real_file {
-            eprintln!("Skipping video in archive");
+            // we *could* probably adapt this to also work based on streams, but really when would you want to search for videos within archives?
+            writeln!(oup, "{}[rga: skipping video in archive]", line_prefix,)?;
             return Ok(());
         }
         let inp_fname = filepath_hint;

src/adapters/spawning.rs

@@ -20,14 +20,14 @@ pub fn postproc_line_prefix(
     let mut reader = BufReader::with_capacity(1 << 12, inp);
     let fourk = reader.fill_buf()?;
     if fourk.contains(&0u8) {
-        oup.write_all(format!("{}[binary data]\n", line_prefix).as_bytes())?;
+        writeln!(oup, "{}[rga: binary data]\n", line_prefix)?;
         return Ok(());
     }
     // intentionally do not call reader.consume
     for line in reader.split(b'\n') {
         let line = line?;
         if line.contains(&0u8) {
-            oup.write_all(format!("{}[binary data]\n", line_prefix).as_bytes())?;
+            writeln!(oup, "{}[rga: binary data]\n", line_prefix)?;
             return Ok(());
         }
         oup.write_all(line_prefix.as_bytes())?;

src/adapters/tar.rs

@@ -57,6 +57,7 @@ impl FileAdapter for TarAdapter {
             mut inp,
             oup,
             line_prefix,
+            archive_recursion_depth,
             ..
         } = ai;
 
@@ -76,6 +77,7 @@ impl FileAdapter for TarAdapter {
                 let ai2: AdaptInfo = AdaptInfo {
                     filepath_hint: &path,
                     is_real_file: false,
+                    archive_recursion_depth: archive_recursion_depth + 1,
                     inp: &mut file,
                     oup,
                     line_prefix,

src/adapters/zip.rs

@@ -4,7 +4,6 @@ use ::zip::read::ZipFile;
 use failure::*;
 use lazy_static::lazy_static;
 
-
 // todo:
 // maybe todo: read list of extensions from
 //ffmpeg -demuxers | tail -n+5 | awk '{print $2}' | while read demuxer; do echo MUX=$demuxer; ffmpeg -h demuxer=$demuxer | grep 'Common extensions'; done 2>/dev/null
@@ -50,6 +49,7 @@ impl FileAdapter for ZipAdapter {
             mut inp,
             oup,
             line_prefix,
+            archive_recursion_depth,
             ..
         } = ai;
         loop {
@@ -74,6 +74,7 @@ impl FileAdapter for ZipAdapter {
                             inp: &mut file,
                             oup,
                             line_prefix,
+                            archive_recursion_depth,
                         },
                         None,
                     )?;

src/preproc.rs

@@ -18,8 +18,8 @@ pub fn open_cache_db() -> Result<std::sync::Arc<std::sync::RwLock<rkv::Rkv>>, Er
         .get_or_create(app_cache.as_path(), |p| {
             let mut builder = rkv::Rkv::environment_builder();
             builder
-                .set_flags(rkv::EnvironmentFlags::NO_SYNC | rkv::EnvironmentFlags::WRITE_MAP) // not durable
+                .set_flags(rkv::EnvironmentFlags::NO_SYNC | rkv::EnvironmentFlags::WRITE_MAP) // not durable cuz it's a cache
-                // i'm not sure why this is needed. otherwise LMDB transactions (open readers) will keep piling up until it fails with
+                // i'm not sure why NO_TLS is needed. otherwise LMDB transactions (open readers) will keep piling up until it fails with
                 // LmdbError(ReadersFull)
                 // hope it doesn't break integrity
                 .set_flags(rkv::EnvironmentFlags::NO_TLS)
@@ -113,6 +113,7 @@ pub fn rga_preproc<'a>(
                             is_real_file,
                             inp,
                             oup: &mut compbuf,
+                            archive_recursion_depth: 0,
                         })?;
                         let compressed = compbuf
                             .into_inner()
@@ -144,6 +145,7 @@ pub fn rga_preproc<'a>(
                     is_real_file,
                     inp,
                     oup,
+                    archive_recursion_depth: 0,
                 })?;
                 Ok(())
             }