From dfc10cbb53eadc32e6b14277182c74126a80016a Mon Sep 17 00:00:00 2001 From: phiresky Date: Wed, 30 Sep 2020 16:49:51 +0200 Subject: [PATCH] fix custom adapters --- src/adapters.rs | 39 ++++++++++----------------------------- src/adapters/postproc.rs | 6 ++++-- src/adapters/spawning.rs | 28 ++++++++++++++++++++++++---- src/adapters/zip.rs | 22 +++++++++++----------- src/bin/rga.rs | 14 +++++++------- src/config.rs | 6 +++--- src/lib.rs | 1 + src/preproc.rs | 2 +- src/read_iter.rs | 24 ++++++++++++++++++++++++ src/recurse.rs | 8 ++++---- src/test_utils.rs | 1 + 11 files changed, 90 insertions(+), 61 deletions(-) create mode 100644 src/read_iter.rs diff --git a/src/adapters.rs b/src/adapters.rs index 796965e..3840c51 100644 --- a/src/adapters.rs +++ b/src/adapters.rs @@ -1,18 +1,19 @@ -// pub mod custom; +pub mod custom; // pub mod decompress; // pub mod ffmpeg; pub mod postproc; // pub mod pdfpages; -// pub mod spawning; +pub mod spawning; // pub mod sqlite; // pub mod tar; // pub mod tesseract; // pub mod writing; pub mod zip; -use crate::{config::RgaConfig, matching::*}; +use crate::{config::RgaConfig, matching::*, read_iter::ReadIterBox}; use anyhow::*; // use custom::builtin_spawning_adapters; //use custom::CustomAdapterConfig; +use custom::CustomAdapterConfig; use log::*; use std::borrow::Cow; @@ -23,7 +24,6 @@ use std::path::PathBuf; use std::rc::Rc; pub type ReadBox<'a> = Box; -pub type ReadIterBox<'a> = Box; pub struct AdapterMeta { /// unique short name of this adapter (a-z0-9 only) pub name: String, @@ -84,26 +84,7 @@ pub trait FileAdapter: GetMetadata { &self, a: AdaptInfo<'a>, detection_reason: &FileMatcher, - ) -> Result>; -} - -pub trait ReadIter { - // next takes a 'a-lived reference and returns a Read that lives as long as the reference - fn next<'a>(&'a mut self) -> Option>; -} - -pub struct SingleReadIter<'a> { - ai: Option>, -} -impl SingleReadIter<'_> { - pub fn new<'a>(ai: AdaptInfo<'a>) -> SingleReadIter<'a> { - SingleReadIter { ai: Some(ai) } - } -} -impl ReadIter for SingleReadIter<'_> { - fn next<'a>(&'a mut self) -> Option> { - self.ai.take() - } + ) -> Result>; } pub struct AdaptInfo<'a> { @@ -124,14 +105,14 @@ pub struct AdaptInfo<'a> { /// (enabledAdapters, disabledAdapters) type AdaptersTuple = (Vec>, Vec>); -pub fn get_all_adapters(/*custom_adapters: Option>*/) -> AdaptersTuple { +pub fn get_all_adapters(custom_adapters: Option>) -> AdaptersTuple { // order in descending priority let mut adapters: Vec> = vec![]; - /*if let Some(custom_adapters) = custom_adapters { + if let Some(custom_adapters) = custom_adapters { for adapter_config in custom_adapters { adapters.push(Rc::new(adapter_config.to_adapter())); } - }*/ + } let internal_adapters: Vec> = vec![ //Rc::new(ffmpeg::FFmpegAdapter::new()), @@ -163,10 +144,10 @@ pub fn get_all_adapters(/*custom_adapters: Option>*/) - * - "+a,b" means use default list but also a and b (a,b will be prepended to the list so given higher priority) */ pub fn get_adapters_filtered>( - /*custom_adapters: Option>,*/ + custom_adapters: Option>, adapter_names: &Vec, ) -> Result>> { - let (def_enabled_adapters, def_disabled_adapters) = get_all_adapters(/*custom_adapters*/); + let (def_enabled_adapters, def_disabled_adapters) = get_all_adapters(custom_adapters); let adapters = if !adapter_names.is_empty() { let adapters_map: HashMap<_, _> = def_enabled_adapters .iter() diff --git a/src/adapters/postproc.rs b/src/adapters/postproc.rs index d24ef09..fa2d0ac 100644 --- a/src/adapters/postproc.rs +++ b/src/adapters/postproc.rs @@ -6,7 +6,9 @@ use anyhow::Result; use std::{cmp::min, io::Read}; -use super::{AdaptInfo, AdapterMeta, FileAdapter, GetMetadata, SingleReadIter}; +use crate::read_iter::{ReadIterBox, SingleReadIter}; + +use super::{AdaptInfo, AdapterMeta, FileAdapter, GetMetadata}; struct ByteReplacer where @@ -97,7 +99,7 @@ impl FileAdapter for PostprocPrefix { &self, a: super::AdaptInfo<'a>, _detection_reason: &crate::matching::FileMatcher, - ) -> Result> { + ) -> Result> { let read = postproc_prefix(&a.line_prefix, a.inp)?; // keep adapt info (filename etc) except replace inp let ai = AdaptInfo { diff --git a/src/adapters/spawning.rs b/src/adapters/spawning.rs index 7327c65..cf785ff 100644 --- a/src/adapters/spawning.rs +++ b/src/adapters/spawning.rs @@ -1,12 +1,14 @@ +use crate::read_iter::SingleReadIter; + use super::*; use anyhow::*; use encoding_rs_io::DecodeReaderBytesBuilder; use log::*; -use std::io::prelude::*; use std::io::BufReader; use std::process::Command; use std::process::{Child, Stdio}; +use std::{io::prelude::*, path::Path}; /** * Copy a Read to a Write, while prefixing every line with a prefix. @@ -54,6 +56,8 @@ pub fn postproc_line_prefix( } Ok(()) } + +// TODO: don't separate the trait and the struct pub trait SpawningFileAdapterTrait: GetMetadata { fn get_exe(&self) -> &str; fn command(&self, filepath_hint: &Path, command: Command) -> Result; @@ -138,12 +142,19 @@ pub fn pipe_output<'a>( } impl FileAdapter for SpawningFileAdapter { - fn adapt<'a>(&self, ai: AdaptInfo<'a>, _detection_reason: &FileMatcher) -> Result> { + fn adapt<'a>( + &self, + ai: AdaptInfo<'a>, + _detection_reason: &FileMatcher, + ) -> Result> { let AdaptInfo { filepath_hint, mut inp, line_prefix, - .. + archive_recursion_depth, + postprocess, + config, + is_real_file, } = ai; let cmd = Command::new(self.inner.get_exe()); @@ -152,6 +163,15 @@ impl FileAdapter for SpawningFileAdapter { .command(&filepath_hint, cmd) .with_context(|| format!("Could not set cmd arguments for {}", self.inner.get_exe()))?; debug!("executing {:?}", cmd); - pipe_output(&line_prefix, cmd, &mut inp, self.inner.get_exe(), "") + let output = pipe_output(&line_prefix, cmd, &mut inp, self.inner.get_exe(), "")?; + Ok(Box::new(SingleReadIter::new(AdaptInfo { + filepath_hint, + inp: output, + line_prefix, + is_real_file: false, + archive_recursion_depth, + postprocess, + config, + }))) } } diff --git a/src/adapters/zip.rs b/src/adapters/zip.rs index 67c1fab..2c1398b 100644 --- a/src/adapters/zip.rs +++ b/src/adapters/zip.rs @@ -1,5 +1,5 @@ use super::*; -use crate::print_bytes; +use crate::{print_bytes, read_iter::ReadIter}; use anyhow::*; use lazy_static::lazy_static; use log::*; @@ -35,6 +35,16 @@ impl GetMetadata for ZipAdapter { } } +impl FileAdapter for ZipAdapter { + fn adapt<'a>( + &self, + inp: AdaptInfo<'a>, + _detection_reason: &FileMatcher, + ) -> Result> { + Ok(Box::new(ZipAdaptIter { inp })) + } +} + struct ZipAdaptIter<'a> { inp: AdaptInfo<'a>, } @@ -72,16 +82,6 @@ impl<'a> ReadIter for ZipAdaptIter<'a> { } } -impl FileAdapter for ZipAdapter { - fn adapt<'a>( - &self, - inp: AdaptInfo<'a>, - _detection_reason: &FileMatcher, - ) -> Result> { - Ok(Box::new(ZipAdaptIter { inp })) - } -} - #[cfg(test)] mod test { use super::*; diff --git a/src/bin/rga.rs b/src/bin/rga.rs index 48a550a..b994287 100644 --- a/src/bin/rga.rs +++ b/src/bin/rga.rs @@ -67,16 +67,16 @@ fn main() -> anyhow::Result<()> { env_logger::init(); - let (args, mut passthrough_args) = split_args(false)?; + let (config, mut passthrough_args) = split_args(false)?; - if args.print_config_schema { + if config.print_config_schema { println!("{}", serde_json::to_string_pretty(&schema_for!(RgaConfig))?); return Ok(()); } - if args.list_adapters { - return list_adapters(args); + if config.list_adapters { + return list_adapters(config); } - if let Some(path) = args.fzf_path { + if let Some(path) = config.fzf_path { if path == "_" { // fzf found no result, ignore everything and return println!("[no file found]"); @@ -92,9 +92,9 @@ fn main() -> anyhow::Result<()> { return Ok(()); } - let adapters = get_adapters_filtered(args.custom_adapters.clone(), &args.adapters)?; + let adapters = get_adapters_filtered(config.custom_adapters.clone(), &config.adapters)?; - let pre_glob = if !args.accurate { + let pre_glob = if !config.accurate { let extensions = adapters .iter() .flat_map(|a| &a.metadata().fast_matchers) diff --git a/src/config.rs b/src/config.rs index b734912..db5ff85 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,4 @@ -use crate::project_dirs; +use crate::{adapters::custom::CustomAdapterConfig, project_dirs}; use anyhow::*; use derive_more::FromStr; use log::*; @@ -151,12 +151,12 @@ pub struct RgaConfig { )] pub max_archive_recursion: MaxArchiveRecursion, - /* ////////////////////////////////////////// + ////////////////////////////////////////// //////////////////////////// Config file only ////////////////////////////////////////// #[serde(default, skip_serializing_if = "is_default")] #[structopt(skip)] - pub custom_adapters: Option>,*/ + pub custom_adapters: Option>, ////////////////////////////////////////// //////////////////////////// CMD line only ////////////////////////////////////////// diff --git a/src/lib.rs b/src/lib.rs index 3db6c1d..6956804 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,7 @@ pub mod matching; pub mod pipe; pub mod preproc; pub mod preproc_cache; +pub mod read_iter; pub mod recurse; #[cfg(test)] pub mod test_utils; diff --git a/src/preproc.rs b/src/preproc.rs index 9f511de..5d3122c 100644 --- a/src/preproc.rs +++ b/src/preproc.rs @@ -31,7 +31,7 @@ pub fn rga_preproc(ai: AdaptInfo) -> Result { } = ai; debug!("path (hint) to preprocess: {:?}", filepath_hint); let filtered_adapters = - get_adapters_filtered(/*config.custom_adapters.clone(),*/ &config.adapters)?; + get_adapters_filtered(config.custom_adapters.clone(), &config.adapters)?; let adapters = adapter_matcher(&filtered_adapters, config.accurate)?; let filename = filepath_hint .file_name() diff --git a/src/read_iter.rs b/src/read_iter.rs new file mode 100644 index 0000000..35e4e08 --- /dev/null +++ b/src/read_iter.rs @@ -0,0 +1,24 @@ +use crate::adapters::AdaptInfo; + +// TODO: using iterator trait possible?? should basically be Iterator +pub trait ReadIter { + // next takes a 'a-lived reference and returns an AdaptInfo that lives as long as the reference + fn next<'a>(&'a mut self) -> Option>; +} + +/// A single AdaptInfo +pub struct SingleReadIter<'a> { + ai: Option>, +} +impl SingleReadIter<'_> { + pub fn new<'a>(ai: AdaptInfo<'a>) -> SingleReadIter<'a> { + SingleReadIter { ai: Some(ai) } + } +} +impl ReadIter for SingleReadIter<'_> { + fn next<'a>(&'a mut self) -> Option> { + self.ai.take() + } +} + +pub type ReadIterBox<'a> = Box; diff --git a/src/recurse.rs b/src/recurse.rs index f449625..1c94fba 100644 --- a/src/recurse.rs +++ b/src/recurse.rs @@ -1,15 +1,15 @@ -use crate::adapters::*; use crate::preproc::rga_preproc; +use crate::{adapters::*, read_iter::ReadIterBox}; use anyhow::*; use std::io::Read; pub struct RecursingConcattyReader<'a> { - inp: Box, + inp: ReadIterBox<'a>, cur: Option>, } impl<'a> RecursingConcattyReader<'a> { - pub fn concat(inp: Box) -> Result> { + pub fn concat(inp: ReadIterBox<'a>) -> Result> { let mut r = RecursingConcattyReader { inp, cur: None }; r.ascend()?; Ok(Box::new(r)) @@ -20,7 +20,7 @@ impl<'a> RecursingConcattyReader<'a> { // we only need to access the inp: ReadIter when the inner reader is done, so this should be safe let ai = unsafe { // would love to make this safe, but how? something like OwnedRef - (*(inp as *mut Box)).next() + (*(inp as *mut ReadIterBox<'a>)).next() }; self.cur = match ai { Some(ai) => Some(rga_preproc(ai)?), diff --git a/src/test_utils.rs b/src/test_utils.rs index b686693..5bbaebc 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -20,6 +20,7 @@ pub fn simple_adapt_info<'a>(filepath: &Path, inp: ReadBox<'a>) -> (AdaptInfo<'a inp, line_prefix: "PREFIX:".to_string(), config: RgaConfig::default(), + postprocess: true, }, FastFileMatcher::FileExtension( filepath.extension().unwrap().to_string_lossy().into_owned(),