fix custom adapters

This commit is contained in:
phiresky 2020-09-30 16:49:51 +02:00
parent ab5ddcad2e
commit dfc10cbb53
11 changed files with 90 additions and 61 deletions

View File

@ -1,18 +1,19 @@
// pub mod custom; pub mod custom;
// pub mod decompress; // pub mod decompress;
// pub mod ffmpeg; // pub mod ffmpeg;
pub mod postproc; pub mod postproc;
// pub mod pdfpages; // pub mod pdfpages;
// pub mod spawning; pub mod spawning;
// pub mod sqlite; // pub mod sqlite;
// pub mod tar; // pub mod tar;
// pub mod tesseract; // pub mod tesseract;
// pub mod writing; // pub mod writing;
pub mod zip; pub mod zip;
use crate::{config::RgaConfig, matching::*}; use crate::{config::RgaConfig, matching::*, read_iter::ReadIterBox};
use anyhow::*; use anyhow::*;
// use custom::builtin_spawning_adapters; // use custom::builtin_spawning_adapters;
//use custom::CustomAdapterConfig; //use custom::CustomAdapterConfig;
use custom::CustomAdapterConfig;
use log::*; use log::*;
use std::borrow::Cow; use std::borrow::Cow;
@ -23,7 +24,6 @@ use std::path::PathBuf;
use std::rc::Rc; use std::rc::Rc;
pub type ReadBox<'a> = Box<dyn Read + 'a>; pub type ReadBox<'a> = Box<dyn Read + 'a>;
pub type ReadIterBox<'a> = Box<dyn ReadIter + 'a>;
pub struct AdapterMeta { pub struct AdapterMeta {
/// unique short name of this adapter (a-z0-9 only) /// unique short name of this adapter (a-z0-9 only)
pub name: String, pub name: String,
@ -84,26 +84,7 @@ pub trait FileAdapter: GetMetadata {
&self, &self,
a: AdaptInfo<'a>, a: AdaptInfo<'a>,
detection_reason: &FileMatcher, detection_reason: &FileMatcher,
) -> Result<Box<dyn ReadIter + 'a>>; ) -> Result<ReadIterBox<'a>>;
}
pub trait ReadIter {
// next takes a 'a-lived reference and returns a Read that lives as long as the reference
fn next<'a>(&'a mut self) -> Option<AdaptInfo<'a>>;
}
pub struct SingleReadIter<'a> {
ai: Option<AdaptInfo<'a>>,
}
impl SingleReadIter<'_> {
pub fn new<'a>(ai: AdaptInfo<'a>) -> SingleReadIter<'a> {
SingleReadIter { ai: Some(ai) }
}
}
impl ReadIter for SingleReadIter<'_> {
fn next<'a>(&'a mut self) -> Option<AdaptInfo<'a>> {
self.ai.take()
}
} }
pub struct AdaptInfo<'a> { pub struct AdaptInfo<'a> {
@ -124,14 +105,14 @@ pub struct AdaptInfo<'a> {
/// (enabledAdapters, disabledAdapters) /// (enabledAdapters, disabledAdapters)
type AdaptersTuple = (Vec<Rc<dyn FileAdapter>>, Vec<Rc<dyn FileAdapter>>); type AdaptersTuple = (Vec<Rc<dyn FileAdapter>>, Vec<Rc<dyn FileAdapter>>);
pub fn get_all_adapters(/*custom_adapters: Option<Vec<CustomAdapterConfig>>*/) -> AdaptersTuple { pub fn get_all_adapters(custom_adapters: Option<Vec<CustomAdapterConfig>>) -> AdaptersTuple {
// order in descending priority // order in descending priority
let mut adapters: Vec<Rc<dyn FileAdapter>> = vec![]; let mut adapters: Vec<Rc<dyn FileAdapter>> = vec![];
/*if let Some(custom_adapters) = custom_adapters { if let Some(custom_adapters) = custom_adapters {
for adapter_config in custom_adapters { for adapter_config in custom_adapters {
adapters.push(Rc::new(adapter_config.to_adapter())); adapters.push(Rc::new(adapter_config.to_adapter()));
} }
}*/ }
let internal_adapters: Vec<Rc<dyn FileAdapter>> = vec![ let internal_adapters: Vec<Rc<dyn FileAdapter>> = vec![
//Rc::new(ffmpeg::FFmpegAdapter::new()), //Rc::new(ffmpeg::FFmpegAdapter::new()),
@ -163,10 +144,10 @@ pub fn get_all_adapters(/*custom_adapters: Option<Vec<CustomAdapterConfig>>*/) -
* - "+a,b" means use default list but also a and b (a,b will be prepended to the list so given higher priority) * - "+a,b" means use default list but also a and b (a,b will be prepended to the list so given higher priority)
*/ */
pub fn get_adapters_filtered<T: AsRef<str>>( pub fn get_adapters_filtered<T: AsRef<str>>(
/*custom_adapters: Option<Vec<CustomAdapterConfig>>,*/ custom_adapters: Option<Vec<CustomAdapterConfig>>,
adapter_names: &Vec<T>, adapter_names: &Vec<T>,
) -> Result<Vec<Rc<dyn FileAdapter>>> { ) -> Result<Vec<Rc<dyn FileAdapter>>> {
let (def_enabled_adapters, def_disabled_adapters) = get_all_adapters(/*custom_adapters*/); let (def_enabled_adapters, def_disabled_adapters) = get_all_adapters(custom_adapters);
let adapters = if !adapter_names.is_empty() { let adapters = if !adapter_names.is_empty() {
let adapters_map: HashMap<_, _> = def_enabled_adapters let adapters_map: HashMap<_, _> = def_enabled_adapters
.iter() .iter()

View File

@ -6,7 +6,9 @@ use anyhow::Result;
use std::{cmp::min, io::Read}; use std::{cmp::min, io::Read};
use super::{AdaptInfo, AdapterMeta, FileAdapter, GetMetadata, SingleReadIter}; use crate::read_iter::{ReadIterBox, SingleReadIter};
use super::{AdaptInfo, AdapterMeta, FileAdapter, GetMetadata};
struct ByteReplacer<R> struct ByteReplacer<R>
where where
@ -97,7 +99,7 @@ impl FileAdapter for PostprocPrefix {
&self, &self,
a: super::AdaptInfo<'a>, a: super::AdaptInfo<'a>,
_detection_reason: &crate::matching::FileMatcher, _detection_reason: &crate::matching::FileMatcher,
) -> Result<Box<dyn super::ReadIter + 'a>> { ) -> Result<ReadIterBox<'a>> {
let read = postproc_prefix(&a.line_prefix, a.inp)?; let read = postproc_prefix(&a.line_prefix, a.inp)?;
// keep adapt info (filename etc) except replace inp // keep adapt info (filename etc) except replace inp
let ai = AdaptInfo { let ai = AdaptInfo {

View File

@ -1,12 +1,14 @@
use crate::read_iter::SingleReadIter;
use super::*; use super::*;
use anyhow::*; use anyhow::*;
use encoding_rs_io::DecodeReaderBytesBuilder; use encoding_rs_io::DecodeReaderBytesBuilder;
use log::*; use log::*;
use std::io::prelude::*;
use std::io::BufReader; use std::io::BufReader;
use std::process::Command; use std::process::Command;
use std::process::{Child, Stdio}; use std::process::{Child, Stdio};
use std::{io::prelude::*, path::Path};
/** /**
* Copy a Read to a Write, while prefixing every line with a prefix. * Copy a Read to a Write, while prefixing every line with a prefix.
@ -54,6 +56,8 @@ pub fn postproc_line_prefix(
} }
Ok(()) Ok(())
} }
// TODO: don't separate the trait and the struct
pub trait SpawningFileAdapterTrait: GetMetadata { pub trait SpawningFileAdapterTrait: GetMetadata {
fn get_exe(&self) -> &str; fn get_exe(&self) -> &str;
fn command(&self, filepath_hint: &Path, command: Command) -> Result<Command>; fn command(&self, filepath_hint: &Path, command: Command) -> Result<Command>;
@ -138,12 +142,19 @@ pub fn pipe_output<'a>(
} }
impl FileAdapter for SpawningFileAdapter { impl FileAdapter for SpawningFileAdapter {
fn adapt<'a>(&self, ai: AdaptInfo<'a>, _detection_reason: &FileMatcher) -> Result<ReadBox<'a>> { fn adapt<'a>(
&self,
ai: AdaptInfo<'a>,
_detection_reason: &FileMatcher,
) -> Result<ReadIterBox<'a>> {
let AdaptInfo { let AdaptInfo {
filepath_hint, filepath_hint,
mut inp, mut inp,
line_prefix, line_prefix,
.. archive_recursion_depth,
postprocess,
config,
is_real_file,
} = ai; } = ai;
let cmd = Command::new(self.inner.get_exe()); let cmd = Command::new(self.inner.get_exe());
@ -152,6 +163,15 @@ impl FileAdapter for SpawningFileAdapter {
.command(&filepath_hint, cmd) .command(&filepath_hint, cmd)
.with_context(|| format!("Could not set cmd arguments for {}", self.inner.get_exe()))?; .with_context(|| format!("Could not set cmd arguments for {}", self.inner.get_exe()))?;
debug!("executing {:?}", cmd); debug!("executing {:?}", cmd);
pipe_output(&line_prefix, cmd, &mut inp, self.inner.get_exe(), "") let output = pipe_output(&line_prefix, cmd, &mut inp, self.inner.get_exe(), "")?;
Ok(Box::new(SingleReadIter::new(AdaptInfo {
filepath_hint,
inp: output,
line_prefix,
is_real_file: false,
archive_recursion_depth,
postprocess,
config,
})))
} }
} }

View File

@ -1,5 +1,5 @@
use super::*; use super::*;
use crate::print_bytes; use crate::{print_bytes, read_iter::ReadIter};
use anyhow::*; use anyhow::*;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use log::*; use log::*;
@ -35,6 +35,16 @@ impl GetMetadata for ZipAdapter {
} }
} }
impl FileAdapter for ZipAdapter {
fn adapt<'a>(
&self,
inp: AdaptInfo<'a>,
_detection_reason: &FileMatcher,
) -> Result<Box<dyn ReadIter + 'a>> {
Ok(Box::new(ZipAdaptIter { inp }))
}
}
struct ZipAdaptIter<'a> { struct ZipAdaptIter<'a> {
inp: AdaptInfo<'a>, inp: AdaptInfo<'a>,
} }
@ -72,16 +82,6 @@ impl<'a> ReadIter for ZipAdaptIter<'a> {
} }
} }
impl FileAdapter for ZipAdapter {
fn adapt<'a>(
&self,
inp: AdaptInfo<'a>,
_detection_reason: &FileMatcher,
) -> Result<Box<dyn ReadIter + 'a>> {
Ok(Box::new(ZipAdaptIter { inp }))
}
}
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::*;

View File

@ -67,16 +67,16 @@ fn main() -> anyhow::Result<()> {
env_logger::init(); env_logger::init();
let (args, mut passthrough_args) = split_args(false)?; let (config, mut passthrough_args) = split_args(false)?;
if args.print_config_schema { if config.print_config_schema {
println!("{}", serde_json::to_string_pretty(&schema_for!(RgaConfig))?); println!("{}", serde_json::to_string_pretty(&schema_for!(RgaConfig))?);
return Ok(()); return Ok(());
} }
if args.list_adapters { if config.list_adapters {
return list_adapters(args); return list_adapters(config);
} }
if let Some(path) = args.fzf_path { if let Some(path) = config.fzf_path {
if path == "_" { if path == "_" {
// fzf found no result, ignore everything and return // fzf found no result, ignore everything and return
println!("[no file found]"); println!("[no file found]");
@ -92,9 +92,9 @@ fn main() -> anyhow::Result<()> {
return Ok(()); return Ok(());
} }
let adapters = get_adapters_filtered(args.custom_adapters.clone(), &args.adapters)?; let adapters = get_adapters_filtered(config.custom_adapters.clone(), &config.adapters)?;
let pre_glob = if !args.accurate { let pre_glob = if !config.accurate {
let extensions = adapters let extensions = adapters
.iter() .iter()
.flat_map(|a| &a.metadata().fast_matchers) .flat_map(|a| &a.metadata().fast_matchers)

View File

@ -1,4 +1,4 @@
use crate::project_dirs; use crate::{adapters::custom::CustomAdapterConfig, project_dirs};
use anyhow::*; use anyhow::*;
use derive_more::FromStr; use derive_more::FromStr;
use log::*; use log::*;
@ -151,12 +151,12 @@ pub struct RgaConfig {
)] )]
pub max_archive_recursion: MaxArchiveRecursion, pub max_archive_recursion: MaxArchiveRecursion,
/* ////////////////////////////////////////// //////////////////////////////////////////
//////////////////////////// Config file only //////////////////////////// Config file only
////////////////////////////////////////// //////////////////////////////////////////
#[serde(default, skip_serializing_if = "is_default")] #[serde(default, skip_serializing_if = "is_default")]
#[structopt(skip)] #[structopt(skip)]
pub custom_adapters: Option<Vec<CustomAdapterConfig>>,*/ pub custom_adapters: Option<Vec<CustomAdapterConfig>>,
////////////////////////////////////////// //////////////////////////////////////////
//////////////////////////// CMD line only //////////////////////////// CMD line only
////////////////////////////////////////// //////////////////////////////////////////

View File

@ -9,6 +9,7 @@ pub mod matching;
pub mod pipe; pub mod pipe;
pub mod preproc; pub mod preproc;
pub mod preproc_cache; pub mod preproc_cache;
pub mod read_iter;
pub mod recurse; pub mod recurse;
#[cfg(test)] #[cfg(test)]
pub mod test_utils; pub mod test_utils;

View File

@ -31,7 +31,7 @@ pub fn rga_preproc(ai: AdaptInfo) -> Result<ReadBox> {
} = ai; } = ai;
debug!("path (hint) to preprocess: {:?}", filepath_hint); debug!("path (hint) to preprocess: {:?}", filepath_hint);
let filtered_adapters = let filtered_adapters =
get_adapters_filtered(/*config.custom_adapters.clone(),*/ &config.adapters)?; get_adapters_filtered(config.custom_adapters.clone(), &config.adapters)?;
let adapters = adapter_matcher(&filtered_adapters, config.accurate)?; let adapters = adapter_matcher(&filtered_adapters, config.accurate)?;
let filename = filepath_hint let filename = filepath_hint
.file_name() .file_name()

24
src/read_iter.rs Normal file
View File

@ -0,0 +1,24 @@
use crate::adapters::AdaptInfo;
// TODO: using iterator trait possible?? should basically be Iterator<AdaptInfo>
pub trait ReadIter {
// next takes a 'a-lived reference and returns an AdaptInfo that lives as long as the reference
fn next<'a>(&'a mut self) -> Option<AdaptInfo<'a>>;
}
/// A single AdaptInfo
pub struct SingleReadIter<'a> {
ai: Option<AdaptInfo<'a>>,
}
impl SingleReadIter<'_> {
pub fn new<'a>(ai: AdaptInfo<'a>) -> SingleReadIter<'a> {
SingleReadIter { ai: Some(ai) }
}
}
impl ReadIter for SingleReadIter<'_> {
fn next<'a>(&'a mut self) -> Option<AdaptInfo<'a>> {
self.ai.take()
}
}
pub type ReadIterBox<'a> = Box<dyn ReadIter + 'a>;

View File

@ -1,15 +1,15 @@
use crate::adapters::*;
use crate::preproc::rga_preproc; use crate::preproc::rga_preproc;
use crate::{adapters::*, read_iter::ReadIterBox};
use anyhow::*; use anyhow::*;
use std::io::Read; use std::io::Read;
pub struct RecursingConcattyReader<'a> { pub struct RecursingConcattyReader<'a> {
inp: Box<dyn ReadIter + 'a>, inp: ReadIterBox<'a>,
cur: Option<ReadBox<'a>>, cur: Option<ReadBox<'a>>,
} }
impl<'a> RecursingConcattyReader<'a> { impl<'a> RecursingConcattyReader<'a> {
pub fn concat(inp: Box<dyn ReadIter + 'a>) -> Result<Box<dyn Read + 'a>> { pub fn concat(inp: ReadIterBox<'a>) -> Result<Box<dyn Read + 'a>> {
let mut r = RecursingConcattyReader { inp, cur: None }; let mut r = RecursingConcattyReader { inp, cur: None };
r.ascend()?; r.ascend()?;
Ok(Box::new(r)) Ok(Box::new(r))
@ -20,7 +20,7 @@ impl<'a> RecursingConcattyReader<'a> {
// we only need to access the inp: ReadIter when the inner reader is done, so this should be safe // we only need to access the inp: ReadIter when the inner reader is done, so this should be safe
let ai = unsafe { let ai = unsafe {
// would love to make this safe, but how? something like OwnedRef<inp, cur> // would love to make this safe, but how? something like OwnedRef<inp, cur>
(*(inp as *mut Box<dyn ReadIter + 'a>)).next() (*(inp as *mut ReadIterBox<'a>)).next()
}; };
self.cur = match ai { self.cur = match ai {
Some(ai) => Some(rga_preproc(ai)?), Some(ai) => Some(rga_preproc(ai)?),

View File

@ -20,6 +20,7 @@ pub fn simple_adapt_info<'a>(filepath: &Path, inp: ReadBox<'a>) -> (AdaptInfo<'a
inp, inp,
line_prefix: "PREFIX:".to_string(), line_prefix: "PREFIX:".to_string(),
config: RgaConfig::default(), config: RgaConfig::default(),
postprocess: true,
}, },
FastFileMatcher::FileExtension( FastFileMatcher::FileExtension(
filepath.extension().unwrap().to_string_lossy().into_owned(), filepath.extension().unwrap().to_string_lossy().into_owned(),