fix custom adapters

This commit is contained in:
phiresky 2020-09-30 16:49:51 +02:00
parent ab5ddcad2e
commit dfc10cbb53
11 changed files with 90 additions and 61 deletions

View File

@ -1,18 +1,19 @@
// pub mod custom;
pub mod custom;
// pub mod decompress;
// pub mod ffmpeg;
pub mod postproc;
// pub mod pdfpages;
// pub mod spawning;
pub mod spawning;
// pub mod sqlite;
// pub mod tar;
// pub mod tesseract;
// pub mod writing;
pub mod zip;
use crate::{config::RgaConfig, matching::*};
use crate::{config::RgaConfig, matching::*, read_iter::ReadIterBox};
use anyhow::*;
// use custom::builtin_spawning_adapters;
//use custom::CustomAdapterConfig;
use custom::CustomAdapterConfig;
use log::*;
use std::borrow::Cow;
@ -23,7 +24,6 @@ use std::path::PathBuf;
use std::rc::Rc;
pub type ReadBox<'a> = Box<dyn Read + 'a>;
pub type ReadIterBox<'a> = Box<dyn ReadIter + 'a>;
pub struct AdapterMeta {
/// unique short name of this adapter (a-z0-9 only)
pub name: String,
@ -84,26 +84,7 @@ pub trait FileAdapter: GetMetadata {
&self,
a: AdaptInfo<'a>,
detection_reason: &FileMatcher,
) -> Result<Box<dyn ReadIter + 'a>>;
}
pub trait ReadIter {
// next takes a 'a-lived reference and returns a Read that lives as long as the reference
fn next<'a>(&'a mut self) -> Option<AdaptInfo<'a>>;
}
pub struct SingleReadIter<'a> {
ai: Option<AdaptInfo<'a>>,
}
impl SingleReadIter<'_> {
pub fn new<'a>(ai: AdaptInfo<'a>) -> SingleReadIter<'a> {
SingleReadIter { ai: Some(ai) }
}
}
impl ReadIter for SingleReadIter<'_> {
fn next<'a>(&'a mut self) -> Option<AdaptInfo<'a>> {
self.ai.take()
}
) -> Result<ReadIterBox<'a>>;
}
pub struct AdaptInfo<'a> {
@ -124,14 +105,14 @@ pub struct AdaptInfo<'a> {
/// (enabledAdapters, disabledAdapters)
type AdaptersTuple = (Vec<Rc<dyn FileAdapter>>, Vec<Rc<dyn FileAdapter>>);
pub fn get_all_adapters(/*custom_adapters: Option<Vec<CustomAdapterConfig>>*/) -> AdaptersTuple {
pub fn get_all_adapters(custom_adapters: Option<Vec<CustomAdapterConfig>>) -> AdaptersTuple {
// order in descending priority
let mut adapters: Vec<Rc<dyn FileAdapter>> = vec![];
/*if let Some(custom_adapters) = custom_adapters {
if let Some(custom_adapters) = custom_adapters {
for adapter_config in custom_adapters {
adapters.push(Rc::new(adapter_config.to_adapter()));
}
}*/
}
let internal_adapters: Vec<Rc<dyn FileAdapter>> = vec![
//Rc::new(ffmpeg::FFmpegAdapter::new()),
@ -163,10 +144,10 @@ pub fn get_all_adapters(/*custom_adapters: Option<Vec<CustomAdapterConfig>>*/) -
* - "+a,b" means use default list but also a and b (a,b will be prepended to the list so given higher priority)
*/
pub fn get_adapters_filtered<T: AsRef<str>>(
/*custom_adapters: Option<Vec<CustomAdapterConfig>>,*/
custom_adapters: Option<Vec<CustomAdapterConfig>>,
adapter_names: &Vec<T>,
) -> Result<Vec<Rc<dyn FileAdapter>>> {
let (def_enabled_adapters, def_disabled_adapters) = get_all_adapters(/*custom_adapters*/);
let (def_enabled_adapters, def_disabled_adapters) = get_all_adapters(custom_adapters);
let adapters = if !adapter_names.is_empty() {
let adapters_map: HashMap<_, _> = def_enabled_adapters
.iter()

View File

@ -6,7 +6,9 @@ use anyhow::Result;
use std::{cmp::min, io::Read};
use super::{AdaptInfo, AdapterMeta, FileAdapter, GetMetadata, SingleReadIter};
use crate::read_iter::{ReadIterBox, SingleReadIter};
use super::{AdaptInfo, AdapterMeta, FileAdapter, GetMetadata};
struct ByteReplacer<R>
where
@ -97,7 +99,7 @@ impl FileAdapter for PostprocPrefix {
&self,
a: super::AdaptInfo<'a>,
_detection_reason: &crate::matching::FileMatcher,
) -> Result<Box<dyn super::ReadIter + 'a>> {
) -> Result<ReadIterBox<'a>> {
let read = postproc_prefix(&a.line_prefix, a.inp)?;
// keep adapt info (filename etc) except replace inp
let ai = AdaptInfo {

View File

@ -1,12 +1,14 @@
use crate::read_iter::SingleReadIter;
use super::*;
use anyhow::*;
use encoding_rs_io::DecodeReaderBytesBuilder;
use log::*;
use std::io::prelude::*;
use std::io::BufReader;
use std::process::Command;
use std::process::{Child, Stdio};
use std::{io::prelude::*, path::Path};
/**
* Copy a Read to a Write, while prefixing every line with a prefix.
@ -54,6 +56,8 @@ pub fn postproc_line_prefix(
}
Ok(())
}
// TODO: don't separate the trait and the struct
pub trait SpawningFileAdapterTrait: GetMetadata {
fn get_exe(&self) -> &str;
fn command(&self, filepath_hint: &Path, command: Command) -> Result<Command>;
@ -138,12 +142,19 @@ pub fn pipe_output<'a>(
}
impl FileAdapter for SpawningFileAdapter {
fn adapt<'a>(&self, ai: AdaptInfo<'a>, _detection_reason: &FileMatcher) -> Result<ReadBox<'a>> {
fn adapt<'a>(
&self,
ai: AdaptInfo<'a>,
_detection_reason: &FileMatcher,
) -> Result<ReadIterBox<'a>> {
let AdaptInfo {
filepath_hint,
mut inp,
line_prefix,
..
archive_recursion_depth,
postprocess,
config,
is_real_file,
} = ai;
let cmd = Command::new(self.inner.get_exe());
@ -152,6 +163,15 @@ impl FileAdapter for SpawningFileAdapter {
.command(&filepath_hint, cmd)
.with_context(|| format!("Could not set cmd arguments for {}", self.inner.get_exe()))?;
debug!("executing {:?}", cmd);
pipe_output(&line_prefix, cmd, &mut inp, self.inner.get_exe(), "")
let output = pipe_output(&line_prefix, cmd, &mut inp, self.inner.get_exe(), "")?;
Ok(Box::new(SingleReadIter::new(AdaptInfo {
filepath_hint,
inp: output,
line_prefix,
is_real_file: false,
archive_recursion_depth,
postprocess,
config,
})))
}
}

View File

@ -1,5 +1,5 @@
use super::*;
use crate::print_bytes;
use crate::{print_bytes, read_iter::ReadIter};
use anyhow::*;
use lazy_static::lazy_static;
use log::*;
@ -35,6 +35,16 @@ impl GetMetadata for ZipAdapter {
}
}
impl FileAdapter for ZipAdapter {
fn adapt<'a>(
&self,
inp: AdaptInfo<'a>,
_detection_reason: &FileMatcher,
) -> Result<Box<dyn ReadIter + 'a>> {
Ok(Box::new(ZipAdaptIter { inp }))
}
}
struct ZipAdaptIter<'a> {
inp: AdaptInfo<'a>,
}
@ -72,16 +82,6 @@ impl<'a> ReadIter for ZipAdaptIter<'a> {
}
}
impl FileAdapter for ZipAdapter {
fn adapt<'a>(
&self,
inp: AdaptInfo<'a>,
_detection_reason: &FileMatcher,
) -> Result<Box<dyn ReadIter + 'a>> {
Ok(Box::new(ZipAdaptIter { inp }))
}
}
#[cfg(test)]
mod test {
use super::*;

View File

@ -67,16 +67,16 @@ fn main() -> anyhow::Result<()> {
env_logger::init();
let (args, mut passthrough_args) = split_args(false)?;
let (config, mut passthrough_args) = split_args(false)?;
if args.print_config_schema {
if config.print_config_schema {
println!("{}", serde_json::to_string_pretty(&schema_for!(RgaConfig))?);
return Ok(());
}
if args.list_adapters {
return list_adapters(args);
if config.list_adapters {
return list_adapters(config);
}
if let Some(path) = args.fzf_path {
if let Some(path) = config.fzf_path {
if path == "_" {
// fzf found no result, ignore everything and return
println!("[no file found]");
@ -92,9 +92,9 @@ fn main() -> anyhow::Result<()> {
return Ok(());
}
let adapters = get_adapters_filtered(args.custom_adapters.clone(), &args.adapters)?;
let adapters = get_adapters_filtered(config.custom_adapters.clone(), &config.adapters)?;
let pre_glob = if !args.accurate {
let pre_glob = if !config.accurate {
let extensions = adapters
.iter()
.flat_map(|a| &a.metadata().fast_matchers)

View File

@ -1,4 +1,4 @@
use crate::project_dirs;
use crate::{adapters::custom::CustomAdapterConfig, project_dirs};
use anyhow::*;
use derive_more::FromStr;
use log::*;
@ -151,12 +151,12 @@ pub struct RgaConfig {
)]
pub max_archive_recursion: MaxArchiveRecursion,
/* //////////////////////////////////////////
//////////////////////////////////////////
//////////////////////////// Config file only
//////////////////////////////////////////
#[serde(default, skip_serializing_if = "is_default")]
#[structopt(skip)]
pub custom_adapters: Option<Vec<CustomAdapterConfig>>,*/
pub custom_adapters: Option<Vec<CustomAdapterConfig>>,
//////////////////////////////////////////
//////////////////////////// CMD line only
//////////////////////////////////////////

View File

@ -9,6 +9,7 @@ pub mod matching;
pub mod pipe;
pub mod preproc;
pub mod preproc_cache;
pub mod read_iter;
pub mod recurse;
#[cfg(test)]
pub mod test_utils;

View File

@ -31,7 +31,7 @@ pub fn rga_preproc(ai: AdaptInfo) -> Result<ReadBox> {
} = ai;
debug!("path (hint) to preprocess: {:?}", filepath_hint);
let filtered_adapters =
get_adapters_filtered(/*config.custom_adapters.clone(),*/ &config.adapters)?;
get_adapters_filtered(config.custom_adapters.clone(), &config.adapters)?;
let adapters = adapter_matcher(&filtered_adapters, config.accurate)?;
let filename = filepath_hint
.file_name()

24
src/read_iter.rs Normal file
View File

@ -0,0 +1,24 @@
use crate::adapters::AdaptInfo;
// TODO: using iterator trait possible?? should basically be Iterator<AdaptInfo>
pub trait ReadIter {
// next takes a 'a-lived reference and returns an AdaptInfo that lives as long as the reference
fn next<'a>(&'a mut self) -> Option<AdaptInfo<'a>>;
}
/// A single AdaptInfo
pub struct SingleReadIter<'a> {
ai: Option<AdaptInfo<'a>>,
}
impl SingleReadIter<'_> {
pub fn new<'a>(ai: AdaptInfo<'a>) -> SingleReadIter<'a> {
SingleReadIter { ai: Some(ai) }
}
}
impl ReadIter for SingleReadIter<'_> {
fn next<'a>(&'a mut self) -> Option<AdaptInfo<'a>> {
self.ai.take()
}
}
pub type ReadIterBox<'a> = Box<dyn ReadIter + 'a>;

View File

@ -1,15 +1,15 @@
use crate::adapters::*;
use crate::preproc::rga_preproc;
use crate::{adapters::*, read_iter::ReadIterBox};
use anyhow::*;
use std::io::Read;
pub struct RecursingConcattyReader<'a> {
inp: Box<dyn ReadIter + 'a>,
inp: ReadIterBox<'a>,
cur: Option<ReadBox<'a>>,
}
impl<'a> RecursingConcattyReader<'a> {
pub fn concat(inp: Box<dyn ReadIter + 'a>) -> Result<Box<dyn Read + 'a>> {
pub fn concat(inp: ReadIterBox<'a>) -> Result<Box<dyn Read + 'a>> {
let mut r = RecursingConcattyReader { inp, cur: None };
r.ascend()?;
Ok(Box::new(r))
@ -20,7 +20,7 @@ impl<'a> RecursingConcattyReader<'a> {
// we only need to access the inp: ReadIter when the inner reader is done, so this should be safe
let ai = unsafe {
// would love to make this safe, but how? something like OwnedRef<inp, cur>
(*(inp as *mut Box<dyn ReadIter + 'a>)).next()
(*(inp as *mut ReadIterBox<'a>)).next()
};
self.cur = match ai {
Some(ai) => Some(rga_preproc(ai)?),

View File

@ -20,6 +20,7 @@ pub fn simple_adapt_info<'a>(filepath: &Path, inp: ReadBox<'a>) -> (AdaptInfo<'a
inp,
line_prefix: "PREFIX:".to_string(),
config: RgaConfig::default(),
postprocess: true,
},
FastFileMatcher::FileExtension(
filepath.extension().unwrap().to_string_lossy().into_owned(),