mirror of
https://github.com/FliegendeWurst/ripgrep-all.git
synced 2024-11-24 12:24:56 +00:00
fix cache key
This commit is contained in:
parent
5447c4ac3a
commit
5901cdcb5d
@ -1,6 +1,7 @@
|
|||||||
# 0.9.1 (2019-06-16)
|
# 0.9.1 (2019-06-16)
|
||||||
|
|
||||||
- Add enabled adapters to cache key if caching for archive
|
- Add enabled adapters to cache key if caching for archive
|
||||||
|
- Prevent empty trailing page output in pdf reader
|
||||||
|
|
||||||
# 0.9.0 (2019-06-16)
|
# 0.9.0 (2019-06-16)
|
||||||
|
|
||||||
|
@ -26,7 +26,9 @@ pub struct AdapterMeta {
|
|||||||
/// version identifier. used to key cache entries, change if your output format changes
|
/// version identifier. used to key cache entries, change if your output format changes
|
||||||
pub version: i32,
|
pub version: i32,
|
||||||
pub description: String,
|
pub description: String,
|
||||||
/// list of matchers (interpreted as ORed)
|
/// indicates whether this adapter can descend (=call rga_preproc again). if true, the cache key needs to include the list of active adapters
|
||||||
|
pub recurses: bool,
|
||||||
|
/// list of matchers (interpreted as a OR b OR ...)
|
||||||
pub fast_matchers: Vec<FastMatcher>,
|
pub fast_matchers: Vec<FastMatcher>,
|
||||||
/// list of matchers when we have mime type detection active (interpreted as ORed)
|
/// list of matchers when we have mime type detection active (interpreted as ORed)
|
||||||
/// warning: this *overrides* the fast matchers
|
/// warning: this *overrides* the fast matchers
|
||||||
@ -71,7 +73,6 @@ pub struct AdaptInfo<'a> {
|
|||||||
pub oup: &'a mut (dyn Write + Send),
|
pub oup: &'a mut (dyn Write + Send),
|
||||||
/// prefix every output line with this string to better indicate the file's location if it is in some archive
|
/// prefix every output line with this string to better indicate the file's location if it is in some archive
|
||||||
pub line_prefix: &'a str,
|
pub line_prefix: &'a str,
|
||||||
// pub adapt_subobject: &'a dyn Fn(AdaptInfo) -> Fallible<()>,
|
|
||||||
pub config: PreprocConfig<'a>,
|
pub config: PreprocConfig<'a>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -19,6 +19,7 @@ lazy_static! {
|
|||||||
description:
|
description:
|
||||||
"Reads compressed file as a stream and runs a different extractor on the contents."
|
"Reads compressed file as a stream and runs a different extractor on the contents."
|
||||||
.to_owned(),
|
.to_owned(),
|
||||||
|
recurses: true,
|
||||||
fast_matchers: EXTENSIONS
|
fast_matchers: EXTENSIONS
|
||||||
.iter()
|
.iter()
|
||||||
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
||||||
|
@ -16,6 +16,7 @@ lazy_static! {
|
|||||||
name: "ffmpeg".to_owned(),
|
name: "ffmpeg".to_owned(),
|
||||||
version: 1,
|
version: 1,
|
||||||
description: "Uses ffmpeg to extract video metadata/chapters and subtitles".to_owned(),
|
description: "Uses ffmpeg to extract video metadata/chapters and subtitles".to_owned(),
|
||||||
|
recurses: false,
|
||||||
fast_matchers: EXTENSIONS
|
fast_matchers: EXTENSIONS
|
||||||
.iter()
|
.iter()
|
||||||
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
||||||
@ -47,7 +48,7 @@ struct FFprobeStream {
|
|||||||
codec_type: String, // video,audio,subtitle
|
codec_type: String, // video,audio,subtitle
|
||||||
}
|
}
|
||||||
impl FileAdapter for FFmpegAdapter {
|
impl FileAdapter for FFmpegAdapter {
|
||||||
fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()> {
|
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||||
let AdaptInfo {
|
let AdaptInfo {
|
||||||
is_real_file,
|
is_real_file,
|
||||||
filepath_hint,
|
filepath_hint,
|
||||||
|
@ -48,6 +48,7 @@ lazy_static! {
|
|||||||
description:
|
description:
|
||||||
"Uses pandoc to convert binary/unreadable text documents to plain markdown-like text"
|
"Uses pandoc to convert binary/unreadable text documents to plain markdown-like text"
|
||||||
.to_owned(),
|
.to_owned(),
|
||||||
|
recurses: false,
|
||||||
fast_matchers: EXTENSIONS
|
fast_matchers: EXTENSIONS
|
||||||
.iter()
|
.iter()
|
||||||
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
||||||
|
@ -17,6 +17,7 @@ lazy_static! {
|
|||||||
name: "pdfpages".to_owned(),
|
name: "pdfpages".to_owned(),
|
||||||
version: 1,
|
version: 1,
|
||||||
description: "Converts a pdf to it's individual pages as png files. Only useful in combination with tesseract".to_owned(),
|
description: "Converts a pdf to it's individual pages as png files. Only useful in combination with tesseract".to_owned(),
|
||||||
|
recurses: true,
|
||||||
fast_matchers: EXTENSIONS
|
fast_matchers: EXTENSIONS
|
||||||
.iter()
|
.iter()
|
||||||
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
||||||
@ -42,7 +43,7 @@ impl GetMetadata for PdfPagesAdapter {
|
|||||||
/// A pdf is basically converted to a zip that has Page X.png files.
|
/// A pdf is basically converted to a zip that has Page X.png files.
|
||||||
/// This way, something like tesseract can process the pages individually
|
/// This way, something like tesseract can process the pages individually
|
||||||
impl FileAdapter for PdfPagesAdapter {
|
impl FileAdapter for PdfPagesAdapter {
|
||||||
fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()> {
|
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||||
let AdaptInfo {
|
let AdaptInfo {
|
||||||
filepath_hint,
|
filepath_hint,
|
||||||
is_real_file,
|
is_real_file,
|
||||||
|
@ -12,6 +12,7 @@ lazy_static! {
|
|||||||
version: 1,
|
version: 1,
|
||||||
description: "Uses pdftotext (from poppler-utils) to extract plain text from PDF files"
|
description: "Uses pdftotext (from poppler-utils) to extract plain text from PDF files"
|
||||||
.to_owned(),
|
.to_owned(),
|
||||||
|
recurses: false,
|
||||||
fast_matchers: EXTENSIONS
|
fast_matchers: EXTENSIONS
|
||||||
.iter()
|
.iter()
|
||||||
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
||||||
@ -50,6 +51,9 @@ impl SpawningFileAdapter for PopplerAdapter {
|
|||||||
// page break
|
// page break
|
||||||
line = line.replace('\x0c', "");
|
line = line.replace('\x0c', "");
|
||||||
page += 1;
|
page += 1;
|
||||||
|
if line.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
oup.write_all(format!("{}Page {}: {}\n", line_prefix, page, line).as_bytes())?;
|
oup.write_all(format!("{}Page {}: {}\n", line_prefix, page, line).as_bytes())?;
|
||||||
}
|
}
|
||||||
|
@ -93,7 +93,7 @@ impl<T> FileAdapter for T
|
|||||||
where
|
where
|
||||||
T: SpawningFileAdapter,
|
T: SpawningFileAdapter,
|
||||||
{
|
{
|
||||||
fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()> {
|
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||||
let AdaptInfo {
|
let AdaptInfo {
|
||||||
filepath_hint,
|
filepath_hint,
|
||||||
mut inp,
|
mut inp,
|
||||||
|
@ -14,6 +14,7 @@ lazy_static! {
|
|||||||
description:
|
description:
|
||||||
"Uses sqlite bindings to convert sqlite databases into a simple plain text format"
|
"Uses sqlite bindings to convert sqlite databases into a simple plain text format"
|
||||||
.to_owned(),
|
.to_owned(),
|
||||||
|
recurses: false, // set to true if we decide to make sqlite blobs searchable (gz blob in db is kinda common I think)
|
||||||
fast_matchers: EXTENSIONS
|
fast_matchers: EXTENSIONS
|
||||||
.iter()
|
.iter()
|
||||||
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
||||||
@ -56,7 +57,7 @@ fn format_blob(b: ValueRef) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl FileAdapter for SqliteAdapter {
|
impl FileAdapter for SqliteAdapter {
|
||||||
fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()> {
|
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||||
let AdaptInfo {
|
let AdaptInfo {
|
||||||
is_real_file,
|
is_real_file,
|
||||||
filepath_hint,
|
filepath_hint,
|
||||||
|
@ -13,6 +13,7 @@ lazy_static! {
|
|||||||
name: "tar".to_owned(),
|
name: "tar".to_owned(),
|
||||||
version: 1,
|
version: 1,
|
||||||
description: "Reads a tar file as a stream and recurses down into its contents".to_owned(),
|
description: "Reads a tar file as a stream and recurses down into its contents".to_owned(),
|
||||||
|
recurses: true,
|
||||||
fast_matchers: EXTENSIONS
|
fast_matchers: EXTENSIONS
|
||||||
.iter()
|
.iter()
|
||||||
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
||||||
@ -35,7 +36,7 @@ impl GetMetadata for TarAdapter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl FileAdapter for TarAdapter {
|
impl FileAdapter for TarAdapter {
|
||||||
fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()> {
|
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||||
let AdaptInfo {
|
let AdaptInfo {
|
||||||
filepath_hint,
|
filepath_hint,
|
||||||
mut inp,
|
mut inp,
|
||||||
|
@ -10,6 +10,7 @@ lazy_static! {
|
|||||||
name: "tesseract".to_owned(),
|
name: "tesseract".to_owned(),
|
||||||
version: 1,
|
version: 1,
|
||||||
description: "Uses tesseract to run OCR on images to make them searchable. May need -j1 to prevent overloading the system. Make sure you have tesseract installed.".to_owned(),
|
description: "Uses tesseract to run OCR on images to make them searchable. May need -j1 to prevent overloading the system. Make sure you have tesseract installed.".to_owned(),
|
||||||
|
recurses: false,
|
||||||
fast_matchers: EXTENSIONS
|
fast_matchers: EXTENSIONS
|
||||||
.iter()
|
.iter()
|
||||||
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
||||||
|
@ -14,6 +14,7 @@ lazy_static! {
|
|||||||
name: "zip".to_owned(),
|
name: "zip".to_owned(),
|
||||||
version: 1,
|
version: 1,
|
||||||
description: "Reads a zip file as a stream and recurses down into its contents".to_owned(),
|
description: "Reads a zip file as a stream and recurses down into its contents".to_owned(),
|
||||||
|
recurses: true,
|
||||||
fast_matchers: EXTENSIONS
|
fast_matchers: EXTENSIONS
|
||||||
.iter()
|
.iter()
|
||||||
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
.map(|s| FastMatcher::FileExtension(s.to_string()))
|
||||||
@ -45,7 +46,7 @@ fn is_dir(f: &ZipFile) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl FileAdapter for ZipAdapter {
|
impl FileAdapter for ZipAdapter {
|
||||||
fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()> {
|
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||||
let AdaptInfo {
|
let AdaptInfo {
|
||||||
filepath_hint,
|
filepath_hint,
|
||||||
mut inp,
|
mut inp,
|
||||||
|
@ -72,13 +72,22 @@ pub fn rga_preproc(ai: AdaptInfo) -> Result<(), Error> {
|
|||||||
let clean_path = filepath_hint.to_owned().clean();
|
let clean_path = filepath_hint.to_owned().clean();
|
||||||
let meta = std::fs::metadata(&filepath_hint)?;
|
let meta = std::fs::metadata(&filepath_hint)?;
|
||||||
|
|
||||||
|
if adapter.metadata().recurses {
|
||||||
|
let key = (
|
||||||
|
clean_path,
|
||||||
|
meta.modified().expect("weird OS that can't into mtime"),
|
||||||
|
&args.adapters[..],
|
||||||
|
);
|
||||||
|
eprintln!("cache key: {:?}", key);
|
||||||
|
bincode::serialize(&key).expect("could not serialize path") // key in the cache database
|
||||||
|
} else {
|
||||||
let key = (
|
let key = (
|
||||||
clean_path,
|
clean_path,
|
||||||
meta.modified().expect("weird OS that can't into mtime"),
|
meta.modified().expect("weird OS that can't into mtime"),
|
||||||
);
|
);
|
||||||
eprintln!("cache key: {:?}", key);
|
eprintln!("cache key: {:?}", key);
|
||||||
|
|
||||||
bincode::serialize(&key).expect("could not serialize path") // key in the cache database
|
bincode::serialize(&key).expect("could not serialize path") // key in the cache database
|
||||||
|
}
|
||||||
};
|
};
|
||||||
cache.write().unwrap().get_or_run(
|
cache.write().unwrap().get_or_run(
|
||||||
&db_name,
|
&db_name,
|
||||||
|
Loading…
Reference in New Issue
Block a user