From f2d9393832503dfe5456e05fa6ec33119f853c7a Mon Sep 17 00:00:00 2001 From: phiresky Date: Sat, 6 Jun 2020 12:57:43 +0200 Subject: [PATCH] switch failure -> anyhow, parse byte suffix in cache-max-blob-len --- Cargo.lock | 160 +++++++++++++++---------------------- Cargo.toml | 21 +++-- README.md | 2 +- src/adapters.rs | 6 +- src/adapters/decompress.rs | 6 +- src/adapters/ffmpeg.rs | 4 +- src/adapters/pdfpages.rs | 4 +- src/adapters/poppler.rs | 2 +- src/adapters/spawning.rs | 14 ++-- src/adapters/sqlite.rs | 4 +- src/adapters/tar.rs | 4 +- src/adapters/zip.rs | 4 +- src/args.rs | 47 ++++++++--- src/bin/rga-preproc.rs | 2 +- src/bin/rga.rs | 6 +- src/caching_writer.rs | 8 +- src/matching.rs | 4 +- src/preproc.rs | 8 +- src/preproc_cache.rs | 56 ++++++++----- 19 files changed, 189 insertions(+), 173 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 15fc7d6..37864e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,14 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "addr2line" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456d75cbb82da1ad150c8a9d97285ffcd21c9931dcb11e995903e7d75141b38b" -dependencies = [ - "gimli", -] - [[package]] name = "adler32" version = "1.0.4" @@ -33,6 +24,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "anyhow" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" + [[package]] name = "arrayref" version = "0.3.6" @@ -62,19 +59,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" -[[package]] -name = "backtrace" -version = "0.3.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0df2f85c8a2abbe3b7d7e748052fdd9b76a0458fdeb16ad4223f5eca78c7c130" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "object", - "rustc-demangle", -] - [[package]] name = "bincode" version = "1.2.1" @@ -109,12 +93,13 @@ dependencies = [ [[package]] name = "bzip2-sys" -version = "0.1.8+1.0.8" +version = "0.1.9+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05305b41c5034ff0e93937ac64133d109b5a2660114ec45e9760bc6816d83038" +checksum = "ad3b39a260062fca31f7b0b12f207e8f2590a67d32ec7d59c20484b07ea7285e" dependencies = [ "cc", "libc", + "pkg-config", ] [[package]] @@ -125,9 +110,9 @@ checksum = "c06509d1f4ffa658939bd23f076cd929ef218241363796551528e7eec69128c8" [[package]] name = "cc" -version = "1.0.53" +version = "1.0.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "404b1fe4f65288577753b17e3b36a04596ee784493ec249bf81c7f2d2acd751c" +checksum = "7bbb73db36c1246e9034e307d0fba23f9a2e251faa47ade70c1bd252220c8311" dependencies = [ "jobserver", ] @@ -235,9 +220,9 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" +checksum = "ab6bffe714b6bb07e42f201352c34f51fefd355ace793f9e638ebd52d23f98d2" dependencies = [ "cfg-if", "crossbeam-utils", @@ -254,6 +239,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "either" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" + [[package]] name = "encoding_rs" version = "0.8.23" @@ -285,22 +276,12 @@ dependencies = [ "termcolor", ] -[[package]] -name = "exitfailure" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ff5bd832af37f366c6c194d813a11cd90ac484f124f079294f28e357ae40515" -dependencies = [ - "failure", -] - [[package]] name = "failure" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" dependencies = [ - "backtrace", "failure_derive", ] @@ -390,12 +371,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "gimli" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc8e0c9bce37868955864dbecd2b1ab2bdf967e6f28066d65aaac620444b65c" - [[package]] name = "glob" version = "0.3.0" @@ -440,6 +415,15 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "itertools" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.5" @@ -463,9 +447,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.70" +version = "0.2.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3baa92041a6fec78c687fa0cc2b3fae8884f743d672cf551bed1d6dac6988d0f" +checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" [[package]] name = "libsqlite3-sys" @@ -651,12 +635,6 @@ dependencies = [ "autocfg 1.0.0", ] -[[package]] -name = "object" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cbca9424c482ee628fa549d9c812e2cd22f1180b9222c9200fdfa6eb31aecb2" - [[package]] name = "ordered-float" version = "1.0.2" @@ -701,9 +679,9 @@ dependencies = [ [[package]] name = "paste" -version = "0.1.12" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a229b1c58c692edcaa5b9b0948084f130f55d2dcc15b02fcc5340b2b4521476" +checksum = "d508492eeb1e5c38ee696371bf7b9fc33c83d46a7d451606b96458fbbbdc2dec" dependencies = [ "paste-impl", "proc-macro-hack", @@ -711,9 +689,9 @@ dependencies = [ [[package]] name = "paste-impl" -version = "0.1.12" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e0bf239e447e67ff6d16a8bb5e4d4bd2343acf5066061c0e8e06ac5ba8ca68c" +checksum = "84f328a6a63192b333fce5fbb4be79db6758a4d518dfac6d54412f1492f72d32" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -751,9 +729,9 @@ checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" [[package]] name = "podio" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780fb4b6698bbf9cf2444ea5d22411cef2953f0824b98f33cf454ec5615645bd" +checksum = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19" [[package]] name = "ppv-lite86" @@ -789,15 +767,15 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63" +checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" [[package]] name = "proc-macro2" -version = "1.0.13" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f5ffe53a6b28e37c9c1ce74893477864d64f74778a93a4beb43c8fa167f639" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" dependencies = [ "unicode-xid", ] @@ -981,9 +959,9 @@ checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" [[package]] name = "regex" -version = "1.3.7" +version = "1.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" dependencies = [ "aho-corasick", "memchr", @@ -993,9 +971,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" [[package]] name = "remove_dir_all" @@ -1010,6 +988,7 @@ dependencies = [ name = "ripgrep_all" version = "0.9.7-alpha.0" dependencies = [ + "anyhow", "bincode", "bzip2", "cachedir", @@ -1019,8 +998,6 @@ dependencies = [ "encoding_rs", "encoding_rs_io", "env_logger", - "exitfailure", - "failure", "flate2", "glob", "lazy_static", @@ -1079,12 +1056,6 @@ dependencies = [ "time", ] -[[package]] -name = "rustc-demangle" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" - [[package]] name = "rustc_version" version = "0.2.3" @@ -1096,9 +1067,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" [[package]] name = "scopeguard" @@ -1123,18 +1094,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c" +checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984" +checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250" dependencies = [ "proc-macro2", "quote", @@ -1209,9 +1180,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.22" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1425de3c33b0941002740a420b1a906a350b88d08b82b2c8a01035a3f9447bac" +checksum = "93a56fabc59dce20fe48b6c832cc249c713e7ed88fa28b0ee0a3bfcaae5fe4e2" dependencies = [ "proc-macro2", "quote", @@ -1243,9 +1214,9 @@ dependencies = [ [[package]] name = "tar" -version = "0.4.26" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" +checksum = "5c058ad0bd6ccb84faa24cc44d4fc99bee8a5d7ba9ff33aa4d993122d1aeeac2" dependencies = [ "filetime", "libc", @@ -1389,9 +1360,9 @@ checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" [[package]] name = "vcpkg" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" +checksum = "55d1e41d56121e07f1e223db0a4def204e45c85425f6a16d462fd07c8d10d74c" [[package]] name = "vec_map" @@ -1401,9 +1372,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" [[package]] name = "wasi" @@ -1475,18 +1446,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.5.1+zstd.1.4.4" +version = "0.5.2+zstd.1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5d978b793ae64375b80baf652919b148f6a496ac8802922d9999f5a553194f" +checksum = "644352b10ce7f333d6e0af85bd4f5322dc449416dc1211c6308e95bca8923db4" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "2.0.3+zstd.1.4.4" +version = "2.0.4+zstd.1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee25eac9753cfedd48133fa1736cbd23b774e253d89badbeac7d12b23848d3f" +checksum = "7113c0c9aed2c55181f2d9f5b0a36e7d2c0183b11c058ab40b35987479efe4d7" dependencies = [ "libc", "zstd-sys", @@ -1494,11 +1465,12 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.4.15+zstd.1.4.4" +version = "1.4.16+zstd.1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89719b034dc22d240d5b407fb0a3fe6d29952c181cff9a9f95c0bd40b4f8f7d8" +checksum = "c442965efc45353be5a9b9969c9b0872fff6828c7e06d118dda2cb2d0bb11d5a" dependencies = [ "cc", "glob", + "itertools", "libc", ] diff --git a/Cargo.toml b/Cargo.toml index 50c9a2e..2fa202d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,32 +17,31 @@ exclude = [ [dependencies] tree_magic = { package = "tree_magic_fork", version = "0.2.2" } -regex = "1.3.6" +regex = "1.3.9" rkv = "0.10.4" cachedir = "0.1.1" path-clean = "0.1.0" bincode = "1.2.1" -serde = { version = "1.0.106", features = ["derive"] } -zstd = "0.5.1" +serde = { version = "1.0.111", features = ["derive"] } +zstd = "0.5.2" lazy_static = "1.4.0" -serde_json = "1.0.51" -failure = "0.1.7" +serde_json = "1.0.53" zip = "0.5.5" crossbeam = "0.7.3" -clap = { version = "2.33.0", features = ["wrap_help"] } +clap = { version = "2.33.1", features = ["wrap_help"] } log = "0.4.8" env_logger = "0.7.1" xz2 = "0.1.6" flate2 = "1.0.14" bzip2 = "0.3.3" -tar = "0.4.26" +tar = "0.4.28" chrono = "0.4.11" -encoding_rs = "0.8.22" +encoding_rs = "0.8.23" encoding_rs_io = "0.1.7" rusqlite = { version = "0.23.1", features = ["vtab", "bundled"] } size_format = "1.0.2" -structopt = "0.3.12" -paste = "0.1.10" +structopt = "0.3.14" +paste = "0.1.16" tempfile = "3.1.0" glob = "0.3.0" -exitfailure = "0.5.1" +anyhow = "1.0.31" diff --git a/README.md b/README.md index 680c499..330d37c 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,7 @@ The following adapters are disabled by default, and can be enabled using '--rga- > \[default: 12\] -**\--rga-cache-max-blob-len** \ +**\--rga-cache-max-blob-len**=\ > Max compressed size to cache > diff --git a/src/adapters.rs b/src/adapters.rs index 70868be..8d5f942 100644 --- a/src/adapters.rs +++ b/src/adapters.rs @@ -10,7 +10,7 @@ pub mod tesseract; pub mod zip; use crate::matching::*; use crate::preproc::PreprocConfig; -use failure::*; +use anyhow::*; use log::*; use regex::Regex; use std::borrow::Cow; @@ -58,7 +58,7 @@ pub trait FileAdapter: GetMetadata { /// adapt a file. /// /// detection_reason is the Matcher that was used to identify this file. Unless --rga-accurate was given, it is always a FastMatcher - fn adapt(&self, a: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()>; + fn adapt(&self, a: AdaptInfo, detection_reason: &SlowMatcher) -> Result<()>; } pub struct AdaptInfo<'a> { /// file path. May not be an actual file on the file system (e.g. in an archive). Used for matching file extensions. @@ -107,7 +107,7 @@ pub fn get_all_adapters() -> AdaptersTuple { */ pub fn get_adapters_filtered>( adapter_names: &[T], -) -> Fallible>> { +) -> Result>> { let (def_enabled_adapters, def_disabled_adapters) = get_all_adapters(); let adapters = if !adapter_names.is_empty() { let adapters_map: HashMap<_, _> = def_enabled_adapters diff --git a/src/adapters/decompress.rs b/src/adapters/decompress.rs index 0b51c91..60fc6ac 100644 --- a/src/adapters/decompress.rs +++ b/src/adapters/decompress.rs @@ -1,6 +1,6 @@ use super::*; use crate::preproc::rga_preproc; -use failure::*; +use anyhow::*; use lazy_static::lazy_static; use std::path::PathBuf; @@ -46,7 +46,7 @@ impl GetMetadata for DecompressAdapter { } } -fn decompress_any<'a, R>(reason: &SlowMatcher, inp: &'a mut R) -> Fallible> +fn decompress_any<'a, R>(reason: &SlowMatcher, inp: &'a mut R) -> Result> where R: Read, { @@ -91,7 +91,7 @@ fn get_inner_filename(filename: &Path) -> PathBuf { } impl FileAdapter for DecompressAdapter { - fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()> { + fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Result<()> { let AdaptInfo { filepath_hint, mut inp, diff --git a/src/adapters/ffmpeg.rs b/src/adapters/ffmpeg.rs index 4c9b94f..f7791ed 100644 --- a/src/adapters/ffmpeg.rs +++ b/src/adapters/ffmpeg.rs @@ -1,6 +1,6 @@ use super::spawning::map_exe_error; use super::*; -use failure::*; +use anyhow::*; use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; use std::io::BufReader; @@ -48,7 +48,7 @@ struct FFprobeStream { codec_type: String, // video,audio,subtitle } impl FileAdapter for FFmpegAdapter { - fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> { + fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> { let AdaptInfo { is_real_file, filepath_hint, diff --git a/src/adapters/pdfpages.rs b/src/adapters/pdfpages.rs index 27a611a..5984dcf 100644 --- a/src/adapters/pdfpages.rs +++ b/src/adapters/pdfpages.rs @@ -43,7 +43,7 @@ impl GetMetadata for PdfPagesAdapter { /// A pdf is basically converted to a zip that has Page X.png files. /// This way, something like tesseract can process the pages individually impl FileAdapter for PdfPagesAdapter { - fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> { + fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> { let AdaptInfo { filepath_hint, is_real_file, @@ -114,7 +114,7 @@ fn split_by_seq<'a>( split_seq: &'a [u8], split_inx: usize, read: &mut Read, -) -> Fallible + 'a> { +) -> Result + 'a> { let regex = split_seq .iter() .map(|c| format!("\\x{:0>2x}", c)) diff --git a/src/adapters/poppler.rs b/src/adapters/poppler.rs index 42df503..5a474c2 100644 --- a/src/adapters/poppler.rs +++ b/src/adapters/poppler.rs @@ -42,7 +42,7 @@ impl SpawningFileAdapter for PopplerAdapter { cmd.arg("-").arg("-"); cmd } - fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Fallible<()> { + fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Result<()> { // prepend Page X to each line let mut page = 1; for line in BufReader::new(inp).lines() { diff --git a/src/adapters/spawning.rs b/src/adapters/spawning.rs index 8c632bd..d111950 100644 --- a/src/adapters/spawning.rs +++ b/src/adapters/spawning.rs @@ -1,6 +1,6 @@ use super::*; +use anyhow::*; use encoding_rs_io::DecodeReaderBytesBuilder; -use failure::*; use log::*; use std::io::prelude::*; use std::io::BufReader; @@ -20,7 +20,7 @@ pub fn postproc_line_prefix( line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write, -) -> Fallible<()> { +) -> Result<()> { // TODO: parse these options from ripgrep's configuration let encoding = None; // detect bom but usually assume utf8 let bom_sniffing = true; @@ -57,7 +57,7 @@ pub trait SpawningFileAdapter: GetMetadata { fn get_exe(&self) -> &str; fn command(&self, filepath_hint: &Path, command: Command) -> Command; - fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Fallible<()> { + fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Result<()> { postproc_line_prefix(line_prefix, inp, oup) } } @@ -78,8 +78,8 @@ pub fn pipe_output( oup: &mut (dyn Write + Send), exe_name: &str, help: &str, - cp: fn(line_prefix: &str, &mut dyn Read, &mut dyn Write) -> Fallible<()>, -) -> Fallible<()> { + cp: fn(line_prefix: &str, &mut dyn Read, &mut dyn Write) -> Result<()>, +) -> Result<()> { let mut cmd = cmd .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -90,7 +90,7 @@ pub fn pipe_output( // TODO: how to handle this copying better? // do we really need threads for this? - crossbeam::scope(|s| -> Fallible<()> { + crossbeam::scope(|s| -> Result<()> { s.spawn(|_| cp(line_prefix, &mut stdo, oup).unwrap()); // errors? std::io::copy(inp, &mut stdi)?; drop(stdi); // NEEDED! otherwise deadlock @@ -109,7 +109,7 @@ impl FileAdapter for T where T: SpawningFileAdapter, { - fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> { + fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> { let AdaptInfo { filepath_hint, mut inp, diff --git a/src/adapters/sqlite.rs b/src/adapters/sqlite.rs index 7a59f33..ed89a8c 100644 --- a/src/adapters/sqlite.rs +++ b/src/adapters/sqlite.rs @@ -1,5 +1,5 @@ use super::*; -use failure::*; +use anyhow::Result; use lazy_static::lazy_static; use log::*; use rusqlite::types::ValueRef; @@ -58,7 +58,7 @@ fn format_blob(b: ValueRef) -> String { } impl FileAdapter for SqliteAdapter { - fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> { + fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> { let AdaptInfo { is_real_file, filepath_hint, diff --git a/src/adapters/tar.rs b/src/adapters/tar.rs index c798b05..fd559b1 100644 --- a/src/adapters/tar.rs +++ b/src/adapters/tar.rs @@ -1,7 +1,7 @@ use super::*; use crate::preproc::rga_preproc; use ::tar::EntryType::Regular; -use failure::*; +use anyhow::*; use lazy_static::lazy_static; use log::*; use std::path::PathBuf; @@ -36,7 +36,7 @@ impl GetMetadata for TarAdapter { } impl FileAdapter for TarAdapter { - fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> { + fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> { let AdaptInfo { filepath_hint, mut inp, diff --git a/src/adapters/zip.rs b/src/adapters/zip.rs index cbcc27e..1e07780 100644 --- a/src/adapters/zip.rs +++ b/src/adapters/zip.rs @@ -1,7 +1,7 @@ use super::*; use crate::preproc::rga_preproc; use ::zip::read::ZipFile; -use failure::*; +use anyhow::*; use lazy_static::lazy_static; use log::*; @@ -47,7 +47,7 @@ fn is_dir(f: &ZipFile) -> bool { } impl FileAdapter for ZipAdapter { - fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> { + fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> { let AdaptInfo { filepath_hint, mut inp, diff --git a/src/args.rs b/src/args.rs index 748b4f0..271081c 100644 --- a/src/args.rs +++ b/src/args.rs @@ -1,16 +1,42 @@ -use failure::Fallible; +use anyhow::*; use log::*; + use serde::{Deserialize, Serialize}; use std::ffi::OsString; -use std::iter::IntoIterator; +use std::{iter::IntoIterator, str::FromStr}; use structopt::StructOpt; + +#[derive(Debug, Deserialize, Serialize)] +struct ReadableBytesCount(i64); + +fn parse_readable_bytes_str(s: &str) -> Result { + let suffix = s.chars().last(); + if let Some(suffix) = suffix { + match suffix { + 'k' | 'M' | 'G' => i64::from_str(s.trim_end_matches(suffix)) + .with_context(|| format!("Could not parse int")) + .map(|e| { + e * match suffix { + 'k' => 1000, + 'M' => 1000_000, + 'G' => 1000_000_000, + _ => panic!("impossible"), + } + }), + _ => i64::from_str(s).with_context(|| format!("Could not parse int")), + } + } else { + Err(format_err!("empty byte input")) + } +} + fn is_default(t: &T) -> bool { t == &T::default() } -// ugly, but serde and structopt use different methods to define defaults +// ugly, but serde and structopt use different methods to define defaults, so need to declare defaults twice macro_rules! set_default { ($name:ident, $val:expr, $type:ty) => { paste::item! { @@ -25,7 +51,7 @@ macro_rules! set_default { } set_default!(cache_compression_level, 12, u32); -set_default!(cache_max_blob_len, 2000000, u32); +set_default!(cache_max_blob_len, 2000000, i64); set_default!(max_archive_recursion, 4, i32); #[derive(StructOpt, Debug, Deserialize, Serialize)] @@ -83,12 +109,13 @@ pub struct RgaArgs { long = "--rga-cache-max-blob-len", default_value = "2000000", hidden_short_help = true, - require_equals = true + require_equals = true, + parse(try_from_str = parse_readable_bytes_str) )] /// Max compressed size to cache /// - /// Longest byte length (after compression) to store in cache. Longer adapter outputs will not be cached and recomputed every time. - pub cache_max_blob_len: u32, + /// Longest byte length (after compression) to store in cache. Longer adapter outputs will not be cached and recomputed every time. Allowed suffixes: k M G + pub cache_max_blob_len: i64, #[serde( default = "def_cache_compression_level", @@ -102,6 +129,8 @@ pub struct RgaArgs { help = "" )] /// ZSTD compression level to apply to adapter outputs before storing in cache db + /// + /// Ranges from 1 - 22 pub cache_compression_level: u32, #[serde( @@ -133,7 +162,7 @@ pub struct RgaArgs { static RGA_CONFIG: &str = "RGA_CONFIG"; -pub fn parse_args(args: I) -> Fallible +pub fn parse_args(args: I) -> Result where I: IntoIterator, I::Item: Into + Clone, @@ -158,7 +187,7 @@ where } /// Split arguments into the ones we care about and the ones rg cares about -pub fn split_args() -> Fallible<(RgaArgs, Vec)> { +pub fn split_args() -> Result<(RgaArgs, Vec)> { let mut app = RgaArgs::clap(); app.p.create_help_and_version(); diff --git a/src/bin/rga-preproc.rs b/src/bin/rga-preproc.rs index ee97348..fb5cc48 100644 --- a/src/bin/rga-preproc.rs +++ b/src/bin/rga-preproc.rs @@ -4,7 +4,7 @@ use ripgrep_all as rga; use std::fs::File; -fn main() -> Result<(), exitfailure::ExitFailure> { +fn main() -> anyhow::Result<()> { env_logger::init(); let mut arg_arr: Vec = std::env::args_os().collect(); let last = arg_arr.pop().expect("No filename specified"); diff --git a/src/bin/rga.rs b/src/bin/rga.rs index e0c844c..d56612d 100644 --- a/src/bin/rga.rs +++ b/src/bin/rga.rs @@ -1,4 +1,4 @@ -use failure::Fallible; +use anyhow::Result; use log::*; use rga::adapters::spawning::map_exe_error; use rga::adapters::*; @@ -9,7 +9,7 @@ use structopt::StructOpt; use std::process::Command; -fn main() -> Result<(), exitfailure::ExitFailure> { +fn main() -> anyhow::Result<()> { env_logger::init(); let (args, passthrough_args) = split_args()?; @@ -113,7 +113,7 @@ fn main() -> Result<(), exitfailure::ExitFailure> { } /// add the directory that contains `rga` to PATH, so rga-preproc can find pandoc etc (if we are on Windows where we include dependent binaries) -fn add_exe_to_path() -> Fallible<()> { +fn add_exe_to_path() -> Result<()> { use std::env; let mut exe = env::current_exe().expect("Could not get executable location"); // let preproc_exe = exe.with_file_name("rga-preproc"); diff --git a/src/caching_writer.rs b/src/caching_writer.rs index 86439fd..b6e22c7 100644 --- a/src/caching_writer.rs +++ b/src/caching_writer.rs @@ -1,4 +1,4 @@ -use failure::Fallible; +use anyhow::Result; use log::*; use std::io::Write; @@ -13,11 +13,7 @@ pub struct CachingWriter { out: W, } impl CachingWriter { - pub fn new( - out: W, - max_cache_size: usize, - compression_level: i32, - ) -> Fallible> { + pub fn new(out: W, max_cache_size: usize, compression_level: i32) -> Result> { Ok(CachingWriter { out, max_cache_size, diff --git a/src/matching.rs b/src/matching.rs index 2f4c6ef..5cf6f05 100644 --- a/src/matching.rs +++ b/src/matching.rs @@ -3,7 +3,7 @@ */ use crate::adapters::*; -use failure::*; +use anyhow::*; use regex::{Regex, RegexSet}; @@ -49,7 +49,7 @@ pub fn extension_to_regex(extension: &str) -> Regex { pub fn adapter_matcher>( adapter_names: &[T], slow: bool, -) -> Fallible Option<(Rc, SlowMatcher)>> { +) -> Result Option<(Rc, SlowMatcher)>> { let adapters = get_adapters_filtered(adapter_names)?; // need order later let adapter_names: Vec = adapters.iter().map(|e| e.metadata().name.clone()).collect(); diff --git a/src/preproc.rs b/src/preproc.rs index 36b3a40..f0f5643 100644 --- a/src/preproc.rs +++ b/src/preproc.rs @@ -2,7 +2,7 @@ use crate::adapters::*; use crate::args::RgaArgs; use crate::matching::*; use crate::CachingWriter; -use failure::*; +use anyhow::*; use log::*; use path_clean::PathClean; use std::convert::TryInto; @@ -22,7 +22,7 @@ pub struct PreprocConfig<'a> { * If a cache is passed, read/write to it. * */ -pub fn rga_preproc(ai: AdaptInfo) -> Fallible<()> { +pub fn rga_preproc(ai: AdaptInfo) -> Result<()> { let AdaptInfo { filepath_hint, is_real_file, @@ -98,7 +98,7 @@ pub fn rga_preproc(ai: AdaptInfo) -> Fallible<()> { cache.write().unwrap().get_or_run( &db_name, &cache_key, - Box::new(|| -> Fallible>> { + Box::new(|| -> Result>> { // wrapping BufWriter here gives ~10% perf boost let mut compbuf = BufWriter::new(CachingWriter::new( oup, @@ -120,7 +120,7 @@ pub fn rga_preproc(ai: AdaptInfo) -> Fallible<()> { )?; let compressed = compbuf .into_inner() - .map_err(|_| "could not finish zstd") + .map_err(|_| "could not finish zstd") // can't use with_context here .unwrap() .finish()?; if let Some(cached) = compressed { diff --git a/src/preproc_cache.rs b/src/preproc_cache.rs index 7352555..b80cf88 100644 --- a/src/preproc_cache.rs +++ b/src/preproc_cache.rs @@ -1,8 +1,11 @@ -use failure::{format_err, Fallible}; +use anyhow::{format_err, Context, Result}; use log::*; -use std::sync::{Arc, RwLock}; +use std::{ + fmt::Display, + sync::{Arc, RwLock}, +}; -pub fn open() -> Fallible>> { +pub fn open() -> Result>> { Ok(Arc::new(RwLock::new(LmdbCache::open()?))) } pub trait PreprocCache { @@ -11,13 +14,13 @@ pub trait PreprocCache { &mut self, db_name: &str, key: &[u8], - runner: Box Fallible>> + 'a>, - callback: Box Fallible<()> + 'a>, - ) -> Fallible<()>; + runner: Box Result>> + 'a>, + callback: Box Result<()> + 'a>, + ) -> Result<()>; } /// opens a LMDB cache -fn open_cache_db() -> Fallible>> { +fn open_cache_db() -> Result>> { let app_cache = cachedir::CacheDirConfig::new("rga").get_cache_dir()?; let db_arc = rkv::Manager::singleton() @@ -46,30 +49,42 @@ pub struct LmdbCache { } impl LmdbCache { - pub fn open() -> Fallible { + pub fn open() -> Result { Ok(LmdbCache { db_arc: open_cache_db()?, }) } } + +#[derive(Debug)] +struct RkvErrWrap(rkv::StoreError); +impl Display for RkvErrWrap { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} +impl std::error::Error for RkvErrWrap {} + impl PreprocCache for LmdbCache { // possible without second lambda? fn get_or_run<'a>( &mut self, db_name: &str, key: &[u8], - runner: Box Fallible>> + 'a>, - callback: Box Fallible<()> + 'a>, - ) -> Fallible<()> { + runner: Box Result>> + 'a>, + callback: Box Result<()> + 'a>, + ) -> Result<()> { let db_env = self.db_arc.read().unwrap(); let db = db_env .open_single(db_name, rkv::store::Options::create()) - .map_err(|p| format_err!("could not open db store: {:?}", p))?; + .map_err(RkvErrWrap) + .with_context(|| format_err!("could not open cache db store"))?; let reader = db_env.read().expect("could not get reader"); let cached = db .get(&reader, &key) - .map_err(|p| format_err!("could not read from db: {:?}", p))?; + .map_err(RkvErrWrap) + .with_context(|| format_err!("could not read from db"))?; match cached { Some(rkv::Value::Blob(cached)) => { @@ -81,12 +96,17 @@ impl PreprocCache for LmdbCache { debug!("did not get cached"); drop(reader); if let Some(got) = runner()? { - let mut writer = db_env.write().map_err(|p| { - format_err!("could not open write handle to cache: {:?}", p) - })?; + let mut writer = db_env + .write() + .map_err(RkvErrWrap) + .with_context(|| format_err!("could not open write handle to cache"))?; db.put(&mut writer, &key, &rkv::Value::Blob(&got)) - .map_err(|p| format_err!("could not write to cache: {:?}", p))?; - writer.commit()?; + .map_err(RkvErrWrap) + .with_context(|| format_err!("could not write to cache"))?; + writer + .commit() + .map_err(RkvErrWrap) + .with_context(|| format!("could not write cache"))?; } } };