mirror of
https://github.com/FliegendeWurst/ripgrep-all.git
synced 2024-11-24 04:14:57 +00:00
switch failure -> anyhow, parse byte suffix in cache-max-blob-len
This commit is contained in:
phiresky
parent
5d97c2ad5f
commit
f2d9393832
160
Cargo.lock
generated
160
Cargo.lock
generated
@ -1,14 +1,5 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
[[package]]
|
||||
name = "addr2line"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "456d75cbb82da1ad150c8a9d97285ffcd21c9931dcb11e995903e7d75141b38b"
|
||||
dependencies = [
|
||||
"gimli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "adler32"
|
||||
version = "1.0.4"
|
||||
@ -33,6 +24,12 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f"
|
||||
|
||||
[[package]]
|
||||
name = "arrayref"
|
||||
version = "0.3.6"
|
||||
@ -62,19 +59,6 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.48"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0df2f85c8a2abbe3b7d7e748052fdd9b76a0458fdeb16ad4223f5eca78c7c130"
|
||||
dependencies = [
|
||||
"addr2line",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"object",
|
||||
"rustc-demangle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.2.1"
|
||||
@ -109,12 +93,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bzip2-sys"
|
||||
version = "0.1.8+1.0.8"
|
||||
version = "0.1.9+1.0.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05305b41c5034ff0e93937ac64133d109b5a2660114ec45e9760bc6816d83038"
|
||||
checksum = "ad3b39a260062fca31f7b0b12f207e8f2590a67d32ec7d59c20484b07ea7285e"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -125,9 +110,9 @@ checksum = "c06509d1f4ffa658939bd23f076cd929ef218241363796551528e7eec69128c8"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.53"
|
||||
version = "1.0.54"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "404b1fe4f65288577753b17e3b36a04596ee784493ec249bf81c7f2d2acd751c"
|
||||
checksum = "7bbb73db36c1246e9034e307d0fba23f9a2e251faa47ade70c1bd252220c8311"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
]
|
||||
@ -235,9 +220,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.2.1"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db"
|
||||
checksum = "ab6bffe714b6bb07e42f201352c34f51fefd355ace793f9e638ebd52d23f98d2"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
@ -254,6 +239,12 @@ dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.23"
|
||||
@ -285,22 +276,12 @@ dependencies = [
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "exitfailure"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ff5bd832af37f366c6c194d813a11cd90ac484f124f079294f28e357ae40515"
|
||||
dependencies = [
|
||||
"failure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "failure"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"failure_derive",
|
||||
]
|
||||
|
||||
@ -390,12 +371,6 @@ dependencies = [
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gimli"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bcc8e0c9bce37868955864dbecd2b1ab2bdf967e6f28066d65aaac620444b65c"
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.0"
|
||||
@ -440,6 +415,15 @@ dependencies = [
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "0.4.5"
|
||||
@ -463,9 +447,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.70"
|
||||
version = "0.2.71"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3baa92041a6fec78c687fa0cc2b3fae8884f743d672cf551bed1d6dac6988d0f"
|
||||
checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49"
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
@ -651,12 +635,6 @@ dependencies = [
|
||||
"autocfg 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9cbca9424c482ee628fa549d9c812e2cd22f1180b9222c9200fdfa6eb31aecb2"
|
||||
|
||||
[[package]]
|
||||
name = "ordered-float"
|
||||
version = "1.0.2"
|
||||
@ -701,9 +679,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "paste"
|
||||
version = "0.1.12"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a229b1c58c692edcaa5b9b0948084f130f55d2dcc15b02fcc5340b2b4521476"
|
||||
checksum = "d508492eeb1e5c38ee696371bf7b9fc33c83d46a7d451606b96458fbbbdc2dec"
|
||||
dependencies = [
|
||||
"paste-impl",
|
||||
"proc-macro-hack",
|
||||
@ -711,9 +689,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "paste-impl"
|
||||
version = "0.1.12"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e0bf239e447e67ff6d16a8bb5e4d4bd2343acf5066061c0e8e06ac5ba8ca68c"
|
||||
checksum = "84f328a6a63192b333fce5fbb4be79db6758a4d518dfac6d54412f1492f72d32"
|
||||
dependencies = [
|
||||
"proc-macro-hack",
|
||||
"proc-macro2",
|
||||
@ -751,9 +729,9 @@ checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
|
||||
|
||||
[[package]]
|
||||
name = "podio"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "780fb4b6698bbf9cf2444ea5d22411cef2953f0824b98f33cf454ec5615645bd"
|
||||
checksum = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
@ -789,15 +767,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro-hack"
|
||||
version = "0.5.15"
|
||||
version = "0.5.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63"
|
||||
checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.13"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "53f5ffe53a6b28e37c9c1ce74893477864d64f74778a93a4beb43c8fa167f639"
|
||||
checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
@ -981,9 +959,9 @@ checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.3.7"
|
||||
version = "1.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692"
|
||||
checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@ -993,9 +971,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.17"
|
||||
version = "0.6.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae"
|
||||
checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
@ -1010,6 +988,7 @@ dependencies = [
|
||||
name = "ripgrep_all"
|
||||
version = "0.9.7-alpha.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bincode",
|
||||
"bzip2",
|
||||
"cachedir",
|
||||
@ -1019,8 +998,6 @@ dependencies = [
|
||||
"encoding_rs",
|
||||
"encoding_rs_io",
|
||||
"env_logger",
|
||||
"exitfailure",
|
||||
"failure",
|
||||
"flate2",
|
||||
"glob",
|
||||
"lazy_static",
|
||||
@ -1079,12 +1056,6 @@ dependencies = [
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.2.3"
|
||||
@ -1096,9 +1067,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.4"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1"
|
||||
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
@ -1123,18 +1094,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.110"
|
||||
version = "1.0.111"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c"
|
||||
checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.110"
|
||||
version = "1.0.111"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984"
|
||||
checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@ -1209,9 +1180,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.22"
|
||||
version = "1.0.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1425de3c33b0941002740a420b1a906a350b88d08b82b2c8a01035a3f9447bac"
|
||||
checksum = "93a56fabc59dce20fe48b6c832cc249c713e7ed88fa28b0ee0a3bfcaae5fe4e2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@ -1243,9 +1214,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tar"
|
||||
version = "0.4.26"
|
||||
version = "0.4.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
|
||||
checksum = "5c058ad0bd6ccb84faa24cc44d4fc99bee8a5d7ba9ff33aa4d993122d1aeeac2"
|
||||
dependencies = [
|
||||
"filetime",
|
||||
"libc",
|
||||
@ -1389,9 +1360,9 @@ checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.8"
|
||||
version = "0.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168"
|
||||
checksum = "55d1e41d56121e07f1e223db0a4def204e45c85425f6a16d462fd07c8d10d74c"
|
||||
|
||||
[[package]]
|
||||
name = "vec_map"
|
||||
@ -1401,9 +1372,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.1"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
|
||||
checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
@ -1475,18 +1446,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.5.1+zstd.1.4.4"
|
||||
version = "0.5.2+zstd.1.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c5d978b793ae64375b80baf652919b148f6a496ac8802922d9999f5a553194f"
|
||||
checksum = "644352b10ce7f333d6e0af85bd4f5322dc449416dc1211c6308e95bca8923db4"
|
||||
dependencies = [
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd-safe"
|
||||
version = "2.0.3+zstd.1.4.4"
|
||||
version = "2.0.4+zstd.1.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bee25eac9753cfedd48133fa1736cbd23b774e253d89badbeac7d12b23848d3f"
|
||||
checksum = "7113c0c9aed2c55181f2d9f5b0a36e7d2c0183b11c058ab40b35987479efe4d7"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"zstd-sys",
|
||||
@ -1494,11 +1465,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zstd-sys"
|
||||
version = "1.4.15+zstd.1.4.4"
|
||||
version = "1.4.16+zstd.1.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89719b034dc22d240d5b407fb0a3fe6d29952c181cff9a9f95c0bd40b4f8f7d8"
|
||||
checksum = "c442965efc45353be5a9b9969c9b0872fff6828c7e06d118dda2cb2d0bb11d5a"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"glob",
|
||||
"itertools",
|
||||
"libc",
|
||||
]
|
||||
|
||||
21
Cargo.toml
21
Cargo.toml
@ -17,32 +17,31 @@ exclude = [
|
||||
|
||||
[dependencies]
|
||||
tree_magic = { package = "tree_magic_fork", version = "0.2.2" }
|
||||
regex = "1.3.6"
|
||||
regex = "1.3.9"
|
||||
rkv = "0.10.4"
|
||||
cachedir = "0.1.1"
|
||||
path-clean = "0.1.0"
|
||||
bincode = "1.2.1"
|
||||
serde = { version = "1.0.106", features = ["derive"] }
|
||||
zstd = "0.5.1"
|
||||
serde = { version = "1.0.111", features = ["derive"] }
|
||||
zstd = "0.5.2"
|
||||
lazy_static = "1.4.0"
|
||||
serde_json = "1.0.51"
|
||||
failure = "0.1.7"
|
||||
serde_json = "1.0.53"
|
||||
zip = "0.5.5"
|
||||
crossbeam = "0.7.3"
|
||||
clap = { version = "2.33.0", features = ["wrap_help"] }
|
||||
clap = { version = "2.33.1", features = ["wrap_help"] }
|
||||
log = "0.4.8"
|
||||
env_logger = "0.7.1"
|
||||
xz2 = "0.1.6"
|
||||
flate2 = "1.0.14"
|
||||
bzip2 = "0.3.3"
|
||||
tar = "0.4.26"
|
||||
tar = "0.4.28"
|
||||
chrono = "0.4.11"
|
||||
encoding_rs = "0.8.22"
|
||||
encoding_rs = "0.8.23"
|
||||
encoding_rs_io = "0.1.7"
|
||||
rusqlite = { version = "0.23.1", features = ["vtab", "bundled"] }
|
||||
size_format = "1.0.2"
|
||||
structopt = "0.3.12"
|
||||
paste = "0.1.10"
|
||||
structopt = "0.3.14"
|
||||
paste = "0.1.16"
|
||||
tempfile = "3.1.0"
|
||||
glob = "0.3.0"
|
||||
exitfailure = "0.5.1"
|
||||
anyhow = "1.0.31"
|
||||
|
||||
@ -211,7 +211,7 @@ The following adapters are disabled by default, and can be enabled using '--rga-
|
||||
|
||||
> \[default: 12\]
|
||||
|
||||
**\--rga-cache-max-blob-len** \<cache-max-blob-len\>
|
||||
**\--rga-cache-max-blob-len**=\<cache-max-blob-len\>
|
||||
|
||||
> Max compressed size to cache
|
||||
>
|
||||
|
||||
@ -10,7 +10,7 @@ pub mod tesseract;
|
||||
pub mod zip;
|
||||
use crate::matching::*;
|
||||
use crate::preproc::PreprocConfig;
|
||||
use failure::*;
|
||||
use anyhow::*;
|
||||
use log::*;
|
||||
use regex::Regex;
|
||||
use std::borrow::Cow;
|
||||
@ -58,7 +58,7 @@ pub trait FileAdapter: GetMetadata {
|
||||
/// adapt a file.
|
||||
///
|
||||
/// detection_reason is the Matcher that was used to identify this file. Unless --rga-accurate was given, it is always a FastMatcher
|
||||
fn adapt(&self, a: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()>;
|
||||
fn adapt(&self, a: AdaptInfo, detection_reason: &SlowMatcher) -> Result<()>;
|
||||
}
|
||||
pub struct AdaptInfo<'a> {
|
||||
/// file path. May not be an actual file on the file system (e.g. in an archive). Used for matching file extensions.
|
||||
@ -107,7 +107,7 @@ pub fn get_all_adapters() -> AdaptersTuple {
|
||||
*/
|
||||
pub fn get_adapters_filtered<T: AsRef<str>>(
|
||||
adapter_names: &[T],
|
||||
) -> Fallible<Vec<Rc<dyn FileAdapter>>> {
|
||||
) -> Result<Vec<Rc<dyn FileAdapter>>> {
|
||||
let (def_enabled_adapters, def_disabled_adapters) = get_all_adapters();
|
||||
let adapters = if !adapter_names.is_empty() {
|
||||
let adapters_map: HashMap<_, _> = def_enabled_adapters
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
use super::*;
|
||||
use crate::preproc::rga_preproc;
|
||||
use failure::*;
|
||||
use anyhow::*;
|
||||
use lazy_static::lazy_static;
|
||||
|
||||
use std::path::PathBuf;
|
||||
@ -46,7 +46,7 @@ impl GetMetadata for DecompressAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
fn decompress_any<'a, R>(reason: &SlowMatcher, inp: &'a mut R) -> Fallible<Box<dyn Read + 'a>>
|
||||
fn decompress_any<'a, R>(reason: &SlowMatcher, inp: &'a mut R) -> Result<Box<dyn Read + 'a>>
|
||||
where
|
||||
R: Read,
|
||||
{
|
||||
@ -91,7 +91,7 @@ fn get_inner_filename(filename: &Path) -> PathBuf {
|
||||
}
|
||||
|
||||
impl FileAdapter for DecompressAdapter {
|
||||
fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||
fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Result<()> {
|
||||
let AdaptInfo {
|
||||
filepath_hint,
|
||||
mut inp,
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
use super::spawning::map_exe_error;
|
||||
use super::*;
|
||||
use failure::*;
|
||||
use anyhow::*;
|
||||
use lazy_static::lazy_static;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::io::BufReader;
|
||||
@ -48,7 +48,7 @@ struct FFprobeStream {
|
||||
codec_type: String, // video,audio,subtitle
|
||||
}
|
||||
impl FileAdapter for FFmpegAdapter {
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
|
||||
let AdaptInfo {
|
||||
is_real_file,
|
||||
filepath_hint,
|
||||
|
||||
@ -43,7 +43,7 @@ impl GetMetadata for PdfPagesAdapter {
|
||||
/// A pdf is basically converted to a zip that has Page X.png files.
|
||||
/// This way, something like tesseract can process the pages individually
|
||||
impl FileAdapter for PdfPagesAdapter {
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
|
||||
let AdaptInfo {
|
||||
filepath_hint,
|
||||
is_real_file,
|
||||
@ -114,7 +114,7 @@ fn split_by_seq<'a>(
|
||||
split_seq: &'a [u8],
|
||||
split_inx: usize,
|
||||
read: &mut Read,
|
||||
) -> Fallible<impl IntoIterator<Item = impl Read> + 'a> {
|
||||
) -> Result<impl IntoIterator<Item = impl Read> + 'a> {
|
||||
let regex = split_seq
|
||||
.iter()
|
||||
.map(|c| format!("\\x{:0>2x}", c))
|
||||
|
||||
@ -42,7 +42,7 @@ impl SpawningFileAdapter for PopplerAdapter {
|
||||
cmd.arg("-").arg("-");
|
||||
cmd
|
||||
}
|
||||
fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Fallible<()> {
|
||||
fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Result<()> {
|
||||
// prepend Page X to each line
|
||||
let mut page = 1;
|
||||
for line in BufReader::new(inp).lines() {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
use super::*;
|
||||
use anyhow::*;
|
||||
use encoding_rs_io::DecodeReaderBytesBuilder;
|
||||
use failure::*;
|
||||
use log::*;
|
||||
use std::io::prelude::*;
|
||||
use std::io::BufReader;
|
||||
@ -20,7 +20,7 @@ pub fn postproc_line_prefix(
|
||||
line_prefix: &str,
|
||||
inp: &mut dyn Read,
|
||||
oup: &mut dyn Write,
|
||||
) -> Fallible<()> {
|
||||
) -> Result<()> {
|
||||
// TODO: parse these options from ripgrep's configuration
|
||||
let encoding = None; // detect bom but usually assume utf8
|
||||
let bom_sniffing = true;
|
||||
@ -57,7 +57,7 @@ pub trait SpawningFileAdapter: GetMetadata {
|
||||
fn get_exe(&self) -> &str;
|
||||
fn command(&self, filepath_hint: &Path, command: Command) -> Command;
|
||||
|
||||
fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Fallible<()> {
|
||||
fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Result<()> {
|
||||
postproc_line_prefix(line_prefix, inp, oup)
|
||||
}
|
||||
}
|
||||
@ -78,8 +78,8 @@ pub fn pipe_output(
|
||||
oup: &mut (dyn Write + Send),
|
||||
exe_name: &str,
|
||||
help: &str,
|
||||
cp: fn(line_prefix: &str, &mut dyn Read, &mut dyn Write) -> Fallible<()>,
|
||||
) -> Fallible<()> {
|
||||
cp: fn(line_prefix: &str, &mut dyn Read, &mut dyn Write) -> Result<()>,
|
||||
) -> Result<()> {
|
||||
let mut cmd = cmd
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
@ -90,7 +90,7 @@ pub fn pipe_output(
|
||||
|
||||
// TODO: how to handle this copying better?
|
||||
// do we really need threads for this?
|
||||
crossbeam::scope(|s| -> Fallible<()> {
|
||||
crossbeam::scope(|s| -> Result<()> {
|
||||
s.spawn(|_| cp(line_prefix, &mut stdo, oup).unwrap()); // errors?
|
||||
std::io::copy(inp, &mut stdi)?;
|
||||
drop(stdi); // NEEDED! otherwise deadlock
|
||||
@ -109,7 +109,7 @@ impl<T> FileAdapter for T
|
||||
where
|
||||
T: SpawningFileAdapter,
|
||||
{
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
|
||||
let AdaptInfo {
|
||||
filepath_hint,
|
||||
mut inp,
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
use super::*;
|
||||
use failure::*;
|
||||
use anyhow::Result;
|
||||
use lazy_static::lazy_static;
|
||||
use log::*;
|
||||
use rusqlite::types::ValueRef;
|
||||
@ -58,7 +58,7 @@ fn format_blob(b: ValueRef) -> String {
|
||||
}
|
||||
|
||||
impl FileAdapter for SqliteAdapter {
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
|
||||
let AdaptInfo {
|
||||
is_real_file,
|
||||
filepath_hint,
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
use super::*;
|
||||
use crate::preproc::rga_preproc;
|
||||
use ::tar::EntryType::Regular;
|
||||
use failure::*;
|
||||
use anyhow::*;
|
||||
use lazy_static::lazy_static;
|
||||
use log::*;
|
||||
use std::path::PathBuf;
|
||||
@ -36,7 +36,7 @@ impl GetMetadata for TarAdapter {
|
||||
}
|
||||
|
||||
impl FileAdapter for TarAdapter {
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
|
||||
let AdaptInfo {
|
||||
filepath_hint,
|
||||
mut inp,
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
use super::*;
|
||||
use crate::preproc::rga_preproc;
|
||||
use ::zip::read::ZipFile;
|
||||
use failure::*;
|
||||
use anyhow::*;
|
||||
use lazy_static::lazy_static;
|
||||
use log::*;
|
||||
|
||||
@ -47,7 +47,7 @@ fn is_dir(f: &ZipFile) -> bool {
|
||||
}
|
||||
|
||||
impl FileAdapter for ZipAdapter {
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
|
||||
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
|
||||
let AdaptInfo {
|
||||
filepath_hint,
|
||||
mut inp,
|
||||
|
||||
47
src/args.rs
47
src/args.rs
@ -1,16 +1,42 @@
|
||||
use failure::Fallible;
|
||||
use anyhow::*;
|
||||
use log::*;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use std::ffi::OsString;
|
||||
use std::iter::IntoIterator;
|
||||
use std::{iter::IntoIterator, str::FromStr};
|
||||
|
||||
use structopt::StructOpt;
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
struct ReadableBytesCount(i64);
|
||||
|
||||
fn parse_readable_bytes_str(s: &str) -> Result<i64, Error> {
|
||||
let suffix = s.chars().last();
|
||||
if let Some(suffix) = suffix {
|
||||
match suffix {
|
||||
'k' | 'M' | 'G' => i64::from_str(s.trim_end_matches(suffix))
|
||||
.with_context(|| format!("Could not parse int"))
|
||||
.map(|e| {
|
||||
e * match suffix {
|
||||
'k' => 1000,
|
||||
'M' => 1000_000,
|
||||
'G' => 1000_000_000,
|
||||
_ => panic!("impossible"),
|
||||
}
|
||||
}),
|
||||
_ => i64::from_str(s).with_context(|| format!("Could not parse int")),
|
||||
}
|
||||
} else {
|
||||
Err(format_err!("empty byte input"))
|
||||
}
|
||||
}
|
||||
|
||||
fn is_default<T: Default + PartialEq>(t: &T) -> bool {
|
||||
t == &T::default()
|
||||
}
|
||||
|
||||
// ugly, but serde and structopt use different methods to define defaults
|
||||
// ugly, but serde and structopt use different methods to define defaults, so need to declare defaults twice
|
||||
macro_rules! set_default {
|
||||
($name:ident, $val:expr, $type:ty) => {
|
||||
paste::item! {
|
||||
@ -25,7 +51,7 @@ macro_rules! set_default {
|
||||
}
|
||||
|
||||
set_default!(cache_compression_level, 12, u32);
|
||||
set_default!(cache_max_blob_len, 2000000, u32);
|
||||
set_default!(cache_max_blob_len, 2000000, i64);
|
||||
set_default!(max_archive_recursion, 4, i32);
|
||||
|
||||
#[derive(StructOpt, Debug, Deserialize, Serialize)]
|
||||
@ -83,12 +109,13 @@ pub struct RgaArgs {
|
||||
long = "--rga-cache-max-blob-len",
|
||||
default_value = "2000000",
|
||||
hidden_short_help = true,
|
||||
require_equals = true
|
||||
require_equals = true,
|
||||
parse(try_from_str = parse_readable_bytes_str)
|
||||
)]
|
||||
/// Max compressed size to cache
|
||||
///
|
||||
/// Longest byte length (after compression) to store in cache. Longer adapter outputs will not be cached and recomputed every time.
|
||||
pub cache_max_blob_len: u32,
|
||||
/// Longest byte length (after compression) to store in cache. Longer adapter outputs will not be cached and recomputed every time. Allowed suffixes: k M G
|
||||
pub cache_max_blob_len: i64,
|
||||
|
||||
#[serde(
|
||||
default = "def_cache_compression_level",
|
||||
@ -102,6 +129,8 @@ pub struct RgaArgs {
|
||||
help = ""
|
||||
)]
|
||||
/// ZSTD compression level to apply to adapter outputs before storing in cache db
|
||||
///
|
||||
/// Ranges from 1 - 22
|
||||
pub cache_compression_level: u32,
|
||||
|
||||
#[serde(
|
||||
@ -133,7 +162,7 @@ pub struct RgaArgs {
|
||||
|
||||
static RGA_CONFIG: &str = "RGA_CONFIG";
|
||||
|
||||
pub fn parse_args<I>(args: I) -> Fallible<RgaArgs>
|
||||
pub fn parse_args<I>(args: I) -> Result<RgaArgs>
|
||||
where
|
||||
I: IntoIterator,
|
||||
I::Item: Into<OsString> + Clone,
|
||||
@ -158,7 +187,7 @@ where
|
||||
}
|
||||
|
||||
/// Split arguments into the ones we care about and the ones rg cares about
|
||||
pub fn split_args() -> Fallible<(RgaArgs, Vec<OsString>)> {
|
||||
pub fn split_args() -> Result<(RgaArgs, Vec<OsString>)> {
|
||||
let mut app = RgaArgs::clap();
|
||||
|
||||
app.p.create_help_and_version();
|
||||
|
||||
@ -4,7 +4,7 @@ use ripgrep_all as rga;
|
||||
|
||||
use std::fs::File;
|
||||
|
||||
fn main() -> Result<(), exitfailure::ExitFailure> {
|
||||
fn main() -> anyhow::Result<()> {
|
||||
env_logger::init();
|
||||
let mut arg_arr: Vec<std::ffi::OsString> = std::env::args_os().collect();
|
||||
let last = arg_arr.pop().expect("No filename specified");
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
use failure::Fallible;
|
||||
use anyhow::Result;
|
||||
use log::*;
|
||||
use rga::adapters::spawning::map_exe_error;
|
||||
use rga::adapters::*;
|
||||
@ -9,7 +9,7 @@ use structopt::StructOpt;
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
fn main() -> Result<(), exitfailure::ExitFailure> {
|
||||
fn main() -> anyhow::Result<()> {
|
||||
env_logger::init();
|
||||
|
||||
let (args, passthrough_args) = split_args()?;
|
||||
@ -113,7 +113,7 @@ fn main() -> Result<(), exitfailure::ExitFailure> {
|
||||
}
|
||||
|
||||
/// add the directory that contains `rga` to PATH, so rga-preproc can find pandoc etc (if we are on Windows where we include dependent binaries)
|
||||
fn add_exe_to_path() -> Fallible<()> {
|
||||
fn add_exe_to_path() -> Result<()> {
|
||||
use std::env;
|
||||
let mut exe = env::current_exe().expect("Could not get executable location");
|
||||
// let preproc_exe = exe.with_file_name("rga-preproc");
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
use failure::Fallible;
|
||||
use anyhow::Result;
|
||||
use log::*;
|
||||
use std::io::Write;
|
||||
|
||||
@ -13,11 +13,7 @@ pub struct CachingWriter<W: Write> {
|
||||
out: W,
|
||||
}
|
||||
impl<W: Write> CachingWriter<W> {
|
||||
pub fn new(
|
||||
out: W,
|
||||
max_cache_size: usize,
|
||||
compression_level: i32,
|
||||
) -> Fallible<CachingWriter<W>> {
|
||||
pub fn new(out: W, max_cache_size: usize, compression_level: i32) -> Result<CachingWriter<W>> {
|
||||
Ok(CachingWriter {
|
||||
out,
|
||||
max_cache_size,
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
*/
|
||||
use crate::adapters::*;
|
||||
|
||||
use failure::*;
|
||||
use anyhow::*;
|
||||
|
||||
use regex::{Regex, RegexSet};
|
||||
|
||||
@ -49,7 +49,7 @@ pub fn extension_to_regex(extension: &str) -> Regex {
|
||||
pub fn adapter_matcher<T: AsRef<str>>(
|
||||
adapter_names: &[T],
|
||||
slow: bool,
|
||||
) -> Fallible<impl Fn(FileMeta) -> Option<(Rc<dyn FileAdapter>, SlowMatcher)>> {
|
||||
) -> Result<impl Fn(FileMeta) -> Option<(Rc<dyn FileAdapter>, SlowMatcher)>> {
|
||||
let adapters = get_adapters_filtered(adapter_names)?;
|
||||
// need order later
|
||||
let adapter_names: Vec<String> = adapters.iter().map(|e| e.metadata().name.clone()).collect();
|
||||
|
||||
@ -2,7 +2,7 @@ use crate::adapters::*;
|
||||
use crate::args::RgaArgs;
|
||||
use crate::matching::*;
|
||||
use crate::CachingWriter;
|
||||
use failure::*;
|
||||
use anyhow::*;
|
||||
use log::*;
|
||||
use path_clean::PathClean;
|
||||
use std::convert::TryInto;
|
||||
@ -22,7 +22,7 @@ pub struct PreprocConfig<'a> {
|
||||
* If a cache is passed, read/write to it.
|
||||
*
|
||||
*/
|
||||
pub fn rga_preproc(ai: AdaptInfo) -> Fallible<()> {
|
||||
pub fn rga_preproc(ai: AdaptInfo) -> Result<()> {
|
||||
let AdaptInfo {
|
||||
filepath_hint,
|
||||
is_real_file,
|
||||
@ -98,7 +98,7 @@ pub fn rga_preproc(ai: AdaptInfo) -> Fallible<()> {
|
||||
cache.write().unwrap().get_or_run(
|
||||
&db_name,
|
||||
&cache_key,
|
||||
Box::new(|| -> Fallible<Option<Vec<u8>>> {
|
||||
Box::new(|| -> Result<Option<Vec<u8>>> {
|
||||
// wrapping BufWriter here gives ~10% perf boost
|
||||
let mut compbuf = BufWriter::new(CachingWriter::new(
|
||||
oup,
|
||||
@ -120,7 +120,7 @@ pub fn rga_preproc(ai: AdaptInfo) -> Fallible<()> {
|
||||
)?;
|
||||
let compressed = compbuf
|
||||
.into_inner()
|
||||
.map_err(|_| "could not finish zstd")
|
||||
.map_err(|_| "could not finish zstd") // can't use with_context here
|
||||
.unwrap()
|
||||
.finish()?;
|
||||
if let Some(cached) = compressed {
|
||||
|
||||
@ -1,8 +1,11 @@
|
||||
use failure::{format_err, Fallible};
|
||||
use anyhow::{format_err, Context, Result};
|
||||
use log::*;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::{
|
||||
fmt::Display,
|
||||
sync::{Arc, RwLock},
|
||||
};
|
||||
|
||||
pub fn open() -> Fallible<Arc<RwLock<dyn PreprocCache>>> {
|
||||
pub fn open() -> Result<Arc<RwLock<dyn PreprocCache>>> {
|
||||
Ok(Arc::new(RwLock::new(LmdbCache::open()?)))
|
||||
}
|
||||
pub trait PreprocCache {
|
||||
@ -11,13 +14,13 @@ pub trait PreprocCache {
|
||||
&mut self,
|
||||
db_name: &str,
|
||||
key: &[u8],
|
||||
runner: Box<dyn FnOnce() -> Fallible<Option<Vec<u8>>> + 'a>,
|
||||
callback: Box<dyn FnOnce(&[u8]) -> Fallible<()> + 'a>,
|
||||
) -> Fallible<()>;
|
||||
runner: Box<dyn FnOnce() -> Result<Option<Vec<u8>>> + 'a>,
|
||||
callback: Box<dyn FnOnce(&[u8]) -> Result<()> + 'a>,
|
||||
) -> Result<()>;
|
||||
}
|
||||
|
||||
/// opens a LMDB cache
|
||||
fn open_cache_db() -> Fallible<std::sync::Arc<std::sync::RwLock<rkv::Rkv>>> {
|
||||
fn open_cache_db() -> Result<std::sync::Arc<std::sync::RwLock<rkv::Rkv>>> {
|
||||
let app_cache = cachedir::CacheDirConfig::new("rga").get_cache_dir()?;
|
||||
|
||||
let db_arc = rkv::Manager::singleton()
|
||||
@ -46,30 +49,42 @@ pub struct LmdbCache {
|
||||
}
|
||||
|
||||
impl LmdbCache {
|
||||
pub fn open() -> Fallible<LmdbCache> {
|
||||
pub fn open() -> Result<LmdbCache> {
|
||||
Ok(LmdbCache {
|
||||
db_arc: open_cache_db()?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct RkvErrWrap(rkv::StoreError);
|
||||
impl Display for RkvErrWrap {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
self.0.fmt(f)
|
||||
}
|
||||
}
|
||||
impl std::error::Error for RkvErrWrap {}
|
||||
|
||||
impl PreprocCache for LmdbCache {
|
||||
// possible without second lambda?
|
||||
fn get_or_run<'a>(
|
||||
&mut self,
|
||||
db_name: &str,
|
||||
key: &[u8],
|
||||
runner: Box<dyn FnOnce() -> Fallible<Option<Vec<u8>>> + 'a>,
|
||||
callback: Box<dyn FnOnce(&[u8]) -> Fallible<()> + 'a>,
|
||||
) -> Fallible<()> {
|
||||
runner: Box<dyn FnOnce() -> Result<Option<Vec<u8>>> + 'a>,
|
||||
callback: Box<dyn FnOnce(&[u8]) -> Result<()> + 'a>,
|
||||
) -> Result<()> {
|
||||
let db_env = self.db_arc.read().unwrap();
|
||||
let db = db_env
|
||||
.open_single(db_name, rkv::store::Options::create())
|
||||
.map_err(|p| format_err!("could not open db store: {:?}", p))?;
|
||||
.map_err(RkvErrWrap)
|
||||
.with_context(|| format_err!("could not open cache db store"))?;
|
||||
|
||||
let reader = db_env.read().expect("could not get reader");
|
||||
let cached = db
|
||||
.get(&reader, &key)
|
||||
.map_err(|p| format_err!("could not read from db: {:?}", p))?;
|
||||
.map_err(RkvErrWrap)
|
||||
.with_context(|| format_err!("could not read from db"))?;
|
||||
|
||||
match cached {
|
||||
Some(rkv::Value::Blob(cached)) => {
|
||||
@ -81,12 +96,17 @@ impl PreprocCache for LmdbCache {
|
||||
debug!("did not get cached");
|
||||
drop(reader);
|
||||
if let Some(got) = runner()? {
|
||||
let mut writer = db_env.write().map_err(|p| {
|
||||
format_err!("could not open write handle to cache: {:?}", p)
|
||||
})?;
|
||||
let mut writer = db_env
|
||||
.write()
|
||||
.map_err(RkvErrWrap)
|
||||
.with_context(|| format_err!("could not open write handle to cache"))?;
|
||||
db.put(&mut writer, &key, &rkv::Value::Blob(&got))
|
||||
.map_err(|p| format_err!("could not write to cache: {:?}", p))?;
|
||||
writer.commit()?;
|
||||
.map_err(RkvErrWrap)
|
||||
.with_context(|| format_err!("could not write to cache"))?;
|
||||
writer
|
||||
.commit()
|
||||
.map_err(RkvErrWrap)
|
||||
.with_context(|| format!("could not write cache"))?;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user