arnekeller/ripgrep-all
1
0
Fork 0
mirror of https://github.com/FliegendeWurst/ripgrep-all.git synced 2024-11-10 06:50:38 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #148 from lafrenierejm/nix-flake

Browse Source 
Add Nix Flake and direnv configs and instructions
This commit is contained in:
phiresky 2022-12-21 13:49:20 +01:00 committed by GitHub
commit 0d75d5bcc2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 446 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.envrc Normal file
View File

@ -0,0 +1 @@
use flake

2
.gitignore vendored
View File

@ -1,5 +1,7 @@
/result
/target
/exampledir.2
/.idea
/.pre-commit-config.yaml
/.vscode/settings.json
**/*.rs.bk

12
README.md
View File

@ -282,3 +282,15 @@ Also remember to disable caching with `--rga-no-cache` or clear the cache
(`~/Library/Caches/rga` on macOS, `~/.cache/rga` on other Unixes,
or `C:\Users\username\AppData\Local\rga` on Windows)
to debug the adapters.
### Nix and Direnv
You can use the provided [`flake.nix`](./flake.nix) to setup all build- and
run-time dependencies:
1. Enable [Flakes](https://nixos.wiki/wiki/Flakes) in your Nix configuration.
1. Add [`direnv`](https://direnv.net/) to your profile:
`nix profile install nixpkgs#direnv`
1. `cd` into the directory where you have cloned this directory.
1. Allow use of [`.envrc`](./.envrc): `direnv allow`
1. After the dependencies have been installed, your shell will now have all of
the necessary development dependencies.

271
flake.lock Normal file
View File

@ -0,0 +1,271 @@
{
"nodes": {
"advisory-db": {
"flake": false,
"locked": {
"lastModified": 1670452192,
"narHash": "sha256-f8NIFbqSbCzpssgDUK4srfgKaVaMhDScEptw4uuxGAc=",
"owner": "rustsec",
"repo": "advisory-db",
"rev": "0a2faeb87195392b23333a8097309d29f2c5d31d",
"type": "github"
},
"original": {
"owner": "rustsec",
"repo": "advisory-db",
"type": "github"
}
},
"crane": {
"inputs": {
"flake-compat": "flake-compat",
"flake-utils": "flake-utils",
"nixpkgs": [
"nixpkgs"
],
"rust-overlay": "rust-overlay"
},
"locked": {
"lastModified": 1670546681,
"narHash": "sha256-S33bhME0zPHPEZyZPCsrdQL/4WW/A020PwN+a3z7Q+I=",
"owner": "ipetkov",
"repo": "crane",
"rev": "63f80ee278897e72a1468090278716b5befa5128",
"type": "github"
},
"original": {
"owner": "ipetkov",
"repo": "crane",
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1668681692,
"narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "009399224d5e398d03b22badca40a37ac85412a1",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_2": {
"flake": false,
"locked": {
"lastModified": 1668681692,
"narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "009399224d5e398d03b22badca40a37ac85412a1",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-utils": {
"locked": {
"lastModified": 1667395993,
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_2": {
"locked": {
"lastModified": 1667395993,
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_3": {
"locked": {
"lastModified": 1667395993,
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"gitignore": {
"inputs": {
"nixpkgs": [
"pre-commit-hooks",
"nixpkgs"
]
},
"locked": {
"lastModified": 1660459072,
"narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
"owner": "hercules-ci",
"repo": "gitignore.nix",
"rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "gitignore.nix",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1670525689,
"narHash": "sha256-YIjGzxrRQa5LYO0zlnH/ardcwXsRgsnHe3TkGkvCxbc=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "f21f11aa2a02cb78651c6d57546c7d7541f9240c",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-stable": {
"locked": {
"lastModified": 1668984258,
"narHash": "sha256-0gDMJ2T3qf58xgcSbYoXiRGUkPWmKyr5C3vcathWhKs=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "cf63ade6f74bbc9d2a017290f1b2e33e8fbfa70a",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.05",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1668994630,
"narHash": "sha256-1lqx6HLyw6fMNX/hXrrETG1vMvZRGm2XVC9O/Jt0T6c=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "af50806f7c6ab40df3e6b239099e8f8385f6c78b",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"pre-commit-hooks": {
"inputs": {
"flake-compat": "flake-compat_2",
"flake-utils": "flake-utils_3",
"gitignore": "gitignore",
"nixpkgs": "nixpkgs_2",
"nixpkgs-stable": "nixpkgs-stable"
},
"locked": {
"lastModified": 1670413394,
"narHash": "sha256-M7sWqrKtOqUv9euX1t3HCxis8cPy9MNiZxQmUf0KF1o=",
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"rev": "1303a1a76e9eb074075bfe566518c413f6fc104e",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "pre-commit-hooks.nix",
"type": "github"
}
},
"root": {
"inputs": {
"advisory-db": "advisory-db",
"crane": "crane",
"flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs",
"pre-commit-hooks": "pre-commit-hooks",
"rust-overlay": "rust-overlay_2"
}
},
"rust-overlay": {
"inputs": {
"flake-utils": [
"crane",
"flake-utils"
],
"nixpkgs": [
"crane",
"nixpkgs"
]
},
"locked": {
"lastModified": 1670034122,
"narHash": "sha256-EqmuOKucPWtMvCZtHraHr3Q3bgVszq1x2PoZtQkUuEk=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "a0d5773275ecd4f141d792d3a0376277c0fc0b65",
"type": "github"
},
"original": {
"owner": "oxalica",
"repo": "rust-overlay",
"type": "github"
}
},
"rust-overlay_2": {
"inputs": {
"flake-utils": [
"flake-utils"
],
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1670552927,
"narHash": "sha256-lCE51eAGrAFS4k9W5aDGFpVtOAwQQ/rFMN80PCDh0vo=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "a0fdafd18c9cf599fde17fbaf07dbb20fa57eecb",
"type": "github"
},
"original": {
"owner": "oxalica",
"repo": "rust-overlay",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

123
flake.nix Normal file
View File

@ -0,0 +1,123 @@
{
description =
"ripgrep, but also search in PDFs, E-Books, Office documents, zip, tar.gz, etc.";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
crane = {
url = "github:ipetkov/crane";
inputs.nixpkgs.follows = "nixpkgs";
};
flake-utils.url = "github:numtide/flake-utils";
rust-overlay = {
url = "github:oxalica/rust-overlay";
inputs = {
nixpkgs.follows = "nixpkgs";
flake-utils.follows = "flake-utils";
};
};
advisory-db = {
url = "github:rustsec/advisory-db";
flake = false;
};
pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix";
};
outputs = { self, nixpkgs, crane, flake-utils, rust-overlay, advisory-db
, pre-commit-hooks }:
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = import nixpkgs {
inherit system;
overlays = [ (import rust-overlay) ];
};
inherit (pkgs) lib;
craneLib = crane.lib.${system};
src = craneLib.cleanCargoSource ./.;
buildInputs = with pkgs;
[ ffmpeg imagemagick pandoc poppler_utils ripgrep tesseract ]
++ lib.optionals pkgs.stdenv.isDarwin [
# Additional darwin specific inputs can be set here
pkgs.libiconv
];
# Build *just* the cargo dependencies, so we can reuse
# all of that work (e.g. via cachix) when running in CI
cargoArtifacts = craneLib.buildDepsOnly { inherit src buildInputs; };
# Build the actual crate itself, reusing the dependency
# artifacts from above.
rga = craneLib.buildPackage {
inherit cargoArtifacts src buildInputs;
doCheck = false;
};
pre-commit = pre-commit-hooks.lib."${system}".run;
in {
checks = {
# Build the crate as part of `nix flake check` for convenience
inherit rga;
# Run clippy (and deny all warnings) on the crate source,
# again, resuing the dependency artifacts from above.
#
# Note that this is done as a separate derivation so that
# we can block the CI if there are issues here, but not
# prevent downstream consumers from building our crate by itself.
rga-clippy = craneLib.cargoClippy {
inherit cargoArtifacts src buildInputs;
cargoClippyExtraArgs = "--all-targets -- --deny warnings";
};
rga-doc = craneLib.cargoDoc { inherit cargoArtifacts src; };
# Check formatting
rga-fmt = craneLib.cargoFmt { inherit src; };
# Audit dependencies
rga-audit = craneLib.cargoAudit { inherit src advisory-db; };
# Run tests with cargo-nextest.
rga-nextest = craneLib.cargoNextest {
inherit cargoArtifacts src buildInputs;
partitions = 1;
partitionType = "count";
};
inherit pre-commit;
pre-commit-check = pre-commit {
src = ./.;
hooks = {
nixfmt.enable = true;
rustfmt.enable = true;
cargo-check.enable = true;
};
};
} // lib.optionalAttrs (system == "x86_64-linux") {
# NB: cargo-tarpaulin only supports x86_64 systems
# Check code coverage (note: this will not upload coverage anywhere)
rga-coverage =
craneLib.cargoTarpaulin { inherit cargoArtifacts src; };
};
# `nix build`
packages.default = rga;
# `nix run`
apps.default = flake-utils.lib.mkApp { drv = rga; };
# `nix develop`
devShells.default = pkgs.mkShell {
inherit (self.checks.${system}.pre-commit-check) shellHook;
inputsFrom = builtins.attrValues self.checks;
buildInputs = buildInputs;
};
});
}

4
src/adapters.rs
View File

@ -17,13 +17,13 @@ use custom::CustomAdapterConfig;
use log::*;
use tokio::io::AsyncRead;
use core::fmt::Debug;
use std::borrow::Cow;
use std::collections::HashMap;
use std::iter::Iterator;
use std::path::PathBuf;
use std::pin::Pin;
use std::rc::Rc;
use core::fmt::Debug;
pub type ReadBox = Pin<Box<dyn AsyncRead + Send>>;
pub struct AdapterMeta {
@ -78,7 +78,7 @@ impl AdapterMeta {
pub trait GetMetadata {
fn metadata(&self) -> &AdapterMeta;
}
pub trait FileAdapter: GetMetadata + Send + Sync{
pub trait FileAdapter: GetMetadata + Send + Sync {
/// adapt a file.
///
/// detection_reason is the Matcher that was used to identify this file. Unless --rga-accurate was given, it is always a FastMatcher

37
src/adapters/postproc.rs
View File

@ -4,19 +4,17 @@
use anyhow::Context;
use anyhow::Result;
use async_stream::stream;
use bytes::Bytes;
use encoding_rs_io::DecodeReaderBytesBuilder;
use tokio::io::{AsyncRead, AsyncReadExt};
use async_stream::stream;
use tokio_util::io::ReaderStream;
use tokio_util::io::StreamReader;
use std::cmp::min;
use std::io::Cursor;
use std::pin::Pin;
use std::{
cmp::min,
};
use tokio::io::{AsyncRead, AsyncReadExt};
use tokio_util::io::ReaderStream;
use tokio_util::io::StreamReader;
use crate::adapted_iter::{AdaptedFilesIterBox};
use crate::adapted_iter::AdaptedFilesIterBox;
use super::{AdaptInfo, AdapterMeta, FileAdapter, GetMetadata};
@ -177,18 +175,30 @@ pub fn postproc_pagebreaks(line_prefix: &str, inp: impl AsyncRead) -> impl Async
mod tests {
use super::*;
use anyhow::Result;
use tokio::pin;
use std::io::Read;
use tokio::pin;
async fn test_from_strs(pagebreaks: bool, line_prefix: &str, a: &'static str, b: &str) -> Result<()> {
async fn test_from_strs(
pagebreaks: bool,
line_prefix: &str,
a: &'static str,
b: &str,
) -> Result<()> {
test_from_bytes(pagebreaks, line_prefix, a.as_bytes(), b).await
}
async fn test_from_bytes(pagebreaks: bool, line_prefix: &str, a: &'static [u8], b: &str) -> Result<()> {
async fn test_from_bytes(
pagebreaks: bool,
line_prefix: &str,
a: &'static [u8],
b: &str,
) -> Result<()> {
let mut oup = Vec::new();
let inp = postproc_encoding("", a)?;
if pagebreaks {
postproc_pagebreaks(line_prefix, inp).read_to_end(&mut oup).await?;
postproc_pagebreaks(line_prefix, inp)
.read_to_end(&mut oup)
.await?;
} else {
let x = postproc_prefix(line_prefix, inp);
pin!(x);
@ -231,7 +241,8 @@ mod tests {
"foo:",
"this is a test \n\n \0 foo",
"foo:[rga: binary data]",
).await?;
)
.await?;
test_from_strs(false, "foo:", "\0", "foo:[rga: binary data]").await?;
Ok(())

1
src/adapters/spawning.rs
View File

@ -1,4 +1,3 @@
use super::*;
use anyhow::Result;
use async_stream::{stream, AsyncStream};

6
src/bin/rga-preproc.rs
View File

@ -5,7 +5,7 @@ use ripgrep_all as rga;
use anyhow::Context;
use log::debug;
use std::{time::Instant};
use std::time::Instant;
use tokio::fs::File;
#[tokio::main]
@ -20,7 +20,9 @@ async fn main() -> anyhow::Result<()> {
std::env::current_dir()?.join(&filepath)
};
let i = File::open(&path).await.context("Specified input file not found")?;
let i = File::open(&path)
.await
.context("Specified input file not found")?;
let mut o = tokio::io::stdout();
let ai = AdaptInfo {
inp: Box::pin(i),

4
src/caching_writer.rs
View File

@ -17,11 +17,11 @@ use crate::adapters::ReadBox;
* unless more than max_cache_size bytes is written, then the cache is dropped and it is pure passthrough.
*/
pub fn async_read_and_write_to_cache<'a>(
inp: impl AsyncRead + Send +'a,
inp: impl AsyncRead + Send + 'a,
max_cache_size: usize,
compression_level: i32,
on_finish: Box<dyn FnOnce((u64, Option<Vec<u8>>)) -> Result<()> + Send>,
) -> Result<Pin<Box<dyn AsyncRead + Send +'a>>> {
) -> Result<Pin<Box<dyn AsyncRead + Send + 'a>>> {
let inp = Box::pin(inp);
let mut zstd_writer = Some(ZstdEncoder::with_quality(
Vec::new(),

4
src/config.rs
View File

@ -1,5 +1,5 @@
use crate::{adapters::custom::CustomAdapterConfig, project_dirs};
use anyhow::{Result, Context};
use anyhow::{Context, Result};
use derive_more::FromStr;
use log::*;
use schemars::JsonSchema;
@ -237,7 +237,7 @@ pub struct CacheConfig {
default_value,
long = "--rga-cache-path",
hidden_short_help = true,
require_equals = true,
require_equals = true
)]
pub path: CachePath,
}

2
src/preproc.rs
View File

@ -13,11 +13,11 @@ use async_compression::tokio::bufread::ZstdDecoder;
use async_stream::stream;
use log::*;
use path_clean::PathClean;
use std::sync::Arc;
use postproc::PostprocPrefix;
use std::convert::TryInto;
use std::io::Cursor;
use std::path::Path;
use std::sync::Arc;
use tokio::io::AsyncBufRead;
use tokio::io::AsyncBufReadExt;
use tokio::io::BufReader;

7
src/recurse.rs
View File

@ -3,10 +3,7 @@ use tokio_util::io::{ReaderStream, StreamReader};
use crate::{adapted_iter::AdaptedFilesIterBox, adapters::*};
use async_stream::stream;
pub fn concat_read_streams(
input: AdaptedFilesIterBox,
) -> ReadBox {
pub fn concat_read_streams(input: AdaptedFilesIterBox) -> ReadBox {
let s = stream! {
for await output in input {
let stream = ReaderStream::new(output.inp);
@ -16,4 +13,4 @@ pub fn concat_read_streams(
}
};
Box::pin(StreamReader::new(s))
}
}