Merge pull request #148 from lafrenierejm/nix-flake

Add Nix Flake and direnv configs and instructions

.envrc

@@ -0,0 +1 @@
+use flake

.gitignore

@@ -1,5 +1,7 @@
+/result
 /target
 /exampledir.2
 /.idea
+/.pre-commit-config.yaml
 /.vscode/settings.json
 **/*.rs.bk

README.md

@@ -282,3 +282,15 @@ Also remember to disable caching with `--rga-no-cache` or clear the cache
 (`~/Library/Caches/rga` on macOS, `~/.cache/rga` on other Unixes,
 or `C:\Users\username\AppData\Local\rga` on Windows)
 to debug the adapters.
+
+### Nix and Direnv
+
+You can use the provided [`flake.nix`](./flake.nix) to setup all build- and
+run-time dependencies:
+1. Enable [Flakes](https://nixos.wiki/wiki/Flakes) in your Nix configuration.
+1. Add [`direnv`](https://direnv.net/) to your profile:
+   `nix profile install nixpkgs#direnv`
+1. `cd` into the directory where you have cloned this directory.
+1. Allow use of [`.envrc`](./.envrc): `direnv allow`
+1. After the dependencies have been installed, your shell will now have all of
+   the necessary development dependencies.

flake.lock

@@ -0,0 +1,271 @@
+{
+  "nodes": {
+    "advisory-db": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1670452192,
+        "narHash": "sha256-f8NIFbqSbCzpssgDUK4srfgKaVaMhDScEptw4uuxGAc=",
+        "owner": "rustsec",
+        "repo": "advisory-db",
+        "rev": "0a2faeb87195392b23333a8097309d29f2c5d31d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rustsec",
+        "repo": "advisory-db",
+        "type": "github"
+      }
+    },
+    "crane": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1670546681,
+        "narHash": "sha256-S33bhME0zPHPEZyZPCsrdQL/4WW/A020PwN+a3z7Q+I=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "63f80ee278897e72a1468090278716b5befa5128",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1668681692,
+        "narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "009399224d5e398d03b22badca40a37ac85412a1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1668681692,
+        "narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "009399224d5e398d03b22badca40a37ac85412a1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1660459072,
+        "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1670525689,
+        "narHash": "sha256-YIjGzxrRQa5LYO0zlnH/ardcwXsRgsnHe3TkGkvCxbc=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "f21f11aa2a02cb78651c6d57546c7d7541f9240c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1668984258,
+        "narHash": "sha256-0gDMJ2T3qf58xgcSbYoXiRGUkPWmKyr5C3vcathWhKs=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "cf63ade6f74bbc9d2a017290f1b2e33e8fbfa70a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-22.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1668994630,
+        "narHash": "sha256-1lqx6HLyw6fMNX/hXrrETG1vMvZRGm2XVC9O/Jt0T6c=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "af50806f7c6ab40df3e6b239099e8f8385f6c78b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": "flake-compat_2",
+        "flake-utils": "flake-utils_3",
+        "gitignore": "gitignore",
+        "nixpkgs": "nixpkgs_2",
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1670413394,
+        "narHash": "sha256-M7sWqrKtOqUv9euX1t3HCxis8cPy9MNiZxQmUf0KF1o=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "1303a1a76e9eb074075bfe566518c413f6fc104e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "advisory-db": "advisory-db",
+        "crane": "crane",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks": "pre-commit-hooks",
+        "rust-overlay": "rust-overlay_2"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "flake-utils": [
+          "crane",
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "crane",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1670034122,
+        "narHash": "sha256-EqmuOKucPWtMvCZtHraHr3Q3bgVszq1x2PoZtQkUuEk=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "a0d5773275ecd4f141d792d3a0376277c0fc0b65",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
+    "rust-overlay_2": {
+      "inputs": {
+        "flake-utils": [
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1670552927,
+        "narHash": "sha256-lCE51eAGrAFS4k9W5aDGFpVtOAwQQ/rFMN80PCDh0vo=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "a0fdafd18c9cf599fde17fbaf07dbb20fa57eecb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,123 @@
+{
+  description =
+    "ripgrep, but also search in PDFs, E-Books, Office documents, zip, tar.gz, etc.";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+
+    crane = {
+      url = "github:ipetkov/crane";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+
+    flake-utils.url = "github:numtide/flake-utils";
+
+    rust-overlay = {
+      url = "github:oxalica/rust-overlay";
+      inputs = {
+        nixpkgs.follows = "nixpkgs";
+        flake-utils.follows = "flake-utils";
+      };
+    };
+
+    advisory-db = {
+      url = "github:rustsec/advisory-db";
+      flake = false;
+    };
+
+    pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix";
+  };
+
+  outputs = { self, nixpkgs, crane, flake-utils, rust-overlay, advisory-db
+    , pre-commit-hooks }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = import nixpkgs {
+          inherit system;
+          overlays = [ (import rust-overlay) ];
+        };
+        inherit (pkgs) lib;
+
+        craneLib = crane.lib.${system};
+        src = craneLib.cleanCargoSource ./.;
+
+        buildInputs = with pkgs;
+          [ ffmpeg imagemagick pandoc poppler_utils ripgrep tesseract ]
+          ++ lib.optionals pkgs.stdenv.isDarwin [
+            # Additional darwin specific inputs can be set here
+            pkgs.libiconv
+          ];
+
+        # Build *just* the cargo dependencies, so we can reuse
+        # all of that work (e.g. via cachix) when running in CI
+        cargoArtifacts = craneLib.buildDepsOnly { inherit src buildInputs; };
+
+        # Build the actual crate itself, reusing the dependency
+        # artifacts from above.
+        rga = craneLib.buildPackage {
+          inherit cargoArtifacts src buildInputs;
+          doCheck = false;
+        };
+
+        pre-commit = pre-commit-hooks.lib."${system}".run;
+      in {
+        checks = {
+          # Build the crate as part of `nix flake check` for convenience
+          inherit rga;
+
+          # Run clippy (and deny all warnings) on the crate source,
+          # again, resuing the dependency artifacts from above.
+          #
+          # Note that this is done as a separate derivation so that
+          # we can block the CI if there are issues here, but not
+          # prevent downstream consumers from building our crate by itself.
+          rga-clippy = craneLib.cargoClippy {
+            inherit cargoArtifacts src buildInputs;
+            cargoClippyExtraArgs = "--all-targets -- --deny warnings";
+          };
+
+          rga-doc = craneLib.cargoDoc { inherit cargoArtifacts src; };
+
+          # Check formatting
+          rga-fmt = craneLib.cargoFmt { inherit src; };
+
+          # Audit dependencies
+          rga-audit = craneLib.cargoAudit { inherit src advisory-db; };
+
+          # Run tests with cargo-nextest.
+          rga-nextest = craneLib.cargoNextest {
+            inherit cargoArtifacts src buildInputs;
+            partitions = 1;
+            partitionType = "count";
+          };
+
+          inherit pre-commit;
+          pre-commit-check = pre-commit {
+            src = ./.;
+            hooks = {
+              nixfmt.enable = true;
+              rustfmt.enable = true;
+              cargo-check.enable = true;
+            };
+          };
+        } // lib.optionalAttrs (system == "x86_64-linux") {
+          # NB: cargo-tarpaulin only supports x86_64 systems
+          # Check code coverage (note: this will not upload coverage anywhere)
+          rga-coverage =
+            craneLib.cargoTarpaulin { inherit cargoArtifacts src; };
+        };
+
+        # `nix build`
+        packages.default = rga;
+
+        # `nix run`
+        apps.default = flake-utils.lib.mkApp { drv = rga; };
+
+        # `nix develop`
+        devShells.default = pkgs.mkShell {
+          inherit (self.checks.${system}.pre-commit-check) shellHook;
+          inputsFrom = builtins.attrValues self.checks;
+          buildInputs = buildInputs;
+        };
+      });
+}

src/adapters.rs

@@ -17,13 +17,13 @@ use custom::CustomAdapterConfig;
 use log::*;
 use tokio::io::AsyncRead;
 
+use core::fmt::Debug;
 use std::borrow::Cow;
 use std::collections::HashMap;
 use std::iter::Iterator;
 use std::path::PathBuf;
 use std::pin::Pin;
 use std::rc::Rc;
-use core::fmt::Debug;
 
 pub type ReadBox = Pin<Box<dyn AsyncRead + Send>>;
 pub struct AdapterMeta {
@@ -78,7 +78,7 @@ impl AdapterMeta {
 pub trait GetMetadata {
     fn metadata(&self) -> &AdapterMeta;
 }
-pub trait FileAdapter: GetMetadata + Send + Sync{
+pub trait FileAdapter: GetMetadata + Send + Sync {
     /// adapt a file.
     ///
     /// detection_reason is the Matcher that was used to identify this file. Unless --rga-accurate was given, it is always a FastMatcher

src/adapters/postproc.rs

@@ -4,19 +4,17 @@
 
 use anyhow::Context;
 use anyhow::Result;
+use async_stream::stream;
 use bytes::Bytes;
 use encoding_rs_io::DecodeReaderBytesBuilder;
-use tokio::io::{AsyncRead, AsyncReadExt};
+use std::cmp::min;
-use async_stream::stream;
-use tokio_util::io::ReaderStream;
-use tokio_util::io::StreamReader;
 use std::io::Cursor;
 use std::pin::Pin;
-use std::{
+use tokio::io::{AsyncRead, AsyncReadExt};
-    cmp::min,
+use tokio_util::io::ReaderStream;
-};
+use tokio_util::io::StreamReader;
 
-use crate::adapted_iter::{AdaptedFilesIterBox};
+use crate::adapted_iter::AdaptedFilesIterBox;
 
 use super::{AdaptInfo, AdapterMeta, FileAdapter, GetMetadata};
 
@@ -177,18 +175,30 @@ pub fn postproc_pagebreaks(line_prefix: &str, inp: impl AsyncRead) -> impl Async
 mod tests {
     use super::*;
     use anyhow::Result;
-    use tokio::pin;
     use std::io::Read;
+    use tokio::pin;
 
-    async fn test_from_strs(pagebreaks: bool, line_prefix: &str, a: &'static str, b: &str) -> Result<()> {
+    async fn test_from_strs(
+        pagebreaks: bool,
+        line_prefix: &str,
+        a: &'static str,
+        b: &str,
+    ) -> Result<()> {
         test_from_bytes(pagebreaks, line_prefix, a.as_bytes(), b).await
     }
 
-    async fn test_from_bytes(pagebreaks: bool, line_prefix: &str, a: &'static [u8], b: &str) -> Result<()> {
+    async fn test_from_bytes(
+        pagebreaks: bool,
+        line_prefix: &str,
+        a: &'static [u8],
+        b: &str,
+    ) -> Result<()> {
         let mut oup = Vec::new();
         let inp = postproc_encoding("", a)?;
         if pagebreaks {
-            postproc_pagebreaks(line_prefix, inp).read_to_end(&mut oup).await?;
+            postproc_pagebreaks(line_prefix, inp)
+                .read_to_end(&mut oup)
+                .await?;
         } else {
             let x = postproc_prefix(line_prefix, inp);
             pin!(x);
@@ -231,7 +241,8 @@ mod tests {
             "foo:",
             "this is a test \n\n \0 foo",
             "foo:[rga: binary data]",
-        ).await?;
+        )
+        .await?;
         test_from_strs(false, "foo:", "\0", "foo:[rga: binary data]").await?;
 
         Ok(())

src/adapters/spawning.rs

@@ -1,4 +1,3 @@
-
 use super::*;
 use anyhow::Result;
 use async_stream::{stream, AsyncStream};

src/bin/rga-preproc.rs

@@ -5,7 +5,7 @@ use ripgrep_all as rga;
 
 use anyhow::Context;
 use log::debug;
-use std::{time::Instant};
+use std::time::Instant;
 use tokio::fs::File;
 
 #[tokio::main]
@@ -20,7 +20,9 @@ async fn main() -> anyhow::Result<()> {
         std::env::current_dir()?.join(&filepath)
     };
 
-    let i = File::open(&path).await.context("Specified input file not found")?;
+    let i = File::open(&path)
+        .await
+        .context("Specified input file not found")?;
     let mut o = tokio::io::stdout();
     let ai = AdaptInfo {
         inp: Box::pin(i),

src/caching_writer.rs

@@ -17,11 +17,11 @@ use crate::adapters::ReadBox;
  * unless more than max_cache_size bytes is written, then the cache is dropped and it is pure passthrough.
  */
 pub fn async_read_and_write_to_cache<'a>(
-    inp: impl AsyncRead + Send +'a,
+    inp: impl AsyncRead + Send + 'a,
     max_cache_size: usize,
     compression_level: i32,
     on_finish: Box<dyn FnOnce((u64, Option<Vec<u8>>)) -> Result<()> + Send>,
-) -> Result<Pin<Box<dyn AsyncRead + Send +'a>>> {
+) -> Result<Pin<Box<dyn AsyncRead + Send + 'a>>> {
     let inp = Box::pin(inp);
     let mut zstd_writer = Some(ZstdEncoder::with_quality(
         Vec::new(),

src/config.rs

@@ -1,5 +1,5 @@
 use crate::{adapters::custom::CustomAdapterConfig, project_dirs};
-use anyhow::{Result, Context};
+use anyhow::{Context, Result};
 use derive_more::FromStr;
 use log::*;
 use schemars::JsonSchema;
@@ -237,7 +237,7 @@ pub struct CacheConfig {
         default_value,
         long = "--rga-cache-path",
         hidden_short_help = true,
-        require_equals = true,
+        require_equals = true
     )]
     pub path: CachePath,
 }

src/preproc.rs

@@ -13,11 +13,11 @@ use async_compression::tokio::bufread::ZstdDecoder;
 use async_stream::stream;
 use log::*;
 use path_clean::PathClean;
-use std::sync::Arc;
 use postproc::PostprocPrefix;
 use std::convert::TryInto;
 use std::io::Cursor;
 use std::path::Path;
+use std::sync::Arc;
 use tokio::io::AsyncBufRead;
 use tokio::io::AsyncBufReadExt;
 use tokio::io::BufReader;

src/recurse.rs

@@ -3,10 +3,7 @@ use tokio_util::io::{ReaderStream, StreamReader};
 use crate::{adapted_iter::AdaptedFilesIterBox, adapters::*};
 use async_stream::stream;
 
-
+pub fn concat_read_streams(input: AdaptedFilesIterBox) -> ReadBox {
-pub fn concat_read_streams(
-    input: AdaptedFilesIterBox,
-) -> ReadBox {
     let s = stream! {
         for await output in input {
             let stream = ReaderStream::new(output.inp);