Merge pull request #16 from phiresky/windows

Windows support and fixes

.travis.yml

@@ -1,4 +1,5 @@
 language: rust
+cache: cargo
 dist: xenial
 env:
     global:
@@ -36,30 +37,6 @@ matrix:
           rust: nightly
           # XML_CATALOG_FILES is apparently necessary for asciidoc on macOS.
           env: TARGET=x86_64-apple-darwin XML_CATALOG_FILES=/usr/local/etc/xml/catalog
-        #- os: linux
-        #  rust: nightly
-        #  env: TARGET=arm-unknown-linux-gnueabihf GCC_VERSION=4.8
-        #  addons:
-        #    apt:
-        #      packages:
-        #        - gcc-4.8-arm-linux-gnueabihf
-        #        - binutils-arm-linux-gnueabihf
-        #        - libc6-armhf-cross
-        #        - libc6-dev-armhf-cross
-        #        # For generating man page.
-        #        - libxslt1-dev
-        #        - asciidoc
-        #        - docbook-xsl
-        #        - xsltproc
-        #        - libxml2-utils
-        # Beta channel. We enable these to make sure there are no regressions in
-        # Rust beta releases.
-        - os: linux
-          rust: beta
-          env: TARGET=x86_64-unknown-linux-musl
-        - os: linux
-          rust: beta
-          env: TARGET=x86_64-unknown-linux-gnu
         # Minimum Rust supported channel. We enable these to make sure ripgrep
         # continues to work on the advertised minimum Rust version.
         - os: linux
@@ -84,17 +61,19 @@ matrix:
         #        - docbook-xsl
         #        - xsltproc
         #        - libxml2-utils
+        - os: windows
+          rust: nightly
+          env: TARGET=x86_64-pc-windows-msvc
 install: ci/install.sh
 script: ci/script.sh
 before_deploy: ci/before_deploy.sh
 deploy:
     provider: releases
     file_glob: true
-    file: deployment/${PROJECT_NAME}-${TRAVIS_TAG}-${TARGET}.tar.gz
+    file: deployment/${PROJECT_NAME}-${TRAVIS_TAG}-${TARGET}.*
     skip_cleanup: true
     on:
         condition: $TRAVIS_RUST_VERSION = nightly
-        branch: master # i guess we do need this after all?
         tags: true
     api_key:
         secure: BACN/S5fv590pb0bq7rUNX553yxrUDk3ZDcQ9JcG0DXFkv3/XIRzoc88YbDAp065Yk9DJ8SrT9X/Al9l8/MKoLfVmcz3jdFUJAsppkO4fouOxdtiufYq5MFFlrP6SYMHy7Gx95lZsh++NSwbq+1fjMkESOTWqz1ptUmAa7ERecGuX4t/4QoPfyTYNWb84LLDqeD9Geabj7HuCsjsa4gdJUFew13zvE1SFRUkVPVGo09j/+fYVZRoY0ObqHVYQEOlj4HtHHjaYnsLgKHcGGigc252N15sm7zFM0+/lRMYIx7LLv6SmFc/eHqHy8D9gHzUx1tEYnScuNEDDHSS7hXfKGyTrOPfENAqzWCIAftveUVI/+rWmMIOWDxzxt5s9P9k9c93GP7L2L/HJlEZfr0UKQhHfcv/uOaS48vhh4WyzybgBRLFcNwrGKSoHgJhGRHdnQbTc8JMNPBEoJKdtSosQk56ZievMg2rDxF/GArpLTTe58+kMrhFbK9bGBP0YcZjaLnIZuuRf2LZQFtT9gSPFuvXjhwDaz/5a3Gp+9ZIEF50Ad/nf2xKf/rwFaFFJ0RkPdw20TCuerasQUnbpDmZRYnJNrdVlYKIIxQ6dF+PqO8/4RJ1tRYaWk79G8fDVRWp/IfvtduLKqgh/mjNgefOc9qf0DoBVxK7LJx5CHHomkE=
@@ -102,6 +81,7 @@ branches:
     only:
         # Pushes and PR to the master branch
         - master
+        - windows
         # Ruby regex to match tags. Required, or travis won't trigger deploys when
         # a new tag is pushed.
         - /^\d+\.\d+\.\d+.*$/

CHANGELOG.md

@@ -3,6 +3,8 @@
 -   Fix file ending regex ([#13](https://github.com/phiresky/ripgrep-all/issues/13))
 -   Fix decoding of UTF16 with BOM ([#5](https://github.com/phiresky/ripgrep-all/issues/5))
 -   Shorten the output on failure to two lines (https://github.com/phiresky/ripgrep-all/issues/7), you can use `--no-messages` to completely suppress errors.
+-   Better installations instructions in readme for each OS
+-   Add windows binaries! Including all dependencies!
 
 # 0.9.1 (2019-06-16)
 

Cargo.lock

@@ -305,6 +305,14 @@ dependencies = [
  "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "exitfailure"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "failure"
 version = "0.1.5"
@@ -939,6 +947,7 @@ dependencies = [
  "encoding_rs 0.8.17 (registry+https://github.com/rust-lang/crates.io-index)",
  "encoding_rs_io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "env_logger 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "exitfailure 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "flate2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1425,6 +1434,7 @@ dependencies = [
 "checksum encoding_rs 0.8.17 (registry+https://github.com/rust-lang/crates.io-index)" = "4155785c79f2f6701f185eb2e6b4caf0555ec03477cb4c70db67b465311620ed"
 "checksum encoding_rs_io 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9619ee7a2bf4e777e020b95c1439abaf008f8ea8041b78a0552c4f1bcf4df32c"
 "checksum env_logger 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b61fa891024a945da30a9581546e8cfaf5602c7b3f4c137a2805cf388f92075a"
+"checksum exitfailure 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2ff5bd832af37f366c6c194d813a11cd90ac484f124f079294f28e357ae40515"
 "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2"
 "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1"
 "checksum fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"

Cargo.toml

@@ -44,3 +44,4 @@ structopt = "0.2.16"
 paste = "0.1.5"
 tempfile = "3.0.8"
 glob = "0.3.0"
+exitfailure = "0.5.1"

README.md

@@ -28,6 +28,44 @@ demo/
 
 ![rga output](doc/demodir.png)
 
+## INSTALLATION
+
+Linux x64, OSX and Windows binaries are available [in GitHub Releases][latestrelease].
+
+[latestrelease]: https://github.com/phiresky/ripgrep-all/releases/latest
+
+### Linux
+
+On Arch Linux, you can simply install from AUR: `yay -S ripgrep-all`.
+
+On Debian-based distributions you can download the [rga binary][latestrelease] and get the dependencies like this:
+
+`apt install ripgrep pandoc poppler-utils ffmpeg cargo`
+
+If ripgrep is not included in your package sources, get it from [here](https://github.com/BurntSushi/ripgrep/releases).
+
+rga will search for all binaries it calls in \$PATH and the directory itself is in.
+
+### Windows
+
+Just unzip the [Windows binary release][latestrelease] anywhere, possibly somewhere in your \$PATH. It includes all necessary and optional dependencies.
+
+### OSX
+
+To get all necessary and optional dependencies:
+
+`brew install ripgrep pandoc poppler tesseract ffmpeg`
+
+### Compile from source
+
+rga should compile with stable Rust (v1.35.0+, check with `rustc --version`). To build it, run the following (or the equivalent in your OS):
+
+```
+   ~$ apt install build-essential pandoc poppler-utils ffmpeg ripgrep cargo
+   ~$ cargo install ripgrep_all
+   ~$ rga --version    # this should work now
+```
+
 ## Available Adapters
 
 ```
@@ -100,23 +138,11 @@ The following adapters are disabled by default, and can be enabled using '--rga-
 
     Extensions: .jpg, .png
 
-## INSTALLATION:
+## USAGE
-
-rga should compile with stable Rust.  To build it, run the following (or the equivalent in your OS):
-
-```
-   ~$ apt install build-essential pandoc poppler-utils ffmpeg ripgrep cargo
-   ~$ cargo install ripgrep_all
-   ~$ rga --version    # this should work now
-```
-
-You could do `cargo build`, instead of `cargo install ripgrep_all`, to just build rga in the local tree.
-
-## USAGE:
 
 > rga \[FLAGS\] \[OPTIONS\] PATTERN \[PATH ...\]
 
-## FLAGS:
+## FLAGS
 
 **\--rga-accurate**
 

appveyor.yml

@@ -1,83 +0,0 @@
-# adapted from https://raw.githubusercontent.com/BurntSushi/ripgrep/master/appveyor.yml
-cache:
-    - c:\cargo\registry
-    - c:\cargo\git
-
-init:
-    - mkdir c:\cargo
-    - mkdir c:\rustup
-    - SET PATH=c:\cargo\bin;%PATH%
-
-clone_folder: c:\projects\ripgrep_all
-
-environment:
-    CARGO_HOME: "c:\\cargo"
-    RUSTUP_HOME: "c:\\rustup"
-    CARGO_TARGET_DIR: "c:\\projects\\ripgrep_all\\target"
-    global:
-        PROJECT_NAME: ripgrep_all
-        RUST_BACKTRACE: full
-    matrix:
-        - TARGET: x86_64-pc-windows-gnu
-          CHANNEL: stable
-          BITS: 64
-          MSYS2: 1
-        - TARGET: x86_64-pc-windows-msvc
-          CHANNEL: stable
-          BITS: 64
-        - TARGET: i686-pc-windows-gnu
-          CHANNEL: stable
-          BITS: 32
-          MSYS2: 1
-        - TARGET: i686-pc-windows-msvc
-          CHANNEL: stable
-          BITS: 32
-
-matrix:
-    fast_finish: true
-
-# Install Rust and Cargo
-# (Based on from https://github.com/rust-lang/libc/blob/master/appveyor.yml)
-install:
-    - curl -sSf -o rustup-init.exe https://win.rustup.rs/
-    - rustup-init.exe -y --default-host %TARGET%
-    - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin
-    - if defined MSYS2 set PATH=C:\msys64\mingw%BITS%\bin;%PATH%
-    - rustc -V
-    - cargo -V
-
-# Hack to work around a harmless warning in Appveyor builds?
-build: false
-
-# Equivalent to Travis' `script` phase
-test_script:
-    - cargo test --verbose --all
-
-before_deploy:
-    # Generate artifacts for release
-    - cargo build --release
-    - mkdir staging
-    - copy target\release\rga.exe staging
-    - copy target\release\rga-preproc.exe staging
-    - ps: copy target\release\build\ripgrep*\out\_rg.ps1 staging
-    - cd staging
-    # release zipfile will look like 'ripgrep-1.2.3-x86_64-pc-windows-msvc'
-    - 7z a ../%PROJECT_NAME%-%APPVEYOR_REPO_TAG_NAME%-%TARGET%.zip *
-    - appveyor PushArtifact ../%PROJECT_NAME%-%APPVEYOR_REPO_TAG_NAME%-%TARGET%.zip
-
-deploy:
-    description: "Automatically deployed release"
-    # All the zipped artifacts will be deployed
-    artifact: /.*\.zip/
-    auth_token:
-        secure: vv4vBCEosGlyQjaEC1+kraP2P6O4CQSa+Tw50oHWFTGcmuXxaWS0/yEXbxsIRLpw
-    provider: GitHub
-    # deploy when a new tag is pushed and only on the stable channel
-    on:
-        CHANNEL: stable
-        appveyor_repo_tag: true
-
-branches:
-    only:
-        - /^\d+\.\d+\.\d+$/
-        - master

ci/before_deploy.sh

@@ -12,6 +12,38 @@ mk_artifacts() {
 	"$CARGO" build --target "$TARGET" --release
 }
 
+# run from tmpdir, put results in $1/
+# currently windows only, because other OS probably have a package manager
+# also currently just a fixed version of each tool since it doesn't matter much
+download_other_binaries() {
+    outdir="$1"
+    mkdir -p "$outdir/licenses" "$outdir/lib"
+
+    # ffmpeg
+    wget -q https://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-4.1.3-win64-static.zip -O ffmpeg.zip
+    unzip ffmpeg.zip
+    cp ffmpeg-*/bin/{ffmpeg,ffprobe}.exe "$outdir/lib"
+    cp ffmpeg-*/LICENSE.txt "$outdir/licenses/ffmpeg"
+
+    # poppler
+    wget -q https://blog.alivate.com.au/wp-content/uploads/2018/10/poppler-0.68.0_x86.7z -O poppler.7z
+    7z x poppler.7z
+    for f in pdftotext.exe libpoppler-79.dll libgcc_s_dw2-1.dll libstdc++-6.dll jpeg62.dll libpng16-16.dll libtiff3.dll zlib1.dll freetype6.dll libpoppler-79.dll; do
+        cp poppler-*/bin/"$f" "$outdir/lib"
+    done
+    cp poppler-*/bin/COPYING3 "$outdir/licenses/poppler"
+    
+    wget -q https://github.com/jgm/pandoc/releases/download/2.7.3/pandoc-2.7.3-windows-x86_64.zip -O pandoc.zip
+    unzip pandoc.zip
+    cp pandoc-*/pandoc.exe "$outdir/lib"
+    cp pandoc-*/COPYRIGHT.txt "$outdir/licenses/pandoc"
+
+    wget -q https://github.com/BurntSushi/ripgrep/releases/download/11.0.1/ripgrep-11.0.1-x86_64-pc-windows-msvc.zip -O ripgrep.zip
+    unzip ripgrep.zip
+    cp rg.exe "$outdir/lib"
+
+}
+
 mk_tarball() {
     # When cross-compiling, use the right `strip` tool on the binary.
     local gcc_prefix="$(gcc_prefix)"
@@ -20,7 +52,8 @@ mk_tarball() {
     local tmpdir="$(mktemp -d)"
     local name="${PROJECT_NAME}-${TRAVIS_TAG}-${TARGET}"
     local staging="$tmpdir/$name"
-    mkdir -p "$staging"/{complete,doc}
+    mkdir -p "$staging/"
+    # mkdir -p "$staging"/{complete,doc}
     # The deployment directory is where the final archive will reside.
     # This path is known by the .travis.yml configuration.
     local out_dir="$(pwd)/deployment"
@@ -29,10 +62,15 @@ mk_tarball() {
     # contains shell completion files and the man page.
     local cargo_out_dir="$(cargo_out_dir "target/$TARGET")"
 
+    bin_ext=""
+    if is_windows; then
+        bin_ext=".exe"
+    fi
+
     # Copy the binaries and strip it.
     for binary in rga rga-preproc; do
-        cp "target/$TARGET/release/$binary" "$staging/$binary"
+        cp "target/$TARGET/release/$binary$bin_ext" "$staging/$binary$bin_ext"
-        "${gcc_prefix}strip" "$staging/$binary"
+        # "${gcc_prefix}strip" "$staging/$binary"
     done
     # Copy the licenses and README.
     cp {README.md,LICENSE.md} "$staging/"
@@ -46,7 +84,12 @@ mk_tarball() {
     # cp "$cargo_out_dir"/{rg.bash,rg.fish,_rg.ps1} "$staging/complete/"
     # cp complete/_rg "$staging/complete/"
 
-    (cd "$tmpdir" && tar czf "$out_dir/$name.tar.gz" "$name")
+    if is_windows; then
+        (cd "$tmpdir" && download_other_binaries "$name")
+        (cd "$tmpdir" && 7z a "$out_dir/$name.7z" "$name")
+    else
+        (cd "$tmpdir" && tar czf "$out_dir/$name.tar.gz" "$name")
+    fi
     rm -rf "$tmpdir"
 }
 

ci/utils.sh

@@ -24,6 +24,8 @@ host() {
         osx)
             echo x86_64-apple-darwin
             ;;
+        windows)
+            echo x86_64-pc-windows-msvc
     esac
 }
 
@@ -97,6 +99,13 @@ is_osx() {
     esac
 }
 
+is_windows() {
+    case "$TRAVIS_OS_NAME" in
+        windows) return 0 ;;
+        *)   return 1 ;;
+    esac
+}
+
 builder() {
     if is_musl && is_x86_64; then
         cargo install cross

src/adapters/decompress.rs

@@ -131,7 +131,7 @@ mod tests {
             ("hi/test.hi.bz2", "hi/test.hi"),
             ("hello.tar.gz", "hello.tar"),
         ] {
-            assert_eq!(get_inner_filename(&PathBuf::from(a)).to_string_lossy(), *b);
+            assert_eq!(get_inner_filename(&PathBuf::from(a)), PathBuf::from(*b));
         }
     }
 }

src/bin/rga-preproc.rs

@@ -5,7 +5,7 @@ use ripgrep_all as rga;
 
 use std::fs::File;
 
-fn main() -> Fallible<()> {
+fn main() -> Result<(), exitfailure::ExitFailure> {
     env_logger::init();
     let mut arg_arr: Vec<std::ffi::OsString> = std::env::args_os().collect();
     let last = arg_arr.pop().expect("No filename specified");
@@ -32,12 +32,6 @@ fn main() -> Fallible<()> {
         archive_recursion_depth: 0,
         config: PreprocConfig { cache, args: &args },
     };
-
+    rga_preproc(ai)?;
-    match rga_preproc(ai) {
+    Ok(())
-        Ok(()) => Ok(()),
-        Err(e) => {
-            eprintln!("preproc error: {}", e);
-            std::process::exit(1);
-        }
-    }
 }

src/bin/rga.rs

@@ -8,7 +8,7 @@ use structopt::StructOpt;
 
 use std::process::Command;
 
-fn main() -> Fallible<()> {
+fn main() -> Result<(), exitfailure::ExitFailure> {
     env_logger::init();
 
     let (args, passthrough_args) = split_args()?;
@@ -85,8 +85,7 @@ fn main() -> Fallible<()> {
         "*".to_owned()
     };
 
-    let exe = std::env::current_exe().expect("Could not get executable location");
+    add_exe_to_path()?;
-    let preproc_exe = exe.with_file_name("rga-preproc");
 
     let rg_args = vec![
         "--no-line-number",
@@ -98,7 +97,7 @@ fn main() -> Fallible<()> {
     let mut child = Command::new("rg")
         .args(rg_args)
         .arg("--pre")
-        .arg(preproc_exe)
+        .arg("rga-preproc")
         .arg("--pre-glob")
         .arg(pre_glob)
         .args(passthrough_args)
@@ -108,3 +107,19 @@ fn main() -> Fallible<()> {
     child.wait()?;
     Ok(())
 }
+
+/// add the directory that contains `rga` to PATH, so ripgrep can find rga-preproc and rga-preproc can find pandoc etc (if we are on Windows where we include dependent binaries)
+fn add_exe_to_path() -> Fallible<()> {
+    use std::env;
+    let mut exe = env::current_exe().expect("Could not get executable location");
+    // let preproc_exe = exe.with_file_name("rga-preproc");
+    exe.pop(); // dirname
+
+    let path = env::var_os("PATH").unwrap_or("".into());
+    let mut paths = env::split_paths(&path).collect::<Vec<_>>();
+    paths.push(exe.to_owned()); // append: this way system PATH gets higher priority than bundled versions
+    paths.push(exe.join("lib"));
+    let new_path = env::join_paths(paths)?;
+    env::set_var("PATH", &new_path);
+    Ok(())
+}

src/preproc.rs

@@ -2,8 +2,7 @@ use crate::adapters::*;
 use crate::args::RgaArgs;
 use crate::matching::*;
 use crate::CachingWriter;
-use failure::Fallible;
+use failure::*;
-use failure::{format_err, Error};
 use log::*;
 use path_clean::PathClean;
 use std::convert::TryInto;