diff --git a/Cargo.lock b/Cargo.lock index 5813ba7..116f2c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,6 +371,14 @@ name = "fuchsia-cprng" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "generic-array" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "glob" version = "0.2.11" @@ -541,6 +549,26 @@ name = "nom" version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "num" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-iter 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-complex" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "num-integer" version = "0.1.41" @@ -550,6 +578,25 @@ dependencies = [ "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "num-iter" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-rational" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "num-traits" version = "0.2.8" @@ -840,6 +887,7 @@ dependencies = [ "rusqlite 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", + "size_format 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", "tree_magic_fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "xz2 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -874,6 +922,7 @@ dependencies = [ "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "fallible-iterator 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "fallible-streaming-iterator 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "libsqlite3-sys 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -949,6 +998,15 @@ dependencies = [ "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "size_format" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "smallvec" version = "0.6.9" @@ -1048,6 +1106,11 @@ dependencies = [ "petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "typenum" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "ucd-util" version = "0.1.3" @@ -1246,6 +1309,7 @@ dependencies = [ "checksum flate2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f87e68aa82b2de08a6e037f1385455759df6e445a8df5e005b4297191dbf18aa" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +"checksum generic-array 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c0f28c2f5bfb5960175af447a2da7c18900693738343dc896ffbcabd9839592" "checksum glob 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8be18de09a56b60ed0edf84bc9df007e30040691af7acd1c41874faac5895bfb" "checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" @@ -1269,7 +1333,11 @@ dependencies = [ "checksum miniz_oxide_c_api 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b7fe927a42e3807ef71defb191dc87d4e24479b221e67015fe38ae2b7b447bab" "checksum nodrop 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9667ddcc6cc8a43afc9b7917599d7216aa09c463919ea32c59ed6cac8bc945" "checksum nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff" +"checksum num 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cf4825417e1e1406b3782a8ce92f4d53f26ec055e3622e1881ca8e9f5f9e08db" +"checksum num-complex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "107b9be86cd2481930688277b675b0114578227f034674726605b8a482d8baf8" "checksum num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "b85e541ef8255f6cf42bbfe4ef361305c6c135d10919ecc26126c4e5ae94bc09" +"checksum num-iter 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "76bd5272412d173d6bf9afdf98db8612bbabc9a7a830b7bfc9c188911716132e" +"checksum num-rational 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4e96f040177bb3da242b5b1ecf3f54b5d5af3efbbfb18608977a5d2767b22f10" "checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32" "checksum num_cpus 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a23f0ed30a54abaa0c7e83b1d2d87ada7c3c23078d1d87815af3e3b6385fbba" "checksum numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" @@ -1314,6 +1382,7 @@ dependencies = [ "checksum serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "32746bf0f26eab52f06af0d0aa1984f641341d06d8d673c693871da2d188c9be" "checksum serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "46a3223d0c9ba936b61c0d2e3e559e3217dbfb8d65d06d26e8b3c25de38bae3e" "checksum serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" +"checksum size_format 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6ed5f6ab2122c6dec69dca18c72fa4590a27e581ad20d44960fe74c032a0b23b" "checksum smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c4488ae950c49d403731982257768f48fada354a5203fe81f9bb6f43ca9002be" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" "checksum syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)" = "a1393e4a97a19c01e900df2aec855a29f71cf02c402e2f443b8d2747c25c5dbe" @@ -1325,6 +1394,7 @@ dependencies = [ "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" "checksum tree_magic_fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aab921ca9b828f83389f3f3c5e77404612547081e5222eb3a23d06184f6813af" +"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" "checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86" "checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" "checksum unicode-normalization 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "141339a08b982d942be2ca06ff8b076563cbe223d1befd5450716790d44e2426" diff --git a/Cargo.toml b/Cargo.toml index 051f70f..c28b6b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rga" -description = "ripgrep, except for pdf, ebooks, Office documents, etc" +description = "ripgrep but for pdf, ebooks, Office documents, archives, etc" license = "AGPL-3.0-or-later" version = "0.4.0" repository = "https://github.com/phiresky/rga" @@ -37,4 +37,5 @@ tar = "0.4.26" chrono = "0.4.6" encoding_rs = "0.8.17" encoding_rs_io = "0.1.6" -rusqlite = "0.18.0" +rusqlite = { version = "0.18.0", features=["vtab"] } # "bundled" +size_format = "1.0.2" diff --git a/src/adapters.rs b/src/adapters/mod.rs similarity index 96% rename from src/adapters.rs rename to src/adapters/mod.rs index f0e76dd..59b0069 100644 --- a/src/adapters.rs +++ b/src/adapters/mod.rs @@ -2,6 +2,7 @@ pub mod ffmpeg; pub mod pandoc; pub mod poppler; pub mod spawning; +pub mod sqlite; pub mod tar; pub mod zip; use failure::*; @@ -23,6 +24,7 @@ pub enum Matcher { pub struct AdapterMeta { pub name: String, + // version identifier. used to key cache entries, change if your output format changes pub version: i32, pub matchers: Vec, } @@ -67,6 +69,7 @@ pub fn get_adapters() -> Vec> { Rc::new(poppler::PopplerAdapter), Rc::new(zip::ZipAdapter), Rc::new(tar::TarAdapter), + Rc::new(sqlite::SqliteAdapter), ]; adapters } diff --git a/src/adapters/sqlite.rs b/src/adapters/sqlite.rs new file mode 100644 index 0000000..5386aa6 --- /dev/null +++ b/src/adapters/sqlite.rs @@ -0,0 +1,111 @@ +use super::spawning::map_exe_error; +use super::*; +use failure::*; +use lazy_static::lazy_static; +use rusqlite::types::{ToSql, ValueRef}; +use rusqlite::*; +use serde::{Deserialize, Serialize}; +use std::convert::TryInto; +use std::io::BufReader; +use std::process::*; + +static EXTENSIONS: &[&str] = &["db", "db3", "sqlite", "sqlite3"]; + +lazy_static! { + static ref METADATA: AdapterMeta = AdapterMeta { + name: "sqlite".to_owned(), + version: 1, + matchers: EXTENSIONS + .iter() + .map(|s| Matcher::FileExtension(s.to_string())) + .collect(), + }; +} + +#[derive(Default)] +pub struct SqliteAdapter; + +impl SqliteAdapter { + pub fn new() -> SqliteAdapter { + SqliteAdapter + } +} +impl GetMetadata for SqliteAdapter { + fn metadata(&self) -> &AdapterMeta { + &METADATA + } +} + +fn format_blob(b: ValueRef) -> String { + use ValueRef::*; + match b { + Null => "NULL".to_owned(), + Integer(i) => format!("{}", i), + Real(i) => format!("{}", i), + Text(i) => format!("'{}'", i.replace("'", "''")), + Blob(b) => format!( + "[blob {}B]", + size_format::SizeFormatterSI::new( + // can't be larger than 2GB anyways + b.len().try_into().unwrap() + ) + ), + } +} + +impl FileAdapter for SqliteAdapter { + fn adapt(&self, ai: AdaptInfo) -> Fallible<()> { + let AdaptInfo { + is_real_file, + filepath_hint, + oup, + line_prefix, + .. + } = ai; + if !is_real_file { + // todo: read to memory and then use that blob if size < max + writeln!(oup, "{}[rga: skipping sqlite in archive]", line_prefix,)?; + return Ok(()); + } + let inp_fname = filepath_hint; + + let conn = Connection::open_with_flags(inp_fname, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + let tables: Vec = conn + .prepare("select name from sqlite_master where type='table'")? + .query_map(NO_PARAMS, |r| r.get::<_, String>(0))? + .filter_map(|e| e.ok()) + .collect(); + eprintln!("db has {} tables", tables.len()); + for table in tables { + // can't use query param at that position + let mut sel = conn.prepare(&format!( + "select * from {}", + rusqlite::vtab::escape_double_quote(&table) + ))?; + let mut z = sel.query(NO_PARAMS)?; + let col_names: Vec = z + .column_names() + .ok_or_else(|| format_err!("no column names"))? + .into_iter() + .map(|e| e.to_owned()) + .collect(); + // writeln!(oup, "{}: {}", table, cols.join(", "))?; + + // kind of shitty (lossy) output. maybe output real csv or something? + while let Some(row) = z.next()? { + writeln!( + oup, + "{}: {}", + table, + col_names + .iter() + .enumerate() + .map(|(i, e)| format!("{}={}", e, format_blob(row.get_raw(i)))) + .collect::>() + .join(", ") + )?; + } + } + Ok(()) + } +}