From 1529a678e0f4c78a24c8cc2cd236ce12ea8a78dc Mon Sep 17 00:00:00 2001 From: FliegendeWurst <2012gdwu@web.de> Date: Tue, 21 Apr 2020 22:41:28 +0200 Subject: [PATCH] Initial commit --- .gitignore | 1 + Cargo.lock | 1759 +++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 20 + README.md | 26 + src/errors.rs | 64 ++ src/main.rs | 484 ++++++++++++++ 6 files changed, 2354 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 src/errors.rs create mode 100644 src/main.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eb5a316 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..b8e1569 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1759 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "KIT-ILIAS-downloader" +version = "0.1.0" +dependencies = [ + "error-chain", + "futures-util", + "lazy_static", + "parking_lot", + "regex", + "reqwest", + "scraper", + "serde_json", + "tokio", + "url 2.1.1", +] + +[[package]] +name = "aho-corasick" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" +dependencies = [ + "memchr", +] + +[[package]] +name = "arc-swap" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62" + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "backtrace" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e692897359247cc6bb902933361652380af0f1b7651ae5c5013407f30e109e" +dependencies = [ + "backtrace-sys", + "cfg-if", + "libc", + "rustc-demangle", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78848718ee1255a2485d1309ad9cdecfc2e7d0362dd11c6829364c6b35ae1bc7" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bumpalo" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ae9db68ad7fac5fe51304d20f016c911539251075a214f8e663babefa35187" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "bytes" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1" + +[[package]] +name = "cc" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "cookie" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5" +dependencies = [ + "time", + "url 1.7.2", +] + +[[package]] +name = "cookie_store" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d3b52e7c82f1af00203337cd15c56be6fac37397a26439d03cedbbdbc487ef" +dependencies = [ + "cookie", + "idna 0.2.0", + "log", + "publicsuffix", + "serde", + "serde_json", + "time", + "url 2.1.1", +] + +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + +[[package]] +name = "cssparser" +version = "0.25.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbe18ca4efb9ba3716c6da66cc3d7e673bf59fa576353011f48c4cfddbdd740e" +dependencies = [ + "autocfg 0.1.7", + "cssparser-macros", + "dtoa-short", + "itoa", + "matches", + "phf", + "proc-macro2", + "procedural-masquerade", + "quote", + "smallvec 0.6.13", + "syn", +] + +[[package]] +name = "cssparser-macros" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb1c84e87c717666564ec056105052331431803d606bd45529b28547b611eef" +dependencies = [ + "phf_codegen", + "proc-macro2", + "procedural-masquerade", + "quote", + "syn", +] + +[[package]] +name = "dtoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" + +[[package]] +name = "dtoa-short" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59020b8513b76630c49d918c33db9f4c91638e7d3404a28084083b87e33f76f2" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" + +[[package]] +name = "encoding_rs" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8d03faa7fe0c1431609dfad7bbe827af30f82e1e2ae6f7ee4fca6bd764bc28" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "error-chain" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d371106cc88ffdfb1eabd7111e432da544f16f3e2d7bf1dfe8bf575f1df045cd" +dependencies = [ + "backtrace", + "version_check", +] + +[[package]] +name = "fnv" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c9c1ce3fa9336301af935ab852c437817d14cd33690446569392e65170aac3b" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] +name = "futures-channel" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a" + +[[package]] +name = "futures-macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6" + +[[package]] +name = "futures-task" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" + +[[package]] +name = "futures-util" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "h2" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "377038bf3c89d18d6ca1431e7a5027194fbd724ca10592b9487ede5e8e144f42" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "log", + "slab", + "tokio", + "tokio-util", +] + +[[package]] +name = "hermit-abi" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d737e0f947a1864e93d33fdef4af8445a00d1ed8dc0c8ddb73139ea6abf15" +dependencies = [ + "libc", +] + +[[package]] +name = "html5ever" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025483b0a1e4577bb28578318c886ee5f817dda6eb62473269349044406644cb" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "hyper" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96816e1d921eca64d208a85aab4f7798455a8e34229ee5a88c935bdee1b78b14" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "log", + "net2", + "pin-project", + "time", + "tokio", + "tower-service", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-tls", +] + +[[package]] +name = "idna" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" +dependencies = [ + "autocfg 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" + +[[package]] +name = "js-sys" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a27d435371a2fa5b6d2b028a74bbdb1234f308da363226a2854ca3ff8ba7055" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65381d9d47506b8592b97c4efd936afcf673b09b059f2bef39c7211ee78b9d03" +dependencies = [ + "log", + "phf", + "phf_codegen", + "serde", + "serde_derive", + "serde_json", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "mio" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f" +dependencies = [ + "cfg-if", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.1", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "mio-named-pipes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3" +dependencies = [ + "log", + "mio", + "miow 0.3.3", + "winapi 0.3.8", +] + +[[package]] +name = "mio-uds" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125" +dependencies = [ + "iovec", + "libc", + "mio", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396aa0f2003d7df8395cb93e09871561ccc3e785f0acb369170e8cc74ddf9226" +dependencies = [ + "socket2", + "winapi 0.3.8", +] + +[[package]] +name = "native-tls" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "net2" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" +dependencies = [ + "cfg-if", + "libc", + "winapi 0.3.8", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "openssl" +version = "0.10.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee6d85f4cb4c4f59a6a85d5b68a233d280c82e29e822913b9c8b129fbf20bdd" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7717097d810a0f2e2323f9e5d11e71608355e24828410b55b9d4f18aa5f9a5d8" +dependencies = [ + "autocfg 1.0.0", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e136c1904604defe99ce5fd71a28d473fa60a12255d511aa78a9ddf11237aeb" +dependencies = [ + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "smallvec 1.3.0", + "winapi 0.3.8", +] + +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "phf" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +dependencies = [ + "phf_shared", + "rand 0.6.5", +] + +[[package]] +name = "phf_shared" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f6a7f5eee6292c559c793430c55c00aea9d3b3d1905e855806ca4d7253426a2" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8988430ce790d8682672117bc06dda364c0be32d3abd738234f19f3240bad99a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae" + +[[package]] +name = "pin-utils" +version = "0.1.0-alpha.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" + +[[package]] +name = "pkg-config" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" + +[[package]] +name = "ppv-lite86" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "proc-macro-hack" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63" + +[[package]] +name = "proc-macro-nested" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" + +[[package]] +name = "proc-macro2" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "procedural-masquerade" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a1574a51c3fd37b26d2c0032b649d08a7d51d4cca9c41bbc5bf7118fa4509d0" + +[[package]] +name = "publicsuffix" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b" +dependencies = [ + "error-chain", + "idna 0.2.0", + "lazy_static", + "regex", + "url 2.1.1", +] + +[[package]] +name = "quote" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi 0.3.8", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi 0.3.8", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi 0.3.8", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" + +[[package]] +name = "regex" +version = "1.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" + +[[package]] +name = "remove_dir_all" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" +dependencies = [ + "winapi 0.3.8", +] + +[[package]] +name = "reqwest" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b81e49ddec5109a9dcfc5f2a317ff53377c915e9ae9d4f2fb50914b85614e2" +dependencies = [ + "base64", + "bytes", + "cookie", + "cookie_store", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "js-sys", + "lazy_static", + "log", + "mime", + "mime_guess", + "native-tls", + "percent-encoding 2.1.0", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "time", + "tokio", + "tokio-tls", + "url 2.1.1", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + +[[package]] +name = "ryu" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" + +[[package]] +name = "schannel" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "039c25b130bd8c1321ee2d7de7fde2659fa9c2744e4bb29711cfc852ea53cd19" +dependencies = [ + "lazy_static", + "winapi 0.3.8", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "scraper" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77e013440e2ab1c25065551adda5a9773698bb5be3dbab06ff2deac3d5c1a793" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "matches", + "selectors", + "smallvec 0.6.13", + "tendril", +] + +[[package]] +name = "security-framework" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572dfa3a0785509e7a44b5b4bebcf94d41ba34e9ed9eb9df722545c3b3c4144a" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb15a5fec93b7021b8a9e96009c5d8d51c15673569f7c0f6b7204e5b7b404f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "selectors" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b86b100bede4f651059740afc3b6cb83458d7401cb7c1ad96d8a11e91742c86" +dependencies = [ + "bitflags", + "cssparser", + "fxhash", + "log", + "matches", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec 0.6.13", + "thin-slice", +] + +[[package]] +name = "serde" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +dependencies = [ + "dtoa", + "itoa", + "serde", + "url 2.1.1", +] + +[[package]] +name = "servo_arc" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" +dependencies = [ + "nodrop", + "stable_deref_trait", +] + +[[package]] +name = "signal-hook-registry" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41" +dependencies = [ + "arc-swap", + "libc", +] + +[[package]] +name = "siphasher" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smallvec" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a" + +[[package]] +name = "socket2" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi 0.3.8", +] + +[[package]] +name = "stable_deref_trait" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" + +[[package]] +name = "string_cache" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89c058a82f9fd69b1becf8c274f412281038877c553182f1d02eb027045a2d67" +dependencies = [ + "lazy_static", + "new_debug_unreachable", + "phf_shared", + "precomputed-hash", + "serde", + "string_cache_codegen", + "string_cache_shared", +] + +[[package]] +name = "string_cache_codegen" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "string_cache_shared", +] + +[[package]] +name = "string_cache_shared" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc" + +[[package]] +name = "syn" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if", + "libc", + "rand 0.7.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.8", +] + +[[package]] +name = "tendril" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707feda9f2582d5d680d733e38755547a3e8fb471e7ba11452ecfd9ce93a5d3b" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "thin-slice" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi 0.3.8", +] + +[[package]] +name = "tokio" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ef16d072d2b6dc8b4a56c70f5c5ced1a37752116f8e7c1e80c659aa7cb6713" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio", + "mio-named-pipes", + "mio-uds", + "num_cpus", + "pin-project-lite", + "signal-hook-registry", + "slab", + "tokio-macros", + "winapi 0.3.8", +] + +[[package]] +name = "tokio-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bde02a3a5291395f59b06ec6945a3077602fac2b07eeeaf0dee2122f3619828" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "try-lock" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" +dependencies = [ + "smallvec 1.3.0", +] + +[[package]] +name = "unicode-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" + +[[package]] +name = "url" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" +dependencies = [ + "idna 0.1.5", + "matches", + "percent-encoding 1.0.1", +] + +[[package]] +name = "url" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +dependencies = [ + "idna 0.2.0", + "matches", + "percent-encoding 2.1.0", +] + +[[package]] +name = "utf-8" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" + +[[package]] +name = "vcpkg" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" + +[[package]] +name = "version_check" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasm-bindgen" +version = "0.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc57ce05287f8376e998cbddfb4c8cb43b84a7ec55cf4551d7c00eef317a47f" +dependencies = [ + "cfg-if", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d967d37bf6c16cca2973ca3af071d0a2523392e4a594548155d89a678f4237cd" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7add542ea1ac7fdaa9dc25e031a6af33b7d63376292bd24140c637d00d1c312a" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bd151b63e1ea881bb742cd20e1d6127cef28399558f3b5d415289bc41eee3a4" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68a5b36eef1be7868f668632863292e37739656a80fc4b9acec7b0bd35a4931" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf76fe7d25ac79748a37538b7daeed1c7a6867c92d3245c12c6222e4a20d639" + +[[package]] +name = "web-sys" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6f51648d8c56c366144378a33290049eafdd784071077f6fe37dae64c1c4cb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi 0.3.8", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..3d172c5 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "KIT-ILIAS-downloader" +version = "0.1.0" +authors = ["FliegendeWurst <2012gdwu@web.de>"] +license = "GPL-2.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +reqwest = { version = "0.10.4", features = ["cookies", "json", "stream"] } +error-chain = "0.12.2" +tokio = { version = "0.2", features = ["full"] } +serde_json = "1.0.51" +scraper = "0.11.0" +url = "2.1.1" +futures-util = "0.3.4" +regex = "1.3.7" +lazy_static = "1.4.0" +parking_lot = "0.10.2" diff --git a/README.md b/README.md new file mode 100644 index 0000000..f8af1d6 --- /dev/null +++ b/README.md @@ -0,0 +1,26 @@ +# KIT-ILIAS-downloader + +Download content from ILIAS. That includes: + +* files +* Opencast lectures + +## Installation + +Go to the [releases](releases) and get the executable for your operating system. Alternatively compile from source: +```sh +$ git clone https://github.com/FliegendeWurst/KIT-ILIAS-downloader +... +$ cd KIT-ILIAS-downloader +$ cargo build --release +... +$ cp target/release/KIT-ILIAS-downloader [directory in $PATH] +``` + +## Usage + +TBD + +## Credits + +Inspired by https://github.com/brantsch/kit-ilias-fuse. \ No newline at end of file diff --git a/src/errors.rs b/src/errors.rs new file mode 100644 index 0000000..594b0fd --- /dev/null +++ b/src/errors.rs @@ -0,0 +1,64 @@ +use error_chain::error_chain; + +use super::*; + +error_chain! { + // The type defined for this error. These are the conventional + // and recommended names, but they can be arbitrarily chosen. + // + // It is also possible to leave this section out entirely, or + // leave it empty, and these names will be used automatically. + types { + Error, ErrorKind, ResultExt, Result; + } + + // Without the `Result` wrapper: + // + // types { + // Error, ErrorKind, ResultExt; + // } + + // Automatic conversions between this error chain and other + // error chains. In this case, it will e.g. generate an + // `ErrorKind` variant called `Another` which in turn contains + // the `other_error::ErrorKind`, with conversions from + // `other_error::Error`. + // + // Optionally, some attributes can be added to a variant. + // + // This section can be empty. + links { + // Another(other_error::Error, other_error::ErrorKind) #[cfg(unix)]; + } + + // Automatic conversions between this error chain and other + // error types not defined by the `error_chain!`. These will be + // wrapped in a new error with, in the first case, the + // `ErrorKind::Fmt` variant. The description and cause will + // forward to the description and cause of the original error. + // + // Optionally, some attributes can be added to a variant. + // + // This section can be empty. + foreign_links { + //ALSA(alsa::Error); + //Channel(crossbeam_channel::SendError); // TODO: requires type argument + //Discord(serenity::Error); + Io(std::io::Error); + Reqwest(reqwest::Error); + //Pulse(pulse::error::PAErr); + } + + // Define additional `ErrorKind` variants. Define custom responses with the + // `description` and `display` calls. + errors { + UnsupportedChannel(x: String) { + description("unsupported channel kind") + display("unsupported channel kind {:?}", x) + } + } + + // If this annotation is left off, a variant `Msg(s: String)` will be added, and `From` + // impls will be provided for `String` and `&str` + //skip_msg_variant +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..299dde3 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,484 @@ +use futures_util::stream::TryStreamExt; +use lazy_static::lazy_static; +use parking_lot::Mutex; +use regex::Regex; +use reqwest::Client; +use scraper::{ElementRef, Html, Selector}; +use serde_json::json; +use tokio::fs::File as AsyncFile; +use tokio::io::{stream_reader, BufWriter}; +use tokio::task; +use url::Url; + +use std::collections::VecDeque; +use std::default::Default; +use std::fs; +use std::io; +use std::panic; +use std::path::PathBuf; +use std::sync::Arc; + +mod errors; +use errors::*; + +const ILIAS_URL: &'static str = "https://ilias.studium.kit.edu/"; + +struct ILIAS { + user: String, + pass: String, + path_prefix: PathBuf, + client: Client +} + +#[derive(Debug)] +struct Dashboard { + items: Vec +} + +#[derive(Debug)] +enum Object { + Course { + name: String, + url: URL + }, + Folder { + name: String, + url: URL + }, + File { + name: String, + url: URL + }, + Forum { + name: String, + url: URL + }, + Wiki { + name: String, + url: URL + }, + ExerciseHandler { + name: String, + url: URL + }, + PluginDispatch { + name: String, + url: URL + }, + Video { + url: String, + }, + Generic { + name: String, + url: URL + }, +} + +use Object::*; + +impl Object { + fn name(&self) -> &str { + match self { + Course { name, .. } => &name, + Folder { name, .. } => &name, + File { name, .. } => &name, + Forum { name, .. } => &name, + Wiki { name, .. } => &name, + ExerciseHandler { name, .. } => &name, + PluginDispatch { name, .. } => &name, + Video { url } => &url, + Generic { name, .. } => &name, + } + } + + fn kind(&self) -> &str { + match self { + Course { .. } => "course", + Folder { .. } => "folder", + File { .. } => "file", + Forum { .. } => "forum", + Wiki { .. } => "wiki", + ExerciseHandler { .. } => "exercise handler", + PluginDispatch { .. } => "plugin dispatch", + Video { .. } => "video", + Generic { .. } => "generic", + } + } + + fn from_link(item: ElementRef, link: ElementRef) -> Self { + let mut name = link.text().collect::().replace('/', "-"); + let url = URL::from_href(link.value().attr("href").unwrap()); + + if url.url.starts_with("https://ilias.studium.kit.edu/goto.php") { + let item_prop = Selector::parse("span.il_ItemProperty").unwrap(); + let mut item_props = item.select(&item_prop); + let ext = item_props.next().unwrap(); + let version = item_props.nth(1).unwrap().text().collect::(); + let version = version.trim(); + if version.starts_with("Version: ") { + name.push_str("_v"); + name.push_str(&version[9..]); + } + return File { name: format!("{}.{}", name, ext.text().collect::().trim()), url }; + } + + if url.cmd.as_ref().map(|x| &**x) == Some("showThreads") { + return Forum { name, url }; + } + + match &*url.baseClass { + "ilExerciseHandlerGUI" => ExerciseHandler { name, url }, + "ililWikiHandlerGUI" => Wiki { name, url }, + "ilrepositorygui" => match url.cmd.as_deref() { + Some("view") => Folder { name, url }, + Some(_) => Generic { name, url }, + None => Course { name, url }, + }, + "ilObjPluginDispatchGUI" => PluginDispatch { name, url }, + _ => Generic { name, url } + } + } +} + +#[allow(non_snake_case)] +#[derive(Debug)] +struct URL { + url: String, + baseClass: String, + cmdClass: Option, + cmdNode: Option, + cmd: Option, + forwardCmd: Option, + ref_id: String, +} + +#[allow(non_snake_case)] +impl URL { + fn from_href(href: &str) -> Self { + let url = Url::parse(&format!("http://domain/{}", href)).unwrap(); + let mut baseClass = String::new(); + let mut cmdClass = None; + let mut cmdNode = None; + let mut cmd = None; + let mut forwardCmd = None; + let mut ref_id = String::new(); + for (k, v) in url.query_pairs() { + match &*k { + "baseClass" => baseClass = v.into_owned(), + "cmdClass" => cmdClass = Some(v.into_owned()), + "cmdNode" => cmdNode = Some(v.into_owned()), + "cmd" => cmd = Some(v.into_owned()), + "forwardCmd" => forwardCmd = Some(v.into_owned()), + "ref_id" => ref_id = v.into_owned(), + _ => {} + } + } + URL { + url: href.to_owned(), + baseClass, + cmdClass, + cmdNode, + cmd, + forwardCmd, + ref_id + } + } +} + +impl ILIAS { + async fn login(user: S1, pass: S1) -> Result where S1: Into, S2: Into { + let user = user.into(); + let pass = pass.into(); + let client = Client::builder() + .cookie_store(true) + .user_agent("KIT-ILIAS-fuse/0.0.1-dev") + .build()?; + println!("Logging into Shibboleth.."); + let session_establishment = client + .post("https://ilias.studium.kit.edu/Shibboleth.sso/Login") + .form(&json!({ + "sendLogin": "1", + "idp_selection": "https://idp.scc.kit.edu/idp/shibboleth", + "target": "https://ilias.studium.kit.edu/shib_login.php?target=", + "home_organization_selection": "Mit KIT-Account anmelden" + })) + .send().await?; + println!("Logging into identity provider.."); + let login_response = client + .post(session_establishment.url().clone()) + .form(&json!({ + "j_username": &user, + "j_password": &pass, + "_eventId_proceed": "" + })) + .send().await?.text().await?; + let dom = Html::parse_document(&login_response); + /* TODO: OTP + login_soup = BeautifulSoup(login_response.text, 'lxml') + otp_inp = login_soup.find("input", attrs={"name": "j_tokenNumber"}) + if otp_inp: + print("OTP Detected.") + otp = input("OTP token: ") + otp_url = otp_inp.parent.parent.parent['action'] + otp_response = self.post('https://idp.scc.kit.edu'+otp_url, data={'j_tokenNumber':otp, "_eventId_proceed": ""}) + login_soup = BeautifulSoup(otp_response.text, 'lxml') + */ + let saml = Selector::parse(r#"input[name="SAMLResponse"]"#).unwrap(); + let saml = dom.select(&saml).next().expect("no SAML response"); + let relay_state = Selector::parse(r#"input[name="RelayState"]"#).unwrap(); + let relay_state = dom.select(&relay_state).next().expect("no relay state"); + println!("Logging into ILIAS.."); + client + .post("https://ilias.studium.kit.edu/Shibboleth.sso/SAML2/POST") + .form(&json!({ + "SAMLResponse": saml.value().attr("value").unwrap(), + "RelayState": relay_state.value().attr("value").unwrap() + })) + .send().await?; + println!("Logged in!"); + let path_prefix = PathBuf::from(env!("ILIAS_DIR")); + Ok(ILIAS { + client, user, pass, path_prefix + }) + } + + async fn personal_desktop(&mut self) -> Result { + let html = self.get_html("https://ilias.studium.kit.edu/ilias.php?baseClass=ilPersonalDesktopGUI&cmd=jumpToSelectedItems").await?; + let items = ILIAS::get_items(&html); + Ok(Dashboard { + items + }) + } + + fn get_items(html: &Html) -> Vec { + let container_items = Selector::parse("div.il_ContainerListItem").unwrap(); + let container_item_title = Selector::parse("a.il_ContainerItemTitle").unwrap(); + html.select(&container_items).map(|item| { + let link = item.select(&container_item_title).next().unwrap(); + Object::from_link(item, link) + }).collect() + } + + async fn get_html(&self, url: &str) -> Result { + let text = self.client.get(url).send().await?.text().await?; + Ok(Html::parse_document(&text)) + } + + async fn get_course_content(&self, url: &URL) -> Result> { + let html = self.get_html(&format!("{}{}", ILIAS_URL, url.url)).await?; + Ok(ILIAS::get_items(&html)) + } + + async fn download(&self, url: &str) -> Result { + //let url = format!("{}{}", ILIAS_URL, url.url); + if VERBOSITY > 0 { + println!("Downloading {}", url); + } + Ok(self.client.get(url).send().await?) + } +} + +const DOWNLOAD_FILES: bool = true; +const DOWNLOAD_VIDEOS: bool = true; + +const SKIP_EXISTING: bool = true; + +const VERBOSITY: usize = 1; + +#[tokio::main] +async fn main() { + *PANIC_HOOK.lock() = panic::take_hook(); + panic::set_hook(Box::new(|info| { + *TASKS_RUNNING.lock() -= 1; + PANIC_HOOK.lock()(info); + })); + // TODO: config at runtime.. + // it's literally in the executable currently + let mut ilias = match ILIAS::login::<&str, &str>(env!("ILIAS_USER"), env!("ILIAS_PASS")).await { + Ok(ilias) => ilias, + Err(e) => panic!("error: {:?}", e) + }; + let desktop = ilias.personal_desktop().await.unwrap(); + let mut queue = VecDeque::new(); + for item in desktop.items { + let mut path = ilias.path_prefix.clone(); + path.push(item.name()); + queue.push_back((path, item)); + } + let ilias = Arc::new(ilias); + while let Some((path, obj)) = queue.pop_front() { + let ilias = Arc::clone(&ilias); + task::spawn(async { + *TASKS_RUNNING.lock() += 1; + process(ilias, path, obj).await; + *TASKS_RUNNING.lock() -= 1; + }); + } + while *TASKS_RUNNING.lock() > 0 { + tokio::time::delay_for(std::time::Duration::from_millis(500)).await; + } +} + +lazy_static!{ + static ref TASKS_RUNNING: Mutex = Mutex::default(); + + static ref PANIC_HOOK: Mutex> = Mutex::new(Box::new(|_| {})); +} + +// see https://github.com/rust-lang/rust/issues/53690#issuecomment-418911229 +//async fn process(ilias: Arc, path: PathBuf, obj: Object) { +fn process(ilias: Arc, path: PathBuf, obj: Object) -> impl std::future::Future + Send { async move { + if VERBOSITY > 0 { + println!("Syncing {} {}..", obj.kind(), path.strip_prefix(&ilias.path_prefix).unwrap().to_string_lossy()); + } + match &obj { + Course { name, url } => { + if let Err(e) = fs::create_dir(&path) { + if e.kind() != io::ErrorKind::AlreadyExists { + println!("error: {:?}", e); + } + } + let content = ilias.get_course_content(&url).await.unwrap(); + for item in content { + let mut path = path.clone(); + path.push(item.name()); + let ilias = Arc::clone(&ilias); + task::spawn(async { + *TASKS_RUNNING.lock() += 1; + process(ilias, path, item).await; + *TASKS_RUNNING.lock() -= 1; + }); + } + }, + Folder { name, url } => { + if let Err(e) = fs::create_dir(&path) { + if e.kind() != io::ErrorKind::AlreadyExists { + println!("error: {:?}", e); + } + } + let content = ilias.get_course_content(&url).await.unwrap(); + for item in content { + let mut path = path.clone(); + path.push(item.name()); + let ilias = Arc::clone(&ilias); + task::spawn(async { + *TASKS_RUNNING.lock() += 1; + process(ilias, path, item).await; + *TASKS_RUNNING.lock() -= 1; + }); + } + }, + File { name, url } => { + if !DOWNLOAD_FILES { + return; + } + if SKIP_EXISTING && fs::metadata(&path).is_ok() { + if VERBOSITY > 1 { + println!("Skipping download, file exists already"); + } + return; + } + let data = ilias.download(&url.url).await; + match data { + Ok(resp) => { + let mut reader = stream_reader(resp.bytes_stream().map_err(|x| { + io::Error::new(io::ErrorKind::Other, x) + })); + let file = AsyncFile::create(&path).await.unwrap(); + let mut file = BufWriter::new(file); + tokio::io::copy(&mut reader, &mut file).await.unwrap(); + }, + Err(e) => println!("error: {:?}", e) + } + }, + PluginDispatch { name, url } => { + if !DOWNLOAD_VIDEOS { + return; + } + if let Err(e) = fs::create_dir(&path) { + if e.kind() != io::ErrorKind::AlreadyExists { + println!("error: {:?}", e); + } + } + let list_url = format!("{}ilias.php?ref_id={}&cmdClass=xocteventgui&cmdNode=n7:mz:14p&baseClass=ilObjPluginDispatchGUI&lang=de&limit=20&cmd=asyncGetTableGUI&cmdMode=asynch", ILIAS_URL, url.ref_id); + let data = ilias.download(&list_url); + let html = data.await.unwrap().text().await.unwrap(); + let html = Html::parse_fragment(&html); + let tr = Selector::parse("tr").unwrap(); + let td = Selector::parse("td").unwrap(); + let a = Selector::parse(r#"a[target="_blank"]"#).unwrap(); + for row in html.select(&tr) { + let link = row.select(&a).next(); + if link.is_none() { + continue; + } + let link = link.unwrap(); + let mut cells = row.select(&td); + if let Some(title) = cells.nth(2) { + let title = title.inner_html(); + let title = title.trim(); + if title.starts_with(" 0 { + println!("Found video: {}", title); + } + let video = Video { + url: link.value().attr("href").unwrap().to_owned() + }; + let ilias = Arc::clone(&ilias); + task::spawn(async { + *TASKS_RUNNING.lock() += 1; + process(ilias, path, video).await; + *TASKS_RUNNING.lock() -= 1; + }); + } + + } + }, + Video { url } => { + lazy_static!{ + static ref XOCT_REGEX: Regex = Regex::new(r#"(?m)"#).unwrap(); + } + if !DOWNLOAD_VIDEOS { + return; + } + if SKIP_EXISTING && fs::metadata(&path).is_ok() { + if VERBOSITY > 1 { + println!("Skipping download, file exists already"); + } + return; + } + let url = format!("{}{}", ILIAS_URL, url); + let data = ilias.download(&url); + let html = data.await.unwrap().text().await.unwrap(); + //println!("{}", html); + let json: serde_json::Value = { + let mut json_capture = XOCT_REGEX.captures_iter(&html); + let json = &json_capture.next().unwrap()[1]; + //println!("{}", json); + let json = json.split(",\n").nth(0).unwrap(); + serde_json::from_str(&json.trim()).unwrap() + }; + //println!("{}", json); + let url = json["streams"][0]["sources"]["mp4"][0]["src"].as_str().unwrap(); + let resp = ilias.download(&url).await.unwrap(); + let mut reader = stream_reader(resp.bytes_stream().map_err(|x| { + io::Error::new(io::ErrorKind::Other, x) + })); + if VERBOSITY > 0 { + println!("Saving video to {:?}", path); + } + let file = AsyncFile::create(&path).await.unwrap(); + let mut file = BufWriter::new(file); + tokio::io::copy(&mut reader, &mut file).await.unwrap(); + }, + o => { + if VERBOSITY > 0 { + println!("ignoring {:#?}", o) + } + } + } +}}