Store thumbnails in rocksdb.
This commit is contained in:
parent
d07542b544
commit
914e30365e
302
Cargo.lock
generated
302
Cargo.lock
generated
@ -53,6 +53,28 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e4036b9bf40f3cf16aba72a3d65e8a520fc4bafcdc7079aea8f848c58c5b5536"
|
||||
dependencies = [
|
||||
"backtrace-sys",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"rustc-demangle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backtrace-sys"
|
||||
version = "0.1.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.10.1"
|
||||
@ -68,6 +90,29 @@ version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.49.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c07087f3d5731bf3fb375a81841b99597e25dc11bd3bc72d16d43adf6624a6e"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cexpr",
|
||||
"cfg-if",
|
||||
"clang-sys",
|
||||
"clap",
|
||||
"env_logger",
|
||||
"fxhash",
|
||||
"lazy_static 1.4.0",
|
||||
"log 0.4.8",
|
||||
"peeking_take_while",
|
||||
"proc-macro2 0.4.30",
|
||||
"quote 0.6.13",
|
||||
"regex",
|
||||
"shlex",
|
||||
"which",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.2.1"
|
||||
@ -166,6 +211,18 @@ name = "cc"
|
||||
version = "1.0.50"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d"
|
||||
dependencies = [
|
||||
"nom 4.2.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
@ -185,6 +242,17 @@ dependencies = [
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"libc",
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "2.33.0"
|
||||
@ -334,6 +402,28 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"humantime",
|
||||
"log 0.4.8",
|
||||
"regex",
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "failure"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fake-simd"
|
||||
version = "0.1.2"
|
||||
@ -433,8 +523,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7"
|
||||
dependencies = [
|
||||
"proc-macro-hack",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
@ -467,6 +557,15 @@ dependencies = [
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fxhash"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
||||
dependencies = [
|
||||
"byteorder 1.3.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.12.3"
|
||||
@ -497,6 +596,12 @@ dependencies = [
|
||||
"lzw",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
|
||||
|
||||
[[package]]
|
||||
name = "google-photoslibrary1"
|
||||
version = "0.1.0-20200203"
|
||||
@ -535,8 +640,8 @@ name = "google_field_selector_derive"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/google-apis-rs/generator#a848301466556f16d48df64ef16b92a8327a8406"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"serde_derive_internals",
|
||||
"syn",
|
||||
]
|
||||
@ -681,6 +786,15 @@ version = "1.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
|
||||
dependencies = [
|
||||
"quick-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.12.35"
|
||||
@ -873,6 +987,15 @@ version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jpeg-decoder"
|
||||
version = "0.1.18"
|
||||
@ -920,6 +1043,28 @@ version = "0.2.66"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "librocksdb-sys"
|
||||
version = "6.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a0785e816e1e11e7599388a492c61ef80ddc2afc91e313e61662cce537809be"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
"glob",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.3.3"
|
||||
@ -1253,6 +1398,12 @@ dependencies = [
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "peeking_take_while"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "1.0.1"
|
||||
@ -1318,11 +1469,13 @@ dependencies = [
|
||||
"prometheus",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rocksdb",
|
||||
"rust-embed",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"stderrlog",
|
||||
"structopt",
|
||||
"tempdir",
|
||||
"warp",
|
||||
"yup-oauth2",
|
||||
]
|
||||
@ -1342,8 +1495,8 @@ version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
@ -1390,8 +1543,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "875077759af22fa20b610ad4471d8155b321c89c3f2785526c9839b099be4e0a"
|
||||
dependencies = [
|
||||
"proc-macro-error-attr",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"rustversion",
|
||||
"syn",
|
||||
]
|
||||
@ -1402,8 +1555,8 @@ version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c5717d9fa2664351a01ed73ba5ef6df09c01a521cb42cb65a061432a826f3c7a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"rustversion",
|
||||
"syn",
|
||||
"syn-mid",
|
||||
@ -1415,8 +1568,8 @@ version = "0.5.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
@ -1426,13 +1579,22 @@ version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "0.4.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
|
||||
dependencies = [
|
||||
"unicode-xid 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
"unicode-xid 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1475,13 +1637,35 @@ version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "0.6.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.30",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"proc-macro2 1.0.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
|
||||
dependencies = [
|
||||
"fuchsia-cprng",
|
||||
"libc",
|
||||
"rand_core 0.3.1",
|
||||
"rdrand",
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1761,6 +1945,16 @@ dependencies = [
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rocksdb"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12069b106981c6103d3eab7dd1c86751482d0779a520b7c14954c8b586c1e643"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"librocksdb-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed"
|
||||
version = "5.3.0"
|
||||
@ -1778,7 +1972,7 @@ version = "5.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50212d0e652f580e6d297537c31237d4b2f4497e5912eebe25fde97ac06a51df"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"quote 1.0.2",
|
||||
"rust-embed-utils",
|
||||
"syn",
|
||||
"walkdir",
|
||||
@ -1793,6 +1987,12 @@ dependencies = [
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.1.7"
|
||||
@ -1842,8 +2042,8 @@ version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
@ -1963,8 +2163,8 @@ version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
@ -1974,8 +2174,8 @@ version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1dbab34ca63057a1f15280bdf3c39f2b1eb1b54c17e98360e511637aef7418c6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
@ -2026,6 +2226,12 @@ dependencies = [
|
||||
"opaque-debug",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "0.2.3"
|
||||
@ -2112,8 +2318,8 @@ checksum = "095064aa1f5b94d14e635d0a5684cf140c43ae40a0fd990708d38f5d669e5f64"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro-error",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
@ -2123,9 +2329,9 @@ version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af6f3550d8dff9ef7dc34d384ac6f107e5d31c8f57d9f28e0081503f547ac8f5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-xid",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"unicode-xid 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -2134,11 +2340,21 @@ version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempdir"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
|
||||
dependencies = [
|
||||
"rand 0.4.6",
|
||||
"remove_dir_all",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.1.0"
|
||||
@ -2592,6 +2808,12 @@ version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.0"
|
||||
@ -2762,8 +2984,8 @@ dependencies = [
|
||||
"bumpalo",
|
||||
"lazy_static 1.4.0",
|
||||
"log 0.4.8",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
@ -2786,7 +3008,7 @@ version = "0.2.58"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "574094772ce6921576fb6f2e3f7497b8a76273b6db092be18fc48a082de09dc3"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"quote 1.0.2",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
@ -2796,8 +3018,8 @@ version = "0.2.58"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e85031354f25eaebe78bb7db1c3d86140312a911a106b2e29f9cc440ce3e7668"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn",
|
||||
"wasm-bindgen-backend",
|
||||
"wasm-bindgen-shared",
|
||||
@ -2818,8 +3040,8 @@ dependencies = [
|
||||
"anyhow",
|
||||
"heck",
|
||||
"log 0.4.8",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn",
|
||||
"wasm-bindgen-backend",
|
||||
"weedle",
|
||||
@ -2866,6 +3088,16 @@ dependencies = [
|
||||
"nom 4.2.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164"
|
||||
dependencies = [
|
||||
"failure",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.2.8"
|
||||
|
||||
@ -24,7 +24,11 @@ serde = { version = "1.0.104", features = ["derive"] }
|
||||
image = "0.23.0"
|
||||
rust-embed = "5.2.0"
|
||||
mime_guess = "2.0.1"
|
||||
rocksdb = "0.13.0"
|
||||
|
||||
[dependencies.prometheus]
|
||||
features = ["process"]
|
||||
version = "0.7.0"
|
||||
|
||||
[dev-dependencies]
|
||||
tempdir = "0.3.7"
|
||||
|
||||
136
src/library.rs
136
src/library.rs
@ -3,22 +3,33 @@ use std::fs::File;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use google_photoslibrary1 as photos;
|
||||
use log::error;
|
||||
use log::info;
|
||||
use log::warn;
|
||||
use photos::schemas::Album;
|
||||
use photos::schemas::MediaItem;
|
||||
use rocksdb::DB;
|
||||
|
||||
// Used to ensure DB is invalidated after schema changes.
|
||||
const LIBRARY_GENERATION: &'static str = "1";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Library {
|
||||
root: PathBuf,
|
||||
originals_dir: PathBuf,
|
||||
cache_db: Arc<DB>,
|
||||
}
|
||||
|
||||
impl Library {
|
||||
pub fn new(root: PathBuf) -> io::Result<Library> {
|
||||
pub fn new(root: PathBuf) -> Result<Library, Box<dyn std::error::Error>> {
|
||||
let db = DB::open_default(root.join("cache"))?;
|
||||
let cache_db = Arc::new(db);
|
||||
let lib = Library {
|
||||
originals_dir: root.join("images").join("originals"),
|
||||
cache_db,
|
||||
root,
|
||||
};
|
||||
if !lib.originals_dir.exists() {
|
||||
@ -30,6 +41,20 @@ impl Library {
|
||||
}
|
||||
Ok(lib)
|
||||
}
|
||||
// Removes all data in the database from older schema.
|
||||
pub fn clean_db(&self) -> Result<(), rocksdb::Error> {
|
||||
Library::gc(LIBRARY_GENERATION, &self.cache_db)
|
||||
}
|
||||
fn gc(generation: &str, db: &DB) -> Result<(), rocksdb::Error> {
|
||||
let gen = format!("{}/", generation);
|
||||
for (k, _v) in db.iterator(rocksdb::IteratorMode::Start) {
|
||||
if !k.starts_with(gen.as_bytes()) {
|
||||
info!("deleting stale key: {}", String::from_utf8_lossy(&k));
|
||||
db.delete(k)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn create_album_index(&self, albums: &Vec<Album>) -> io::Result<()> {
|
||||
// Serialize it to a JSON string.
|
||||
let j = serde_json::to_string(albums)?;
|
||||
@ -102,4 +127,113 @@ impl Library {
|
||||
None
|
||||
}
|
||||
}
|
||||
// TODO(wathiede): make this a macro like format! to skip the second string create and copy.
|
||||
fn generational_key(generation: &str, key: &str) -> String {
|
||||
format!("{}/{}", generation, key)
|
||||
}
|
||||
pub fn thumbnail(&self, media_items_id: &str, dimensions: (u32, u32)) -> Option<Vec<u8>> {
|
||||
fn generate_thumbnail(
|
||||
lib: &Library,
|
||||
media_items_id: &str,
|
||||
dimensions: (u32, u32),
|
||||
) -> Result<Vec<u8>, io::Error> {
|
||||
let (w, h) = dimensions;
|
||||
match lib.original(&media_items_id) {
|
||||
None => {
|
||||
warn!("Couldn't find original {}", &media_items_id);
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::NotFound,
|
||||
format!("{}", media_items_id),
|
||||
))
|
||||
}
|
||||
Some(path) => {
|
||||
let orig_img = image::io::Reader::open(&path)?
|
||||
.with_guessed_format()?
|
||||
.decode()
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
|
||||
// TODO (wathiede) resize..fill
|
||||
let img = orig_img.resize(w, h, image::imageops::FilterType::CatmullRom);
|
||||
let mut buf = Vec::new();
|
||||
img.write_to(&mut buf, image::ImageFormat::Jpeg)
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
|
||||
Ok(buf)
|
||||
}
|
||||
}
|
||||
}
|
||||
fn cache_key(media_items_id: &str, dimensions: (u32, u32)) -> String {
|
||||
let (w, h) = dimensions;
|
||||
Library::generational_key(
|
||||
LIBRARY_GENERATION,
|
||||
&format!("{}-w={}-h={}", media_items_id, w, h),
|
||||
)
|
||||
}
|
||||
let key = cache_key(media_items_id, dimensions);
|
||||
let db = self.cache_db.clone();
|
||||
match db.get(key.as_bytes()) {
|
||||
// Cache hit, return bytes as-is.
|
||||
Ok(Some(bytes)) => {
|
||||
info!("cache HIT {}", key);
|
||||
Some(bytes)
|
||||
}
|
||||
// Cache miss, fill cache and return.
|
||||
Ok(None) => {
|
||||
info!("cache MISS {}", key);
|
||||
let bytes = match generate_thumbnail(self, media_items_id, dimensions) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
error!("Failed to generate thumbnail for {}: {}", media_items_id, e);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
match db.put(key.as_bytes(), &bytes) {
|
||||
Ok(_) => Some(bytes),
|
||||
Err(e) => {
|
||||
error!("Failed to put bytes to {}: {}", key, e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
// RocksDB error.
|
||||
Err(e) => {
|
||||
error!("Failed to search DB for {}: {}", key, e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use tempdir::TempDir;
|
||||
|
||||
#[test]
|
||||
fn clean_db() {
|
||||
let td = TempDir::new("photosync_test").expect("failed to create temporary directory");
|
||||
eprintln!("creating database in {}", td.path().to_string_lossy());
|
||||
let db = DB::open_default(td.path()).expect("failed to open DB");
|
||||
let keys = vec!["one", "two", "three"];
|
||||
|
||||
fn get_keys(db: &DB) -> Vec<String> {
|
||||
db.iterator(rocksdb::IteratorMode::Start)
|
||||
.map(|(k, _v)| String::from_utf8(k.to_vec()).expect("key not utf-8"))
|
||||
.collect()
|
||||
}
|
||||
for k in &keys {
|
||||
for g in vec!["1", "2", "3"] {
|
||||
db.put(Library::generational_key(g, k), k)
|
||||
.expect("failed to put");
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
get_keys(&db),
|
||||
vec![
|
||||
"1/one", "1/three", "1/two", "2/one", "2/three", "2/two", "3/one", "3/three",
|
||||
"3/two"
|
||||
]
|
||||
);
|
||||
Library::gc("2", &db).expect("failed to GC DB");
|
||||
assert_eq!(get_keys(&db), vec!["2/one", "2/three", "2/two",]);
|
||||
}
|
||||
}
|
||||
|
||||
43
src/web.rs
43
src/web.rs
@ -2,7 +2,6 @@ use std::error::Error;
|
||||
use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use log::info;
|
||||
use log::warn;
|
||||
use prometheus::Encoder;
|
||||
use rust_embed::RustEmbed;
|
||||
@ -32,6 +31,7 @@ fn metrics() -> impl Filter<Extract = (impl warp::reply::Reply,), Error = Reject
|
||||
.with(warp::reply::with::headers(text_headers))
|
||||
}
|
||||
|
||||
// TODO(wathiede): add caching for hashed files. Add at least etag for everything.
|
||||
fn index(path: warp::path::FullPath) -> Result<impl warp::Reply, warp::Rejection> {
|
||||
let path = path.as_str();
|
||||
let path = if path.ends_with("/") {
|
||||
@ -47,7 +47,6 @@ fn index(path: warp::path::FullPath) -> Result<impl warp::Reply, warp::Rejection
|
||||
|
||||
Ok(warp::http::Response::builder()
|
||||
.header("Content-Type", mime.essence_str())
|
||||
.header("Content-Length", bytes.len())
|
||||
.body(bytes.into_owned()))
|
||||
}
|
||||
None => Err(warp::reject::not_found()),
|
||||
@ -74,7 +73,6 @@ fn album(lib: Library, id: String) -> Result<impl warp::Reply, warp::Rejection>
|
||||
struct ImageParams {
|
||||
w: Option<u32>,
|
||||
h: Option<u32>,
|
||||
c: Option<bool>,
|
||||
}
|
||||
|
||||
fn image(
|
||||
@ -82,42 +80,17 @@ fn image(
|
||||
media_items_id: String,
|
||||
params: ImageParams,
|
||||
) -> Result<impl warp::Reply, warp::Rejection> {
|
||||
info!("image_id {} params {:?}", media_items_id, params);
|
||||
|
||||
match lib.original(&media_items_id) {
|
||||
match lib.thumbnail(
|
||||
&media_items_id,
|
||||
(params.w.unwrap_or(0), params.h.unwrap_or(0)),
|
||||
) {
|
||||
None => {
|
||||
warn!("Couldn't find original {}", &media_items_id);
|
||||
Err(warp::reject::not_found())
|
||||
}
|
||||
Some(path) => {
|
||||
let orig_img = image::io::Reader::open(&path)
|
||||
.map_err(|e| {
|
||||
warn!("Couldn't open {}: {}", path.to_string_lossy(), e);
|
||||
warp::reject::not_found()
|
||||
})?
|
||||
.with_guessed_format()
|
||||
.map_err(|e| {
|
||||
warn!("Couldn't guess format {}: {}", path.to_string_lossy(), e);
|
||||
warp::reject::not_found()
|
||||
})?
|
||||
.decode()
|
||||
.map_err(|e| {
|
||||
warn!("Couldn't decode {}: {}", path.to_string_lossy(), e);
|
||||
warp::reject::not_found()
|
||||
})?;
|
||||
let img = orig_img.resize(
|
||||
params.w.unwrap_or(0),
|
||||
params.h.unwrap_or(0),
|
||||
image::imageops::FilterType::CatmullRom,
|
||||
);
|
||||
let mut buf = Vec::new();
|
||||
img.write_to(&mut buf, image::ImageFormat::Jpeg)
|
||||
.map_err(|e| {
|
||||
warn!("Couldn't write_to {}: {}", path.to_string_lossy(), e);
|
||||
warp::reject::not_found()
|
||||
})?;
|
||||
Ok(buf)
|
||||
}
|
||||
Some(bytes) => Ok(warp::http::Response::builder()
|
||||
.header("Content-Type", "image/jpeg")
|
||||
.body(bytes)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user