Store thumbnails in rocksdb.

Cargo.lock

@@ -53,6 +53,28 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
 
+[[package]]
+name = "backtrace"
+version = "0.3.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4036b9bf40f3cf16aba72a3d65e8a520fc4bafcdc7079aea8f848c58c5b5536"
+dependencies = [
+ "backtrace-sys",
+ "cfg-if",
+ "libc",
+ "rustc-demangle",
+]
+
+[[package]]
+name = "backtrace-sys"
+version = "0.1.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "base64"
 version = "0.10.1"
@@ -68,6 +90,29 @@ version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
 
+[[package]]
+name = "bindgen"
+version = "0.49.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c07087f3d5731bf3fb375a81841b99597e25dc11bd3bc72d16d43adf6624a6e"
+dependencies = [
+ "bitflags",
+ "cexpr",
+ "cfg-if",
+ "clang-sys",
+ "clap",
+ "env_logger",
+ "fxhash",
+ "lazy_static 1.4.0",
+ "log 0.4.8",
+ "peeking_take_while",
+ "proc-macro2 0.4.30",
+ "quote 0.6.13",
+ "regex",
+ "shlex",
+ "which",
+]
+
 [[package]]
 name = "bitflags"
 version = "1.2.1"
@@ -166,6 +211,18 @@ name = "cc"
 version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd"
+dependencies = [
+ "jobserver",
+]
+
+[[package]]
+name = "cexpr"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d"
+dependencies = [
+ "nom 4.2.3",
+]
 
 [[package]]
 name = "cfg-if"
@@ -185,6 +242,17 @@ dependencies = [
  "time",
 ]
 
+[[package]]
+name = "clang-sys"
+version = "0.28.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
 [[package]]
 name = "clap"
 version = "2.33.0"
@@ -334,6 +402,28 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "env_logger"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3"
+dependencies = [
+ "atty",
+ "humantime",
+ "log 0.4.8",
+ "regex",
+ "termcolor",
+]
+
+[[package]]
+name = "failure"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9"
+dependencies = [
+ "backtrace",
+]
+
 [[package]]
 name = "fake-simd"
 version = "0.1.2"
@@ -433,8 +523,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7"
 dependencies = [
  "proc-macro-hack",
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "syn",
 ]
 
@@ -467,6 +557,15 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "fxhash"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+dependencies = [
+ "byteorder 1.3.4",
+]
+
 [[package]]
 name = "generic-array"
 version = "0.12.3"
@@ -497,6 +596,12 @@ dependencies = [
  "lzw",
 ]
 
+[[package]]
+name = "glob"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
+
 [[package]]
 name = "google-photoslibrary1"
 version = "0.1.0-20200203"
@@ -535,8 +640,8 @@ name = "google_field_selector_derive"
 version = "0.1.0"
 source = "git+https://github.com/google-apis-rs/generator#a848301466556f16d48df64ef16b92a8327a8406"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "serde_derive_internals",
  "syn",
 ]
@@ -681,6 +786,15 @@ version = "1.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
 
+[[package]]
+name = "humantime"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
+dependencies = [
+ "quick-error",
+]
+
 [[package]]
 name = "hyper"
 version = "0.12.35"
@@ -873,6 +987,15 @@ version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e"
 
+[[package]]
+name = "jobserver"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "jpeg-decoder"
 version = "0.1.18"
@@ -920,6 +1043,28 @@ version = "0.2.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
 
+[[package]]
+name = "libloading"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
+dependencies = [
+ "cc",
+ "winapi 0.3.8",
+]
+
+[[package]]
+name = "librocksdb-sys"
+version = "6.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a0785e816e1e11e7599388a492c61ef80ddc2afc91e313e61662cce537809be"
+dependencies = [
+ "bindgen",
+ "cc",
+ "glob",
+ "libc",
+]
+
 [[package]]
 name = "lock_api"
 version = "0.3.3"
@@ -1253,6 +1398,12 @@ dependencies = [
  "winapi 0.3.8",
 ]
 
+[[package]]
+name = "peeking_take_while"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
+
 [[package]]
 name = "percent-encoding"
 version = "1.0.1"
@@ -1318,11 +1469,13 @@ dependencies = [
  "prometheus",
  "regex",
  "reqwest",
+ "rocksdb",
  "rust-embed",
  "serde",
  "serde_json",
  "stderrlog",
  "structopt",
+ "tempdir",
  "warp",
  "yup-oauth2",
 ]
@@ -1342,8 +1495,8 @@ version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "syn",
 ]
 
@@ -1390,8 +1543,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "875077759af22fa20b610ad4471d8155b321c89c3f2785526c9839b099be4e0a"
 dependencies = [
  "proc-macro-error-attr",
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "rustversion",
  "syn",
 ]
@@ -1402,8 +1555,8 @@ version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c5717d9fa2664351a01ed73ba5ef6df09c01a521cb42cb65a061432a826f3c7a"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "rustversion",
  "syn",
  "syn-mid",
@@ -1415,8 +1568,8 @@ version = "0.5.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "syn",
 ]
 
@@ -1426,13 +1579,22 @@ version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e"
 
+[[package]]
+name = "proc-macro2"
+version = "0.4.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
+dependencies = [
+ "unicode-xid 0.1.0",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548"
 dependencies = [
- "unicode-xid",
+ "unicode-xid 0.2.0",
 ]
 
 [[package]]
@@ -1475,13 +1637,35 @@ version = "1.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
+[[package]]
+name = "quote"
+version = "0.6.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
+dependencies = [
+ "proc-macro2 0.4.30",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.8",
+]
+
+[[package]]
+name = "rand"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
+dependencies = [
+ "fuchsia-cprng",
+ "libc",
+ "rand_core 0.3.1",
+ "rdrand",
+ "winapi 0.3.8",
 ]
 
 [[package]]
@@ -1761,6 +1945,16 @@ dependencies = [
  "winapi 0.3.8",
 ]
 
+[[package]]
+name = "rocksdb"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12069b106981c6103d3eab7dd1c86751482d0779a520b7c14954c8b586c1e643"
+dependencies = [
+ "libc",
+ "librocksdb-sys",
+]
+
 [[package]]
 name = "rust-embed"
 version = "5.3.0"
@@ -1778,7 +1972,7 @@ version = "5.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "50212d0e652f580e6d297537c31237d4b2f4497e5912eebe25fde97ac06a51df"
 dependencies = [
- "quote",
+ "quote 1.0.2",
  "rust-embed-utils",
  "syn",
  "walkdir",
@@ -1793,6 +1987,12 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "rustc-demangle"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
+
 [[package]]
 name = "rustc_version"
 version = "0.1.7"
@@ -1842,8 +2042,8 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "syn",
 ]
 
@@ -1963,8 +2163,8 @@ version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "syn",
 ]
 
@@ -1974,8 +2174,8 @@ version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1dbab34ca63057a1f15280bdf3c39f2b1eb1b54c17e98360e511637aef7418c6"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "syn",
 ]
 
@@ -2026,6 +2226,12 @@ dependencies = [
  "opaque-debug",
 ]
 
+[[package]]
+name = "shlex"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
+
 [[package]]
 name = "siphasher"
 version = "0.2.3"
@@ -2112,8 +2318,8 @@ checksum = "095064aa1f5b94d14e635d0a5684cf140c43ae40a0fd990708d38f5d669e5f64"
 dependencies = [
  "heck",
  "proc-macro-error",
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "syn",
 ]
 
@@ -2123,9 +2329,9 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "af6f3550d8dff9ef7dc34d384ac6f107e5d31c8f57d9f28e0081503f547ac8f5"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
- "unicode-xid",
+ "unicode-xid 0.2.0",
 ]
 
 [[package]]
@@ -2134,11 +2340,21 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "syn",
 ]
 
+[[package]]
+name = "tempdir"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
+dependencies = [
+ "rand 0.4.6",
+ "remove_dir_all",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.1.0"
@@ -2592,6 +2808,12 @@ version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479"
 
+[[package]]
+name = "unicode-xid"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.0"
@@ -2762,8 +2984,8 @@ dependencies = [
  "bumpalo",
  "lazy_static 1.4.0",
  "log 0.4.8",
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "syn",
  "wasm-bindgen-shared",
 ]
@@ -2786,7 +3008,7 @@ version = "0.2.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "574094772ce6921576fb6f2e3f7497b8a76273b6db092be18fc48a082de09dc3"
 dependencies = [
- "quote",
+ "quote 1.0.2",
  "wasm-bindgen-macro-support",
 ]
 
@@ -2796,8 +3018,8 @@ version = "0.2.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e85031354f25eaebe78bb7db1c3d86140312a911a106b2e29f9cc440ce3e7668"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "syn",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
@@ -2818,8 +3040,8 @@ dependencies = [
  "anyhow",
  "heck",
  "log 0.4.8",
- "proc-macro2",
+ "proc-macro2 1.0.8",
- "quote",
+ "quote 1.0.2",
  "syn",
  "wasm-bindgen-backend",
  "weedle",
@@ -2866,6 +3088,16 @@ dependencies = [
  "nom 4.2.3",
 ]
 
+[[package]]
+name = "which"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164"
+dependencies = [
+ "failure",
+ "libc",
+]
+
 [[package]]
 name = "winapi"
 version = "0.2.8"

Cargo.toml

@@ -24,7 +24,11 @@ serde = { version = "1.0.104", features = ["derive"] }
 image = "0.23.0"
 rust-embed = "5.2.0"
 mime_guess = "2.0.1"
+rocksdb = "0.13.0"
 
 [dependencies.prometheus]
 features = ["process"]
 version = "0.7.0"
+
+[dev-dependencies]
+tempdir = "0.3.7"

src/library.rs

@@ -3,22 +3,33 @@ use std::fs::File;
 use std::io;
 use std::path::Path;
 use std::path::PathBuf;
+use std::sync::Arc;
 
 use google_photoslibrary1 as photos;
+use log::error;
 use log::info;
+use log::warn;
 use photos::schemas::Album;
 use photos::schemas::MediaItem;
+use rocksdb::DB;
+
+// Used to ensure DB is invalidated after schema changes.
+const LIBRARY_GENERATION: &'static str = "1";
 
 #[derive(Clone)]
 pub struct Library {
     root: PathBuf,
     originals_dir: PathBuf,
+    cache_db: Arc<DB>,
 }
 
 impl Library {
-    pub fn new(root: PathBuf) -> io::Result<Library> {
+    pub fn new(root: PathBuf) -> Result<Library, Box<dyn std::error::Error>> {
+        let db = DB::open_default(root.join("cache"))?;
+        let cache_db = Arc::new(db);
         let lib = Library {
             originals_dir: root.join("images").join("originals"),
+            cache_db,
             root,
         };
         if !lib.originals_dir.exists() {
@@ -30,6 +41,20 @@ impl Library {
         }
         Ok(lib)
     }
+    // Removes all data in the database from older schema.
+    pub fn clean_db(&self) -> Result<(), rocksdb::Error> {
+        Library::gc(LIBRARY_GENERATION, &self.cache_db)
+    }
+    fn gc(generation: &str, db: &DB) -> Result<(), rocksdb::Error> {
+        let gen = format!("{}/", generation);
+        for (k, _v) in db.iterator(rocksdb::IteratorMode::Start) {
+            if !k.starts_with(gen.as_bytes()) {
+                info!("deleting stale key: {}", String::from_utf8_lossy(&k));
+                db.delete(k)?;
+            }
+        }
+        Ok(())
+    }
     pub fn create_album_index(&self, albums: &Vec<Album>) -> io::Result<()> {
         // Serialize it to a JSON string.
         let j = serde_json::to_string(albums)?;
@@ -102,4 +127,113 @@ impl Library {
             None
         }
     }
+    // TODO(wathiede): make this a macro like format! to skip the second string create and copy.
+    fn generational_key(generation: &str, key: &str) -> String {
+        format!("{}/{}", generation, key)
+    }
+    pub fn thumbnail(&self, media_items_id: &str, dimensions: (u32, u32)) -> Option<Vec<u8>> {
+        fn generate_thumbnail(
+            lib: &Library,
+            media_items_id: &str,
+            dimensions: (u32, u32),
+        ) -> Result<Vec<u8>, io::Error> {
+            let (w, h) = dimensions;
+            match lib.original(&media_items_id) {
+                None => {
+                    warn!("Couldn't find original {}", &media_items_id);
+                    Err(io::Error::new(
+                        io::ErrorKind::NotFound,
+                        format!("{}", media_items_id),
+                    ))
+                }
+                Some(path) => {
+                    let orig_img = image::io::Reader::open(&path)?
+                        .with_guessed_format()?
+                        .decode()
+                        .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
+                    // TODO (wathiede) resize..fill
+                    let img = orig_img.resize(w, h, image::imageops::FilterType::CatmullRom);
+                    let mut buf = Vec::new();
+                    img.write_to(&mut buf, image::ImageFormat::Jpeg)
+                        .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
+                    Ok(buf)
+                }
+            }
+        }
+        fn cache_key(media_items_id: &str, dimensions: (u32, u32)) -> String {
+            let (w, h) = dimensions;
+            Library::generational_key(
+                LIBRARY_GENERATION,
+                &format!("{}-w={}-h={}", media_items_id, w, h),
+            )
+        }
+        let key = cache_key(media_items_id, dimensions);
+        let db = self.cache_db.clone();
+        match db.get(key.as_bytes()) {
+            // Cache hit, return bytes as-is.
+            Ok(Some(bytes)) => {
+                info!("cache HIT {}", key);
+                Some(bytes)
+            }
+            // Cache miss, fill cache and return.
+            Ok(None) => {
+                info!("cache MISS {}", key);
+                let bytes = match generate_thumbnail(self, media_items_id, dimensions) {
+                    Ok(bytes) => bytes,
+                    Err(e) => {
+                        error!("Failed to generate thumbnail for {}: {}", media_items_id, e);
+                        return None;
+                    }
+                };
+                match db.put(key.as_bytes(), &bytes) {
+                    Ok(_) => Some(bytes),
+                    Err(e) => {
+                        error!("Failed to put bytes to {}: {}", key, e);
+                        None
+                    }
+                }
+            }
+            // RocksDB error.
+            Err(e) => {
+                error!("Failed to search DB for {}: {}", key, e);
+                None
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use tempdir::TempDir;
+
+    #[test]
+    fn clean_db() {
+        let td = TempDir::new("photosync_test").expect("failed to create temporary directory");
+        eprintln!("creating database in {}", td.path().to_string_lossy());
+        let db = DB::open_default(td.path()).expect("failed to open DB");
+        let keys = vec!["one", "two", "three"];
+
+        fn get_keys(db: &DB) -> Vec<String> {
+            db.iterator(rocksdb::IteratorMode::Start)
+                .map(|(k, _v)| String::from_utf8(k.to_vec()).expect("key not utf-8"))
+                .collect()
+        }
+        for k in &keys {
+            for g in vec!["1", "2", "3"] {
+                db.put(Library::generational_key(g, k), k)
+                    .expect("failed to put");
+            }
+        }
+
+        assert_eq!(
+            get_keys(&db),
+            vec![
+                "1/one", "1/three", "1/two", "2/one", "2/three", "2/two", "3/one", "3/three",
+                "3/two"
+            ]
+        );
+        Library::gc("2", &db).expect("failed to GC DB");
+        assert_eq!(get_keys(&db), vec!["2/one", "2/three", "2/two",]);
+    }
 }

src/web.rs

@@ -2,7 +2,6 @@ use std::error::Error;
 use std::net::SocketAddr;
 use std::path::PathBuf;
 
-use log::info;
 use log::warn;
 use prometheus::Encoder;
 use rust_embed::RustEmbed;
@@ -32,6 +31,7 @@ fn metrics() -> impl Filter<Extract = (impl warp::reply::Reply,), Error = Reject
         .with(warp::reply::with::headers(text_headers))
 }
 
+// TODO(wathiede): add caching for hashed files.  Add at least etag for everything.
 fn index(path: warp::path::FullPath) -> Result<impl warp::Reply, warp::Rejection> {
     let path = path.as_str();
     let path = if path.ends_with("/") {
@@ -47,7 +47,6 @@ fn index(path: warp::path::FullPath) -> Result<impl warp::Reply, warp::Rejection
 
             Ok(warp::http::Response::builder()
                 .header("Content-Type", mime.essence_str())
-                .header("Content-Length", bytes.len())
                 .body(bytes.into_owned()))
         }
         None => Err(warp::reject::not_found()),
@@ -74,7 +73,6 @@ fn album(lib: Library, id: String) -> Result<impl warp::Reply, warp::Rejection>
 struct ImageParams {
     w: Option<u32>,
     h: Option<u32>,
-    c: Option<bool>,
 }
 
 fn image(
@@ -82,42 +80,17 @@ fn image(
     media_items_id: String,
     params: ImageParams,
 ) -> Result<impl warp::Reply, warp::Rejection> {
-    info!("image_id {} params {:?}", media_items_id, params);
+    match lib.thumbnail(
-
+        &media_items_id,
-    match lib.original(&media_items_id) {
+        (params.w.unwrap_or(0), params.h.unwrap_or(0)),
+    ) {
         None => {
             warn!("Couldn't find original {}", &media_items_id);
             Err(warp::reject::not_found())
         }
-        Some(path) => {
+        Some(bytes) => Ok(warp::http::Response::builder()
-            let orig_img = image::io::Reader::open(&path)
+            .header("Content-Type", "image/jpeg")
-                .map_err(|e| {
+            .body(bytes)),
-                    warn!("Couldn't open {}: {}", path.to_string_lossy(), e);
-                    warp::reject::not_found()
-                })?
-                .with_guessed_format()
-                .map_err(|e| {
-                    warn!("Couldn't guess format {}: {}", path.to_string_lossy(), e);
-                    warp::reject::not_found()
-                })?
-                .decode()
-                .map_err(|e| {
-                    warn!("Couldn't decode {}: {}", path.to_string_lossy(), e);
-                    warp::reject::not_found()
-                })?;
-            let img = orig_img.resize(
-                params.w.unwrap_or(0),
-                params.h.unwrap_or(0),
-                image::imageops::FilterType::CatmullRom,
-            );
-            let mut buf = Vec::new();
-            img.write_to(&mut buf, image::ImageFormat::Jpeg)
-                .map_err(|e| {
-                    warn!("Couldn't write_to {}: {}", path.to_string_lossy(), e);
-                    warp::reject::not_found()
-                })?;
-            Ok(buf)
-        }
     }
 }