diff --git a/Cargo.lock b/Cargo.lock index 37adfb2..90fe71a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,6 +30,18 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c" +[[package]] +name = "arc-swap" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + [[package]] name = "arrayvec" version = "0.5.1" @@ -125,6 +137,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "blake2b_simd" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.7.3" @@ -224,6 +247,16 @@ dependencies = [ "ppv-lite86", ] +[[package]] +name = "cacher" +version = "0.1.0" +dependencies = [ + "log 0.4.8", + "rusoto_core", + "rusoto_credential", + "rusoto_s3", +] + [[package]] name = "cast" version = "0.2.3" @@ -310,6 +343,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd" +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "core-foundation" version = "0.6.4" @@ -377,7 +416,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca" dependencies = [ "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-utils 0.7.0", ] [[package]] @@ -388,12 +427,21 @@ checksum = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac" dependencies = [ "autocfg 0.1.7", "cfg-if", - "crossbeam-utils", + "crossbeam-utils 0.7.0", "lazy_static 1.4.0", "memoffset", "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" +dependencies = [ + "crossbeam-utils 0.6.6", +] + [[package]] name = "crossbeam-queue" version = "0.2.1" @@ -401,7 +449,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" dependencies = [ "cfg-if", - "crossbeam-utils", + "crossbeam-utils 0.7.0", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if", + "lazy_static 1.4.0", ] [[package]] @@ -415,6 +473,16 @@ dependencies = [ "lazy_static 1.4.0", ] +[[package]] +name = "crypto-mac" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "csv" version = "1.1.3" @@ -465,6 +533,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.8", +] + [[package]] name = "dtoa" version = "0.4.5" @@ -850,6 +929,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + [[package]] name = "hexihasher" version = "0.1.0" @@ -859,6 +944,16 @@ dependencies = [ "sha2", ] +[[package]] +name = "hmac" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" +dependencies = [ + "crypto-mac", + "digest", +] + [[package]] name = "http" version = "0.1.21" @@ -1006,6 +1101,19 @@ dependencies = [ "webpki", ] +[[package]] +name = "hyper-tls" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" +dependencies = [ + "bytes 0.4.12", + "futures", + "hyper 0.12.35", + "native-tls", + "tokio-io", +] + [[package]] name = "hyper-tls" version = "0.4.1" @@ -1294,6 +1402,12 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.3.0" @@ -1368,12 +1482,24 @@ dependencies = [ "kernel32-sys", "libc", "log 0.4.8", - "miow", + "miow 0.2.1", "net2", "slab", "winapi 0.2.8", ] +[[package]] +name = "mio-named-pipes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3" +dependencies = [ + "log 0.4.8", + "mio", + "miow 0.3.5", + "winapi 0.3.8", +] + [[package]] name = "mio-uds" version = "0.6.7" @@ -1397,6 +1523,16 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "miow" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e" +dependencies = [ + "socket2", + "winapi 0.3.8", +] + [[package]] name = "mozjpeg" version = "0.8.15" @@ -1682,6 +1818,7 @@ dependencies = [ name = "photosync" version = "0.1.0" dependencies = [ + "cacher", "criterion", "google-photoslibrary1", "google_api_auth", @@ -2073,8 +2210,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9" dependencies = [ "crossbeam-deque", - "crossbeam-queue", - "crossbeam-utils", + "crossbeam-queue 0.2.1", + "crossbeam-utils 0.7.0", "lazy_static 1.4.0", "num_cpus", ] @@ -2103,6 +2240,17 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "redox_users" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b23093265f8d200fa7b4c2c76297f47e681c655f6f1285a8780d6a022f7431" +dependencies = [ + "getrandom", + "redox_syscall", + "rust-argon2", +] + [[package]] name = "regex" version = "1.3.4" @@ -2154,7 +2302,7 @@ dependencies = [ "http-body 0.3.1", "hyper 0.13.2", "hyper-rustls 0.19.1", - "hyper-tls", + "hyper-tls 0.4.1", "js-sys", "lazy_static 1.4.0", "log 0.4.8", @@ -2216,6 +2364,100 @@ dependencies = [ "librocksdb-sys", ] +[[package]] +name = "rusoto_core" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d1ecfe8dac29878a713fbc4c36b0a84a48f7a6883541841cdff9fdd2ba7dfb" +dependencies = [ + "base64 0.11.0", + "bytes 0.4.12", + "futures", + "http 0.1.21", + "hyper 0.12.35", + "hyper-tls 0.3.2", + "lazy_static 1.4.0", + "log 0.4.8", + "rusoto_credential", + "rusoto_signature", + "rustc_version 0.2.3", + "serde", + "serde_derive", + "serde_json", + "time", + "tokio 0.1.22", + "tokio-timer", + "xml-rs", +] + +[[package]] +name = "rusoto_credential" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8632e41d289db90dd40d0389c71a23c5489e3afd448424226529113102e2a002" +dependencies = [ + "chrono", + "dirs", + "futures", + "hyper 0.12.35", + "lazy_static 1.4.0", + "regex", + "serde", + "serde_derive", + "serde_json", + "shlex", + "tokio-process", + "tokio-timer", +] + +[[package]] +name = "rusoto_s3" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fedcadf3d73c2925b05d547b66787f2219c5e727a98c893fff5cf2197dbd678" +dependencies = [ + "bytes 0.4.12", + "futures", + "rusoto_core", + "xml-rs", +] + +[[package]] +name = "rusoto_signature" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7063a70614eb4b36f49bcf4f6f6bb30cc765e3072b317d6afdfe51e7a9f482d1" +dependencies = [ + "base64 0.11.0", + "bytes 0.4.12", + "futures", + "hex", + "hmac", + "http 0.1.21", + "hyper 0.12.35", + "log 0.4.8", + "md5", + "percent-encoding 2.1.0", + "rusoto_credential", + "rustc_version 0.2.3", + "serde", + "sha2", + "time", + "tokio 0.1.22", +] + +[[package]] +name = "rust-argon2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017" +dependencies = [ + "base64 0.11.0", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils 0.7.0", +] + [[package]] name = "rust-embed" version = "5.3.0" @@ -2493,6 +2735,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" +[[package]] +name = "signal-hook-registry" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41" +dependencies = [ + "arc-swap", + "libc", +] + [[package]] name = "siphasher" version = "0.2.3" @@ -2520,6 +2772,18 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" +[[package]] +name = "socket2" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi 0.3.8", +] + [[package]] name = "sourcefile" version = "0.1.4" @@ -2594,6 +2858,12 @@ dependencies = [ "syn", ] +[[package]] +name = "subtle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" + [[package]] name = "syn" version = "1.0.14" @@ -2814,7 +3084,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.0", "futures", ] @@ -2840,13 +3110,32 @@ dependencies = [ "log 0.4.8", ] +[[package]] +name = "tokio-process" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382d90f43fa31caebe5d3bc6cfd854963394fff3b8cb59d5146607aaae7e7e43" +dependencies = [ + "crossbeam-queue 0.1.2", + "futures", + "lazy_static 1.4.0", + "libc", + "log 0.4.8", + "mio", + "mio-named-pipes", + "tokio-io", + "tokio-reactor", + "tokio-signal", + "winapi 0.3.8", +] + [[package]] name = "tokio-reactor" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.0", "futures", "lazy_static 1.4.0", "log 0.4.8", @@ -2885,6 +3174,23 @@ dependencies = [ "webpki", ] +[[package]] +name = "tokio-signal" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c34c6e548f101053321cba3da7cbb87a610b85555884c41b07da2eb91aff12" +dependencies = [ + "futures", + "libc", + "mio", + "mio-uds", + "signal-hook-registry", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "winapi 0.3.8", +] + [[package]] name = "tokio-sync" version = "0.1.8" @@ -2916,8 +3222,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" dependencies = [ "crossbeam-deque", - "crossbeam-queue", - "crossbeam-utils", + "crossbeam-queue 0.2.1", + "crossbeam-utils 0.7.0", "futures", "lazy_static 1.4.0", "log 0.4.8", @@ -2932,7 +3238,7 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.0", "futures", "slab", "tokio-executor", @@ -3214,9 +3520,9 @@ dependencies = [ [[package]] name = "warp" -version = "0.1.20" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3921463c44f680d24f1273ea55efd985f31206a22a02dee207a2ec72684285ca" +checksum = "99b53196ba54e91e31ba1e90309a31053218cc1d4697f6e48f7c7e3d255e64fc" dependencies = [ "bytes 0.4.12", "futures", @@ -3441,6 +3747,12 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + [[package]] name = "yup-oauth2" version = "3.1.1" diff --git a/Cargo.toml b/Cargo.toml index a2cefec..339fa88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ mime_guess = "2.0.1" rocksdb = "0.13.0" jpeg-decoder = "0.1.18" imageutils = { git = "https://git.z.xinu.tv/wathiede/imageutils" } +cacher = { path = "../cacher" } [dependencies.prometheus] features = ["process"] diff --git a/src/library.rs b/src/library.rs index 5802069..4858167 100644 --- a/src/library.rs +++ b/src/library.rs @@ -5,6 +5,7 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use cacher::Cacher; use google_photoslibrary1 as photos; use image::imageops; use imageutils::{load_image, resize, resize_to_fill, save_to_jpeg_bytes, FilterType}; @@ -21,20 +22,28 @@ use rocksdb::DB; const LIBRARY_GENERATION: &'static str = "14"; #[derive(Clone)] -pub struct Library { +pub struct Library +where + C: Cacher, +{ root: PathBuf, originals_dir: PathBuf, cache_db: Arc, + image_cache: C, } -impl Library { - pub fn new(root: PathBuf) -> Result> { +impl Library +where + C: Cacher, +{ + pub fn new(root: PathBuf, image_cache: C) -> Result, Box> { let db = DB::open_default(root.join("cache"))?; let cache_db = Arc::new(db); let lib = Library { originals_dir: root.join("images").join("originals"), cache_db, root, + image_cache, }; let cnt = lib.clean_db()?; if cnt != 0 { @@ -51,7 +60,7 @@ impl Library { } // Removes all data in the database from older schema. pub fn clean_db(&self) -> Result { - Library::gc(LIBRARY_GENERATION, &self.cache_db) + Library::::gc(LIBRARY_GENERATION, &self.cache_db) } fn gc(generation: &str, db: &DB) -> Result { let gen = format!("{}/", generation); @@ -184,16 +193,22 @@ impl Library { dimensions: (Option, Option), fill: bool, ) -> Option> { - fn cache_key(media_items_id: &str, dimensions: (Option, Option)) -> String { + fn cache_key( + media_items_id: &str, + dimensions: (Option, Option), + ) -> String { let dim = match dimensions { (Some(w), Some(h)) => format!("-w={}-h={}", w, h), (Some(w), None) => format!("-w={}", w), (None, Some(h)) => format!("-h={}", h), (None, None) => "".to_string(), }; - Library::generational_key(LIBRARY_GENERATION, &format!("{}{}", media_items_id, dim)) + Library::::generational_key( + LIBRARY_GENERATION, + &format!("{}{}", media_items_id, dim), + ) } - let key = cache_key(media_items_id, dimensions); + let key = cache_key::(media_items_id, dimensions); let db = self.cache_db.clone(); match db.get(key.as_bytes()) { // Cache hit, return bytes as-is. diff --git a/src/main.rs b/src/main.rs index 7aa547e..6636d36 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,7 @@ use std::path::PathBuf; use std::thread; use std::time; +use cacher::S3Cacher; use google_api_auth; use google_photoslibrary1 as photos; use hexihasher; @@ -320,6 +321,7 @@ fn main() -> Result<(), Box> { .init() .unwrap(); debug!("opt: {:?}", opt); + let image_cache = S3Cacher::new("photosync")?; match opt.cmd { Command::ListAlbums { auth, title_filter } => { let client = new_client(&auth.credentials, &auth.token_cache)?; @@ -340,14 +342,14 @@ fn main() -> Result<(), Box> { }, } => { let client = new_client(&auth.credentials, &auth.token_cache)?; - let lib = Library::new(root)?; + let lib = Library::new(root, image_cache)?; sync_albums(&client, &title_filter, &lib)?; Ok(()) } Command::Serve { serve: Serve { addr, root }, } => { - let lib = Library::new(root)?; + let lib = Library::new(root, image_cache)?; serve(addr, lib) } Command::ServeAndSync { @@ -361,7 +363,7 @@ fn main() -> Result<(), Box> { addr, } => { let client = new_client(&auth.credentials, &auth.token_cache)?; - let lib = Library::new(root)?; + let lib = Library::new(root, image_cache)?; background_sync(client, interval, title_filter, lib.clone())?; serve(addr, lib)?; Ok(()) diff --git a/src/web.rs b/src/web.rs index 8d832e8..fcac90d 100644 --- a/src/web.rs +++ b/src/web.rs @@ -2,6 +2,7 @@ use std::error::Error; use std::io::Write; use std::net::SocketAddr; +use cacher::Cacher; use log::warn; use prometheus::Encoder; use rust_embed::RustEmbed; @@ -53,7 +54,7 @@ fn index(path: warp::path::FullPath) -> Result Result { +fn albums(lib: Library) -> Result { let albums = lib.albums().map_err(|e| { warn!("Couldn't find albums: {}", e); warp::reject::not_found() @@ -61,7 +62,7 @@ fn albums(lib: Library) -> Result { Ok(warp::reply::json(&albums)) } -fn album(lib: Library, id: String) -> Result { +fn album(lib: Library, id: String) -> Result { let album = lib.album(&id).map_err(|e| { warn!("Couldn't find album {}: {}", id, e); warp::reject::not_found() @@ -76,8 +77,8 @@ struct ImageParams { fill: Option, } -fn image( - lib: Library, +fn image( + lib: Library, media_items_id: String, params: ImageParams, ) -> Result { @@ -122,24 +123,24 @@ fn embedz() -> Result { .body(w)) } -pub fn run(addr: SocketAddr, lib: Library) -> Result<(), Box> { +pub fn run(addr: SocketAddr, lib: Library) -> Result<(), Box> { let lib = warp::any().map(move || lib.clone()); let index = warp::get2().and(warp::path::full()).and_then(index); - let albums = warp::path("albums").and(lib.clone()).and_then(albums); + let albums = warp::path("albums").and(lib.clone()).and_then(index); let embedz = warp::path("embedz").and_then(embedz); let album = warp::path("album") .and(lib.clone()) .and(warp::path::param()) - .and_then(album); + .and_then(index); let image = warp::path("image") .and(lib.clone()) .and(warp::path::param()) .and(warp::query::()) - .and_then(image); + .and_then(index); let api = albums.or(album).or(image); let api = warp::path("api").and(api);