Attempt to create a cacher but I can't make it work with warp.

This commit is contained in:
Bill Thiede 2020-06-19 20:37:51 -07:00
parent 4fc0bc9d01
commit a19874fe47
5 changed files with 363 additions and 32 deletions

340
Cargo.lock generated
View File

@ -30,6 +30,18 @@ version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c"
[[package]]
name = "arc-swap"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034"
[[package]]
name = "arrayref"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
[[package]]
name = "arrayvec"
version = "0.5.1"
@ -125,6 +137,17 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "blake2b_simd"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a"
dependencies = [
"arrayref",
"arrayvec",
"constant_time_eq",
]
[[package]]
name = "block-buffer"
version = "0.7.3"
@ -224,6 +247,16 @@ dependencies = [
"ppv-lite86",
]
[[package]]
name = "cacher"
version = "0.1.0"
dependencies = [
"log 0.4.8",
"rusoto_core",
"rusoto_credential",
"rusoto_s3",
]
[[package]]
name = "cast"
version = "0.2.3"
@ -310,6 +343,12 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dbbb57365263e881e805dc77d94697c9118fd94d8da011240555aa7b23445bd"
[[package]]
name = "constant_time_eq"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
[[package]]
name = "core-foundation"
version = "0.6.4"
@ -377,7 +416,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
"crossbeam-utils 0.7.0",
]
[[package]]
@ -388,12 +427,21 @@ checksum = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac"
dependencies = [
"autocfg 0.1.7",
"cfg-if",
"crossbeam-utils",
"crossbeam-utils 0.7.0",
"lazy_static 1.4.0",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-queue"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
dependencies = [
"crossbeam-utils 0.6.6",
]
[[package]]
name = "crossbeam-queue"
version = "0.2.1"
@ -401,7 +449,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db"
dependencies = [
"cfg-if",
"crossbeam-utils",
"crossbeam-utils 0.7.0",
]
[[package]]
name = "crossbeam-utils"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
dependencies = [
"cfg-if",
"lazy_static 1.4.0",
]
[[package]]
@ -415,6 +473,16 @@ dependencies = [
"lazy_static 1.4.0",
]
[[package]]
name = "crypto-mac"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5"
dependencies = [
"generic-array",
"subtle",
]
[[package]]
name = "csv"
version = "1.1.3"
@ -465,6 +533,17 @@ dependencies = [
"generic-array",
]
[[package]]
name = "dirs"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901"
dependencies = [
"libc",
"redox_users",
"winapi 0.3.8",
]
[[package]]
name = "dtoa"
version = "0.4.5"
@ -850,6 +929,12 @@ dependencies = [
"libc",
]
[[package]]
name = "hex"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35"
[[package]]
name = "hexihasher"
version = "0.1.0"
@ -859,6 +944,16 @@ dependencies = [
"sha2",
]
[[package]]
name = "hmac"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695"
dependencies = [
"crypto-mac",
"digest",
]
[[package]]
name = "http"
version = "0.1.21"
@ -1006,6 +1101,19 @@ dependencies = [
"webpki",
]
[[package]]
name = "hyper-tls"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f"
dependencies = [
"bytes 0.4.12",
"futures",
"hyper 0.12.35",
"native-tls",
"tokio-io",
]
[[package]]
name = "hyper-tls"
version = "0.4.1"
@ -1294,6 +1402,12 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]]
name = "md5"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "memchr"
version = "2.3.0"
@ -1368,12 +1482,24 @@ dependencies = [
"kernel32-sys",
"libc",
"log 0.4.8",
"miow",
"miow 0.2.1",
"net2",
"slab",
"winapi 0.2.8",
]
[[package]]
name = "mio-named-pipes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3"
dependencies = [
"log 0.4.8",
"mio",
"miow 0.3.5",
"winapi 0.3.8",
]
[[package]]
name = "mio-uds"
version = "0.6.7"
@ -1397,6 +1523,16 @@ dependencies = [
"ws2_32-sys",
]
[[package]]
name = "miow"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e"
dependencies = [
"socket2",
"winapi 0.3.8",
]
[[package]]
name = "mozjpeg"
version = "0.8.15"
@ -1682,6 +1818,7 @@ dependencies = [
name = "photosync"
version = "0.1.0"
dependencies = [
"cacher",
"criterion",
"google-photoslibrary1",
"google_api_auth",
@ -2073,8 +2210,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9"
dependencies = [
"crossbeam-deque",
"crossbeam-queue",
"crossbeam-utils",
"crossbeam-queue 0.2.1",
"crossbeam-utils 0.7.0",
"lazy_static 1.4.0",
"num_cpus",
]
@ -2103,6 +2240,17 @@ dependencies = [
"redox_syscall",
]
[[package]]
name = "redox_users"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09b23093265f8d200fa7b4c2c76297f47e681c655f6f1285a8780d6a022f7431"
dependencies = [
"getrandom",
"redox_syscall",
"rust-argon2",
]
[[package]]
name = "regex"
version = "1.3.4"
@ -2154,7 +2302,7 @@ dependencies = [
"http-body 0.3.1",
"hyper 0.13.2",
"hyper-rustls 0.19.1",
"hyper-tls",
"hyper-tls 0.4.1",
"js-sys",
"lazy_static 1.4.0",
"log 0.4.8",
@ -2216,6 +2364,100 @@ dependencies = [
"librocksdb-sys",
]
[[package]]
name = "rusoto_core"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1d1ecfe8dac29878a713fbc4c36b0a84a48f7a6883541841cdff9fdd2ba7dfb"
dependencies = [
"base64 0.11.0",
"bytes 0.4.12",
"futures",
"http 0.1.21",
"hyper 0.12.35",
"hyper-tls 0.3.2",
"lazy_static 1.4.0",
"log 0.4.8",
"rusoto_credential",
"rusoto_signature",
"rustc_version 0.2.3",
"serde",
"serde_derive",
"serde_json",
"time",
"tokio 0.1.22",
"tokio-timer",
"xml-rs",
]
[[package]]
name = "rusoto_credential"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8632e41d289db90dd40d0389c71a23c5489e3afd448424226529113102e2a002"
dependencies = [
"chrono",
"dirs",
"futures",
"hyper 0.12.35",
"lazy_static 1.4.0",
"regex",
"serde",
"serde_derive",
"serde_json",
"shlex",
"tokio-process",
"tokio-timer",
]
[[package]]
name = "rusoto_s3"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fedcadf3d73c2925b05d547b66787f2219c5e727a98c893fff5cf2197dbd678"
dependencies = [
"bytes 0.4.12",
"futures",
"rusoto_core",
"xml-rs",
]
[[package]]
name = "rusoto_signature"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7063a70614eb4b36f49bcf4f6f6bb30cc765e3072b317d6afdfe51e7a9f482d1"
dependencies = [
"base64 0.11.0",
"bytes 0.4.12",
"futures",
"hex",
"hmac",
"http 0.1.21",
"hyper 0.12.35",
"log 0.4.8",
"md5",
"percent-encoding 2.1.0",
"rusoto_credential",
"rustc_version 0.2.3",
"serde",
"sha2",
"time",
"tokio 0.1.22",
]
[[package]]
name = "rust-argon2"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017"
dependencies = [
"base64 0.11.0",
"blake2b_simd",
"constant_time_eq",
"crossbeam-utils 0.7.0",
]
[[package]]
name = "rust-embed"
version = "5.3.0"
@ -2493,6 +2735,16 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
[[package]]
name = "signal-hook-registry"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41"
dependencies = [
"arc-swap",
"libc",
]
[[package]]
name = "siphasher"
version = "0.2.3"
@ -2520,6 +2772,18 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc"
[[package]]
name = "socket2"
version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"winapi 0.3.8",
]
[[package]]
name = "sourcefile"
version = "0.1.4"
@ -2594,6 +2858,12 @@ dependencies = [
"syn",
]
[[package]]
name = "subtle"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee"
[[package]]
name = "syn"
version = "1.0.14"
@ -2814,7 +3084,7 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671"
dependencies = [
"crossbeam-utils",
"crossbeam-utils 0.7.0",
"futures",
]
@ -2840,13 +3110,32 @@ dependencies = [
"log 0.4.8",
]
[[package]]
name = "tokio-process"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "382d90f43fa31caebe5d3bc6cfd854963394fff3b8cb59d5146607aaae7e7e43"
dependencies = [
"crossbeam-queue 0.1.2",
"futures",
"lazy_static 1.4.0",
"libc",
"log 0.4.8",
"mio",
"mio-named-pipes",
"tokio-io",
"tokio-reactor",
"tokio-signal",
"winapi 0.3.8",
]
[[package]]
name = "tokio-reactor"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351"
dependencies = [
"crossbeam-utils",
"crossbeam-utils 0.7.0",
"futures",
"lazy_static 1.4.0",
"log 0.4.8",
@ -2885,6 +3174,23 @@ dependencies = [
"webpki",
]
[[package]]
name = "tokio-signal"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0c34c6e548f101053321cba3da7cbb87a610b85555884c41b07da2eb91aff12"
dependencies = [
"futures",
"libc",
"mio",
"mio-uds",
"signal-hook-registry",
"tokio-executor",
"tokio-io",
"tokio-reactor",
"winapi 0.3.8",
]
[[package]]
name = "tokio-sync"
version = "0.1.8"
@ -2916,8 +3222,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89"
dependencies = [
"crossbeam-deque",
"crossbeam-queue",
"crossbeam-utils",
"crossbeam-queue 0.2.1",
"crossbeam-utils 0.7.0",
"futures",
"lazy_static 1.4.0",
"log 0.4.8",
@ -2932,7 +3238,7 @@ version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296"
dependencies = [
"crossbeam-utils",
"crossbeam-utils 0.7.0",
"futures",
"slab",
"tokio-executor",
@ -3214,9 +3520,9 @@ dependencies = [
[[package]]
name = "warp"
version = "0.1.20"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3921463c44f680d24f1273ea55efd985f31206a22a02dee207a2ec72684285ca"
checksum = "99b53196ba54e91e31ba1e90309a31053218cc1d4697f6e48f7c7e3d255e64fc"
dependencies = [
"bytes 0.4.12",
"futures",
@ -3441,6 +3747,12 @@ dependencies = [
"winapi-build",
]
[[package]]
name = "xml-rs"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a"
[[package]]
name = "yup-oauth2"
version = "3.1.1"

View File

@ -27,6 +27,7 @@ mime_guess = "2.0.1"
rocksdb = "0.13.0"
jpeg-decoder = "0.1.18"
imageutils = { git = "https://git.z.xinu.tv/wathiede/imageutils" }
cacher = { path = "../cacher" }
[dependencies.prometheus]
features = ["process"]

View File

@ -5,6 +5,7 @@ use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use cacher::Cacher;
use google_photoslibrary1 as photos;
use image::imageops;
use imageutils::{load_image, resize, resize_to_fill, save_to_jpeg_bytes, FilterType};
@ -21,20 +22,28 @@ use rocksdb::DB;
const LIBRARY_GENERATION: &'static str = "14";
#[derive(Clone)]
pub struct Library {
pub struct Library<C>
where
C: Cacher,
{
root: PathBuf,
originals_dir: PathBuf,
cache_db: Arc<DB>,
image_cache: C,
}
impl Library {
pub fn new(root: PathBuf) -> Result<Library, Box<dyn std::error::Error>> {
impl<C> Library<C>
where
C: Cacher,
{
pub fn new(root: PathBuf, image_cache: C) -> Result<Library<C>, Box<dyn std::error::Error>> {
let db = DB::open_default(root.join("cache"))?;
let cache_db = Arc::new(db);
let lib = Library {
originals_dir: root.join("images").join("originals"),
cache_db,
root,
image_cache,
};
let cnt = lib.clean_db()?;
if cnt != 0 {
@ -51,7 +60,7 @@ impl Library {
}
// Removes all data in the database from older schema.
pub fn clean_db(&self) -> Result<usize, rocksdb::Error> {
Library::gc(LIBRARY_GENERATION, &self.cache_db)
Library::<C>::gc(LIBRARY_GENERATION, &self.cache_db)
}
fn gc(generation: &str, db: &DB) -> Result<usize, rocksdb::Error> {
let gen = format!("{}/", generation);
@ -184,16 +193,22 @@ impl Library {
dimensions: (Option<u32>, Option<u32>),
fill: bool,
) -> Option<Vec<u8>> {
fn cache_key(media_items_id: &str, dimensions: (Option<u32>, Option<u32>)) -> String {
fn cache_key<C: Cacher>(
media_items_id: &str,
dimensions: (Option<u32>, Option<u32>),
) -> String {
let dim = match dimensions {
(Some(w), Some(h)) => format!("-w={}-h={}", w, h),
(Some(w), None) => format!("-w={}", w),
(None, Some(h)) => format!("-h={}", h),
(None, None) => "".to_string(),
};
Library::generational_key(LIBRARY_GENERATION, &format!("{}{}", media_items_id, dim))
Library::<C>::generational_key(
LIBRARY_GENERATION,
&format!("{}{}", media_items_id, dim),
)
}
let key = cache_key(media_items_id, dimensions);
let key = cache_key::<C>(media_items_id, dimensions);
let db = self.cache_db.clone();
match db.get(key.as_bytes()) {
// Cache hit, return bytes as-is.

View File

@ -5,6 +5,7 @@ use std::path::PathBuf;
use std::thread;
use std::time;
use cacher::S3Cacher;
use google_api_auth;
use google_photoslibrary1 as photos;
use hexihasher;
@ -320,6 +321,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
.init()
.unwrap();
debug!("opt: {:?}", opt);
let image_cache = S3Cacher::new("photosync")?;
match opt.cmd {
Command::ListAlbums { auth, title_filter } => {
let client = new_client(&auth.credentials, &auth.token_cache)?;
@ -340,14 +342,14 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
},
} => {
let client = new_client(&auth.credentials, &auth.token_cache)?;
let lib = Library::new(root)?;
let lib = Library::new(root, image_cache)?;
sync_albums(&client, &title_filter, &lib)?;
Ok(())
}
Command::Serve {
serve: Serve { addr, root },
} => {
let lib = Library::new(root)?;
let lib = Library::new(root, image_cache)?;
serve(addr, lib)
}
Command::ServeAndSync {
@ -361,7 +363,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
addr,
} => {
let client = new_client(&auth.credentials, &auth.token_cache)?;
let lib = Library::new(root)?;
let lib = Library::new(root, image_cache)?;
background_sync(client, interval, title_filter, lib.clone())?;
serve(addr, lib)?;
Ok(())

View File

@ -2,6 +2,7 @@ use std::error::Error;
use std::io::Write;
use std::net::SocketAddr;
use cacher::Cacher;
use log::warn;
use prometheus::Encoder;
use rust_embed::RustEmbed;
@ -53,7 +54,7 @@ fn index(path: warp::path::FullPath) -> Result<impl warp::Reply, warp::Rejection
}
}
fn albums(lib: Library) -> Result<impl warp::Reply, warp::Rejection> {
fn albums<C: Cacher>(lib: Library<C>) -> Result<impl warp::Reply, warp::Rejection> {
let albums = lib.albums().map_err(|e| {
warn!("Couldn't find albums: {}", e);
warp::reject::not_found()
@ -61,7 +62,7 @@ fn albums(lib: Library) -> Result<impl warp::Reply, warp::Rejection> {
Ok(warp::reply::json(&albums))
}
fn album(lib: Library, id: String) -> Result<impl warp::Reply, warp::Rejection> {
fn album<C: Cacher>(lib: Library<C>, id: String) -> Result<impl warp::Reply, warp::Rejection> {
let album = lib.album(&id).map_err(|e| {
warn!("Couldn't find album {}: {}", id, e);
warp::reject::not_found()
@ -76,8 +77,8 @@ struct ImageParams {
fill: Option<bool>,
}
fn image(
lib: Library,
fn image<C: Cacher>(
lib: Library<C>,
media_items_id: String,
params: ImageParams,
) -> Result<impl warp::Reply, warp::Rejection> {
@ -122,24 +123,24 @@ fn embedz() -> Result<impl warp::Reply, warp::Rejection> {
.body(w))
}
pub fn run(addr: SocketAddr, lib: Library) -> Result<(), Box<dyn Error>> {
pub fn run<C: Cacher>(addr: SocketAddr, lib: Library<C>) -> Result<(), Box<dyn Error>> {
let lib = warp::any().map(move || lib.clone());
let index = warp::get2().and(warp::path::full()).and_then(index);
let albums = warp::path("albums").and(lib.clone()).and_then(albums);
let albums = warp::path("albums").and(lib.clone()).and_then(index);
let embedz = warp::path("embedz").and_then(embedz);
let album = warp::path("album")
.and(lib.clone())
.and(warp::path::param())
.and_then(album);
.and_then(index);
let image = warp::path("image")
.and(lib.clone())
.and(warp::path::param())
.and(warp::query::<ImageParams>())
.and_then(image);
.and_then(index);
let api = albums.or(album).or(image);
let api = warp::path("api").and(api);