diff --git a/Cargo.lock b/Cargo.lock index ebebcee..03b91b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,16 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +[[package]] +name = "arrayvec" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06f59fe10306bb78facd90d28c2038ad23ffaaefa85bac43c8a434cde383334f" +dependencies = [ + "nodrop", + "odds", +] + [[package]] name = "arrayvec" version = "0.5.1" @@ -126,28 +136,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" -[[package]] -name = "backtrace" -version = "0.3.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4036b9bf40f3cf16aba72a3d65e8a520fc4bafcdc7079aea8f848c58c5b5536" -dependencies = [ - "backtrace-sys", - "cfg-if", - "libc", - "rustc-demangle", -] - -[[package]] -name = "backtrace-sys" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "base64" version = "0.9.3" @@ -179,29 +167,6 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e223af0dc48c96d4f8342ec01a4974f139df863896b316681efd36742f22cc67" -[[package]] -name = "bindgen" -version = "0.49.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c07087f3d5731bf3fb375a81841b99597e25dc11bd3bc72d16d43adf6624a6e" -dependencies = [ - "bitflags", - "cexpr", - "cfg-if", - "clang-sys", - "clap", - "env_logger", - "fxhash", - "lazy_static 1.4.0", - "log 0.4.8", - "peeking_take_while", - "proc-macro2 0.4.30", - "quote 0.6.13", - "regex", - "shlex", - "which", -] - [[package]] name = "bitflags" version = "1.2.1" @@ -215,7 +180,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" dependencies = [ "arrayref", - "arrayvec", + "arrayvec 0.5.1", "constant_time_eq", ] @@ -261,16 +226,6 @@ dependencies = [ "serde", ] -[[package]] -name = "buf_redux" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" -dependencies = [ - "memchr", - "safemem", -] - [[package]] name = "bumpalo" version = "3.2.0" @@ -330,11 +285,13 @@ dependencies = [ [[package]] name = "cacher" version = "0.1.0" +source = "git+https://git.z.xinu.tv/wathiede/cacher#43d364cf47a1d179d75c4e8370d344ffbe747522" dependencies = [ "log 0.4.8", "rusoto_core", "rusoto_credential", "rusoto_s3", + "thiserror", ] [[package]] @@ -355,15 +312,6 @@ dependencies = [ "jobserver", ] -[[package]] -name = "cexpr" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d" -dependencies = [ - "nom 4.2.3", -] - [[package]] name = "cfg-if" version = "0.1.10" @@ -382,17 +330,6 @@ dependencies = [ "time", ] -[[package]] -name = "clang-sys" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clap" version = "2.33.0" @@ -461,6 +398,17 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" +[[package]] +name = "cr2" +version = "0.1.0" +source = "git+https://git.z.xinu.tv/wathiede/cr2#a666d4580c008cdf172faff024c746c38f8acd70" +dependencies = [ + "hexdump", + "lazy_static 1.4.0", + "memmap", + "nom 5.1.2", +] + [[package]] name = "crc32fast" version = "1.2.0" @@ -481,7 +429,7 @@ dependencies = [ "clap", "criterion-plot", "csv", - "itertools", + "itertools 0.8.2", "lazy_static 1.4.0", "num-traits", "oorandom", @@ -502,7 +450,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a01e15e0ea58e8234f96146b1f91fa9d0e4dd7a38da93ff7a75d42c0b9d3a545" dependencies = [ "cast", - "itertools", + "itertools 0.8.2", ] [[package]] @@ -699,28 +647,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "env_logger" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" -dependencies = [ - "atty", - "humantime", - "log 0.4.8", - "regex", - "termcolor", -] - -[[package]] -name = "failure" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" -dependencies = [ - "backtrace", -] - [[package]] name = "fake-simd" version = "0.1.2" @@ -912,15 +838,6 @@ dependencies = [ "slab", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder 1.3.4", -] - [[package]] name = "gcc" version = "0.3.55" @@ -1053,32 +970,6 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "headers" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ca7d8722f33ce2c2db44f95425d6267ed59ca96ce02acbe58320054ceb642" -dependencies = [ - "base64 0.10.1", - "bitflags", - "bytes 0.4.12", - "headers-core", - "http 0.1.21", - "mime 0.3.16", - "sha-1", - "time", -] - -[[package]] -name = "headers-core" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "967131279aaa9f7c20c7205b45a391638a83ab118e6509b2d0ccbe08de044237" -dependencies = [ - "bytes 0.4.12", - "http 0.1.21", -] - [[package]] name = "heck" version = "0.3.1" @@ -1103,6 +994,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" +[[package]] +name = "hexdump" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "850f3f2c33d20c0f96c4485e087dd580ff041d720988ebf4c84a42acf739262b" +dependencies = [ + "arrayvec 0.3.25", + "itertools 0.4.19", +] + [[package]] name = "hexihasher" version = "0.1.0" @@ -1182,15 +1083,6 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] - [[package]] name = "hyper" version = "0.10.16" @@ -1367,10 +1259,14 @@ dependencies = [ [[package]] name = "imageutils" version = "0.1.0" -source = "git+https://git.z.xinu.tv/wathiede/imageutils#d6804fa0f8e0afe2deb54354acc983a1ab59d794" +source = "git+https://git.z.xinu.tv/wathiede/imageutils#b49d30e30fd7d89d242ba7c96b36018a8f9f361e" dependencies = [ + "cr2", "image", "jpeg-decoder", + "log 0.4.8", + "memmap", + "rexif 0.5.0", ] [[package]] @@ -1417,15 +1313,6 @@ dependencies = [ "libc", ] -[[package]] -name = "input_buffer" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1b822cc844905551931d6f81608ed5f50a79c1078a4e2b4d42dbc7c1eedfbf" -dependencies = [ - "bytes 0.4.12", -] - [[package]] name = "iovec" version = "0.1.4" @@ -1435,6 +1322,12 @@ dependencies = [ "libc", ] +[[package]] +name = "itertools" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f" + [[package]] name = "itertools" version = "0.8.2" @@ -1534,34 +1427,25 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "lexical-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616" +dependencies = [ + "arrayvec 0.5.1", + "bitflags", + "cfg-if", + "ryu", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" -[[package]] -name = "libloading" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" -dependencies = [ - "cc", - "winapi 0.3.8", -] - -[[package]] -name = "librocksdb-sys" -version = "6.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a0785e816e1e11e7599388a492c61ef80ddc2afc91e313e61662cce537809be" -dependencies = [ - "bindgen", - "cc", - "glob", - "libc", -] - [[package]] name = "load_image" version = "2.12.0" @@ -1572,7 +1456,7 @@ dependencies = [ "lcms2", "lodepng", "mozjpeg", - "rexif", + "rexif 0.3.7", "rgb", ] @@ -1643,6 +1527,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi 0.3.8", +] + [[package]] name = "memoffset" version = "0.5.3" @@ -1667,18 +1561,6 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" -[[package]] -name = "mime_guess" -version = "1.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d977de9ee851a0b16e932979515c0f3da82403183879811bc97d50bd9cc50f7" -dependencies = [ - "mime 0.2.6", - "phf", - "phf_codegen", - "unicase 1.4.2", -] - [[package]] name = "mime_guess" version = "2.0.1" @@ -1780,7 +1662,7 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f672ed5c96e386d6436643104c9bdc175d4b79ae3eb30538b7a58eb55f3bf318" dependencies = [ - "arrayvec", + "arrayvec 0.5.1", "libc", "mozjpeg-sys", "rgb", @@ -1798,24 +1680,6 @@ dependencies = [ "nasm-rs", ] -[[package]] -name = "multipart" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136eed74cadb9edd2651ffba732b19a450316b680e4f48d6c79e905799e19d01" -dependencies = [ - "buf_redux", - "httparse", - "log 0.4.8", - "mime 0.2.6", - "mime_guess 1.8.7", - "quick-error", - "rand 0.6.5", - "safemem", - "tempfile", - "twoway", -] - [[package]] name = "nasm-rs" version = "0.1.7" @@ -1854,6 +1718,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + [[package]] name = "nom" version = "1.2.4" @@ -1870,6 +1740,17 @@ dependencies = [ "version_check 0.1.5", ] +[[package]] +name = "nom" +version = "5.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" +dependencies = [ + "lexical-core", + "memchr", + "version_check 0.9.1", +] + [[package]] name = "notify" version = "4.0.15" @@ -1888,6 +1769,41 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "num" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg 1.0.0", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg 1.0.0", + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.42" @@ -1916,6 +1832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" dependencies = [ "autocfg 1.0.0", + "num-bigint", "num-integer", "num-traits", ] @@ -1945,6 +1862,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" +[[package]] +name = "odds" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eae0151b9dacf24fcc170d9995e511669a082856a91f958a2fe380bfab3fb22" + [[package]] name = "oorandom" version = "11.1.0" @@ -2038,12 +1961,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "percent-encoding" version = "1.0.1" @@ -2056,45 +1973,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -[[package]] -name = "phf" -version = "0.7.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_codegen" -version = "0.7.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e" -dependencies = [ - "phf_generator", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.7.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" -dependencies = [ - "phf_shared", - "rand 0.6.5", -] - -[[package]] -name = "phf_shared" -version = "0.7.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" -dependencies = [ - "siphasher", - "unicase 1.4.2", -] - [[package]] name = "photosync" version = "0.1.0" @@ -2110,13 +1988,14 @@ dependencies = [ "lazy_static 1.4.0", "load_image", "log 0.4.8", - "mime_guess 2.0.1", + "mime_guess", "prometheus", "regex", "reqwest", "rocket", "rocket_contrib", - "rocksdb", + "rusoto_core", + "rusoto_s3", "rust-embed", "serde", "serde_json", @@ -2124,7 +2003,7 @@ dependencies = [ "stderrlog", "structopt", "tempdir", - "warp", + "thiserror", "yup-oauth2", ] @@ -2600,7 +2479,7 @@ dependencies = [ "lazy_static 1.4.0", "log 0.4.8", "mime 0.3.16", - "mime_guess 2.0.1", + "mime_guess", "native-tls", "percent-encoding 2.1.0", "pin-project-lite", @@ -2626,6 +2505,15 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958f3866c3a81ceaca304745792491944d0c9ed36e46eea43984b88cfac016f2" +[[package]] +name = "rexif" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b18541ee0a90d5ba60c094fd70b8d31cb903d6bc75cdba0f795507077bf6c1dd" +dependencies = [ + "num", +] + [[package]] name = "rgb" version = "0.8.16" @@ -2713,16 +2601,6 @@ dependencies = [ "unicode-xid 0.1.0", ] -[[package]] -name = "rocksdb" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12069b106981c6103d3eab7dd1c86751482d0779a520b7c14954c8b586c1e643" -dependencies = [ - "libc", - "librocksdb-sys", -] - [[package]] name = "rusoto_core" version = "0.42.0" @@ -2849,12 +2727,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "rustc-demangle" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" - [[package]] name = "rustc_version" version = "0.1.7" @@ -2940,12 +2812,6 @@ dependencies = [ "winapi 0.3.8", ] -[[package]] -name = "scoped-tls" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" - [[package]] name = "scoped_threadpool" version = "0.1.9" @@ -3064,18 +2930,6 @@ dependencies = [ "url 2.1.1", ] -[[package]] -name = "sha-1" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" -dependencies = [ - "block-buffer", - "digest", - "fake-simd", - "opaque-debug", -] - [[package]] name = "sha2" version = "0.8.2" @@ -3104,12 +2958,6 @@ dependencies = [ "libc", ] -[[package]] -name = "siphasher" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" - [[package]] name = "slab" version = "0.4.2" @@ -3161,6 +3009,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "stb_image" version = "0.2.2" @@ -3336,6 +3190,26 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thiserror" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793" +dependencies = [ + "proc-macro2 1.0.8", + "quote 1.0.2", + "syn 1.0.14", +] + [[package]] name = "thread_local" version = "0.3.4" @@ -3710,34 +3584,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" -[[package]] -name = "tungstenite" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0c2bd5aeb7dcd2bb32e472c8872759308495e5eccc942e929a513cd8d36110" -dependencies = [ - "base64 0.11.0", - "byteorder 1.3.4", - "bytes 0.4.12", - "http 0.1.21", - "httparse", - "input_buffer", - "log 0.4.8", - "rand 0.7.3", - "sha-1", - "url 2.1.1", - "utf-8", -] - -[[package]] -name = "twoway" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" -dependencies = [ - "memchr", -] - [[package]] name = "typeable" version = "0.1.2" @@ -3857,18 +3703,6 @@ dependencies = [ "percent-encoding 2.1.0", ] -[[package]] -name = "urlencoding" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df3561629a8bb4c57e5a2e4c43348d9e29c7c29d9b1c4c1f47166deca8f37ed" - -[[package]] -name = "utf-8" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" - [[package]] name = "vcpkg" version = "0.2.8" @@ -3931,32 +3765,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "warp" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99b53196ba54e91e31ba1e90309a31053218cc1d4697f6e48f7c7e3d255e64fc" -dependencies = [ - "bytes 0.4.12", - "futures", - "headers", - "http 0.1.21", - "hyper 0.12.35", - "log 0.4.8", - "mime 0.3.16", - "mime_guess 2.0.1", - "multipart", - "scoped-tls", - "serde", - "serde_json", - "serde_urlencoded", - "tokio 0.1.22", - "tokio-io", - "tokio-threadpool", - "tungstenite", - "urlencoding", -] - [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" @@ -4088,16 +3896,6 @@ dependencies = [ "nom 4.2.3", ] -[[package]] -name = "which" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164" -dependencies = [ - "failure", - "libc", -] - [[package]] name = "winapi" version = "0.2.8" @@ -4184,7 +3982,7 @@ dependencies = [ "http 0.1.21", "hyper 0.12.35", "hyper-rustls 0.17.1", - "itertools", + "itertools 0.8.2", "log 0.3.9", "rustls", "serde", diff --git a/Cargo.toml b/Cargo.toml index 5e40965..7b21805 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,16 +19,17 @@ serde_json = "1.0.46" stderrlog = "0.4.3" structopt = "0.3.9" yup-oauth2 = "^3.1" -warp = "0.1" serde = { version = "1.0.104", features = ["derive"] } image = { version = "0.23.2" } #, default-features = false, features = ["jpeg"] } rust-embed = "5.2.0" mime_guess = "2.0.1" -rocksdb = "0.13.0" jpeg-decoder = "0.1.18" imageutils = { git = "https://git.z.xinu.tv/wathiede/imageutils" } -cacher = { path = "../cacher" } +cacher = { git = "https://git.z.xinu.tv/wathiede/cacher" } rocket = "0.4.5" +thiserror = "1.0.20" +rusoto_s3 = "0.42.0" +rusoto_core = "0.42.0" [dependencies.prometheus] features = ["process"] diff --git a/src/library.rs b/src/library.rs index d257373..a665d95 100644 --- a/src/library.rs +++ b/src/library.rs @@ -1,178 +1,102 @@ -use std::fs; -use std::fs::File; use std::io; use std::io::Read; -use std::path::Path; use std::path::PathBuf; -use std::sync::Arc; -use std::sync::Mutex; -use cacher::Cacher; +use cacher::s3::S3CacherError; +use cacher::S3Cacher; use google_photoslibrary1 as photos; use image::imageops; -use imageutils::{load_image, resize, resize_to_fill, save_to_jpeg_bytes, FilterType}; -use log::error; -use log::info; -use log::warn; -use photos::schemas::Album; -use photos::schemas::MediaItem; -use rocksdb::Direction; -use rocksdb::IteratorMode; -use rocksdb::DB; +use imageutils::{load_image_buffer, resize, resize_to_fill, save_to_jpeg_bytes, FilterType}; +use log::{error, info}; +use photos::schemas::{Album, MediaItem}; +use rusoto_core::RusotoError; +use rusoto_s3::GetObjectError; +use thiserror::Error; // Used to ensure DB is invalidated after schema changes. -const LIBRARY_GENERATION: &'static str = "14"; +const LIBRARY_GENERATION: &'static str = "16"; + +#[derive(Error, Debug)] +pub enum LibraryError { + #[error("IO error: {0}")] + IoError(#[from] std::io::Error), + #[error("s3 error: {0}")] + S3CacherError(#[from] S3CacherError), + #[error("json error: {0}")] + JsonError(#[from] serde_json::Error), +} #[derive(Clone)] pub struct Library { - root: PathBuf, - originals_dir: PathBuf, - cache_db: Arc, - image_cache: Arc>>, + s3: S3Cacher, } impl Library { - pub fn new( - root: PathBuf, - image_cache: Arc>>, - ) -> Result> { - let db = DB::open_default(root.join("cache"))?; - let cache_db = Arc::new(db); - let lib = Library { - originals_dir: root.join("images").join("originals"), - cache_db, - root, - image_cache, - }; - let cnt = lib.clean_db()?; - if cnt != 0 { - info!("Deleted {} entries", cnt); - } - if !lib.originals_dir.exists() { - info!( - "create originals dir {}", - &lib.originals_dir.to_string_lossy() - ); - fs::create_dir_all(&lib.originals_dir)?; - } + pub fn new(s3: S3Cacher) -> Result> { + let lib = Library { s3 }; Ok(lib) } - // Removes all data in the database from older schema. - pub fn clean_db(&self) -> Result { - Library::gc(LIBRARY_GENERATION, &self.cache_db) - } - fn gc(generation: &str, db: &DB) -> Result { - let gen = format!("{}/", generation); - // '0' is the next character after '/', so iterator's starting there would be after the - // last `gen` entry. - let next_gen = format!("{}0", generation); - let mut del_cnt = 0; - for (k, _v) in db.iterator(IteratorMode::From(gen.as_bytes(), Direction::Reverse)) { - if !k.starts_with(gen.as_bytes()) { - info!("deleting stale key: {}", String::from_utf8_lossy(&k)); - db.delete(k)?; - del_cnt += 1; - } - } - for (k, _v) in db.iterator(IteratorMode::From(next_gen.as_bytes(), Direction::Forward)) { - if !k.starts_with(gen.as_bytes()) { - info!("deleting stale key: {}", String::from_utf8_lossy(&k)); - db.delete(k)?; - del_cnt += 1; - } - } - Ok(del_cnt) - } - pub fn create_album_index(&self, albums: &Vec) -> io::Result<()> { + pub fn create_album_index(&self, albums: &Vec) -> Result<(), LibraryError> { // Serialize it to a JSON string. let j = serde_json::to_string(albums)?; - let path = self.root.join("albums.json"); - info!("saving {}", path.to_string_lossy()); - fs::write(path, j) + let filename = "albums.json"; + + self.s3 + .set(&Library::generational_key(filename), j.as_ref())?; + Ok(()) } - pub fn create_album>( + pub fn create_album( &self, - album_id: P, + album_id: &str, media_items: &Vec, - ) -> io::Result<()> { - let album_dir = self.root.join(album_id); - if !album_dir.exists() { - info!("making album directory {}", album_dir.to_string_lossy()); - fs::create_dir_all(&album_dir)?; - } + ) -> Result<(), LibraryError> { + let relpath = format!("{}.json", &album_id); let j = serde_json::to_string(&media_items)?; - let path = album_dir.join("album.json"); - info!("saving {}", path.to_string_lossy()); - fs::write(path, j) + + self.s3 + .set(&Library::generational_key(&relpath), j.as_ref())?; + Ok(()) } pub fn albums(&self) -> Result, Box> { - let albums_path = self.root.join("albums.json"); - info!("loading {}", albums_path.to_string_lossy()); - let bytes = fs::read(albums_path)?; - Ok(serde_json::from_slice(&bytes)?) + let filename = "albums.json"; + + let bytes = self.s3.get(&Library::generational_key(filename))?; + let album: Vec = serde_json::from_slice(&bytes)?; + Ok(album) } pub fn album(&self, album_id: &str) -> Result, Box> { - let album_path = self.root.join(album_id).join("album.json"); - let bytes = fs::read(album_path)?; - Ok(serde_json::from_slice(&bytes)?) + let relpath = format!("{}.json", &album_id); + let bytes = self.s3.get(&Library::generational_key(&relpath))?; + let mis: Vec = serde_json::from_slice(&bytes)?; + Ok(mis) } pub fn download_image( &self, - filename: &str, + _filename: &str, media_items_id: &str, base_url: &str, ) -> Result> { - // Put images from all albums in common directory. - let image_path = self.originals_dir.join(media_items_id); - if image_path.exists() { - info!( - "Skipping already downloaded {} @ {}", - &filename, - image_path.to_string_lossy() - ); - } else { - let download_path = image_path.with_extension("download"); - let c = Arc::clone(&self.image_cache); - let mut c = c.lock().unwrap(); - match c.get(media_items_id) { - Some(bytes) => { - info!( - "saving local copy of original from cache {}", - media_items_id - ); - fs::write(&download_path, bytes)?; - } - None => { - let url = format!("{}=d", base_url); - let mut r = reqwest::blocking::get(&url)?; - let mut buf = Vec::new(); - info!("Downloading {}", &url); - r.read_to_end(&mut buf)?; - fs::write(&download_path, &buf); - c.set(media_items_id, &buf); - } - }; - info!( - "Rename {} -> {}", - download_path.to_string_lossy(), - image_path.to_string_lossy() - ); - fs::rename(download_path, &image_path)?; + let filename = Library::generational_key(&format!("images/originals/{}", media_items_id)); + if !self.s3.contains_key(&filename) { + let url = format!("{}=d", base_url); + let mut r = reqwest::blocking::get(&url)?; + let mut buf = Vec::new(); + info!("Downloading {}", &url); + r.read_to_end(&mut buf)?; + self.s3.set(&filename, &buf)?; } - Ok(image_path) + Ok(filename.into()) } - pub fn original(&self, media_items_id: &str) -> Option { - let path = self.originals_dir.join(media_items_id); - if path.exists() { - Some(path) - } else { - None - } + + pub fn original_buffer(&self, media_items_id: &str) -> Result, LibraryError> { + let filename = Library::generational_key(&format!("images/originals/{}", media_items_id)); + let bytes = self.s3.get(&filename)?; + Ok(bytes) } // TODO(wathiede): make this a macro like format! to skip the second string create and copy. - fn generational_key(generation: &str, key: &str) -> String { - format!("{}/{}", generation, key) + fn generational_key(key: &str) -> String { + format!("{}/{}", LIBRARY_GENERATION, key) } pub fn generate_thumbnail( @@ -182,24 +106,22 @@ impl Library { filter: FilterType, fill: bool, ) -> Result, Box> { - match self.original(&media_items_id) { - None => { - warn!("Couldn't find original {}", &media_items_id); - Err(io::Error::new(io::ErrorKind::NotFound, format!("{}", media_items_id)).into()) - } - Some(path) => { - let orig_img = load_image(&path, dimensions.0, dimensions.1)?; - //.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; - let img = if fill { - resize_to_fill(&orig_img, dimensions, filter) - } else { - resize(&orig_img, dimensions, filter) - }; - let buf = save_to_jpeg_bytes(&img) - .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; - Ok(buf) - } - } + let buf = self.original_buffer(&media_items_id)?; + let dimension_hint = match dimensions { + (Some(w), Some(h)) => Some((w, h)), + // Partial dimensions should be handled by the caller of this function. So all + // other options are None. + _ => None, + }; + let orig_img = load_image_buffer(buf, dimension_hint)?; + //.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + let img = if fill { + resize_to_fill(&orig_img, dimensions, filter) + } else { + resize(&orig_img, dimensions, filter) + }; + let buf = save_to_jpeg_bytes(&img).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + Ok(buf) } pub fn thumbnail( &self, @@ -214,96 +136,33 @@ impl Library { (None, Some(h)) => format!("-h={}", h), (None, None) => "".to_string(), }; - Library::generational_key(LIBRARY_GENERATION, &format!("{}{}", media_items_id, dim)) + Library::generational_key(&format!("images/thumbnails/{}-{}", media_items_id, dim)) } let key = cache_key(media_items_id, dimensions); - let db = self.cache_db.clone(); - match db.get(key.as_bytes()) { - // Cache hit, return bytes as-is. - Ok(Some(bytes)) => Some(bytes), - // Cache miss, fill cache and return. - Ok(None) => { - // TODO(wathiede): use cache for thumbnail like download_image does. - let c = Arc::clone(&self.image_cache); - let mut c = c.lock().unwrap(); - let bytes = match c.get(&key) { - Some(bytes) => { - info!( - "saving local copy of thumbnail from cache {}", - media_items_id - ); - bytes - } - None => { - info!("cache MISS {}", key); - let bytes = match self.generate_thumbnail( - media_items_id, - dimensions, - FilterType::Builtin(imageops::FilterType::Lanczos3), - fill, - ) { - Ok(bytes) => bytes, - Err(e) => { - error!( - "Failed to generate thumbnail for {}: {}", - media_items_id, e - ); - return None; - } - }; - c.set(&key, &bytes); - bytes - } - }; - match db.put(key.as_bytes(), &bytes) { - Ok(_) => Some(bytes), - Err(e) => { - error!("Failed to put bytes to {}: {}", key, e); - None - } - } - } - // RocksDB error. + match self.s3.get(&key) { + Ok(bytes) => return Some(bytes), + Err(S3CacherError::GetObjectError(RusotoError::Service( + GetObjectError::NoSuchKey(msg), + ))) => info!("Missing thumbnail {} in s3: {}", key, msg), + Err(e) => error!("Error fetching thumbnail {} from s3: {}", key, e), + }; + + info!("cache MISS {}", key); + let bytes = match self.generate_thumbnail( + media_items_id, + dimensions, + FilterType::Builtin(imageops::FilterType::Lanczos3), + fill, + ) { + Ok(bytes) => bytes, Err(e) => { - error!("Failed to search DB for {}: {}", key, e); - None + error!("Failed to generate thumbnail for {}: {}", media_items_id, e); + return None; } + }; + if let Err(e) = self.s3.set(&key, &bytes) { + error!("Failed to put thumbnail {}: {}", &key, e); } - } -} - -#[cfg(test)] -mod test { - use super::*; - use tempdir::TempDir; - - #[test] - fn clean_db() { - let td = TempDir::new("photosync_test").expect("failed to create temporary directory"); - eprintln!("creating database in {}", td.path().to_string_lossy()); - let db = DB::open_default(td.path()).expect("failed to open DB"); - let keys = vec!["one", "two", "three"]; - - fn get_keys(db: &DB) -> Vec { - db.iterator(rocksdb::IteratorMode::Start) - .map(|(k, _v)| String::from_utf8(k.to_vec()).expect("key not utf-8")) - .collect() - } - for k in &keys { - for g in vec!["1", "2", "3"] { - db.put(Library::generational_key(g, k), k) - .expect("failed to put"); - } - } - - assert_eq!( - get_keys(&db), - vec![ - "1/one", "1/three", "1/two", "2/one", "2/three", "2/two", "3/one", "3/three", - "3/two" - ] - ); - Library::gc("2", &db).expect("failed to GC DB"); - assert_eq!(get_keys(&db), vec!["2/one", "2/three", "2/two",]); + Some(bytes) } } diff --git a/src/main.rs b/src/main.rs index 81ae0f4..13169ee 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,11 +2,10 @@ use std::collections::HashMap; use std::error::Error; use std::net::SocketAddr; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; use std::thread; use std::time; -use cacher::{Cacher, S3Cacher}; +use cacher::S3Cacher; use google_api_auth; use google_photoslibrary1 as photos; use hexihasher; @@ -32,14 +31,13 @@ struct Sync { /// Optional album title to filter. Default will mirror all albums. #[structopt(short, long)] title_filter: Option, - /// Directory to store sync. - root: PathBuf, + /// S3 bucket holding metadata and images. + #[structopt(long, default_value = "photosync-dev")] + s3_bucket: String, } #[derive(Debug, StructOpt)] struct Serve { - /// Directory of data fetched by `sync`. - root: PathBuf, /// HTTP address to listen for web requests. #[structopt(long = "addr", default_value = "0.0.0.0:0")] addr: SocketAddr, @@ -65,6 +63,9 @@ enum Command { Serve { #[structopt(flatten)] serve: Serve, + /// S3 bucket holding metadata and images. + #[structopt(default_value = "photosync-dev")] + s3_bucket: String, }, ServeAndSync { /// Sync albums at given interval. @@ -322,9 +323,6 @@ fn main() -> Result<(), Box> { .init() .unwrap(); debug!("opt: {:?}", opt); - let image_cache: Mutex> = - Mutex::new(Box::new(S3Cacher::new("photosync".to_string())?)); - let image_cache = Arc::new(image_cache); match opt.cmd { Command::ListAlbums { auth, title_filter } => { let client = new_client(&auth.credentials, &auth.token_cache)?; @@ -341,18 +339,21 @@ fn main() -> Result<(), Box> { Sync { auth, title_filter, - root, + s3_bucket, }, } => { + let s3 = S3Cacher::new(s3_bucket.clone())?; let client = new_client(&auth.credentials, &auth.token_cache)?; - let lib = Library::new(root, image_cache)?; + let lib = Library::new(s3)?; sync_albums(&client, &title_filter, &lib)?; Ok(()) } Command::Serve { - serve: Serve { addr, root }, + serve: Serve { addr }, + s3_bucket, } => { - let lib = Library::new(root, image_cache)?; + let s3 = S3Cacher::new(s3_bucket.clone())?; + let lib = Library::new(s3)?; serve(addr, lib) } Command::ServeAndSync { @@ -361,12 +362,14 @@ fn main() -> Result<(), Box> { Sync { auth, title_filter, - root, + s3_bucket, }, + addr, } => { + let s3 = S3Cacher::new(s3_bucket.clone())?; let client = new_client(&auth.credentials, &auth.token_cache)?; - let lib = Library::new(root, image_cache)?; + let lib = Library::new(s3)?; background_sync(client, interval, title_filter, lib.clone())?; serve(addr, lib)?; Ok(()) diff --git a/src/rweb.rs b/src/rweb.rs index 66df9c3..43b7b9e 100644 --- a/src/rweb.rs +++ b/src/rweb.rs @@ -7,6 +7,7 @@ use google_photoslibrary1 as photos; use log::error; use photos::schemas::{Album, MediaItem}; use prometheus::Encoder; +use rocket::config::{Config, Environment}; use rocket::http::ContentType; use rocket::response::status::NotFound; use rocket::response::Content; @@ -116,8 +117,13 @@ fn embedz() -> Content> { Content(ContentType::HTML, w) } -pub fn run(_addr: SocketAddr, lib: Library) -> Result<(), Box> { - let e = rocket::ignite() +pub fn run(addr: SocketAddr, lib: Library) -> Result<(), Box> { + let config = Config::build(Environment::Development) + .address(addr.ip().to_string()) + .port(addr.port()) + .finalize()?; + + let e = rocket::custom(config) .manage(lib) .mount( "/",