wathiede/photosync
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

Rewrite to use s3 instead of local files and rocksdb.

Browse Source
This commit is contained in:
Bill Thiede 2020-06-22 20:48:15 -07:00
parent 337f5dfd49
commit 1bca4c3642
5 changed files with 303 additions and 636 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

546
Cargo.lock generated
View File

@ -97,6 +97,16 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
[[package]]
name = "arrayvec"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06f59fe10306bb78facd90d28c2038ad23ffaaefa85bac43c8a434cde383334f"
dependencies = [
"nodrop",
"odds",
]
[[package]] [[package]]
name = "arrayvec" name = "arrayvec"
version = "0.5.1" version = "0.5.1"
@ -126,28 +136,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
[[package]]
name = "backtrace"
version = "0.3.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4036b9bf40f3cf16aba72a3d65e8a520fc4bafcdc7079aea8f848c58c5b5536"
dependencies = [
"backtrace-sys",
"cfg-if",
"libc",
"rustc-demangle",
]
[[package]]
name = "backtrace-sys"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491"
dependencies = [
"cc",
"libc",
]
[[package]] [[package]]
name = "base64" name = "base64"
version = "0.9.3" version = "0.9.3"
@ -179,29 +167,6 @@ version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e223af0dc48c96d4f8342ec01a4974f139df863896b316681efd36742f22cc67" checksum = "e223af0dc48c96d4f8342ec01a4974f139df863896b316681efd36742f22cc67"
[[package]]
name = "bindgen"
version = "0.49.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c07087f3d5731bf3fb375a81841b99597e25dc11bd3bc72d16d43adf6624a6e"
dependencies = [
"bitflags",
"cexpr",
"cfg-if",
"clang-sys",
"clap",
"env_logger",
"fxhash",
"lazy_static 1.4.0",
"log 0.4.8",
"peeking_take_while",
"proc-macro2 0.4.30",
"quote 0.6.13",
"regex",
"shlex",
"which",
]
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.2.1" version = "1.2.1"
@ -215,7 +180,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a"
dependencies = [ dependencies = [
"arrayref", "arrayref",
"arrayvec", "arrayvec 0.5.1",
"constant_time_eq", "constant_time_eq",
] ]
@ -261,16 +226,6 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "buf_redux"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f"
dependencies = [
"memchr",
"safemem",
]
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.2.0" version = "3.2.0"
@ -330,11 +285,13 @@ dependencies = [
[[package]] [[package]]
name = "cacher" name = "cacher"
version = "0.1.0" version = "0.1.0"
source = "git+https://git.z.xinu.tv/wathiede/cacher#43d364cf47a1d179d75c4e8370d344ffbe747522"
dependencies = [ dependencies = [
"log 0.4.8", "log 0.4.8",
"rusoto_core", "rusoto_core",
"rusoto_credential", "rusoto_credential",
"rusoto_s3", "rusoto_s3",
"thiserror",
] ]
[[package]] [[package]]
@ -355,15 +312,6 @@ dependencies = [
"jobserver", "jobserver",
] ]
[[package]]
name = "cexpr"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d"
dependencies = [
"nom 4.2.3",
]
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
version = "0.1.10" version = "0.1.10"
@ -382,17 +330,6 @@ dependencies = [
"time", "time",
] ]
[[package]]
name = "clang-sys"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853"
dependencies = [
"glob",
"libc",
"libloading",
]
[[package]] [[package]]
name = "clap" name = "clap"
version = "2.33.0" version = "2.33.0"
@ -461,6 +398,17 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b"
[[package]]
name = "cr2"
version = "0.1.0"
source = "git+https://git.z.xinu.tv/wathiede/cr2#a666d4580c008cdf172faff024c746c38f8acd70"
dependencies = [
"hexdump",
"lazy_static 1.4.0",
"memmap",
"nom 5.1.2",
]
[[package]] [[package]]
name = "crc32fast" name = "crc32fast"
version = "1.2.0" version = "1.2.0"
@ -481,7 +429,7 @@ dependencies = [
"clap", "clap",
"criterion-plot", "criterion-plot",
"csv", "csv",
"itertools", "itertools 0.8.2",
"lazy_static 1.4.0", "lazy_static 1.4.0",
"num-traits", "num-traits",
"oorandom", "oorandom",
@ -502,7 +450,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a01e15e0ea58e8234f96146b1f91fa9d0e4dd7a38da93ff7a75d42c0b9d3a545" checksum = "a01e15e0ea58e8234f96146b1f91fa9d0e4dd7a38da93ff7a75d42c0b9d3a545"
dependencies = [ dependencies = [
"cast", "cast",
"itertools", "itertools 0.8.2",
] ]
[[package]] [[package]]
@ -699,28 +647,6 @@ dependencies = [
"cfg-if", "cfg-if",
] ]
[[package]]
name = "env_logger"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3"
dependencies = [
"atty",
"humantime",
"log 0.4.8",
"regex",
"termcolor",
]
[[package]]
name = "failure"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9"
dependencies = [
"backtrace",
]
[[package]] [[package]]
name = "fake-simd" name = "fake-simd"
version = "0.1.2" version = "0.1.2"
@ -912,15 +838,6 @@ dependencies = [
"slab", "slab",
] ]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder 1.3.4",
]
[[package]] [[package]]
name = "gcc" name = "gcc"
version = "0.3.55" version = "0.3.55"
@ -1053,32 +970,6 @@ dependencies = [
"tokio-util", "tokio-util",
] ]
[[package]]
name = "headers"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "882ca7d8722f33ce2c2db44f95425d6267ed59ca96ce02acbe58320054ceb642"
dependencies = [
"base64 0.10.1",
"bitflags",
"bytes 0.4.12",
"headers-core",
"http 0.1.21",
"mime 0.3.16",
"sha-1",
"time",
]
[[package]]
name = "headers-core"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "967131279aaa9f7c20c7205b45a391638a83ab118e6509b2d0ccbe08de044237"
dependencies = [
"bytes 0.4.12",
"http 0.1.21",
]
[[package]] [[package]]
name = "heck" name = "heck"
version = "0.3.1" version = "0.3.1"
@ -1103,6 +994,16 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35"
[[package]]
name = "hexdump"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "850f3f2c33d20c0f96c4485e087dd580ff041d720988ebf4c84a42acf739262b"
dependencies = [
"arrayvec 0.3.25",
"itertools 0.4.19",
]
[[package]] [[package]]
name = "hexihasher" name = "hexihasher"
version = "0.1.0" version = "0.1.0"
@ -1182,15 +1083,6 @@ version = "1.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
[[package]]
name = "humantime"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
dependencies = [
"quick-error",
]
[[package]] [[package]]
name = "hyper" name = "hyper"
version = "0.10.16" version = "0.10.16"
@ -1367,10 +1259,14 @@ dependencies = [
[[package]] [[package]]
name = "imageutils" name = "imageutils"
version = "0.1.0" version = "0.1.0"
source = "git+https://git.z.xinu.tv/wathiede/imageutils#d6804fa0f8e0afe2deb54354acc983a1ab59d794" source = "git+https://git.z.xinu.tv/wathiede/imageutils#b49d30e30fd7d89d242ba7c96b36018a8f9f361e"
dependencies = [ dependencies = [
"cr2",
"image", "image",
"jpeg-decoder", "jpeg-decoder",
"log 0.4.8",
"memmap",
"rexif 0.5.0",
] ]
[[package]] [[package]]
@ -1417,15 +1313,6 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "input_buffer"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e1b822cc844905551931d6f81608ed5f50a79c1078a4e2b4d42dbc7c1eedfbf"
dependencies = [
"bytes 0.4.12",
]
[[package]] [[package]]
name = "iovec" name = "iovec"
version = "0.1.4" version = "0.1.4"
@ -1435,6 +1322,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "itertools"
version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f"
[[package]] [[package]]
name = "itertools" name = "itertools"
version = "0.8.2" version = "0.8.2"
@ -1534,34 +1427,25 @@ dependencies = [
"pkg-config", "pkg-config",
] ]
[[package]]
name = "lexical-core"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616"
dependencies = [
"arrayvec 0.5.1",
"bitflags",
"cfg-if",
"ryu",
"static_assertions",
]
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.66" version = "0.2.66"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
[[package]]
name = "libloading"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
dependencies = [
"cc",
"winapi 0.3.8",
]
[[package]]
name = "librocksdb-sys"
version = "6.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a0785e816e1e11e7599388a492c61ef80ddc2afc91e313e61662cce537809be"
dependencies = [
"bindgen",
"cc",
"glob",
"libc",
]
[[package]] [[package]]
name = "load_image" name = "load_image"
version = "2.12.0" version = "2.12.0"
@ -1572,7 +1456,7 @@ dependencies = [
"lcms2", "lcms2",
"lodepng", "lodepng",
"mozjpeg", "mozjpeg",
"rexif", "rexif 0.3.7",
"rgb", "rgb",
] ]
@ -1643,6 +1527,16 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" checksum = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223"
[[package]]
name = "memmap"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
dependencies = [
"libc",
"winapi 0.3.8",
]
[[package]] [[package]]
name = "memoffset" name = "memoffset"
version = "0.5.3" version = "0.5.3"
@ -1667,18 +1561,6 @@ version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "mime_guess"
version = "1.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d977de9ee851a0b16e932979515c0f3da82403183879811bc97d50bd9cc50f7"
dependencies = [
"mime 0.2.6",
"phf",
"phf_codegen",
"unicase 1.4.2",
]
[[package]] [[package]]
name = "mime_guess" name = "mime_guess"
version = "2.0.1" version = "2.0.1"
@ -1780,7 +1662,7 @@ version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f672ed5c96e386d6436643104c9bdc175d4b79ae3eb30538b7a58eb55f3bf318" checksum = "f672ed5c96e386d6436643104c9bdc175d4b79ae3eb30538b7a58eb55f3bf318"
dependencies = [ dependencies = [
"arrayvec", "arrayvec 0.5.1",
"libc", "libc",
"mozjpeg-sys", "mozjpeg-sys",
"rgb", "rgb",
@ -1798,24 +1680,6 @@ dependencies = [
"nasm-rs", "nasm-rs",
] ]
[[package]]
name = "multipart"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "136eed74cadb9edd2651ffba732b19a450316b680e4f48d6c79e905799e19d01"
dependencies = [
"buf_redux",
"httparse",
"log 0.4.8",
"mime 0.2.6",
"mime_guess 1.8.7",
"quick-error",
"rand 0.6.5",
"safemem",
"tempfile",
"twoway",
]
[[package]] [[package]]
name = "nasm-rs" name = "nasm-rs"
version = "0.1.7" version = "0.1.7"
@ -1854,6 +1718,12 @@ dependencies = [
"winapi 0.3.8", "winapi 0.3.8",
] ]
[[package]]
name = "nodrop"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
[[package]] [[package]]
name = "nom" name = "nom"
version = "1.2.4" version = "1.2.4"
@ -1870,6 +1740,17 @@ dependencies = [
"version_check 0.1.5", "version_check 0.1.5",
] ]
[[package]]
name = "nom"
version = "5.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
dependencies = [
"lexical-core",
"memchr",
"version_check 0.9.1",
]
[[package]] [[package]]
name = "notify" name = "notify"
version = "4.0.15" version = "4.0.15"
@ -1888,6 +1769,41 @@ dependencies = [
"winapi 0.3.8", "winapi 0.3.8",
] ]
[[package]]
name = "num"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"
dependencies = [
"num-bigint",
"num-complex",
"num-integer",
"num-iter",
"num-rational",
"num-traits",
]
[[package]]
name = "num-bigint"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
dependencies = [
"autocfg 1.0.0",
"num-integer",
"num-traits",
]
[[package]]
name = "num-complex"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
dependencies = [
"autocfg 1.0.0",
"num-traits",
]
[[package]] [[package]]
name = "num-integer" name = "num-integer"
version = "0.1.42" version = "0.1.42"
@ -1916,6 +1832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3"
dependencies = [ dependencies = [
"autocfg 1.0.0", "autocfg 1.0.0",
"num-bigint",
"num-integer", "num-integer",
"num-traits", "num-traits",
] ]
@ -1945,6 +1862,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef"
[[package]]
name = "odds"
version = "0.2.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4eae0151b9dacf24fcc170d9995e511669a082856a91f958a2fe380bfab3fb22"
[[package]] [[package]]
name = "oorandom" name = "oorandom"
version = "11.1.0" version = "11.1.0"
@ -2038,12 +1961,6 @@ dependencies = [
"yansi", "yansi",
] ]
[[package]]
name = "peeking_take_while"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]] [[package]]
name = "percent-encoding" name = "percent-encoding"
version = "1.0.1" version = "1.0.1"
@ -2056,45 +1973,6 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "phf"
version = "0.7.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18"
dependencies = [
"phf_shared",
]
[[package]]
name = "phf_codegen"
version = "0.7.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e"
dependencies = [
"phf_generator",
"phf_shared",
]
[[package]]
name = "phf_generator"
version = "0.7.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662"
dependencies = [
"phf_shared",
"rand 0.6.5",
]
[[package]]
name = "phf_shared"
version = "0.7.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0"
dependencies = [
"siphasher",
"unicase 1.4.2",
]
[[package]] [[package]]
name = "photosync" name = "photosync"
version = "0.1.0" version = "0.1.0"
@ -2110,13 +1988,14 @@ dependencies = [
"lazy_static 1.4.0", "lazy_static 1.4.0",
"load_image", "load_image",
"log 0.4.8", "log 0.4.8",
"mime_guess 2.0.1", "mime_guess",
"prometheus", "prometheus",
"regex", "regex",
"reqwest", "reqwest",
"rocket", "rocket",
"rocket_contrib", "rocket_contrib",
"rocksdb", "rusoto_core",
"rusoto_s3",
"rust-embed", "rust-embed",
"serde", "serde",
"serde_json", "serde_json",
@ -2124,7 +2003,7 @@ dependencies = [
"stderrlog", "stderrlog",
"structopt", "structopt",
"tempdir", "tempdir",
"warp", "thiserror",
"yup-oauth2", "yup-oauth2",
] ]
@ -2600,7 +2479,7 @@ dependencies = [
"lazy_static 1.4.0", "lazy_static 1.4.0",
"log 0.4.8", "log 0.4.8",
"mime 0.3.16", "mime 0.3.16",
"mime_guess 2.0.1", "mime_guess",
"native-tls", "native-tls",
"percent-encoding 2.1.0", "percent-encoding 2.1.0",
"pin-project-lite", "pin-project-lite",
@ -2626,6 +2505,15 @@ version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "958f3866c3a81ceaca304745792491944d0c9ed36e46eea43984b88cfac016f2" checksum = "958f3866c3a81ceaca304745792491944d0c9ed36e46eea43984b88cfac016f2"
[[package]]
name = "rexif"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b18541ee0a90d5ba60c094fd70b8d31cb903d6bc75cdba0f795507077bf6c1dd"
dependencies = [
"num",
]
[[package]] [[package]]
name = "rgb" name = "rgb"
version = "0.8.16" version = "0.8.16"
@ -2713,16 +2601,6 @@ dependencies = [
"unicode-xid 0.1.0", "unicode-xid 0.1.0",
] ]
[[package]]
name = "rocksdb"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12069b106981c6103d3eab7dd1c86751482d0779a520b7c14954c8b586c1e643"
dependencies = [
"libc",
"librocksdb-sys",
]
[[package]] [[package]]
name = "rusoto_core" name = "rusoto_core"
version = "0.42.0" version = "0.42.0"
@ -2849,12 +2727,6 @@ dependencies = [
"walkdir", "walkdir",
] ]
[[package]]
name = "rustc-demangle"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
[[package]] [[package]]
name = "rustc_version" name = "rustc_version"
version = "0.1.7" version = "0.1.7"
@ -2940,12 +2812,6 @@ dependencies = [
"winapi 0.3.8", "winapi 0.3.8",
] ]
[[package]]
name = "scoped-tls"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
[[package]] [[package]]
name = "scoped_threadpool" name = "scoped_threadpool"
version = "0.1.9" version = "0.1.9"
@ -3064,18 +2930,6 @@ dependencies = [
"url 2.1.1", "url 2.1.1",
] ]
[[package]]
name = "sha-1"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df"
dependencies = [
"block-buffer",
"digest",
"fake-simd",
"opaque-debug",
]
[[package]] [[package]]
name = "sha2" name = "sha2"
version = "0.8.2" version = "0.8.2"
@ -3104,12 +2958,6 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "siphasher"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.2" version = "0.4.2"
@ -3161,6 +3009,12 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028" checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028"
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]] [[package]]
name = "stb_image" name = "stb_image"
version = "0.2.2" version = "0.2.2"
@ -3336,6 +3190,26 @@ dependencies = [
"unicode-width", "unicode-width",
] ]
[[package]]
name = "thiserror"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793"
dependencies = [
"proc-macro2 1.0.8",
"quote 1.0.2",
"syn 1.0.14",
]
[[package]] [[package]]
name = "thread_local" name = "thread_local"
version = "0.3.4" version = "0.3.4"
@ -3710,34 +3584,6 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
[[package]]
name = "tungstenite"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a0c2bd5aeb7dcd2bb32e472c8872759308495e5eccc942e929a513cd8d36110"
dependencies = [
"base64 0.11.0",
"byteorder 1.3.4",
"bytes 0.4.12",
"http 0.1.21",
"httparse",
"input_buffer",
"log 0.4.8",
"rand 0.7.3",
"sha-1",
"url 2.1.1",
"utf-8",
]
[[package]]
name = "twoway"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "typeable" name = "typeable"
version = "0.1.2" version = "0.1.2"
@ -3857,18 +3703,6 @@ dependencies = [
"percent-encoding 2.1.0", "percent-encoding 2.1.0",
] ]
[[package]]
name = "urlencoding"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3df3561629a8bb4c57e5a2e4c43348d9e29c7c29d9b1c4c1f47166deca8f37ed"
[[package]]
name = "utf-8"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7"
[[package]] [[package]]
name = "vcpkg" name = "vcpkg"
version = "0.2.8" version = "0.2.8"
@ -3931,32 +3765,6 @@ dependencies = [
"try-lock", "try-lock",
] ]
[[package]]
name = "warp"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99b53196ba54e91e31ba1e90309a31053218cc1d4697f6e48f7c7e3d255e64fc"
dependencies = [
"bytes 0.4.12",
"futures",
"headers",
"http 0.1.21",
"hyper 0.12.35",
"log 0.4.8",
"mime 0.3.16",
"mime_guess 2.0.1",
"multipart",
"scoped-tls",
"serde",
"serde_json",
"serde_urlencoded",
"tokio 0.1.22",
"tokio-io",
"tokio-threadpool",
"tungstenite",
"urlencoding",
]
[[package]] [[package]]
name = "wasi" name = "wasi"
version = "0.9.0+wasi-snapshot-preview1" version = "0.9.0+wasi-snapshot-preview1"
@ -4088,16 +3896,6 @@ dependencies = [
"nom 4.2.3", "nom 4.2.3",
] ]
[[package]]
name = "which"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164"
dependencies = [
"failure",
"libc",
]
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.2.8" version = "0.2.8"
@ -4184,7 +3982,7 @@ dependencies = [
"http 0.1.21", "http 0.1.21",
"hyper 0.12.35", "hyper 0.12.35",
"hyper-rustls 0.17.1", "hyper-rustls 0.17.1",
"itertools", "itertools 0.8.2",
"log 0.3.9", "log 0.3.9",
"rustls", "rustls",
"serde", "serde",

7
Cargo.toml
View File

@ -19,16 +19,17 @@ serde_json = "1.0.46"
stderrlog = "0.4.3" stderrlog = "0.4.3"
structopt = "0.3.9" structopt = "0.3.9"
yup-oauth2 = "^3.1" yup-oauth2 = "^3.1"
warp = "0.1"
serde = { version = "1.0.104", features = ["derive"] } serde = { version = "1.0.104", features = ["derive"] }
image = { version = "0.23.2" } #, default-features = false, features = ["jpeg"] } image = { version = "0.23.2" } #, default-features = false, features = ["jpeg"] }
rust-embed = "5.2.0" rust-embed = "5.2.0"
mime_guess = "2.0.1" mime_guess = "2.0.1"
rocksdb = "0.13.0"
jpeg-decoder = "0.1.18" jpeg-decoder = "0.1.18"
imageutils = { git = "https://git.z.xinu.tv/wathiede/imageutils" } imageutils = { git = "https://git.z.xinu.tv/wathiede/imageutils" }
cacher = { path = "../cacher" } cacher = { git = "https://git.z.xinu.tv/wathiede/cacher" }
rocket = "0.4.5" rocket = "0.4.5"
thiserror = "1.0.20"
rusoto_s3 = "0.42.0"
rusoto_core = "0.42.0"
[dependencies.prometheus] [dependencies.prometheus]
features = ["process"] features = ["process"]

299
src/library.rs
View File

@ -1,178 +1,102 @@
use std::fs;
use std::fs::File;
use std::io; use std::io;
use std::io::Read; use std::io::Read;
use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use cacher::Cacher; use cacher::s3::S3CacherError;
use cacher::S3Cacher;
use google_photoslibrary1 as photos; use google_photoslibrary1 as photos;
use image::imageops; use image::imageops;
use imageutils::{load_image, resize, resize_to_fill, save_to_jpeg_bytes, FilterType}; use imageutils::{load_image_buffer, resize, resize_to_fill, save_to_jpeg_bytes, FilterType};
use log::error; use log::{error, info};
use log::info; use photos::schemas::{Album, MediaItem};
use log::warn; use rusoto_core::RusotoError;
use photos::schemas::Album; use rusoto_s3::GetObjectError;
use photos::schemas::MediaItem; use thiserror::Error;
use rocksdb::Direction;
use rocksdb::IteratorMode;
use rocksdb::DB;
// Used to ensure DB is invalidated after schema changes. // Used to ensure DB is invalidated after schema changes.
const LIBRARY_GENERATION: &'static str = "14"; const LIBRARY_GENERATION: &'static str = "16";
#[derive(Error, Debug)]
pub enum LibraryError {
#[error("IO error: {0}")]
IoError(#[from] std::io::Error),
#[error("s3 error: {0}")]
S3CacherError(#[from] S3CacherError),
#[error("json error: {0}")]
JsonError(#[from] serde_json::Error),
}
#[derive(Clone)] #[derive(Clone)]
pub struct Library { pub struct Library {
root: PathBuf, s3: S3Cacher,
originals_dir: PathBuf,
cache_db: Arc<DB>,
image_cache: Arc<Mutex<Box<dyn Cacher>>>,
} }
impl Library { impl Library {
pub fn new( pub fn new(s3: S3Cacher) -> Result<Library, Box<dyn std::error::Error>> {
root: PathBuf, let lib = Library { s3 };
image_cache: Arc<Mutex<Box<dyn Cacher>>>,
) -> Result<Library, Box<dyn std::error::Error>> {
let db = DB::open_default(root.join("cache"))?;
let cache_db = Arc::new(db);
let lib = Library {
originals_dir: root.join("images").join("originals"),
cache_db,
root,
image_cache,
};
let cnt = lib.clean_db()?;
if cnt != 0 {
info!("Deleted {} entries", cnt);
}
if !lib.originals_dir.exists() {
info!(
"create originals dir {}",
&lib.originals_dir.to_string_lossy()
);
fs::create_dir_all(&lib.originals_dir)?;
}
Ok(lib) Ok(lib)
} }
// Removes all data in the database from older schema. pub fn create_album_index(&self, albums: &Vec<Album>) -> Result<(), LibraryError> {
pub fn clean_db(&self) -> Result<usize, rocksdb::Error> {
Library::gc(LIBRARY_GENERATION, &self.cache_db)
}
fn gc(generation: &str, db: &DB) -> Result<usize, rocksdb::Error> {
let gen = format!("{}/", generation);
// '0' is the next character after '/', so iterator's starting there would be after the
// last `gen` entry.
let next_gen = format!("{}0", generation);
let mut del_cnt = 0;
for (k, _v) in db.iterator(IteratorMode::From(gen.as_bytes(), Direction::Reverse)) {
if !k.starts_with(gen.as_bytes()) {
info!("deleting stale key: {}", String::from_utf8_lossy(&k));
db.delete(k)?;
del_cnt += 1;
}
}
for (k, _v) in db.iterator(IteratorMode::From(next_gen.as_bytes(), Direction::Forward)) {
if !k.starts_with(gen.as_bytes()) {
info!("deleting stale key: {}", String::from_utf8_lossy(&k));
db.delete(k)?;
del_cnt += 1;
}
}
Ok(del_cnt)
}
pub fn create_album_index(&self, albums: &Vec<Album>) -> io::Result<()> {
// Serialize it to a JSON string. // Serialize it to a JSON string.
let j = serde_json::to_string(albums)?; let j = serde_json::to_string(albums)?;
let path = self.root.join("albums.json"); let filename = "albums.json";
info!("saving {}", path.to_string_lossy());
fs::write(path, j) self.s3
.set(&Library::generational_key(filename), j.as_ref())?;
Ok(())
} }
pub fn create_album<P: AsRef<Path>>( pub fn create_album(
&self, &self,
album_id: P, album_id: &str,
media_items: &Vec<MediaItem>, media_items: &Vec<MediaItem>,
) -> io::Result<()> { ) -> Result<(), LibraryError> {
let album_dir = self.root.join(album_id); let relpath = format!("{}.json", &album_id);
if !album_dir.exists() {
info!("making album directory {}", album_dir.to_string_lossy());
fs::create_dir_all(&album_dir)?;
}
let j = serde_json::to_string(&media_items)?; let j = serde_json::to_string(&media_items)?;
let path = album_dir.join("album.json");
info!("saving {}", path.to_string_lossy()); self.s3
fs::write(path, j) .set(&Library::generational_key(&relpath), j.as_ref())?;
Ok(())
} }
pub fn albums(&self) -> Result<Vec<Album>, Box<dyn std::error::Error>> { pub fn albums(&self) -> Result<Vec<Album>, Box<dyn std::error::Error>> {
let albums_path = self.root.join("albums.json"); let filename = "albums.json";
info!("loading {}", albums_path.to_string_lossy());
let bytes = fs::read(albums_path)?; let bytes = self.s3.get(&Library::generational_key(filename))?;
Ok(serde_json::from_slice(&bytes)?) let album: Vec<Album> = serde_json::from_slice(&bytes)?;
Ok(album)
} }
pub fn album(&self, album_id: &str) -> Result<Vec<MediaItem>, Box<dyn std::error::Error>> { pub fn album(&self, album_id: &str) -> Result<Vec<MediaItem>, Box<dyn std::error::Error>> {
let album_path = self.root.join(album_id).join("album.json"); let relpath = format!("{}.json", &album_id);
let bytes = fs::read(album_path)?; let bytes = self.s3.get(&Library::generational_key(&relpath))?;
Ok(serde_json::from_slice(&bytes)?) let mis: Vec<MediaItem> = serde_json::from_slice(&bytes)?;
Ok(mis)
} }
pub fn download_image( pub fn download_image(
&self, &self,
filename: &str, _filename: &str,
media_items_id: &str, media_items_id: &str,
base_url: &str, base_url: &str,
) -> Result<PathBuf, Box<dyn std::error::Error>> { ) -> Result<PathBuf, Box<dyn std::error::Error>> {
// Put images from all albums in common directory. let filename = Library::generational_key(&format!("images/originals/{}", media_items_id));
let image_path = self.originals_dir.join(media_items_id); if !self.s3.contains_key(&filename) {
if image_path.exists() {
info!(
"Skipping already downloaded {} @ {}",
&filename,
image_path.to_string_lossy()
);
} else {
let download_path = image_path.with_extension("download");
let c = Arc::clone(&self.image_cache);
let mut c = c.lock().unwrap();
match c.get(media_items_id) {
Some(bytes) => {
info!(
"saving local copy of original from cache {}",
media_items_id
);
fs::write(&download_path, bytes)?;
}
None => {
let url = format!("{}=d", base_url); let url = format!("{}=d", base_url);
let mut r = reqwest::blocking::get(&url)?; let mut r = reqwest::blocking::get(&url)?;
let mut buf = Vec::new(); let mut buf = Vec::new();
info!("Downloading {}", &url); info!("Downloading {}", &url);
r.read_to_end(&mut buf)?; r.read_to_end(&mut buf)?;
fs::write(&download_path, &buf); self.s3.set(&filename, &buf)?;
c.set(media_items_id, &buf);
} }
}; Ok(filename.into())
info!(
"Rename {} -> {}",
download_path.to_string_lossy(),
image_path.to_string_lossy()
);
fs::rename(download_path, &image_path)?;
}
Ok(image_path)
}
pub fn original(&self, media_items_id: &str) -> Option<PathBuf> {
let path = self.originals_dir.join(media_items_id);
if path.exists() {
Some(path)
} else {
None
} }
pub fn original_buffer(&self, media_items_id: &str) -> Result<Vec<u8>, LibraryError> {
let filename = Library::generational_key(&format!("images/originals/{}", media_items_id));
let bytes = self.s3.get(&filename)?;
Ok(bytes)
} }
// TODO(wathiede): make this a macro like format! to skip the second string create and copy. // TODO(wathiede): make this a macro like format! to skip the second string create and copy.
fn generational_key(generation: &str, key: &str) -> String { fn generational_key(key: &str) -> String {
format!("{}/{}", generation, key) format!("{}/{}", LIBRARY_GENERATION, key)
} }
pub fn generate_thumbnail( pub fn generate_thumbnail(
@ -182,25 +106,23 @@ impl Library {
filter: FilterType, filter: FilterType,
fill: bool, fill: bool,
) -> Result<Vec<u8>, Box<dyn std::error::Error>> { ) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
match self.original(&media_items_id) { let buf = self.original_buffer(&media_items_id)?;
None => { let dimension_hint = match dimensions {
warn!("Couldn't find original {}", &media_items_id); (Some(w), Some(h)) => Some((w, h)),
Err(io::Error::new(io::ErrorKind::NotFound, format!("{}", media_items_id)).into()) // Partial dimensions should be handled by the caller of this function. So all
} // other options are None.
Some(path) => { _ => None,
let orig_img = load_image(&path, dimensions.0, dimensions.1)?; };
let orig_img = load_image_buffer(buf, dimension_hint)?;
//.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; //.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
let img = if fill { let img = if fill {
resize_to_fill(&orig_img, dimensions, filter) resize_to_fill(&orig_img, dimensions, filter)
} else { } else {
resize(&orig_img, dimensions, filter) resize(&orig_img, dimensions, filter)
}; };
let buf = save_to_jpeg_bytes(&img) let buf = save_to_jpeg_bytes(&img).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
Ok(buf) Ok(buf)
} }
}
}
pub fn thumbnail( pub fn thumbnail(
&self, &self,
media_items_id: &str, media_items_id: &str,
@ -214,27 +136,17 @@ impl Library {
(None, Some(h)) => format!("-h={}", h), (None, Some(h)) => format!("-h={}", h),
(None, None) => "".to_string(), (None, None) => "".to_string(),
}; };
Library::generational_key(LIBRARY_GENERATION, &format!("{}{}", media_items_id, dim)) Library::generational_key(&format!("images/thumbnails/{}-{}", media_items_id, dim))
} }
let key = cache_key(media_items_id, dimensions); let key = cache_key(media_items_id, dimensions);
let db = self.cache_db.clone(); match self.s3.get(&key) {
match db.get(key.as_bytes()) { Ok(bytes) => return Some(bytes),
// Cache hit, return bytes as-is. Err(S3CacherError::GetObjectError(RusotoError::Service(
Ok(Some(bytes)) => Some(bytes), GetObjectError::NoSuchKey(msg),
// Cache miss, fill cache and return. ))) => info!("Missing thumbnail {} in s3: {}", key, msg),
Ok(None) => { Err(e) => error!("Error fetching thumbnail {} from s3: {}", key, e),
// TODO(wathiede): use cache for thumbnail like download_image does. };
let c = Arc::clone(&self.image_cache);
let mut c = c.lock().unwrap();
let bytes = match c.get(&key) {
Some(bytes) => {
info!(
"saving local copy of thumbnail from cache {}",
media_items_id
);
bytes
}
None => {
info!("cache MISS {}", key); info!("cache MISS {}", key);
let bytes = match self.generate_thumbnail( let bytes = match self.generate_thumbnail(
media_items_id, media_items_id,
@ -244,66 +156,13 @@ impl Library {
) { ) {
Ok(bytes) => bytes, Ok(bytes) => bytes,
Err(e) => { Err(e) => {
error!( error!("Failed to generate thumbnail for {}: {}", media_items_id, e);
"Failed to generate thumbnail for {}: {}",
media_items_id, e
);
return None; return None;
} }
}; };
c.set(&key, &bytes); if let Err(e) = self.s3.set(&key, &bytes) {
bytes error!("Failed to put thumbnail {}: {}", &key, e);
}
};
match db.put(key.as_bytes(), &bytes) {
Ok(_) => Some(bytes),
Err(e) => {
error!("Failed to put bytes to {}: {}", key, e);
None
}
}
}
// RocksDB error.
Err(e) => {
error!("Failed to search DB for {}: {}", key, e);
None
}
} }
} Some(bytes)
}
#[cfg(test)]
mod test {
use super::*;
use tempdir::TempDir;
#[test]
fn clean_db() {
let td = TempDir::new("photosync_test").expect("failed to create temporary directory");
eprintln!("creating database in {}", td.path().to_string_lossy());
let db = DB::open_default(td.path()).expect("failed to open DB");
let keys = vec!["one", "two", "three"];
fn get_keys(db: &DB) -> Vec<String> {
db.iterator(rocksdb::IteratorMode::Start)
.map(|(k, _v)| String::from_utf8(k.to_vec()).expect("key not utf-8"))
.collect()
}
for k in &keys {
for g in vec!["1", "2", "3"] {
db.put(Library::generational_key(g, k), k)
.expect("failed to put");
}
}
assert_eq!(
get_keys(&db),
vec![
"1/one", "1/three", "1/two", "2/one", "2/three", "2/two", "3/one", "3/three",
"3/two"
]
);
Library::gc("2", &db).expect("failed to GC DB");
assert_eq!(get_keys(&db), vec!["2/one", "2/three", "2/two",]);
} }
} }

33
src/main.rs
View File

@ -2,11 +2,10 @@ use std::collections::HashMap;
use std::error::Error; use std::error::Error;
use std::net::SocketAddr; use std::net::SocketAddr;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use std::thread; use std::thread;
use std::time; use std::time;
use cacher::{Cacher, S3Cacher}; use cacher::S3Cacher;
use google_api_auth; use google_api_auth;
use google_photoslibrary1 as photos; use google_photoslibrary1 as photos;
use hexihasher; use hexihasher;
@ -32,14 +31,13 @@ struct Sync {
/// Optional album title to filter. Default will mirror all albums. /// Optional album title to filter. Default will mirror all albums.
#[structopt(short, long)] #[structopt(short, long)]
title_filter: Option<Regex>, title_filter: Option<Regex>,
/// Directory to store sync. /// S3 bucket holding metadata and images.
root: PathBuf, #[structopt(long, default_value = "photosync-dev")]
s3_bucket: String,
} }
#[derive(Debug, StructOpt)] #[derive(Debug, StructOpt)]
struct Serve { struct Serve {
/// Directory of data fetched by `sync`.
root: PathBuf,
/// HTTP address to listen for web requests. /// HTTP address to listen for web requests.
#[structopt(long = "addr", default_value = "0.0.0.0:0")] #[structopt(long = "addr", default_value = "0.0.0.0:0")]
addr: SocketAddr, addr: SocketAddr,
@ -65,6 +63,9 @@ enum Command {
Serve { Serve {
#[structopt(flatten)] #[structopt(flatten)]
serve: Serve, serve: Serve,
/// S3 bucket holding metadata and images.
#[structopt(default_value = "photosync-dev")]
s3_bucket: String,
}, },
ServeAndSync { ServeAndSync {
/// Sync albums at given interval. /// Sync albums at given interval.
@ -322,9 +323,6 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
.init() .init()
.unwrap(); .unwrap();
debug!("opt: {:?}", opt); debug!("opt: {:?}", opt);
let image_cache: Mutex<Box<dyn Cacher>> =
Mutex::new(Box::new(S3Cacher::new("photosync".to_string())?));
let image_cache = Arc::new(image_cache);
match opt.cmd { match opt.cmd {
Command::ListAlbums { auth, title_filter } => { Command::ListAlbums { auth, title_filter } => {
let client = new_client(&auth.credentials, &auth.token_cache)?; let client = new_client(&auth.credentials, &auth.token_cache)?;
@ -341,18 +339,21 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
Sync { Sync {
auth, auth,
title_filter, title_filter,
root, s3_bucket,
}, },
} => { } => {
let s3 = S3Cacher::new(s3_bucket.clone())?;
let client = new_client(&auth.credentials, &auth.token_cache)?; let client = new_client(&auth.credentials, &auth.token_cache)?;
let lib = Library::new(root, image_cache)?; let lib = Library::new(s3)?;
sync_albums(&client, &title_filter, &lib)?; sync_albums(&client, &title_filter, &lib)?;
Ok(()) Ok(())
} }
Command::Serve { Command::Serve {
serve: Serve { addr, root }, serve: Serve { addr },
s3_bucket,
} => { } => {
let lib = Library::new(root, image_cache)?; let s3 = S3Cacher::new(s3_bucket.clone())?;
let lib = Library::new(s3)?;
serve(addr, lib) serve(addr, lib)
} }
Command::ServeAndSync { Command::ServeAndSync {
@ -361,12 +362,14 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
Sync { Sync {
auth, auth,
title_filter, title_filter,
root, s3_bucket,
}, },
addr, addr,
} => { } => {
let s3 = S3Cacher::new(s3_bucket.clone())?;
let client = new_client(&auth.credentials, &auth.token_cache)?; let client = new_client(&auth.credentials, &auth.token_cache)?;
let lib = Library::new(root, image_cache)?; let lib = Library::new(s3)?;
background_sync(client, interval, title_filter, lib.clone())?; background_sync(client, interval, title_filter, lib.clone())?;
serve(addr, lib)?; serve(addr, lib)?;
Ok(()) Ok(())

10
src/rweb.rs
View File

@ -7,6 +7,7 @@ use google_photoslibrary1 as photos;
use log::error; use log::error;
use photos::schemas::{Album, MediaItem}; use photos::schemas::{Album, MediaItem};
use prometheus::Encoder; use prometheus::Encoder;
use rocket::config::{Config, Environment};
use rocket::http::ContentType; use rocket::http::ContentType;
use rocket::response::status::NotFound; use rocket::response::status::NotFound;
use rocket::response::Content; use rocket::response::Content;
@ -116,8 +117,13 @@ fn embedz() -> Content<Vec<u8>> {
Content(ContentType::HTML, w) Content(ContentType::HTML, w)
} }
pub fn run(_addr: SocketAddr, lib: Library) -> Result<(), Box<dyn Error>> { pub fn run(addr: SocketAddr, lib: Library) -> Result<(), Box<dyn Error>> {
let e = rocket::ignite() let config = Config::build(Environment::Development)
.address(addr.ip().to_string())
.port(addr.port())
.finalize()?;
let e = rocket::custom(config)
.manage(lib) .manage(lib)
.mount( .mount(
"/", "/",