Rewrite to use s3 instead of local files and rocksdb.
This commit is contained in:
parent
337f5dfd49
commit
1bca4c3642
546
Cargo.lock
generated
546
Cargo.lock
generated
@ -97,6 +97,16 @@ version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.3.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06f59fe10306bb78facd90d28c2038ad23ffaaefa85bac43c8a434cde383334f"
|
||||
dependencies = [
|
||||
"nodrop",
|
||||
"odds",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.5.1"
|
||||
@ -126,28 +136,6 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e4036b9bf40f3cf16aba72a3d65e8a520fc4bafcdc7079aea8f848c58c5b5536"
|
||||
dependencies = [
|
||||
"backtrace-sys",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"rustc-demangle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backtrace-sys"
|
||||
version = "0.1.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.9.3"
|
||||
@ -179,29 +167,6 @@ version = "0.12.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e223af0dc48c96d4f8342ec01a4974f139df863896b316681efd36742f22cc67"
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.49.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c07087f3d5731bf3fb375a81841b99597e25dc11bd3bc72d16d43adf6624a6e"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cexpr",
|
||||
"cfg-if",
|
||||
"clang-sys",
|
||||
"clap",
|
||||
"env_logger",
|
||||
"fxhash",
|
||||
"lazy_static 1.4.0",
|
||||
"log 0.4.8",
|
||||
"peeking_take_while",
|
||||
"proc-macro2 0.4.30",
|
||||
"quote 0.6.13",
|
||||
"regex",
|
||||
"shlex",
|
||||
"which",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.2.1"
|
||||
@ -215,7 +180,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"arrayvec",
|
||||
"arrayvec 0.5.1",
|
||||
"constant_time_eq",
|
||||
]
|
||||
|
||||
@ -261,16 +226,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "buf_redux"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"safemem",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.2.0"
|
||||
@ -330,11 +285,13 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "cacher"
|
||||
version = "0.1.0"
|
||||
source = "git+https://git.z.xinu.tv/wathiede/cacher#43d364cf47a1d179d75c4e8370d344ffbe747522"
|
||||
dependencies = [
|
||||
"log 0.4.8",
|
||||
"rusoto_core",
|
||||
"rusoto_credential",
|
||||
"rusoto_s3",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -355,15 +312,6 @@ dependencies = [
|
||||
"jobserver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d"
|
||||
dependencies = [
|
||||
"nom 4.2.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "0.1.10"
|
||||
@ -382,17 +330,6 @@ dependencies = [
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"libc",
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "2.33.0"
|
||||
@ -461,6 +398,17 @@ version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b"
|
||||
|
||||
[[package]]
|
||||
name = "cr2"
|
||||
version = "0.1.0"
|
||||
source = "git+https://git.z.xinu.tv/wathiede/cr2#a666d4580c008cdf172faff024c746c38f8acd70"
|
||||
dependencies = [
|
||||
"hexdump",
|
||||
"lazy_static 1.4.0",
|
||||
"memmap",
|
||||
"nom 5.1.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.2.0"
|
||||
@ -481,7 +429,7 @@ dependencies = [
|
||||
"clap",
|
||||
"criterion-plot",
|
||||
"csv",
|
||||
"itertools",
|
||||
"itertools 0.8.2",
|
||||
"lazy_static 1.4.0",
|
||||
"num-traits",
|
||||
"oorandom",
|
||||
@ -502,7 +450,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a01e15e0ea58e8234f96146b1f91fa9d0e4dd7a38da93ff7a75d42c0b9d3a545"
|
||||
dependencies = [
|
||||
"cast",
|
||||
"itertools",
|
||||
"itertools 0.8.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -699,28 +647,6 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"humantime",
|
||||
"log 0.4.8",
|
||||
"regex",
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "failure"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fake-simd"
|
||||
version = "0.1.2"
|
||||
@ -912,15 +838,6 @@ dependencies = [
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fxhash"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
|
||||
dependencies = [
|
||||
"byteorder 1.3.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gcc"
|
||||
version = "0.3.55"
|
||||
@ -1053,32 +970,6 @@ dependencies = [
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "headers"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "882ca7d8722f33ce2c2db44f95425d6267ed59ca96ce02acbe58320054ceb642"
|
||||
dependencies = [
|
||||
"base64 0.10.1",
|
||||
"bitflags",
|
||||
"bytes 0.4.12",
|
||||
"headers-core",
|
||||
"http 0.1.21",
|
||||
"mime 0.3.16",
|
||||
"sha-1",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "headers-core"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "967131279aaa9f7c20c7205b45a391638a83ab118e6509b2d0ccbe08de044237"
|
||||
dependencies = [
|
||||
"bytes 0.4.12",
|
||||
"http 0.1.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.3.1"
|
||||
@ -1103,6 +994,16 @@ version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35"
|
||||
|
||||
[[package]]
|
||||
name = "hexdump"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "850f3f2c33d20c0f96c4485e087dd580ff041d720988ebf4c84a42acf739262b"
|
||||
dependencies = [
|
||||
"arrayvec 0.3.25",
|
||||
"itertools 0.4.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hexihasher"
|
||||
version = "0.1.0"
|
||||
@ -1182,15 +1083,6 @@ version = "1.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
|
||||
dependencies = [
|
||||
"quick-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.10.16"
|
||||
@ -1367,10 +1259,14 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "imageutils"
|
||||
version = "0.1.0"
|
||||
source = "git+https://git.z.xinu.tv/wathiede/imageutils#d6804fa0f8e0afe2deb54354acc983a1ab59d794"
|
||||
source = "git+https://git.z.xinu.tv/wathiede/imageutils#b49d30e30fd7d89d242ba7c96b36018a8f9f361e"
|
||||
dependencies = [
|
||||
"cr2",
|
||||
"image",
|
||||
"jpeg-decoder",
|
||||
"log 0.4.8",
|
||||
"memmap",
|
||||
"rexif 0.5.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -1417,15 +1313,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "input_buffer"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e1b822cc844905551931d6f81608ed5f50a79c1078a4e2b4d42dbc7c1eedfbf"
|
||||
dependencies = [
|
||||
"bytes 0.4.12",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iovec"
|
||||
version = "0.1.4"
|
||||
@ -1435,6 +1322,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.4.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f"
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.8.2"
|
||||
@ -1534,34 +1427,25 @@ dependencies = [
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lexical-core"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616"
|
||||
dependencies = [
|
||||
"arrayvec 0.5.1",
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"ryu",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.66"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "librocksdb-sys"
|
||||
version = "6.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0a0785e816e1e11e7599388a492c61ef80ddc2afc91e313e61662cce537809be"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
"glob",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "load_image"
|
||||
version = "2.12.0"
|
||||
@ -1572,7 +1456,7 @@ dependencies = [
|
||||
"lcms2",
|
||||
"lodepng",
|
||||
"mozjpeg",
|
||||
"rexif",
|
||||
"rexif 0.3.7",
|
||||
"rgb",
|
||||
]
|
||||
|
||||
@ -1643,6 +1527,16 @@ version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223"
|
||||
|
||||
[[package]]
|
||||
name = "memmap"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.5.3"
|
||||
@ -1667,18 +1561,6 @@ version = "0.3.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
|
||||
|
||||
[[package]]
|
||||
name = "mime_guess"
|
||||
version = "1.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d977de9ee851a0b16e932979515c0f3da82403183879811bc97d50bd9cc50f7"
|
||||
dependencies = [
|
||||
"mime 0.2.6",
|
||||
"phf",
|
||||
"phf_codegen",
|
||||
"unicase 1.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime_guess"
|
||||
version = "2.0.1"
|
||||
@ -1780,7 +1662,7 @@ version = "0.8.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f672ed5c96e386d6436643104c9bdc175d4b79ae3eb30538b7a58eb55f3bf318"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"arrayvec 0.5.1",
|
||||
"libc",
|
||||
"mozjpeg-sys",
|
||||
"rgb",
|
||||
@ -1798,24 +1680,6 @@ dependencies = [
|
||||
"nasm-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multipart"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "136eed74cadb9edd2651ffba732b19a450316b680e4f48d6c79e905799e19d01"
|
||||
dependencies = [
|
||||
"buf_redux",
|
||||
"httparse",
|
||||
"log 0.4.8",
|
||||
"mime 0.2.6",
|
||||
"mime_guess 1.8.7",
|
||||
"quick-error",
|
||||
"rand 0.6.5",
|
||||
"safemem",
|
||||
"tempfile",
|
||||
"twoway",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nasm-rs"
|
||||
version = "0.1.7"
|
||||
@ -1854,6 +1718,12 @@ dependencies = [
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nodrop"
|
||||
version = "0.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "1.2.4"
|
||||
@ -1870,6 +1740,17 @@ dependencies = [
|
||||
"version_check 0.1.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "5.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
|
||||
dependencies = [
|
||||
"lexical-core",
|
||||
"memchr",
|
||||
"version_check 0.9.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "notify"
|
||||
version = "4.0.15"
|
||||
@ -1888,6 +1769,41 @@ dependencies = [
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"
|
||||
dependencies = [
|
||||
"num-bigint",
|
||||
"num-complex",
|
||||
"num-integer",
|
||||
"num-iter",
|
||||
"num-rational",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
|
||||
dependencies = [
|
||||
"autocfg 1.0.0",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-complex"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
|
||||
dependencies = [
|
||||
"autocfg 1.0.0",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.42"
|
||||
@ -1916,6 +1832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3"
|
||||
dependencies = [
|
||||
"autocfg 1.0.0",
|
||||
"num-bigint",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
@ -1945,6 +1862,12 @@ version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef"
|
||||
|
||||
[[package]]
|
||||
name = "odds"
|
||||
version = "0.2.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4eae0151b9dacf24fcc170d9995e511669a082856a91f958a2fe380bfab3fb22"
|
||||
|
||||
[[package]]
|
||||
name = "oorandom"
|
||||
version = "11.1.0"
|
||||
@ -2038,12 +1961,6 @@ dependencies = [
|
||||
"yansi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "peeking_take_while"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "1.0.1"
|
||||
@ -2056,45 +1973,6 @@ version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.7.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18"
|
||||
dependencies = [
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_codegen"
|
||||
version = "0.7.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.7.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662"
|
||||
dependencies = [
|
||||
"phf_shared",
|
||||
"rand 0.6.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.7.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
"unicase 1.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "photosync"
|
||||
version = "0.1.0"
|
||||
@ -2110,13 +1988,14 @@ dependencies = [
|
||||
"lazy_static 1.4.0",
|
||||
"load_image",
|
||||
"log 0.4.8",
|
||||
"mime_guess 2.0.1",
|
||||
"mime_guess",
|
||||
"prometheus",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rocket",
|
||||
"rocket_contrib",
|
||||
"rocksdb",
|
||||
"rusoto_core",
|
||||
"rusoto_s3",
|
||||
"rust-embed",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@ -2124,7 +2003,7 @@ dependencies = [
|
||||
"stderrlog",
|
||||
"structopt",
|
||||
"tempdir",
|
||||
"warp",
|
||||
"thiserror",
|
||||
"yup-oauth2",
|
||||
]
|
||||
|
||||
@ -2600,7 +2479,7 @@ dependencies = [
|
||||
"lazy_static 1.4.0",
|
||||
"log 0.4.8",
|
||||
"mime 0.3.16",
|
||||
"mime_guess 2.0.1",
|
||||
"mime_guess",
|
||||
"native-tls",
|
||||
"percent-encoding 2.1.0",
|
||||
"pin-project-lite",
|
||||
@ -2626,6 +2505,15 @@ version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "958f3866c3a81ceaca304745792491944d0c9ed36e46eea43984b88cfac016f2"
|
||||
|
||||
[[package]]
|
||||
name = "rexif"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b18541ee0a90d5ba60c094fd70b8d31cb903d6bc75cdba0f795507077bf6c1dd"
|
||||
dependencies = [
|
||||
"num",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rgb"
|
||||
version = "0.8.16"
|
||||
@ -2713,16 +2601,6 @@ dependencies = [
|
||||
"unicode-xid 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rocksdb"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12069b106981c6103d3eab7dd1c86751482d0779a520b7c14954c8b586c1e643"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"librocksdb-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusoto_core"
|
||||
version = "0.42.0"
|
||||
@ -2849,12 +2727,6 @@ dependencies = [
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
|
||||
|
||||
[[package]]
|
||||
name = "rustc_version"
|
||||
version = "0.1.7"
|
||||
@ -2940,12 +2812,6 @@ dependencies = [
|
||||
"winapi 0.3.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "scoped-tls"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
|
||||
|
||||
[[package]]
|
||||
name = "scoped_threadpool"
|
||||
version = "0.1.9"
|
||||
@ -3064,18 +2930,6 @@ dependencies = [
|
||||
"url 2.1.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha-1"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"digest",
|
||||
"fake-simd",
|
||||
"opaque-debug",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.8.2"
|
||||
@ -3104,12 +2958,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.2"
|
||||
@ -3161,6 +3009,12 @@ version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7345c971d1ef21ffdbd103a75990a15eb03604fc8b8852ca8cb418ee1a099028"
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "stb_image"
|
||||
version = "0.2.2"
|
||||
@ -3336,6 +3190,26 @@ dependencies = [
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.8",
|
||||
"quote 1.0.2",
|
||||
"syn 1.0.14",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "0.3.4"
|
||||
@ -3710,34 +3584,6 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a0c2bd5aeb7dcd2bb32e472c8872759308495e5eccc942e929a513cd8d36110"
|
||||
dependencies = [
|
||||
"base64 0.11.0",
|
||||
"byteorder 1.3.4",
|
||||
"bytes 0.4.12",
|
||||
"http 0.1.21",
|
||||
"httparse",
|
||||
"input_buffer",
|
||||
"log 0.4.8",
|
||||
"rand 0.7.3",
|
||||
"sha-1",
|
||||
"url 2.1.1",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "twoway"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typeable"
|
||||
version = "0.1.2"
|
||||
@ -3857,18 +3703,6 @@ dependencies = [
|
||||
"percent-encoding 2.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "urlencoding"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3df3561629a8bb4c57e5a2e4c43348d9e29c7c29d9b1c4c1f47166deca8f37ed"
|
||||
|
||||
[[package]]
|
||||
name = "utf-8"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.8"
|
||||
@ -3931,32 +3765,6 @@ dependencies = [
|
||||
"try-lock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "warp"
|
||||
version = "0.1.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99b53196ba54e91e31ba1e90309a31053218cc1d4697f6e48f7c7e3d255e64fc"
|
||||
dependencies = [
|
||||
"bytes 0.4.12",
|
||||
"futures",
|
||||
"headers",
|
||||
"http 0.1.21",
|
||||
"hyper 0.12.35",
|
||||
"log 0.4.8",
|
||||
"mime 0.3.16",
|
||||
"mime_guess 2.0.1",
|
||||
"multipart",
|
||||
"scoped-tls",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"tokio 0.1.22",
|
||||
"tokio-io",
|
||||
"tokio-threadpool",
|
||||
"tungstenite",
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.9.0+wasi-snapshot-preview1"
|
||||
@ -4088,16 +3896,6 @@ dependencies = [
|
||||
"nom 4.2.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164"
|
||||
dependencies = [
|
||||
"failure",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.2.8"
|
||||
@ -4184,7 +3982,7 @@ dependencies = [
|
||||
"http 0.1.21",
|
||||
"hyper 0.12.35",
|
||||
"hyper-rustls 0.17.1",
|
||||
"itertools",
|
||||
"itertools 0.8.2",
|
||||
"log 0.3.9",
|
||||
"rustls",
|
||||
"serde",
|
||||
|
||||
@ -19,16 +19,17 @@ serde_json = "1.0.46"
|
||||
stderrlog = "0.4.3"
|
||||
structopt = "0.3.9"
|
||||
yup-oauth2 = "^3.1"
|
||||
warp = "0.1"
|
||||
serde = { version = "1.0.104", features = ["derive"] }
|
||||
image = { version = "0.23.2" } #, default-features = false, features = ["jpeg"] }
|
||||
rust-embed = "5.2.0"
|
||||
mime_guess = "2.0.1"
|
||||
rocksdb = "0.13.0"
|
||||
jpeg-decoder = "0.1.18"
|
||||
imageutils = { git = "https://git.z.xinu.tv/wathiede/imageutils" }
|
||||
cacher = { path = "../cacher" }
|
||||
cacher = { git = "https://git.z.xinu.tv/wathiede/cacher" }
|
||||
rocket = "0.4.5"
|
||||
thiserror = "1.0.20"
|
||||
rusoto_s3 = "0.42.0"
|
||||
rusoto_core = "0.42.0"
|
||||
|
||||
[dependencies.prometheus]
|
||||
features = ["process"]
|
||||
|
||||
343
src/library.rs
343
src/library.rs
@ -1,178 +1,102 @@
|
||||
use std::fs;
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::io::Read;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use cacher::Cacher;
|
||||
use cacher::s3::S3CacherError;
|
||||
use cacher::S3Cacher;
|
||||
use google_photoslibrary1 as photos;
|
||||
use image::imageops;
|
||||
use imageutils::{load_image, resize, resize_to_fill, save_to_jpeg_bytes, FilterType};
|
||||
use log::error;
|
||||
use log::info;
|
||||
use log::warn;
|
||||
use photos::schemas::Album;
|
||||
use photos::schemas::MediaItem;
|
||||
use rocksdb::Direction;
|
||||
use rocksdb::IteratorMode;
|
||||
use rocksdb::DB;
|
||||
use imageutils::{load_image_buffer, resize, resize_to_fill, save_to_jpeg_bytes, FilterType};
|
||||
use log::{error, info};
|
||||
use photos::schemas::{Album, MediaItem};
|
||||
use rusoto_core::RusotoError;
|
||||
use rusoto_s3::GetObjectError;
|
||||
use thiserror::Error;
|
||||
|
||||
// Used to ensure DB is invalidated after schema changes.
|
||||
const LIBRARY_GENERATION: &'static str = "14";
|
||||
const LIBRARY_GENERATION: &'static str = "16";
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum LibraryError {
|
||||
#[error("IO error: {0}")]
|
||||
IoError(#[from] std::io::Error),
|
||||
#[error("s3 error: {0}")]
|
||||
S3CacherError(#[from] S3CacherError),
|
||||
#[error("json error: {0}")]
|
||||
JsonError(#[from] serde_json::Error),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Library {
|
||||
root: PathBuf,
|
||||
originals_dir: PathBuf,
|
||||
cache_db: Arc<DB>,
|
||||
image_cache: Arc<Mutex<Box<dyn Cacher>>>,
|
||||
s3: S3Cacher,
|
||||
}
|
||||
|
||||
impl Library {
|
||||
pub fn new(
|
||||
root: PathBuf,
|
||||
image_cache: Arc<Mutex<Box<dyn Cacher>>>,
|
||||
) -> Result<Library, Box<dyn std::error::Error>> {
|
||||
let db = DB::open_default(root.join("cache"))?;
|
||||
let cache_db = Arc::new(db);
|
||||
let lib = Library {
|
||||
originals_dir: root.join("images").join("originals"),
|
||||
cache_db,
|
||||
root,
|
||||
image_cache,
|
||||
};
|
||||
let cnt = lib.clean_db()?;
|
||||
if cnt != 0 {
|
||||
info!("Deleted {} entries", cnt);
|
||||
}
|
||||
if !lib.originals_dir.exists() {
|
||||
info!(
|
||||
"create originals dir {}",
|
||||
&lib.originals_dir.to_string_lossy()
|
||||
);
|
||||
fs::create_dir_all(&lib.originals_dir)?;
|
||||
}
|
||||
pub fn new(s3: S3Cacher) -> Result<Library, Box<dyn std::error::Error>> {
|
||||
let lib = Library { s3 };
|
||||
Ok(lib)
|
||||
}
|
||||
// Removes all data in the database from older schema.
|
||||
pub fn clean_db(&self) -> Result<usize, rocksdb::Error> {
|
||||
Library::gc(LIBRARY_GENERATION, &self.cache_db)
|
||||
}
|
||||
fn gc(generation: &str, db: &DB) -> Result<usize, rocksdb::Error> {
|
||||
let gen = format!("{}/", generation);
|
||||
// '0' is the next character after '/', so iterator's starting there would be after the
|
||||
// last `gen` entry.
|
||||
let next_gen = format!("{}0", generation);
|
||||
let mut del_cnt = 0;
|
||||
for (k, _v) in db.iterator(IteratorMode::From(gen.as_bytes(), Direction::Reverse)) {
|
||||
if !k.starts_with(gen.as_bytes()) {
|
||||
info!("deleting stale key: {}", String::from_utf8_lossy(&k));
|
||||
db.delete(k)?;
|
||||
del_cnt += 1;
|
||||
}
|
||||
}
|
||||
for (k, _v) in db.iterator(IteratorMode::From(next_gen.as_bytes(), Direction::Forward)) {
|
||||
if !k.starts_with(gen.as_bytes()) {
|
||||
info!("deleting stale key: {}", String::from_utf8_lossy(&k));
|
||||
db.delete(k)?;
|
||||
del_cnt += 1;
|
||||
}
|
||||
}
|
||||
Ok(del_cnt)
|
||||
}
|
||||
pub fn create_album_index(&self, albums: &Vec<Album>) -> io::Result<()> {
|
||||
pub fn create_album_index(&self, albums: &Vec<Album>) -> Result<(), LibraryError> {
|
||||
// Serialize it to a JSON string.
|
||||
let j = serde_json::to_string(albums)?;
|
||||
|
||||
let path = self.root.join("albums.json");
|
||||
info!("saving {}", path.to_string_lossy());
|
||||
fs::write(path, j)
|
||||
let filename = "albums.json";
|
||||
|
||||
self.s3
|
||||
.set(&Library::generational_key(filename), j.as_ref())?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn create_album<P: AsRef<Path>>(
|
||||
pub fn create_album(
|
||||
&self,
|
||||
album_id: P,
|
||||
album_id: &str,
|
||||
media_items: &Vec<MediaItem>,
|
||||
) -> io::Result<()> {
|
||||
let album_dir = self.root.join(album_id);
|
||||
if !album_dir.exists() {
|
||||
info!("making album directory {}", album_dir.to_string_lossy());
|
||||
fs::create_dir_all(&album_dir)?;
|
||||
}
|
||||
) -> Result<(), LibraryError> {
|
||||
let relpath = format!("{}.json", &album_id);
|
||||
let j = serde_json::to_string(&media_items)?;
|
||||
let path = album_dir.join("album.json");
|
||||
info!("saving {}", path.to_string_lossy());
|
||||
fs::write(path, j)
|
||||
|
||||
self.s3
|
||||
.set(&Library::generational_key(&relpath), j.as_ref())?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn albums(&self) -> Result<Vec<Album>, Box<dyn std::error::Error>> {
|
||||
let albums_path = self.root.join("albums.json");
|
||||
info!("loading {}", albums_path.to_string_lossy());
|
||||
let bytes = fs::read(albums_path)?;
|
||||
Ok(serde_json::from_slice(&bytes)?)
|
||||
let filename = "albums.json";
|
||||
|
||||
let bytes = self.s3.get(&Library::generational_key(filename))?;
|
||||
let album: Vec<Album> = serde_json::from_slice(&bytes)?;
|
||||
Ok(album)
|
||||
}
|
||||
pub fn album(&self, album_id: &str) -> Result<Vec<MediaItem>, Box<dyn std::error::Error>> {
|
||||
let album_path = self.root.join(album_id).join("album.json");
|
||||
let bytes = fs::read(album_path)?;
|
||||
Ok(serde_json::from_slice(&bytes)?)
|
||||
let relpath = format!("{}.json", &album_id);
|
||||
let bytes = self.s3.get(&Library::generational_key(&relpath))?;
|
||||
let mis: Vec<MediaItem> = serde_json::from_slice(&bytes)?;
|
||||
Ok(mis)
|
||||
}
|
||||
pub fn download_image(
|
||||
&self,
|
||||
filename: &str,
|
||||
_filename: &str,
|
||||
media_items_id: &str,
|
||||
base_url: &str,
|
||||
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
||||
// Put images from all albums in common directory.
|
||||
let image_path = self.originals_dir.join(media_items_id);
|
||||
if image_path.exists() {
|
||||
info!(
|
||||
"Skipping already downloaded {} @ {}",
|
||||
&filename,
|
||||
image_path.to_string_lossy()
|
||||
);
|
||||
} else {
|
||||
let download_path = image_path.with_extension("download");
|
||||
let c = Arc::clone(&self.image_cache);
|
||||
let mut c = c.lock().unwrap();
|
||||
match c.get(media_items_id) {
|
||||
Some(bytes) => {
|
||||
info!(
|
||||
"saving local copy of original from cache {}",
|
||||
media_items_id
|
||||
);
|
||||
fs::write(&download_path, bytes)?;
|
||||
}
|
||||
None => {
|
||||
let url = format!("{}=d", base_url);
|
||||
let mut r = reqwest::blocking::get(&url)?;
|
||||
let mut buf = Vec::new();
|
||||
info!("Downloading {}", &url);
|
||||
r.read_to_end(&mut buf)?;
|
||||
fs::write(&download_path, &buf);
|
||||
c.set(media_items_id, &buf);
|
||||
}
|
||||
};
|
||||
info!(
|
||||
"Rename {} -> {}",
|
||||
download_path.to_string_lossy(),
|
||||
image_path.to_string_lossy()
|
||||
);
|
||||
fs::rename(download_path, &image_path)?;
|
||||
let filename = Library::generational_key(&format!("images/originals/{}", media_items_id));
|
||||
if !self.s3.contains_key(&filename) {
|
||||
let url = format!("{}=d", base_url);
|
||||
let mut r = reqwest::blocking::get(&url)?;
|
||||
let mut buf = Vec::new();
|
||||
info!("Downloading {}", &url);
|
||||
r.read_to_end(&mut buf)?;
|
||||
self.s3.set(&filename, &buf)?;
|
||||
}
|
||||
Ok(image_path)
|
||||
Ok(filename.into())
|
||||
}
|
||||
pub fn original(&self, media_items_id: &str) -> Option<PathBuf> {
|
||||
let path = self.originals_dir.join(media_items_id);
|
||||
if path.exists() {
|
||||
Some(path)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
|
||||
pub fn original_buffer(&self, media_items_id: &str) -> Result<Vec<u8>, LibraryError> {
|
||||
let filename = Library::generational_key(&format!("images/originals/{}", media_items_id));
|
||||
let bytes = self.s3.get(&filename)?;
|
||||
Ok(bytes)
|
||||
}
|
||||
// TODO(wathiede): make this a macro like format! to skip the second string create and copy.
|
||||
fn generational_key(generation: &str, key: &str) -> String {
|
||||
format!("{}/{}", generation, key)
|
||||
fn generational_key(key: &str) -> String {
|
||||
format!("{}/{}", LIBRARY_GENERATION, key)
|
||||
}
|
||||
|
||||
pub fn generate_thumbnail(
|
||||
@ -182,24 +106,22 @@ impl Library {
|
||||
filter: FilterType,
|
||||
fill: bool,
|
||||
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
|
||||
match self.original(&media_items_id) {
|
||||
None => {
|
||||
warn!("Couldn't find original {}", &media_items_id);
|
||||
Err(io::Error::new(io::ErrorKind::NotFound, format!("{}", media_items_id)).into())
|
||||
}
|
||||
Some(path) => {
|
||||
let orig_img = load_image(&path, dimensions.0, dimensions.1)?;
|
||||
//.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
|
||||
let img = if fill {
|
||||
resize_to_fill(&orig_img, dimensions, filter)
|
||||
} else {
|
||||
resize(&orig_img, dimensions, filter)
|
||||
};
|
||||
let buf = save_to_jpeg_bytes(&img)
|
||||
.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
|
||||
Ok(buf)
|
||||
}
|
||||
}
|
||||
let buf = self.original_buffer(&media_items_id)?;
|
||||
let dimension_hint = match dimensions {
|
||||
(Some(w), Some(h)) => Some((w, h)),
|
||||
// Partial dimensions should be handled by the caller of this function. So all
|
||||
// other options are None.
|
||||
_ => None,
|
||||
};
|
||||
let orig_img = load_image_buffer(buf, dimension_hint)?;
|
||||
//.map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
|
||||
let img = if fill {
|
||||
resize_to_fill(&orig_img, dimensions, filter)
|
||||
} else {
|
||||
resize(&orig_img, dimensions, filter)
|
||||
};
|
||||
let buf = save_to_jpeg_bytes(&img).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
|
||||
Ok(buf)
|
||||
}
|
||||
pub fn thumbnail(
|
||||
&self,
|
||||
@ -214,96 +136,33 @@ impl Library {
|
||||
(None, Some(h)) => format!("-h={}", h),
|
||||
(None, None) => "".to_string(),
|
||||
};
|
||||
Library::generational_key(LIBRARY_GENERATION, &format!("{}{}", media_items_id, dim))
|
||||
Library::generational_key(&format!("images/thumbnails/{}-{}", media_items_id, dim))
|
||||
}
|
||||
let key = cache_key(media_items_id, dimensions);
|
||||
let db = self.cache_db.clone();
|
||||
match db.get(key.as_bytes()) {
|
||||
// Cache hit, return bytes as-is.
|
||||
Ok(Some(bytes)) => Some(bytes),
|
||||
// Cache miss, fill cache and return.
|
||||
Ok(None) => {
|
||||
// TODO(wathiede): use cache for thumbnail like download_image does.
|
||||
let c = Arc::clone(&self.image_cache);
|
||||
let mut c = c.lock().unwrap();
|
||||
let bytes = match c.get(&key) {
|
||||
Some(bytes) => {
|
||||
info!(
|
||||
"saving local copy of thumbnail from cache {}",
|
||||
media_items_id
|
||||
);
|
||||
bytes
|
||||
}
|
||||
None => {
|
||||
info!("cache MISS {}", key);
|
||||
let bytes = match self.generate_thumbnail(
|
||||
media_items_id,
|
||||
dimensions,
|
||||
FilterType::Builtin(imageops::FilterType::Lanczos3),
|
||||
fill,
|
||||
) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Failed to generate thumbnail for {}: {}",
|
||||
media_items_id, e
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
c.set(&key, &bytes);
|
||||
bytes
|
||||
}
|
||||
};
|
||||
match db.put(key.as_bytes(), &bytes) {
|
||||
Ok(_) => Some(bytes),
|
||||
Err(e) => {
|
||||
error!("Failed to put bytes to {}: {}", key, e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
// RocksDB error.
|
||||
match self.s3.get(&key) {
|
||||
Ok(bytes) => return Some(bytes),
|
||||
Err(S3CacherError::GetObjectError(RusotoError::Service(
|
||||
GetObjectError::NoSuchKey(msg),
|
||||
))) => info!("Missing thumbnail {} in s3: {}", key, msg),
|
||||
Err(e) => error!("Error fetching thumbnail {} from s3: {}", key, e),
|
||||
};
|
||||
|
||||
info!("cache MISS {}", key);
|
||||
let bytes = match self.generate_thumbnail(
|
||||
media_items_id,
|
||||
dimensions,
|
||||
FilterType::Builtin(imageops::FilterType::Lanczos3),
|
||||
fill,
|
||||
) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
error!("Failed to search DB for {}: {}", key, e);
|
||||
None
|
||||
error!("Failed to generate thumbnail for {}: {}", media_items_id, e);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if let Err(e) = self.s3.set(&key, &bytes) {
|
||||
error!("Failed to put thumbnail {}: {}", &key, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use tempdir::TempDir;
|
||||
|
||||
#[test]
|
||||
fn clean_db() {
|
||||
let td = TempDir::new("photosync_test").expect("failed to create temporary directory");
|
||||
eprintln!("creating database in {}", td.path().to_string_lossy());
|
||||
let db = DB::open_default(td.path()).expect("failed to open DB");
|
||||
let keys = vec!["one", "two", "three"];
|
||||
|
||||
fn get_keys(db: &DB) -> Vec<String> {
|
||||
db.iterator(rocksdb::IteratorMode::Start)
|
||||
.map(|(k, _v)| String::from_utf8(k.to_vec()).expect("key not utf-8"))
|
||||
.collect()
|
||||
}
|
||||
for k in &keys {
|
||||
for g in vec!["1", "2", "3"] {
|
||||
db.put(Library::generational_key(g, k), k)
|
||||
.expect("failed to put");
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
get_keys(&db),
|
||||
vec![
|
||||
"1/one", "1/three", "1/two", "2/one", "2/three", "2/two", "3/one", "3/three",
|
||||
"3/two"
|
||||
]
|
||||
);
|
||||
Library::gc("2", &db).expect("failed to GC DB");
|
||||
assert_eq!(get_keys(&db), vec!["2/one", "2/three", "2/two",]);
|
||||
Some(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
33
src/main.rs
33
src/main.rs
@ -2,11 +2,10 @@ use std::collections::HashMap;
|
||||
use std::error::Error;
|
||||
use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
use std::time;
|
||||
|
||||
use cacher::{Cacher, S3Cacher};
|
||||
use cacher::S3Cacher;
|
||||
use google_api_auth;
|
||||
use google_photoslibrary1 as photos;
|
||||
use hexihasher;
|
||||
@ -32,14 +31,13 @@ struct Sync {
|
||||
/// Optional album title to filter. Default will mirror all albums.
|
||||
#[structopt(short, long)]
|
||||
title_filter: Option<Regex>,
|
||||
/// Directory to store sync.
|
||||
root: PathBuf,
|
||||
/// S3 bucket holding metadata and images.
|
||||
#[structopt(long, default_value = "photosync-dev")]
|
||||
s3_bucket: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, StructOpt)]
|
||||
struct Serve {
|
||||
/// Directory of data fetched by `sync`.
|
||||
root: PathBuf,
|
||||
/// HTTP address to listen for web requests.
|
||||
#[structopt(long = "addr", default_value = "0.0.0.0:0")]
|
||||
addr: SocketAddr,
|
||||
@ -65,6 +63,9 @@ enum Command {
|
||||
Serve {
|
||||
#[structopt(flatten)]
|
||||
serve: Serve,
|
||||
/// S3 bucket holding metadata and images.
|
||||
#[structopt(default_value = "photosync-dev")]
|
||||
s3_bucket: String,
|
||||
},
|
||||
ServeAndSync {
|
||||
/// Sync albums at given interval.
|
||||
@ -322,9 +323,6 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
.init()
|
||||
.unwrap();
|
||||
debug!("opt: {:?}", opt);
|
||||
let image_cache: Mutex<Box<dyn Cacher>> =
|
||||
Mutex::new(Box::new(S3Cacher::new("photosync".to_string())?));
|
||||
let image_cache = Arc::new(image_cache);
|
||||
match opt.cmd {
|
||||
Command::ListAlbums { auth, title_filter } => {
|
||||
let client = new_client(&auth.credentials, &auth.token_cache)?;
|
||||
@ -341,18 +339,21 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
Sync {
|
||||
auth,
|
||||
title_filter,
|
||||
root,
|
||||
s3_bucket,
|
||||
},
|
||||
} => {
|
||||
let s3 = S3Cacher::new(s3_bucket.clone())?;
|
||||
let client = new_client(&auth.credentials, &auth.token_cache)?;
|
||||
let lib = Library::new(root, image_cache)?;
|
||||
let lib = Library::new(s3)?;
|
||||
sync_albums(&client, &title_filter, &lib)?;
|
||||
Ok(())
|
||||
}
|
||||
Command::Serve {
|
||||
serve: Serve { addr, root },
|
||||
serve: Serve { addr },
|
||||
s3_bucket,
|
||||
} => {
|
||||
let lib = Library::new(root, image_cache)?;
|
||||
let s3 = S3Cacher::new(s3_bucket.clone())?;
|
||||
let lib = Library::new(s3)?;
|
||||
serve(addr, lib)
|
||||
}
|
||||
Command::ServeAndSync {
|
||||
@ -361,12 +362,14 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
Sync {
|
||||
auth,
|
||||
title_filter,
|
||||
root,
|
||||
s3_bucket,
|
||||
},
|
||||
|
||||
addr,
|
||||
} => {
|
||||
let s3 = S3Cacher::new(s3_bucket.clone())?;
|
||||
let client = new_client(&auth.credentials, &auth.token_cache)?;
|
||||
let lib = Library::new(root, image_cache)?;
|
||||
let lib = Library::new(s3)?;
|
||||
background_sync(client, interval, title_filter, lib.clone())?;
|
||||
serve(addr, lib)?;
|
||||
Ok(())
|
||||
|
||||
10
src/rweb.rs
10
src/rweb.rs
@ -7,6 +7,7 @@ use google_photoslibrary1 as photos;
|
||||
use log::error;
|
||||
use photos::schemas::{Album, MediaItem};
|
||||
use prometheus::Encoder;
|
||||
use rocket::config::{Config, Environment};
|
||||
use rocket::http::ContentType;
|
||||
use rocket::response::status::NotFound;
|
||||
use rocket::response::Content;
|
||||
@ -116,8 +117,13 @@ fn embedz() -> Content<Vec<u8>> {
|
||||
Content(ContentType::HTML, w)
|
||||
}
|
||||
|
||||
pub fn run(_addr: SocketAddr, lib: Library) -> Result<(), Box<dyn Error>> {
|
||||
let e = rocket::ignite()
|
||||
pub fn run(addr: SocketAddr, lib: Library) -> Result<(), Box<dyn Error>> {
|
||||
let config = Config::build(Environment::Development)
|
||||
.address(addr.ip().to_string())
|
||||
.port(addr.port())
|
||||
.finalize()?;
|
||||
|
||||
let e = rocket::custom(config)
|
||||
.manage(lib)
|
||||
.mount(
|
||||
"/",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user