From 249db6500e5b2ce69be8024e1c7f3c77e4482bea Mon Sep 17 00:00:00 2001 From: Bill Thiede Date: Sun, 9 Feb 2020 21:26:03 -0800 Subject: [PATCH] sync: download fullsize images. --- Cargo.lock | 293 +++++++++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 15 +-- src/main.rs | 79 ++++++++++++-- 3 files changed, 370 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f5b9f48..1b23551 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,12 +68,39 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + [[package]] name = "bumpalo" version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f359dc14ff8911330a51ef78022d376f25ed00248912803b58f00cb1c27f742" +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + [[package]] name = "byteorder" version = "1.3.4" @@ -97,6 +124,15 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1" +[[package]] +name = "c2-chacha" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" +dependencies = [ + "ppv-lite86", +] + [[package]] name = "cc" version = "1.0.50" @@ -215,6 +251,15 @@ dependencies = [ "sct", ] +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + [[package]] name = "dtoa" version = "0.4.5" @@ -236,12 +281,39 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "faster-hex" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348138dd23e03bb0018caef99647fb1a5befec5ff4b501991de88f09854d4c28" + [[package]] name = "fnv" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -342,6 +414,26 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "getrandom" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "google-photoslibrary1" version = "0.1.0-20200203" @@ -441,6 +533,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hexihasher" +version = "0.1.0" +source = "git+https://git.z.xinu.tv/wathiede/hexihasher#1cf4cdf1cc797e28260705bb232e437274a3df7e" +dependencies = [ + "faster-hex", + "sha2", +] + [[package]] name = "http" version = "0.1.21" @@ -579,6 +680,19 @@ dependencies = [ "webpki", ] +[[package]] +name = "hyper-tls" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa" +dependencies = [ + "bytes 0.5.4", + "hyper 0.13.2", + "native-tls", + "tokio 0.2.11", + "tokio-tls", +] + [[package]] name = "idna" version = "0.1.5" @@ -783,6 +897,24 @@ dependencies = [ "ws2_32-sys", ] +[[package]] +name = "native-tls" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e" +dependencies = [ + "lazy_static 1.4.0", + "libc", + "log 0.4.8", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "net2" version = "0.2.33" @@ -839,12 +971,45 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "openssl" +version = "0.10.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973293749822d7dd6370d6da1e523b0d1db19f06c459134c658b2a4261378b52" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "lazy_static 1.4.0", + "libc", + "openssl-sys", +] + [[package]] name = "openssl-probe" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" +[[package]] +name = "openssl-sys" +version = "0.9.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1024c0a59774200a555087a6da3f253a9095a5f344e353b212ac4c8b8e450986" +dependencies = [ + "autocfg 1.0.0", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking_lot" version = "0.9.0" @@ -889,8 +1054,11 @@ version = "0.1.0" dependencies = [ "google-photoslibrary1", "google_api_auth", + "hexihasher", + "lazy_static 1.4.0", "log 0.4.8", "regex", + "reqwest", "serde_json", "stderrlog", "structopt", @@ -929,6 +1097,18 @@ version = "0.1.0-alpha.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" +[[package]] +name = "pkg-config" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" + +[[package]] +name = "ppv-lite86" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" + [[package]] name = "proc-macro-error" version = "0.4.8" @@ -998,9 +1178,9 @@ checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" dependencies = [ "autocfg 0.1.7", "libc", - "rand_chacha", + "rand_chacha 0.1.1", "rand_core 0.4.2", - "rand_hc", + "rand_hc 0.1.0", "rand_isaac", "rand_jitter", "rand_os", @@ -1009,6 +1189,19 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha 0.2.1", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + [[package]] name = "rand_chacha" version = "0.1.1" @@ -1019,6 +1212,16 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "rand_chacha" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" +dependencies = [ + "c2-chacha", + "rand_core 0.5.1", +] + [[package]] name = "rand_core" version = "0.3.1" @@ -1034,6 +1237,15 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + [[package]] name = "rand_hc" version = "0.1.0" @@ -1043,6 +1255,15 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rand_isaac" version = "0.1.1" @@ -1138,6 +1359,15 @@ version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b28dfe3fe9badec5dbf0a79a9cccad2cfc2ab5484bdb3e44cbd1ae8b3ba2be06" +[[package]] +name = "remove_dir_all" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" +dependencies = [ + "winapi 0.3.8", +] + [[package]] name = "reqwest" version = "0.10.1" @@ -1153,11 +1383,13 @@ dependencies = [ "http-body 0.3.1", "hyper 0.13.2", "hyper-rustls 0.19.1", + "hyper-tls", "js-sys", "lazy_static 1.4.0", "log 0.4.8", "mime", "mime_guess", + "native-tls", "percent-encoding 2.1.0", "pin-project-lite", "rustls", @@ -1167,6 +1399,7 @@ dependencies = [ "time", "tokio 0.2.11", "tokio-rustls 0.12.2", + "tokio-tls", "url 2.1.1", "wasm-bindgen", "wasm-bindgen-futures", @@ -1357,6 +1590,18 @@ dependencies = [ "url 2.1.1", ] +[[package]] +name = "sha2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + [[package]] name = "slab" version = "0.4.2" @@ -1464,6 +1709,20 @@ dependencies = [ "syn", ] +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if", + "libc", + "rand 0.7.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.8", +] + [[package]] name = "termcolor" version = "1.1.0" @@ -1494,7 +1753,7 @@ dependencies = [ "base64 0.10.1", "byteorder", "chrono", - "rand", + "rand 0.6.5", "serde", "serde_derive", ] @@ -1741,6 +2000,16 @@ dependencies = [ "tokio-executor", ] +[[package]] +name = "tokio-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bde02a3a5291395f59b06ec6945a3077602fac2b07eeeaf0dee2122f3619828" +dependencies = [ + "native-tls", + "tokio 0.2.11", +] + [[package]] name = "tokio-udp" version = "0.1.6" @@ -1800,6 +2069,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" +[[package]] +name = "typenum" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" + [[package]] name = "unicase" version = "2.6.0" @@ -1882,6 +2157,12 @@ dependencies = [ "percent-encoding 2.1.0", ] +[[package]] +name = "vcpkg" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" + [[package]] name = "vec_map" version = "0.8.1" @@ -1927,6 +2208,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasm-bindgen" version = "0.2.58" diff --git a/Cargo.toml b/Cargo.toml index 6cc5ab1..e27a7c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,12 +7,15 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -yup-oauth2 = "^3.1" -google_api_auth = { git = "https://github.com/google-apis-rs/generator", features = ["with-yup-oauth2"] } # TODO, use https://git.z.xinu.tv/wathiede/google-api-photoslibrary and figure out auth story. -google-photoslibrary1 = { git = "https://git.z.xinu.tv/wathiede/google-api-photoslibrary" } -structopt = "0.3.9" -regex = "1.3.4" +google-photoslibrary1 = { git = "https://git.z.xinu.tv/wathiede/google-api-photoslibrary" } +google_api_auth = { git = "https://github.com/google-apis-rs/generator", features = ["with-yup-oauth2"] } +hexihasher = { git = "https://git.z.xinu.tv/wathiede/hexihasher" } +lazy_static = "1.4.0" log = "0.4.8" -stderrlog = "0.4.3" +regex = "1.3.4" +reqwest = { version = "0.10.1", features = ["blocking"] } serde_json = "1.0.46" +stderrlog = "0.4.3" +structopt = "0.3.9" +yup-oauth2 = "^3.1" diff --git a/src/main.rs b/src/main.rs index 521fe86..36aabcd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,12 +1,18 @@ +use std::collections::HashMap; use std::error::Error; use std::fs; +use std::fs::File; +use std::io; use std::path::PathBuf; use google_api_auth; use google_photoslibrary1 as photos; +use hexihasher; +use lazy_static::lazy_static; use log::{debug, info}; use photos::schemas::{Album, MediaItem, SearchMediaItemsRequest}; use regex::Regex; +use reqwest; use structopt::StructOpt; use yup_oauth2::{Authenticator, InstalledFlow}; @@ -125,10 +131,15 @@ impl<'a> Iterator for SearchIter<'a> { fn print_media_items(media_items: Vec) { for mi in &media_items { + let id = mi + .id + .as_ref() + .map_or("NO ID".to_string(), |s| s.to_string()); println!( - "{} {}", - mi.id.as_ref().unwrap_or(&"NO ID".to_string()), - mi.filename.as_ref().unwrap_or(&"NO FILENAME".to_string()) + "media item: {}\n\t{}\n\t{}", + mi.filename.as_ref().unwrap_or(&"NO FILENAME".to_string()), + hexihasher::sha256(id.as_bytes()), + id, ); } println!("({}) items total", media_items.len()); @@ -136,12 +147,12 @@ fn print_media_items(media_items: Vec) { fn search_media_items( client: &photos::Client, - album_id: String, + album_id: &str, ) -> Result, Box> { let media_items = SearchIter::new( &client, SearchMediaItemsRequest { - album_id: Some(album_id.clone()), + album_id: Some(album_id.to_string()), // 100 is the documented max. page_size: Some(100), ..Default::default() @@ -152,6 +163,17 @@ fn search_media_items( Ok(media_items) } +lazy_static! { + static ref MIME_TO_EXT: HashMap<&'static str, &'static str> = [ + ("image/gif", "gif"), + ("image/heif", "heic"), + ("image/jpeg", "jpg"), + ] + .iter() + .copied() + .collect(); +} + fn sync_albums( client: &photos::Client, title_filter: Option, @@ -159,13 +181,54 @@ fn sync_albums( ) -> Result<(), Box> { let albums = list_albums(client, title_filter)?; for a in &albums { - let album_dir = output_dir.join(a.id.as_ref().expect("missing album id")); + let album_id = a.id.as_ref().expect("unset album id").to_string(); + let album_dir = output_dir.join(&album_id); if !album_dir.exists() { info!("making album directory {}", album_dir.to_string_lossy()); fs::create_dir_all(&album_dir)?; } - let album = search_media_items(client, a.id.as_ref().expect("unset album id").to_string())?; + let album = search_media_items(client, &album_id)?; + for (i, mi) in album.iter().enumerate() { + let mi_id = mi.id.as_ref().expect("unset media item id").to_string(); + let filename = mi + .filename + .as_ref() + .map_or("NO_FILENAME".to_string(), |s| s.to_string()); + // Put images from all albums in common directory. + let image_path = output_dir.join("images").join(&mi_id); + if !image_path.exists() { + fs::create_dir_all(&image_path)?; + } + let image_path = image_path.join(&filename); + if image_path.exists() { + info!( + "Skipping already downloaded {} @ {}", + &filename, + image_path.to_string_lossy() + ); + } else { + let download_path = image_path.with_extension("download"); + info!( + "({}/{}) Downloading {} -> {}", + i + 1, + &album.len(), + &filename, + download_path.to_string_lossy() + ); + let base_url = mi.base_url.as_ref().expect("missing base_url"); + let url = format!("{}=d", base_url); + let mut r = reqwest::blocking::get(&url)?; + let mut w = File::create(&download_path)?; + let _n = io::copy(&mut r, &mut w)?; + info!( + "Rename {} -> {}", + download_path.to_string_lossy(), + image_path.to_string_lossy() + ); + fs::rename(download_path, &image_path)?; + } + } let j = serde_json::to_string(&album)?; let path = album_dir.join("album.json"); info!("saving {}", path.to_string_lossy()); @@ -230,7 +293,7 @@ fn main() -> Result<(), Box> { Ok(()) } Command::SearchMediaItems { album_id } => { - print_media_items(search_media_items(&client, album_id)?); + print_media_items(search_media_items(&client, &album_id)?); Ok(()) } Command::Sync {