wathiede/photosync
1
0
Fork 1
Code Issues Pull Requests Releases Wiki Activity

sync: download fullsize images.

Browse Source
This commit is contained in:
Bill Thiede 2020-02-09 21:26:03 -08:00
parent 991b5a1469
commit 249db6500e
3 changed files with 370 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

293
Cargo.lock generated
View File

@ -68,12 +68,39 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "block-buffer"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
dependencies = [
"block-padding",
"byte-tools",
"byteorder",
"generic-array",
]
[[package]]
name = "block-padding"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
dependencies = [
"byte-tools",
]
[[package]]
name = "bumpalo"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f359dc14ff8911330a51ef78022d376f25ed00248912803b58f00cb1c27f742"
[[package]]
name = "byte-tools"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
[[package]]
name = "byteorder"
version = "1.3.4"
@ -97,6 +124,15 @@ version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1"
[[package]]
name = "c2-chacha"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
dependencies = [
"ppv-lite86",
]
[[package]]
name = "cc"
version = "1.0.50"
@ -215,6 +251,15 @@ dependencies = [
"sct",
]
[[package]]
name = "digest"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
dependencies = [
"generic-array",
]
[[package]]
name = "dtoa"
version = "0.4.5"
@ -236,12 +281,39 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "fake-simd"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
[[package]]
name = "faster-hex"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "348138dd23e03bb0018caef99647fb1a5befec5ff4b501991de88f09854d4c28"
[[package]]
name = "fnv"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
@ -342,6 +414,26 @@ dependencies = [
"slab",
]
[[package]]
name = "generic-array"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec"
dependencies = [
"typenum",
]
[[package]]
name = "getrandom"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "google-photoslibrary1"
version = "0.1.0-20200203"
@ -441,6 +533,15 @@ dependencies = [
"libc",
]
[[package]]
name = "hexihasher"
version = "0.1.0"
source = "git+https://git.z.xinu.tv/wathiede/hexihasher#1cf4cdf1cc797e28260705bb232e437274a3df7e"
dependencies = [
"faster-hex",
"sha2",
]
[[package]]
name = "http"
version = "0.1.21"
@ -579,6 +680,19 @@ dependencies = [
"webpki",
]
[[package]]
name = "hyper-tls"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3adcd308402b9553630734e9c36b77a7e48b3821251ca2493e8cd596763aafaa"
dependencies = [
"bytes 0.5.4",
"hyper 0.13.2",
"native-tls",
"tokio 0.2.11",
"tokio-tls",
]
[[package]]
name = "idna"
version = "0.1.5"
@ -783,6 +897,24 @@ dependencies = [
"ws2_32-sys",
]
[[package]]
name = "native-tls"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e"
dependencies = [
"lazy_static 1.4.0",
"libc",
"log 0.4.8",
"openssl",
"openssl-probe",
"openssl-sys",
"schannel",
"security-framework",
"security-framework-sys",
"tempfile",
]
[[package]]
name = "net2"
version = "0.2.33"
@ -839,12 +971,45 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef"
[[package]]
name = "opaque-debug"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
[[package]]
name = "openssl"
version = "0.10.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "973293749822d7dd6370d6da1e523b0d1db19f06c459134c658b2a4261378b52"
dependencies = [
"bitflags",
"cfg-if",
"foreign-types",
"lazy_static 1.4.0",
"libc",
"openssl-sys",
]
[[package]]
name = "openssl-probe"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
[[package]]
name = "openssl-sys"
version = "0.9.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1024c0a59774200a555087a6da3f253a9095a5f344e353b212ac4c8b8e450986"
dependencies = [
"autocfg 1.0.0",
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "parking_lot"
version = "0.9.0"
@ -889,8 +1054,11 @@ version = "0.1.0"
dependencies = [
"google-photoslibrary1",
"google_api_auth",
"hexihasher",
"lazy_static 1.4.0",
"log 0.4.8",
"regex",
"reqwest",
"serde_json",
"stderrlog",
"structopt",
@ -929,6 +1097,18 @@ version = "0.1.0-alpha.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587"
[[package]]
name = "pkg-config"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
[[package]]
name = "ppv-lite86"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
[[package]]
name = "proc-macro-error"
version = "0.4.8"
@ -998,9 +1178,9 @@ checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
dependencies = [
"autocfg 0.1.7",
"libc",
"rand_chacha",
"rand_chacha 0.1.1",
"rand_core 0.4.2",
"rand_hc",
"rand_hc 0.1.0",
"rand_isaac",
"rand_jitter",
"rand_os",
@ -1009,6 +1189,19 @@ dependencies = [
"winapi 0.3.8",
]
[[package]]
name = "rand"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
"getrandom",
"libc",
"rand_chacha 0.2.1",
"rand_core 0.5.1",
"rand_hc 0.2.0",
]
[[package]]
name = "rand_chacha"
version = "0.1.1"
@ -1019,6 +1212,16 @@ dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "rand_chacha"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853"
dependencies = [
"c2-chacha",
"rand_core 0.5.1",
]
[[package]]
name = "rand_core"
version = "0.3.1"
@ -1034,6 +1237,15 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
[[package]]
name = "rand_core"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
dependencies = [
"getrandom",
]
[[package]]
name = "rand_hc"
version = "0.1.0"
@ -1043,6 +1255,15 @@ dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "rand_hc"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
dependencies = [
"rand_core 0.5.1",
]
[[package]]
name = "rand_isaac"
version = "0.1.1"
@ -1138,6 +1359,15 @@ version = "0.6.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b28dfe3fe9badec5dbf0a79a9cccad2cfc2ab5484bdb3e44cbd1ae8b3ba2be06"
[[package]]
name = "remove_dir_all"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
dependencies = [
"winapi 0.3.8",
]
[[package]]
name = "reqwest"
version = "0.10.1"
@ -1153,11 +1383,13 @@ dependencies = [
"http-body 0.3.1",
"hyper 0.13.2",
"hyper-rustls 0.19.1",
"hyper-tls",
"js-sys",
"lazy_static 1.4.0",
"log 0.4.8",
"mime",
"mime_guess",
"native-tls",
"percent-encoding 2.1.0",
"pin-project-lite",
"rustls",
@ -1167,6 +1399,7 @@ dependencies = [
"time",
"tokio 0.2.11",
"tokio-rustls 0.12.2",
"tokio-tls",
"url 2.1.1",
"wasm-bindgen",
"wasm-bindgen-futures",
@ -1357,6 +1590,18 @@ dependencies = [
"url 2.1.1",
]
[[package]]
name = "sha2"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0"
dependencies = [
"block-buffer",
"digest",
"fake-simd",
"opaque-debug",
]
[[package]]
name = "slab"
version = "0.4.2"
@ -1464,6 +1709,20 @@ dependencies = [
"syn",
]
[[package]]
name = "tempfile"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
dependencies = [
"cfg-if",
"libc",
"rand 0.7.3",
"redox_syscall",
"remove_dir_all",
"winapi 0.3.8",
]
[[package]]
name = "termcolor"
version = "1.1.0"
@ -1494,7 +1753,7 @@ dependencies = [
"base64 0.10.1",
"byteorder",
"chrono",
"rand",
"rand 0.6.5",
"serde",
"serde_derive",
]
@ -1741,6 +2000,16 @@ dependencies = [
"tokio-executor",
]
[[package]]
name = "tokio-tls"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bde02a3a5291395f59b06ec6945a3077602fac2b07eeeaf0dee2122f3619828"
dependencies = [
"native-tls",
"tokio 0.2.11",
]
[[package]]
name = "tokio-udp"
version = "0.1.6"
@ -1800,6 +2069,12 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
[[package]]
name = "typenum"
version = "1.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9"
[[package]]
name = "unicase"
version = "2.6.0"
@ -1882,6 +2157,12 @@ dependencies = [
"percent-encoding 2.1.0",
]
[[package]]
name = "vcpkg"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168"
[[package]]
name = "vec_map"
version = "0.8.1"
@ -1927,6 +2208,12 @@ dependencies = [
"try-lock",
]
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]]
name = "wasm-bindgen"
version = "0.2.58"

15
Cargo.toml
View File

@ -7,12 +7,15 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
yup-oauth2 = "^3.1"
google_api_auth = { git = "https://github.com/google-apis-rs/generator", features = ["with-yup-oauth2"] }
# TODO, use https://git.z.xinu.tv/wathiede/google-api-photoslibrary and figure out auth story.
google-photoslibrary1 = { git = "https://git.z.xinu.tv/wathiede/google-api-photoslibrary" }
structopt = "0.3.9"
regex = "1.3.4"
google-photoslibrary1 = { git = "https://git.z.xinu.tv/wathiede/google-api-photoslibrary" }
google_api_auth = { git = "https://github.com/google-apis-rs/generator", features = ["with-yup-oauth2"] }
hexihasher = { git = "https://git.z.xinu.tv/wathiede/hexihasher" }
lazy_static = "1.4.0"
log = "0.4.8"
stderrlog = "0.4.3"
regex = "1.3.4"
reqwest = { version = "0.10.1", features = ["blocking"] }
serde_json = "1.0.46"
stderrlog = "0.4.3"
structopt = "0.3.9"
yup-oauth2 = "^3.1"

79
src/main.rs
View File

@ -1,12 +1,18 @@
use std::collections::HashMap;
use std::error::Error;
use std::fs;
use std::fs::File;
use std::io;
use std::path::PathBuf;
use google_api_auth;
use google_photoslibrary1 as photos;
use hexihasher;
use lazy_static::lazy_static;
use log::{debug, info};
use photos::schemas::{Album, MediaItem, SearchMediaItemsRequest};
use regex::Regex;
use reqwest;
use structopt::StructOpt;
use yup_oauth2::{Authenticator, InstalledFlow};
@ -125,10 +131,15 @@ impl<'a> Iterator for SearchIter<'a> {
fn print_media_items(media_items: Vec<MediaItem>) {
for mi in &media_items {
let id = mi
.id
.as_ref()
.map_or("NO ID".to_string(), |s| s.to_string());
println!(
"{} {}",
mi.id.as_ref().unwrap_or(&"NO ID".to_string()),
mi.filename.as_ref().unwrap_or(&"NO FILENAME".to_string())
"media item: {}\n\t{}\n\t{}",
mi.filename.as_ref().unwrap_or(&"NO FILENAME".to_string()),
hexihasher::sha256(id.as_bytes()),
id,
);
}
println!("({}) items total", media_items.len());
@ -136,12 +147,12 @@ fn print_media_items(media_items: Vec<MediaItem>) {
fn search_media_items(
client: &photos::Client,
album_id: String,
album_id: &str,
) -> Result<Vec<MediaItem>, Box<dyn Error>> {
let media_items = SearchIter::new(
&client,
SearchMediaItemsRequest {
album_id: Some(album_id.clone()),
album_id: Some(album_id.to_string()),
// 100 is the documented max.
page_size: Some(100),
..Default::default()
@ -152,6 +163,17 @@ fn search_media_items(
Ok(media_items)
}
lazy_static! {
static ref MIME_TO_EXT: HashMap<&'static str, &'static str> = [
("image/gif", "gif"),
("image/heif", "heic"),
("image/jpeg", "jpg"),
]
.iter()
.copied()
.collect();
}
fn sync_albums(
client: &photos::Client,
title_filter: Option<Regex>,
@ -159,13 +181,54 @@ fn sync_albums(
) -> Result<(), Box<dyn Error>> {
let albums = list_albums(client, title_filter)?;
for a in &albums {
let album_dir = output_dir.join(a.id.as_ref().expect("missing album id"));
let album_id = a.id.as_ref().expect("unset album id").to_string();
let album_dir = output_dir.join(&album_id);
if !album_dir.exists() {
info!("making album directory {}", album_dir.to_string_lossy());
fs::create_dir_all(&album_dir)?;
}
let album = search_media_items(client, a.id.as_ref().expect("unset album id").to_string())?;
let album = search_media_items(client, &album_id)?;
for (i, mi) in album.iter().enumerate() {
let mi_id = mi.id.as_ref().expect("unset media item id").to_string();
let filename = mi
.filename
.as_ref()
.map_or("NO_FILENAME".to_string(), |s| s.to_string());
// Put images from all albums in common directory.
let image_path = output_dir.join("images").join(&mi_id);
if !image_path.exists() {
fs::create_dir_all(&image_path)?;
}
let image_path = image_path.join(&filename);
if image_path.exists() {
info!(
"Skipping already downloaded {} @ {}",
&filename,
image_path.to_string_lossy()
);
} else {
let download_path = image_path.with_extension("download");
info!(
"({}/{}) Downloading {} -> {}",
i + 1,
&album.len(),
&filename,
download_path.to_string_lossy()
);
let base_url = mi.base_url.as_ref().expect("missing base_url");
let url = format!("{}=d", base_url);
let mut r = reqwest::blocking::get(&url)?;
let mut w = File::create(&download_path)?;
let _n = io::copy(&mut r, &mut w)?;
info!(
"Rename {} -> {}",
download_path.to_string_lossy(),
image_path.to_string_lossy()
);
fs::rename(download_path, &image_path)?;
}
}
let j = serde_json::to_string(&album)?;
let path = album_dir.join("album.json");
info!("saving {}", path.to_string_lossy());
@ -230,7 +293,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
Ok(())
}
Command::SearchMediaItems { album_id } => {
print_media_items(search_media_items(&client, album_id)?);
print_media_items(search_media_items(&client, &album_id)?);
Ok(())
}
Command::Sync {