wathiede/superdeduper
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix sample command.

Browse Source 
Add largest_dimension and unitest.
Use const for metadata filenames.
This commit is contained in:
Bill Thiede 2019-11-03 20:59:31 -08:00
parent 185c3cde2d
commit 8b809060ec
2 changed files with 156 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

166
src/lib.rs
View File

@ -27,7 +27,10 @@ use serde::Deserialize;
use serde::Serialize; use serde::Serialize;
use serde_json::Value; use serde_json::Value;
#[derive(Clone, Deserialize, Debug, Serialize)] const FULL_METADATA_FILENAME: &str = "metadata.json";
const COMPACT_METADATA_FILENAME: &str = "metadata.compact.json";
#[derive(Clone, Deserialize, Debug, PartialEq, Serialize)]
pub struct Resolution(usize, usize); pub struct Resolution(usize, usize);
impl Display for Resolution { impl Display for Resolution {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
@ -57,7 +60,7 @@ where
} }
#[derive(Clone, Deserialize, Debug, Serialize)] #[derive(Clone, Deserialize, Debug, Serialize)]
pub struct Format { struct Format {
#[serde(deserialize_with = "from_str")] #[serde(deserialize_with = "from_str")]
bit_rate: usize, bit_rate: usize,
#[serde(deserialize_with = "from_str")] #[serde(deserialize_with = "from_str")]
@ -83,7 +86,7 @@ impl Tags {
// TODO(wathiede): make strem an enum with the tag type stored in codec_type? // TODO(wathiede): make strem an enum with the tag type stored in codec_type?
#[derive(Clone, Deserialize, Debug, Serialize)] #[derive(Clone, Deserialize, Debug, Serialize)]
#[serde(tag = "codec_type")] #[serde(tag = "codec_type")]
pub enum Stream { enum Stream {
#[serde(rename = "video")] #[serde(rename = "video")]
Video { Video {
#[serde(default, deserialize_with = "option_from_str")] #[serde(default, deserialize_with = "option_from_str")]
@ -127,7 +130,7 @@ impl Stream {
} }
#[derive(Clone, Deserialize, Debug, Serialize)] #[derive(Clone, Deserialize, Debug, Serialize)]
pub struct Metadata { struct Metadata {
format: Format, format: Format,
streams: Vec<Stream>, streams: Vec<Stream>,
} }
@ -156,6 +159,20 @@ pub struct VideoFormat {
language: Option<String>, language: Option<String>,
} }
#[cfg(test)]
impl Default for VideoFormat {
fn default() -> Self {
VideoFormat {
short_name: "UNAMED_SHORT".to_string(),
long_name: "UNAMED_LONG".to_string(),
height: 0,
width: 0,
title: None,
language: None,
}
}
}
#[derive(Clone, Deserialize, Debug, Serialize)] #[derive(Clone, Deserialize, Debug, Serialize)]
pub struct AudioFormat { pub struct AudioFormat {
short_name: String, short_name: String,
@ -180,24 +197,54 @@ pub struct SubtitleFormat {
#[derive(Clone, Deserialize, Debug, Serialize)] #[derive(Clone, Deserialize, Debug, Serialize)]
pub struct CompactMetadata { pub struct CompactMetadata {
#[serde(deserialize_with = "from_str")]
bit_rate: usize, bit_rate: usize,
#[serde(deserialize_with = "from_str")] pub duration: f32,
duration: f32,
filename: String, filename: String,
format_name: String, format_name: String,
#[serde(deserialize_with = "from_str")] pub size: usize,
size: usize,
video: Vec<VideoFormat>, video: Vec<VideoFormat>,
audio: Vec<AudioFormat>, audio: Vec<AudioFormat>,
#[serde(skip_serializing_if = "Vec::is_empty")]
subtitle: Vec<SubtitleFormat>, subtitle: Vec<SubtitleFormat>,
} }
impl CompactMetadata {
pub fn largest_dimension(&self) -> Option<Resolution> {
if self.video.is_empty() {
return None;
}
Some(self.video.iter().fold(
Resolution(0, 0),
|acc, VideoFormat { width, height, .. }| {
if acc.0 * acc.1 < width * height {
Resolution(*width, *height)
} else {
acc
}
},
))
}
}
#[cfg(test)]
impl Default for CompactMetadata {
fn default() -> Self {
CompactMetadata {
bit_rate: 0,
duration: 0.,
filename: "UNSET".to_string(),
format_name: "UNKNOWN".to_string(),
size: 0,
video: Vec::new(),
audio: Vec::new(),
subtitle: Vec::new(),
}
}
}
#[derive(Deserialize, Debug, Serialize)] #[derive(Deserialize, Debug, Serialize)]
pub struct MetadataFile { pub struct MetadataFile {
#[serde(flatten)] #[serde(flatten)]
pub metadata: HashMap<String, Metadata>, metadata: HashMap<String, Metadata>,
} }
pub struct MovieLibrary { pub struct MovieLibrary {
@ -241,7 +288,15 @@ impl MovieLibrary {
} }
pub fn compact_metadata(&self) -> Result<(), Error> { pub fn compact_metadata(&self) -> Result<(), Error> {
let mdf = read_metadata_from_file(Path::new(&self.root).join("metadata.json"))?; let path = Path::new(&self.root).join(FULL_METADATA_FILENAME);
// Open the file in read-only mode with buffer.
let f = File::open(&path).context(format!("open {}", path.display()))?;
let r = BufReader::new(f);
// Read the JSON contents of the file as an instance of `User`.
let mdf: MetadataFile = serde_json::from_reader(r)
.context(format!("serde_json::from_reader {}", path.display()))?;
info!("Read metadata, {} videos found", mdf.metadata.len()); info!("Read metadata, {} videos found", mdf.metadata.len());
let metadata: HashMap<String, CompactMetadata> = mdf let metadata: HashMap<String, CompactMetadata> = mdf
@ -340,13 +395,13 @@ impl MovieLibrary {
}) })
.collect(); .collect();
let f = File::create(Path::new(&self.root).join("metadata.compact.json"))?; let f = File::create(Path::new(&self.root).join(COMPACT_METADATA_FILENAME))?;
let f = BufWriter::new(f); let f = BufWriter::new(f);
Ok(serde_json::ser::to_writer_pretty(f, &metadata)?) Ok(serde_json::ser::to_writer_pretty(f, &metadata)?)
} }
pub fn update_metadata(&self) -> Result<Vec<String>, Error> { pub fn update_metadata(&self) -> Result<Vec<String>, Error> {
let path = Path::new(&self.root).join("metadata.json"); let path = Path::new(&self.root).join(FULL_METADATA_FILENAME);
// Open the file in read-only mode with buffer. // Open the file in read-only mode with buffer.
let f = File::open(&path).context(format!("open {}", path.display()))?; let f = File::open(&path).context(format!("open {}", path.display()))?;
let r = BufReader::new(f); let r = BufReader::new(f);
@ -389,14 +444,14 @@ impl MovieLibrary {
}) })
.map(|(path, json)| (path, serde_json::from_str::<Value>(&json).unwrap())) .map(|(path, json)| (path, serde_json::from_str::<Value>(&json).unwrap()))
.collect(); .collect();
let new_movies = metadata.keys().cloned().collect(); let new_videos = metadata.keys().cloned().collect();
info!("Adding {} new videos", metadata.len()); info!("Adding {} new videos", metadata.len());
metadata.extend(old_metadata); metadata.extend(old_metadata);
let f = File::create(Path::new(&self.root).join("metadata.json"))?; let f = File::create(Path::new(&self.root).join(FULL_METADATA_FILENAME))?;
let f = BufWriter::new(f); let f = BufWriter::new(f);
serde_json::ser::to_writer_pretty(f, &metadata)?; serde_json::ser::to_writer_pretty(f, &metadata)?;
Ok(new_movies) Ok(new_videos)
} }
fn iter_video_files(&self) -> impl Send + Iterator<Item = Result<PathBuf, glob::GlobError>> { fn iter_video_files(&self) -> impl Send + Iterator<Item = Result<PathBuf, glob::GlobError>> {
@ -415,25 +470,16 @@ impl MovieLibrary {
}) })
} }
pub fn movies(&self, include_stale: bool) -> Result<(HashMap<String, Metadata>), Error> { pub fn videos(&self, include_stale: bool) -> Result<(HashMap<String, CompactMetadata>), Error> {
let mut movies = HashMap::new(); // TODO(wathiede): implement include_stale.
for md in glob(&format!("{}/*/metadata.json", self.root))? { let path = Path::new(&self.root).join(COMPACT_METADATA_FILENAME);
let path = md?; // Open the file in read-only mode with buffer.
let mdf = read_metadata_from_file(&path)?; let f = File::open(&path).context(format!("open {}", path.display()))?;
for (name, md) in mdf.metadata { let r = BufReader::new(f);
if include_stale {
movies.insert(name, md); // Read the JSON contents of the file as an instance of `User`.
} else { Ok(serde_json::from_reader(r)
// Filter out files that don't exist .context(format!("serde_json::from_reader {}", path.display()))?)
let mut p = PathBuf::from(&self.root);
p.push(&name);
if p.is_file() {
movies.insert(name, md);
}
}
}
}
Ok(movies)
} }
} }
@ -459,10 +505,54 @@ mod tests {
format!("{}/testdata", env::var("CARGO_MANIFEST_DIR").unwrap()) format!("{}/testdata", env::var("CARGO_MANIFEST_DIR").unwrap())
} }
#[test]
fn largest_dimension() {
let md = CompactMetadata {
..Default::default()
};
assert_eq!(md.largest_dimension(), None);
let md = CompactMetadata {
video: vec![
VideoFormat {
height: 3,
width: 4,
..Default::default()
},
VideoFormat {
width: 640,
height: 480,
..Default::default()
},
],
..Default::default()
};
assert_eq!(md.largest_dimension(), Some(Resolution(640, 480)));
let md = CompactMetadata {
video: vec![
VideoFormat {
width: 640,
height: 480,
..Default::default()
},
VideoFormat {
height: 3,
width: 4,
..Default::default()
},
],
..Default::default()
};
assert_eq!(md.largest_dimension(), Some(Resolution(640, 480)));
}
#[test] #[test]
fn test_read_full_metadata() { fn test_read_full_metadata() {
let mdf = read_metadata_from_file(Path::new(&testdata_dir()).join("Movies/metadata.json")) let mdf = read_metadata_from_file(
.expect("failed to read metadata"); Path::new(&testdata_dir()).join(&format!("Movies/{}", FULL_METADATA_FILENAME)),
)
.expect("failed to read metadata");
assert_eq!(mdf.metadata.len(), 1214); assert_eq!(mdf.metadata.len(), 1214);
} }

56
src/main.rs
View File

@ -9,7 +9,7 @@ use log::info;
use regex::Regex; use regex::Regex;
use structopt::StructOpt; use structopt::StructOpt;
use superdeduper::Metadata; use superdeduper::CompactMetadata;
use superdeduper::MovieLibrary; use superdeduper::MovieLibrary;
const MOVIE_DIR: &str = "/home/wathiede/Movies"; const MOVIE_DIR: &str = "/home/wathiede/Movies";
@ -25,35 +25,35 @@ fn clean_path_parent<P: AsRef<Path>>(path: P) -> PathBuf {
PathBuf::from(path) PathBuf::from(path)
} }
fn print_movie_groups(movie_groups: &HashMap<PathBuf, Vec<(String, Metadata)>>) { fn print_video_groups(video_groups: &HashMap<PathBuf, Vec<(String, CompactMetadata)>>) {
let mut names = movie_groups.keys().collect::<Vec<_>>(); let mut names = video_groups.keys().collect::<Vec<_>>();
names.sort(); names.sort();
let mut fmtr = Formatter::new(); let mut fmtr = Formatter::new();
fmtr.with_separator(""); fmtr.with_separator("");
fmtr.with_scales(Scales::Binary()); fmtr.with_scales(Scales::Binary());
for name in names { for name in names {
let paths = &movie_groups[name]; let paths = &video_groups[name];
if paths.len() < 2 { if paths.len() < 2 {
continue; continue;
} }
let mut file: Vec<_> = movie_groups[name].iter().collect(); let mut file: Vec<_> = video_groups[name].iter().collect();
file.sort_by(|(n1, _), (n2, _)| n1.partial_cmp(n2).unwrap()); file.sort_by(|(n1, _), (n2, _)| n1.partial_cmp(n2).unwrap());
println!("{}:", name.display()); println!("{}:", name.display());
for (p, md) in file { for (p, md) in file {
println!( println!(
" {:>9} {:>9} {} {}", " {:>9} {:>9} {} {}",
md.dimension().unwrap(), md.largest_dimension().unwrap(),
fmtr.format(md.size() as f64), fmtr.format(md.size as f64),
md.duration(), md.duration,
&p[p.rfind("/").unwrap() + 1..] &p[p.rfind("/").unwrap() + 1..]
); );
} }
} }
} }
fn print_movies(movies: &HashMap<String, Metadata>, filter: Option<&Regex>) { fn print_videos(videos: &HashMap<String, CompactMetadata>, filter: Option<&Regex>) {
let mut names = movies.keys().collect::<Vec<_>>(); let mut names = videos.keys().collect::<Vec<_>>();
names.sort(); names.sort();
let mut fmtr = Formatter::new(); let mut fmtr = Formatter::new();
fmtr.with_separator(""); fmtr.with_separator("");
@ -64,12 +64,12 @@ fn print_movies(movies: &HashMap<String, Metadata>, filter: Option<&Regex>) {
continue; continue;
} }
} }
let md = &movies[name]; let md = &videos[name];
info!( info!(
"{:>9} {:>8} {} {}", "{:>9} {:>8} {} {}",
md.dimension().unwrap(), md.largest_dimension().unwrap(),
fmtr.format(md.size() as f64), fmtr.format(md.size as f64),
md.duration(), md.duration,
&name[MOVIE_DIR.len() + 1..] &name[MOVIE_DIR.len() + 1..]
); );
println!("mv '{}' '{}'", name, TO_BE_REMOVED_DIR); println!("mv '{}' '{}'", name, TO_BE_REMOVED_DIR);
@ -78,9 +78,9 @@ fn print_movies(movies: &HashMap<String, Metadata>, filter: Option<&Regex>) {
#[derive(StructOpt)] #[derive(StructOpt)]
enum Command { enum Command {
#[structopt(name = "samples", about = "Print movie files deemed to be samples")] #[structopt(name = "samples", about = "Print video files deemed to be samples")]
Samples, Samples,
#[structopt(name = "groups", about = "Print movies grouped by root name")] #[structopt(name = "groups", about = "Print videos grouped by root name")]
Groups, Groups,
#[structopt( #[structopt(
name = "compact-metadata", name = "compact-metadata",
@ -99,7 +99,7 @@ enum Command {
#[derive(StructOpt)] #[derive(StructOpt)]
#[structopt( #[structopt(
name = "superdeduper", name = "superdeduper",
about = "Tool for pruning extra movies in collection" about = "Tool for pruning extra videos in collection"
)] )]
struct SuperDeduper { struct SuperDeduper {
#[structopt( #[structopt(
@ -130,23 +130,23 @@ fn main() -> Result<(), Box<dyn Error>> {
match app.cmd { match app.cmd {
Command::Samples => { Command::Samples => {
let lib = MovieLibrary::new(MOVIE_DIR); let lib = MovieLibrary::new(MOVIE_DIR);
let movies = lib.movies(false)?; let videos = lib.videos(false)?;
let samples_re = Regex::new(r"(?i).*sample.*").unwrap(); let samples_re = Regex::new(r"(?i).*sample.*").unwrap();
print_movies(&movies, Some(&samples_re)); print_videos(&videos, Some(&samples_re));
} }
Command::Groups => { Command::Groups => {
let lib = MovieLibrary::new(MOVIE_DIR); let lib = MovieLibrary::new(MOVIE_DIR);
let movies = lib.movies(false)?; let videos = lib.videos(false)?;
let mut movie_groups: HashMap<PathBuf, Vec<(String, Metadata)>> = HashMap::new(); let mut video_groups: HashMap<PathBuf, Vec<(String, CompactMetadata)>> = HashMap::new();
for (name, md) in movies.into_iter() { for (name, md) in videos.into_iter() {
let clean_name = clean_path_parent(&name); let clean_name = clean_path_parent(&name);
let paths = movie_groups.entry(clean_name).or_insert(Vec::new()); let paths = video_groups.entry(clean_name).or_insert(Vec::new());
paths.push((name.to_string(), md)); paths.push((name.to_string(), md));
} }
print_movie_groups(&movie_groups); print_video_groups(&video_groups);
} }
Command::CompactMetadata => { Command::CompactMetadata => {
let lib = MovieLibrary::new(MOVIE_DIR); let lib = MovieLibrary::new(MOVIE_DIR);
@ -158,11 +158,11 @@ fn main() -> Result<(), Box<dyn Error>> {
} }
Command::UpdateAndCompactMetadata => { Command::UpdateAndCompactMetadata => {
let lib = MovieLibrary::new(MOVIE_DIR); let lib = MovieLibrary::new(MOVIE_DIR);
let new_movies = lib.update_metadata()?; let new_videos = lib.update_metadata()?;
if !new_movies.is_empty() { if !new_videos.is_empty() {
info!( info!(
"{} new movies added, recompacting metadata", "{} new videos added, recompacting metadata",
new_movies.len() new_videos.len()
); );
lib.compact_metadata()?; lib.compact_metadata()?;
} }