src/lib.rs

@@ -1,4 +1,3 @@
-use std::cmp::Ordering;
 use std::collections::HashMap;
 use std::env;
 use std::ffi::OsStr;
@@ -32,20 +31,7 @@ use serde_json::Value;
 const FULL_METADATA_FILENAME: &str = "metadata.json";
 const COMPACT_METADATA_FILENAME: &str = "metadata.compact.json";
 
-pub fn clean_path_parent<P: AsRef<Path>>(path: P) -> PathBuf {
+#[derive(Clone, Deserialize, Debug, PartialEq, Serialize)]
-    let path = path.as_ref();
-    let path = path.parent().unwrap();
-    let path = path.to_str().unwrap();
-    /*
-    // Strip year
-    if path.ends_with(')') {
-        path = &path[..path.len() - 7];
-    }
-    */
-    PathBuf::from(path)
-}
-
-#[derive(Clone, Deserialize, Debug, Eq, PartialEq, Serialize)]
 pub struct Resolution(usize, usize);
 impl From<(usize, usize)> for Resolution {
     fn from(res: (usize, usize)) -> Self {
@@ -60,20 +46,6 @@ impl Display for Resolution {
     }
 }
 
-impl Ord for Resolution {
-    fn cmp(&self, other: &Resolution) -> Ordering {
-        let pixels = self.0.checked_mul(self.1).unwrap_or(usize::max_value());
-        let other_pixels = other.0.checked_mul(other.1).unwrap_or(usize::max_value());
-        pixels.cmp(&other_pixels)
-    }
-}
-
-impl PartialOrd for Resolution {
-    fn partial_cmp(&self, other: &Resolution) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
 fn option_from_str<'de, T, D>(deserializer: D) -> Result<Option<T>, D::Error>
 where
     T: FromStr,
@@ -319,47 +291,8 @@ lazy_static! {
 }
 
 #[derive(Default, Debug, PartialEq)]
-pub struct Movie {
+struct Movie {
-    pub files: Vec<(String, CompactMetadata)>,
+    files: Vec<(String, CompactMetadata)>,
-}
-
-impl Movie {
-    fn min_bit_rate(&self) -> Option<usize> {
-        if self.files.is_empty() {
-            None
-        } else {
-            Some(self.files.iter().fold(usize::max_value(), |acc, (_, cmd)| {
-                std::cmp::min(acc, cmd.bit_rate)
-            }))
-        }
-    }
-
-    fn min_resolution(&self) -> Option<Resolution> {
-        if self.files.is_empty() {
-            None
-        } else {
-            Some(self.files.iter().fold(
-                Resolution(usize::max_value(), usize::max_value()),
-                |acc, (_, cmd)| std::cmp::min(acc, cmd.largest_dimension().unwrap()),
-            ))
-        }
-    }
-}
-
-impl Display for Movie {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        let p = &self.files.first().unwrap().0;
-        write!(f, "{}", &p[..p.find("/").unwrap()])?;
-        for (path, cmd) in &self.files {
-            write!(
-                f,
-                " {} {}",
-                &path[path.rfind("/").unwrap()..],
-                cmd.largest_dimension().unwrap()
-            )?;
-        }
-        Ok(())
-    }
 }
 
 #[derive(Debug, PartialEq)]
@@ -367,41 +300,6 @@ pub struct Movies {
     movies: Vec<Movie>,
 }
 
-impl Movies {
-    /// Find all movies with multiple copies.  The returned vec contains a tuple of (Movie to keep,
-    /// One or more Movies to remove).  The highest resolution movie is kept, TODO(wathiede): with
-    /// higher bit rate breaking ties.
-    pub fn duplicate_candidates(&self) -> Vec<(&Movie, Vec<&Movie>)> {
-        let mut movie_counter = HashMap::new();
-        for m in &self.movies {
-            let (path, _cmd) = m.files.first().unwrap();
-            let parent = clean_path_parent(path).to_string_lossy().to_string();
-            movie_counter.entry(parent).or_insert(Vec::new()).push(m);
-        }
-        let mut dupes = Vec::new();
-        for (_parent, mut movies) in movie_counter.into_iter() {
-            if movies.len() > 1 {
-                // Sort, lowest bit_rate movie first
-                movies.sort_by(|a, b| a.min_bit_rate().cmp(&b.min_bit_rate()));
-                // Flip order, we care about the largest.
-                movies.reverse();
-                // Take the largest image, return the rest for removal.
-                let tuple = (movies.remove(0), movies);
-                dupes.push(tuple);
-            }
-        }
-        for d in &dupes {
-            let (biggest, deletes) = d;
-            eprintln!("biggest: {}", biggest);
-            for (i, delete) in deletes.iter().enumerate() {
-                eprintln!("{}. delete: {}", i + 1, delete);
-            }
-        }
-
-        dupes
-    }
-}
-
 fn movies_from_paths_compact_metadata(mut p_cmd: HashMap<String, CompactMetadata>) -> Movies {
     let multidisc = collapse_multidisc(&p_cmd.keys().map(|s| s.to_string()).collect());
     let movies = multidisc
@@ -438,6 +336,7 @@ impl MovieLibrary {
         let f = File::open(&path).context(format!("open {}", path.display()))?;
         let r = BufReader::new(f);
 
+        // Read the JSON contents of the file as an instance of `User`.
         let mdf: MetadataFile = serde_json::from_reader(r)
             .context(format!("serde_json::from_reader {}", path.display()))?;
 
@@ -547,7 +446,7 @@ impl MovieLibrary {
     pub fn update_metadata(&self) -> Result<Vec<String>, Error> {
         let path = Path::new(&self.root).join(FULL_METADATA_FILENAME);
 
-        let mut old_metadata: HashMap<String, Value> = match File::open(&path) {
+        let old_metadata: HashMap<String, Value> = match File::open(&path) {
             Ok(f) => {
                 let r = BufReader::new(f);
                 serde_json::from_reader(r)?
@@ -559,29 +458,6 @@ impl MovieLibrary {
         };
 
         info!("Read metadata, {} videos found", old_metadata.len());
-        // Filter out stale metadata (where the file no longer exists).
-        let old_metadata: HashMap<String, Value> = self
-            .iter_video_files()
-            .filter(|r| r.is_ok())
-            .filter_map(|r| {
-                let path = r
-                    .as_ref()
-                    .unwrap()
-                    .strip_prefix(&self.root)
-                    .unwrap()
-                    .to_str()
-                    .unwrap()
-                    .to_owned();
-                match old_metadata.remove(&path) {
-                    Some(v) => Some((path, v)),
-                    None => None,
-                }
-            })
-            .collect();
-        info!(
-            "After removing stale metadata, {} videos found",
-            old_metadata.len()
-        );
 
         let mut metadata: HashMap<_, _> = self
             .iter_video_files()

src/main.rs

@@ -1,6 +1,8 @@
 use std::collections::HashMap;
 use std::error::Error;
 use std::io::Write;
+use std::path::Path;
+use std::path::PathBuf;
 use std::time::Duration;
 
 use human_format::Formatter;
@@ -12,13 +14,26 @@ use regex::Regex;
 use structopt::StructOpt;
 use tabwriter::TabWriter;
 
-use superdeduper::clean_path_parent;
+use superdeduper::is_multidisc;
 use superdeduper::CompactMetadata;
 use superdeduper::MovieLibrary;
 
 const MOVIE_DIR: &str = "/home/wathiede/Movies";
 const TO_BE_REMOVED_DIR: &str = "/storage/media/to-be-deleted/";
 
+fn clean_path_parent<P: AsRef<Path>>(path: P) -> PathBuf {
+    let path = path.as_ref();
+    let path = path.parent().unwrap();
+    let path = path.to_str().unwrap();
+    /*
+    // Strip year
+    if path.ends_with(')') {
+        path = &path[..path.len() - 7];
+    }
+    */
+    PathBuf::from(path)
+}
+
 lazy_static! {
     static ref CLEAN_TITLE_CHARS: Regex = Regex::new("[^ 0-9[:alpha:]]").unwrap();
 }
@@ -32,44 +47,55 @@ fn normalize(path: &str) -> String {
 lazy_static! {
     static ref YEAR_SUFFIX: Regex = Regex::new(r" \d{4}$").unwrap();
 }
+fn print_dupes(videos: HashMap<String, CompactMetadata>) {
+    let mut video_groups: HashMap<String, Vec<(String, CompactMetadata)>> = HashMap::new();
+    for (name, md) in videos.into_iter() {
+        let clean_name = normalize(clean_path_parent(&name).to_str().unwrap());
+        let paths = video_groups.entry(clean_name).or_insert(Vec::new());
+        paths.push((name.to_string(), md));
+    }
 
-fn print_dupes(lib: &MovieLibrary) {
+    let mut names = video_groups.keys().collect::<Vec<_>>();
-    let videos = lib.movies().expect("couldn't get videos from library");
+    names.sort();
+
+    for name in &names {
+        if YEAR_SUFFIX.is_match(&name) {
+            let yearless = &name[..&name.len() - 5];
+            info!("is '{}' in map", yearless);
+            if let Some(yearless_vg) = video_groups.get(yearless) {
+                println!("Possible dupe between movie with year and no year:");
+                println!("  {:?}", video_groups.get(name.as_str()).unwrap());
+                println!("  {:?}", yearless_vg);
+            }
+        }
+    }
     let mut fmtr = Formatter::new();
     fmtr.with_separator("");
     fmtr.with_scales(Scales::Binary());
-    let mut delete_paths = Vec::new();
+    for name in names {
-    for (keep, deletes) in videos.duplicate_candidates() {
+        let possible_dupes = &video_groups[name];
-        let p = &keep.files.first().unwrap().0;
+        if possible_dupes.len() < 2 {
-        println!("{}", &p[..p.find("/").unwrap()]);
+            continue;
-        println!("  Keeping:");
+        }
-        for (p, md) in &keep.files {
+        let paths: Vec<String> = possible_dupes
+            .iter()
+            .map(|(name, _)| name.to_string())
+            .collect();
+        if is_multidisc(&paths) {
+            continue;
+        }
+        let mut file: Vec<_> = video_groups[name].iter().collect();
+        file.sort_by(|(n1, _), (n2, _)| n1.partial_cmp(n2).unwrap());
+        println!("{}:", name);
+        for (p, md) in file {
             println!(
-                "    {:>9} {:>9} {} {}",
+                "  {:>9} {:>9} {} {}",
                 md.largest_dimension().unwrap(),
                 fmtr.format(md.size as f64),
                 humantime::Duration::from(Duration::from_secs(md.duration as u64)),
                 &p[p.rfind("/").unwrap() + 1..]
             );
         }
-        println!("  Need to remove:");
-        for delete in &deletes {
-            for (p, md) in &delete.files {
-                delete_paths.push(p);
-                println!(
-                    "    {:>9} {:>9} {} {}",
-                    md.largest_dimension().unwrap(),
-                    fmtr.format(md.size as f64),
-                    humantime::Duration::from(Duration::from_secs(md.duration as u64)),
-                    &p[p.rfind("/").unwrap() + 1..]
-                );
-            }
-        }
-        println!();
-    }
-    delete_paths.sort();
-    for path in &delete_paths {
-        println!(r#"mv "{}" /storage/media/to-be-deleted/"#, path);
     }
 }
 
@@ -231,7 +257,9 @@ fn main() -> Result<(), Box<dyn Error>> {
         }
         Command::PrintDupes => {
             let lib = MovieLibrary::new(MOVIE_DIR);
-            print_dupes(&lib);
+            let videos = lib.videos()?;
+
+            print_dupes(videos);
         }
         Command::PrintAll => {
             let lib = MovieLibrary::new(MOVIE_DIR);
@@ -245,8 +273,14 @@ fn main() -> Result<(), Box<dyn Error>> {
         }
         Command::UpdateAndCompactMetadata => {
             let lib = MovieLibrary::new(MOVIE_DIR);
-            lib.update_metadata()?;
+            let new_videos = lib.update_metadata()?;
-            lib.compact_metadata()?;
+            if !new_videos.is_empty() {
+                info!(
+                    "{} new videos added, recompacting metadata",
+                    new_videos.len()
+                );
+                lib.compact_metadata()?;
+            }
         }
     }
     Ok(())

src/movielibrary_test.rs

@@ -49,7 +49,7 @@ fn test_simple_library() {
     );
 }
 
-fn build_tuple<R>(path: &str, res: R, bit_rate: usize) -> (String, CompactMetadata)
+fn build_tuple<R>(path: &str, res: R) -> (String, CompactMetadata)
 where
     R: Into<Resolution>,
 {
@@ -58,7 +58,7 @@ where
         path.to_string(),
         CompactMetadata {
             filename: format!("./{}", path),
-            bit_rate,
+            bit_rate: 1,
             duration: 1.0,
             format_name: "test_format".to_string(),
             size: 1,
@@ -72,14 +72,14 @@ where
     )
 }
 
-fn build_movie<R>(paths: Vec<(&str, R, usize)>) -> Movie
+fn build_movie<R>(paths: Vec<(&str, R)>) -> Movie
 where
     R: Into<Resolution>,
 {
     Movie {
         files: paths
             .into_iter()
-            .map(|(path, res, bit_rate)| build_tuple(path, res, bit_rate))
+            .map(|(path, res)| build_tuple(path, res))
             .collect(),
     }
 }
@@ -89,32 +89,26 @@ fn build_complex_metadata() -> HashMap<String, CompactMetadata> {
         build_tuple(
             "One Movie With Year (2019)/abcdef123456789.mkv",
             (1920, 1080),
-            1,
         ),
         build_tuple(
             "One Movie With Two Parts (2019)/abcdef123456789 part 1.mkv",
             (1280, 720),
-            1,
         ),
         build_tuple(
             "One Movie With Two Parts (2019)/abcdef123456789 part 2.mkv",
             (1280, 720),
-            1,
         ),
         build_tuple(
             "Two Movies With Multi Parts (2019)/abcdef123456789 part 1.mkv",
             (1280, 720),
-            1000,
         ),
         build_tuple(
             "Two Movies With Multi Parts (2019)/abcdef123456789 part 2.mkv",
             (1280, 720),
-            1000,
         ),
         build_tuple(
             "Two Movies With Multi Parts (2019)/somethingelse.mkv",
             (1920, 1080),
-            5000,
         ),
     ]
     .into_iter()
@@ -127,36 +121,30 @@ fn build_complex_movies() -> Movies {
             build_movie(vec![(
                 "One Movie With Year (2019)/abcdef123456789.mkv",
                 (1920, 1080),
-                1,
             )]),
             build_movie(vec![
                 (
                     "One Movie With Two Parts (2019)/abcdef123456789 part 1.mkv",
                     (1280, 720),
-                    1,
                 ),
                 (
                     "One Movie With Two Parts (2019)/abcdef123456789 part 2.mkv",
                     (1280, 720),
-                    1,
                 ),
             ]),
             build_movie(vec![
                 (
                     "Two Movies With Multi Parts (2019)/abcdef123456789 part 1.mkv",
                     (1280, 720),
-                    1000,
                 ),
                 (
                     "Two Movies With Multi Parts (2019)/abcdef123456789 part 2.mkv",
                     (1280, 720),
-                    1000,
                 ),
             ]),
             build_movie(vec![(
                 "Two Movies With Multi Parts (2019)/somethingelse.mkv",
                 (1920, 1080),
-                5000,
             )]),
         ],
     };
@@ -185,122 +173,6 @@ fn test_roundtrip_library() -> Result<(), Box<dyn Error>> {
     let got = ml.movies().expect("failed to build movies");
     assert_eq!(got.movies.len(), want.movies.len());
     assert_eq!(got, want);
-    Ok(())
+    //assert_eq!(got, want, "Got {:#?}\nWant {:#?}", got, want);
-}
-
-fn validate_duplicates(got: Vec<(&Movie, Vec<&Movie>)>, want: Vec<(Movie, Vec<Movie>)>) {
-    assert_eq!(got.len(), want.len());
-    for (g, w) in got.iter().zip(&want) {
-        assert_eq!(g.0, &w.0);
-        assert_eq!(g.1, w.1.iter().map(|v| v).collect::<Vec<_>>());
-    }
-}
-
-#[test]
-fn test_duplicate_candidates() -> Result<(), Box<dyn Error>> {
-    let movies = build_complex_movies();
-    let got = movies.duplicate_candidates();
-    let want = vec![(
-        build_movie(vec![(
-            "Two Movies With Multi Parts (2019)/somethingelse.mkv",
-            (1920, 1080),
-            5000,
-        )]),
-        vec![build_movie(vec![
-            (
-                "Two Movies With Multi Parts (2019)/abcdef123456789 part 1.mkv",
-                (1280, 720),
-                1000,
-            ),
-            (
-                "Two Movies With Multi Parts (2019)/abcdef123456789 part 2.mkv",
-                (1280, 720),
-                1000,
-            ),
-        ])],
-    )];
-    validate_duplicates(got, want);
-    Ok(())
-}
-
-#[test]
-fn test_fullmetal() -> Result<(), Box<dyn Error>> {
-    let mut movies = Movies {
-        movies: vec![
-            build_movie(vec![(
-                "Full Metal Jacket (1987)/Full Metal Jacket.mp4",
-                (1280, 720),
-                2581935,
-            )]),
-            build_movie(vec![(
-                "Full Metal Jacket (1987)/1776f8e2fb614a6fb77a66cde601bb45.mkv",
-                (1920, 1080),
-                5719802,
-            )]),
-        ],
-    };
-    movies.movies.sort_by(|a, b| {
-        a.files
-            .first()
-            .unwrap()
-            .0
-            .partial_cmp(&b.files.first().unwrap().0)
-            .unwrap()
-    });
-    let got = movies.duplicate_candidates();
-    let want = vec![(
-        build_movie(vec![(
-            "Full Metal Jacket (1987)/1776f8e2fb614a6fb77a66cde601bb45.mkv",
-            (1920, 1080),
-            5719802,
-        )]),
-        vec![build_movie(vec![(
-            "Full Metal Jacket (1987)/Full Metal Jacket.mp4",
-            (1280, 720),
-            2581935,
-        )])],
-    )];
-    validate_duplicates(got, want);
-    Ok(())
-}
-
-#[test]
-fn test_keep_lower_res_higher_bit_rate() -> Result<(), Box<dyn Error>> {
-    let mut movies = Movies {
-        movies: vec![
-            build_movie(vec![(
-                "X Men The Last Stand (2006)/X.Men.The.Last.Stand.2006.1080p.BluRay.x264.DTS-ES.PRoDJi.mkv",
-                (1920, 800),
-                11349705,
-            )]),
-            build_movie(vec![(
-                "X Men The Last Stand (2006)/948f08a4ba784626ac13de77b77559dd.mkv",
-                (1920, 1080),
-                6574160,
-            )]),
-        ],
-    };
-    movies.movies.sort_by(|a, b| {
-        a.files
-            .first()
-            .unwrap()
-            .0
-            .partial_cmp(&b.files.first().unwrap().0)
-            .unwrap()
-    });
-    let got = movies.duplicate_candidates();
-    let want = vec![(
-        build_movie(vec![(
-                "X Men The Last Stand (2006)/X.Men.The.Last.Stand.2006.1080p.BluRay.x264.DTS-ES.PRoDJi.mkv",
-                (1920, 800),
-                11349705,
-        )]),
-        vec![build_movie(vec![(
-                "X Men The Last Stand (2006)/948f08a4ba784626ac13de77b77559dd.mkv",
-                (1920, 1080),
-                6574160,
-        )])],
-    )];
-    validate_duplicates(got, want);
     Ok(())
 }