package main import ( "bytes" "flag" "fmt" "io/ioutil" "os" "path/filepath" "strings" "xinu.tv/email" "xinu.tv/email/maildir" "github.com/golang/glog" "github.com/syndtr/goleveldb/leveldb" ) var ( levelDBPath = flag.String("db", "", "level DB path for storing email") maildirPath = flag.String("maildir", filepath.Join(os.Getenv("HOME"), "Maildir"), "Maildir path") skipFiles = flag.String("skip", ".notmuch,maildirfolder,log,msgid.cache,razor-agent.log", "comma separated files to skip") ) const maxMessageSize int64 = 50 << 20 func load(dbPath, mdPath string) error { db, err := leveldb.OpenFile(*levelDBPath, nil) if err != nil { return fmt.Errorf("error opening leveldb: %v", err) } defer func() { if err := db.Close(); err != nil { glog.Errorf("Error closing %q: %v", *levelDBPath, err) } }() skipPats := strings.Split(*skipFiles, ",") glog.Infoln("Skip files", skipPats) imported, count := 0, 0 if err := maildir.Walk(mdPath, func(path string) error { r, err := os.Open(path) if err != nil { return fmt.Errorf("%q open: %v", path, err) } b, err := ioutil.ReadAll(r) if err != nil { return fmt.Errorf("%q read: %v", path, err) } chksum, err := email.HashReader(bytes.NewReader(b)) if err != nil { return fmt.Errorf("%q checksum: %v", path, err) } count++ if count%1000 == 0 { glog.Infof("Processed %d files, %d imported", count, imported) } key := []byte(chksum) ok, err := db.Has(key, nil) if err != nil { return fmt.Errorf("%q error Has(%q): %v", path, key, err) } if ok { return nil } //glog.Infof("Processing %v %q", chksum, path) imported++ if err := db.Put(key, b, nil); err != nil { return fmt.Errorf("%q put: %v", path, err) } return nil }); err != nil { return fmt.Errorf("walk error: %v", err) } return nil } func main() { flag.Parse() defer glog.Flush() if *levelDBPath == "" { glog.Exitf("-db required") } if *maildirPath == "" { glog.Exitf("-maildir required") } if err := load(*levelDBPath, *maildirPath); err != nil { glog.Exitf("Failed to load: %v", err) } }