94 lines
2.1 KiB
Go
94 lines
2.1 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"flag"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"xinu.tv/email"
|
|
"xinu.tv/email/maildir"
|
|
|
|
"github.com/golang/glog"
|
|
"github.com/syndtr/goleveldb/leveldb"
|
|
)
|
|
|
|
var (
|
|
levelDBPath = flag.String("db", "", "level DB path for storing email")
|
|
maildirPath = flag.String("maildir", filepath.Join(os.Getenv("HOME"), "Maildir"), "Maildir path")
|
|
skipFiles = flag.String("skip", ".notmuch,maildirfolder,log,msgid.cache,razor-agent.log",
|
|
"comma separated files to skip")
|
|
)
|
|
|
|
const maxMessageSize int64 = 50 << 20
|
|
|
|
func load(dbPath, mdPath string) error {
|
|
db, err := leveldb.OpenFile(*levelDBPath, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("error opening leveldb: %v", err)
|
|
}
|
|
defer func() {
|
|
if err := db.Close(); err != nil {
|
|
glog.Errorf("Error closing %q: %v", *levelDBPath, err)
|
|
}
|
|
}()
|
|
|
|
skipPats := strings.Split(*skipFiles, ",")
|
|
glog.Infoln("Skip files", skipPats)
|
|
imported, count := 0, 0
|
|
if err := maildir.Walk(mdPath, func(path string) error {
|
|
r, err := os.Open(path)
|
|
if err != nil {
|
|
return fmt.Errorf("%q open: %v", path, err)
|
|
}
|
|
b, err := ioutil.ReadAll(r)
|
|
if err != nil {
|
|
return fmt.Errorf("%q read: %v", path, err)
|
|
}
|
|
|
|
chksum, err := email.HashReader(bytes.NewReader(b))
|
|
if err != nil {
|
|
return fmt.Errorf("%q checksum: %v", path, err)
|
|
}
|
|
|
|
count++
|
|
if count%1000 == 0 {
|
|
glog.Infof("Processed %d files, %d imported", count, imported)
|
|
}
|
|
key := []byte(chksum)
|
|
ok, err := db.Has(key, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("%q error Has(%q): %v", path, key, err)
|
|
}
|
|
if ok {
|
|
return nil
|
|
}
|
|
//glog.Infof("Processing %v %q", chksum, path)
|
|
imported++
|
|
if err := db.Put(key, b, nil); err != nil {
|
|
return fmt.Errorf("%q put: %v", path, err)
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
return fmt.Errorf("walk error: %v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func main() {
|
|
flag.Parse()
|
|
defer glog.Flush()
|
|
if *levelDBPath == "" {
|
|
glog.Exitf("-db required")
|
|
}
|
|
if *maildirPath == "" {
|
|
glog.Exitf("-maildir required")
|
|
}
|
|
if err := load(*levelDBPath, *maildirPath); err != nil {
|
|
glog.Exitf("Failed to load: %v", err)
|
|
}
|
|
}
|