LevelDB test programs.

2016-12-21 21:05:10 -08:00
parent f2ef02a033
commit a4f102ad99
2 changed files with 265 additions and 0 deletions
--- a/cmd/md2leveldb/md2leveldb.go
+++ b/cmd/md2leveldb/md2leveldb.go
@@ -0,0 +1,121 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"xinu.tv/email"
+
+	"github.com/golang/glog"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+var (
+	levelDBPath = flag.String("db", "", "level DB path for storing email")
+	maildirPath = flag.String("maildir", filepath.Join(os.Getenv("HOME"), "Maildir"), "Maildir path")
+	skipFiles   = flag.String("skip", ".notmuch,maildirfolder,log,msgid.cache,razor-agent.log",
+		"comma separated files to skip")
+)
+
+const maxMessageSize int64 = 50 << 20
+
+func load(dbPath, mdPath string) error {
+	db, err := leveldb.OpenFile(*levelDBPath, nil)
+	if err != nil {
+		return fmt.Errorf("error opening leveldb: %v", err)
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			glog.Errorf("Error closing %q: %v", *levelDBPath, err)
+		}
+	}()
+
+	skipPats := strings.Split(*skipFiles, ",")
+	glog.Infoln("Skip files", skipPats)
+	imported, count := 0, 0
+	if err := filepath.Walk(mdPath, func(path string, info os.FileInfo, err error) error {
+		base := filepath.Base(path)
+		if info.IsDir() {
+			glog.Infoln(path)
+			for _, pat := range skipPats {
+				ok, err := filepath.Match(pat, base)
+				if err != nil {
+					return err
+				}
+				if ok {
+					return filepath.SkipDir
+
+				}
+			}
+		}
+
+		if !info.Mode().IsRegular() {
+			return nil
+		}
+		for _, pat := range skipPats {
+			ok, err := filepath.Match(pat, base)
+			if err != nil {
+				return err
+			}
+			if ok {
+				return nil
+
+			}
+		}
+
+		r, err := os.Open(path)
+		if err != nil {
+			return fmt.Errorf("%q open: %v", path, err)
+		}
+		b, err := ioutil.ReadAll(r)
+		if err != nil {
+			return fmt.Errorf("%q read: %v", path, err)
+		}
+
+		chksum, err := email.HashReader(bytes.NewReader(b))
+		if err != nil {
+			return fmt.Errorf("%q checksum: %v", path, err)
+		}
+
+		count++
+		if count%1000 == 0 {
+			glog.Infof("Processed %d files, %d imported", count, imported)
+		}
+		key := []byte(chksum)
+		ok, err := db.Has(key, nil)
+		if err != nil {
+			return fmt.Errorf("%q error Has(%q): %v", path, key, err)
+		}
+		if ok {
+			return nil
+		}
+		//glog.Infof("Processing %v %q", chksum, path)
+		imported++
+		if err := db.Put(key, b, nil); err != nil {
+			return fmt.Errorf("%q put: %v", path, err)
+		}
+		return nil
+	}); err != nil {
+		return fmt.Errorf("walk error: %v", err)
+	}
+	return nil
+}
+
+func main() {
+	flag.Parse()
+	defer glog.Flush()
+	if *levelDBPath == "" {
+		glog.Exitf("-db required")
+	}
+	if *maildirPath == "" {
+		glog.Exitf("-maildir required")
+	}
+	if err := load(*levelDBPath, *maildirPath); err != nil {
+		glog.Exitf("Failed to load: %v", err)
+	}
+}