diff --git a/cmd/md2pq/md2pq.go b/cmd/md2pq/md2pq.go index 2af8299..a2649b3 100644 --- a/cmd/md2pq/md2pq.go +++ b/cmd/md2pq/md2pq.go @@ -4,11 +4,13 @@ package main import ( "bytes" "database/sql" + "expvar" "flag" "fmt" "io/ioutil" "os" "path/filepath" + "strconv" "strings" "time" @@ -28,6 +30,10 @@ var ( // Hashed over fields from each message. headers = []string{"to", "from", "cc", "date", "subject", "message-id"} + + total = expvar.NewInt("bytes-parsed") + cnt = expvar.NewInt("messages-parsed") + dupCnt = expvar.NewInt("duplicates-found") ) var CRCR = []byte("\n\n") @@ -35,11 +41,9 @@ var CRCR = []byte("\n\n") func Load(db *sql.DB, uid int, root string, skip *set.StringSet) error { dup := set.NewStrings() start := time.Now() - cnt := 0 - dupCnt := 0 defer func() { - glog.Infof("%d messages processed in %s", cnt, time.Since(start)) - glog.Infof("%d dups found", dupCnt) + glog.Infof("%s messages processed in %s", cnt, time.Since(start)) + glog.Infof("%s dups found", dupCnt) }() txn, err := db.Begin() @@ -65,7 +69,6 @@ func Load(db *sql.DB, uid int, root string, skip *set.StringSet) error { return err } - var total int err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -106,24 +109,32 @@ func Load(db *sql.DB, uid int, root string, skip *set.StringSet) error { if dup.Contains(chksum) { glog.Warningln("Dup ", chksum, path[len(root)+1:], len(b)) - dupCnt++ + dupCnt.Add(1) return nil } dup.Add(chksum) n := len(b) - total += n + total.Add(int64(n)) delta := time.Since(start) - if cnt%1000 == 0 { - glog.Infof("%d messages processed in %s: %.2f msg/s %s/s", cnt, - delta, float64(cnt)/delta.Seconds(), - types.Base2Size(float64(total)/delta.Seconds())) + i, err := strconv.Atoi(cnt.String()) + if err != nil { + return err + } + if i%1000 == 0 { + t, err := strconv.Atoi(total.String()) + if err != nil { + return err + } + glog.Infof("%d messages processed in %s: %.2f msg/s %s/s", i, + delta, float64(i)/delta.Seconds(), + types.Base2Size(float64(t)/delta.Seconds())) } if _, err := stmt.Exec(uid, chksum, hdr_size, n, b); err != nil { return err } - cnt++ + cnt.Add(1) return nil })