Use expvar for message stats.

cmd/md2pq/md2pq.go

@@ -4,11 +4,13 @@ package main
 import (
 	"bytes"
 	"database/sql"
+	"expvar"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -28,6 +30,10 @@ var (
 
 	// Hashed over fields from each message.
 	headers = []string{"to", "from", "cc", "date", "subject", "message-id"}
+
+	total  = expvar.NewInt("bytes-parsed")
+	cnt    = expvar.NewInt("messages-parsed")
+	dupCnt = expvar.NewInt("duplicates-found")
 )
 
 var CRCR = []byte("\n\n")
@@ -35,11 +41,9 @@ var CRCR = []byte("\n\n")
 func Load(db *sql.DB, uid int, root string, skip *set.StringSet) error {
 	dup := set.NewStrings()
 	start := time.Now()
-	cnt := 0
-	dupCnt := 0
 	defer func() {
-		glog.Infof("%d messages processed in %s", cnt, time.Since(start))
-		glog.Infof("%d dups found", dupCnt)
+		glog.Infof("%s messages processed in %s", cnt, time.Since(start))
+		glog.Infof("%s dups found", dupCnt)
 	}()
 
 	txn, err := db.Begin()
@@ -65,7 +69,6 @@ func Load(db *sql.DB, uid int, root string, skip *set.StringSet) error {
 		return err
 	}
 
-	var total int
 	err = filepath.Walk(root,
 		func(path string, info os.FileInfo, err error) error {
 			if err != nil {
@@ -106,24 +109,32 @@ func Load(db *sql.DB, uid int, root string, skip *set.StringSet) error {
 
 			if dup.Contains(chksum) {
 				glog.Warningln("Dup ", chksum, path[len(root)+1:], len(b))
-				dupCnt++
+				dupCnt.Add(1)
 				return nil
 			}
 			dup.Add(chksum)
 
 			n := len(b)
-			total += n
+			total.Add(int64(n))
 			delta := time.Since(start)
-			if cnt%1000 == 0 {
-				glog.Infof("%d messages processed in %s: %.2f msg/s %s/s", cnt,
-					delta, float64(cnt)/delta.Seconds(),
-					types.Base2Size(float64(total)/delta.Seconds()))
+			i, err := strconv.Atoi(cnt.String())
+			if err != nil {
+				return err
+			}
+			if i%1000 == 0 {
+				t, err := strconv.Atoi(total.String())
+				if err != nil {
+					return err
+				}
+				glog.Infof("%d messages processed in %s: %.2f msg/s %s/s", i,
+					delta, float64(i)/delta.Seconds(),
+					types.Base2Size(float64(t)/delta.Seconds()))
 			}
 			if _, err := stmt.Exec(uid, chksum, hdr_size, n, b); err != nil {
 				return err
 			}
 
-			cnt++
+			cnt.Add(1)
 			return nil
 		})