From e7d568551fdf5db66db58e8a0548d2fc42847cde Mon Sep 17 00:00:00 2001
From: Bill Thiede <git@xinu.tv>
Date: Mon, 9 Oct 2017 20:55:06 -0700
Subject: [PATCH] Export metrics via prometheus.

---
 zfs_replication_exporter.go | 115 +++++++++++++++++++++++++-----------
 1 file changed, 82 insertions(+), 33 deletions(-)

diff --git a/zfs_replication_exporter.go b/zfs_replication_exporter.go
index 325d0fa..bdf21e6 100644
--- a/zfs_replication_exporter.go
+++ b/zfs_replication_exporter.go
@@ -6,27 +6,51 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 	"regexp"
 	"time"
 
 	"github.com/golang/glog"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"golang.org/x/crypto/ssh"
 )
 
 var (
-	host = flag.String("host", "localhost:22", "host:port to scp from")
-	user = flag.String("user", os.Getenv("USER"), "ssh user")
+	addr            = flag.String("addr", "localhost:9999", "HTTP listen address for prometheus /metrics")
+	host            = flag.String("host", "localhost:22", "host:port to scp from")
+	refreshInterval = flag.Duration("refresh", 5*time.Minute, "refresh interval time")
+	user            = flag.String("user", os.Getenv("USER"), "ssh user")
 )
 
+const snapshotListCmd = "/sbin/zfs list -t snapshot -H -o name -s name"
+
 // Example: @auto-20171001.1400-2w
 var (
 	snapshotPattern = regexp.MustCompile(`^([^@]+)@auto-(\d{8}\.\d{4})-\d+[mwy]$`)
 	snapshotFormat  = "20060102.1504"
+
+	fetchRequestDurationMetric = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "ssh_fetch_duration_seconds",
+		Help: "Time to fetch and parse snapshot age over SSH",
+	})
+	snapshotAgesMetric = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "zfs_snapshot_age_seconds",
+			Help: "Duration in seconds for most recent snapshot for `filesystem`",
+		},
+		[]string{"filesystem"},
+	)
 )
 
+func init() {
+	prometheus.MustRegister(fetchRequestDurationMetric)
+	prometheus.MustRegister(snapshotAgesMetric)
+}
+
 func newPublicKey() ([]ssh.AuthMethod, error) {
 	var signers []ssh.AuthMethod
 
@@ -61,40 +85,25 @@ func newPublicKey() ([]ssh.AuthMethod, error) {
 	return signers, nil
 }
 
-func main() {
-	flag.Parse()
-	defer glog.Flush()
-
-	ams, err := newPublicKey()
-	if err != nil {
-		glog.Exitf("Error fetching public keys: %v", err)
-	}
-
-	// An SSH client is represented with a ClientConn.
-	//
-	// To authenticate with the remote server you must pass at least one
-	// implementation of ClientAuth via the Auth field in ClientConfig.
-	config := &ssh.ClientConfig{
-		User: *user,
-		Auth: ams,
-		// TODO(wathiede); use FixedHostKey?
-		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-	}
-	client, err := ssh.Dial("tcp", *host, config)
-	if err != nil {
-		glog.Exitf("Error dialing %q: %v", *host, err)
-	}
-
+func updateMetrics(c *ssh.Client) error {
+	now := time.Now()
+	defer func() {
+		delta := time.Since(now)
+		fetchRequestDurationMetric.Set(delta.Seconds())
+		glog.V(2).Infof("Update took %s", delta)
+	}()
 	// Each ClientConn can support multiple interactive sessions,
 	// represented by a Session.
-	s, err := client.NewSession()
+	s, err := c.NewSession()
 	if err != nil {
-		glog.Exitf("Error creating new session: %v", err)
+		return fmt.Errorf("error creating new session: %v", err)
 	}
-	cmd := "/sbin/zfs list -t snapshot -H -o name -s name"
-	b, err := s.CombinedOutput(cmd)
+	defer s.Close()
+
+	glog.V(2).Infof("Running %q", snapshotListCmd)
+	b, err := s.CombinedOutput(snapshotListCmd)
 	if err != nil {
-		glog.Exitf("Error running %q: %v", cmd, err)
+		return fmt.Errorf("error running %q: %v", snapshotListCmd, err)
 	}
 	scanner := bufio.NewScanner(bytes.NewReader(b))
 	snapshotAges := map[string]time.Time{}
@@ -115,9 +124,49 @@ func main() {
 		}
 	}
 	if err := scanner.Err(); err != nil {
-		glog.Exitf("Failed to scan response: %v", err)
+		return fmt.Errorf("failed to scan response: %v", err)
 	}
+
 	for snapshot, snapshotTime := range snapshotAges {
-		fmt.Println(snapshot, snapshotTime, time.Since(snapshotTime))
+		snapshotAgesMetric.WithLabelValues(snapshot).Set(now.Sub(snapshotTime).Seconds())
 	}
+	return nil
+}
+
+func main() {
+	flag.Parse()
+	defer glog.Flush()
+
+	ams, err := newPublicKey()
+	if err != nil {
+		glog.Exitf("Error fetching public keys: %v", err)
+	}
+
+	// An SSH client is represented with a ClientConn.
+	//
+	// To authenticate with the remote server you must pass at least one
+	// implementation of ClientAuth via the Auth field in ClientConfig.
+	config := &ssh.ClientConfig{
+		User: *user,
+		Auth: ams,
+		// TODO(wathiede); use FixedHostKey?
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+	c, err := ssh.Dial("tcp", *host, config)
+	if err != nil {
+		glog.Exitf("Error dialing %q: %v", *host, err)
+	}
+
+	go func() {
+		for {
+			if err := updateMetrics(c); err != nil {
+				glog.Errorf("Failed to update metrics: %v", err)
+			}
+			time.Sleep(*refreshInterval)
+		}
+	}()
+
+	// Expose the registered metrics via HTTP.
+	http.Handle("/metrics", promhttp.Handler())
+	glog.Exitf("Failed to ListenAndServe: %v", http.ListenAndServe(*addr, nil))
 }