Add health check API (#41)

* Uptick go version

* Add test stage

* Add health check for metric provider servers

* Address review comments

Signed-off-by: Abdul Qadeer <aqadeer@paypal.com>
(cherry picked from commit 32392ed)

Author: Abdul Qadeer

.github/workflows/ci.yml

@@ -20,3 +20,6 @@ jobs:
 
     - name: Build
       run: go build -o load-watcher main.go
+
+    - name: Test
+      run: go test ./...

go.mod

@@ -1,6 +1,6 @@
 module github.com/paypal/load-watcher
 
-go 1.15
+go 1.16
 
 require (
 	github.com/francoispqt/gojay v1.2.13

pkg/watcher/internal/metricsprovider/k8s.go

@@ -18,6 +18,8 @@ package metricsprovider
 
 import (
 	"context"
+	"fmt"
+	"net/http"
 	"os"
 
 	"github.com/paypal/load-watcher/pkg/watcher"
@@ -153,3 +155,12 @@ func (m metricsServerClient) FetchAllHostsMetrics(window *watcher.Window) (map[s
 
 	return metrics, nil
 }
+
+func (m metricsServerClient) Health() (int, error) {
+	var status int
+	m.metricsClientSet.RESTClient().Verb("HEAD").Do(context.Background()).StatusCode(&status)
+	if status != http.StatusOK {
+		return -1, fmt.Errorf("received response status code: %v", status)
+	}
+	return 0, nil
+}

pkg/watcher/internal/metricsprovider/prometheus.go

@@ -19,6 +19,7 @@ package metricsprovider
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"time"
 
 	"github.com/paypal/load-watcher/pkg/watcher"
@@ -133,6 +134,21 @@ func (s promClient) FetchAllHostsMetrics(window *watcher.Window) (map[string][]w
 	return hostMetrics, anyerr
 }
 
+func (s promClient) Health() (int, error) {
+	req, err := http.NewRequest("HEAD", DefaultPromAddress, nil)
+	if err != nil {
+		return -1, err
+	}
+	resp, _, err := s.client.Do(context.Background(), req)
+	if err != nil {
+		return -1, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return -1, fmt.Errorf("received response status code: %v", resp.StatusCode)
+	}
+	return 0, nil
+}
+
 func (s promClient) buildPromQuery(host string, metric string, method string, rollup string) string {
 	var promQuery string
 

pkg/watcher/internal/metricsprovider/signalfx.go

@@ -191,6 +191,27 @@ func (s signalFxClient) FetchAllHostsMetrics(window *watcher.Window) (map[string
 	return metrics, nil
 }
 
+func (s signalFxClient) Health() (int, error) {
+	return Ping(s.client, s.signalFxAddress)
+}
+
+// Simple ping utility to a given URL
+// Returns -1 if unhealthy, 0 if healthy along with error if any
+func Ping(client http.Client, url string) (int, error) {
+	req, err := http.NewRequest("HEAD", url, nil)
+	if err != nil {
+		return -1, err
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return -1, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return -1, fmt.Errorf("received response code: %v", resp.StatusCode)
+	}
+	return 0, nil
+}
+
 func addMetadata(metric *watcher.Metric, metricType string) {
 	metric.Operator = watcher.Average
 	if metricType == cpuUtilizationMetric {

pkg/watcher/metricsprovider.go

@@ -50,11 +50,14 @@ type MetricsProviderClient interface {
 	FetchHostMetrics(host string, window *Window) ([]Metric, error)
 	// Fetch metrics for all hosts
 	FetchAllHostsMetrics(window *Window) (map[string][]Metric, error)
+	// Get metric provider server health status
+	// Returns 0 if healthy, -1 if unhealthy along with error if any
+	Health() (int, error)
 }
 
 // Generic metrics provider options
 type MetricsProviderOpts struct {
 	Name      string
 	Address   string
 	AuthToken string
-}
+}

pkg/watcher/testserver.go

@@ -115,3 +115,7 @@ func (t testServerClient) FetchAllHostsMetrics(window *Window) (map[string][]Met
 
 	return FifteenMinutesMetricsMap, nil
 }
+
+func (t testServerClient) Health() (int, error) {
+	return 0, nil
+}

pkg/watcher/watcher.go

@@ -37,6 +37,7 @@ import (
 
 const (
 	BaseUrl         = "/watcher"
+	HealthCheckUrl  = "/watcher/health"
 	FifteenMinutes  = "15m"
 	TenMinutes      = "10m"
 	FiveMinutes     = "5m"
@@ -153,6 +154,7 @@ func (w *Watcher) StartWatching() {
 	}
 
 	http.HandleFunc(BaseUrl, w.handler)
+	http.HandleFunc(HealthCheckUrl, w.healthCheckHandler)
 	server := &http.Server{
 		Addr:    ":2020",
 		Handler: http.DefaultServeMux,
@@ -176,6 +178,7 @@ func (w *Watcher) StartWatching() {
 	w.mutex.Lock()
 	w.isStarted = true
 	w.mutex.Unlock()
+	log.Info("Started watching metrics")
 }
 
 // StartWatching() should be called before calling this.
@@ -297,6 +300,16 @@ func (w *Watcher) handler(resp http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// Simple server status handler
+func (w *Watcher) healthCheckHandler(resp http.ResponseWriter, r *http.Request) {
+	if status, err := w.client.Health(); status != 0 {
+		log.Warnf("health check failed with: %v", err)
+		resp.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	resp.WriteHeader(http.StatusOK)
+}
+
 // Utility functions
 
 func metricMapToWatcherMetrics(metricMap map[string][]Metric, clientName string, window Window) WatcherMetrics {

pkg/watcher/watcher_test.go

@@ -120,6 +120,17 @@ func TestWatcherInternalServerError(t *testing.T) {
 	assert.Equal(t, http.StatusInternalServerError, rr.Code)
 }
 
+func TestWatcherHealthCheck(t *testing.T) {
+	req, err := http.NewRequest("GET", HealthCheckUrl, nil)
+	require.Nil(t, err)
+
+	rr := httptest.NewRecorder()
+	handler := http.HandlerFunc(w.handler)
+
+	handler.ServeHTTP(rr, req)
+	require.Equal(t, http.StatusOK, rr.Code)
+}
+
 func TestMain(m *testing.M) {
 	client := NewTestMetricsServerClient()
 	w = NewWatcher(client)