[WIP] Add a command to run tests on queries based on a jsonnet test file

common.libsonnet

@@ -0,0 +1,33 @@
+local formatLabels = function(labels)
+  local lf = std.join(', ', std.map(function(l) '%s="%s"' % [l, labels[l]], std.objectFields(labels)));
+  '{%s}' % [lf];
+
+// returns a series object with correctly formatted labels.
+// labels can be modified post creation using `_labels`.
+local series = function(name, labels, values) {
+  _name:: name,
+  _labels:: labels,
+  series: self._name + formatLabels(self._labels),
+  values: values,
+};
+
+// returns a test object with the given series and samples. Sample interval is 30s
+// the evaluation time is set one hour in the future since all our queries operate on a 1h window
+local test = function(name, series, query, samples, interval='30s', eval_time='1h') {
+  name: name,
+  interval: interval,
+  input_series: if std.isArray(series) then series else std.objectValues(series),
+  promql_expr_test: [
+    {
+      expr: query,
+      eval_time: eval_time,
+      exp_samples: if std.isArray(samples) then samples else [samples],
+    },
+  ],
+};
+
+{
+  series: series,
+  formatLabels: formatLabels,
+  test: test,
+}

main.go

@@ -66,6 +66,7 @@ func newApp() (context.Context, context.CancelFunc, *cli.App) {
 		},
 		Commands: []*cli.Command{
 			newReportCommand(),
+			newQueryTestCommand(),
 		},
 		ExitErrHandler: func(context *cli.Context, err error) {
 			if err == nil {

pkg/querycheck/querycheck.go

@@ -0,0 +1,52 @@
+package querycheck
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"github.com/google/go-jsonnet"
+
+	"github.com/appuio/appuio-reporting/pkg/testsuite"
+)
+
+func RunTestQueries(filepath string, extcodes *map[string]string) error {
+	tmp, err := renderJsonnet(filepath, extcodes)
+	if err != nil {
+		return err
+	}
+	return runPromtool(tmp)
+}
+
+func runPromtool(tmp string) error {
+	cmd := exec.Command(testsuite.PromtoolBin, "test", "rules", tmp)
+	var stderr, stdout strings.Builder
+	cmd.Stderr = &stderr
+	cmd.Stdout = &stdout
+	err := cmd.Run()
+	// Not using t.Log to keep formatting sane
+	fmt.Println("STDOUT")
+	fmt.Println(stdout.String())
+	fmt.Println("STDERR")
+	fmt.Println(stderr.String())
+	return err
+}
+
+func renderJsonnet(tFile string, extcodes *map[string]string) (string, error) {
+	vm := jsonnet.MakeVM()
+
+	for key := range(*extcodes) {
+		vm.ExtCode(key, (*extcodes)[key])
+	}
+
+	ev, err := vm.EvaluateFile(tFile)
+	if err != nil {
+		return "", err
+	}
+
+	tmp := path.Join("/tmp", "test.json")
+	err = os.WriteFile(tmp, []byte(ev), 0644)
+	return tmp, err
+}

query_test_command.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/appuio/appuio-reporting/pkg/querycheck"
+	"github.com/urfave/cli/v2"
+	"gopkg.in/yaml.v3"
+)
+
+type queryTestCommand struct {
+	testFilePath string
+	additionalYamlFiles cli.StringSlice
+}
+
+var queryTestCommandName = "test"
+
+func newQueryTestCommand() *cli.Command {
+	command := &queryTestCommand{}
+	return &cli.Command{
+		Name:   queryTestCommandName,
+		Usage:  "Run Prometheus tests on a set of query test cases",
+		Before: command.before,
+		Action: command.execute,
+		Flags: []cli.Flag{
+			&cli.StringFlag{Name: "test-file", Usage: "Path of the jsonnet test file from which to test queries",
+				EnvVars: envVars("TEST_FILE"), Destination: &command.testFilePath, Value: "./test.jsonnet"},
+			&cli.StringSliceFlag{Name: "add-yaml-file", Usage: "Additional yaml files to include into the test (available to jsonnet as extVar)",
+				EnvVars: envVars("ADD_YAML_FILE"), Destination: &command.additionalYamlFiles},
+		},
+	}
+}
+
+func (cmd *queryTestCommand) before(context *cli.Context) error {
+	fmt.Println("begin!")
+	return nil
+}
+
+func (cmd *queryTestCommand) execute(cliCtx *cli.Context) error {
+	ctx := cliCtx.Context
+	log := AppLogger(ctx).WithName(queryTestCommandName)
+
+	extVars, err := cmd.buildExtVars()
+	if err != nil {
+		log.Error(err, "Query test setup failed")
+	}
+	err = querycheck.RunTestQueries(cmd.testFilePath, extVars)
+
+	if err != nil {
+		log.Error(err, "Query test failed")
+	}
+	log.Info("Done")
+	return err 
+}
+
+func (cmd *queryTestCommand) buildExtVars() (*map[string]string, error) {
+	extVars := make(map[string]string)
+	for file := range(cmd.additionalYamlFiles.Value()) {
+		path := cmd.additionalYamlFiles.Value()[file]
+		name := filepath.Base(path)
+		fileHandle, err := os.Open(path)
+		if err != nil {
+			return nil, err
+		}
+		defer fileHandle.Close()
+		fileContent, err := io.ReadAll(fileHandle)
+		if err != nil {
+			return nil, err
+		}
+
+		var parsed map[string]interface{}
+		err = yaml.Unmarshal(fileContent, &parsed)
+		if err != nil {
+			return nil, err
+		}
+		jsonStr, err := json.Marshal(parsed)
+		if err != nil {
+			return nil, err
+		}
+		extVars[name] = string(jsonStr)
+	}
+	return &extVars, nil
+}