Merge pull request #5 from klouddb/unused_hba_lines

added hba unused line handling in logparser

Author: klouddb

.github/workflows/integration.yml

@@ -70,4 +70,4 @@ jobs:
           integrationtest setup --prefix "${{ matrix.log_prefix }}"
           sudo chmod -R 777 pglog/log0
           integrationtest test -p "${{ matrix.log_prefix }}" -f "pglog/log0/postgresql*.log"
-          docker-compose down -v
+          docker compose down -v

cmd/ciscollector/main.go

@@ -22,6 +22,7 @@ import (
 	"github.com/klouddb/klouddbshield/passwordmanager"
 	"github.com/klouddb/klouddbshield/pkg/config"
 	cons "github.com/klouddb/klouddbshield/pkg/const"
+	"github.com/klouddb/klouddbshield/pkg/hbarules"
 	"github.com/klouddb/klouddbshield/pkg/logger"
 	"github.com/klouddb/klouddbshield/pkg/mysqldb"
 	"github.com/klouddb/klouddbshield/pkg/parselog"
@@ -110,6 +111,8 @@ func main() {
 			runInactiveUSersLogParser(ctx, cnf, store)
 		// case cons.LogParserCMD_MismatchIPs:
 		// 	runMismatchIPsLogParser(ctx, cnf)
+		case cons.LogParserCMD_HBAUnusedLines:
+			runHBAUnusedLinesLogParser(ctx, cnf, store)
 		default:
 			fmt.Println("Invalid command for log parser")
 			os.Exit(1)
@@ -146,6 +149,62 @@ func updatePgSettings(ctx context.Context, store *sql.DB, pgSettings *model.PgSe
 	pgSettings.LogConnections = ps.LogConnections
 }
 
+func runHBAUnusedLinesLogParser(ctx context.Context, cnf *config.Config, store *sql.DB) {
+
+	// check if postgres setting contains required variable or connection logs
+	if !strings.Contains(cnf.LogParser.PgSettings.LogLinePrefix, "%h") && !strings.Contains(cnf.LogParser.PgSettings.LogLinePrefix, "%r") {
+		fmt.Println("Please set log_line_prefix to '%h' or '%r' or enable log_connections")
+		return
+	}
+
+	if !strings.Contains(cnf.LogParser.PgSettings.LogLinePrefix, "%u") || !strings.Contains(cnf.LogParser.PgSettings.LogLinePrefix, "%d") {
+		fmt.Printf("In logline prefix, please set '%s' and '%s'\n", "%u", "%d") // using printf to avoid the warning for %d in println
+		return
+	}
+
+	baseParser := parselog.GetDynamicBaseParser(cnf.LogParser.PgSettings.LogLinePrefix)
+
+	var hbaRules []model.HBAFIleRules
+
+	// if user is passing hba conf file manually then he or she are expecting that file to be scanned
+	if cnf.LogParser.HbaConfFile != "" {
+		var err error
+		hbaRules, err = hbarules.ScanHBAFile(ctx, store, cnf.LogParser.HbaConfFile)
+		if err != nil {
+			fmt.Println("Got error while scanning hba file:", err)
+			return
+		}
+	} else if store != nil {
+		var err error
+		hbaRules, err = utils.GetDatabaseAndHostForUSerFromHbaFileRules(ctx, store)
+		if err != nil {
+			fmt.Println("Got error while getting hba rules:", err)
+			return
+		}
+	} else {
+		fmt.Println("Please provide hba file or database connection")
+		return
+	}
+
+	hbaValidator, err := hbarules.ParseHBAFileRules(hbaRules)
+	if err != nil {
+		fmt.Println("Got error while parsing hba rules:", err)
+		return
+	}
+
+	hbaUnusedLineParser := parselog.NewHbaUnusedLines(cnf, baseParser, hbaValidator)
+	runner.RunFastParser(ctx, cnf, hbaUnusedLineParser.Feed, parselog.GetBaseParserValidator(baseParser))
+
+	if ctx.Err() != nil {
+		fmt.Println("file parsing is taking longer then expected, please check the file or errors in" + logger.GetLogFileName())
+		return
+	}
+
+	fmt.Println("")
+	fmt.Println("Unused lines found from given log file:", hbaValidator.GetUnusedLines())
+	fmt.Println("")
+}
+
 func runMismatchIPsLogParser(ctx context.Context, cnf *config.Config) {
 
 	// check if postgres setting contains required variable or connection logs

docker_testing/database

@@ -0,0 +1,2 @@
+newdbs
+teestingdbs

docker_testing/integrationtest/setup.go

@@ -53,7 +53,7 @@ func init() {
 }
 
 func startPostgres() {
-	cmd := exec.Command("docker-compose", "up", "--build", "-d", "postgres")
+	cmd := exec.Command("docker", "compose", "up", "--build", "-d", "postgres")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	cmd.Stdin = os.Stdin
@@ -72,7 +72,7 @@ func startPostgres() {
 }
 
 func createUSers() {
-	cmd := exec.Command("docker-compose", "run", "--rm", "createuser")
+	cmd := exec.Command("docker", "compose", "run", "--rm", "createuser")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	cmd.Stdin = os.Stdin
@@ -93,7 +93,7 @@ func createUSers() {
 func execPgbench(wg *sync.WaitGroup, pgUsers, ip string) {
 	defer wg.Done()
 	fmt.Println("executing pgbench command for users:", pgUsers, "and ip:", ip)
-	cmd := exec.Command("docker-compose", "run", "--rm", "pgbench")
+	cmd := exec.Command("docker", "compose", "run", "--rm", "pgbench")
 	cmd.Env = append(os.Environ(),
 		"PGUSERS="+pgUsers,
 		"IP="+ip,

docker_testing/integrationtest/testcase.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"os/exec"
@@ -20,6 +21,7 @@ func init() {
 			testInactiveUser(prefix, filename)
 			// testMissingIPs(prefix, filename)
 			testUniqueIPs(prefix, filename)
+			testUnusedHbaLines(prefix, filename)
 		},
 	}
 
@@ -152,3 +154,46 @@ func testMissingIPs(prefix, file string) {
 
 	fmt.Println("mismatch ip test is working fine for prefix:", prefix)
 }
+
+func testUnusedHbaLines(prefix, file string) {
+	cmd := exec.Command("ciscollector",
+		"-logparser", cons.LogParserCMD_HBAUnusedLines,
+		"-prefix", prefix,
+		"-file-path", file,
+		"-output-type", "json",
+		"-hba-file", "./pg_hba.conf",
+	)
+
+	// create io.Writer to store output and print it later
+	var buf bytes.Buffer
+
+	cmd.Stdout = &buf
+	cmd.Stderr = os.Stderr
+	cmd.Stdin = os.Stdin
+
+	err := cmd.Run()
+	if err != nil {
+		fmt.Println("Got error while parsing file:", err)
+		os.Exit(1)
+	}
+
+	out := buf.String()
+	if strings.Contains(out, "In logline prefix, please set '%u' and '%d'") || strings.Contains(out, "Please set log_line_prefix to '%h' or '%r' or enable log_connections") {
+		fmt.Println("skipping test for unused files as required details are not available in prefix:", prefix)
+		return
+	}
+
+	if !strings.Contains(out, "Successfully parsed all files") {
+		fmt.Println("Got error while parsing file:", out)
+		// fail the command
+		os.Exit(1)
+	}
+
+	if strings.Contains(out, `Unused lines found from given log file: [11 23 28]`) {
+		fmt.Println("unused lines test is working fine for prefix:", prefix)
+		return
+	}
+
+	fmt.Println("not getting valid unused lines:", out)
+	os.Exit(1)
+}

docker_testing/pg_hba.conf

@@ -0,0 +1,28 @@
+# Allow any user on the local system to connect to any database with
+# any database user name using Unix-domain sockets (the default for local
+# connections).
+#
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+local   all             all                                     trust
+
+# The same using local loopback TCP/IP connections.
+#
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+host    all             all             192.168.1.1/24            trust
+
+# The same as the previous line, but using a separate netmask column
+#
+# TYPE  DATABASE        USER            IP-ADDRESS      IP-MASK             METHOD
+host    all             user4           192.168.0.1       255.255.255.0     trust
+host    all             all             192.168.0.1       255.255.255.0     trust
+# host    all             all             all                    trust
+
+# The same over IPv6.
+#
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+host    all             all             ::1/128                 trust
+
+# The same using a host name (would typically cover both IPv4 and IPv6).
+#
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+host    all             all             localhost               trust

docker_testing/runner.sh

@@ -7,7 +7,7 @@
 ############################ widely used prefixes #################################
 # PREFIXES=('%t %h %u %m ' '%m (%h:%u) ' '%m from %h by %u ' '%m in %d by %u@%h ' '%t %h %u [%p] ' '%m (%h:%u:%p) ' '%m from %h by %u pid=%p ' '%m in %d by %u@%h pid=%p ' '%t %h %u db=%d %m ' '%m in %d by %u@%h db=%d')
 ####################################################################################
-PREFIXES=('%t [%p]: ')
+PREFIXES=('%m in %d by %u@%h ')
 
 FILE_SIZE=12MB # to modify file size
 
@@ -42,4 +42,4 @@ done
 
 
 # Set permissions for pglog directory
-sudo chmod 777 pglog/*
+sudo chmod 777 pglog/*

docker_testing/users

@@ -0,0 +1,5 @@
+user0
+user1
+user2
+user3
+user4

go.mod

@@ -3,6 +3,7 @@ module github.com/klouddb/klouddbshield
 go 1.18
 
 require (
+	github.com/DATA-DOG/go-sqlmock v1.5.0
 	github.com/go-sql-driver/mysql v1.7.0
 	github.com/hashicorp/go-version v1.6.0
 	github.com/jackc/pgx/v4 v4.18.3

go.sum

@@ -38,6 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
+github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=