[5.17 backport] kms: support key rotation for vault

.github/workflows/run_hac_test.yml

@@ -13,7 +13,13 @@ jobs:
         uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version-file: go.mod
+          check-latest: true
+          cache: true
+          cache-dependency-path: |
+            **/go.sum
+      - name: Set up kind
+        uses: helm/kind-action@v1
 
       - name: Set environment variables
         run: |

.github/workflows/run_kms_azure_vault_test.yml

@@ -13,7 +13,13 @@ jobs:
         uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version-file: go.mod
+          check-latest: true
+          cache: true
+          cache-dependency-path: |
+            **/go.sum
+      - name: Set up kind
+        uses: helm/kind-action@v1
 
       - name: Set environment variables
         run: |

.github/workflows/run_kms_dev_test.yml

@@ -13,7 +13,13 @@ jobs:
         uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version-file: go.mod
+          check-latest: true
+          cache: true
+          cache-dependency-path: |
+            **/go.sum
+      - name: Set up kind
+        uses: helm/kind-action@v1
 
       - name: Set environment variables
         run: |

.github/workflows/run_kms_ibm_kp_test.yml

@@ -16,7 +16,13 @@ jobs:
         uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version-file: go.mod
+          check-latest: true
+          cache: true
+          cache-dependency-path: |
+            **/go.sum
+      - name: Set up kind
+        uses: helm/kind-action@v1
 
       - name: Set environment variables
         run: |

.github/workflows/run_kms_kmip_test.yml

@@ -13,22 +13,28 @@ jobs:
         uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version-file: go.mod
+          check-latest: true
+          cache: true
+          cache-dependency-path: |
+            **/go.sum
+      - name: Set up kind
+        uses: helm/kind-action@v1
 
       - name: Set environment variables
         run: |
           echo PATH=$PATH:$HOME/go/bin                                          >> $GITHUB_ENV
           echo OPERATOR_IMAGE=localhost:5000/noobaa/noobaa-operator:integration >> $GITHUB_ENV
           echo PYKMIP_IMAGE=localhost:5000/noobaa/pykmip:integration >> $GITHUB_ENV
-      
+
       - name: Deploy Dependencies
         run: |
           set -x
           bash .travis/install-5nodes-kind-cluster.sh
           go get -v github.com/onsi/ginkgo/ginkgo
           go install -v github.com/onsi/ginkgo/ginkgo
           ginkgo version
-          
+
       - name: Build NooBaa
         run: |
           make cli

.github/workflows/run_kms_rotate_test.yml

@@ -13,7 +13,13 @@ jobs:
         uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version-file: go.mod
+          check-latest: true
+          cache: true
+          cache-dependency-path: |
+            **/go.sum
+      - name: Set up kind
+        uses: helm/kind-action@v1
 
       - name: Set environment variables
         run: |

.github/workflows/run_kms_tls_sa_test.yml

@@ -13,8 +13,13 @@ jobs:
         uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
-
+          go-version-file: go.mod
+          check-latest: true
+          cache: true
+          cache-dependency-path: |
+            **/go.sum
+      - name: Set up kind
+        uses: helm/kind-action@v1
       - name: Set environment variables
         run: |
           echo PATH=$PATH:$HOME/go/bin                                          >> $GITHUB_ENV

.github/workflows/run_kms_tls_token_test.yml

@@ -13,7 +13,13 @@ jobs:
         uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version-file: go.mod
+          check-latest: true
+          cache: true
+          cache-dependency-path: |
+            **/go.sum
+      - name: Set up kind
+        uses: helm/kind-action@v1
 
       - name: Set environment variables
         run: |

pkg/system/phase2_creating.go

@@ -1052,6 +1052,13 @@ func (r *Reconciler) keyRotate() error {
 		return err
 	}
 
+	err = k.Get()
+	if err != nil {
+		r.Logger.Errorf("keyRotate, KMS Get error %v", err)
+		r.setKMSConditionStatus(nbv1.ConditionKMSErrorRead)
+		return err
+	}
+
 	// Generate new random root key and set it in the KMS
 	// Key - rotate begins
 	err = k.Set(util.RandomBase64(32))

pkg/util/kms/kms_azure.go

@@ -21,6 +21,8 @@ const (
 // AzureVault is a azure kms driver
 type AzureVault struct {
 	UID string // NooBaa system UID
+	name string // NooBaa system name
+	ns   string // NooBaa system namespace
 }
 
 // NewAzureVault is azure driver constructor
@@ -29,7 +31,7 @@ func NewAzureVault(
 	namespace string,
 	uid string,
 ) Driver {
-	return &AzureVault{uid}
+	return &AzureVault{uid, name, namespace}
 }
 
 //
@@ -107,8 +109,8 @@ func createCertTempFile(config map[string]interface{}, namespace string) error {
 
 // Version returns the current driver KMS version
 // either single string or map, i.e. rotating key
-func (*AzureVault) Version(kms *KMS) Version {
-	return &VersionSingleSecret{kms, nil}
+func (k *AzureVault) Version(kms *KMS) Version {
+	return &VersionRotatingSecret{VersionBase{kms, nil}, k.name, k.ns}
 }
 
 // Register Azure driver with KMS layer

pkg/util/kms/kms_vault.go

@@ -24,7 +24,9 @@ const (
 
 // Vault is a vault driver
 type Vault struct {
-	UID string // NooBaa system UID
+	UID  string // NooBaa system UID
+	name string // NooBaa system name
+	ns   string // NooBaa system namespace
 }
 
 // NewVault is vault driver constructor
@@ -33,7 +35,7 @@ func NewVault(
 	namespace string,
 	uid string,
 ) Driver {
-	return &Vault{uid}
+	return &Vault{uid, name, namespace}
 }
 
 //
@@ -179,8 +181,8 @@ func writeCrtsToFile(secretName string, namespace string, secretValue []byte, en
 
 // Version returns the current driver KMS version
 // either single string or map, i.e. rotating key
-func (*Vault) Version(kms *KMS) Version {
-	return &VersionSingleSecret{kms, nil}
+func (k *Vault) Version(kms *KMS) Version {
+	return &VersionRotatingSecret{VersionBase{kms, nil}, k.name, k.ns}
 }
 
 // Register Vault driver with KMS layer

pkg/util/kms/kms_version.go

@@ -102,7 +102,7 @@ func (v *VersionRotatingSecret) Reconcile(r SecretReconciler) error {
 
 // Get implements SecretStorage interface for the secret map, i.e. rotating master root key
 func (v *VersionRotatingSecret) Get() error {
-	s, _, err := v.k.GetSecret(v.backendSecretName(), v.k.driver.GetContext())
+	s, _, err := v.k.GetSecret(v.BackendSecretName(), v.k.driver.GetContext())
 	if err != nil {
 		// handle k8s get from non-existent secret
 		if strings.Contains(err.Error(), "not found") || strings.Contains(err.Error(), "does not exist") {
@@ -119,8 +119,8 @@ func (v *VersionRotatingSecret) Get() error {
 	return nil
 }
 
-// backendSecretName returns the rotating secret backend secret name
-func (v *VersionRotatingSecret) backendSecretName() string {
+// BackendSecretName returns the rotating secret backend secret name
+func (v *VersionRotatingSecret) BackendSecretName() string {
 	return v.name + "-root-master-key-backend"
 }
 
@@ -136,7 +136,7 @@ func (v *VersionRotatingSecret) Set(val string) error {
 	s[ActiveRootKey] = key
 	s[key] = val
 	v.data = s
-	_, err := v.k.PutSecret(v.backendSecretName(), toInterfaceMap(s), v.k.driver.SetContext())
+	_, err := v.k.PutSecret(v.BackendSecretName(), toInterfaceMap(s), v.k.driver.SetContext())
 	return err
 }
 
@@ -153,11 +153,15 @@ func (v *VersionRotatingSecret) deleteSingleStringSecret() bool {
 func (v *VersionRotatingSecret) Delete() error {
 	// Delete rotating secret backend
 	backendSecret := &corev1.Secret{}
-	backendSecret.Name = v.backendSecretName()
+	backendSecret.Name = v.BackendSecretName()
 	backendSecret.Namespace = v.ns
 	if !util.KubeDelete(backendSecret) {
 		return fmt.Errorf("KMS Delete error for the rotating master root secret backend")
+	}
 
+	err := v.k.DeleteSecret(v.BackendSecretName(), v.k.driver.DeleteContext())
+	if err != nil {
+		return err
 	}
 
 	return nil

pkg/util/kms/test/azure-vault/kms_azure_vault_test.go

@@ -3,6 +3,7 @@ package kmsazurevaulttest
 import (
 	"os"
 
+	"github.com/libopenstorage/secrets"
 	"github.com/libopenstorage/secrets/azure"
 	nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1"
 	"github.com/noobaa/noobaa-operator/v5/pkg/options"
@@ -49,17 +50,6 @@ var _ = Describe("KMS - Azure Vault", func() {
 		Specify("Create KMS Noobaa", func() {
 			Expect(util.KubeCreateFailExisting(noobaa)).To(BeTrue())
 		})
-		// TODO: As of now azure key vault is a cloud service and to test
-		// this case, an account needs to be created at azure side.
-		// Create Azure key vault and provide the parameters
-		// Below condition always be corev1.ConditionStatus = "Invalid"
-		// utill we provide the actual azure key vault credentials
-		// Change Expect(util.NooBaaCondStatus(noobaa, nbv1.ConditionKMSInit)).To(BeFalse())
-		// to Expect(util.NooBaaCondStatus(noobaa, nbv1.ConditionKMSInit)).To(BeTrue())
-		// once we have azure valut in place
-		Specify("Verify KMS condition status Init", func() {
-			Expect(util.NooBaaCondStatus(noobaa, nbv1.ConditionKMSInit)).To(BeFalse())
-		})
 		Specify("Restart NooBaa operator", func() {
 			podList := &corev1.PodList{}
 			podSelector, _ := labels.Parse("noobaa-operator=deployment")
@@ -105,4 +95,48 @@ var _ = Describe("KMS - Azure Vault", func() {
 		})
 	})
 
+	Context("Verify Rotate", func() {
+		noobaa := getMiniNooBaa()
+		azureVaultURL, azureVaultURLFound := os.LookupEnv("AZURE_VAULT_URL")
+		k := azureKMSSpec(azureVaultURL)
+		noobaa.Spec.Security.KeyManagementService = k
+		noobaa.Spec.Security.KeyManagementService.EnableKeyRotation = true
+		noobaa.Spec.Security.KeyManagementService.Schedule = "* * * * *" // every min
+
+		Specify("Verify API Address", func() {
+			Expect(azureVaultURLFound).To(BeTrue())
+		})
+		Specify("Create key rotate schedule system", func() {
+			Expect(util.KubeCreateFailExisting(noobaa)).To(BeTrue())
+		})
+		// Change here to .To(BeTrue()) once fixed issue in line 53
+		Specify("Verify KMS condition Type", func() {
+			Expect(util.NooBaaCondition(noobaa, nbv1.ConditionTypeKMSType, secrets.TypeAzure)).To(BeFalse())
+		})
+		// Change here to .To(BeTrue()) once fixed issue in line 53
+		Specify("Verify KMS condition status Init", func() {
+			Expect(util.NooBaaCondStatus(noobaa, nbv1.ConditionKMSInit)).To(BeFalse())
+		})
+		Specify("Restart NooBaa operator", func() {
+			podList := &corev1.PodList{}
+			podSelector, _ := labels.Parse("noobaa-operator=deployment")
+			listOptions := client.ListOptions{Namespace: options.Namespace, LabelSelector: podSelector}
+
+			Expect(util.KubeList(podList, &listOptions)).To(BeTrue())
+			Expect(len(podList.Items)).To(BeEquivalentTo(1))
+			Expect(util.KubeDelete(&podList.Items[0])).To(BeTrue())
+		})
+		// Change here to .To(BeTrue()) once fixed issue in line 53
+		Specify("Verify KMS condition status Sync", func() {
+			Expect(util.NooBaaCondStatus(noobaa, nbv1.ConditionKMSSync)).To(BeFalse())
+		})
+		// Change here to .To(BeTrue()) once fixed issue in line 53
+		Specify("Verify KMS condition status Key Rotate", func() {
+			Expect(util.NooBaaCondStatus(noobaa, nbv1.ConditionKMSKeyRotate)).To(BeFalse())
+		})
+		Specify("Delete NooBaa", func() {
+			Expect(util.KubeDelete(noobaa)).To(BeTrue())
+		})
+	})
+
 })