diff --git a/apis/v1/secretproviderclasspodstatus_types.go b/apis/v1/secretproviderclasspodstatus_types.go index 3616d5077..8d7f6b009 100644 --- a/apis/v1/secretproviderclasspodstatus_types.go +++ b/apis/v1/secretproviderclasspodstatus_types.go @@ -32,6 +32,7 @@ type SecretProviderClassPodStatusStatus struct { Mounted bool `json:"mounted,omitempty"` TargetPath string `json:"targetPath,omitempty"` Objects []SecretProviderClassObject `json:"objects,omitempty"` + FSGroup string `json:"fsGroup,omitempty"` } // SecretProviderClassObject defines the object fetched from external secrets store diff --git a/config/crd/bases/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml b/config/crd/bases/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml index 5ae5c2531..8c9021297 100644 --- a/config/crd/bases/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml +++ b/config/crd/bases/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml @@ -41,6 +41,8 @@ spec: description: SecretProviderClassPodStatusStatus defines the observed state of SecretProviderClassPodStatus properties: + fsGroup: + type: string mounted: type: boolean objects: diff --git a/manifest_staging/charts/secrets-store-csi-driver/crds/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml b/manifest_staging/charts/secrets-store-csi-driver/crds/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml index 5ae5c2531..8c9021297 100644 --- a/manifest_staging/charts/secrets-store-csi-driver/crds/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml +++ b/manifest_staging/charts/secrets-store-csi-driver/crds/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml @@ -41,6 +41,8 @@ spec: description: SecretProviderClassPodStatusStatus defines the observed state of SecretProviderClassPodStatus properties: + fsGroup: + type: string mounted: type: boolean objects: diff --git a/manifest_staging/deploy/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml b/manifest_staging/deploy/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml index 5ae5c2531..8c9021297 100644 --- a/manifest_staging/deploy/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml +++ b/manifest_staging/deploy/secrets-store.csi.x-k8s.io_secretproviderclasspodstatuses.yaml @@ -41,6 +41,8 @@ spec: description: SecretProviderClassPodStatusStatus defines the observed state of SecretProviderClassPodStatus properties: + fsGroup: + type: string mounted: type: boolean objects: diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go new file mode 100644 index 000000000..b2d4e9aac --- /dev/null +++ b/pkg/constants/constants.go @@ -0,0 +1,20 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package constants + +const ( + NoGID = int64(-1) // Use the default gid -1 to indicate no change in FSGroup +) diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index dc3a56ef4..3506dbd6d 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -44,5 +44,6 @@ const ( // PodVolumeNotFound error PodVolumeNotFound = "PodVolumeNotFound" // FileWriteError error - FileWriteError = "FileWriteError" + FileWriteError = "FileWriteError" + FailedToParseFSGroup = "FailedToParseFSGroup" ) diff --git a/pkg/rotation/reconciler.go b/pkg/rotation/reconciler.go index e82d12f79..1942f0d24 100644 --- a/pkg/rotation/reconciler.go +++ b/pkg/rotation/reconciler.go @@ -21,12 +21,14 @@ import ( "encoding/json" "fmt" "os" + "strconv" "strings" "time" secretsstorev1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" "sigs.k8s.io/secrets-store-csi-driver/controllers" secretsStoreClient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" + "sigs.k8s.io/secrets-store-csi-driver/pkg/constants" internalerrors "sigs.k8s.io/secrets-store-csi-driver/pkg/errors" "sigs.k8s.io/secrets-store-csi-driver/pkg/k8s" secretsstore "sigs.k8s.io/secrets-store-csi-driver/pkg/secrets-store" @@ -398,7 +400,18 @@ func (r *Reconciler) reconcile(ctx context.Context, spcps *secretsstorev1.Secret r.generateEvent(pod, corev1.EventTypeWarning, mountRotationFailedReason, fmt.Sprintf("failed to lookup provider client: %q", providerName)) return fmt.Errorf("failed to lookup provider client: %q", providerName) } - newObjectVersions, errorReason, err := secretsstore.MountContent(ctx, providerClient, string(paramsJSON), string(secretsJSON), spcps.Status.TargetPath, string(permissionJSON), oldObjectVersions) + gid := constants.NoGID + if len(spcps.Status.FSGroup) > 0 { + gid, err = strconv.ParseInt(spcps.Status.FSGroup, 10, 64) + if err != nil { + errorReason = internalerrors.FailedToParseFSGroup + errStr := fmt.Sprintf("failed to rotate objects for pod %s/%s, invalid FSGroup:%s", spcps.Namespace, spcps.Status.PodName, spcps.Status.FSGroup) + r.generateEvent(pod, corev1.EventTypeWarning, mountRotationFailedReason, fmt.Sprintf("%s, err: %v", errStr, err)) + return fmt.Errorf("%s, err: %w", errStr, err) + } + } + klog.V(5).InfoS("updating the secret content", "pod", klog.ObjectRef{Namespace: spcps.Namespace, Name: spcps.Status.PodName}, "FSGroup", gid) + newObjectVersions, errorReason, err := secretsstore.MountContent(ctx, providerClient, string(paramsJSON), string(secretsJSON), spcps.Status.TargetPath, string(permissionJSON), oldObjectVersions, gid) if err != nil { r.generateEvent(pod, corev1.EventTypeWarning, mountRotationFailedReason, fmt.Sprintf("provider mount err: %+v", err)) return fmt.Errorf("failed to rotate objects for pod %s/%s, err: %w", spcps.Namespace, spcps.Status.PodName, err) diff --git a/pkg/rotation/reconciler_test.go b/pkg/rotation/reconciler_test.go index db79450b7..565ccf642 100644 --- a/pkg/rotation/reconciler_test.go +++ b/pkg/rotation/reconciler_test.go @@ -87,6 +87,93 @@ func newTestReconciler(client client.Reader, kubeClient kubernetes.Interface, cr }, nil } +func getSPC(customize func(*secretsstorev1.SecretProviderClass)) *secretsstorev1.SecretProviderClass { + var spc = &secretsstorev1.SecretProviderClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "spc1", + Namespace: "default", + }, + Spec: secretsstorev1.SecretProviderClassSpec{ + SecretObjects: []*secretsstorev1.SecretObject{ + { + Data: []*secretsstorev1.SecretObjectData{ + { + ObjectName: "object1", + Key: "foo", + }, + }, + }, + }, + Provider: "provider1", + }, + } + customize(spc) + return spc +} + +func getSPCPS(t *testing.T, customize func(*secretsstorev1.SecretProviderClassPodStatus)) *secretsstorev1.SecretProviderClassPodStatus { + var spcps = &secretsstorev1.SecretProviderClassPodStatus{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1-default-spc1", + Namespace: "default", + Labels: map[string]string{secretsstorev1.InternalNodeLabel: "nodeName"}, + }, + Status: secretsstorev1.SecretProviderClassPodStatusStatus{ + SecretProviderClassName: "spc1", + PodName: "pod1", + TargetPath: getTestTargetPath(t, "foo", "csi-volume"), + Objects: []secretsstorev1.SecretProviderClassObject{ + { + ID: "secret/object1", + Version: "v1", + }, + }, + }, + } + customize(spcps) + return spcps +} + +func getPod(customize func(*corev1.Pod)) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + UID: types.UID("foo"), + }, + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "csi-volume", + VolumeSource: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: "secrets-store.csi.k8s.io", + VolumeAttributes: map[string]string{"secretProviderClass": "spc1"}, + NodePublishSecretRef: &corev1.LocalObjectReference{ + Name: "secret1", + }, + }, + }, + }, + }, + }, + } + customize(pod) + return pod +} + +func GetNodePublishSecretRefSecret() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret1", + Namespace: "default", + Labels: map[string]string{ + controllers.SecretUsedLabel: "true", + }, + }, + Data: map[string][]byte{"clientid": []byte("clientid")}, + } +} func TestReconcileError(t *testing.T) { g := NewWithT(t) @@ -103,185 +190,47 @@ func TestReconcileError(t *testing.T) { expectedErrorEvents bool }{ { - name: "secret provider class not found", - rotationPollInterval: 60 * time.Second, - secretProviderClassPodStatusToProcess: &secretsstorev1.SecretProviderClassPodStatus{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1-default-spc1", - Namespace: "default", - Labels: map[string]string{secretsstorev1.InternalNodeLabel: "nodeName"}, - }, - Status: secretsstorev1.SecretProviderClassPodStatusStatus{ - SecretProviderClassName: "spc1", - PodName: "pod1", - }, - }, - secretProviderClassToAdd: &secretsstorev1.SecretProviderClass{}, - podToAdd: &corev1.Pod{}, - socketPath: t.TempDir(), - secretToAdd: &corev1.Secret{}, - expectedErr: true, + name: "secret provider class not found", + rotationPollInterval: 60 * time.Second, + secretProviderClassPodStatusToProcess: getSPCPS(t, func(*secretsstorev1.SecretProviderClassPodStatus) {}), + secretProviderClassToAdd: &secretsstorev1.SecretProviderClass{}, + podToAdd: &corev1.Pod{}, + socketPath: t.TempDir(), + secretToAdd: &corev1.Secret{}, + expectedErr: true, }, { - name: "failed to get pod", - rotationPollInterval: 60 * time.Second, - secretProviderClassPodStatusToProcess: &secretsstorev1.SecretProviderClassPodStatus{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1-default-spc1", - Namespace: "default", - Labels: map[string]string{secretsstorev1.InternalNodeLabel: "nodeName"}, - }, - Status: secretsstorev1.SecretProviderClassPodStatusStatus{ - SecretProviderClassName: "spc1", - PodName: "pod1", - }, - }, - secretProviderClassToAdd: &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "spc1", - Namespace: "default", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - SecretObjects: []*secretsstorev1.SecretObject{ - { - Data: []*secretsstorev1.SecretObjectData{ - { - ObjectName: "object1", - Key: "foo", - }, - }, - }, - }, - }, - }, + name: "failed to get pod", + rotationPollInterval: 60 * time.Second, + secretProviderClassPodStatusToProcess: getSPCPS(t, func(*secretsstorev1.SecretProviderClassPodStatus) {}), + secretProviderClassToAdd: getSPC(func(s *secretsstorev1.SecretProviderClass) { + s.Spec.Provider = "" + }), podToAdd: &corev1.Pod{}, socketPath: t.TempDir(), secretToAdd: &corev1.Secret{}, expectedErr: true, }, { - name: "failed to get NodePublishSecretRef secret", - rotationPollInterval: 60 * time.Second, - secretProviderClassPodStatusToProcess: &secretsstorev1.SecretProviderClassPodStatus{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1-default-spc1", - Namespace: "default", - Labels: map[string]string{secretsstorev1.InternalNodeLabel: "nodeName"}, - }, - Status: secretsstorev1.SecretProviderClassPodStatusStatus{ - SecretProviderClassName: "spc1", - PodName: "pod1", - TargetPath: getTestTargetPath(t, "foo", "csi-volume"), - }, - }, - secretProviderClassToAdd: &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "spc1", - Namespace: "default", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - SecretObjects: []*secretsstorev1.SecretObject{ - { - Data: []*secretsstorev1.SecretObjectData{ - { - ObjectName: "object1", - Key: "foo", - }, - }, - }, - }, - Provider: "provider1", - }, - }, - podToAdd: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1", - Namespace: "default", - UID: types.UID("foo"), - }, - Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: "csi-volume", - VolumeSource: corev1.VolumeSource{ - CSI: &corev1.CSIVolumeSource{ - Driver: "secrets-store.csi.k8s.io", - VolumeAttributes: map[string]string{"secretProviderClass": "spc1"}, - NodePublishSecretRef: &corev1.LocalObjectReference{ - Name: "secret1", - }, - }, - }, - }, - }, - }, - }, - socketPath: t.TempDir(), - secretToAdd: &corev1.Secret{}, - expectedErr: true, - expectedErrorEvents: true, + name: "failed to get NodePublishSecretRef secret", + rotationPollInterval: 60 * time.Second, + secretProviderClassPodStatusToProcess: getSPCPS(t, func(*secretsstorev1.SecretProviderClassPodStatus) {}), + secretProviderClassToAdd: getSPC(func(*secretsstorev1.SecretProviderClass) {}), + podToAdd: getPod(func(*corev1.Pod) {}), + socketPath: t.TempDir(), + secretToAdd: &corev1.Secret{}, + expectedErr: true, + expectedErrorEvents: true, }, { name: "failed to validate targetpath UID", rotationPollInterval: 60 * time.Second, - secretProviderClassPodStatusToProcess: &secretsstorev1.SecretProviderClassPodStatus{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1-default-spc1", - Namespace: "default", - Labels: map[string]string{secretsstorev1.InternalNodeLabel: "nodeName"}, - }, - Status: secretsstorev1.SecretProviderClassPodStatusStatus{ - SecretProviderClassName: "spc1", - PodName: "pod1", - TargetPath: getTestTargetPath(t, "bad-uid", "csi-volume"), - Objects: []secretsstorev1.SecretProviderClassObject{ - { - ID: "secret/object1", - Version: "v1", - }, - }, - }, - }, - secretProviderClassToAdd: &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "spc1", - Namespace: "default", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - SecretObjects: []*secretsstorev1.SecretObject{ - { - Data: []*secretsstorev1.SecretObjectData{ - { - ObjectName: "object1", - Key: "foo", - }, - }, - }, - }, - Provider: "provider1", - }, - }, - podToAdd: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1", - Namespace: "default", - UID: types.UID("foo"), - }, - Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: "csi-volume", - VolumeSource: corev1.VolumeSource{ - CSI: &corev1.CSIVolumeSource{ - Driver: "secrets-store.csi.k8s.io", - VolumeAttributes: map[string]string{"secretProviderClass": "spc1"}, - }, - }, - }, - }, - }, - }, - socketPath: t.TempDir(), + secretProviderClassPodStatusToProcess: getSPCPS(t, func(s *secretsstorev1.SecretProviderClassPodStatus) { + s.Status.TargetPath = getTestTargetPath(t, "bad-uid", "csi-volume") + }), + secretProviderClassToAdd: getSPC(func(*secretsstorev1.SecretProviderClass) {}), + podToAdd: getPod(func(*corev1.Pod) {}), + socketPath: t.TempDir(), secretToAdd: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "object1", @@ -297,64 +246,12 @@ func TestReconcileError(t *testing.T) { { name: "failed to validate targetpath volume name", rotationPollInterval: 60 * time.Second, - secretProviderClassPodStatusToProcess: &secretsstorev1.SecretProviderClassPodStatus{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1-default-spc1", - Namespace: "default", - Labels: map[string]string{secretsstorev1.InternalNodeLabel: "nodeName"}, - }, - Status: secretsstorev1.SecretProviderClassPodStatusStatus{ - SecretProviderClassName: "spc1", - PodName: "pod1", - TargetPath: getTestTargetPath(t, "foo", "bad-volume-name"), - Objects: []secretsstorev1.SecretProviderClassObject{ - { - ID: "secret/object1", - Version: "v1", - }, - }, - }, - }, - secretProviderClassToAdd: &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "spc1", - Namespace: "default", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - SecretObjects: []*secretsstorev1.SecretObject{ - { - Data: []*secretsstorev1.SecretObjectData{ - { - ObjectName: "object1", - Key: "foo", - }, - }, - }, - }, - Provider: "provider1", - }, - }, - podToAdd: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1", - Namespace: "default", - UID: types.UID("foo"), - }, - Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: "csi-volume", - VolumeSource: corev1.VolumeSource{ - CSI: &corev1.CSIVolumeSource{ - Driver: "secrets-store.csi.k8s.io", - VolumeAttributes: map[string]string{"secretProviderClass": "spc1"}, - }, - }, - }, - }, - }, - }, - socketPath: t.TempDir(), + secretProviderClassPodStatusToProcess: getSPCPS(t, func(s *secretsstorev1.SecretProviderClassPodStatus) { + s.Status.TargetPath = getTestTargetPath(t, "foo", "bad-volume-name") + }), + secretProviderClassToAdd: getSPC(func(*secretsstorev1.SecretProviderClass) {}), + podToAdd: getPod(func(*corev1.Pod) {}), + socketPath: t.TempDir(), secretToAdd: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "object1", @@ -368,73 +265,31 @@ func TestReconcileError(t *testing.T) { expectedErrorEvents: false, }, { - name: "failed to lookup provider client", - rotationPollInterval: 60 * time.Second, - secretProviderClassPodStatusToProcess: &secretsstorev1.SecretProviderClassPodStatus{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1-default-spc1", - Namespace: "default", - Labels: map[string]string{secretsstorev1.InternalNodeLabel: "nodeName"}, - }, - Status: secretsstorev1.SecretProviderClassPodStatusStatus{ - SecretProviderClassName: "spc1", - PodName: "pod1", - TargetPath: getTestTargetPath(t, "foo", "csi-volume"), - }, - }, - secretProviderClassToAdd: &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "spc1", - Namespace: "default", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - SecretObjects: []*secretsstorev1.SecretObject{ - { - Data: []*secretsstorev1.SecretObjectData{ - { - ObjectName: "object1", - Key: "foo", - }, - }, - }, - }, - Provider: "wrongprovider", - }, - }, - podToAdd: &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1", - Namespace: "default", - UID: types.UID("foo"), - }, - Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: "csi-volume", - VolumeSource: corev1.VolumeSource{ - CSI: &corev1.CSIVolumeSource{ - Driver: "secrets-store.csi.k8s.io", - VolumeAttributes: map[string]string{"secretProviderClass": "spc1"}, - NodePublishSecretRef: &corev1.LocalObjectReference{ - Name: "secret1", - }, - }, - }, - }, - }, - }, - }, - socketPath: t.TempDir(), - secretToAdd: &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "secret1", - Namespace: "default", - }, - Data: map[string][]byte{"clientid": []byte("clientid")}, - }, + name: "failed to lookup provider client", + rotationPollInterval: 60 * time.Second, + secretProviderClassPodStatusToProcess: getSPCPS(t, func(s *secretsstorev1.SecretProviderClassPodStatus) {}), + secretProviderClassToAdd: getSPC(func(s *secretsstorev1.SecretProviderClass) { + s.Spec.Provider = "wrongprovider" + }), + podToAdd: getPod(func(*corev1.Pod) {}), + socketPath: t.TempDir(), + secretToAdd: GetNodePublishSecretRefSecret(), expectedErr: true, expectedErrorEvents: true, }, + { + name: "failed to parse FSGroup", + rotationPollInterval: 60 * time.Second, + secretProviderClassPodStatusToProcess: getSPCPS(t, func(s *secretsstorev1.SecretProviderClassPodStatus) { + s.Status.FSGroup = "INVALID" + }), + secretProviderClassToAdd: getSPC(func(*secretsstorev1.SecretProviderClass) {}), + podToAdd: getPod(func(*corev1.Pod) {}), + socketPath: t.TempDir(), + secretToAdd: GetNodePublishSecretRefSecret(), + expectedErr: true, + expectedErrorEvents: true, + }, } scheme, err := setupScheme() @@ -484,183 +339,129 @@ func TestReconcileNoError(t *testing.T) { g := NewWithT(t) tests := []struct { - name string - nodePublishSecretRefSecretToAdd *corev1.Secret + name string + nodePublishSecretRefSecretToAdd *corev1.Secret + secretProviderClassPodStatusToProcess *secretsstorev1.SecretProviderClassPodStatus }{ { - name: "filtered watch for nodePublishSecretRef", - nodePublishSecretRefSecretToAdd: &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "secret1", - Namespace: "default", - Labels: map[string]string{ - controllers.SecretUsedLabel: "true", - }, - }, - Data: map[string][]byte{"clientid": []byte("clientid")}, - }, + name: "filtered watch for nodePublishSecretRef", + nodePublishSecretRefSecretToAdd: GetNodePublishSecretRefSecret(), + secretProviderClassPodStatusToProcess: getSPCPS(t, func(*secretsstorev1.SecretProviderClassPodStatus) {}), + }, + { + name: "reconcile with FSGroup", + nodePublishSecretRefSecretToAdd: GetNodePublishSecretRefSecret(), + secretProviderClassPodStatusToProcess: getSPCPS(t, func(s *secretsstorev1.SecretProviderClassPodStatus) { + s.Status.FSGroup = "1004" + }), }, } for _, test := range tests { - secretProviderClassPodStatusToProcess := &secretsstorev1.SecretProviderClassPodStatus{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1-default-spc1", - Namespace: "default", - Labels: map[string]string{secretsstorev1.InternalNodeLabel: "nodeName"}, - }, - Status: secretsstorev1.SecretProviderClassPodStatusStatus{ - SecretProviderClassName: "spc1", - PodName: "pod1", - TargetPath: getTestTargetPath(t, "foo", "csi-volume"), - Objects: []secretsstorev1.SecretProviderClassObject{ - { - ID: "secret/object1", - Version: "v1", - }, - }, - }, - } - secretProviderClassToAdd := &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "spc1", - Namespace: "default", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - SecretObjects: []*secretsstorev1.SecretObject{ - { - Data: []*secretsstorev1.SecretObjectData{ - { - ObjectName: "object1", - Key: "foo", - }, - }, - SecretName: "foosecret", - Type: "Opaque", - }, - }, - Provider: "provider1", - }, - } - podToAdd := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "pod1", - Namespace: "default", - UID: types.UID("foo"), - }, - Spec: corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: "csi-volume", - VolumeSource: corev1.VolumeSource{ - CSI: &corev1.CSIVolumeSource{ - Driver: "secrets-store.csi.k8s.io", - VolumeAttributes: map[string]string{"secretProviderClass": "spc1"}, - NodePublishSecretRef: &corev1.LocalObjectReference{ - Name: "secret1", - }, - }, - }, + t.Run(test.name, func(t *testing.T) { + secretProviderClassPodStatusToProcess := test.secretProviderClassPodStatusToProcess + secretProviderClassToAdd := getSPC(func(s *secretsstorev1.SecretProviderClass) { + s.Spec.SecretObjects[0].SecretName = "foosecret" + s.Spec.SecretObjects[0].Type = "Opaque" + }) + podToAdd := getPod(func(*corev1.Pod) {}) + secretToBeRotated := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foosecret", + Namespace: "default", + ResourceVersion: "12352", + Labels: map[string]string{ + controllers.SecretManagedLabel: "true", }, }, - }, - } - secretToBeRotated := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foosecret", - Namespace: "default", - ResourceVersion: "12352", - Labels: map[string]string{ - controllers.SecretManagedLabel: "true", - }, - }, - Data: map[string][]byte{"foo": []byte("olddata")}, - } - - socketPath := t.TempDir() - expectedObjectVersions := map[string]string{"secret/object1": "v2"} - scheme, err := setupScheme() - g.Expect(err).NotTo(HaveOccurred()) - - kubeClient := fake.NewSimpleClientset(podToAdd, test.nodePublishSecretRefSecretToAdd, secretToBeRotated) - crdClient := secretsStoreFakeClient.NewSimpleClientset(secretProviderClassPodStatusToProcess, secretProviderClassToAdd) - - initObjects := []client.Object{ - podToAdd, - secretToBeRotated, - test.nodePublishSecretRefSecretToAdd, - secretProviderClassPodStatusToProcess, - secretProviderClassToAdd, - } - ctrlClient := controllerfake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjects...).Build() - - testReconciler, err := newTestReconciler(ctrlClient, kubeClient, crdClient, 60*time.Second, socketPath) - g.Expect(err).NotTo(HaveOccurred()) - err = testReconciler.secretStore.Run(wait.NeverStop) - g.Expect(err).NotTo(HaveOccurred()) - - serverEndpoint := fmt.Sprintf("%s/%s.sock", socketPath, "provider1") - defer os.Remove(serverEndpoint) - - server, err := providerfake.NewMocKCSIProviderServer(serverEndpoint) - g.Expect(err).NotTo(HaveOccurred()) - server.SetObjects(expectedObjectVersions) - err = server.Start() - g.Expect(err).NotTo(HaveOccurred()) - - err = os.WriteFile(secretProviderClassPodStatusToProcess.Status.TargetPath+"/object1", []byte("newdata"), secretsstore.FilePermission) - g.Expect(err).NotTo(HaveOccurred()) - - err = testReconciler.reconcile(context.TODO(), secretProviderClassPodStatusToProcess) - g.Expect(err).NotTo(HaveOccurred()) - - // validate the secret provider class pod status versions have been updated - updatedSPCPodStatus, err := crdClient.SecretsstoreV1().SecretProviderClassPodStatuses(corev1.NamespaceDefault).Get(context.TODO(), "pod1-default-spc1", metav1.GetOptions{}) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(updatedSPCPodStatus.Status.Objects).To(Equal([]secretsstorev1.SecretProviderClassObject{{ID: "secret/object1", Version: "v2"}})) - - // validate the secret data has been updated to the latest value - updatedSecret, err := kubeClient.CoreV1().Secrets(corev1.NamespaceDefault).Get(context.TODO(), "foosecret", metav1.GetOptions{}) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(updatedSecret.Data["foo"]).To(Equal([]byte("newdata"))) - - // 2 normal events - one for successfully updating the mounted contents and - // second for successfully rotating the K8s secret - g.Expect(len(fakeRecorder.Events)).To(BeNumerically("==", 2)) - for len(fakeRecorder.Events) > 0 { - <-fakeRecorder.Events - } - - // test with pod being terminated - podToAdd.DeletionTimestamp = &metav1.Time{Time: time.Now()} - kubeClient = fake.NewSimpleClientset(podToAdd, test.nodePublishSecretRefSecretToAdd) - initObjects = []client.Object{ - podToAdd, - test.nodePublishSecretRefSecretToAdd, - } - ctrlClient = controllerfake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjects...).Build() - testReconciler, err = newTestReconciler(ctrlClient, kubeClient, crdClient, 60*time.Second, socketPath) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(err).NotTo(HaveOccurred()) - - err = testReconciler.reconcile(context.TODO(), secretProviderClassPodStatusToProcess) - g.Expect(err).NotTo(HaveOccurred()) - - // test with pod being in succeeded phase - podToAdd.DeletionTimestamp = nil - podToAdd.Status.Phase = corev1.PodSucceeded - kubeClient = fake.NewSimpleClientset(podToAdd, test.nodePublishSecretRefSecretToAdd) - initObjects = []client.Object{ - podToAdd, - test.nodePublishSecretRefSecretToAdd, - } - ctrlClient = controllerfake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjects...).Build() - testReconciler, err = newTestReconciler(ctrlClient, kubeClient, crdClient, 60*time.Second, socketPath) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(err).NotTo(HaveOccurred()) - - err = testReconciler.reconcile(context.TODO(), secretProviderClassPodStatusToProcess) - g.Expect(err).NotTo(HaveOccurred()) + Data: map[string][]byte{"foo": []byte("olddata")}, + } + + socketPath := t.TempDir() + expectedObjectVersions := map[string]string{"secret/object1": "v2"} + scheme, err := setupScheme() + g.Expect(err).NotTo(HaveOccurred()) + + kubeClient := fake.NewSimpleClientset(podToAdd, test.nodePublishSecretRefSecretToAdd, secretToBeRotated) + crdClient := secretsStoreFakeClient.NewSimpleClientset(secretProviderClassPodStatusToProcess, secretProviderClassToAdd) + + initObjects := []client.Object{ + podToAdd, + secretToBeRotated, + test.nodePublishSecretRefSecretToAdd, + secretProviderClassPodStatusToProcess, + secretProviderClassToAdd, + } + ctrlClient := controllerfake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjects...).Build() + + testReconciler, err := newTestReconciler(ctrlClient, kubeClient, crdClient, 60*time.Second, socketPath) + g.Expect(err).NotTo(HaveOccurred()) + err = testReconciler.secretStore.Run(wait.NeverStop) + g.Expect(err).NotTo(HaveOccurred()) + + serverEndpoint := fmt.Sprintf("%s/%s.sock", socketPath, "provider1") + defer os.Remove(serverEndpoint) + + server, err := providerfake.NewMocKCSIProviderServer(serverEndpoint) + g.Expect(err).NotTo(HaveOccurred()) + server.SetObjects(expectedObjectVersions) + err = server.Start() + g.Expect(err).NotTo(HaveOccurred()) + + err = os.WriteFile(secretProviderClassPodStatusToProcess.Status.TargetPath+"/object1", []byte("newdata"), secretsstore.FilePermission) + g.Expect(err).NotTo(HaveOccurred()) + + err = testReconciler.reconcile(context.TODO(), secretProviderClassPodStatusToProcess) + g.Expect(err).NotTo(HaveOccurred()) + + // validate the secret provider class pod status versions have been updated + updatedSPCPodStatus, err := crdClient.SecretsstoreV1().SecretProviderClassPodStatuses(corev1.NamespaceDefault).Get(context.TODO(), "pod1-default-spc1", metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(updatedSPCPodStatus.Status.Objects).To(Equal([]secretsstorev1.SecretProviderClassObject{{ID: "secret/object1", Version: "v2"}})) + + // validate the secret data has been updated to the latest value + updatedSecret, err := kubeClient.CoreV1().Secrets(corev1.NamespaceDefault).Get(context.TODO(), "foosecret", metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(updatedSecret.Data["foo"]).To(Equal([]byte("newdata"))) + + // 2 normal events - one for successfully updating the mounted contents and + // second for successfully rotating the K8s secret + g.Expect(len(fakeRecorder.Events)).To(BeNumerically("==", 2)) + for len(fakeRecorder.Events) > 0 { + <-fakeRecorder.Events + } + + // test with pod being terminated + podToAdd.DeletionTimestamp = &metav1.Time{Time: time.Now()} + kubeClient = fake.NewSimpleClientset(podToAdd, test.nodePublishSecretRefSecretToAdd) + initObjects = []client.Object{ + podToAdd, + test.nodePublishSecretRefSecretToAdd, + } + ctrlClient = controllerfake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjects...).Build() + testReconciler, err = newTestReconciler(ctrlClient, kubeClient, crdClient, 60*time.Second, socketPath) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(err).NotTo(HaveOccurred()) + + err = testReconciler.reconcile(context.TODO(), secretProviderClassPodStatusToProcess) + g.Expect(err).NotTo(HaveOccurred()) + + // test with pod being in succeeded phase + podToAdd.DeletionTimestamp = nil + podToAdd.Status.Phase = corev1.PodSucceeded + kubeClient = fake.NewSimpleClientset(podToAdd, test.nodePublishSecretRefSecretToAdd) + initObjects = []client.Object{ + podToAdd, + test.nodePublishSecretRefSecretToAdd, + } + ctrlClient = controllerfake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjects...).Build() + testReconciler, err = newTestReconciler(ctrlClient, kubeClient, crdClient, 60*time.Second, socketPath) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(err).NotTo(HaveOccurred()) + + err = testReconciler.reconcile(context.TODO(), secretProviderClassPodStatusToProcess) + g.Expect(err).NotTo(HaveOccurred()) + }) } } diff --git a/pkg/secrets-store/nodeserver.go b/pkg/secrets-store/nodeserver.go index e2af8dcd0..7d7c2b3b5 100644 --- a/pkg/secrets-store/nodeserver.go +++ b/pkg/secrets-store/nodeserver.go @@ -23,8 +23,10 @@ import ( "fmt" "os" "path/filepath" + "strconv" "time" + "sigs.k8s.io/secrets-store-csi-driver/pkg/constants" internalerrors "sigs.k8s.io/secrets-store-csi-driver/pkg/errors" "sigs.k8s.io/secrets-store-csi-driver/pkg/k8s" "sigs.k8s.io/secrets-store-csi-driver/pkg/util/fileutil" @@ -140,7 +142,22 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis return &csi.NodePublishVolumeResponse{}, nil } - klog.V(2).InfoS("node publish volume", "target", targetPath, "volumeId", volumeID, "mount flags", mountFlags) + klog.V(2).InfoS("node publish volume", "target", targetPath, "volumeId", volumeID, "mount flags", mountFlags, "volumeCapabilities", req.VolumeCapability.String()) + + gid := constants.NoGID // Group ID to Chown the volume contents to + mountVol := req.VolumeCapability.GetMount() + if len(mountVol.GetVolumeMountGroup()) > 0 { + fsGroupStr := mountVol.GetVolumeMountGroup() + klog.V(5).Info("fsGroupStr: %v\n", fsGroupStr) + gid, err = strconv.ParseInt(fsGroupStr, 10, 64) + klog.V(5).Info("converted gid: %v\n", gid) + if err != nil { + klog.ErrorS(err, "failed to mount secrets store object content", "pod", klog.ObjectRef{Namespace: podNamespace, Name: podName}, "FSGroup: ", fsGroupStr) + return nil, status.Error(codes.InvalidArgument, "Error parsing FSGroup") + } + } else { + klog.V(5).InfoS("mount group not set", "targetPath", targetPath, "pod", klog.ObjectRef{Namespace: podNamespace, Name: podName}) + } if isMockProvider(providerName) { // mock provider is used only for running sanity tests against the driver @@ -236,8 +253,9 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis } } mounted = true + var objectVersions map[string]string - if objectVersions, errorReason, err = ns.mountSecretsStoreObjectContent(ctx, providerName, string(parametersStr), string(secretStr), targetPath, string(permissionStr), podName); err != nil { + if objectVersions, errorReason, err = ns.mountSecretsStoreObjectContent(ctx, providerName, string(parametersStr), string(secretStr), targetPath, string(permissionStr), podName, gid); err != nil { klog.ErrorS(err, "failed to mount secrets store object content", "pod", klog.ObjectRef{Namespace: podNamespace, Name: podName}) return nil, fmt.Errorf("failed to mount secrets store objects for pod %s/%s, err: %w", podNamespace, podName, err) } @@ -246,7 +264,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis // SPCPS is created the first time after the pod mount is complete. Update is required in scenarios where // the pod with same name (pods created by statefulsets) is moved to a different node and the old SPCPS // has not yet been garbage collected. - if err = createOrUpdateSecretProviderClassPodStatus(ctx, ns.client, ns.reader, podName, podNamespace, podUID, secretProviderClass, targetPath, ns.nodeID, true, objectVersions); err != nil { + if err = createOrUpdateSecretProviderClassPodStatus(ctx, ns.client, ns.reader, podName, podNamespace, podUID, secretProviderClass, targetPath, ns.nodeID, true, objectVersions, gid); err != nil { return nil, fmt.Errorf("failed to create secret provider class pod status for pod %s/%s, err: %w", podNamespace, podName, err) } @@ -334,7 +352,7 @@ func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag return &csi.NodeUnstageVolumeResponse{}, nil } -func (ns *nodeServer) mountSecretsStoreObjectContent(ctx context.Context, providerName, attributes, secrets, targetPath, permission, podName string) (map[string]string, string, error) { +func (ns *nodeServer) mountSecretsStoreObjectContent(ctx context.Context, providerName, attributes, secrets, targetPath, permission, podName string, gid int64) (map[string]string, string, error) { if len(attributes) == 0 { return nil, "", errors.New("missing attributes") } @@ -352,7 +370,7 @@ func (ns *nodeServer) mountSecretsStoreObjectContent(ctx context.Context, provid klog.InfoS("Using gRPC client", "provider", providerName, "pod", podName) - return MountContent(ctx, client, attributes, secrets, targetPath, permission, nil) + return MountContent(ctx, client, attributes, secrets, targetPath, permission, nil, gid) } func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { @@ -376,6 +394,13 @@ func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC }, }, }, + { + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_VOLUME_MOUNT_GROUP, + }, + }, + }, } return &csi.NodeGetCapabilitiesResponse{ diff --git a/pkg/secrets-store/nodeserver_test.go b/pkg/secrets-store/nodeserver_test.go index cef12021f..e0c80ae33 100644 --- a/pkg/secrets-store/nodeserver_test.go +++ b/pkg/secrets-store/nodeserver_test.go @@ -59,6 +59,40 @@ func testNodeServer(t *testing.T, client client.Client, reporter StatsReporter) return newNodeServer("testnode", mount.NewFakeMounter([]mount.MountPoint{}), providerClients, client, client, reporter, k8s.NewTokenClient(fakeclient.NewSimpleClientset(), "test-driver", 1*time.Second)) } +func getInitObjects(customize func(*secretsstorev1.SecretProviderClass)) []client.Object { + var spc = &secretsstorev1.SecretProviderClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "provider1", + Namespace: "default", + }, + Spec: secretsstorev1.SecretProviderClassSpec{ + Provider: "provider1", + Parameters: map[string]string{"parameter1": "value1"}, + }, + } + customize(spc) + var initObjects = []client.Object{ + spc, + } + return initObjects +} + +func getRequest(t *testing.T, customize func(*csi.NodePublishVolumeRequest)) *csi.NodePublishVolumeRequest { + var request = &csi.NodePublishVolumeRequest{ + VolumeCapability: &csi.VolumeCapability{}, + VolumeId: "testvolid1", + VolumeContext: map[string]string{ + "secretProviderClass": "provider1", + CSIPodName: "pod1", + CSIPodNamespace: "default", + CSIPodUID: "poduid1", + }, + TargetPath: targetPath(t), + Readonly: true, + } + customize(request) + return request +} func TestNodePublishVolume_Errors(t *testing.T) { tests := []struct { name string @@ -106,117 +140,59 @@ func TestNodePublishVolume_Errors(t *testing.T) { want: codes.Unknown, }, { - name: "spc missing", - nodePublishVolReq: &csi.NodePublishVolumeRequest{ - VolumeCapability: &csi.VolumeCapability{}, - VolumeId: "testvolid1", - TargetPath: targetPath(t), - VolumeContext: map[string]string{"secretProviderClass": "provider1", CSIPodName: "pod1", CSIPodNamespace: "default", CSIPodUID: "poduid1", "providerName": "provider1"}, - Readonly: true, - }, - initObjects: []client.Object{ - &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "provider1", - Namespace: "incorrect_namespace", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - Provider: "provider1", - Parameters: map[string]string{"parameter1": "value1"}, - }, - }, - }, + name: "spc missing", + nodePublishVolReq: getRequest(t, func(*csi.NodePublishVolumeRequest) {}), + initObjects: getInitObjects(func(s *secretsstorev1.SecretProviderClass) { + s.ObjectMeta.Namespace = "incorrect_namespace" + }), want: codes.Unknown, }, { - name: "provider not set in secret provider class", - nodePublishVolReq: &csi.NodePublishVolumeRequest{ - VolumeCapability: &csi.VolumeCapability{}, - VolumeId: "testvolid1", - TargetPath: targetPath(t), - VolumeContext: map[string]string{"secretProviderClass": "provider1", CSIPodName: "pod1", CSIPodNamespace: "default"}, - }, - initObjects: []client.Object{ - &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "provider1", - Namespace: "default", - }, - }, - }, + name: "provider not set in secret provider class", + nodePublishVolReq: getRequest(t, func(*csi.NodePublishVolumeRequest) {}), + initObjects: getInitObjects(func(s *secretsstorev1.SecretProviderClass) { + s.Spec = secretsstorev1.SecretProviderClassSpec{} + }), want: codes.Unknown, }, { - name: "parameters not set in secret provider class", - nodePublishVolReq: &csi.NodePublishVolumeRequest{ - VolumeCapability: &csi.VolumeCapability{}, - VolumeId: "testvolid1", - TargetPath: targetPath(t), - VolumeContext: map[string]string{"secretProviderClass": "provider1", CSIPodName: "pod1", CSIPodNamespace: "default"}, - }, - initObjects: []client.Object{ - &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "provider1", - Namespace: "default", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - Provider: "provider1", - }, - }, - }, + name: "parameters not set in secret provider class", + nodePublishVolReq: getRequest(t, func(*csi.NodePublishVolumeRequest) {}), + initObjects: getInitObjects(func(s *secretsstorev1.SecretProviderClass) { + s.Spec.Parameters = map[string]string{} + }), want: codes.Unknown, }, { name: "read only is not set to true", - nodePublishVolReq: &csi.NodePublishVolumeRequest{ - VolumeCapability: &csi.VolumeCapability{}, - VolumeId: "testvolid1", - TargetPath: targetPath(t), - VolumeContext: map[string]string{"secretProviderClass": "provider1", CSIPodName: "pod1", CSIPodNamespace: "default"}, - }, - initObjects: []client.Object{ - &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "provider1", - Namespace: "default", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - Provider: "provider1", - Parameters: map[string]string{"parameter1": "value1"}, - }, - }, - }, - want: codes.InvalidArgument, + nodePublishVolReq: getRequest(t, func(r *csi.NodePublishVolumeRequest) { + r.Readonly = false + }), + initObjects: getInitObjects(func(*secretsstorev1.SecretProviderClass) {}), + want: codes.InvalidArgument, }, { - name: "provider not installed", - nodePublishVolReq: &csi.NodePublishVolumeRequest{ - VolumeCapability: &csi.VolumeCapability{}, - VolumeId: "testvolid1", - TargetPath: targetPath(t), - VolumeContext: map[string]string{ - "secretProviderClass": "provider1", - CSIPodName: "pod1", - CSIPodNamespace: "default", - CSIPodUID: "poduid1", - }, - Readonly: true, - }, - initObjects: []client.Object{ - &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "provider1", - Namespace: "default", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - Provider: "provider_not_installed", - Parameters: map[string]string{"parameter1": "value1"}, - }, - }, - }, + name: "provider not installed", + nodePublishVolReq: getRequest(t, func(*csi.NodePublishVolumeRequest) {}), + initObjects: getInitObjects(func(s *secretsstorev1.SecretProviderClass) { + s.Spec.Provider = "provider_not_installed" + }), want: codes.Unknown, }, + { + name: "Invalid FSGroup", + nodePublishVolReq: getRequest(t, func(r *csi.NodePublishVolumeRequest) { + r.VolumeCapability = &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{ + VolumeMountGroup: "INVALID", + }, + }, + } + }), + initObjects: getInitObjects(func(*secretsstorev1.SecretProviderClass) {}), + want: codes.InvalidArgument, + }, } s := scheme.Scheme @@ -272,61 +248,31 @@ func TestNodePublishVolume(t *testing.T) { initObjects []client.Object }{ { - name: "volume mount", - nodePublishVolReq: &csi.NodePublishVolumeRequest{ - VolumeCapability: &csi.VolumeCapability{}, - VolumeId: "testvolid1", - TargetPath: targetPath(t), - VolumeContext: map[string]string{ - "secretProviderClass": "provider1", - CSIPodName: "pod1", - CSIPodNamespace: "default", - CSIPodUID: "poduid1", - }, - Readonly: true, - }, - initObjects: []client.Object{ - &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "provider1", - Namespace: "default", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - Provider: "provider1", - Parameters: map[string]string{"parameter1": "value1"}, - }, - }, - }, + name: "volume mount", + nodePublishVolReq: getRequest(t, func(*csi.NodePublishVolumeRequest) {}), + initObjects: getInitObjects(func(*secretsstorev1.SecretProviderClass) {}), }, { name: "volume mount with refresh token", - nodePublishVolReq: &csi.NodePublishVolumeRequest{ - VolumeCapability: &csi.VolumeCapability{}, - VolumeId: "testvolid1", - TargetPath: targetPath(t), - VolumeContext: map[string]string{ - "secretProviderClass": "provider1", - CSIPodName: "pod1", - CSIPodNamespace: "default", - CSIPodUID: "poduid1", - // not a real token, just for testing - CSIPodServiceAccountTokens: `{"https://kubernetes.default.svc":{"token":"eyJhbGciOiJSUzI1NiIsImtpZCI6IjEyMyJ9.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjIl0sImV4cCI6MTYxMTk1OTM5NiwiaWF0IjoxNjExOTU4Nzk2LCJpc3MiOiJodHRwczovL2t1YmVybmV0ZXMuZGVmYXVsdC5zdmMiLCJrdWJlcm5ldGVzLmlvIjp7Im5hbWVzcGFjZSI6ImRlZmF1bHQiLCJzZXJ2aWNlYWNjb3VudCI6eyJuYW1lIjoiZGVmYXVsdCIsInVpZCI6IjA5MWUyNTU3LWJkODYtNDhhMC1iZmNmLWI1YTI4ZjRjODAyNCJ9fSwibmJmIjoxNjExOTU4Nzk2LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6ZGVmYXVsdDpkZWZhdWx0In0.YNU2Z_gEE84DGCt8lh9GuE8gmoof-Pk_7emp3fsyj9pq16DRiDaLtOdprH-njpOYqvtT5Uf_QspFc_RwD_pdq9UJWCeLxFkRTsYR5WSjhMFcl767c4Cwp_oZPYhaHd1x7aU1emH-9oarrM__tr1hSmGoAc2I0gUSkAYFueaTUSy5e5d9QKDfjVljDRc7Yrp6qAAfd1OuDdk1XYIjrqTHk1T1oqGGlcd3lRM_dKSsW5I_YqgKMrjwNt8yOKcdKBrgQhgC42GZbFDRVJDJHs_Hq32xo-2s3PJ8UZ_alN4wv8EbuwB987_FHBTc_XAULHPvp0mCv2C5h0V2A7gzccv30A","expirationTimestamp":"2021-01-29T22:29:56Z"}}`, - "providerName": "provider1", - }, - Readonly: true, - }, - initObjects: []client.Object{ - &secretsstorev1.SecretProviderClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "provider1", - Namespace: "default", - }, - Spec: secretsstorev1.SecretProviderClassSpec{ - Provider: "provider1", - Parameters: map[string]string{"parameter1": "value1"}, + nodePublishVolReq: getRequest(t, func(r *csi.NodePublishVolumeRequest) { + // not a real token, just for testing + r.VolumeContext[CSIPodServiceAccountTokens] = `{"https://kubernetes.default.svc":{"token":"eyJhbGciOiJSUzI1NiIsImtpZCI6IjEyMyJ9.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjIl0sImV4cCI6MTYxMTk1OTM5NiwiaWF0IjoxNjExOTU4Nzk2LCJpc3MiOiJodHRwczovL2t1YmVybmV0ZXMuZGVmYXVsdC5zdmMiLCJrdWJlcm5ldGVzLmlvIjp7Im5hbWVzcGFjZSI6ImRlZmF1bHQiLCJzZXJ2aWNlYWNjb3VudCI6eyJuYW1lIjoiZGVmYXVsdCIsInVpZCI6IjA5MWUyNTU3LWJkODYtNDhhMC1iZmNmLWI1YTI4ZjRjODAyNCJ9fSwibmJmIjoxNjExOTU4Nzk2LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6ZGVmYXVsdDpkZWZhdWx0In0.YNU2Z_gEE84DGCt8lh9GuE8gmoof-Pk_7emp3fsyj9pq16DRiDaLtOdprH-njpOYqvtT5Uf_QspFc_RwD_pdq9UJWCeLxFkRTsYR5WSjhMFcl767c4Cwp_oZPYhaHd1x7aU1emH-9oarrM__tr1hSmGoAc2I0gUSkAYFueaTUSy5e5d9QKDfjVljDRc7Yrp6qAAfd1OuDdk1XYIjrqTHk1T1oqGGlcd3lRM_dKSsW5I_YqgKMrjwNt8yOKcdKBrgQhgC42GZbFDRVJDJHs_Hq32xo-2s3PJ8UZ_alN4wv8EbuwB987_FHBTc_XAULHPvp0mCv2C5h0V2A7gzccv30A","expirationTimestamp":"2021-01-29T22:29:56Z"}}` + r.VolumeContext["providerName"] = "provider1" + }), + initObjects: getInitObjects(func(*secretsstorev1.SecretProviderClass) {}), + }, + { + name: "volume mount with valid FSGroup", + nodePublishVolReq: getRequest(t, func(r *csi.NodePublishVolumeRequest) { + r.VolumeCapability = &csi.VolumeCapability{ + AccessType: &csi.VolumeCapability_Mount{ + Mount: &csi.VolumeCapability_MountVolume{ + VolumeMountGroup: "1004", + }, }, - }, - }, + } + }), + initObjects: getInitObjects(func(*secretsstorev1.SecretProviderClass) {}), }, } diff --git a/pkg/secrets-store/provider_client.go b/pkg/secrets-store/provider_client.go index 2672f53b3..ab43e7b43 100644 --- a/pkg/secrets-store/provider_client.go +++ b/pkg/secrets-store/provider_client.go @@ -224,7 +224,7 @@ func (p *PluginClientBuilder) HealthCheck(ctx context.Context, interval time.Dur // MountContent calls the client's Mount() RPC with helpers to format the // request and interpret the response. -func MountContent(ctx context.Context, client v1alpha1.CSIDriverProviderClient, attributes, secrets, targetPath, permission string, oldObjectVersions map[string]string) (map[string]string, string, error) { +func MountContent(ctx context.Context, client v1alpha1.CSIDriverProviderClient, attributes, secrets, targetPath, permission string, oldObjectVersions map[string]string, gid int64) (map[string]string, string, error) { var objVersions []*v1alpha1.ObjectVersion for obj, version := range oldObjectVersions { objVersions = append(objVersions, &v1alpha1.ObjectVersion{Id: obj, Version: version}) @@ -272,7 +272,7 @@ func MountContent(ctx context.Context, client v1alpha1.CSIDriverProviderClient, return objectVersions, "", nil } - if err := fileutil.WritePayloads(targetPath, resp.GetFiles()); err != nil { + if err := fileutil.WritePayloads(targetPath, resp.GetFiles(), gid); err != nil { return nil, internalerrors.FileWriteError, err } klog.V(5).Info("mount response files written.") diff --git a/pkg/secrets-store/provider_client_test.go b/pkg/secrets-store/provider_client_test.go index 2e5eadb30..d62819a78 100644 --- a/pkg/secrets-store/provider_client_test.go +++ b/pkg/secrets-store/provider_client_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "sigs.k8s.io/secrets-store-csi-driver/pkg/constants" "sigs.k8s.io/secrets-store-csi-driver/pkg/util/fileutil" "sigs.k8s.io/secrets-store-csi-driver/provider/fake" "sigs.k8s.io/secrets-store-csi-driver/provider/v1alpha1" @@ -195,7 +196,7 @@ func TestMountContent(t *testing.T) { t.Fatalf("expected err to be nil, got: %+v", err) } - objectVersions, _, err := MountContent(context.TODO(), client, "{}", "{}", targetPath, test.permission, nil) + objectVersions, _, err := MountContent(context.TODO(), client, "{}", "{}", targetPath, test.permission, nil, constants.NoGID) if err != nil { t.Errorf("expected err to be nil, got: %+v", err) } @@ -253,7 +254,7 @@ func TestMountContent_TooLarge(t *testing.T) { } // rpc error: code = ResourceExhausted desc = grpc: received message larger than max (28 vs. 5) - _, errorCode, err := MountContent(context.TODO(), client, "{}", "{}", targetPath, "777", nil) + _, errorCode, err := MountContent(context.TODO(), client, "{}", "{}", targetPath, "777", nil, constants.NoGID) if err == nil { t.Errorf("expected err to be not nil") } @@ -347,7 +348,7 @@ func TestMountContentError(t *testing.T) { t.Fatalf("expected err to be nil, got: %+v", err) } - objectVersions, errorCode, err := MountContent(context.TODO(), client, test.attributes, test.secrets, test.targetPath, test.permission, nil) + objectVersions, errorCode, err := MountContent(context.TODO(), client, test.attributes, test.secrets, test.targetPath, test.permission, nil, constants.NoGID) if err == nil { t.Errorf("expected err to be not nil") } diff --git a/pkg/secrets-store/utils.go b/pkg/secrets-store/utils.go index e709b8033..8e1bc28ee 100644 --- a/pkg/secrets-store/utils.go +++ b/pkg/secrets-store/utils.go @@ -20,9 +20,11 @@ import ( "context" "fmt" "os" + "strconv" "strings" secretsstorev1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" + "sigs.k8s.io/secrets-store-csi-driver/pkg/constants" "sigs.k8s.io/secrets-store-csi-driver/pkg/util/runtimeutil" "sigs.k8s.io/secrets-store-csi-driver/pkg/util/spcpsutil" @@ -90,7 +92,7 @@ func getSecretProviderItem(ctx context.Context, c client.Client, name, namespace // createOrUpdateSecretProviderClassPodStatus creates secret provider class pod status if not exists. // if the secret provider class pod status already exists, it'll update the status and owner references. -func createOrUpdateSecretProviderClassPodStatus(ctx context.Context, c client.Client, reader client.Reader, podname, namespace, podUID, spcName, targetPath, nodeID string, mounted bool, objects map[string]string) error { +func createOrUpdateSecretProviderClassPodStatus(ctx context.Context, c client.Client, reader client.Reader, podname, namespace, podUID, spcName, targetPath, nodeID string, mounted bool, objects map[string]string, gid int64) error { var o []secretsstorev1.SecretProviderClassObject var err error spcpsName := podname + "-" + namespace + "-" + spcName @@ -99,7 +101,11 @@ func createOrUpdateSecretProviderClassPodStatus(ctx context.Context, c client.Cl o = append(o, secretsstorev1.SecretProviderClassObject{ID: k, Version: v}) } o = spcpsutil.OrderSecretProviderClassObjectByID(o) - + fsGroup := "" + if gid != constants.NoGID { + fsGroup = strconv.FormatInt(gid, 10) + } + klog.V(5).InfoS("setting gid in spcps", "pod", klog.ObjectRef{Namespace: namespace, Name: podname}, "gid", gid, "gidStr", fsGroup) spcPodStatus := &secretsstorev1.SecretProviderClassPodStatus{ ObjectMeta: metav1.ObjectMeta{ Name: spcpsName, @@ -112,6 +118,7 @@ func createOrUpdateSecretProviderClassPodStatus(ctx context.Context, c client.Cl Mounted: mounted, SecretProviderClassName: spcName, Objects: o, + FSGroup: fsGroup, }, } @@ -125,7 +132,6 @@ func createOrUpdateSecretProviderClassPodStatus(ctx context.Context, c client.Cl UID: types.UID(podUID), }, }) - if err = c.Create(ctx, spcPodStatus); err == nil || !apierrors.IsAlreadyExists(err) { return err } diff --git a/pkg/secrets-store/utils_test.go b/pkg/secrets-store/utils_test.go index 212809ef4..98ca6c0b8 100644 --- a/pkg/secrets-store/utils_test.go +++ b/pkg/secrets-store/utils_test.go @@ -30,6 +30,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/secrets-store-csi-driver/pkg/constants" ) var ( @@ -137,7 +138,7 @@ func TestCreateOrUpdateSecretProviderClassPodStatus(t *testing.T) { cb := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.initObjects...) client := cb.Build() - err := createOrUpdateSecretProviderClassPodStatus(context.TODO(), client, client, testPodName, testNamespace, testPodUID, testSPCName, testTargetPath, tt.nodeID, true, tt.objects) + err := createOrUpdateSecretProviderClassPodStatus(context.TODO(), client, client, testPodName, testNamespace, testPodUID, testSPCName, testTargetPath, tt.nodeID, true, tt.objects, constants.NoGID) if err != nil { t.Errorf("Unexpected error: %v", err) } diff --git a/pkg/util/fileutil/atomic_writer.go b/pkg/util/fileutil/atomic_writer.go index 34081a176..77d7d578b 100644 --- a/pkg/util/fileutil/atomic_writer.go +++ b/pkg/util/fileutil/atomic_writer.go @@ -18,6 +18,8 @@ limitations under the License. // * tag: v1.20.6, // * commit: 8a62859e515889f07e3e3be6a1080413f17cf2c3 // * link: https://github.com/kubernetes/kubernetes/blob/8a62859e515889f07e3e3be6a1080413f17cf2c3/pkg/volume/util/atomic_writer.go +// In addition, FileProjection::FSUser has been changed to FileProjection::FSGroup +// to facilitate support for FSGroup csi.NodeServiceCapability_RPC_VOLUME_MOUNT_GROUP package fileutil @@ -67,9 +69,9 @@ type AtomicWriter struct { // FileProjection contains file Data and access Mode type FileProjection struct { - Data []byte - Mode int32 - FsUser *int64 + Data []byte + Mode int32 + FsGroup *int64 } // NewAtomicWriter creates a new AtomicWriter configured to write to the given @@ -410,11 +412,11 @@ func (w *AtomicWriter) writePayloadToDir(payload map[string]FileProjection, dir return err } - if fileProjection.FsUser == nil { + if fileProjection.FsGroup == nil || runtimeutil.IsRuntimeWindows() { continue } - if err := os.Chown(fullPath, int(*fileProjection.FsUser), -1); err != nil { - klog.ErrorS(err, "unable to change file with owner", "logContext", w.logContext, "fullPath", fullPath, "owner", int(*fileProjection.FsUser)) + if err := os.Chown(fullPath, -1, int(*fileProjection.FsGroup)); err != nil { + klog.ErrorS(err, "unable to change file with owner", "logContext", w.logContext, "fullPath", fullPath, "owner", int(*fileProjection.FsGroup)) return err } } diff --git a/pkg/util/fileutil/writer.go b/pkg/util/fileutil/writer.go index 69c540cf9..e4e8f8c05 100644 --- a/pkg/util/fileutil/writer.go +++ b/pkg/util/fileutil/writer.go @@ -42,7 +42,7 @@ func Validate(payloads []*v1alpha1.File) error { // WritePayloads writes the files to target directory. This helper builds the // atomic writer and converts the v1alpha1.File proto to the FileProjection type // used by the atomic writer. -func WritePayloads(path string, payloads []*v1alpha1.File) error { +func WritePayloads(path string, payloads []*v1alpha1.File, gid int64) error { if err := Validate(payloads); err != nil { return err } @@ -62,8 +62,9 @@ func WritePayloads(path string, payloads []*v1alpha1.File) error { files := make(map[string]FileProjection, len(payloads)) for _, payload := range payloads { files[payload.GetPath()] = FileProjection{ - Data: payload.GetContents(), - Mode: payload.GetMode(), + Data: payload.GetContents(), + Mode: payload.GetMode(), + FsGroup: &gid, } } diff --git a/pkg/util/fileutil/writer_test.go b/pkg/util/fileutil/writer_test.go index 2b9ff69d8..7ddab8086 100644 --- a/pkg/util/fileutil/writer_test.go +++ b/pkg/util/fileutil/writer_test.go @@ -27,6 +27,7 @@ import ( "strings" "testing" + "sigs.k8s.io/secrets-store-csi-driver/pkg/constants" "sigs.k8s.io/secrets-store-csi-driver/pkg/util/runtimeutil" "sigs.k8s.io/secrets-store-csi-driver/provider/v1alpha1" ) @@ -372,7 +373,7 @@ func TestWritePayloads(t *testing.T) { dir := t.TempDir() // check that the first write succeeds and the contents match - if err := WritePayloads(dir, tc.first); err != nil { + if err := WritePayloads(dir, tc.first, constants.NoGID); err != nil { t.Errorf("WritePayload(first) got error: %v", err) } @@ -382,7 +383,7 @@ func TestWritePayloads(t *testing.T) { // check that the second write succeeds and the contents match, // ensuring that the files have the updated values - if err := WritePayloads(dir, tc.second); err != nil { + if err := WritePayloads(dir, tc.second, constants.NoGID); err != nil { t.Errorf("WritePayload(second) got error: %v", err) } @@ -421,7 +422,7 @@ func TestWritePayloads_BackwardCompatible(t *testing.T) { want := []byte("new") - if err := WritePayloads(dir, payload); err != nil { + if err := WritePayloads(dir, payload, constants.NoGID); err != nil { t.Fatalf("could not write new file: %s", err) } diff --git a/test/bats/e2e-provider.bats b/test/bats/e2e-provider.bats index a73b27200..f70d485bc 100644 --- a/test/bats/e2e-provider.bats +++ b/test/bats/e2e-provider.bats @@ -17,6 +17,8 @@ export SECRET_NAME=${SECRET_NAME:-foo} export SECRET_VERSION=${SECRET_VERSION:-"v1"} # default secret value returned by the mock provider export SECRET_VALUE=${SECRET_VALUE:-"secret"} +# default secret mode returned by the mock provider +export SECRET_MODE=${SECRET_MODE:-'"0644"'} # export key vars export KEY_NAME=${KEY_NAME:-fookey} @@ -25,6 +27,8 @@ export KEY_VERSION=${KEY_VERSION:-"v1"} # default key value returned by mock provider. # base64 encoded content comparision is easier in case of very long multiline string. export KEY_VALUE_CONTAINS=${KEY_VALUE:-"LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KVGhpcyBpcyBtb2NrIGtleQotLS0tLUVORCBQVUJMSUMgS0VZLS0tLS0K"} +# defualt version value returned by mock provider +export KEY_MODE=${KEY_MODE:-'"0644"'} # export node selector var export NODE_SELECTOR_OS=$NODE_SELECTOR_OS @@ -33,11 +37,90 @@ export LABEL_VALUE=${LABEL_VALUE:-"test"} # export the secrets-store API version to be used export API_VERSION=$(get_secrets_store_api_version) +export NAMESPACE=${NAMESPACE:-"default"} +export SPC_NAME=${SPC_NAME:-"e2e-provider"} +export POD_NAME=${POD_NAME:-"secrets-store-inline-crd"} +export POD_SECURITY_CONTEXT=${POD_SECURITY_CONTEXT:-""} +export CONTAINER_SECURITY_CONTEXT=${CONTAINER_SECURITY_CONTEXT:-""} # export the token requests audience configured in the CSIDriver # refer to https://kubernetes-csi.github.io/docs/token-requests.html for more information export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) +######################################################### +# begin: Utility functions to perform common operations # +######################################################### +function create_spc() { + envsubst < $BATS_TESTS_DIR/e2e_provider_secretproviderclass.yaml | kubectl -n $NAMESPACE apply -f - + + kubectl -n $NAMESPACE wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io + + cmd="kubectl -n $NAMESPACE get secretproviderclasses.secrets-store.csi.x-k8s.io/$SPC_NAME -o yaml | grep $SPC_NAME" + wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd" +} + +function create_pod() { + envsubst < $BATS_TESTS_DIR/pod-secrets-store-inline-volume-crd.yaml | kubectl -n $NAMESPACE apply -f - + kubectl -n $NAMESPACE wait --for=condition=Ready --timeout=180s pod/$POD_NAME + run kubectl -n $NAMESPACE get pod/$POD_NAME + assert_success +} + +# POD_NAME could have come as a parameter in the below 3 functions. +# But keeping the semantics uniform with other functions where envsubst is used +function read_secret() { + wait_for_process $WAIT_TIME $SLEEP_TIME "kubectl -n $NAMESPACE exec $POD_NAME -- ls /mnt/secrets-store/$SECRET_NAME" + local result=$(kubectl -n $NAMESPACE exec $POD_NAME -- cat /mnt/secrets-store/$SECRET_NAME) + [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] +} + +function read_key() { + result=$(kubectl -n $NAMESPACE exec $POD_NAME -- cat /mnt/secrets-store/$KEY_NAME) + result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) + [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] +} + +function delete_pod() { + # On Linux a failure to unmount the tmpfs will block the pod from being + # deleted. + run kubectl -n $NAMESPACE delete pod $POD_NAME + assert_success + + kubectl -n $NAMESPACE wait --for=delete --timeout=${WAIT_TIME}s pod/$POD_NAME + assert_success + + # Sleep to allow time for logs to propagate. + sleep 10 + + # save debug information to archive in case of failure + archive_info + + # On Windows, the failed unmount calls from: https://github.com/kubernetes-sigs/secrets-store-csi-driver/pull/545 + # do not prevent the pod from being deleted. Search through the driver logs + # for the error. + run bash -c "kubectl -n $NAMESPACE logs -l app=$POD_NAME --tail -1 -c secrets-store -n kube-system | grep '^E.*failed to clean and unmount target path.*$'" + assert_failure +} + +function enable_secret_rotation() { + # enable rotation response in mock server + local curl_pod_name=curl-$(openssl rand -hex 5) + kubectl run ${curl_pod_name} -n rotation --image=curlimages/curl:7.75.0 --labels="test=rotation" -- tail -f /dev/null + kubectl wait -n rotation --for=condition=Ready --timeout=60s pod ${curl_pod_name} + local pod_ip=$(kubectl get pod -n kube-system -l app=csi-secrets-store-e2e-provider -o jsonpath="{.items[0].status.podIP}") + run kubectl exec ${curl_pod_name} -n rotation -- curl http://${pod_ip}:8080/rotation?rotated=true + sleep 60 +} + +function disable_secret_rotation() { + local curl_pod_name=$1 + local pod_ip=$(kubectl get pod -n kube-system -l app=csi-secrets-store-e2e-provider -o jsonpath="{.items[0].status.podIP}") + run kubectl exec ${curl_pod_name} -n rotation -- curl http://${pod_ip}:8080/rotation?rotated=false +} +####################################################### +# end: Utility functions to perform common operations # +####################################################### + @test "setup mock provider validation config" { if [[ -n "${VALIDATE_TOKENS_AUDIENCE}" ]]; then # configure the mock provider to validate the token requests @@ -110,90 +193,81 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) @test "[v1alpha1] deploy e2e-provider secretproviderclass crd" { kubectl create namespace test-v1alpha1 --dry-run=client -o yaml | kubectl apply -f - - - envsubst < $BATS_TESTS_DIR/e2e_provider_v1alpha1_secretproviderclass.yaml | kubectl apply -n test-v1alpha1 -f - - - kubectl wait --for condition=established -n test-v1alpha1 --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io - - cmd="kubectl get secretproviderclasses.secrets-store.csi.x-k8s.io/e2e-provider -n test-v1alpha1 -o yaml | grep e2e-provider" - wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd" + NAMESPACE="test-v1alpha1" API_VERSION="secrets-store.csi.x-k8s.io/v1alpha1" create_spc } @test "[v1alpha1] CSI inline volume test with pod portability" { - envsubst < $BATS_TESTS_DIR/pod-secrets-store-inline-volume-crd.yaml | kubectl apply -n test-v1alpha1 -f - - - kubectl wait --for=condition=Ready -n test-v1alpha1 --timeout=180s pod/secrets-store-inline-crd - - run kubectl get pod/secrets-store-inline-crd -n test-v1alpha1 - assert_success + NAMESPACE="test-v1alpha1" create_pod } @test "[v1alpha1] CSI inline volume test with pod portability - read secret from pod" { - wait_for_process $WAIT_TIME $SLEEP_TIME "kubectl exec secrets-store-inline-crd -n test-v1alpha1 -- cat /mnt/secrets-store/$SECRET_NAME | grep '${SECRET_VALUE}'" - - result=$(kubectl exec secrets-store-inline-crd -n test-v1alpha1 -- cat /mnt/secrets-store/$SECRET_NAME) - [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] + NAMESPACE="test-v1alpha1" read_secret } @test "[v1alpha1] CSI inline volume test with pod portability - read key from pod" { - result=$(kubectl exec secrets-store-inline-crd -n test-v1alpha1 -- cat /mnt/secrets-store/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] + NAMESPACE="test-v1alpha1" read_key } -@test "deploy e2e-provider v1 secretproviderclass crd" { - envsubst < $BATS_TESTS_DIR/e2e_provider_v1_secretproviderclass.yaml | kubectl apply -f - - - kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io +@test "[v1alpha1] CSI inline volume test with pod portability - unmount succeeds" { + NAMESPACE="test-v1alpha1" delete_pod +} - cmd="kubectl get secretproviderclasses.secrets-store.csi.x-k8s.io/e2e-provider -o yaml | grep e2e-provider" - wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd" +@test "deploy e2e-provider v1 secretproviderclass crd" { + create_spc } @test "CSI inline volume test with pod portability" { - envsubst < $BATS_TESTS_DIR/pod-secrets-store-inline-volume-crd.yaml | kubectl apply -f - - - kubectl wait --for=condition=Ready --timeout=180s pod/secrets-store-inline-crd - - run kubectl get pod/secrets-store-inline-crd - assert_success + create_pod } @test "CSI inline volume test with pod portability - read secret from pod" { - wait_for_process $WAIT_TIME $SLEEP_TIME "kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$SECRET_NAME | grep '${SECRET_VALUE}'" - - result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$SECRET_NAME) - [[ "${result//$'\r'}" == "${SECRET_VALUE}" ]] + read_secret } @test "CSI inline volume test with pod portability - read key from pod" { - result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$KEY_NAME) - result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS}) - [[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]] + read_key } @test "CSI inline volume test with pod portability - unmount succeeds" { - # On Linux a failure to unmount the tmpfs will block the pod from being - # deleted. - run kubectl delete pod secrets-store-inline-crd - assert_success - - kubectl wait --for=delete --timeout=${WAIT_TIME}s pod/secrets-store-inline-crd - assert_success + delete_pod +} - # Sleep to allow time for logs to propagate. - sleep 10 +@test "deploy e2e-provider v1 secretproviderclass crd with restricted permissions" { + SPC_NAME="e2e-provider-640" SECRET_MODE='"0640"' KEY_MODE='"0640"' create_spc +} - # save debug information to archive in case of failure - archive_info +@test "Non-root POD with no FSGroup - create" { + SPC_NAME="e2e-provider-640" POD_NAME="non-root-with-no-fsgroup" POD_SECURITY_CONTEXT='"runAsNonRoot": true, "runAsUser": 1004, "runAsGroup": 1004' create_pod +} - # On Windows, the failed unmount calls from: https://github.com/kubernetes-sigs/secrets-store-csi-driver/pull/545 - # do not prevent the pod from being deleted. Search through the driver logs - # for the error. - run bash -c "kubectl logs -l app=secrets-store-csi-driver --tail -1 -c secrets-store -n kube-system | grep '^E.*failed to clean and unmount target path.*$'" +@test "Non-root POD with no FSGroup - Should fail to read non world readable secret" { + # use run here as read_secret will run into errors and we want to assert_failure + POD_NAME="non-root-with-no-fsgroup" run read_secret assert_failure } +@test "Non-root POD with no FSGroup - unmount succeeds" { + POD_NAME="non-root-with-no-fsgroup" delete_pod +} + +@test "Non-root POD with FSGroup - create" { + SPC_NAME="e2e-provider-640" POD_NAME="non-root-with-fsgroup" POD_SECURITY_CONTEXT='"runAsNonRoot": true, "runAsUser": 1004, "runAsGroup": 1004, "fsGroup": 1004' create_pod +} + +@test "Non-root POD with FSGroup - should read non world readable secret" { + POD_NAME="non-root-with-fsgroup" read_secret +} + +@test "Non-root POD with FSGroup - rotated secret should also be readable" { + curl_pod_name=$(enable_secret_rotation) + SECRET_VALUE="rotated" POD_NAME="non-root-with-fsgroup" read_secret + disable_secret_rotation $curl_pod_name +} + +@test "Non-root POD with FSGroup - unmount succeeds" { + POD_NAME="non-root-with-fsgroup" delete_pod +} + @test "Sync with K8s secrets - create deployment" { envsubst < $BATS_TESTS_DIR/e2e_provider_synck8s_v1_secretproviderclass.yaml | kubectl apply -f - @@ -378,7 +452,6 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) } @test "Test auto rotation of mount contents and K8s secrets - Create deployment" { - kubectl create namespace rotation --dry-run=client -o yaml | kubectl apply -f - envsubst < $BATS_TESTS_DIR/rotation/e2e_provider_synck8s_v1_secretproviderclass.yaml | kubectl apply -n rotation -f - envsubst < $BATS_TESTS_DIR/rotation/pod-synck8s-e2e-provider.yaml | kubectl apply -n rotation -f - @@ -396,14 +469,8 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d) [[ "${result//$'\r'}" == "secret" ]] - # enable rotation response in mock server - local curl_pod_name=curl-$(openssl rand -hex 5) - kubectl run ${curl_pod_name} -n rotation --image=curlimages/curl:7.75.0 --labels="test=rotation" -- tail -f /dev/null - kubectl wait -n rotation --for=condition=Ready --timeout=60s pod ${curl_pod_name} - local pod_ip=$(kubectl get pod -n kube-system -l app=csi-secrets-store-e2e-provider -o jsonpath="{.items[0].status.podIP}") - run kubectl exec ${curl_pod_name} -n rotation -- curl http://${pod_ip}:8080/rotation?rotated=true + curl_pod_name=$(enable_secret_rotation) - sleep 60 result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/$SECRET_NAME) [[ "${result//$'\r'}" == "rotated" ]] @@ -412,7 +479,8 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) [[ "${result//$'\r'}" == "rotated" ]] # reset rotation response in mock server for all upgrade tests - run kubectl exec ${curl_pod_name} -n rotation -- curl http://${pod_ip}:8080/rotation?rotated=false + disable_secret_rotation $curl_pod_name + } @test "Validate metrics" { @@ -433,6 +501,10 @@ export VALIDATE_TOKENS_AUDIENCE=$(get_token_requests_audience) kubectl delete ns metrics } +setup_file() { + kubectl create namespace rotation --dry-run=client -o yaml | kubectl apply -f - +} + teardown_file() { if [[ "${INPLACE_UPGRADE_TEST}" != "true" ]]; then #cleanup diff --git a/test/bats/tests/e2e_provider/e2e_provider_v1_secretproviderclass.yaml b/test/bats/tests/e2e_provider/e2e_provider_secretproviderclass.yaml similarity index 76% rename from test/bats/tests/e2e_provider/e2e_provider_v1_secretproviderclass.yaml rename to test/bats/tests/e2e_provider/e2e_provider_secretproviderclass.yaml index 52c457f55..d566e67f3 100644 --- a/test/bats/tests/e2e_provider/e2e_provider_v1_secretproviderclass.yaml +++ b/test/bats/tests/e2e_provider/e2e_provider_secretproviderclass.yaml @@ -1,7 +1,7 @@ apiVersion: $API_VERSION kind: SecretProviderClass metadata: - name: e2e-provider + name: $SPC_NAME spec: provider: e2e-provider parameters: @@ -10,6 +10,8 @@ spec: - | objectName: $SECRET_NAME objectVersion: $SECRET_VERSION + filePermission: $SECRET_MODE - | objectName: $KEY_NAME objectVersion: $KEY_VERSION + filePermission: $KEY_MODE diff --git a/test/bats/tests/e2e_provider/e2e_provider_v1alpha1_secretproviderclass.yaml b/test/bats/tests/e2e_provider/e2e_provider_v1alpha1_secretproviderclass.yaml deleted file mode 100644 index 3bd057b38..000000000 --- a/test/bats/tests/e2e_provider/e2e_provider_v1alpha1_secretproviderclass.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: secrets-store.csi.x-k8s.io/v1alpha1 -kind: SecretProviderClass -metadata: - name: e2e-provider -spec: - provider: e2e-provider - parameters: - objects: | - array: - - | - objectName: $SECRET_NAME - objectVersion: $SECRET_VERSION - - | - objectName: $KEY_NAME - objectVersion: $KEY_VERSION diff --git a/test/bats/tests/e2e_provider/pod-secrets-store-inline-volume-crd.yaml b/test/bats/tests/e2e_provider/pod-secrets-store-inline-volume-crd.yaml index 0f8e2e619..ee9e1ef49 100644 --- a/test/bats/tests/e2e_provider/pod-secrets-store-inline-volume-crd.yaml +++ b/test/bats/tests/e2e_provider/pod-secrets-store-inline-volume-crd.yaml @@ -1,8 +1,9 @@ kind: Pod apiVersion: v1 metadata: - name: secrets-store-inline-crd + name: ${POD_NAME} spec: + securityContext: { ${POD_SECURITY_CONTEXT} } terminationGracePeriodSeconds: 0 containers: - image: registry.k8s.io/e2e-test-images/busybox:1.29-4 @@ -15,12 +16,13 @@ spec: - name: secrets-store-inline mountPath: "/mnt/secrets-store" readOnly: true + securityContext: { ${CONTAINER_SECURITY_CONTEXT} } volumes: - name: secrets-store-inline csi: driver: secrets-store.csi.k8s.io readOnly: true volumeAttributes: - secretProviderClass: "e2e-provider" + secretProviderClass: $SPC_NAME nodeSelector: kubernetes.io/os: $NODE_SELECTOR_OS diff --git a/test/e2eprovider/server/server.go b/test/e2eprovider/server/server.go index 1c60079d6..67787c528 100644 --- a/test/e2eprovider/server/server.go +++ b/test/e2eprovider/server/server.go @@ -26,6 +26,7 @@ import ( "net" "net/http" "os" + "strconv" "strings" "sync" @@ -165,6 +166,15 @@ func (s *Server) Mount(ctx context.Context, req *v1alpha1.MountRequest) (*v1alph return nil, fmt.Errorf("failed to get secret, error: %w", err) } + klog.InfoS("Secret Object with", "name", mockSecretsStoreObject.ObjectName, "permission", mockSecretsStoreObject.FilePermission) + if len(mockSecretsStoreObject.FilePermission) > 0 { + mode, err := strconv.ParseUint(mockSecretsStoreObject.FilePermission, 8, 32) + if err != nil || mode > 511 { + return nil, fmt.Errorf("invalid filePermission: %s, error: %w for file: %s", mockSecretsStoreObject.FilePermission, err, mockSecretsStoreObject.ObjectName) + } + secretFile.Mode = int32(mode) + klog.InfoS("Set file mode", "file", secretFile.Path, "mode", os.FileMode(mode)) + } resp.Files = append(resp.Files, secretFile) resp.ObjectVersion = append(resp.ObjectVersion, version) } diff --git a/test/e2eprovider/types/types.go b/test/e2eprovider/types/types.go index 9176071ff..7007587d5 100644 --- a/test/e2eprovider/types/types.go +++ b/test/e2eprovider/types/types.go @@ -25,6 +25,8 @@ type MockSecretsStoreObject struct { ObjectName string `json:"objectName" yaml:"objectName"` // the version of the secret objects ObjectVersion string `json:"objectVersion" yaml:"objectVersion"` + // the filePermission of the secret object + FilePermission string `json:"filePermission,omitempty" yaml:"filePermission,omitempty"` } // StringArray holds a list of objects