Added support for external etcd to CacheServer

config/crd/bases/operator.kcp.io_cacheservers.yaml

@@ -1386,6 +1386,42 @@ spec:
                         type: object
                     type: object
                 type: object
+              etcd:
+                description: |-
+                  Optional: Etcd configures an external etcd connection for this cache server.
+                  If not provided, an embedded etcd is used.
+                properties:
+                  endpoints:
+                    description: Endpoints is a list of http urls at which etcd nodes
+                      are available. The expected format is "https://etcd-hostname:2379".
+                    items:
+                      type: string
+                    type: array
+                  tlsConfig:
+                    description: ClientCert configures the client certificate used
+                      to access etcd.
+                    properties:
+                      secretRef:
+                        description: SecretRef is the reference to a v1.Secret object
+                          that contains the TLS certificate.
+                        properties:
+                          name:
+                            default: ""
+                            description: |-
+                              Name of the referent.
+                              This field is effectively required, but due to backwards compatibility is
+                              allowed to be empty. Instances of this type with an empty value here are
+                              almost certainly wrong.
+                              More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                            type: string
+                        type: object
+                        x-kubernetes-map-type: atomic
+                    required:
+                    - secretRef
+                    type: object
+                required:
+                - endpoints
+                type: object
               image:
                 description: 'Optional: Image overwrites the container image used
                   to deploy the cache server.'

hack/ci/run-e2e-tests.sh

@@ -19,6 +19,11 @@ set -euo pipefail
 cd "$(dirname "$0")/../.."
 source hack/lib.sh
 
+# KCP_TAG (when coming from .prow.yaml) is set to release-* branch name.
+# We stash it into KCP_RELEASE because KCP_TAG is overridden later to a specific
+# git hash of that branch.
+export KCP_RELEASE="${KCP_TAG:-}"
+
 # For periodics especially it's important that we output what versions exactly
 # we're testing against.
 if [ -n "${KCP_TAG:-}" ]; then

internal/resources/cacheserver/deployment.go

@@ -18,6 +18,7 @@ package cacheserver
 
 import (
 	"fmt"
+	"strings"
 
 	"k8c.io/reconciler/pkg/reconciling"
 
@@ -37,8 +38,7 @@ const (
 	ServerContainerName = "cache-server"
 
 	// embeddedEtcdStoragePath is the emptyDir path in the Deployment where the
-	// etcd data is temporarily stored. kcp's cache-server as of v0.30 does not
-	// support external etcd yet.
+	// embedded etcd data is temporarily stored.
 	embeddedEtcdStoragePath = "/var/etcd"
 )
 
@@ -62,13 +62,15 @@ func getCertificateMountPath(certName operatorv1alpha1.Certificate) string {
 	return fmt.Sprintf("/etc/cache-server/tls/%s", certName)
 }
 
+func getEtcdCertificateMountPath() string {
+	return "/etc/cache-server/etcd"
+}
+
 func getCAMountPath(caName operatorv1alpha1.CA) string {
 	return fmt.Sprintf("/etc/cache-server/tls/ca/%s", caName)
 }
 
 func DeploymentReconciler(server *operatorv1alpha1.CacheServer) reconciling.NamedDeploymentReconcilerFactory {
-	const etcdScratchVolume = "etcd-scratch"
-
 	return func() (string, reconciling.DeploymentReconciler) {
 		return resources.GetCacheServerDeploymentName(server), func(dep *appsv1.Deployment) (*appsv1.Deployment, error) {
 			labels := resources.GetCacheServerResourceLabels(server)
@@ -84,28 +86,12 @@ func DeploymentReconciler(server *operatorv1alpha1.CacheServer) reconciling.Name
 
 			dep.Spec.Template.SetLabels(labels)
 
-			secretMounts := []utils.SecretMount{{
-				VolumeName: "serving-cert",
-				SecretName: resources.GetCacheServerCertificateName(server, operatorv1alpha1.ServerCertificate),
-				MountPath:  getCertificateMountPath(operatorv1alpha1.ServerCertificate),
-			}}
-
 			// TODO: Why do we discard the imagePullSecrets?
 			image, _ := resources.GetImageSettings(server.Spec.Image)
 
-			args := getArgs(server)
-			volumes := []corev1.Volume{{
-				Name: etcdScratchVolume,
-				VolumeSource: corev1.VolumeSource{
-					EmptyDir: &corev1.EmptyDirVolumeSource{},
-				},
-			}}
-			volumeMounts := []corev1.VolumeMount{{
-				Name:      etcdScratchVolume,
-				MountPath: embeddedEtcdStoragePath,
-			}}
+			volumes, volumeMounts := getVolumeMounts(server)
 
-			for _, sm := range secretMounts {
+			for _, sm := range getSecretMounts(server) {
 				v, vm := sm.Build()
 				volumes = append(volumes, v)
 				volumeMounts = append(volumeMounts, vm)
@@ -115,7 +101,7 @@ func DeploymentReconciler(server *operatorv1alpha1.CacheServer) reconciling.Name
 				Name:         ServerContainerName,
 				Image:        image,
 				Command:      []string{"/cache-server"},
-				Args:         args,
+				Args:         getArgs(server),
 				VolumeMounts: volumeMounts,
 				Resources:    defaultResourceRequirements,
 				SecurityContext: &corev1.SecurityContext{
@@ -188,18 +174,72 @@ func DeploymentReconciler(server *operatorv1alpha1.CacheServer) reconciling.Name
 	}
 }
 
+func getVolumeMounts(server *operatorv1alpha1.CacheServer) (volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) {
+	const etcdScratchVolume = "etcd-scratch"
+
+	if server.Spec.Etcd == nil {
+		volumes = []corev1.Volume{{
+			Name: etcdScratchVolume,
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
+		}}
+		volumeMounts = []corev1.VolumeMount{{
+			Name:      etcdScratchVolume,
+			MountPath: embeddedEtcdStoragePath,
+		}}
+	}
+
+	return
+}
+
 func getArgs(server *operatorv1alpha1.CacheServer) []string {
 	args := []string{
-		// Configure (lack of) persistence.
-		"--root-directory=",
-		fmt.Sprintf("--embedded-etcd-directory=%s", embeddedEtcdStoragePath),
-
 		// Certificate flags (server, service account signing).
 		fmt.Sprintf("--tls-cert-file=%s/tls.crt", getCertificateMountPath(operatorv1alpha1.ServerCertificate)),
 		fmt.Sprintf("--tls-private-key-file=%s/tls.key", getCertificateMountPath(operatorv1alpha1.ServerCertificate)),
+		// Configure (lack of) persistence.
+		"--root-directory=",
+	}
+
+	if server.Spec.Etcd == nil {
+		// The CacheServer is configured with an embedded etcd store.
+		args = append(args,
+			fmt.Sprintf("--embedded-etcd-directory=%s", embeddedEtcdStoragePath),
+		)
+	} else {
+		// The CacheServer is configured with a dedicated etcd store.
+		args = append(args,
+			fmt.Sprintf("--etcd-servers=%s", strings.Join(server.Spec.Etcd.Endpoints, ",")),
+		)
+		if server.Spec.Etcd.TLSConfig != nil {
+			args = append(args,
+				fmt.Sprintf("--etcd-cafile=%s/ca.crt", getEtcdCertificateMountPath()),
+				fmt.Sprintf("--etcd-certfile=%s/tls.crt", getEtcdCertificateMountPath()),
+				fmt.Sprintf("--etcd-keyfile=%s/tls.key", getEtcdCertificateMountPath()),
+			)
+		}
 	}
 
 	args = append(args, utils.GetLoggingArgs(server.Spec.Logging)...)
 
 	return args
 }
+
+func getSecretMounts(server *operatorv1alpha1.CacheServer) []utils.SecretMount {
+	secretMounts := []utils.SecretMount{{
+		VolumeName: "serving-cert",
+		SecretName: resources.GetCacheServerCertificateName(server, operatorv1alpha1.ServerCertificate),
+		MountPath:  getCertificateMountPath(operatorv1alpha1.ServerCertificate),
+	}}
+
+	if server.Spec.Etcd != nil && server.Spec.Etcd.TLSConfig != nil {
+		secretMounts = append(secretMounts, utils.SecretMount{
+			VolumeName: "etcd-cert",
+			SecretName: server.Spec.Etcd.TLSConfig.SecretRef.Name,
+			MountPath:  getEtcdCertificateMountPath(),
+		})
+	}
+
+	return secretMounts
+}

sdk/apis/operator/v1alpha1/cacheserver_types.go

@@ -45,6 +45,10 @@ type CacheServerSpec struct {
 
 	// Optional: DeploymentTemplate configures the Kubernetes Deployment created for this cache server.
 	DeploymentTemplate *DeploymentTemplate `json:"deploymentTemplate,omitempty"`
+
+	// Optional: Etcd configures an external etcd connection for this cache server.
+	// If not provided, an embedded etcd is used.
+	Etcd *EtcdConfig `json:"etcd,omitempty"`
 }
 
 // CacheServerStatus defines the observed state of CacheServer

test/e2e/cacheserver/cacheserver_test.go

@@ -92,3 +92,68 @@ func TestCacheWithRootShard(t *testing.T) {
 		t.Fatalf("Failed to list secrets in kcp: %v", err)
 	}
 }
+
+func TestCacheWithExternalEtcdAndRootShard(t *testing.T) {
+	switch utils.GetKcpRelease() {
+	// We skip release "" because it currently points to kcp 0.30 which
+	// doesn't currently support cache-server with an external etcd cluster.
+	case "release-0.28", "release-0.29", "release-30", "": // TODO(gman0): remove the empty ("") kcp release!
+		t.Skip("running external etcd with cache-server is only supported with kcp >=0.31")
+		return
+	}
+
+	ctrlruntime.SetLogger(logr.Discard())
+
+	client := utils.GetKubeClient(t)
+	ctx := context.Background()
+
+	// create namespace
+	namespace := utils.CreateSelfDestructingNamespace(t, ctx, client, "cache-with-etcd-rootshard")
+
+	// deploy the cache server
+	cacheServer := utils.DeployCacheServerWithExternalEtcd(ctx, t, client, namespace.Name)
+
+	// deploy a root shard that uses our cache
+	rootShard := utils.DeployRootShard(ctx, t, client, namespace.Name, "example.localhost", func(rs *operatorv1alpha1.RootShard) {
+		rs.Spec.Cache.Reference = &corev1.LocalObjectReference{
+			Name: cacheServer.Name,
+		}
+	})
+
+	// create a kubeconfig to access the root shard
+	configSecretName := fmt.Sprintf("%s-shard-kubeconfig", rootShard.Name)
+
+	rsConfig := operatorv1alpha1.Kubeconfig{}
+	rsConfig.Name = configSecretName
+	rsConfig.Namespace = namespace.Name
+
+	rsConfig.Spec = operatorv1alpha1.KubeconfigSpec{
+		Target: operatorv1alpha1.KubeconfigTarget{
+			RootShardRef: &corev1.LocalObjectReference{
+				Name: rootShard.Name,
+			},
+		},
+		Username: "e2e",
+		Validity: metav1.Duration{Duration: 2 * time.Hour},
+		SecretRef: corev1.LocalObjectReference{
+			Name: configSecretName,
+		},
+		Groups: []string{"system:masters"},
+	}
+
+	t.Log("Creating kubeconfig for RootShard...")
+	if err := client.Create(ctx, &rsConfig); err != nil {
+		t.Fatal(err)
+	}
+	utils.WaitForObject(t, ctx, client, &corev1.Secret{}, types.NamespacedName{Namespace: rsConfig.Namespace, Name: rsConfig.Spec.SecretRef.Name})
+
+	t.Log("Connecting to RootShard...")
+	rootShardClient := utils.ConnectWithKubeconfig(t, ctx, client, namespace.Name, rsConfig.Name, logicalcluster.NewPath("root"))
+
+	// proof of life: list something every logicalcluster in kcp has
+	t.Log("Should be able to list Secrets.")
+	secrets := &corev1.SecretList{}
+	if err := rootShardClient.List(ctx, secrets); err != nil {
+		t.Fatalf("Failed to list secrets in kcp: %v", err)
+	}
+}

test/utils/deploy.go

@@ -284,3 +284,15 @@ func DeployCacheServer(ctx context.Context, t *testing.T, client ctrlruntimeclie
 
 	return server
 }
+
+func DeployCacheServerWithExternalEtcd(ctx context.Context, t *testing.T, client ctrlruntimeclient.Client, namespace string, patches ...func(*operatorv1alpha1.CacheServer)) operatorv1alpha1.CacheServer {
+	t.Helper()
+
+	etcd := DeployEtcd(t, "kachy-etcd", namespace)
+
+	return DeployCacheServer(ctx, t, client, namespace, append(patches, func(server *operatorv1alpha1.CacheServer) {
+		server.Spec.Etcd = &operatorv1alpha1.EtcdConfig{
+			Endpoints: []string{etcd},
+		}
+	})...)
+}

test/utils/utils.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"os"
 	"os/exec"
 	"regexp"
 	"strconv"
@@ -308,3 +309,7 @@ func changeClusterInURL(u string, newCluster logicalcluster.Path) string {
 	// (i.e. accessing the cluster resource X in the kcp cluster also called X)
 	return strings.Replace(u, matches[0], newPath, 1)
 }
+
+func GetKcpRelease() string {
+	return os.Getenv("KCP_RELEASE")
+}