feat: Adding support to the interceptor proxy for TLS on the wire (kedacore#928)

* feat: Adding support to the interceptor proxy for TLS on the wire

Signed-off-by: Joe Wogan <joe.wogan@10xbanking.com>

* chore: adding e2e tests and tidying up changes

Signed-off-by: Joe Wogan <joe.wogan@10xbanking.com>

* chore: changing interceptor tls port for e2e tests

Signed-off-by: Joe Wogan <joe.wogan@10xbanking.com>

---------

Signed-off-by: Joe Wogan <joe.wogan@10xbanking.com>

Author: zorocloud

.gitignore

@@ -355,3 +355,10 @@ admin/Cargo.lock
 
 /target
 .envrc
+
+# locally generated certs for testing TLS
+*.crt
+*.pem
+*.csr
+*.srl
+*.ext

CHANGELOG.md

@@ -29,7 +29,7 @@ This changelog keeps track of work items that have been completed and are ready
 
 ### Improvements
 
-- **General**: TODO ([#TODO](https://github.com/kedacore/http-add-on/issues/TODO))
+- **General**: Add configurable TLS on the wire support to the interceptor proxy ([#907](https://github.com/kedacore/http-add-on/issues/907))
 
 ### Fixes
 

Makefile

@@ -32,6 +32,17 @@ GO_LDFLAGS="-X github.com/kedacore/http-add-on/pkg/build.version=${VERSION} -X g
 GIT_COMMIT  ?= $(shell git rev-list -1 HEAD)
 GIT_COMMIT_SHORT  ?= $(shell git rev-parse --short HEAD)
 
+define DOMAINS
+basicConstraints=CA:FALSE
+keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = localhost
+DNS.2 = *.keda
+DNS.3 = *.interceptor-tls-test-ns
+endef
+export DOMAINS
+
 # Build targets
 
 build-operator:
@@ -45,8 +56,22 @@ build-scaler:
 
 build: build-operator build-interceptor build-scaler
 
+# generate certs for local unit and e2e tests
+rootca-test-certs:
+	mkdir -p certs
+	openssl req -x509 -nodes -new -sha256 -days 1024 -newkey rsa:2048 -keyout certs/RootCA.key -out certs/RootCA.pem -subj "/C=US/CN=Keda-Root-CA"
+	openssl x509 -outform pem -in certs/RootCA.pem -out certs/RootCA.crt
+
+test-certs: rootca-test-certs
+	echo "$$DOMAINS" > certs/domains.ext
+	openssl req -new -nodes -newkey rsa:2048 -keyout certs/tls.key -out certs/tls.csr -subj "/C=US/ST=KedaState/L=KedaCity/O=Keda-Certificates/CN=keda.local"
+	openssl x509 -req -sha256 -days 1024 -in certs/tls.csr -CA certs/RootCA.pem -CAkey certs/RootCA.key -CAcreateserial -extfile certs/domains.ext -out certs/tls.crt
+
+clean-test-certs:
+	rm -r certs || true
+
 # Test targets
-test: fmt vet
+test: fmt vet test-certs
 	go test ./...
 
 e2e-test:
@@ -156,6 +181,12 @@ deploy: manifests kustomize ## Deploy to the K8s cluster specified in ~/.kube/co
 	cd config/interceptor && \
 	$(KUSTOMIZE) edit add patch --path e2e-test/scaledobject.yaml --group keda.sh --kind ScaledObject --name interceptor --version v1alpha1
 
+	cd config/interceptor && \
+	$(KUSTOMIZE) edit add patch --path tls/deployment.yaml --group apps --kind Deployment --name interceptor --version v1
+
+	cd config/interceptor && \
+	$(KUSTOMIZE) edit add patch --path tls/proxy.service.yaml --kind Service --name interceptor-proxy --version v1
+
 	cd config/scaler && \
 	$(KUSTOMIZE) edit set image ghcr.io/kedacore/http-add-on-scaler=${IMAGE_SCALER_VERSIONED_TAG}
 
@@ -174,3 +205,8 @@ kind-load:
 	kind load docker-image ghcr.io/kedacore/http-add-on-operator:${VERSION}
 	kind load docker-image ghcr.io/kedacore/http-add-on-interceptor:${VERSION}
 	kind load docker-image ghcr.io/kedacore/http-add-on-scaler:${VERSION}
+
+k3d-import:
+	k3d image import ghcr.io/kedacore/http-add-on-operator:main
+	k3d image import ghcr.io/kedacore/http-add-on-interceptor:main
+	k3d image import ghcr.io/kedacore/http-add-on-scaler:main

config/interceptor/tls/deployment.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: interceptor
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: interceptor
+        ports:
+        - name: proxy-tls
+          containerPort: 8443
+        env:
+        - name: KEDA_HTTP_PROXY_TLS_ENABLED
+          value: "true"
+        - name: KEDA_HTTP_PROXY_TLS_CERT_PATH
+          value: "/certs/tls.crt"
+        - name: KEDA_HTTP_PROXY_TLS_KEY_PATH
+          value: "/certs/tls.key"
+        - name: KEDA_HTTP_PROXY_TLS_PORT
+          value: "8443"
+        volumeMounts:
+          - readOnly: true
+            mountPath: "/certs"
+            name: certs
+      volumes:
+        - name: certs
+          secret:
+            secretName: keda-tls

config/interceptor/tls/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- deployment.yaml
+- proxy.service.yaml

config/interceptor/tls/proxy.service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: interceptor-proxy
+spec:
+  type: ClusterIP
+  ports:
+  - name: proxy-tls
+    protocol: TCP
+    port: 8443
+    targetPort: proxy-tls

docs/operate.md

@@ -19,3 +19,9 @@ The OTEL exporter can be enabled by setting the `KEDA_HTTP_OTEL_HTTP_EXPORTER_EN
 If the collector is exposed on a unsecured endpoint then you can set the `KEDA_HTTP_OTEL_HTTP_COLLECTOR_INSECURE` environment variable to `true` (`false` by default) which will disable client security on the exporter.
 
 If you need to provide any headers such as authentication details in order to utilise your OTEL collector you can add them into the `KEDA_HTTP_OTEL_HTTP_HEADERS` environment variable. The frequency at which the metrics are exported can be configured by setting `KEDA_HTTP_OTEL_METRIC_EXPORT_INTERVAL` to the number of seconds you require between each export interval (`30` by default).
+
+# Configuring TLS for the KEDA HTTP Add-on interceptor proxy
+
+The interceptor proxy has the ability to run both a HTTP and HTTPS server simultaneously to allow you to scale workloads that use either protocol. By default, the interceptor proxy will only serve over HTTP, but this behavior can be changed by configuring the appropriate environment variables on the deployment.
+
+The TLS server can be enabled by setting the environment variable `KEDA_HTTP_PROXY_TLS_ENABLED` to `true` on the interceptor deployment (`false` by default). The TLS server will start on port `8443` by default, but this can be configured by setting `KEDA_HTTP_PROXY_TLS_PORT` to your desired port number. The TLS server will require valid TLS certificates to start, the path to the certificates can be configured via the `KEDA_HTTP_PROXY_TLS_CERT_PATH` and `KEDA_HTTP_PROXY_TLS_KEY_PATH` environment variables (`/certs/tls.crt` and `/certs/tls.key` by default).

interceptor/config/serving.go

@@ -34,6 +34,15 @@ type Serving struct {
 	//
 	// This is the interval (in milliseconds) representing how often to do a fetch
 	EndpointsCachePollIntervalMS int `envconfig:"KEDA_HTTP_ENDPOINTS_CACHE_POLLING_INTERVAL_MS" default:"250"`
+	// ProxyTLSEnabled is a flag to specify whether the interceptor proxy should
+	// be running using a TLS enabled server
+	ProxyTLSEnabled bool `envconfig:"KEDA_HTTP_PROXY_TLS_ENABLED" default:"false"`
+	// TLSCertPath is the path to read the certificate file from for the TLS server
+	TLSCertPath string `envconfig:"KEDA_HTTP_PROXY_TLS_CERT_PATH" default:"/certs/tls.crt"`
+	// TLSKeyPath is the path to read the private key file from for the TLS server
+	TLSKeyPath string `envconfig:"KEDA_HTTP_PROXY_TLS_KEY_PATH" default:"/certs/tls.key"`
+	// TLSPort is the port that the server should serve on if TLS is enabled
+	TLSPort int `envconfig:"KEDA_HTTP_PROXY_TLS_PORT" default:"8443"`
 }
 
 // Parse parses standard configs using envconfig and returns a pointer to the

interceptor/main.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"flag"
 	"fmt"
@@ -68,6 +70,7 @@ func main() {
 
 	proxyPort := servingCfg.ProxyPort
 	adminPort := servingCfg.AdminPort
+	proxyTLSEnabled := servingCfg.ProxyTLSEnabled
 
 	// setup the configured metrics collectors
 	metrics.NewMetricsCollectors(metricsCfg)
@@ -160,12 +163,29 @@ func main() {
 		})
 	}
 
-	// start the proxy server. this is the server that
+	// start the proxy servers. This is the server that
 	// accepts, holds and forwards user requests
+	// start a proxy server with TLS
+	if proxyTLSEnabled {
+		eg.Go(func() error {
+			proxyTLSConfig := map[string]string{"certificatePath": servingCfg.TLSCertPath, "keyPath": servingCfg.TLSKeyPath}
+			proxyTLSPort := servingCfg.TLSPort
+
+			setupLog.Info("starting the proxy server with TLS enabled", "port", proxyTLSPort)
+
+			if err := runProxyServer(ctx, ctrl.Log, queues, waitFunc, routingTable, timeoutCfg, proxyTLSPort, proxyTLSEnabled, proxyTLSConfig); !util.IsIgnoredErr(err) {
+				setupLog.Error(err, "tls proxy server failed")
+				return err
+			}
+			return nil
+		})
+	}
+
+	// start a proxy server without TLS.
 	eg.Go(func() error {
-		setupLog.Info("starting the proxy server", "port", proxyPort)
+		setupLog.Info("starting the proxy server with TLS disabled", "port", proxyPort)
 
-		if err := runProxyServer(ctx, ctrl.Log, queues, waitFunc, routingTable, timeoutCfg, proxyPort); !util.IsIgnoredErr(err) {
+		if err := runProxyServer(ctx, ctrl.Log, queues, waitFunc, routingTable, timeoutCfg, proxyPort, false, nil); !util.IsIgnoredErr(err) {
 			setupLog.Error(err, "proxy server failed")
 			return err
 		}
@@ -199,7 +219,7 @@ func runAdminServer(
 
 	addr := fmt.Sprintf("0.0.0.0:%d", port)
 	lggr.Info("admin server starting", "address", addr)
-	return kedahttp.ServeContext(ctx, addr, adminServer)
+	return kedahttp.ServeContext(ctx, addr, adminServer, false, nil)
 }
 
 func runMetricsServer(
@@ -209,7 +229,7 @@ func runMetricsServer(
 ) error {
 	lggr.Info("starting the prometheus metrics server", "port", metricsCfg.OtelPrometheusExporterPort, "path", "/metrics")
 	addr := fmt.Sprintf("0.0.0.0:%d", metricsCfg.OtelPrometheusExporterPort)
-	return kedahttp.ServeContext(ctx, addr, promhttp.Handler())
+	return kedahttp.ServeContext(ctx, addr, promhttp.Handler(), false, nil)
 }
 
 func runProxyServer(
@@ -220,6 +240,8 @@ func runProxyServer(
 	routingTable routing.Table,
 	timeouts *config.Timeouts,
 	port int,
+	tlsEnabled bool,
+	tlsConfig map[string]string,
 ) error {
 	dialer := kedanet.NewNetDialer(timeouts.Connect, timeouts.KeepAlive)
 	dialContextFunc := kedanet.DialContextWithRetry(dialer, timeouts.DefaultBackoff())
@@ -229,12 +251,33 @@ func runProxyServer(
 	})
 	go probeHandler.Start(ctx)
 
+	tlsCfg := tls.Config{}
+	if tlsEnabled {
+		caCert, err := os.ReadFile(tlsConfig["certificatePath"])
+		if err != nil {
+			logger.Error(fmt.Errorf("error reading file from TLSCertPath"), "error", err)
+			os.Exit(1)
+		}
+		caCertPool := x509.NewCertPool()
+		caCertPool.AppendCertsFromPEM(caCert)
+		cert, err := tls.LoadX509KeyPair(tlsConfig["certificatePath"], tlsConfig["keyPath"])
+
+		if err != nil {
+			logger.Error(fmt.Errorf("error creating TLS configuration for proxy server"), "error", err)
+			os.Exit(1)
+		}
+
+		tlsCfg.RootCAs = caCertPool
+		tlsCfg.Certificates = []tls.Certificate{cert}
+	}
+
 	var upstreamHandler http.Handler
 	upstreamHandler = newForwardingHandler(
 		logger,
 		dialContextFunc,
 		waitFunc,
 		newForwardingConfigFromTimeouts(timeouts),
+		&tlsCfg,
 	)
 	upstreamHandler = middleware.NewCountingMiddleware(
 		q,
@@ -246,6 +289,7 @@ func runProxyServer(
 		routingTable,
 		probeHandler,
 		upstreamHandler,
+		tlsEnabled,
 	)
 	rootHandler = middleware.NewLogging(
 		logger,
@@ -258,5 +302,5 @@ func runProxyServer(
 
 	addr := fmt.Sprintf("0.0.0.0:%d", port)
 	logger.Info("proxy server starting", "address", addr)
-	return kedahttp.ServeContext(ctx, addr, rootHandler)
+	return kedahttp.ServeContext(ctx, addr, rootHandler, tlsEnabled, tlsConfig)
 }