Skip to content

Commit 12fd2c5

Browse files
bma13bma13
committed
Support of storage tls connection has been added
1 parent da8e30b commit 12fd2c5

File tree

12 files changed

+398
-38
lines changed

12 files changed

+398
-38
lines changed

‎e2e/tests/data/ca.crt

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIC/TCCAeWgAwIBAgIRAKt07W/6Gy+wIe5lk+0YyqwwDQYJKoZIhvcNAQELBQAw
3+
FzEVMBMGA1UEAxMMdGVzdC1yb290LWNhMCAXDTI0MDQxMjA4MTM0NFoYDzIwNTQw
4+
NDA1MDgxMzQ0WjAXMRUwEwYDVQQDEwx0ZXN0LXJvb3QtY2EwggEiMA0GCSqGSIb3
5+
DQEBAQUAA4IBDwAwggEKAoIBAQCs4c67HN45wf9jokQLdxDsfLUO5I3FiPVE4uWZ
6+
Ma2zNSL2pMJBk95Vmj6pP/3HA6llUm3flVotzVzHh3C0j/WBZf6YE31eWlyMokuE
7+
uLAGfKw/qL+gqC6Phoa72f9kJwnGXsVMDZijAEyqNquLZwgkK+4jgQcVhpGi/3ws
8+
fop0qYVcK5LKAT5lGSx0MEuW74jheLDlscMsmUqVl2SCWRC/UGY+nUOTpcKK8228
9+
Corc+DEFstqOIXGH9n/k0ZmBxjh8eU4IRp+LiDcB6x/yI4edAYJK/mnejmSA2i6a
10+
K2mSzCJfBSVnDxwiGWY6xm8eAh6MaDU6iuqqkFFPltGl90CRAgMBAAGjQjBAMA4G
11+
A1UdDwEB/wQEAwICpDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTFXnzlk4tO
12+
SosBlUEM7AiDOYuW7TANBgkqhkiG9w0BAQsFAAOCAQEAJHTeKc1ySltDwwINFVp1
13+
z5kFlIMyp3l146xn6qT5VWzYP4dZWdJz3gjAML56HRCHNe6B3MijjQY8sRObD5YI
14+
589xpEhLMr+JR/DmU3Yol0XGILUdZ6TeK6FK+U3gYJdy3U39rcV2usEGfN5SRV4b
15+
rUZg8asLFWPY7cdWBNIkF2yuJcF6PIpnuhzbfiEtOZ9ucvgnc62XPDnIuMSBKojG
16+
Bj7QfqadEddSOztZFL00FZULIwSVE/8o0+HQvTBGjbZMuvuSBTJswujYUkD5Sy2O
17+
wfc+dMJ6dbcDZR+5Q1kLEg+Oq7jjx/BTS35Axo1fPxO/WumJppTUmuEbFtEoX47o
18+
fQ==
19+
-----END CERTIFICATE-----

‎e2e/tests/data/storage-block-4-2-config-tls.yaml

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
static_erasure: block-4-2
2+
host_configs:
3+
- drive:
4+
- path: SectorMap:1:1
5+
type: SSD
6+
host_config_id: 1
7+
domains_config:
8+
domain:
9+
- name: Root
10+
storage_pool_types:
11+
- kind: ssd
12+
pool_config:
13+
box_id: 1
14+
erasure_species: block-4-2
15+
kind: ssd
16+
pdisk_filter:
17+
- property:
18+
- type: SSD
19+
vdisk_kind: Default
20+
state_storage:
21+
- ring:
22+
node: [1, 2, 3, 4, 5, 6, 7, 8]
23+
nto_select: 5
24+
ssid: 1
25+
table_service_config:
26+
sql_version: 1
27+
actor_system_config:
28+
executor:
29+
- name: System
30+
threads: 1
31+
type: BASIC
32+
- name: User
33+
threads: 1
34+
type: BASIC
35+
- name: Batch
36+
threads: 1
37+
type: BASIC
38+
- name: IO
39+
threads: 1
40+
time_per_mailbox_micro_secs: 100
41+
type: IO
42+
- name: IC
43+
spin_threshold: 10
44+
threads: 4
45+
time_per_mailbox_micro_secs: 100
46+
type: BASIC
47+
scheduler:
48+
progress_threshold: 10000
49+
resolution: 256
50+
spin_threshold: 0
51+
blob_storage_config:
52+
service_set:
53+
groups:
54+
- erasure_species: block-4-2
55+
rings:
56+
- fail_domains:
57+
- vdisk_locations:
58+
- node_id: storage-0
59+
pdisk_category: SSD
60+
path: SectorMap:1:1
61+
- vdisk_locations:
62+
- node_id: storage-1
63+
pdisk_category: SSD
64+
path: SectorMap:1:1
65+
- vdisk_locations:
66+
- node_id: storage-2
67+
pdisk_category: SSD
68+
path: SectorMap:1:1
69+
- vdisk_locations:
70+
- node_id: storage-3
71+
pdisk_category: SSD
72+
path: SectorMap:1:1
73+
- vdisk_locations:
74+
- node_id: storage-4
75+
pdisk_category: SSD
76+
path: SectorMap:1:1
77+
- vdisk_locations:
78+
- node_id: storage-5
79+
pdisk_category: SSD
80+
path: SectorMap:1:1
81+
- vdisk_locations:
82+
- node_id: storage-6
83+
pdisk_category: SSD
84+
path: SectorMap:1:1
85+
- vdisk_locations:
86+
- node_id: storage-7
87+
pdisk_category: SSD
88+
path: SectorMap:1:1
89+
channel_profile_config:
90+
profile:
91+
- channel:
92+
- erasure_species: block-4-2
93+
pdisk_category: 1
94+
storage_pool_kind: ssd
95+
- erasure_species: block-4-2
96+
pdisk_category: 1
97+
storage_pool_kind: ssd
98+
- erasure_species: block-4-2
99+
pdisk_category: 1
100+
storage_pool_kind: ssd
101+
profile_id: 0
102+
grpc_config:
103+
start_grpc_proxy: true
104+
ssl_port: 2135
105+
ca: /tls/grpc/ca.crt
106+
cert: /tls/grpc/tls.crt
107+
key: /tls/grpc/tls.key
108+
grpc_memory_quota_bytes: '1073741824'
109+
host: '[::]'
110+
keep_alive_enable: true
111+
keep_alive_idle_timeout_trigger_sec: 90
112+
keep_alive_max_probe_count: 3
113+
keep_alive_probe_interval_sec: 10
114+
services: [legacy, yql, scripting, cms, discovery, monitoring, import, export, locking, maintenance]
115+
streaming_config: {enable_output_streams: true}

‎e2e/tests/data/tls.crt

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIDYDCCAkigAwIBAgIRAKHQc3G24VDsWff5qQxyl3EwDQYJKoZIhvcNAQELBQAw
3+
FzEVMBMGA1UEAxMMdGVzdC1yb290LWNhMB4XDTI0MDQxMjA4MTgxOVoXDTM5MDQw
4+
OTA4MTgxOVowDzENMAsGA1UEChMEdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEP
5+
ADCCAQoCggEBAMqGcyqm3UIEZCL3Ge8Qh63OU0QykI3ymtj9gDJ57vtZdUOpHF18
6+
QsX4Urf6vGNvI8gWXEtPbK8Dchqo+3Reuejjq8aRFs1RaPVZoWPD8i782p6L6/oX
7+
4k5zAwvCdjC2y/YuUf4GTZqpfwDbhSfH+EdacrfwmYjLxYaehEn7z9M67R5wSekr
8+
1bMuxBKZW1sclmip3JRf3uBuHfMLkoYpTa7KcEycn9YstZ1h2XuWFLk42GS4bMZs
9+
Cq2E3sEqD8LOyCHs/qWWGM3Txz6edLTi5U6XtzPeWwxyM5W7fOpC0Q3iw58myGsm
10+
DDmUngyneS/mzyLvKTywzsu2sRAqdjqsZCUCAwEAAaOBrjCBqzAdBgNVHSUEFjAU
11+
BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBTF
12+
Xnzlk4tOSosBlUEM7AiDOYuW7TBbBgNVHREEVDBSgiJzdG9yYWdlLWdycGMueWRi
13+
LnN2Yy5jbHVzdGVyLmxvY2FsgiwqLnN0b3JhZ2UtaW50ZXJjb25uZWN0LnlkYi5z
14+
dmMuY2x1c3Rlci5sb2NhbDANBgkqhkiG9w0BAQsFAAOCAQEAQYRvaGGFsnR4cp8Y
15+
MJo948t7zI3Pgy20YonmYTriz1zeEYNj9+5t678p04FlCjIx0j4dad1tFC1bNtnI
16+
FOJNMkhiyC1JPKSN7HR90L5P9JfsuunUVTNEHP6EuLj2/VnMXj+30qX9i0kVkcnH
17+
OV9yXk8cpN0GuUUyfKUq5k/WS+/15JRoLJ9F8vS1lm77nHeQ9F7m1Yjqkc131N3k
18+
czFLe3wexWhnKCnNw5OtdFgJsMgL09pYgnN+0xyE4iLY8f0o/b58J/tpbKkeJb9L
19+
YogP1+ultfGvrlX8TE6iAEi8UA+rZCNpGwiqTI2Y9clZNnkYu7UgW7kn2ih5/vgU
20+
xntdsA==
21+
-----END CERTIFICATE-----

‎e2e/tests/data/tls.key

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-----BEGIN RSA PRIVATE KEY-----
2+
MIIEpAIBAAKCAQEAyoZzKqbdQgRkIvcZ7xCHrc5TRDKQjfKa2P2AMnnu+1l1Q6kc
3+
XXxCxfhSt/q8Y28jyBZcS09srwNyGqj7dF656OOrxpEWzVFo9VmhY8PyLvzanovr
4+
+hfiTnMDC8J2MLbL9i5R/gZNmql/ANuFJ8f4R1pyt/CZiMvFhp6ESfvP0zrtHnBJ
5+
6SvVsy7EEplbWxyWaKnclF/e4G4d8wuShilNrspwTJyf1iy1nWHZe5YUuTjYZLhs
6+
xmwKrYTewSoPws7IIez+pZYYzdPHPp50tOLlTpe3M95bDHIzlbt86kLRDeLDnybI
7+
ayYMOZSeDKd5L+bPIu8pPLDOy7axECp2OqxkJQIDAQABAoIBAQCXKyBPl9nTax+r
8+
kbID5dzAeR9h6jRIH+xBR4cnJiih6LZE2LfZd+UHjEGCHl/8AHs+4KHnfNNtFy9W
9+
gweeZw5xrW8MekQA4WFssYhrxVjChe5RJbPwK1+6mtKNNout9OPtT8nXyLCoXxfz
10+
defAN900tWinr6mKmD9KKowoBROtYBsKVwO4DOl+JonZP7A5eF/ao4T8XFSOOju5
11+
aU9flCjw6iagzKuryrJCzt4LsTA3/svuMswPQ7LRMSJm+RsceiBl4oVCuRmJ79tN
12+
pmhHZqG80upp/idgyuGZu+rgePiXSnmtn47iXXvCjZKTNMkabQMc2WZ5P3HWfG0/
13+
28kXH5IBAoGBAPswl0SrvZdcQbalNHjLc1JtMMhZs6LPZe+/QTP4i/5qXvee8Dox
14+
vvGJlvjNQuOGNsPUTsLMY203eKQWJ4+soK2qDhuDCxGmyQJ7NgTTzdhOCULctXlD
15+
pdBHxiSmCUGEtW7Goe3M1/CL3sVfRp85FzMYItHHZEMTHZNzoZ+XXAzBAoGBAM5n
16+
SknfqHM4oBT0LxxkmVFyiFoCi2grXzsGjzK5rk30DWajwT5sAy9x+X2vmG2zKkBJ
17+
tTBYYUFh+SlNtBsY/TW2ZIcmHDpu0w82zXXDLLhUwYjxudXzyG30yU8d4woAwE6u
18+
NPiVmFX6C8U597a/nC36LLsrSnUPvaJ4voFyRlxlAoGAHJa8MLmnO2nppMMKxNDL
19+
EE+TJMpo0pfuTyoiXqrkLBGpO1+gkc8Fn3H8d9bMzR6CbyljyXH/wvd0SKCo4gZQ
20+
x1M6hdEVWm30JM8nJ8d/fyXqkeySzvlvDtSMbbFkDkvvZms/FNSioyMYOLiOTiLu
21+
TAdsNxoNhEDRte2MMKDGfkECgYAGH1o8xr2gbVWSSYv8M5+4osUYpmqsNF0myxME
22+
Vi2tckfTe5gH2fxeM+tKpyLGXkIqlgUh4f1Aiz9w0jU9eIhKR5bDy4Wa1h68nMuL
23+
araw4RK8lS8GAa04VcKC7kgFy+/oZZJ8rTNPmZMvzoBik1x2oK0jAC29OzJM13gP
24+
LuyXYQKBgQCvkar/nthVUlScVa3uPzqKdpNlqmeXQRMVwcmRQJcJoHYWkaF2RRAx
25+
AcjWTkP6/ThUgPhI76O0neQqO2N0P+FnCkaE4qU6Wg+dZcH0q7HVnt2TRuCzsECQ
26+
kxDqdbF+qbXmYfbJ4dKkJdF0dNad4/d2hL/wbvyHXDEEGpuIlZAZrg==
27+
-----END RSA PRIVATE KEY-----

‎e2e/tests/smoke_test.go

+51
Original file line numberDiff line numberDiff line change
@@ -551,6 +551,57 @@ var _ = Describe("Operator smoke test", func() {
551551
}
552552
})
553553

554+
It("using grpcs for storage connection", func() {
555+
By("create secret...")
556+
cert := testobjects.DefaultCertificate(
557+
filepath.Join(".", "data", "tls.crt"),
558+
filepath.Join(".", "data", "tls.key"),
559+
filepath.Join(".", "data", "ca.crt"),
560+
)
561+
Expect(k8sClient.Create(ctx, cert)).Should(Succeed())
562+
defer func() {
563+
Expect(k8sClient.Delete(ctx, cert)).Should(Succeed())
564+
}()
565+
566+
By("create storage...")
567+
storageSample = testobjects.DefaultStorage(filepath.Join(".", "data", "storage-block-4-2-config-tls.yaml"))
568+
storageSample.Spec.Service.GRPC.TLSConfiguration.Enabled = true
569+
storageSample.Spec.Service.GRPC.TLSConfiguration.Certificate = corev1.SecretKeySelector{
570+
LocalObjectReference: corev1.LocalObjectReference{Name: testobjects.CertificateSecretName},
571+
Key: "tls.crt",
572+
}
573+
storageSample.Spec.Service.GRPC.TLSConfiguration.Key = corev1.SecretKeySelector{
574+
LocalObjectReference: corev1.LocalObjectReference{Name: testobjects.CertificateSecretName},
575+
Key: "tls.key",
576+
}
577+
storageSample.Spec.Service.GRPC.TLSConfiguration.CertificateAuthority = corev1.SecretKeySelector{
578+
LocalObjectReference: corev1.LocalObjectReference{Name: testobjects.CertificateSecretName},
579+
Key: "ca.crt",
580+
}
581+
582+
Expect(k8sClient.Create(ctx, storageSample)).Should(Succeed())
583+
defer func() {
584+
Expect(k8sClient.Delete(ctx, storageSample)).Should(Succeed())
585+
}()
586+
By("create database...")
587+
Expect(k8sClient.Create(ctx, databaseSample)).Should(Succeed())
588+
defer func() {
589+
Expect(k8sClient.Delete(ctx, databaseSample)).Should(Succeed())
590+
}()
591+
592+
By("waiting until Storage is ready...")
593+
waitUntilStorageReady(ctx, storageSample.Name, testobjects.YdbNamespace)
594+
595+
By("checking that all the storage pods are running and ready...")
596+
checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-storage", storageSample.Spec.Nodes)
597+
598+
By("waiting until database is ready...")
599+
waitUntilDatabaseReady(ctx, databaseSample.Name, testobjects.YdbNamespace)
600+
601+
By("checking that all the database pods are running and ready...")
602+
checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-database", databaseSample.Spec.Nodes)
603+
})
604+
554605
AfterEach(func() {
555606
Expect(uninstallOperatorWithHelm(testobjects.YdbNamespace)).Should(BeTrue())
556607
Expect(k8sClient.Delete(ctx, &namespace)).Should(Succeed())

‎e2e/tests/test-objects/objects.go

+29-6
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,13 @@ import (
1111
)
1212

1313
const (
14-
YdbImage = "cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17"
15-
YdbNamespace = "ydb"
16-
StorageName = "storage"
17-
DatabaseName = "database"
18-
DefaultDomain = "Root"
19-
ReadyStatus = "Ready"
14+
YdbImage = "cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17"
15+
YdbNamespace = "ydb"
16+
StorageName = "storage"
17+
DatabaseName = "database"
18+
CertificateSecretName = "storage-crt"
19+
DefaultDomain = "Root"
20+
ReadyStatus = "Ready"
2021
)
2122

2223
func constructAntiAffinityFor(key, value string) *corev1.Affinity {
@@ -162,3 +163,25 @@ func DefaultDatabase() *v1alpha1.Database {
162163
},
163164
}
164165
}
166+
167+
func DefaultCertificate(certPath, keyPath, caPath string) *corev1.Secret {
168+
cert, err := os.ReadFile(certPath)
169+
Expect(err).To(BeNil())
170+
key, err := os.ReadFile(keyPath)
171+
Expect(err).To(BeNil())
172+
ca, err := os.ReadFile(caPath)
173+
Expect(err).To(BeNil())
174+
175+
return &corev1.Secret{
176+
ObjectMeta: metav1.ObjectMeta{
177+
Name: CertificateSecretName,
178+
Namespace: YdbNamespace,
179+
},
180+
Type: corev1.SecretTypeOpaque,
181+
Data: map[string][]byte{
182+
"ca.crt": ca,
183+
"tls.crt": cert,
184+
"tls.key": key,
185+
},
186+
}
187+
}

‎internal/cms/tenant.go

+2
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ func (t *Tenant) Create(
3131
ctx context.Context,
3232
database *resources.DatabaseBuilder,
3333
creds ydbCredentials.Credentials,
34+
opts ...ydb.Option,
3435
) error {
3536
logger := log.FromContext(ctx)
3637
createDatabaseURL := fmt.Sprintf(
@@ -42,6 +43,7 @@ func (t *Tenant) Create(
4243
db, err := connection.Open(ctx,
4344
createDatabaseURL,
4445
ydb.WithCredentials(creds),
46+
ydb.MergeOptions(opts...),
4547
)
4648
if err != nil {
4749
logger.Error(err, "Error connecting to YDB storage")

‎internal/connection/connection.go

+21-8
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"crypto/tls"
66
"crypto/x509"
7+
"errors"
78
"fmt"
89
"time"
910

@@ -41,14 +42,26 @@ func Close(ctx context.Context, db *ydb.Driver) {
4142
}
4243
}
4344

44-
func LoadTLSCredentials(secure bool) grpc.DialOption {
45-
if secure {
46-
certPool, _ := x509.SystemCertPool()
47-
tlsConfig := &tls.Config{
48-
MinVersion: tls.VersionTLS12,
49-
RootCAs: certPool,
45+
func LoadTLSCredentials(secure bool, caBundle []byte) (grpc.DialOption, error) {
46+
if !secure {
47+
return grpc.WithTransportCredentials(insecure.NewCredentials()), nil
48+
}
49+
var certPool *x509.CertPool
50+
if len(caBundle) > 0 {
51+
certPool = x509.NewCertPool()
52+
if ok := certPool.AppendCertsFromPEM(caBundle); !ok {
53+
return nil, errors.New("failed to parse CA bundle")
54+
}
55+
} else {
56+
var err error
57+
certPool, err = x509.SystemCertPool()
58+
if err != nil {
59+
return nil, fmt.Errorf("failed to get system cert pool, error: %w", err)
5060
}
51-
return grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
5261
}
53-
return grpc.WithTransportCredentials(insecure.NewCredentials())
62+
tlsConfig := &tls.Config{
63+
MinVersion: tls.VersionTLS12,
64+
RootCAs: certPool,
65+
}
66+
return grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), nil
5467
}

‎internal/controllers/database/init.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,18 @@ func (r *Reconciler) handleTenantCreation(
178178
)
179179
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
180180
}
181+
tlsOptions, err := resources.GetYDBTLSOption(ctx, database.Storage, r.Config)
182+
if err != nil {
183+
r.Recorder.Event(
184+
database,
185+
corev1.EventTypeWarning,
186+
"ControllerError",
187+
fmt.Sprintf("Failed to get YDB TLS options: %s", err),
188+
)
189+
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
190+
}
181191

182-
err = tenant.Create(ctx, database, creds)
192+
err = tenant.Create(ctx, database, creds, tlsOptions)
183193
if err != nil {
184194
r.Recorder.Event(
185195
database,

‎internal/controllers/storage/sync.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -429,8 +429,18 @@ func (r *Reconciler) runSelfCheck(
429429
)
430430
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
431431
}
432+
tlsOptions, err := resources.GetYDBTLSOption(ctx, storage.Unwrap(), r.Config)
433+
if err != nil {
434+
r.Recorder.Event(
435+
storage,
436+
corev1.EventTypeWarning,
437+
"ControllerError",
438+
fmt.Sprintf("Failed to get YDB TLS options: %s", err),
439+
)
440+
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, err
441+
}
432442

433-
result, err := healthcheck.GetSelfCheckResult(ctx, storage, creds)
443+
result, err := healthcheck.GetSelfCheckResult(ctx, storage, creds, tlsOptions)
434444
if err != nil {
435445
r.Log.Error(err, "GetSelfCheckResult error")
436446
return Stop, ctrl.Result{RequeueAfter: SelfCheckRequeueDelay}, err

0 commit comments

Comments
 (0)