Skip to content

Commit 3ab6413

Browse files
kobzonegakobzonega
authored
fix e2e tests check running and ready pods (#177)
1 parent efb7101 commit 3ab6413

File tree

2 files changed

+103
-116
lines changed

2 files changed

+103
-116
lines changed

‎e2e/tests/smoke_test.go

+103-115
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,6 @@ const (
3030
Interval = time.Second * 5
3131
)
3232

33-
var HostPathDirectoryType corev1.HostPathType = "Directory"
34-
3533
func podIsReady(conditions []corev1.PodCondition) bool {
3634
for _, condition := range conditions {
3735
if condition.Type == testobjects.ReadyStatus && condition.Status == "True" {
@@ -41,33 +39,6 @@ func podIsReady(conditions []corev1.PodCondition) bool {
4139
return false
4240
}
4341

44-
func execInPod(namespace string, name string, cmd []string) (string, error) {
45-
args := []string{
46-
"-n",
47-
namespace,
48-
"exec",
49-
name,
50-
"--",
51-
}
52-
args = append(args, cmd...)
53-
result := exec.Command("kubectl", args...)
54-
stdout, err := result.Output()
55-
return string(stdout), err
56-
}
57-
58-
func bringYdbCliToPod(namespace string, name string, ydbHome string) error {
59-
args := []string{
60-
"-n",
61-
namespace,
62-
"cp",
63-
fmt.Sprintf("%v/ydb/bin/ydb", os.ExpandEnv("$HOME")),
64-
fmt.Sprintf("%v:%v/ydb", name, ydbHome),
65-
}
66-
result := exec.Command("kubectl", args...)
67-
_, err := result.Output()
68-
return err
69-
}
70-
7142
func installOperatorWithHelm(namespace string) bool {
7243
args := []string{
7344
"-n",
@@ -145,11 +116,67 @@ func checkPodsRunningAndReady(ctx context.Context, podLabelKey, podLabelValue st
145116
})).Should(Succeed())
146117
g.Expect(len(pods.Items)).Should(BeEquivalentTo(nPods))
147118
for _, pod := range pods.Items {
148-
g.Expect(pod.Status.Phase).To(BeEquivalentTo("Running"))
149-
g.Expect(podIsReady(pod.Status.Conditions)).To(BeTrue())
119+
g.Expect(pod.Status.Phase).Should(BeEquivalentTo("Running"))
120+
g.Expect(podIsReady(pod.Status.Conditions)).Should(BeTrue())
150121
}
151122
return true
152123
}, Timeout, Interval).Should(BeTrue())
124+
125+
Consistently(func(g Gomega) bool {
126+
pods := corev1.PodList{}
127+
g.Expect(k8sClient.List(ctx, &pods, client.InNamespace(testobjects.YdbNamespace), client.MatchingLabels{
128+
podLabelKey: podLabelValue,
129+
})).Should(Succeed())
130+
g.Expect(len(pods.Items)).Should(BeEquivalentTo(nPods))
131+
for _, pod := range pods.Items {
132+
g.Expect(pod.Status.Phase).Should(BeEquivalentTo("Running"))
133+
g.Expect(podIsReady(pod.Status.Conditions)).Should(BeTrue())
134+
}
135+
return true
136+
}, 30*time.Second, Interval).Should(BeTrue())
137+
}
138+
139+
func bringYdbCliToPod(podName, podNamespace string) {
140+
Eventually(func(g Gomega) error {
141+
args := []string{
142+
"-n",
143+
podNamespace,
144+
"cp",
145+
fmt.Sprintf("%v/ydb/bin/ydb", os.ExpandEnv("$HOME")),
146+
fmt.Sprintf("%v:/tmp/ydb", podName),
147+
}
148+
cmd := exec.Command("kubectl", args...)
149+
return cmd.Run()
150+
}, Timeout, Interval).Should(BeNil())
151+
}
152+
153+
func executeSimpleQuery(ctx context.Context, podName, podNamespace, storageEndpoint string) {
154+
Eventually(func(g Gomega) string {
155+
args := []string{
156+
"-n",
157+
podNamespace,
158+
"exec",
159+
podName,
160+
"--",
161+
"/tmp/ydb",
162+
"-d",
163+
"/" + testobjects.DefaultDomain,
164+
"-e" + storageEndpoint,
165+
"yql",
166+
"-s",
167+
"select 1",
168+
}
169+
output, err := exec.Command("kubectl", args...).Output()
170+
g.Expect(err).ShouldNot(HaveOccurred())
171+
172+
// `yql` gives output in the following format:
173+
// ┌─────────┐
174+
// | column0 |
175+
// ├─────────┤
176+
// | 1 |
177+
// └─────────┘
178+
return strings.ReplaceAll(string(output), "\n", "")
179+
}, Timeout, Interval).Should(MatchRegexp(".*column0.*1.*"))
153180
}
154181

155182
var _ = Describe("Operator smoke test", func() {
@@ -206,38 +233,24 @@ var _ = Describe("Operator smoke test", func() {
206233
By("checking that all the database pods are running and ready...")
207234
checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-database", databaseSample.Spec.Nodes)
208235

236+
database := v1alpha1.Database{}
237+
Expect(k8sClient.Get(ctx, types.NamespacedName{
238+
Name: databaseSample.Name,
239+
Namespace: testobjects.YdbNamespace,
240+
}, &database)).Should(Succeed())
241+
storageEndpoint := database.Spec.StorageEndpoint
242+
209243
databasePods := corev1.PodList{}
210-
err := k8sClient.List(ctx, &databasePods, client.InNamespace(testobjects.YdbNamespace), client.MatchingLabels{
244+
Expect(k8sClient.List(ctx, &databasePods, client.InNamespace(testobjects.YdbNamespace), client.MatchingLabels{
211245
"ydb-cluster": "kind-database",
212-
})
213-
Expect(err).To(BeNil())
214-
firstDBPod := databasePods.Items[0].Name
215-
216-
Expect(bringYdbCliToPod(testobjects.YdbNamespace, firstDBPod, testobjects.YdbHome)).To(Succeed())
217-
218-
Eventually(func(g Gomega) {
219-
out, err := execInPod(testobjects.YdbNamespace, firstDBPod, []string{
220-
fmt.Sprintf("%v/ydb", testobjects.YdbHome),
221-
"-d",
222-
"/" + testobjects.DefaultDomain,
223-
"-e",
224-
"grpc://localhost:2135",
225-
"yql",
226-
"-s",
227-
"select 1",
228-
})
246+
})).Should(Succeed())
247+
podName := databasePods.Items[0].Name
229248

230-
g.Expect(err).To(BeNil())
231-
232-
// `yql` gives output in the following format:
233-
// ┌─────────┐
234-
// | column0 |
235-
// ├─────────┤
236-
// | 1 |
237-
// └─────────┘
238-
g.Expect(strings.ReplaceAll(out, "\n", "")).
239-
To(MatchRegexp(".*column0.*1.*"))
240-
})
249+
By("bring YDB CLI inside ydb database pod...")
250+
bringYdbCliToPod(podName, testobjects.YdbNamespace)
251+
252+
By("execute simple query inside ydb database pod...")
253+
executeSimpleQuery(ctx, podName, testobjects.YdbNamespace, storageEndpoint)
241254
})
242255

243256
It("pause and un-pause Storage, should destroy and bring up Pods", func() {
@@ -417,32 +430,33 @@ var _ = Describe("Operator smoke test", func() {
417430
By("checking that all the database pods are running and ready...")
418431
checkPodsRunningAndReady(ctx, "ydb-cluster", "kind-database", databaseSample.Spec.Nodes)
419432

420-
By("delete nodeSetSpec inline to check inheritance...")
421433
database := v1alpha1.Database{}
422-
Expect(k8sClient.Get(ctx, types.NamespacedName{
423-
Name: databaseSample.Name,
424-
Namespace: testobjects.YdbNamespace,
425-
}, &database)).Should(Succeed())
426-
database.Spec.Nodes = 4
427-
database.Spec.NodeSets = []v1alpha1.DatabaseNodeSetSpecInline{
428-
{
429-
Name: testNodeSetName + "-" + strconv.Itoa(1),
430-
DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{
431-
Nodes: 4,
434+
databaseNodeSetList := v1alpha1.DatabaseNodeSetList{}
435+
databasePods := corev1.PodList{}
436+
By("delete nodeSetSpec inline to check inheritance...")
437+
Eventually(func(g Gomega) error {
438+
g.Expect(k8sClient.Get(ctx, types.NamespacedName{
439+
Name: databaseSample.Name,
440+
Namespace: testobjects.YdbNamespace,
441+
}, &database)).Should(Succeed())
442+
database.Spec.Nodes = 4
443+
database.Spec.NodeSets = []v1alpha1.DatabaseNodeSetSpecInline{
444+
{
445+
Name: testNodeSetName + "-" + strconv.Itoa(1),
446+
DatabaseNodeSpec: v1alpha1.DatabaseNodeSpec{
447+
Nodes: 4,
448+
},
432449
},
433-
},
434-
}
435-
Expect(k8sClient.Update(ctx, &database)).Should(Succeed())
450+
}
451+
return k8sClient.Update(ctx, &database)
452+
}, Timeout, Interval).Should(BeNil())
436453

437454
By("check that ObservedDatabaseGeneration changed...")
438455
Eventually(func(g Gomega) bool {
439-
database := v1alpha1.Database{}
440456
g.Expect(k8sClient.Get(ctx, types.NamespacedName{
441457
Name: databaseSample.Name,
442458
Namespace: testobjects.YdbNamespace,
443459
}, &database)).Should(Succeed())
444-
445-
databaseNodeSetList := v1alpha1.DatabaseNodeSetList{}
446460
g.Expect(k8sClient.List(ctx, &databaseNodeSetList,
447461
client.InNamespace(testobjects.YdbNamespace),
448462
)).Should(Succeed())
@@ -456,54 +470,28 @@ var _ = Describe("Operator smoke test", func() {
456470

457471
By("expecting databaseNodeSet pods deletion...")
458472
Eventually(func(g Gomega) bool {
459-
database := v1alpha1.Database{}
460473
g.Expect(k8sClient.Get(ctx, types.NamespacedName{
461474
Name: databaseSample.Name,
462475
Namespace: testobjects.YdbNamespace,
463476
}, &database)).Should(Succeed())
464477

465-
databasePods := corev1.PodList{}
466478
g.Expect(k8sClient.List(ctx, &databasePods, client.InNamespace(testobjects.YdbNamespace), client.MatchingLabels{
467479
"ydb-cluster": "kind-database",
468480
})).Should(Succeed())
469481
return len(databasePods.Items) == int(database.Spec.Nodes)
470482
}, Timeout, Interval).Should(BeTrue())
471483

472-
By("execute simple query inside ydb database pod...")
473-
databasePods := corev1.PodList{}
474-
err := k8sClient.List(ctx, &databasePods,
475-
client.InNamespace(testobjects.YdbNamespace),
476-
client.MatchingLabels{
477-
"ydb-cluster": "kind-database",
478-
})
479-
Expect(err).To(BeNil())
480-
firstDBPod := databasePods.Items[0].Name
481-
482-
Expect(bringYdbCliToPod(testobjects.YdbNamespace, firstDBPod, testobjects.YdbHome)).To(Succeed())
483-
484-
Eventually(func(g Gomega) {
485-
out, err := execInPod(testobjects.YdbNamespace, firstDBPod, []string{
486-
fmt.Sprintf("%v/ydb", testobjects.YdbHome),
487-
"-d",
488-
"/" + testobjects.DefaultDomain,
489-
"-e",
490-
"grpc://localhost:2135",
491-
"yql",
492-
"-s",
493-
"select 1",
494-
})
484+
storageEndpoint := database.Spec.StorageEndpoint
485+
Expect(k8sClient.List(ctx, &databasePods, client.InNamespace(testobjects.YdbNamespace), client.MatchingLabels{
486+
"ydb-cluster": "kind-database",
487+
})).Should(Succeed())
488+
podName := databasePods.Items[0].Name
495489

496-
g.Expect(err).To(BeNil())
497-
498-
// `yql` gives output in the following format:
499-
// ┌─────────┐
500-
// | column0 |
501-
// ├─────────┤
502-
// | 1 |
503-
// └─────────┘
504-
g.Expect(strings.ReplaceAll(out, "\n", "")).
505-
To(MatchRegexp(".*column0.*1.*"))
506-
})
490+
By("bring YDB CLI inside ydb database pod...")
491+
bringYdbCliToPod(podName, testobjects.YdbNamespace)
492+
493+
By("execute simple query inside ydb database pod...")
494+
executeSimpleQuery(ctx, podName, testobjects.YdbNamespace, storageEndpoint)
507495
})
508496

509497
It("operatorConnection check, create storage with default staticCredentials", func() {
@@ -538,7 +526,7 @@ var _ = Describe("Operator smoke test", func() {
538526
By("tracking storage state changes...")
539527
events, err := clientset.CoreV1().Events(testobjects.YdbNamespace).List(context.Background(),
540528
metav1.ListOptions{TypeMeta: metav1.TypeMeta{Kind: "Storage"}})
541-
Expect(err).ToNot(HaveOccurred())
529+
Expect(err).ShouldNot(HaveOccurred())
542530

543531
allowedChanges := map[ClusterState]ClusterState{
544532
StoragePending: StoragePreparing,
@@ -551,7 +539,7 @@ var _ = Describe("Operator smoke test", func() {
551539
for _, event := range events.Items {
552540
if event.Reason == "StatusChanged" {
553541
match := re.FindStringSubmatch(event.Message)
554-
Expect(allowedChanges[ClusterState(match[1])]).To(BeEquivalentTo(ClusterState(match[2])))
542+
Expect(allowedChanges[ClusterState(match[1])]).Should(BeEquivalentTo(ClusterState(match[2])))
555543
}
556544
}
557545
})

‎e2e/tests/test-objects/objects.go

-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import (
1313
const (
1414
YdbImage = "cr.yandex/crptqonuodf51kdj7a7d/ydb:22.4.44"
1515
YdbNamespace = "ydb"
16-
YdbHome = "/home/ydb"
1716
StorageName = "storage"
1817
DatabaseName = "database"
1918
DefaultDomain = "Root"

0 commit comments

Comments
 (0)