Skip to content

Commit 289f021

Browse files
spikecurtisspikecurtis
authored
feat: add scaletest Runner for dynamicparameters load gen (coder#19890)
relates to coder/internal#912 Adds a new scaletest Runner to generate dynamic parameters load. A later PR will add the CLI command, including creating the template & version.
1 parent 5317d30 commit 289f021

File tree

10 files changed

+458
-2
lines changed

10 files changed

+458
-2
lines changed

‎.github/workflows/typos.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
[default]
22
extend-ignore-identifiers-re = ["gho_.*"]
3+
extend-ignore-re = ["(#|//)\\s*spellchecker:ignore-next-line\\n.*"]
34

45
[default.extend-identifiers]
56
alog = "alog"

‎coderd/coderdtest/dynamicparameters.go‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ type DynamicParameterTemplateParams struct {
2020
Plan json.RawMessage
2121
ModulesArchive []byte
2222

23+
// ExtraFiles are additional files to include in the template, beyond the MainTF.
24+
ExtraFiles map[string][]byte
25+
2326
// Uses a zip archive instead of a tar
2427
Zip bool
2528

@@ -36,9 +39,17 @@ type DynamicParameterTemplateParams struct {
3639
func DynamicParameterTemplate(t *testing.T, client *codersdk.Client, org uuid.UUID, args DynamicParameterTemplateParams) (codersdk.Template, codersdk.TemplateVersion) {
3740
t.Helper()
3841

39-
files := echo.WithExtraFiles(map[string][]byte{
42+
// Start with main.tf
43+
extraFiles := map[string][]byte{
4044
"main.tf": []byte(args.MainTF),
41-
})
45+
}
46+
47+
// Add any additional files
48+
for name, content := range args.ExtraFiles {
49+
extraFiles[name] = content
50+
}
51+
52+
files := echo.WithExtraFiles(extraFiles)
4253
files.ProvisionPlan = []*proto.Response{{
4354
Type: &proto.Response_Plan{
4455
Plan: &proto.PlanComplete{

‎provisioner/echo/serve.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"fmt"
88
"os"
99
"path/filepath"
10+
"slices"
1011
"strings"
1112
"text/template"
1213

@@ -359,9 +360,26 @@ func TarWithOptions(ctx context.Context, logger slog.Logger, responses *Response
359360
}
360361
}
361362
}
363+
dirs := []string{}
362364
for name, content := range responses.ExtraFiles {
363365
logger.Debug(ctx, "extra file", slog.F("name", name))
364366

367+
// We need to add directories before any files that use them. But, we only need to do this
368+
// once.
369+
dir := filepath.Dir(name)
370+
if dir != "." && !slices.Contains(dirs, dir) {
371+
logger.Debug(ctx, "adding extra file directory", slog.F("dir", dir))
372+
dirs = append(dirs, dir)
373+
err := writer.WriteHeader(&tar.Header{
374+
Name: dir,
375+
Mode: 0o755,
376+
Typeflag: tar.TypeDir,
377+
})
378+
if err != nil {
379+
return nil, err
380+
}
381+
}
382+
365383
err := writer.WriteHeader(&tar.Header{
366384
Name: name,
367385
Size: int64(len(content)),

‎scaletest/dynamicparameters/config.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package dynamicparameters
2+
3+
import "github.com/google/uuid"
4+
5+
type Config struct {
6+
TemplateVersion uuid.UUID `json:"template_version"`
7+
SessionToken string `json:"session_token"`
8+
Metrics *Metrics `json:"-"`
9+
MetricLabelValues []string `json:"metric_label_values"`
10+
}

‎scaletest/dynamicparameters/metrics.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package dynamicparameters
2+
3+
import "github.com/prometheus/client_golang/prometheus"
4+
5+
type Metrics struct {
6+
LatencyInitialResponseSeconds prometheus.HistogramVec
7+
LatencyChangeResponseSeconds prometheus.HistogramVec
8+
}
9+
10+
func NewMetrics(reg prometheus.Registerer, labelNames ...string) *Metrics {
11+
m := &Metrics{
12+
LatencyInitialResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{
13+
Namespace: "coderd",
14+
Subsystem: "scaletest",
15+
Name: "dynamic_parameters_latency_initial_response_seconds",
16+
Help: "Time in seconds to get the initial dynamic parameters response from start of request.",
17+
}, labelNames),
18+
LatencyChangeResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{
19+
Namespace: "coderd",
20+
Subsystem: "scaletest",
21+
Name: "dynamic_parameters_latency_change_response_seconds",
22+
Help: "Time in seconds to between sending a dynamic parameters change request and receiving the response.",
23+
}, labelNames),
24+
}
25+
reg.MustRegister(m.LatencyInitialResponseSeconds)
26+
reg.MustRegister(m.LatencyChangeResponseSeconds)
27+
return m
28+
}

‎scaletest/dynamicparameters/run.go‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package dynamicparameters
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"slices"
8+
"time"
9+
10+
"golang.org/x/xerrors"
11+
12+
"github.com/coder/coder/v2/codersdk"
13+
"github.com/coder/coder/v2/scaletest/harness"
14+
"github.com/coder/websocket"
15+
)
16+
17+
type Runner struct {
18+
client *codersdk.Client
19+
cfg Config
20+
}
21+
22+
var _ harness.Runnable = &Runner{}
23+
24+
func NewRunner(client *codersdk.Client, cfg Config) *Runner {
25+
clone := codersdk.New(client.URL)
26+
clone.HTTPClient = client.HTTPClient
27+
clone.SetLogger(client.Logger())
28+
clone.SetSessionToken(cfg.SessionToken)
29+
return &Runner{
30+
client: clone,
31+
cfg: cfg,
32+
}
33+
}
34+
35+
// Run executes the dynamic parameters test, which:
36+
//
37+
// 1. connects to the dynamic parameters stream
38+
// 2. waits for the initial response
39+
// 3. sends a change request
40+
// 4. waits for the change response
41+
// 5. closes the stream
42+
func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) (retErr error) {
43+
startTime := time.Now()
44+
stream, err := r.client.TemplateVersionDynamicParameters(ctx, codersdk.Me, r.cfg.TemplateVersion)
45+
if err != nil {
46+
return xerrors.Errorf("connect to dynamic parameters stream: %w", err)
47+
}
48+
defer stream.Close(websocket.StatusNormalClosure)
49+
respCh := stream.Chan()
50+
51+
var initTime time.Time
52+
select {
53+
case <-ctx.Done():
54+
return ctx.Err()
55+
case resp, ok := <-respCh:
56+
if !ok {
57+
return xerrors.Errorf("dynamic parameters stream closed before initial response")
58+
}
59+
initTime = time.Now()
60+
r.cfg.Metrics.LatencyInitialResponseSeconds.
61+
WithLabelValues(r.cfg.MetricLabelValues...).
62+
Observe(initTime.Sub(startTime).Seconds())
63+
_, _ = fmt.Fprintf(logs, "initial response: %+v\n", resp)
64+
if !slices.ContainsFunc(resp.Parameters, func(p codersdk.PreviewParameter) bool {
65+
return p.Name == "zero"
66+
}) {
67+
return xerrors.Errorf("missing expected parameter: 'zero'")
68+
}
69+
if err := checkNoDiagnostics(resp); err != nil {
70+
return xerrors.Errorf("unexpected initial response diagnostics: %w", err)
71+
}
72+
}
73+
74+
err = stream.Send(codersdk.DynamicParametersRequest{
75+
ID: 1,
76+
Inputs: map[string]string{
77+
"zero": "B",
78+
},
79+
})
80+
if err != nil {
81+
return xerrors.Errorf("send change request: %w", err)
82+
}
83+
select {
84+
case <-ctx.Done():
85+
return ctx.Err()
86+
case resp, ok := <-respCh:
87+
if !ok {
88+
return xerrors.Errorf("dynamic parameters stream closed before change response")
89+
}
90+
_, _ = fmt.Fprintf(logs, "change response: %+v\n", resp)
91+
r.cfg.Metrics.LatencyChangeResponseSeconds.
92+
WithLabelValues(r.cfg.MetricLabelValues...).
93+
Observe(time.Since(initTime).Seconds())
94+
if resp.ID != 1 {
95+
return xerrors.Errorf("unexpected response ID: %d", resp.ID)
96+
}
97+
if err := checkNoDiagnostics(resp); err != nil {
98+
return xerrors.Errorf("unexpected change response diagnostics: %w", err)
99+
}
100+
return nil
101+
}
102+
}
103+
104+
func checkNoDiagnostics(resp codersdk.DynamicParametersResponse) error {
105+
if len(resp.Diagnostics) != 0 {
106+
return xerrors.Errorf("unexpected response diagnostics: %v", resp.Diagnostics)
107+
}
108+
for _, param := range resp.Parameters {
109+
if len(param.Diagnostics) != 0 {
110+
return xerrors.Errorf("unexpected parameter diagnostics for '%s': %v", param.Name, param.Diagnostics)
111+
}
112+
}
113+
return nil
114+
}

‎scaletest/dynamicparameters/run_test.go‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package dynamicparameters_test
2+
3+
import (
4+
"strings"
5+
"testing"
6+
7+
"github.com/prometheus/client_golang/prometheus"
8+
"github.com/stretchr/testify/require"
9+
10+
"cdr.dev/slog"
11+
"github.com/coder/coder/v2/coderd/coderdtest"
12+
"github.com/coder/coder/v2/scaletest/dynamicparameters"
13+
"github.com/coder/coder/v2/testutil"
14+
)
15+
16+
func TestRun(t *testing.T) {
17+
t.Parallel()
18+
ctx := testutil.Context(t, testutil.WaitLong)
19+
20+
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
21+
client.SetLogger(testutil.Logger(t).Leveled(slog.LevelDebug))
22+
first := coderdtest.CreateFirstUser(t, client)
23+
userClient, _ := coderdtest.CreateAnotherUser(t, client, first.OrganizationID)
24+
orgID := first.OrganizationID
25+
26+
dynamicParametersTerraformSource, err := dynamicparameters.TemplateContent()
27+
require.NoError(t, err)
28+
29+
template, version := coderdtest.DynamicParameterTemplate(t, client, orgID, coderdtest.DynamicParameterTemplateParams{
30+
MainTF: dynamicParametersTerraformSource,
31+
Plan: nil,
32+
ModulesArchive: nil,
33+
StaticParams: nil,
34+
ExtraFiles: dynamicparameters.GetModuleFiles(),
35+
})
36+
37+
reg := prometheus.NewRegistry()
38+
cfg := dynamicparameters.Config{
39+
TemplateVersion: version.ID,
40+
SessionToken: userClient.SessionToken(),
41+
Metrics: dynamicparameters.NewMetrics(reg, "template", "test_label_name"),
42+
MetricLabelValues: []string{template.Name, "test_label_value"},
43+
}
44+
runner := dynamicparameters.NewRunner(userClient, cfg)
45+
var logs strings.Builder
46+
err = runner.Run(ctx, t.Name(), &logs)
47+
t.Log("Runner logs:\n\n" + logs.String())
48+
require.NoError(t, err)
49+
}

‎scaletest/dynamicparameters/template.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package dynamicparameters
2+
3+
import (
4+
_ "embed"
5+
"encoding/json"
6+
"strings"
7+
"text/template"
8+
9+
"github.com/coder/coder/v2/cryptorand"
10+
)
11+
12+
//go:embed tf/main.tf
13+
var templateContent string
14+
15+
func TemplateContent() (string, error) {
16+
randomString, err := cryptorand.String(8)
17+
if err != nil {
18+
return "", err
19+
}
20+
tmpl, err := template.New("workspace-template").Parse(templateContent)
21+
if err != nil {
22+
return "", err
23+
}
24+
var result strings.Builder
25+
err = tmpl.Execute(&result, map[string]string{
26+
"RandomString": randomString,
27+
})
28+
if err != nil {
29+
return "", err
30+
}
31+
return result.String(), nil
32+
}
33+
34+
//go:embed tf/modules/two/main.tf
35+
var moduleTwoMainTF string
36+
37+
// GetModuleFiles returns a map of module files to be used with ExtraFiles
38+
func GetModuleFiles() map[string][]byte {
39+
// Create the modules.json that Terraform needs to see the module
40+
modulesJSON := struct {
41+
Modules []struct {
42+
Key string `json:"Key"`
43+
Source string `json:"Source"`
44+
Dir string `json:"Dir"`
45+
} `json:"Modules"`
46+
}{
47+
Modules: []struct {
48+
Key string `json:"Key"`
49+
Source string `json:"Source"`
50+
Dir string `json:"Dir"`
51+
}{
52+
{
53+
Key: "",
54+
Source: "",
55+
Dir: ".",
56+
},
57+
{
58+
Key: "two",
59+
Source: "./modules/two",
60+
Dir: "modules/two",
61+
},
62+
},
63+
}
64+
65+
modulesJSONBytes, err := json.Marshal(modulesJSON)
66+
if err != nil {
67+
panic(err) // This should never happen with static data
68+
}
69+
70+
return map[string][]byte{
71+
"modules/two/main.tf": []byte(moduleTwoMainTF),
72+
".terraform/modules/modules.json": modulesJSONBytes,
73+
}
74+
}

0 commit comments

Comments
 (0)