test: fix flake in TestAgent_Metrics_SSH (coder#20447)

Closes coder/internal#921

The flake in the linked issue was caused by the startup script taking longer than 1 second in CI. The existing conditional, that the startup script duration was under a second, was incorrect; the correct conditional is that the metric exists with the `success` label set to `true`.

Author: ethanndickson

agent/agent_test.go

@@ -3481,16 +3481,31 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 	err = session.Shell()
 	require.NoError(t, err)
 
-	expected := []*proto.Stats_Metric{
+	expected := []struct {
+		Name    string
+		Type    proto.Stats_Metric_Type
+		CheckFn func(float64) error
+		Labels  []*proto.Stats_Metric_Label
+	}{
 		{
-			Name:  "agent_reconnecting_pty_connections_total",
-			Type:  proto.Stats_Metric_COUNTER,
-			Value: 0,
+			Name: "agent_reconnecting_pty_connections_total",
+			Type: proto.Stats_Metric_COUNTER,
+			CheckFn: func(v float64) error {
+				if v == 0 {
+					return nil
+				}
+				return xerrors.Errorf("expected 0, got %f", v)
+			},
 		},
 		{
-			Name:  "agent_sessions_total",
-			Type:  proto.Stats_Metric_COUNTER,
-			Value: 1,
+			Name: "agent_sessions_total",
+			Type: proto.Stats_Metric_COUNTER,
+			CheckFn: func(v float64) error {
+				if v == 1 {
+					return nil
+				}
+				return xerrors.Errorf("expected 1, got %f", v)
+			},
 			Labels: []*proto.Stats_Metric_Label{
 				{
 					Name:  "magic_type",
@@ -3503,24 +3518,44 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 			},
 		},
 		{
-			Name:  "agent_ssh_server_failed_connections_total",
-			Type:  proto.Stats_Metric_COUNTER,
-			Value: 0,
+			Name: "agent_ssh_server_failed_connections_total",
+			Type: proto.Stats_Metric_COUNTER,
+			CheckFn: func(v float64) error {
+				if v == 0 {
+					return nil
+				}
+				return xerrors.Errorf("expected 0, got %f", v)
+			},
 		},
 		{
-			Name:  "agent_ssh_server_sftp_connections_total",
-			Type:  proto.Stats_Metric_COUNTER,
-			Value: 0,
+			Name: "agent_ssh_server_sftp_connections_total",
+			Type: proto.Stats_Metric_COUNTER,
+			CheckFn: func(v float64) error {
+				if v == 0 {
+					return nil
+				}
+				return xerrors.Errorf("expected 0, got %f", v)
+			},
 		},
 		{
-			Name:  "agent_ssh_server_sftp_server_errors_total",
-			Type:  proto.Stats_Metric_COUNTER,
-			Value: 0,
+			Name: "agent_ssh_server_sftp_server_errors_total",
+			Type: proto.Stats_Metric_COUNTER,
+			CheckFn: func(v float64) error {
+				if v == 0 {
+					return nil
+				}
+				return xerrors.Errorf("expected 0, got %f", v)
+			},
 		},
 		{
-			Name:  "coderd_agentstats_currently_reachable_peers",
-			Type:  proto.Stats_Metric_GAUGE,
-			Value: 1,
+			Name: "coderd_agentstats_currently_reachable_peers",
+			Type: proto.Stats_Metric_GAUGE,
+			CheckFn: func(v float64) error {
+				if v == 1 {
+					return nil
+				}
+				return xerrors.Errorf("expected 1, got %f", v)
+			},
 			Labels: []*proto.Stats_Metric_Label{
 				{
 					Name:  "connection_type",
@@ -3529,9 +3564,14 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 			},
 		},
 		{
-			Name:  "coderd_agentstats_currently_reachable_peers",
-			Type:  proto.Stats_Metric_GAUGE,
-			Value: 0,
+			Name: "coderd_agentstats_currently_reachable_peers",
+			Type: proto.Stats_Metric_GAUGE,
+			CheckFn: func(f float64) error {
+				if f == 0 {
+					return nil
+				}
+				return xerrors.Errorf("expected 0, got %f", f)
+			},
 			Labels: []*proto.Stats_Metric_Label{
 				{
 					Name:  "connection_type",
@@ -3540,9 +3580,20 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 			},
 		},
 		{
-			Name:  "coderd_agentstats_startup_script_seconds",
-			Type:  proto.Stats_Metric_GAUGE,
-			Value: 1,
+			Name: "coderd_agentstats_startup_script_seconds",
+			Type: proto.Stats_Metric_GAUGE,
+			CheckFn: func(f float64) error {
+				if f >= 0 {
+					return nil
+				}
+				return xerrors.Errorf("expected >= 0, got %f", f)
+			},
+			Labels: []*proto.Stats_Metric_Label{
+				{
+					Name:  "success",
+					Value: "true",
+				},
+			},
 		},
 	}
 
@@ -3564,11 +3615,10 @@ func TestAgent_Metrics_SSH(t *testing.T) {
 		for _, m := range mf.GetMetric() {
 			assert.Equal(t, expected[i].Name, mf.GetName())
 			assert.Equal(t, expected[i].Type.String(), mf.GetType().String())
-			// Value is max expected
 			if expected[i].Type == proto.Stats_Metric_GAUGE {
-				assert.GreaterOrEqualf(t, expected[i].Value, m.GetGauge().GetValue(), "expected %s to be greater than or equal to %f, got %f", expected[i].Name, expected[i].Value, m.GetGauge().GetValue())
+				assert.NoError(t, expected[i].CheckFn(m.GetGauge().GetValue()), "check fn for %s failed", expected[i].Name)
 			} else if expected[i].Type == proto.Stats_Metric_COUNTER {
-				assert.GreaterOrEqualf(t, expected[i].Value, m.GetCounter().GetValue(), "expected %s to be greater than or equal to %f, got %f", expected[i].Name, expected[i].Value, m.GetCounter().GetValue())
+				assert.NoError(t, expected[i].CheckFn(m.GetCounter().GetValue()), "check fn for %s failed", expected[i].Name)
 			}
 			for j, lbl := range expected[i].Labels {
 				assert.Equal(t, m.GetLabel()[j], &promgo.LabelPair{