feat: cancel pending prebuilds from non-active template versions (coder#20387)

## Description

This PR introduces an optimization to automatically cancel pending
prebuild-related jobs from non-active template versions in the
reconciliation loop.

## Problem

Currently, when a template is configured with more prebuild instances
than available provisioners, the provisioner queue can become flooded
with pending prebuild jobs. This issue is worsened when
provisioning/deprovisioning operations take a long time.

When the prebuild reconciliation loop generates jobs faster than
provisioners can process them, pending jobs accumulate in the queue.
Since prebuilt workspaces should always run the latest active template
version, pending prebuild jobs from non-active versions become obsolete
once a new version is promoted.

## Solution

The reconciliation loop cancels pending prebuild-related jobs from
non-active template versions that match the following criteria:

* Build number: 1 (initial build created by the reconciliation loop)
* Job status: `pending`
* Not yet picked up by a provisioner (`worker_id` is `NULL`)
* Owned by the prebuilds system user
* Workspace transition: `start`

This prevents the queue from being cluttered with stale prebuild jobs
that would provision workspaces on an outdated template version that
would consequently need to be deprovisioned.

## Changes

* Added new SQL query `CountPendingNonActivePrebuilds` to identify
presets with pending jobs from non-active versions
* Added new SQL query `UpdatePrebuildProvisionerJobWithCancel` to cancel
jobs for a specific preset
* New reconciliation action type `ActionTypeCancelPending` handles the
cancellation logic
* Cancellation is non-blocking: failures to cancel prebuild jobs are
logged as errors and don't prevent other reconciliation actions

## Follow-up PR

Canceling pending prebuild jobs leaves workspaces in a Canceled state.
While no Terraform resources need to be destroyed (since jobs were
canceled before provisioning started), these database records should
still be cleaned up. This will be addressed in a follow-up PR.

Closes: coder#20242

Author: ssncferreira

coderd/database/dbauthz/dbauthz.go

@@ -1512,6 +1512,13 @@ func (q *querier) CountInProgressPrebuilds(ctx context.Context) ([]database.Coun
 	return q.db.CountInProgressPrebuilds(ctx)
 }
 
+func (q *querier) CountPendingNonActivePrebuilds(ctx context.Context) ([]database.CountPendingNonActivePrebuildsRow, error) {
+	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceWorkspace.All()); err != nil {
+		return nil, err
+	}
+	return q.db.CountPendingNonActivePrebuilds(ctx)
+}
+
 func (q *querier) CountUnreadInboxNotificationsByUserID(ctx context.Context, userID uuid.UUID) (int64, error) {
 	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceInboxNotification.WithOwner(userID.String())); err != nil {
 		return 0, err
@@ -4875,6 +4882,14 @@ func (q *querier) UpdateOrganizationDeletedByID(ctx context.Context, arg databas
 	return deleteQ(q.log, q.auth, q.db.GetOrganizationByID, deleteF)(ctx, arg.ID)
 }
 
+func (q *querier) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]uuid.UUID, error) {
+	// Prebuild operation for canceling pending prebuild jobs from non-active template versions
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourcePrebuiltWorkspace); err != nil {
+		return []uuid.UUID{}, err
+	}
+	return q.db.UpdatePrebuildProvisionerJobWithCancel(ctx, arg)
+}
+
 func (q *querier) UpdatePresetPrebuildStatus(ctx context.Context, arg database.UpdatePresetPrebuildStatusParams) error {
 	preset, err := q.db.GetPresetByID(ctx, arg.PresetID)
 	if err != nil {

coderd/database/dbauthz/dbauthz_test.go

@@ -641,6 +641,16 @@ func (s *MethodTestSuite) TestProvisionerJob() {
 		dbm.EXPECT().UpdateProvisionerJobWithCancelByID(gomock.Any(), arg).Return(nil).AnyTimes()
 		check.Args(arg).Asserts(v.RBACObject(tpl), []policy.Action{policy.ActionRead, policy.ActionUpdate}).Returns()
 	}))
+	s.Run("UpdatePrebuildProvisionerJobWithCancel", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		arg := database.UpdatePrebuildProvisionerJobWithCancelParams{
+			PresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true},
+			Now:      dbtime.Now(),
+		}
+		jobIDs := []uuid.UUID{uuid.New(), uuid.New()}
+
+		dbm.EXPECT().UpdatePrebuildProvisionerJobWithCancel(gomock.Any(), arg).Return(jobIDs, nil).AnyTimes()
+		check.Args(arg).Asserts(rbac.ResourcePrebuiltWorkspace, policy.ActionUpdate).Returns(jobIDs)
+	}))
 	s.Run("GetProvisionerJobsByIDs", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
 		org := testutil.Fake(s.T(), faker, database.Organization{})
 		org2 := testutil.Fake(s.T(), faker, database.Organization{})
@@ -3758,6 +3768,10 @@ func (s *MethodTestSuite) TestPrebuilds() {
 		dbm.EXPECT().CountInProgressPrebuilds(gomock.Any()).Return([]database.CountInProgressPrebuildsRow{}, nil).AnyTimes()
 		check.Args().Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead)
 	}))
+	s.Run("CountPendingNonActivePrebuilds", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
+		dbm.EXPECT().CountPendingNonActivePrebuilds(gomock.Any()).Return([]database.CountPendingNonActivePrebuildsRow{}, nil).AnyTimes()
+		check.Args().Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead)
+	}))
 	s.Run("GetPresetsAtFailureLimit", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
 		dbm.EXPECT().GetPresetsAtFailureLimit(gomock.Any(), int64(0)).Return([]database.GetPresetsAtFailureLimitRow{}, nil).AnyTimes()
 		check.Args(int64(0)).Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights)

coderd/database/dbfake/dbfake.go

@@ -55,14 +55,10 @@ type WorkspaceBuildBuilder struct {
 	resources  []*sdkproto.Resource
 	params     []database.WorkspaceBuildParameter
 	agentToken string
-	dispo      workspaceBuildDisposition
+	jobStatus  database.ProvisionerJobStatus
 	taskAppID  uuid.UUID
 }
 
-type workspaceBuildDisposition struct {
-	starting bool
-}
-
 // WorkspaceBuild generates a workspace build for the provided workspace.
 // Pass a database.Workspace{} with a nil ID to also generate a new workspace.
 // Omitting the template ID on a workspace will also generate a new template
@@ -145,8 +141,17 @@ func (b WorkspaceBuildBuilder) WithTask(seed *sdkproto.App) WorkspaceBuildBuilde
 }
 
 func (b WorkspaceBuildBuilder) Starting() WorkspaceBuildBuilder {
-	//nolint: revive // returns modified struct
-	b.dispo.starting = true
+	b.jobStatus = database.ProvisionerJobStatusRunning
+	return b
+}
+
+func (b WorkspaceBuildBuilder) Pending() WorkspaceBuildBuilder {
+	b.jobStatus = database.ProvisionerJobStatusPending
+	return b
+}
+
+func (b WorkspaceBuildBuilder) Canceled() WorkspaceBuildBuilder {
+	b.jobStatus = database.ProvisionerJobStatusCanceled
 	return b
 }
 
@@ -231,7 +236,11 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 	require.NoError(b.t, err, "insert job")
 	b.logger.Debug(context.Background(), "inserted provisioner job", slog.F("job_id", job.ID))
 
-	if b.dispo.starting {
+	switch b.jobStatus {
+	case database.ProvisionerJobStatusPending:
+		// Provisioner jobs are created in 'pending' status
+		b.logger.Debug(context.Background(), "pending the provisioner job")
+	case database.ProvisionerJobStatusRunning:
 		// might need to do this multiple times if we got a template version
 		// import job as well
 		b.logger.Debug(context.Background(), "looping to acquire provisioner job")
@@ -255,7 +264,23 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse {
 				break
 			}
 		}
-	} else {
+	case database.ProvisionerJobStatusCanceled:
+		// Set provisioner job status to 'canceled'
+		b.logger.Debug(context.Background(), "canceling the provisioner job")
+		err = b.db.UpdateProvisionerJobWithCancelByID(ownerCtx, database.UpdateProvisionerJobWithCancelByIDParams{
+			ID: jobID,
+			CanceledAt: sql.NullTime{
+				Time:  dbtime.Now(),
+				Valid: true,
+			},
+			CompletedAt: sql.NullTime{
+				Time:  dbtime.Now(),
+				Valid: true,
+			},
+		})
+		require.NoError(b.t, err, "cancel job")
+	default:
+		// By default, consider jobs in 'succeeded' status
 		b.logger.Debug(context.Background(), "completing the provisioner job")
 		err = b.db.UpdateProvisionerJobWithCompleteByID(ownerCtx, database.UpdateProvisionerJobWithCompleteByIDParams{
 			ID:        job.ID,
@@ -571,6 +596,12 @@ func (t TemplateVersionBuilder) Do() TemplateVersionResponse {
 		t.params[i] = dbgen.TemplateVersionParameter(t.t, t.db, param)
 	}
 
+	// Update response with template and version
+	if resp.Template.ID == uuid.Nil && version.TemplateID.Valid {
+		template, err := t.db.GetTemplateByID(ownerCtx, version.TemplateID.UUID)
+		require.NoError(t.t, err)
+		resp.Template = template
+	}
 	resp.TemplateVersion = version
 	return resp
 }