fix: record failed lifecycle operations

Adds the ability to record partial failures from resource lifecycle
methods by adding a new `ResourceData.Error` field. This field is
populated when an error occurs during `Create` and `Update` events. In
this case, we'll persist failed resource back to the state before we
terminate the plan. Combined with the existing `NeedsCreate` field, we
can record when an operation needs to be retried on the next run.
Recording partial failures also makes it so resources can be properly
deleted if the user opts to perform a delete rather than a retry.

PLAT-212

Author: jason-lynch

changes/unreleased/Fixed-20260119-095716.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: Fixed a bug that prevented database deletion when we failed to create the Swarm service.
+time: 2026-01-19T09:57:16.746487-05:00

server/internal/resource/event.go

@@ -35,6 +35,13 @@ type Event struct {
 	Diff     jsondiff.Patch `json:"diff,omitempty"`
 }
 
+func (e *Event) ResourceError() error {
+	if e.Resource != nil && e.Resource.Error != "" {
+		return errors.New(e.Resource.Error)
+	}
+	return nil
+}
+
 // Apply applies this event to its resource. It does not modify the state in the
 // given Context.
 func (e *Event) Apply(ctx context.Context, rc *Context) error {
@@ -58,7 +65,10 @@ func (e *Event) Apply(ctx context.Context, rc *Context) error {
 }
 
 func (e *Event) refresh(ctx context.Context, rc *Context, resource Resource) error {
-	var needsRecreate bool
+	// Retain the original Error and NeedsRecreate fields so that they're
+	// available for planCreates.
+	needsRecreate := e.Resource.NeedsRecreate
+	applyErr := e.Resource.Error
 
 	err := resource.Refresh(ctx, rc)
 	if errors.Is(err, ErrNotFound) {
@@ -71,37 +81,48 @@ func (e *Event) refresh(ctx context.Context, rc *Context, resource Resource) err
 	if err != nil {
 		return err
 	}
+
 	updated.NeedsRecreate = needsRecreate
+	updated.Error = applyErr
 
 	e.Resource = updated
 
 	return nil
 }
 
 func (e *Event) create(ctx context.Context, rc *Context, resource Resource) error {
+	var needsRecreate bool
+	var applyErr string
+
 	if err := resource.Create(ctx, rc); err != nil {
-		return fmt.Errorf("failed to create resource %s: %w", resource.Identifier(), err)
+		needsRecreate = true
+		applyErr = fmt.Sprintf("failed to create resource %s: %s", resource.Identifier(), err.Error())
 	}
 
 	updated, err := ToResourceData(resource)
 	if err != nil {
 		return err
 	}
+	updated.NeedsRecreate = needsRecreate
+	updated.Error = applyErr
 
 	e.Resource = updated
 
 	return nil
 }
 
 func (e *Event) update(ctx context.Context, rc *Context, resource Resource) error {
+	var applyErr string
+
 	if err := resource.Update(ctx, rc); err != nil {
-		return fmt.Errorf("failed to update resource %s: %w", resource.Identifier(), err)
+		applyErr = fmt.Sprintf("failed to update resource %s: %s", resource.Identifier(), err.Error())
 	}
 
 	updated, err := ToResourceData(resource)
 	if err != nil {
 		return err
 	}
+	updated.Error = applyErr
 
 	e.Resource = updated
 
@@ -110,6 +131,10 @@ func (e *Event) update(ctx context.Context, rc *Context, resource Resource) erro
 
 func (e *Event) delete(ctx context.Context, rc *Context, resource Resource) error {
 	if err := resource.Delete(ctx, rc); err != nil {
+		// We need to return an error here to indicate that this event should
+		// not be applied to the state. Applying a delete event to the state
+		// removes the resource, so if we didn't return the error it would be
+		// impossible to retry this operation.
 		return fmt.Errorf("failed to delete resource %s: %w", resource.Identifier(), err)
 	}
 

server/internal/resource/event_test.go

@@ -24,13 +24,24 @@ func TestEvent(t *testing.T) {
 			eventType                     resource.EventType
 			notFound                      bool
 			lifecycleError                string
+			originalResourceError         string
+			originalResourceNeedsRecreate bool
 			expectedErr                   string
+			expectedResourceError         string
 			expectedResourceNeedsRecreate bool
 		}{
 			{
 				name:      "refresh success",
 				eventType: resource.EventTypeRefresh,
 			},
+			{
+				name:                          "refresh success retains Error and NeedsRecreate",
+				eventType:                     resource.EventTypeRefresh,
+				originalResourceError:         "some error",
+				originalResourceNeedsRecreate: true,
+				expectedResourceError:         "some error",
+				expectedResourceNeedsRecreate: true,
+			},
 			{
 				name:                          "refresh not found",
 				eventType:                     resource.EventTypeRefresh,
@@ -48,25 +59,44 @@ func TestEvent(t *testing.T) {
 				eventType: resource.EventTypeCreate,
 			},
 			{
-				name:           "create failed",
-				eventType:      resource.EventTypeCreate,
-				lifecycleError: "some error",
-				expectedErr:    "failed to create resource test_resource::test: some error",
+				name:                          "create success clears Error and NeedsRecreate",
+				eventType:                     resource.EventTypeCreate,
+				originalResourceError:         "some error",
+				originalResourceNeedsRecreate: true,
+			},
+			{
+				name:                          "create failed",
+				eventType:                     resource.EventTypeCreate,
+				lifecycleError:                "some error",
+				expectedResourceError:         "failed to create resource test_resource::test: some error",
+				expectedResourceNeedsRecreate: true,
 			},
 			{
 				name:      "update success",
 				eventType: resource.EventTypeUpdate,
 			},
 			{
-				name:           "update failed",
-				eventType:      resource.EventTypeUpdate,
-				lifecycleError: "some error",
-				expectedErr:    "failed to update resource test_resource::test: some error",
+				name:                          "update success clears Error and NeedsRecreate",
+				eventType:                     resource.EventTypeUpdate,
+				originalResourceError:         "some error",
+				originalResourceNeedsRecreate: true,
+			},
+			{
+				name:                  "update failed",
+				eventType:             resource.EventTypeUpdate,
+				lifecycleError:        "some error",
+				expectedResourceError: "failed to update resource test_resource::test: some error",
 			},
 			{
 				name:      "delete success",
 				eventType: resource.EventTypeDelete,
 			},
+			{
+				name:                          "delete success clears Error and NeedsRecreate",
+				eventType:                     resource.EventTypeDelete,
+				originalResourceError:         "some error",
+				originalResourceNeedsRecreate: true,
+			},
 			{
 				name:           "delete failed",
 				eventType:      resource.EventTypeDelete,
@@ -82,8 +112,11 @@ func TestEvent(t *testing.T) {
 				}
 
 				original := r.data(t)
+				original.Error = tc.originalResourceError
+				original.NeedsRecreate = tc.originalResourceNeedsRecreate
 
 				expected := r.data(t)
+				expected.Error = tc.expectedResourceError
 				expected.NeedsRecreate = tc.expectedResourceNeedsRecreate
 
 				event := &resource.Event{

server/internal/resource/resource.go

@@ -43,6 +43,7 @@ type ResourceData struct {
 	DiffIgnore      []string        `json:"diff_ignore"`
 	ResourceVersion string          `json:"resource_version"`
 	PendingDeletion bool            `json:"pending_deletion"`
+	Error           string          `json:"error"`
 }
 
 func (r *ResourceData) Diff(other *ResourceData) (jsondiff.Patch, error) {
@@ -70,6 +71,7 @@ func (r *ResourceData) Clone() *ResourceData {
 		DiffIgnore:      slices.Clone(r.DiffIgnore),
 		ResourceVersion: r.ResourceVersion,
 		PendingDeletion: r.PendingDeletion,
+		Error:           r.Error,
 	}
 }
 

server/internal/resource/state.go

@@ -297,6 +297,12 @@ func (s *State) planCreates(options PlanOptions, desired *State) (Plan, error) {
 					Resource: resource,
 					Reason:   EventReasonNeedsRecreate,
 				}
+			case currentResource.Error != "":
+				event = &Event{
+					Type:     EventTypeUpdate,
+					Resource: resource,
+					Reason:   EventReasonHasError,
+				}
 			case options.ForceUpdate:
 				event = &Event{
 					Type:     EventTypeUpdate,

server/internal/resource/state_test.go

@@ -422,6 +422,97 @@ func TestState(t *testing.T) {
 				assert.ElementsMatch(t, expected[i], phase)
 			}
 		})
+		t.Run("error from previous create", func(t *testing.T) {
+			resource1Data, err := resource.ToResourceData(&testResource{
+				ID: "test1",
+				TestDependencies: []resource.Identifier{
+					testResourceID("test2"),
+				},
+			})
+			require.NoError(t, err)
+
+			resource2Data, err := resource.ToResourceData(&testResource{
+				ID: "test2",
+			})
+			require.NoError(t, err)
+
+			current := resource.NewState()
+			desired := resource.NewState()
+
+			resource2WithError := resource2Data.Clone()
+			resource2WithError.NeedsRecreate = true
+			resource2WithError.Error = "some error"
+
+			current.Add(resource2WithError)
+			desired.Add(resource1Data, resource2Data)
+
+			plan, err := current.Plan(resource.PlanOptions{}, desired)
+			assert.NoError(t, err)
+
+			expected := resource.Plan{
+				{
+					{
+						Type:     resource.EventTypeCreate,
+						Resource: resource2Data,
+						Reason:   resource.EventReasonNeedsRecreate,
+					},
+				},
+				{
+					{
+						Type:     resource.EventTypeCreate,
+						Resource: resource1Data,
+						Reason:   resource.EventReasonDoesNotExist,
+					},
+				},
+			}
+
+			assert.Equal(t, expected, plan)
+		})
+		t.Run("error from previous update", func(t *testing.T) {
+			resource1Data, err := resource.ToResourceData(&testResource{
+				ID: "test1",
+				TestDependencies: []resource.Identifier{
+					testResourceID("test2"),
+				},
+			})
+			require.NoError(t, err)
+
+			resource2Data, err := resource.ToResourceData(&testResource{
+				ID: "test2",
+			})
+			require.NoError(t, err)
+
+			current := resource.NewState()
+			desired := resource.NewState()
+
+			resource2WithError := resource2Data.Clone()
+			resource2WithError.Error = "some error"
+
+			current.Add(resource1Data, resource2WithError)
+			desired.Add(resource1Data, resource2Data)
+
+			plan, err := current.Plan(resource.PlanOptions{}, desired)
+			assert.NoError(t, err)
+
+			expected := resource.Plan{
+				{
+					{
+						Type:     resource.EventTypeUpdate,
+						Resource: resource2Data,
+						Reason:   resource.EventReasonHasError,
+					},
+				},
+				{
+					{
+						Type:     resource.EventTypeUpdate,
+						Resource: resource1Data,
+						Reason:   resource.EventReasonDependencyUpdated,
+					},
+				},
+			}
+
+			assert.Equal(t, expected, plan)
+		})
 		t.Run("missing create dependency", func(t *testing.T) {
 			resource1 := &testResource{
 				ID: "test1",

server/internal/workflows/activities/apply_event.go

@@ -139,9 +139,17 @@ func (a *Activities) logResourceEvent(
 
 	fields["duration_ms"] = duration.Milliseconds()
 
-	if applyErr != nil {
+	var applyErrStr string
+	switch {
+	case applyErr != nil:
+		applyErrStr = applyErr.Error()
+	case event.Type != resource.EventTypeRefresh:
+		applyErrStr = event.Resource.Error
+	}
+
+	if applyErrStr != "" {
 		fields["success"] = false
-		fields["error"] = applyErr.Error()
+		fields["error"] = applyErrStr
 
 		err := log(task.LogEntry{
 			Message: fmt.Sprintf("error while %s resource %s", verb, resourceIdentifier),
@@ -153,18 +161,17 @@ func (a *Activities) logResourceEvent(
 				fmt.Errorf("failed to record event error: %w", err),
 			)
 		}
-		return applyErr
-	}
-
-	fields["success"] = true
+	} else {
+		fields["success"] = true
 
-	err = log(task.LogEntry{
-		Message: fmt.Sprintf("finished %s resource %s (took %s)", verb, resourceIdentifier, duration),
-		Fields:  fields,
-	})
-	if err != nil {
-		return fmt.Errorf("failed to record event completion: %w", err)
+		err = log(task.LogEntry{
+			Message: fmt.Sprintf("finished %s resource %s (took %s)", verb, resourceIdentifier, duration),
+			Fields:  fields,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to record event completion: %w", err)
+		}
 	}
 
-	return nil
+	return applyErr
 }

server/internal/workflows/common.go

@@ -75,6 +75,13 @@ func (w *Workflows) applyEvents(
 				event := phase[i]
 				errs = append(errs, fmt.Errorf("failed to apply %s event from %s to state: %w", event.Type, event.Resource.Identifier, err))
 			}
+			if err := out.Event.ResourceError(); err != nil && out.Event.Type != resource.EventTypeRefresh {
+				// Returns errors that originated from the resource's lifecycle
+				// method. They're already formatted with the event type and the
+				// resource identifier. We still want to apply the event to the
+				// state to record partial creates/updates.
+				errs = append(errs, err)
+			}
 		}
 		if err := errors.Join(errs...); err != nil {
 			return fmt.Errorf("failed while modifying resources: %w", err)