PR #3525
Sections
Review

fix(office): stop the stuck-parent wake sweep from failing every tick on Postgres

main ← feature/office-pg-liststuckparents-6a49a8 2 files +237 −3 PR #3525 ↗

ListStuckParents now uses dialect-aware SQL so the stuck-parent wake sweep parses and runs on Postgres instead of failing every tick.

Why this change

ParentWakeReconciler runs ListStuckParents every tick to recover stuck parents, but the query uses SQLite-only GROUP_CONCAT, json_extract, and IS NOT with a column. Postgres rejects the whole statement at parse time, so the reconciler recovers nothing.

What it does

Architecture, end to end

ParentWakeReconciler ticks ListStuckParents. The query now branches on driver before it hits the database, so the same sweep runs on SQLite and Postgres.

flowchart LR
  Tick[ParentWakeReconciler Tick] --> List[ListStuckParents]
  List --> Driver{DriverName}
  Driver --> Agg[childSetKeyAggregate]
  Driver --> JSON[dialect.JSONExtract]
  Driver --> Cmp[IS DISTINCT FROM]
  Agg --> DB[(tasks / parent_child_wake_receipts / runs)]
  JSON --> DB
  Cmp --> DB
  DB --> Wake[queueChildrenCompletedRun]

Key code changes

Drag to pan. Use the + and − buttons to zoom. Click a node to open the full code. The arrows show how the parts interact.

drag to pan · +/− to zoom · click a node for details

func childSetKeyAggregate(driver string) string
Click for details →

Returns STRING_AGG with ORDER BY on Postgres and GROUP_CONCAT on SQLite so the SQL key matches the Go key byte for byte.

Helper
// childSetKeyAggregate renders the "id:state" concatenation ListStuckParents
// compares against a stored receipt. Its output must match formatChildSetKey
// byte for byte: receipts are written from the Go form and compared against
// this SQL form, so any difference in separator or ordering makes every
// receipt look stale and re-wakes the parent on every tick. Postgres does not
// inherit input ordering from the subquery's ORDER BY, so the ordering is
// restated inside the aggregate.
func childSetKeyAggregate(driver string) string {
	if dialect.IsPostgres(driver) {
		return `STRING_AGG(c.id || ':' || c.state, ',' ORDER BY c.id)`
	}
	return `GROUP_CONCAT(c.id || ':' || c.state, ',')`
}
Usage in ListStuckParents
				COALESCE((
					SELECT GROUP_CONCAT(c.id || ':' || c.state, ',')
					SELECT `+childSetKeyAggregate(driver)+`
					FROM (
						SELECT id, state FROM tasks
						WHERE parent_id = p.id AND archived_at IS NULL
						ORDER BY id
					) c
				), '') AS child_set_key,
func (r *Repository) ListStuckParents(ctx context.Context, reason string, limit int) ([]StuckParentCandidate, error)
Click for details →

Wires driver-aware SQL for receipt comparison and JSON extraction so the sweep parses on Postgres.

Driver wiring
func (r *Repository) ListStuckParents(ctx context.Context, reason string, limit int) ([]StuckParentCandidate, error) {
	driver := r.ro.DriverName()
	var rows []StuckParentCandidate
	err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(`
Receipt comparison
		WHERE s.assignee_agent_profile_id != ''
		  AND ap.status NOT IN ('paused', 'stopped', 'pending_approval')
		  AND (
		      r.child_set_key IS NOT s.child_set_key
		      r.child_set_key IS DISTINCT FROM s.child_set_key
		      OR (
		          NOT EXISTS (
		              SELECT 1 FROM runs delivered
		              WHERE delivered.id = r.delivered_run_id
		          )
		          AND COALESCE(r.delivery_operation_id, '') = ''
		      )
		  )
JSON extraction
		  AND NOT EXISTS (
		      SELECT 1 FROM runs w
		      WHERE json_extract(w.payload, '$.task_id') = s.parent_task_id
		      WHERE `+dialect.JSONExtract(driver, "w.payload", "task_id")+` = s.parent_task_id
		        AND w.reason = ?
		        AND (
		            w.status IN ('queued', 'claimed')
		            OR (
		                w.status IN ('finished', 'failed', 'cancelled')
		                AND w.requested_at >= s.newest_child_updated_at
		            )
		        )
		  )
func TestPostgresListStuckParents(t *testing.T)
Click for details →

Proves the query runs on Postgres, the SQL aggregate matches Go, and covered parents are excluded.

Basic sweep
func TestPostgresListStuckParents(t *testing.T) {
	repo, ctx := newPostgresWakeRepo(t)
	wantKey := seedPostgresStuckParent(t, ctx, repo, "pg-parent-1", "pg-ws-1")
	rows, err := repo.ListStuckParents(ctx, "task_children_completed", 5)
	if err != nil {
		t.Fatalf("ListStuckParents: %v", err)
	}
	if len(rows) != 1 {
		t.Fatalf("len(rows) = %d, want 1", len(rows))
	}
	if rows[0].ChildSetKey != wantKey {
		t.Errorf("ChildSetKey = %q, want %q", rows[0].ChildSetKey, wantKey)
	}
}
Key parity
func TestPostgresListStuckParentsChildSetKeyMatchesGo(t *testing.T) {
	repo, ctx := newPostgresWakeRepo(t)
	seedPostgresTask(t, ctx, repo, "pg-parent-2", "pg-ws-2")
	execPostgres(t, ctx, repo,
		`UPDATE tasks SET project_id = 'office-project' WHERE id = ?`, "pg-parent-2")
	for _, childID := range []string{"pg-parent-2-child-c", "pg-parent-2-child-a", "pg-parent-2-child-b"} {
		seedPostgresTask(t, ctx, repo, childID, "pg-ws-2")
		execPostgres(t, ctx, repo,
			`UPDATE tasks SET parent_id = ?, state = 'COMPLETED' WHERE id = ?`, "pg-parent-2", childID)
	}
	seedPostgresRunner(t, ctx, repo, "pg-parent-2")
	goKey, _ := repo.GetChildSetKey(ctx, "pg-parent-2")
	rows, _ := repo.ListStuckParents(ctx, "task_children_completed", 5)
	if rows[0].ChildSetKey != goKey {
		t.Errorf("ChildSetKey = %q, want %q (GetChildSetKey)", rows[0].ChildSetKey, goKey)
	}
}
Exclusion
func TestPostgresListStuckParentsExcludesCoveredParent(t *testing.T) {
	repo, ctx := newPostgresWakeRepo(t)
	key := seedPostgresStuckParent(t, ctx, repo, "pg-parent-3", "pg-ws-3")
	execPostgres(t, ctx, repo, `
		INSERT INTO parent_child_wake_receipts (parent_task_id, child_set_key, delivery_operation_id, delivered_at)
		VALUES (?, ?, 'op-1', ?)
	`, "pg-parent-3", key, time.Now().UTC())
	rows, _ := repo.ListStuckParents(ctx, "task_children_completed", 5)
	if len(rows) != 0 {
		t.Fatalf("len(rows) = %d, want 0 — receipt already covers this child set", len(rows))
	}
}
Read the changes as a list

Dialect-aware child-set aggregation

apps/backend/internal/office/repository/sqlite/wake_receipts.go

Returns STRING_AGG with ORDER BY on Postgres and GROUP_CONCAT on SQLite so the SQL key matches the Go key byte for byte.

Helper
// childSetKeyAggregate renders the "id:state" concatenation ListStuckParents
// compares against a stored receipt. Its output must match formatChildSetKey
// byte for byte: receipts are written from the Go form and compared against
// this SQL form, so any difference in separator or ordering makes every
// receipt look stale and re-wakes the parent on every tick. Postgres does not
// inherit input ordering from the subquery's ORDER BY, so the ordering is
// restated inside the aggregate.
func childSetKeyAggregate(driver string) string {
	if dialect.IsPostgres(driver) {
		return `STRING_AGG(c.id || ':' || c.state, ',' ORDER BY c.id)`
	}
	return `GROUP_CONCAT(c.id || ':' || c.state, ',')`
}
Usage in ListStuckParents
				COALESCE((
					SELECT GROUP_CONCAT(c.id || ':' || c.state, ',')
					SELECT `+childSetKeyAggregate(driver)+`
					FROM (
						SELECT id, state FROM tasks
						WHERE parent_id = p.id AND archived_at IS NULL
						ORDER BY id
					) c
				), '') AS child_set_key,

Postgres-safe receipt and run filters

apps/backend/internal/office/repository/sqlite/wake_receipts.go

Wires driver-aware SQL for receipt comparison and JSON extraction so the sweep parses on Postgres.

Driver wiring
func (r *Repository) ListStuckParents(ctx context.Context, reason string, limit int) ([]StuckParentCandidate, error) {
	driver := r.ro.DriverName()
	var rows []StuckParentCandidate
	err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(`
Receipt comparison
		WHERE s.assignee_agent_profile_id != ''
		  AND ap.status NOT IN ('paused', 'stopped', 'pending_approval')
		  AND (
		      r.child_set_key IS NOT s.child_set_key
		      r.child_set_key IS DISTINCT FROM s.child_set_key
		      OR (
		          NOT EXISTS (
		              SELECT 1 FROM runs delivered
		              WHERE delivered.id = r.delivered_run_id
		          )
		          AND COALESCE(r.delivery_operation_id, '') = ''
		      )
		  )
JSON extraction
		  AND NOT EXISTS (
		      SELECT 1 FROM runs w
		      WHERE json_extract(w.payload, '$.task_id') = s.parent_task_id
		      WHERE `+dialect.JSONExtract(driver, "w.payload", "task_id")+` = s.parent_task_id
		        AND w.reason = ?
		        AND (
		            w.status IN ('queued', 'claimed')
		            OR (
		                w.status IN ('finished', 'failed', 'cancelled')
		                AND w.requested_at >= s.newest_child_updated_at
		            )
		        )
		  )

Postgres test suite for the sweep

apps/backend/internal/office/repository/sqlite/wake_receipts_postgres_test.go

Proves the query runs on Postgres, the SQL aggregate matches Go, and covered parents are excluded.

Basic sweep
func TestPostgresListStuckParents(t *testing.T) {
	repo, ctx := newPostgresWakeRepo(t)
	wantKey := seedPostgresStuckParent(t, ctx, repo, "pg-parent-1", "pg-ws-1")
	rows, err := repo.ListStuckParents(ctx, "task_children_completed", 5)
	if err != nil {
		t.Fatalf("ListStuckParents: %v", err)
	}
	if len(rows) != 1 {
		t.Fatalf("len(rows) = %d, want 1", len(rows))
	}
	if rows[0].ChildSetKey != wantKey {
		t.Errorf("ChildSetKey = %q, want %q", rows[0].ChildSetKey, wantKey)
	}
}
Key parity
func TestPostgresListStuckParentsChildSetKeyMatchesGo(t *testing.T) {
	repo, ctx := newPostgresWakeRepo(t)
	seedPostgresTask(t, ctx, repo, "pg-parent-2", "pg-ws-2")
	execPostgres(t, ctx, repo,
		`UPDATE tasks SET project_id = 'office-project' WHERE id = ?`, "pg-parent-2")
	for _, childID := range []string{"pg-parent-2-child-c", "pg-parent-2-child-a", "pg-parent-2-child-b"} {
		seedPostgresTask(t, ctx, repo, childID, "pg-ws-2")
		execPostgres(t, ctx, repo,
			`UPDATE tasks SET parent_id = ?, state = 'COMPLETED' WHERE id = ?`, "pg-parent-2", childID)
	}
	seedPostgresRunner(t, ctx, repo, "pg-parent-2")
	goKey, _ := repo.GetChildSetKey(ctx, "pg-parent-2")
	rows, _ := repo.ListStuckParents(ctx, "task_children_completed", 5)
	if rows[0].ChildSetKey != goKey {
		t.Errorf("ChildSetKey = %q, want %q (GetChildSetKey)", rows[0].ChildSetKey, goKey)
	}
}
Exclusion
func TestPostgresListStuckParentsExcludesCoveredParent(t *testing.T) {
	repo, ctx := newPostgresWakeRepo(t)
	key := seedPostgresStuckParent(t, ctx, repo, "pg-parent-3", "pg-ws-3")
	execPostgres(t, ctx, repo, `
		INSERT INTO parent_child_wake_receipts (parent_task_id, child_set_key, delivery_operation_id, delivered_at)
		VALUES (?, ?, 'op-1', ?)
	`, "pg-parent-3", key, time.Now().UTC())
	rows, _ := repo.ListStuckParents(ctx, "task_children_completed", 5)
	if len(rows) != 0 {
		t.Fatalf("len(rows) = %d, want 0 — receipt already covers this child set", len(rows))
	}
}

Data and storage

The sweep compares a SQL-computed child-set key against the last receipt. The key must match the Go form byte for byte.

FieldTypeNotes
parent_child_wake_receipts.child_set_keytextdeterministic id:state list, written by Go, compared by SQL
parent_child_wake_receipts.delivered_run_idtextlegacy run id; existence proves delivery
parent_child_wake_receipts.delivery_operation_idtextworkflow engine operation id; empty means legacy path
runs.payload.task_idjsonextracted via dialect.JSONExtract to find in-flight wakes
tasks.statetextCOMPLETED / CANCELLED are terminal; others block the sweep

Risk

3 / 10 Low
1 low5 medium10 high

Why this score

  • Small SQL-only fix with no schema or API change; rollback is a revert.
  • Query runs every tick, but failure was total on Postgres and fix is narrow.
  • New Postgres tests cover parse, key parity, and exclusion; SQLite path unchanged.

Trade-offs and review notes

Where to look first

  1. Check childSetKeyAggregate output matches formatChildSetKey separator and ORDER BY.
  2. Confirm IS DISTINCT FROM and dialect.JSONExtract produce correct SQL for both drivers.
  3. Verify Postgres tests seed Office-owned parents and runners correctly and assert exclusion.