feat(executors): keep worktree and local agents running across a backend restart ↗

Launcher keeps agentctl alive when survival is on

The launcher skips the parent-liveness pipe and Pdeathsig when survival is enabled, and Stop becomes a no-op so a graceful restart does not kill the server.

l.cmd.SysProcAttr = buildSysProcAttr(l.startupConfig.AgentSurvivalEnabled)

var pipeWrite *os.File
if l.startupConfig.AgentSurvivalEnabled {
    clearInheritedLivenessPipeEnv(l.cmd)
} else {
    pipeWrite, err = setupLivenessPipe(l.cmd)
    if err != nil {
        return err
    }
}

func (l *Launcher) Stop(ctx context.Context) error {
    if l.startupConfig.AgentSurvivalEnabled {
        l.logger.Info("agent-survival capability engaged; leaving agentctl running instead of stopping it")
        return nil
    }
    // ... close pipe, SIGTERM, wait ...
}

agentctl survives SIGPIPE and detached logging

agentctl ignores SIGPIPE and switches to a bounded diagnostic log file when survival is active, so a closed parent pipe does not kill it.

func applySIGPIPEDisposition(cfg *config.Config, ignore func(...os.Signal)) {
    if !cfg.AgentSurvivalEnabled {
        return
    }
    ignore(syscall.SIGPIPE)
}

func resolveRunLoggingConfig(cfg *config.Config) logger.LoggingConfig {
    if cfg.AgentSurvivalEnabled && cfg.DiagnosticLogPath != "" {
        return diagnosticLoggingConfig(cfg.LogLevel, cfg.LogFormat, cfg.DiagnosticLogPath)
    }
    return logger.LoggingConfig{Level: cfg.LogLevel, Format: cfg.LogFormat, OutputPath: "stdout"}
}

waitForShutdown(log, parentDied, controlServer.ShutdownRequested(), func(ctx context.Context) {
    // ... shutdown instances, janitor, http server ...
})

func waitForShutdown(log *logger.Logger, parentDied <-chan struct{}, ownershipShutdown <-chan struct{}, cleanup func(ctx context.Context)) {
    select {
    case sig := <-sigCh:
        log.Info("received signal", zap.String("signal", sig.String()))
    case <-parentDied:
        log.Warn("parent process died, initiating shutdown")
    case <-ownershipShutdown:
        log.Warn("ownership-shutdown operation invoked, initiating shutdown")
    }
}

Adopt the detached control server on startup

The new backend reads the recorded endpoint, checks identity and credential proof, rotates the credential, and persists the new record before it re-tracks any session.

record, err := store.GetControlServerRecord(ctx)
if err != nil {
    return AdoptionOutcome{Reason: AdoptionReasonNoServer}
}
client, err := newClient(record.Endpoint)
if err != nil {
    return AdoptionOutcome{Reason: AdoptionReasonNoServer}
}
contactedAt := time.Now()
identity, err := withAdoptionRetry(ctx, recoveryReadTimeout, recoveryReadRetries, client.GetIdentity)
if err != nil {
    return AdoptionOutcome{Reason: AdoptionReasonNoServer}
}
if !identityMatchesRecordedServer(identity, record) || len(identity.Capabilities) == 0 {
    return AdoptionOutcome{Reason: AdoptionReasonIdentityMismatch, ContactedAt: contactedAt}
}
credential, err := revealControlServerCredential(ctx, secretStore, record.CredentialSecretID)
if err != nil {
    return AdoptionOutcome{Reason: AdoptionReasonCredentialUnavailable, ContactedAt: contactedAt}
}
if !controlServerHoldsCredential(ctx, client, credential, homeDir, recoveryReadTimeout, recoveryReadRetries) {
    return AdoptionOutcome{Reason: AdoptionReasonIdentityMismatch, ContactedAt: contactedAt}
}
client.SetAuthToken(credential)
rotated, err := withAdoptionRetry(ctx, recoveryReadTimeout, recoveryReadRetries, client.RotateCredential)
if err != nil {
    return AdoptionOutcome{Reason: AdoptionReasonAuthenticationFailed, ContactedAt: contactedAt}
}

secretID, err := storeControlServerCredential(ctx, a.secretStore, a.record.CredentialSecretID, a.rotated.Credential)
if err != nil {
    return AdoptionOutcome{Reason: AdoptionReasonCredentialRotationFailed, ContactedAt: a.contactedAt}
}
updated := &models.ControlServerRecord{
    Endpoint: a.record.Endpoint, ServerIdentity: a.identity.ServerIdentity,
    CredentialSecretID: secretID, Capabilities: a.identity.Capabilities,
    DiagnosticLogPath: recordedDiagnosticLogPath(ctx, a.client, a.recoveryReadTimeout, a.recoveryReadRetries),
}
if err := a.store.UpsertControlServerRecord(ctx, updated); err != nil {
    return AdoptionOutcome{Reason: AdoptionReasonCredentialRotationFailed, ContactedAt: a.contactedAt}
}
// Confirm only after both durable writes succeed
withAdoptionRetry(ctx, a.recoveryReadTimeout, a.recoveryReadRetries, func(c context.Context) (struct{}, error) {
    return struct{}{}, a.client.ConfirmCredentialRotation(c, a.rotated.RotationID)
})

Recovery guard and instance re-tracking

The guard is taken before any control-server contact and blocks concurrent launches for sessions that may still be re-tracked.

func TakeStartupRecoveryGuards(ctx context.Context, runningWriter ExecutorRunningWriter, passthroughLookup PassthroughLookup, log *logger.Logger) *RecoveryGuard {
    guard := NewRecoveryGuard()
    lister, ok := runningWriter.(executorRunningLister)
    if !ok {
        return guard
    }
    records, err := lister.ListExecutorsRunningLiveStandalone(ctx)
    if err != nil {
        log.Warn("failed to read live standalone recovery-inventory records", zap.Error(err))
        return guard
    }
    for _, sessionID := range SessionsToGuard(ctx, sessionIDsFromExecutorRunning(records), passthroughLookup) {
        guard.AcquireOrObserve(sessionID)
    }
    return guard
}

func (r *StandaloneExecutor) RecoverInstances(ctx context.Context, records []*models.ExecutorRunning) ([]*ExecutorInstance, error) {
    instances, err := r.listInstancesWithRetry(ctx)
    if err != nil {
        r.logger.Warn("failed to enumerate standalone instances for recovery; leaving every record to the existing repair path", zap.Error(err))
        return nil, nil
    }
    correlation := CorrelateRecoveryInstances(records, instances)
    pending := pendingLoserCounts(correlation.ToStop, correlation.Winners)
    results := r.dispatchRecoveryStops(correlation.ToStop)
    tracker := &jointFailureTracker{exec: r, winners: winnersBySession}
    r.collectRecoveryStops(ctx, results, correlation.Winners, pending, tracker)
    return r.buildRecoveredInstances(correlation.Winners, indexRecordsBySession(records)), nil
}

Control server ownership and credential rotation

The control server exposes identity, proof, and ownership routes with three-tier auth and an unowned-period reaper that self-terminates when no backend claims it.

var adoptionOnlyPaths = map[string]bool{
    "/api/v1/ownership/rotate":   true,
    "/api/v1/ownership/shutdown": true,
}

cs.router.Use(controlCredentialAuth(cs.credentials, adoptionOnlyPaths, "/health", "/auth/handshake", "/identity", "/ownership/prove"))
cs.router.GET("/identity", m.handleIdentity)
cs.router.POST("/ownership/prove", m.handleOwnershipProve)
api.POST("/ownership/claim", m.handleOwnershipClaim)
api.POST("/ownership/rotate", m.handleCredentialRotate)
api.POST("/ownership/confirm", m.handleCredentialConfirm)
api.POST("/ownership/shutdown", m.handleOwnershipShutdown)

func startUnownedReaperIfEnabled(cfg *config.Config, reaper unownedReaper) (stop func()) {
    if cfg.AgentSurvivalEnabled {
        reaper.StartUnownedReaper()
    }
    return reaper.StopUnownedReaper
}