From 98fb34c387c245d5903d27b1968b6111f753db2c Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 15 Oct 2026 19:40:51 -0700 Subject: [PATCH 1/3] fix(mothership): hold desktop leases, the resume wait and first claims to the call deadline --- .../application/executor.integration.ts | 38 ++++++++++++ .../lib/mothership/async-runs/repository.ts | 11 +++- .../mothership/request/lifecycle/run.test.ts | 35 +++++++++++ .../lib/mothership/request/lifecycle/run.ts | 15 ++--- .../mothership/request/tools/executor.test.ts | 61 +++++++++++++++++++ .../lib/mothership/request/tools/executor.ts | 12 ++++ 6 files changed, 164 insertions(+), 8 deletions(-) diff --git a/apps/sim/lib/desktop/application/executor.integration.ts b/apps/sim/lib/desktop/application/executor.integration.ts index edaa8ff5cb4..36b91945d95 100644 --- a/apps/sim/lib/desktop/application/executor.integration.ts +++ b/apps/sim/lib/desktop/application/executor.integration.ts @@ -54,8 +54,10 @@ import { } from '@/lib/desktop/executor/errors' import { isDesktopPresent } from '@/lib/desktop/executor/presence' import { claimDesktopToolCall, requestRunStop } from '@/lib/mothership/async-runs/repository' +import { TOOL_WATCHDOG_RESUME_GRACE_MS } from '@/lib/mothership/constants' import { waitForToolConfirmation } from '@/lib/mothership/persistence/tool-confirm' import { unsealClientToolCompletion } from '@/lib/mothership/request/tools/client-completion-seal.server' +import { settleAbandonedDesktopToolCalls } from '@/lib/mothership/request/tools/desktop-wait' const CAPABILITIES = { executor: 1, browser: true, terminal: true, localFiles: true } @@ -556,6 +558,7 @@ describe.runIf(Boolean(redisUrl))('desktop background executor protocol', () => const toolCallId = await pendingCall(run.runId) const { executionToken } = await claim(desktop.principal, desktop.deviceId, toolCallId) const input = { deviceId: desktop.deviceId, toolCallId, executionToken } + expect((await row(toolCallId)).executionDeadlineAt).toBeNull() await db .update(copilotAsyncToolCalls) @@ -583,6 +586,41 @@ describe.runIf(Boolean(redisUrl))('desktop background executor protocol', () => ).rejects.toBeInstanceOf(DesktopCallRevokedError) }) + it('stops renewing a call once the resume gate gave up on its deadline, then settles it as outcome unknown', async () => { + const desktop = await signedInDesktop() + const run = await boundRun(desktop) + const toolCallId = await pendingCall(run.runId) + const setRow = (fields: Partial) => + db + .update(copilotAsyncToolCalls) + .set(fields) + .where(eq(copilotAsyncToolCalls.toolCallId, toolCallId)) + const deadlinePassedBy = (seconds: number) => + setRow({ executionDeadlineAt: sql`clock_timestamp() - ${seconds} * interval '1 second'` }) + await setRow({ executionDeadlineAt: sql`clock_timestamp() + interval '10 minutes'` }) + const { executionToken } = await claim(desktop.principal, desktop.deviceId, toolCallId) + const input = { deviceId: desktop.deviceId, toolCallId, executionToken } + + // Inside the gate's grace past the deadline, the device keeps the call. + await deadlinePassedBy(TOOL_WATCHDOG_RESUME_GRACE_MS / 1000 - 10) + await expect( + renewDesktopToolLease.execute({ principal: desktop.principal, input }) + ).resolves.toEqual({ renewed: true }) + + await deadlinePassedBy(TOOL_WATCHDOG_RESUME_GRACE_MS / 1000 + 1) + await expect( + renewDesktopToolLease.execute({ principal: desktop.principal, input }) + ).rejects.toBeInstanceOf(DesktopCallRevokedError) + + // Unrenewed, the lease lapses, and the cleanup settles the call as it would any lapse. + await setRow({ executionLeaseExpiresAt: sql`clock_timestamp() - interval '1 second'` }) + await settleAbandonedDesktopToolCalls(0) + expect(await row(toolCallId)).toMatchObject({ + status: 'failed', + error: expect.stringContaining('It may already have taken effect'), + }) + }) + it('after Stop, refuses the stopped calls, revokes the running one and lists it to cancel', async () => { const desktop = await signedInDesktop() const run = await boundRun(desktop) diff --git a/apps/sim/lib/mothership/async-runs/repository.ts b/apps/sim/lib/mothership/async-runs/repository.ts index 7f826b28f96..00015843896 100644 --- a/apps/sim/lib/mothership/async-runs/repository.ts +++ b/apps/sim/lib/mothership/async-runs/repository.ts @@ -55,6 +55,7 @@ import { isAwaitingToolPermission, SIM_TOOL_EXECUTION_VERSION, } from '@/lib/mothership/async-runs/lifecycle' +import { TOOL_WATCHDOG_RESUME_GRACE_MS } from '@/lib/mothership/constants' import { TraceAttr } from '@/lib/mothership/generated/trace-attributes-v1' import { TraceSpan } from '@/lib/mothership/generated/trace-spans-v1' import { @@ -897,6 +898,12 @@ const unfinishedSimExecution = allOf( */ const tooLateToStart = sql`(${copilotAsyncToolCalls.executionDeadlineAt} IS NOT NULL AND ${copilotAsyncToolCalls.executionDeadlineAt} < clock_timestamp() + ${SIM_TOOL_MIN_START_BUDGET_MS} * interval '1 millisecond')` +/** + * A call past its deadline plus the resume grace: the resume wait's ceiling for it (`run.ts`), past + * which nothing still waits for its result. A call without a deadline never matches. + */ +const pastResumeGiveUp = sql`(${copilotAsyncToolCalls.executionDeadlineAt} IS NOT NULL AND ${copilotAsyncToolCalls.executionDeadlineAt} + ${TOOL_WATCHDOG_RESUME_GRACE_MS} * interval '1 millisecond' <= clock_timestamp())` + /** * A Sim execution whose owner died before recording a result: its lease lapsed, which a live * owner's heartbeat never lets happen. A takeover runs it again until it has started @@ -1069,7 +1076,8 @@ export async function claimDesktopToolCall( /** * Expired ownership cannot be renewed, even before a follower has observed the expiry. With * `desktop`, the call must still be running on a run bound to that device, so a call Stop settled - * cannot be renewed either. + * cannot be renewed either, nor one past its deadline plus the resume grace: its lease then lapses + * and the call settles as outcome unknown. */ export async function renewSimToolExecutionLease( owner: SimToolExecutionOwner, @@ -1092,6 +1100,7 @@ export async function renewSimToolExecutionLease( desktop ? and( eq(copilotAsyncToolCalls.status, ASYNC_TOOL_STATUS.running), + not(pastResumeGiveUp), 'deviceId' in desktop ? sql`EXISTS (SELECT 1 FROM ${copilotRuns} r WHERE r.id = ${copilotAsyncToolCalls.runId} AND r.desktop_device_id = ${desktop.deviceId})` : and( diff --git a/apps/sim/lib/mothership/request/lifecycle/run.test.ts b/apps/sim/lib/mothership/request/lifecycle/run.test.ts index a9ceece2e9d..7e4cb2a2658 100644 --- a/apps/sim/lib/mothership/request/lifecycle/run.test.ts +++ b/apps/sim/lib/mothership/request/lifecycle/run.test.ts @@ -3672,6 +3672,41 @@ describe('runCopilotLifecycle', () => { } }) + it('fails a call whose deadline already passed at that deadline plus grace, not a grace from now', async () => { + vi.useFakeTimers() + try { + mothershipAsyncRunsMockFns.mockGetChatViewDesktopLeaseRemainingMs.mockResolvedValue(50_000) + const turn = runImportTurn(Date.now() - 10_000) + await vi.advanceTimersByTimeAsync(19_000) + expect(turn.bodies).toHaveLength(1) + await vi.advanceTimersByTimeAsync(2_000) + expect((await turn.lifecycle).success).toBe(true) + expect(turn.resumedWithLostImport()).toBe(true) + expect( + mothershipAsyncRunsMockFns.mockGetChatViewDesktopLeaseRemainingMs + ).not.toHaveBeenCalled() + } finally { + mothershipAsyncRunsMockFns.mockGetChatViewDesktopLeaseRemainingMs.mockReset() + vi.useRealTimers() + } + }) + + it('a call without a deadline keeps its full budget and grace', async () => { + vi.useFakeTimers() + try { + mothershipAsyncRunsMockFns.mockGetChatViewDesktopLeaseRemainingMs.mockResolvedValue(null) + const turn = runImportTurn() + await vi.advanceTimersByTimeAsync(89_000) + expect(turn.bodies).toHaveLength(1) + await vi.advanceTimersByTimeAsync(2_000) + expect((await turn.lifecycle).success).toBe(true) + expect(turn.resumedWithLostImport()).toBe(true) + } finally { + mothershipAsyncRunsMockFns.mockGetChatViewDesktopLeaseRemainingMs.mockReset() + vi.useRealTimers() + } + }) + it('a lease lookup that fails is checked again instead of failing a renewed import', async () => { vi.useFakeTimers() try { diff --git a/apps/sim/lib/mothership/request/lifecycle/run.ts b/apps/sim/lib/mothership/request/lifecycle/run.ts index d7fed72412c..df28b660535 100644 --- a/apps/sim/lib/mothership/request/lifecycle/run.ts +++ b/apps/sim/lib/mothership/request/lifecycle/run.ts @@ -1450,21 +1450,22 @@ async function runCheckpointLoop( const waitBudgetMs = pendingToolWaitBudgetMs(toolCall, context.desktopDeviceId) + TOOL_WATCHDOG_RESUME_GRACE_MS - const ceilingAt = now + Math.max(waitBudgetMs, CLIENT_TOOL_RESULT_TIMEOUT_MS) + const clientCapAt = now + Math.max(waitBudgetMs, CLIENT_TOOL_RESULT_TIMEOUT_MS) + // Neither the wait nor a lease carries a call past the deadline the worker named for it. + const ceilingAt = + toolCall?.deadlineAt === undefined + ? clientCapAt + : Math.min(clientCapAt, toolCall.deadlineAt + TOOL_WATCHDOG_RESUME_GRACE_MS) pendingWatchdogs.set(toolCallId, { promise, settlement: promise.then( () => ({ toolCallId, promise }), () => ({ toolCallId, promise }) ), - deadlineAt: now + waitBudgetMs, + deadlineAt: Math.min(now + waitBudgetMs, ceilingAt), waitBudgetMs, startedAt: now, - // A lease never carries a call past the deadline the worker named for it. - ceilingAt: - toolCall?.deadlineAt === undefined - ? ceilingAt - : Math.min(ceilingAt, toolCall.deadlineAt + TOOL_WATCHDOG_RESUME_GRACE_MS), + ceilingAt, }) maximumWaitBudgetMs = Math.max(maximumWaitBudgetMs, waitBudgetMs) } diff --git a/apps/sim/lib/mothership/request/tools/executor.test.ts b/apps/sim/lib/mothership/request/tools/executor.test.ts index 3d7406e0c1e..e85d5637ad5 100644 --- a/apps/sim/lib/mothership/request/tools/executor.test.ts +++ b/apps/sim/lib/mothership/request/tools/executor.test.ts @@ -698,6 +698,67 @@ describe('executeToolAndReport provenance isolation', () => { expect(tool.error).toBe(TOOL_DEADLINE_BEFORE_START_MESSAGE) }) + it('settles an unclaimed call past its start window as never run, without claiming it', async () => { + completePendingAsyncToolCall.mockImplementationOnce(async (input) => ({ ...input })) + const tool: ToolCallState = { + ...buildPendingToolCall(), + name: 'generate_audio', + deadlineAt: Date.now() + 30_000, + } + const completion = await executeToolAndReport(tool.id, buildStreamingContext(tool), { + userId: 'user-1', + workflowId: 'workflow-1', + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(), + }) + + expect(claimToolExecution).not.toHaveBeenCalled() + expect(executeTool).not.toHaveBeenCalled() + expect(completePendingAsyncToolCall).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + toolCallId: tool.id, + status: 'failed', + error: TOOL_DEADLINE_BEFORE_START_MESSAGE, + }) + ) + expect(completion.status).toBe('error') + expect(tool.result?.output).toMatchObject({ + error: TOOL_DEADLINE_BEFORE_START_MESSAGE, + notStarted: true, + }) + }) + + it('leaves a call past its start window that is already claimed to its claim', async () => { + completePendingAsyncToolCall.mockResolvedValueOnce(null) + const tool: ToolCallState = { + ...buildPendingToolCall(), + name: 'generate_audio', + deadlineAt: Date.now() + 30_000, + } + await executeToolAndReport(tool.id, buildStreamingContext(tool), { + userId: 'user-1', + workflowId: 'workflow-1', + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(), + }) + + expect(claimToolExecution).toHaveBeenCalledOnce() + expect(executeTool).not.toHaveBeenCalled() + expect(tool.error).toBe(TOOL_DEADLINE_BEFORE_START_MESSAGE) + }) + + it('claims a call without a deadline without trying to settle it first', async () => { + executeTool.mockResolvedValue({ success: true, output: {} }) + const tool: ToolCallState = { ...buildPendingToolCall(), name: 'generate_audio' } + await executeToolAndReport(tool.id, buildStreamingContext(tool), { + userId: 'user-1', + workflowId: 'workflow-1', + resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(), + }) + + expect(completePendingAsyncToolCall).not.toHaveBeenCalled() + expect(claimToolExecution).toHaveBeenCalledOnce() + expect(executeTool).toHaveBeenCalledOnce() + }) + it('runs a call again only until the deadline recorded when it was first handed', async () => { vi.useFakeTimers() let signal: AbortSignal | undefined diff --git a/apps/sim/lib/mothership/request/tools/executor.ts b/apps/sim/lib/mothership/request/tools/executor.ts index 60f10f7662e..d2bcc664fbd 100644 --- a/apps/sim/lib/mothership/request/tools/executor.ts +++ b/apps/sim/lib/mothership/request/tools/executor.ts @@ -754,6 +754,18 @@ async function executeToolAndReportInner( }) /** A re-run after a takeover gets what remains of the call's first deadline, never a fresh one. */ if (record?.executionDeadlineAt) toolCall.deadlineAt = record.executionDeadlineAt.getTime() + // Too late to start: an unclaimed call is settled as never run rather than claimed first, so a + // crash before its handler refuses to start cannot leave it recovered as an unknown outcome. + if ( + toolStartWindowClosed(toolCall.deadlineAt, Date.now()) && + (await failUnstartedToolCall( + toolCall.id, + context, + execContext, + TOOL_DEADLINE_BEFORE_START_MESSAGE + )) + ) + return terminalCompletionFromToolCall(toolCall) if (context.runId) { const claim = await lifetime.claim(context.runId, execContext.userId) if (claim.outcome === 'closed') return settleCancelled('Run stopped before tool admission') From 5457cfa328f1a4942b1026d39d14faf918e01820 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 15 Oct 2026 19:43:54 -0700 Subject: [PATCH 2/3] fix(mothership): type the integration test's row update --- apps/sim/lib/desktop/application/executor.integration.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/desktop/application/executor.integration.ts b/apps/sim/lib/desktop/application/executor.integration.ts index 36b91945d95..d49700d1068 100644 --- a/apps/sim/lib/desktop/application/executor.integration.ts +++ b/apps/sim/lib/desktop/application/executor.integration.ts @@ -37,6 +37,7 @@ import { createDeferred } from '@sim/testing/helpers/deferred' import { generateId } from '@sim/utils/id' import { compareStrings } from '@sim/utils/string' import { eq, inArray, sql } from 'drizzle-orm' +import type { PgUpdateSetSource } from 'drizzle-orm/pg-core' import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' import type { DbTransaction } from '@/lib/db/types' import { @@ -590,7 +591,7 @@ describe.runIf(Boolean(redisUrl))('desktop background executor protocol', () => const desktop = await signedInDesktop() const run = await boundRun(desktop) const toolCallId = await pendingCall(run.runId) - const setRow = (fields: Partial) => + const setRow = (fields: PgUpdateSetSource) => db .update(copilotAsyncToolCalls) .set(fields) From 50e66ae78d0c60e0db7ff8bbb2ec52a9936cca79 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 15 Oct 2026 20:09:20 -0700 Subject: [PATCH 3/3] fix(mothership): settle a too-late call as never run only under open admission --- .../async-runs/orphaned-runs.integration.ts | 50 +++++++++++++++++++ .../lib/mothership/async-runs/repository.ts | 28 +++++++++-- .../request/tools/execution-lifetime.ts | 4 +- .../mothership/request/tools/executor.test.ts | 21 +++++--- .../lib/mothership/request/tools/executor.ts | 26 +++++----- 5 files changed, 101 insertions(+), 28 deletions(-) diff --git a/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts b/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts index 653dce5adb4..bb2cb9becff 100644 --- a/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts +++ b/apps/sim/lib/mothership/async-runs/orphaned-runs.integration.ts @@ -262,6 +262,56 @@ describe.runIf(Boolean(redisUrl))('Chat runs no controller owns', () => { return { ...run, marker: chat?.conversationId ?? null } } + describe('a Sim claim past its start window', () => { + async function lateTool( + runId: string, + fields: Partial<{ status: 'running'; claimedBy: string }> = {} + ) { + const tool = await dispatchedTool(runId) + await db + .update(copilotAsyncToolCalls) + .set({ executionDeadlineAt: sql`clock_timestamp() + interval '30 seconds'`, ...fields }) + .where(eq(copilotAsyncToolCalls.toolCallId, tool.toolCallId)) + return tool + } + + async function storedTool(toolCallId: string) { + const [row] = await db + .select() + .from(copilotAsyncToolCalls) + .where(eq(copilotAsyncToolCalls.toolCallId, toolCallId)) + return row + } + + it('leaves an admitted pending call unclaimed for its caller to settle as never run', async () => { + const run = await admittedRun() + const tool = await lateTool(run.runId) + + expect(await claimToolExecution(tool)).toEqual({ outcome: 'too_late' }) + expect(await storedTool(tool.toolCallId)).toMatchObject({ + status: 'pending', + executionStartedAt: null, + executionOwnerToken: null, + }) + }) + + it('reports a stopped run as closed before the start window', async () => { + const run = await admittedRun({ stopped: true }) + const tool = await lateTool(run.runId) + + expect(await claimToolExecution(tool)).toEqual({ outcome: 'closed' }) + }) + + it('still claims a call a client already claimed, and one without a deadline', async () => { + const run = await admittedRun() + const clientClaimed = await lateTool(run.runId, { status: 'running', claimedBy: 'client' }) + const undated = await dispatchedTool(run.runId) + + expect(await claimToolExecution(clientClaimed)).toEqual({ outcome: 'claimed' }) + expect(await claimToolExecution(undated)).toEqual({ outcome: 'claimed' }) + }) + }) + it('settles a run whose controller died once its recovery window has passed', async () => { const orphan = await admittedRun({ idleMinutes: 90, status: 'paused_waiting_for_tool' }) diff --git a/apps/sim/lib/mothership/async-runs/repository.ts b/apps/sim/lib/mothership/async-runs/repository.ts index 00015843896..cdf07753dae 100644 --- a/apps/sim/lib/mothership/async-runs/repository.ts +++ b/apps/sim/lib/mothership/async-runs/repository.ts @@ -868,6 +868,12 @@ export type ToolExecutionClaim = /** Already claimed or settled. */ | { outcome: 'existing' } +/** A Sim handler's claim, which also reports an unclaimed call too late to start. */ +export type SimToolExecutionClaim = + | ToolExecutionClaim + /** Admitted but unclaimed, with too little of its deadline left to start: nothing ran it. */ + | { outcome: 'too_late' } + /** * Every condition at once, typed as a condition: drizzle's `and` may return undefined, which a * negation would render as `NOT (NULL)` and match nothing. @@ -904,6 +910,13 @@ const tooLateToStart = sql`(${copilotAsyncToolCalls.executionDeadlineAt} IS NOT */ const pastResumeGiveUp = sql`(${copilotAsyncToolCalls.executionDeadlineAt} IS NOT NULL AND ${copilotAsyncToolCalls.executionDeadlineAt} + ${TOOL_WATCHDOG_RESUME_GRACE_MS} * interval '1 millisecond' <= clock_timestamp())` +/** A call nothing claimed or started whose deadline leaves too little to start it. */ +const pendingTooLateToStart = allOf( + eq(copilotAsyncToolCalls.status, ASYNC_TOOL_STATUS.pending), + isNull(copilotAsyncToolCalls.executionStartedAt), + tooLateToStart +) + /** * A Sim execution whose owner died before recording a result: its lease lapsed, which a live * owner's heartbeat never lets happen. A takeover runs it again until it has started @@ -932,11 +945,12 @@ const rerunnableSimExecution = allOf( * Sim claims a call it is about to execute under an execution lease; a terminal result never * releases it. The claim takes a call's first execution, or re-runs one whose owner died * mid-execution: the new owner token fences the dead owner out, since its renewals and its - * result both require the token and an unexpired lease. + * result both require the token and an unexpired lease. A pending call too late to start is left + * unclaimed, under the same admission check, for its caller to settle as never run. */ export async function claimToolExecution( owner: SimToolExecutionOwner -): Promise { +): Promise { return await claimUnderRunAdmission(owner, 'sim-stream', true, async (tx, thisCall) => { const startedAt = new Date() const [claimed] = await traceMothershipQuery('UPDATE', 'copilot_async_tool_calls', () => @@ -961,7 +975,8 @@ export async function claimToolExecution( inArray(copilotAsyncToolCalls.status, [ ASYNC_TOOL_STATUS.pending, ASYNC_TOOL_STATUS.running, - ]) + ]), + not(pendingTooLateToStart) ), interruptedSimExecution ) @@ -971,10 +986,13 @@ export async function claimToolExecution( ) if (claimed) return { outcome: 'claimed' } as const const [record] = await traceMothershipQuery('SELECT', 'copilot_async_tool_calls', () => - tx.select({ id: copilotAsyncToolCalls.id }).from(copilotAsyncToolCalls).where(thisCall) + tx + .select({ tooLate: sql`${pendingTooLateToStart}` }) + .from(copilotAsyncToolCalls) + .where(thisCall) ) if (!record) throw new Error('Tool execution record is unavailable') - return { outcome: 'existing' } as const + return record.tooLate ? ({ outcome: 'too_late' } as const) : ({ outcome: 'existing' } as const) }) } diff --git a/apps/sim/lib/mothership/request/tools/execution-lifetime.ts b/apps/sim/lib/mothership/request/tools/execution-lifetime.ts index f00f3eb197f..98031b32679 100644 --- a/apps/sim/lib/mothership/request/tools/execution-lifetime.ts +++ b/apps/sim/lib/mothership/request/tools/execution-lifetime.ts @@ -15,9 +15,9 @@ import { prepareWorkbenchAccess, recordSimSandboxProcess, renewSimToolExecutionLease, + type SimToolExecutionClaim, settleSimSandboxProcess, settleSimToolExecution, - type ToolExecutionClaim, } from '@/lib/mothership/async-runs/repository' import { TraceAttr } from '@/lib/mothership/generated/trace-attributes-v1' import { TraceSpan } from '@/lib/mothership/generated/trace-spans-v1' @@ -30,7 +30,7 @@ export interface ToolExecutionLifetime { readonly signal: AbortSignal readonly owner?: SimToolExecutionOwner complete(input: CompleteAsyncToolCallInput): Promise - claim(runId: string, userId: string): Promise + claim(runId: string, userId: string): Promise hold(work: Promise): Promise } diff --git a/apps/sim/lib/mothership/request/tools/executor.test.ts b/apps/sim/lib/mothership/request/tools/executor.test.ts index e85d5637ad5..75f74b63904 100644 --- a/apps/sim/lib/mothership/request/tools/executor.test.ts +++ b/apps/sim/lib/mothership/request/tools/executor.test.ts @@ -369,8 +369,10 @@ describe('executeToolAndReport provenance isolation', () => { beforeEach(() => { executeTool.mockReset() completeAsyncToolCall.mockResolvedValue(null) + completePendingAsyncToolCall.mockReset() markAsyncToolRunning.mockResolvedValue(null) upsertAsyncToolCall.mockResolvedValue(null) + claimToolExecution.mockReset() claimToolExecution.mockResolvedValue({ outcome: 'claimed' }) settleSimToolExecution.mockResolvedValue(undefined) waitForToolConfirmation.mockReset() @@ -698,7 +700,8 @@ describe('executeToolAndReport provenance isolation', () => { expect(tool.error).toBe(TOOL_DEADLINE_BEFORE_START_MESSAGE) }) - it('settles an unclaimed call past its start window as never run, without claiming it', async () => { + it('settles a call its claim found too late to start as never run, without claiming it', async () => { + claimToolExecution.mockResolvedValueOnce({ outcome: 'too_late' }) completePendingAsyncToolCall.mockImplementationOnce(async (input) => ({ ...input })) const tool: ToolCallState = { ...buildPendingToolCall(), @@ -711,7 +714,6 @@ describe('executeToolAndReport provenance isolation', () => { resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(), }) - expect(claimToolExecution).not.toHaveBeenCalled() expect(executeTool).not.toHaveBeenCalled() expect(completePendingAsyncToolCall).toHaveBeenCalledExactlyOnceWith( expect.objectContaining({ @@ -727,22 +729,25 @@ describe('executeToolAndReport provenance isolation', () => { }) }) - it('leaves a call past its start window that is already claimed to its claim', async () => { - completePendingAsyncToolCall.mockResolvedValueOnce(null) + it('settles the call of a stopped run as stopped even past its start window', async () => { + claimToolExecution.mockResolvedValueOnce({ outcome: 'closed' }) + completePendingAsyncToolCall.mockImplementation(async (input) => ({ ...input })) const tool: ToolCallState = { ...buildPendingToolCall(), name: 'generate_audio', deadlineAt: Date.now() + 30_000, } - await executeToolAndReport(tool.id, buildStreamingContext(tool), { + const completion = await executeToolAndReport(tool.id, buildStreamingContext(tool), { userId: 'user-1', workflowId: 'workflow-1', resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry(), }) - expect(claimToolExecution).toHaveBeenCalledOnce() - expect(executeTool).not.toHaveBeenCalled() - expect(tool.error).toBe(TOOL_DEADLINE_BEFORE_START_MESSAGE) + expect(completePendingAsyncToolCall).not.toHaveBeenCalled() + expect(completion).toMatchObject({ + status: 'cancelled', + message: 'Run stopped before tool admission', + }) }) it('claims a call without a deadline without trying to settle it first', async () => { diff --git a/apps/sim/lib/mothership/request/tools/executor.ts b/apps/sim/lib/mothership/request/tools/executor.ts index d2bcc664fbd..7e7d053dede 100644 --- a/apps/sim/lib/mothership/request/tools/executor.ts +++ b/apps/sim/lib/mothership/request/tools/executor.ts @@ -754,22 +754,22 @@ async function executeToolAndReportInner( }) /** A re-run after a takeover gets what remains of the call's first deadline, never a fresh one. */ if (record?.executionDeadlineAt) toolCall.deadlineAt = record.executionDeadlineAt.getTime() - // Too late to start: an unclaimed call is settled as never run rather than claimed first, so a - // crash before its handler refuses to start cannot leave it recovered as an unknown outcome. - if ( - toolStartWindowClosed(toolCall.deadlineAt, Date.now()) && - (await failUnstartedToolCall( - toolCall.id, - context, - execContext, - TOOL_DEADLINE_BEFORE_START_MESSAGE - )) - ) - return terminalCompletionFromToolCall(toolCall) if (context.runId) { const claim = await lifetime.claim(context.runId, execContext.userId) if (claim.outcome === 'closed') return settleCancelled('Run stopped before tool admission') - if (claim.outcome === 'existing') { + // Settled as never run instead of claimed, so a crash before the handler refuses to start + // cannot leave it recovered as an unknown outcome. Losing that settlement leaves another writer's. + if ( + claim.outcome === 'too_late' && + (await failUnstartedToolCall( + toolCall.id, + context, + execContext, + TOOL_DEADLINE_BEFORE_START_MESSAGE + )) + ) + return terminalCompletionFromToolCall(toolCall) + if (claim.outcome !== 'claimed') { /** The winning controller owns execution; this promise observes its durable result. */ const completion = await waitForToolConfirmation( toolCall.id,