fix(desktop): tighten computer use input guards for text, tabs, menus and AX failures

apps/desktop/native/computer-use/README.md

@@ -8,13 +8,13 @@ Compile each architecture with `swiftc -parse-as-library -O -target arm64-apple-
 
 Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` returns `{id,result}` or `{id,error:{code,message,dispatchState?}}`. The producer contract is `worker/packages/contracts/src/computer-use.ts` in mothership, copied into the desktop bridge by contract sync. `request_permission` is a local onboarding method, never a model tool; params are `{permission:"accessibility"|"screenCapture"}`. Status/preflight never triggers prompts. Errors go through the protocol, and stdout contains no logs.
 
-The helper observes up to 2000 accessibility nodes (depth 64, queue 4000, traversal deadline 8 seconds, each AX call timeout 250ms). Selected-window and focused-element ancestry are prioritized; menu traversal is deferred and capped at 80 nodes. Electron accessibility is enabled using its documented `AXManualAccessibility` attribute. The helper observes 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every action that reaches the app; a request that fails before sending anything reports `dispatchState: "not_started"` and keeps them valid. Restarts invalidate all references. Screen captures exclude window shadows, preserve the selected window’s coordinate geometry, and are capped to 1600 pixels on the longest side and 8 MiB encoded. Complete JSON replies are capped at 16 MiB: oversized observations omit screenshots first, then truncate accessibility nodes, while preserving action outcomes. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. macOS 26 uses the screenshot-specific capture API; the macOS 14/15 API path includes only the selected window in a display filter and crops its exact frame. That older path requires the window to fit within one display and reports `capture_geometry_unavailable` otherwise.
+The helper observes up to 2000 accessibility nodes (depth 64, queue 4000, traversal deadline 8 seconds, each AX call timeout 250ms). Selected-window and focused-element ancestry are prioritized. The menu bar and menus are read in place, so an open menu's items and their submenus come first; items of closed menus are read last, capped at 80 menu nodes. Electron accessibility is enabled using its documented `AXManualAccessibility` attribute. The helper observes 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every action that reaches the app; a request that fails before sending anything reports `dispatchState: "not_started"` and keeps them valid. An Accessibility failure counts as sent unless Accessibility rejected the request before the app saw it (unsupported attribute or action, illegal argument). Restarts invalidate all references. Screen captures exclude window shadows, preserve the selected window’s coordinate geometry, and are capped to 1600 pixels on the longest side and 8 MiB encoded. Complete JSON replies are capped at 16 MiB: oversized observations omit screenshots first, then truncate accessibility nodes, while preserving action outcomes. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. macOS 26 uses the screenshot-specific capture API; the macOS 14/15 API path includes only the selected window in a display filter and crops its exact frame. That older path requires the window to fit within one display and reports `capture_geometry_unavailable` otherwise.
 
-Coordinates in action inputs are window-local points; node frames are global screen points. Coordinate input without a `windowId` targets the window the snapshot selected; `press_key` with an `elementId` focuses that element first. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps and System Settings are protected targets. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`.
+Coordinates in action inputs are window-local points; node frames are global screen points. Coordinate input without a `windowId` targets the window the snapshot selected, and an element target uses its own window; `press_key` with an `elementId` focuses that element first. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps and System Settings are protected targets. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`.
 
-AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. An action that opens UI (AXPress, AXPick, AXShowMenu) and times out while the app shows a sheet, modal or menu (`cannotComplete`) counts as dispatched; any other action that times out fails as uncertain. A menu-item press that leaves its menu open with the item highlighted (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Open menus and their submenus are observed first, and menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly.
+AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. A hit on a window's own background counts as that window. Background tabs and minimized windows share bounds with the window in front, so they take no coordinate input. An action that opens UI (AXPress, AXPick, AXShowMenu) and times out while the app shows a sheet, modal or menu (`cannotComplete`) counts as dispatched; any other action that times out fails as uncertain. A menu-item press that leaves the item's own menu open with the app highlighting that item (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly.
 
-`type_text` and `input_sequence` target any observed element that can take keyboard focus (text fields, terminals, combo boxes, web editors). `input_sequence` executes 1–32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly.
+`type_text` and `input_sequence` target a text control (text fields and areas, terminals, combo boxes, date fields, or anything with an editable text selection), or a container whose focused editor already holds keyboard focus, such as a web composer; keystrokes are sent only while focus is in a text input. `input_sequence` executes 1–32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly.
 
 ## Cancellation
 

apps/desktop/native/computer-use/tests/live.py

@@ -53,6 +53,14 @@ def mutate(method, snapshot, **params):
     return state()
 
 
+def refused(method, snapshot, **params):
+    try:
+        call(method, bundleId=bundle, snapshotId=snapshot['snapshotId'], **params)
+    except AssertionError as error:
+        return json.loads(str(error).split(': ', 1)[1])
+    raise AssertionError(method + ' was not refused')
+
+
 def coordinate(snapshot, node, fx=0.5, fy=0.5):
     window = next(w for w in snapshot['windows'] if w['windowId'] == snapshot['windowId'])
     return {'windowId': window['windowId'], 'x': node['x'] - window['x'] + node['width'] * fx, 'y': node['y'] - window['y'] + node['height'] * fy}
@@ -106,10 +114,20 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5):
         scroll = find(snapshot, role='AXScrollArea')
         snapshot = mutate('scroll', snapshot, **coordinate(snapshot, scroll), deltaX=0, deltaY=180)
         assert any((node.get('value', '').startswith('Scroll: ') and node['value'] != 'Scroll: 0') or (node.get('label', '').startswith('Scroll: ') and node['label'] != 'Scroll: 0') for node in snapshot['nodes']), 'Scroll did not move fixture content'
+        slider = find(snapshot, role='AXSlider')
+        error = refused('type_text', snapshot, elementId=slider['elementId'], text=' ')
+        assert error['code'] == 'unsupported_action' and error['dispatchState'] == 'not_started', error
+        assert find(state(), role='AXSlider')['value'] == slider['value'], 'Text reached a control that does not take text'
+        snapshot = state()
+        snapshot = mutate('click', snapshot, elementId=find(snapshot, role='AXTextField', label='Fixture editable text')['elementId'])
+        snapshot = mutate('type_text', snapshot, elementId=find(snapshot, role='AXWindow')['elementId'], text=' window')
+        assert find(snapshot, role='AXTextField', label='Fixture editable text')['value'].endswith(' window'), 'Window target did not type into its focused editor'
+        window = next(w for w in snapshot['windows'] if w['windowId'] == snapshot['windowId'])
+        snapshot = mutate('click', snapshot, x=window['width'] * 0.75, y=8)
         button = find(snapshot, role='AXButton', label='Increment')
         snapshot = mutate('perform_action', snapshot, elementId=button['elementId'], accessibilityAction='AXPress')
         assert any(f'Count: {baseline + 2}' in node.get('value', '') or f'Count: {baseline + 2}' in node.get('label', '') for node in snapshot['nodes']), 'Explicit AX action did not increment'
-        print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling; every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
+        print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, a window target typing into its focused editor, a title-bar click; every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED'))
 finally:
     if fixture:
         fixture.terminate()

apps/desktop/src/main/computer-use/service.ts

@@ -180,7 +180,18 @@ export class ComputerUseService {
       return Promise.reject(new Error('Computer Use has too many queued actions.'))
     if (this.queued.has(toolCallId))
       return Promise.reject(new Error('This Computer Use action is already running.'))
-    const input = ComputerUseSchema.parse(raw)
+    // A newer server can send arguments this build predates; nothing has been queued yet.
+    const parsed = ComputerUseSchema.safeParse(raw)
+    if (!parsed.success)
+      return Promise.reject(
+        new ComputerUseError({
+          code: 'invalid_arguments',
+          message:
+            'This desktop app does not support these Computer Use arguments; update Sim Desktop.',
+          dispatchState: 'not_started',
+        })
+      )
+    const input = parsed.data
     const work: ToolWork = { toolCallId, scopeId, input, controller: new AbortController() }
     this.queued.set(toolCallId, work)
     const result = this.tail.then(() => this.run(work))

apps/sim/lib/mothership/generated/computer-use.ts

@@ -17,6 +17,16 @@ const SnapshotId = z
   .describe("Latest unconsumed snapshotId from get_app_state for this app. Actions consume it.");
 const ElementId = z.string().min(1).max(128).describe("Element reference from that snapshot.");
 const WindowId = z.string().min(1).max(128).describe("Window reference from that snapshot.");
+/** Input fields shared across actions keep one description: Anthropic flattens the union and anyOfs differing copies. */
+const TargetWindowId = WindowId.optional().describe(
+  "Window reference from an earlier snapshot of this app. If omitted, get_app_state and press_key use the app's focused window; coordinate actions use the window the snapshot selected.",
+);
+const IncludeScreenshot = z
+  .boolean()
+  .optional()
+  .describe(
+    "Set true to include a screenshot in the returned state. On actions it overrides observeAfter.includeScreenshot.",
+  );
 const Coordinate = z.number().nonnegative().max(100_000);
 const ObserveAfter = {
   observeAfter: z
@@ -25,7 +35,7 @@ const ObserveAfter = {
     .describe(
       "Every mutation returns fresh app state by default. Set includeScreenshot to true to include an image; otherwise no screenshot is captured. Observation failure never retries the action.",
     ),
-  includeScreenshot: z.boolean().optional().describe("Same as observeAfter.includeScreenshot."),
+  includeScreenshot: IncludeScreenshot,
 };
 const InputSequenceStep = z.discriminatedUnion("action", [
   z.strictObject({ action: z.literal("type_text"), text: z.string().max(32_000) }),
@@ -35,9 +45,7 @@ const ElementTarget = { bundleId: BundleId, snapshotId: SnapshotId, elementId: E
 const PointTarget = {
   bundleId: BundleId,
   snapshotId: SnapshotId,
-  windowId: WindowId.optional().describe(
-    "Window reference from that snapshot. Defaults to the window the snapshot selected.",
-  ),
+  windowId: TargetWindowId,
   x: Coordinate,
   y: Coordinate,
 };
@@ -51,13 +59,17 @@ const ScrollOffsets = {
     .min(-10_000)
     .max(10_000)
     .optional()
-    .describe("Horizontal scroll distance in points; positive scrolls right, negative left. Defaults to 0."),
+    .describe(
+      "Horizontal scroll distance in points; positive scrolls right, negative left. Omit for 0 when deltaY is set.",
+    ),
   deltaY: z
     .number()
     .min(-10_000)
     .max(10_000)
     .optional()
-    .describe("Vertical scroll distance in points; positive scrolls down, negative up. Defaults to 0."),
+    .describe(
+      "Vertical scroll distance in points; positive scrolls down, negative up. Omit for 0 when deltaX is set.",
+    ),
 };
 
 const hasScrollDistance = (input: { deltaX?: number | undefined; deltaY?: number | undefined }) =>
@@ -71,11 +83,8 @@ export const ComputerUseSchema = z.union([
   z.strictObject({
     action: z.literal("get_app_state"),
     bundleId: BundleId,
-    windowId: WindowId.optional(),
-    includeScreenshot: z
-      .boolean()
-      .optional()
-      .describe("Set true to include an image of the selected window."),
+    windowId: TargetWindowId,
+    includeScreenshot: IncludeScreenshot,
   }),
   z.strictObject({ action: z.literal("click"), ...ElementTarget, ...ObserveAfter, ...ClickOptions }),
   z.strictObject({ action: z.literal("click"), ...PointTarget, ...ObserveAfter, ...ClickOptions }),
@@ -90,11 +99,15 @@ export const ComputerUseSchema = z.union([
     ...ObserveAfter,
     bundleId: BundleId,
     snapshotId: SnapshotId,
-    windowId: WindowId.optional().describe(
-      "Window reference from that snapshot. Omit to send the key to the app's focused window.",
-    ),
-    elementId: ElementId.optional().describe("Focus this element from the snapshot before pressing the key."),
-    key: z.string().min(1).max(128).describe("One key or chord, such as Enter, Tab, Escape, or Cmd+A."),
+    windowId: TargetWindowId,
+    elementId: ElementId.optional(),
+    key: z
+      .string()
+      .min(1)
+      .max(128)
+      .describe(
+        "One key or chord, such as Enter, Tab, Escape, or Cmd+A. With elementId, focuses that element first.",
+      ),
   }),
   z
     .strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets })