diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 2ae8157cf63..f3129047b73 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -48,12 +48,13 @@ func axCheck(_ result: AXError) throws { } /// A press that opens a sheet, alert or modal panel holds the app's Accessibility reply until that UI /// is up, so for a press `cannotComplete` after the messaging timeout means the request was delivered. -/// For a value or selection change the outcome is unknown: it counts as dispatched but still fails. +/// Any other failure may follow an effect, so it counts as dispatched unless Accessibility rejected the +/// request before the app saw it. func dispatchAX(_ element: AXUIElement, press: Bool = false, _ operation: (AXUIElement) -> AXError) throws { AXUIElementSetMessagingTimeout(element, 2) let result = operation(element) - if result == .cannotComplete { inputDispatched = true; if press { return } } - if result == .success { inputDispatched = true; return } + if ![.illegalArgument, .attributeUnsupported, .actionUnsupported, .notImplemented, .apiDisabled].contains(result) { inputDispatched = true } + if result == .success || result == .cannotComplete && press { return } try axCheck(result) } /// Some apps (Calendar on macOS 26) answer AXPress on a menu item by opening its menu with the item @@ -62,15 +63,12 @@ func completeMenuPress(_ item: AXUIElement, pid: pid_t) async throws { let submenu = (attribute(item, kAXChildrenAttribute) as? [AXUIElement] ?? []).contains { attribute($0, kAXRoleAttribute) as? String == kAXMenuRole } guard attribute(item, kAXRoleAttribute) as? String == kAXMenuItemRole, !submenu else { return } try await Task.sleep(for: .milliseconds(250)) - // An open menu is an on-screen window at the pop-up menu level; a menu that ran its item has none. - let menuLevel = Int(CGWindowLevelForKey(.popUpMenuWindow)) - guard let entries = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]], - entries.contains(where: { ($0[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value == pid && ($0[kCGWindowLayer as String] as? NSNumber)?.intValue == menuLevel }) else { return } - try checkCancellation() - // Return runs whatever is highlighted, so it is sent only when this exact item is. - _ = AXUIElementSetAttributeValue(item, kAXSelectedAttribute as CFString, kCFBooleanTrue) - guard attribute(item, kAXSelectedAttribute) as? Bool == true else { return } try checkCancellation() + // A press that ran its item closes the item's menu (a closed menu has no height). Return runs the + // highlighted item, so it is sent only while this item's own menu is open and the app highlights it. + guard let rawMenu = attribute(item, kAXParentAttribute), CFGetTypeID(rawMenu) == AXUIElementGetTypeID(), + case let menu = unsafeBitCast(rawMenu, to: AXUIElement.self), attribute(menu, kAXRoleAttribute) as? String == kAXMenuRole, + (elementRect(menu)?.height ?? 0) > 0, attribute(item, kAXSelectedAttribute) as? Bool == true else { return } try postKey(pid: pid, code: 36) } /// A menu item's keyboard shortcut in macOS notation, such as ⌥⌘N, so the agent can use real shortcuts. @@ -118,6 +116,20 @@ func holdsFocus(_ focused: CFTypeRef, _ element: AXUIElement) -> Bool { } return false } +/// Whether typed characters land as text: a text, combo or date control, or anything with an editable +/// text selection. Buttons, lists and web areas take characters as commands instead. +func takesText(_ element: AXUIElement) -> Bool { + if [kAXTextFieldRole, kAXTextAreaRole, kAXComboBoxRole, kAXDateFieldRole, kAXTimeFieldRole].contains(attribute(element, kAXRoleAttribute) as? String ?? "") { return true } + var settable: DarwinBoolean = false + return AXUIElementIsAttributeSettable(element, kAXSelectedTextRangeAttribute as CFString, &settable) == .success && settable.boolValue +} +/// The window holding `element`: the element itself when it is a window or sheet, which report no +/// window of their own, otherwise the window it reports. +func containingWindow(_ element: AXUIElement) -> AXUIElement? { + if [kAXWindowRole, kAXSheetRole].contains(attribute(element, kAXRoleAttribute) as? String ?? "") { return element } + guard let raw = attribute(element, kAXWindowAttribute), CFGetTypeID(raw) == AXUIElementGetTypeID() else { return nil } + return unsafeBitCast(raw, to: AXUIElement.self) +} func axActions(_ element: AXUIElement) -> [String] { var names: CFArray? return AXUIElementCopyActionNames(element, &names) == .success ? names as? [String] ?? [] : [] @@ -273,6 +285,10 @@ final class Snapshot { queue.append(AXTraversalEntry(element: element, parent: depth == 0 ? nil : path[depth - 1], depth: depth, menu: false, priority: true)) } } + // The menu bar's titles and any open menu come before window content, which can exhaust the budget. + if let raw = attribute(root, kAXMenuBarAttribute), CFGetTypeID(raw) == AXUIElementGetTypeID() { + queue.append(AXTraversalEntry(element: unsafeBitCast(raw, to: AXUIElement.self), parent: root, depth: 1, menu: false, priority: true)) + } var menuCount = 0 while nodes.count < 2000 && Date().timeIntervalSince(created) < 8, let entry = queue.next() { try checkCancellation() @@ -281,7 +297,7 @@ final class Snapshot { if elements.values.contains(where: { CFEqual($0, element) }) { continue } let role = attribute(element, kAXRoleAttribute) as? String ?? "AXUnknown" let isMenu = entry.menu || [kAXMenuBarRole, kAXMenuRole, kAXMenuItemRole, kAXMenuBarItemRole].contains(role) - if isMenu && !entry.menu && !entry.priority { + if role == kAXMenuItemRole && !entry.menu && !entry.priority { queue.append(AXTraversalEntry(element: element, parent: entry.parent, depth: entry.depth, menu: true, priority: false)); continue } if isMenu && !entry.priority { @@ -316,13 +332,20 @@ final class Snapshot { if !isSecure, entry.depth < 64 { let (children, childrenTruncated) = pagedElements(element, name: kAXChildrenAttribute, limit: 500) truncated = truncated || childrenTruncated - // An open menu is what the user is choosing from, so its items and their submenus come first. - let openMenu = isMenu && (entry.priority || role == kAXMenuRole && (elementRect(element)?.height ?? 0) > 0) - for child in children { queue.append(AXTraversalEntry(element: child, parent: element, depth: entry.depth + 1, menu: isMenu && !openMenu, priority: openMenu)) } + // An open menu's items and their submenus come first; a closed menu (no height) keeps its items last. + let openMenu = role == kAXMenuRole ? (elementRect(element)?.height ?? 0) > 0 : isMenu && entry.priority + let deferChildren = !openMenu && (entry.menu || role == kAXMenuRole) + for child in children { queue.append(AXTraversalEntry(element: child, parent: element, depth: entry.depth + 1, menu: deferChildren, priority: openMenu)) } } else if entry.depth >= 64 { truncated = true } } if queue.hasPending || queue.truncated { truncated = true } } + /// The observed windowId whose window holds `element`; a sheet counts as the window it is attached to. + func windowId(holding element: AXUIElement) -> String? { + guard var window = containingWindow(element) else { return nil } + if attribute(window, kAXRoleAttribute) as? String == kAXSheetRole { guard let owner = sheetOwner(window) else { return nil }; window = owner } + return windowElements.first { CFEqual($0.value, window) }?.key + } func observedWindows() -> [(element: AXUIElement, frame: CGRect)] { windowFrames.compactMap { id, frame in windowElements[id].map { ($0, frame) } } } @@ -432,8 +455,8 @@ func requireForegroundTarget(pid: pid_t, windowID: String, window: AXUIElement, // The point must belong to the observed window (its sheets and popovers report it as their window), // to one of the app's dialogs or floating panels above it, or to an open menu, which has no window. // Anything unconfirmed is refused. - if let rawWindow = attribute(hit, kAXWindowAttribute), CFGetTypeID(rawWindow) == AXUIElementGetTypeID() { - guard acceptsInput(unsafeBitCast(rawWindow, to: AXUIElement.self), observed: [(window, frame)]) else { + if let hitWindow = containingWindow(hit) { + guard acceptsInput(hitWindow, observed: [(window, frame)]) else { throw ComputerError("window_occluded", "Another of this app's windows covers the target point; observe that window and target it instead.") } } else if ![kAXMenuRole, kAXMenuItemRole, kAXMenuBarRole, kAXMenuBarItemRole].contains(attribute(hit, kAXRoleAttribute) as? String ?? "") { @@ -589,14 +612,20 @@ final class Driver { try await snapshot.read(preferredWindowFrame: preferredFrame) let axWindows = snapshot.elements.values.filter { attribute($0, kAXRoleAttribute) as? String == kAXWindowRole } let axWindowFrames = axWindows.compactMap(elementRect) - let windows = windowsFor(app.processIdentifier).prefix(100).compactMap { entry -> [String: Any]? in - guard let id = entry[kCGWindowNumber as String] as? NSNumber, let bounds = entry[kCGWindowBounds as String] as? [String: Any], let rect = CGRect(dictionaryRepresentation: bounds as CFDictionary), rect.width > 0, rect.height > 0, axWindowFrames.contains(rect) else { return nil } + let entries = windowsFor(app.processIdentifier).prefix(100) + func bounds(_ entry: [String: Any]) -> CGRect? { (entry[kCGWindowBounds as String] as? [String: Any]).flatMap { CGRect(dictionaryRepresentation: $0 as CFDictionary) } } + func onScreen(_ entry: [String: Any]) -> Bool { entry[kCGWindowIsOnscreen as String] as? Bool == true } + // A background tab is an off-screen window with the bounds of the tab in front, which is the + // only one Accessibility shows, so it can be neither observed nor targeted. + let onScreenRects = entries.filter(onScreen).compactMap(bounds) + let windows = entries.compactMap { entry -> [String: Any]? in + guard let id = entry[kCGWindowNumber as String] as? NSNumber, let rect = bounds(entry), rect.width > 0, rect.height > 0, axWindowFrames.contains(rect), onScreen(entry) || !onScreenRects.contains(rect) else { return nil } snapshot.windowFrames[id.stringValue] = rect // Public APIs match a window-list entry to its Accessibility window only by bounds; when // two windows share bounds that match is ambiguous, so neither gets an identity and - // input aimed at either is refused. + // input aimed at either is refused. An off-screen (minimized) window takes no input either. let matches = axWindows.filter { elementRect($0) == rect } - if matches.count == 1 { snapshot.windowElements[id.stringValue] = matches[0] } + if matches.count == 1, onScreen(entry) { snapshot.windowElements[id.stringValue] = matches[0] } var result: [String: Any] = ["windowId": id.stringValue, "title": entry[kCGWindowName as String] as? String ?? ""] if let title = result["title"] as? String { result["title"] = String(title.prefix(4096)) } result.merge(rectJSON(rect)) { _, new in new }; return result @@ -604,11 +633,9 @@ final class Driver { if snapshots.count >= 16 { snapshots.removeAll() } snapshots[app.bundleIdentifier!] = snapshot let root = AXUIElementCreateApplication(app.processIdentifier) - var focusedFrame: CGRect? - if let focused = attribute(root, kAXFocusedWindowAttribute), CFGetTypeID(focused) == AXUIElementGetTypeID() { focusedFrame = elementRect(unsafeBitCast(focused, to: AXUIElement.self)) } - let focusedID = snapshot.windowFrames.first { $0.value == focusedFrame }?.key + let focusedID = attribute(root, kAXFocusedWindowAttribute).flatMap { CFGetTypeID($0) == AXUIElementGetTypeID() ? snapshot.windowId(holding: unsafeBitCast($0, to: AXUIElement.self)) : nil } for index in snapshot.nodes.indices { - if let element = snapshot.elements[snapshot.nodes[index]["elementId"] as? String ?? ""], attribute(element, kAXRoleAttribute) as? String == kAXWindowRole, let frame = elementRect(element), let windowID = snapshot.windowFrames.first(where: { $0.value == frame })?.key { snapshot.nodes[index]["windowId"] = windowID } + if let element = snapshot.elements[snapshot.nodes[index]["elementId"] as? String ?? ""], attribute(element, kAXRoleAttribute) as? String == kAXWindowRole, let windowID = snapshot.windowId(holding: element) { snapshot.nodes[index]["windowId"] = windowID } else if let parentID = snapshot.nodes[index]["parentId"] as? String, let parent = snapshot.nodes.first(where: { $0["elementId"] as? String == parentID }), let windowID = parent["windowId"] as? String { snapshot.nodes[index]["windowId"] = windowID } } guard let selectedWindowId = p.windowId ?? focusedID ?? windows.first?["windowId"] as? String else { throw ComputerError("app_window_unavailable", "No accessible app window is available. Open a window, then observe the app again.") } @@ -638,7 +665,9 @@ final class Driver { } catch let error as ComputerError { // Input that reached the app may have changed it, so only an untouched app keeps its references. if inputDispatched { snapshots.removeValue(forKey: bundle); throw error } - throw ComputerError(error.code, error.message + (error.code == "stale_element" ? "" : " Nothing was sent to the app; this snapshot is still valid."), dispatchState: "not_started") + // Only a malformed or unsupported request leaves the observation itself usable. + let requestError = ["invalid_arguments", "invalid_coordinates", "invalid_window", "unsupported_action", "unsupported_key", "element_unavailable", "secure_element"].contains(error.code) + throw ComputerError(error.code, error.message + (requestError ? " Nothing was sent to the app; this snapshot is still valid." : ""), dispatchState: "not_started") } catch { snapshots.removeValue(forKey: bundle); throw error } @@ -670,8 +699,9 @@ final class Driver { } else { try await coordinateAction { dispatch in let position = try point(p, app: app, snapshot: snapshot); let mouse: CGMouseButton = button == "right" ? .right : .left - // Coordinates are window-local to the requested (or selected) window; an element target uses its own. - let windowID = try required(p.windowId ?? (p.elementId == nil ? snapshot.selectedWindowId : snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key), "windowId") + // Coordinates are window-local to the requested (or selected) window. An element targets its + // own window; an out-of-process panel's content reports none and the selected window's sheet check applies. + let windowID = try required(p.windowId ?? p.elementId.flatMap { snapshot.windowId(holding: try snapshot.element($0)) } ?? snapshot.selectedWindowId, "windowId") let frame = try required(snapshot.windowFrames[windowID], "window frame") for click in 1...count { try checkCancellation() @@ -685,29 +715,33 @@ final class Driver { let steps = request.method == "type_text" ? [InputStep(action: "type_text", text: try required(p.text, "text"), key: nil)] : try required(p.steps, "steps") try validateInputSteps(steps) let root = AXUIElementCreateApplication(pid) - // Keystrokes go to whatever holds keyboard focus, so the target is any focusable element - // (text fields, terminals, combo boxes, date fields, web editors), not only AX-writable ones. - var focusable: DarwinBoolean = false - let isFocused = attribute(root, kAXFocusedUIElementAttribute).map { CFEqual($0, element) } == true - guard isFocused || (AXUIElementIsAttributeSettable(element, kAXFocusedAttribute as CFString, &focusable) == .success && focusable.boolValue) else { - throw ComputerError("unsupported_action", "This element cannot take keyboard focus. Type into a text field, text area, combo box or other focusable control, or click it first.") + // Keystrokes go to whatever holds keyboard focus: a text control (terminals included, read-only + // or not), or a container whose focused editor already holds it, such as a web composer. + let targetTakesText = takesText(element) + guard targetTakesText || attribute(root, kAXFocusedUIElementAttribute).map({ holdsFocus($0, element) }) == true else { + throw ComputerError("unsupported_action", "This element does not take text. Type into a text field, text area, combo box or date field, or use press_key for shortcuts.") } if request.method == "input_sequence", p.activateFirst == true { try await activate(app) } // Activation can move focus, so whether the editor holds it is read after activating. - if attribute(root, kAXFocusedUIElementAttribute).map({ CFEqual($0, element) }) != true { + if attribute(root, kAXFocusedUIElementAttribute).map({ holdsFocus($0, element) }) != true { try axCheck(AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue)) try await Task.sleep(for: .milliseconds(50)) } let observedWindows = snapshot.observedWindows() + // The first editor found holding focus receives the whole input, so text never follows focus elsewhere. + var editor: AXUIElement? func validateFocus() throws { try checkCancellation(); try requireNonSecure(element) // The editor must still sit in an observed window (or a dialog above it) at the observed frame. - guard let rawWindow = attribute(element, kAXWindowAttribute), CFGetTypeID(rawWindow) == AXUIElementGetTypeID(), - acceptsInput(unsafeBitCast(rawWindow, to: AXUIElement.self), observed: observedWindows) else { + guard let window = containingWindow(element), acceptsInput(window, observed: observedWindows) else { throw ComputerError("stale_window", "The editor's window moved or is not an observed window; observe again.") } - guard let current = attribute(root, kAXFocusedUIElementAttribute), holdsFocus(current, element) else { throw ComputerError("focus_changed", "The exact editor is no longer focused; remaining input was not dispatched. Observe again.") } - if let focused = attribute(root, kAXFocusedUIElementAttribute), CFGetTypeID(focused) == AXUIElementGetTypeID() { try requireNonSecure(unsafeBitCast(focused, to: AXUIElement.self)) } + guard let current = attribute(root, kAXFocusedUIElementAttribute), holdsFocus(current, element), editor.map({ CFEqual($0, current) }) ?? true else { throw ComputerError("focus_changed", "The exact editor is no longer focused; remaining input was not dispatched. Observe again.") } + // Keystrokes land on the focused element, so it (or the text control targeted) must take text. + let receiver = unsafeBitCast(current, to: AXUIElement.self) + try requireNonSecure(receiver) + guard targetTakesText || takesText(receiver) else { throw ComputerError("unsupported_action", "Keyboard focus is not in a text input; no text was sent. Click into the text field, or use press_key for shortcuts.") } + editor = receiver } // AX focus requests and app activation may settle asynchronously. Wait without // posting input or requesting focus again; every eventual input keeps the same guard. @@ -796,8 +830,8 @@ final class Driver { let position = try point(p, app: app, snapshot: snapshot); let dx = p.deltaX ?? 0; let dy = p.deltaY ?? 0 guard dx.isFinite, dy.isFinite, abs(dx) <= 10000, abs(dy) <= 10000 else { throw ComputerError("invalid_arguments", "Scroll deltas exceed bounds.") } guard let event = CGEvent(scrollWheelEvent2Source: CGEventSource(stateID: .privateState), units: .pixel, wheelCount: 2, wheel1: Int32(-dy), wheel2: Int32(-dx), wheel3: 0) else { throw ComputerError("input_failed", "Could not create scroll event.") } - // Coordinates are window-local to the requested (or selected) window; an element target uses its own. - let windowID = try required(p.windowId ?? (p.elementId == nil ? snapshot.selectedWindowId : snapshot.windowFrames.first(where: { $0.value.contains(position) })?.key), "windowId") + // Coordinates are window-local to the requested (or selected) window; an element targets its own. + let windowID = try required(p.windowId ?? p.elementId.flatMap { snapshot.windowId(holding: try snapshot.element($0)) } ?? snapshot.selectedWindowId, "windowId") let frame = try required(snapshot.windowFrames[windowID], "window frame") try requireForegroundTarget(pid: pid, windowID: windowID, window: try required(snapshot.windowElements[windowID], "observed window"), point: position, frame: frame) event.location = position; dispatch.willDispatch(); event.post(tap: .cghidEventTap) diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md index d2c28de1a9b..3c8d22e8537 100644 --- a/apps/desktop/native/computer-use/README.md +++ b/apps/desktop/native/computer-use/README.md @@ -8,13 +8,13 @@ Compile each architecture with `swiftc -parse-as-library -O -target arm64-apple- Persistent newline-delimited JSON over stdin/stdout: `{id,method,params}` returns `{id,result}` or `{id,error:{code,message,dispatchState?}}`. The producer contract is `worker/packages/contracts/src/computer-use.ts` in mothership, copied into the desktop bridge by contract sync. `request_permission` is a local onboarding method, never a model tool; params are `{permission:"accessibility"|"screenCapture"}`. Status/preflight never triggers prompts. Errors go through the protocol, and stdout contains no logs. -The helper observes up to 2000 accessibility nodes (depth 64, queue 4000, traversal deadline 8 seconds, each AX call timeout 250ms). Selected-window and focused-element ancestry are prioritized; menu traversal is deferred and capped at 80 nodes. Electron accessibility is enabled using its documented `AXManualAccessibility` attribute. The helper observes 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every action that reaches the app; a request that fails before sending anything reports `dispatchState: "not_started"` and keeps them valid. Restarts invalidate all references. Screen captures exclude window shadows, preserve the selected window’s coordinate geometry, and are capped to 1600 pixels on the longest side and 8 MiB encoded. Complete JSON replies are capped at 16 MiB: oversized observations omit screenshots first, then truncate accessibility nodes, while preserving action outcomes. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. macOS 26 uses the screenshot-specific capture API; the macOS 14/15 API path includes only the selected window in a display filter and crops its exact frame. That older path requires the window to fit within one display and reports `capture_geometry_unavailable` otherwise. +The helper observes up to 2000 accessibility nodes (depth 64, queue 4000, traversal deadline 8 seconds, each AX call timeout 250ms). Selected-window and focused-element ancestry are prioritized. The menu bar and menus are read in place, so an open menu's items and their submenus come first; items of closed menus are read last, capped at 80 menu nodes. Electron accessibility is enabled using its documented `AXManualAccessibility` attribute. The helper observes 100 windows and 1000 apps. Requests are bounded to 128 KiB during ingestion. Text/value input is capped at 32000 UTF-16 units. State references are per-process, per-app, expire after 60 seconds, and are consumed by every action that reaches the app; a request that fails before sending anything reports `dispatchState: "not_started"` and keeps them valid. An Accessibility failure counts as sent unless Accessibility rejected the request before the app saw it (unsupported attribute or action, illegal argument). Restarts invalidate all references. Screen captures exclude window shadows, preserve the selected window’s coordinate geometry, and are capped to 1600 pixels on the longest side and 8 MiB encoded. Complete JSON replies are capped at 16 MiB: oversized observations omit screenshots first, then truncate accessibility nodes, while preserving action outcomes. Screenshots are optional and failure is reported in `screenshotError` without discarding accessibility state. macOS 26 uses the screenshot-specific capture API; the macOS 14/15 API path includes only the selected window in a display filter and crops its exact frame. That older path requires the window to fit within one display and reports `capture_geometry_unavailable` otherwise. -Coordinates in action inputs are window-local points; node frames are global screen points. Coordinate input without a `windowId` targets the window the snapshot selected; `press_key` with an `elementId` focuses that element first. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps and System Settings are protected targets. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`. +Coordinates in action inputs are window-local points; node frames are global screen points. Coordinate input without a `windowId` targets the window the snapshot selected, and an element target uses its own window; `press_key` with an `elementId` focuses that element first. Window bounds must still match the observed snapshot. App PID and launch time must match. Secure fields and descendants are unavailable as action targets; secure values and labels are suppressed. Sim's own apps and System Settings are protected targets. `get_app_state` can open an installed bundle ID without activating it. `list_apps` discovers installed apps under `/Applications`, `/System/Applications`, and `~/Applications` (bounded to 5000 filesystem entries and depth 3), merging running regular apps; installed apps omit `pid`. -AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. An action that opens UI (AXPress, AXPick, AXShowMenu) and times out while the app shows a sheet, modal or menu (`cannotComplete`) counts as dispatched; any other action that times out fails as uncertain. A menu-item press that leaves its menu open with the item highlighted (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Open menus and their submenus are observed first, and menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly. +AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. A hit on a window's own background counts as that window. Background tabs and minimized windows share bounds with the window in front, so they take no coordinate input. An action that opens UI (AXPress, AXPick, AXShowMenu) and times out while the app shows a sheet, modal or menu (`cannotComplete`) counts as dispatched; any other action that times out fails as uncertain. A menu-item press that leaves the item's own menu open with the app highlighting that item (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Menu item labels carry their shortcuts (for example ⇧⌘N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly. -`type_text` and `input_sequence` target any observed element that can take keyboard focus (text fields, terminals, combo boxes, web editors). `input_sequence` executes 1–32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly. +`type_text` and `input_sequence` target a text control (text fields and areas, terminals, combo boxes, date fields, or anything with an editable text selection), or a container whose focused editor already holds keyboard focus, such as a web composer; keystrokes are sent only while focus is in a text input. `input_sequence` executes 1–32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly. ## Cancellation diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index 0bac9b9f3d0..0a3bacb4eab 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -53,6 +53,14 @@ def mutate(method, snapshot, **params): return state() +def refused(method, snapshot, **params): + try: + call(method, bundleId=bundle, snapshotId=snapshot['snapshotId'], **params) + except AssertionError as error: + return json.loads(str(error).split(': ', 1)[1]) + raise AssertionError(method + ' was not refused') + + def coordinate(snapshot, node, fx=0.5, fy=0.5): window = next(w for w in snapshot['windows'] if w['windowId'] == snapshot['windowId']) return {'windowId': window['windowId'], 'x': node['x'] - window['x'] + node['width'] * fx, 'y': node['y'] - window['y'] + node['height'] * fy} @@ -106,10 +114,20 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5): scroll = find(snapshot, role='AXScrollArea') snapshot = mutate('scroll', snapshot, **coordinate(snapshot, scroll), deltaX=0, deltaY=180) assert any((node.get('value', '').startswith('Scroll: ') and node['value'] != 'Scroll: 0') or (node.get('label', '').startswith('Scroll: ') and node['label'] != 'Scroll: 0') for node in snapshot['nodes']), 'Scroll did not move fixture content' + slider = find(snapshot, role='AXSlider') + error = refused('type_text', snapshot, elementId=slider['elementId'], text=' ') + assert error['code'] == 'unsupported_action' and error['dispatchState'] == 'not_started', error + assert find(state(), role='AXSlider')['value'] == slider['value'], 'Text reached a control that does not take text' + snapshot = state() + snapshot = mutate('click', snapshot, elementId=find(snapshot, role='AXTextField', label='Fixture editable text')['elementId']) + snapshot = mutate('type_text', snapshot, elementId=find(snapshot, role='AXWindow')['elementId'], text=' window') + assert find(snapshot, role='AXTextField', label='Fixture editable text')['value'].endswith(' window'), 'Window target did not type into its focused editor' + window = next(w for w in snapshot['windows'] if w['windowId'] == snapshot['windowId']) + snapshot = mutate('click', snapshot, x=window['width'] * 0.75, y=8) button = find(snapshot, role='AXButton', label='Increment') snapshot = mutate('perform_action', snapshot, elementId=button['elementId'], accessibilityAction='AXPress') assert any(f'Count: {baseline + 2}' in node.get('value', '') or f'Count: {baseline + 2}' in node.get('label', '') for node in snapshot['nodes']), 'Explicit AX action did not increment' - print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling; every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) + print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, a window target typing into its focused editor, a title-bar click; every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) finally: if fixture: fixture.terminate() diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index e3149dc6aed..3f9cfd3fd02 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -180,7 +180,18 @@ export class ComputerUseService { return Promise.reject(new Error('Computer Use has too many queued actions.')) if (this.queued.has(toolCallId)) return Promise.reject(new Error('This Computer Use action is already running.')) - const input = ComputerUseSchema.parse(raw) + // A newer server can send arguments this build predates; nothing has been queued yet. + const parsed = ComputerUseSchema.safeParse(raw) + if (!parsed.success) + return Promise.reject( + new ComputerUseError({ + code: 'invalid_arguments', + message: + 'This desktop app does not support these Computer Use arguments; update Sim Desktop.', + dispatchState: 'not_started', + }) + ) + const input = parsed.data const work: ToolWork = { toolCallId, scopeId, input, controller: new AbortController() } this.queued.set(toolCallId, work) const result = this.tail.then(() => this.run(work)) diff --git a/apps/sim/lib/mothership/generated/computer-use.ts b/apps/sim/lib/mothership/generated/computer-use.ts index b4ae840cb70..71f908ed036 100644 --- a/apps/sim/lib/mothership/generated/computer-use.ts +++ b/apps/sim/lib/mothership/generated/computer-use.ts @@ -17,6 +17,16 @@ const SnapshotId = z .describe("Latest unconsumed snapshotId from get_app_state for this app. Actions consume it."); const ElementId = z.string().min(1).max(128).describe("Element reference from that snapshot."); const WindowId = z.string().min(1).max(128).describe("Window reference from that snapshot."); +/** Input fields shared across actions keep one description: Anthropic flattens the union and anyOfs differing copies. */ +const TargetWindowId = WindowId.optional().describe( + "Window reference from an earlier snapshot of this app. If omitted, get_app_state and press_key use the app's focused window; coordinate actions use the window the snapshot selected.", +); +const IncludeScreenshot = z + .boolean() + .optional() + .describe( + "Set true to include a screenshot in the returned state. On actions it overrides observeAfter.includeScreenshot.", + ); const Coordinate = z.number().nonnegative().max(100_000); const ObserveAfter = { observeAfter: z @@ -25,7 +35,7 @@ const ObserveAfter = { .describe( "Every mutation returns fresh app state by default. Set includeScreenshot to true to include an image; otherwise no screenshot is captured. Observation failure never retries the action.", ), - includeScreenshot: z.boolean().optional().describe("Same as observeAfter.includeScreenshot."), + includeScreenshot: IncludeScreenshot, }; const InputSequenceStep = z.discriminatedUnion("action", [ z.strictObject({ action: z.literal("type_text"), text: z.string().max(32_000) }), @@ -35,9 +45,7 @@ const ElementTarget = { bundleId: BundleId, snapshotId: SnapshotId, elementId: E const PointTarget = { bundleId: BundleId, snapshotId: SnapshotId, - windowId: WindowId.optional().describe( - "Window reference from that snapshot. Defaults to the window the snapshot selected.", - ), + windowId: TargetWindowId, x: Coordinate, y: Coordinate, }; @@ -51,13 +59,17 @@ const ScrollOffsets = { .min(-10_000) .max(10_000) .optional() - .describe("Horizontal scroll distance in points; positive scrolls right, negative left. Defaults to 0."), + .describe( + "Horizontal scroll distance in points; positive scrolls right, negative left. Omit for 0 when deltaY is set.", + ), deltaY: z .number() .min(-10_000) .max(10_000) .optional() - .describe("Vertical scroll distance in points; positive scrolls down, negative up. Defaults to 0."), + .describe( + "Vertical scroll distance in points; positive scrolls down, negative up. Omit for 0 when deltaX is set.", + ), }; const hasScrollDistance = (input: { deltaX?: number | undefined; deltaY?: number | undefined }) => @@ -71,11 +83,8 @@ export const ComputerUseSchema = z.union([ z.strictObject({ action: z.literal("get_app_state"), bundleId: BundleId, - windowId: WindowId.optional(), - includeScreenshot: z - .boolean() - .optional() - .describe("Set true to include an image of the selected window."), + windowId: TargetWindowId, + includeScreenshot: IncludeScreenshot, }), z.strictObject({ action: z.literal("click"), ...ElementTarget, ...ObserveAfter, ...ClickOptions }), z.strictObject({ action: z.literal("click"), ...PointTarget, ...ObserveAfter, ...ClickOptions }), @@ -90,11 +99,15 @@ export const ComputerUseSchema = z.union([ ...ObserveAfter, bundleId: BundleId, snapshotId: SnapshotId, - windowId: WindowId.optional().describe( - "Window reference from that snapshot. Omit to send the key to the app's focused window.", - ), - elementId: ElementId.optional().describe("Focus this element from the snapshot before pressing the key."), - key: z.string().min(1).max(128).describe("One key or chord, such as Enter, Tab, Escape, or Cmd+A."), + windowId: TargetWindowId, + elementId: ElementId.optional(), + key: z + .string() + .min(1) + .max(128) + .describe( + "One key or chord, such as Enter, Tab, Escape, or Cmd+A. With elementId, focuses that element first.", + ), }), z .strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets }) diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts index b4ae840cb70..71f908ed036 100644 --- a/packages/desktop-bridge/src/computer-use.generated.ts +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -17,6 +17,16 @@ const SnapshotId = z .describe("Latest unconsumed snapshotId from get_app_state for this app. Actions consume it."); const ElementId = z.string().min(1).max(128).describe("Element reference from that snapshot."); const WindowId = z.string().min(1).max(128).describe("Window reference from that snapshot."); +/** Input fields shared across actions keep one description: Anthropic flattens the union and anyOfs differing copies. */ +const TargetWindowId = WindowId.optional().describe( + "Window reference from an earlier snapshot of this app. If omitted, get_app_state and press_key use the app's focused window; coordinate actions use the window the snapshot selected.", +); +const IncludeScreenshot = z + .boolean() + .optional() + .describe( + "Set true to include a screenshot in the returned state. On actions it overrides observeAfter.includeScreenshot.", + ); const Coordinate = z.number().nonnegative().max(100_000); const ObserveAfter = { observeAfter: z @@ -25,7 +35,7 @@ const ObserveAfter = { .describe( "Every mutation returns fresh app state by default. Set includeScreenshot to true to include an image; otherwise no screenshot is captured. Observation failure never retries the action.", ), - includeScreenshot: z.boolean().optional().describe("Same as observeAfter.includeScreenshot."), + includeScreenshot: IncludeScreenshot, }; const InputSequenceStep = z.discriminatedUnion("action", [ z.strictObject({ action: z.literal("type_text"), text: z.string().max(32_000) }), @@ -35,9 +45,7 @@ const ElementTarget = { bundleId: BundleId, snapshotId: SnapshotId, elementId: E const PointTarget = { bundleId: BundleId, snapshotId: SnapshotId, - windowId: WindowId.optional().describe( - "Window reference from that snapshot. Defaults to the window the snapshot selected.", - ), + windowId: TargetWindowId, x: Coordinate, y: Coordinate, }; @@ -51,13 +59,17 @@ const ScrollOffsets = { .min(-10_000) .max(10_000) .optional() - .describe("Horizontal scroll distance in points; positive scrolls right, negative left. Defaults to 0."), + .describe( + "Horizontal scroll distance in points; positive scrolls right, negative left. Omit for 0 when deltaY is set.", + ), deltaY: z .number() .min(-10_000) .max(10_000) .optional() - .describe("Vertical scroll distance in points; positive scrolls down, negative up. Defaults to 0."), + .describe( + "Vertical scroll distance in points; positive scrolls down, negative up. Omit for 0 when deltaX is set.", + ), }; const hasScrollDistance = (input: { deltaX?: number | undefined; deltaY?: number | undefined }) => @@ -71,11 +83,8 @@ export const ComputerUseSchema = z.union([ z.strictObject({ action: z.literal("get_app_state"), bundleId: BundleId, - windowId: WindowId.optional(), - includeScreenshot: z - .boolean() - .optional() - .describe("Set true to include an image of the selected window."), + windowId: TargetWindowId, + includeScreenshot: IncludeScreenshot, }), z.strictObject({ action: z.literal("click"), ...ElementTarget, ...ObserveAfter, ...ClickOptions }), z.strictObject({ action: z.literal("click"), ...PointTarget, ...ObserveAfter, ...ClickOptions }), @@ -90,11 +99,15 @@ export const ComputerUseSchema = z.union([ ...ObserveAfter, bundleId: BundleId, snapshotId: SnapshotId, - windowId: WindowId.optional().describe( - "Window reference from that snapshot. Omit to send the key to the app's focused window.", - ), - elementId: ElementId.optional().describe("Focus this element from the snapshot before pressing the key."), - key: z.string().min(1).max(128).describe("One key or chord, such as Enter, Tab, Escape, or Cmd+A."), + windowId: TargetWindowId, + elementId: ElementId.optional(), + key: z + .string() + .min(1) + .max(128) + .describe( + "One key or chord, such as Enter, Tab, Escape, or Cmd+A. With elementId, focuses that element first.", + ), }), z .strictObject({ action: z.literal("scroll"), ...ElementTarget, ...ObserveAfter, ...ScrollOffsets })