From 2a83d1c28d70bcf852cc36ec2d616bdd29467ad1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 16 Oct 2026 01:43:15 -0700 Subject: [PATCH] fix(desktop): let computer use type into date and time pickers --- apps/desktop/native/computer-use/ComputerUse.swift | 6 +++--- apps/desktop/native/computer-use/README.md | 2 +- apps/desktop/native/computer-use/tests/Fixture.swift | 9 ++++++++- apps/desktop/native/computer-use/tests/live.py | 5 ++++- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index f3129047b73..67a01aae22e 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -116,10 +116,10 @@ func holdsFocus(_ focused: CFTypeRef, _ element: AXUIElement) -> Bool { } return false } -/// Whether typed characters land as text: a text, combo or date control, or anything with an editable -/// text selection. Buttons, lists and web areas take characters as commands instead. +/// Whether typed characters land as text: a text, combo, date or time control (`AXDateTimeArea` is an +/// NSDatePicker's field), or anything with an editable text selection. Buttons, lists and web areas take characters as commands instead. func takesText(_ element: AXUIElement) -> Bool { - if [kAXTextFieldRole, kAXTextAreaRole, kAXComboBoxRole, kAXDateFieldRole, kAXTimeFieldRole].contains(attribute(element, kAXRoleAttribute) as? String ?? "") { return true } + if [kAXTextFieldRole, kAXTextAreaRole, kAXComboBoxRole, kAXDateFieldRole, kAXTimeFieldRole, "AXDateTimeArea"].contains(attribute(element, kAXRoleAttribute) as? String ?? "") { return true } var settable: DarwinBoolean = false return AXUIElementIsAttributeSettable(element, kAXSelectedTextRangeAttribute as CFString, &settable) == .success && settable.boolValue } diff --git a/apps/desktop/native/computer-use/README.md b/apps/desktop/native/computer-use/README.md index 3c8d22e8537..c3971b54d70 100644 --- a/apps/desktop/native/computer-use/README.md +++ b/apps/desktop/native/computer-use/README.md @@ -14,7 +14,7 @@ Coordinates in action inputs are window-local points; node frames are global scr AXPress and AXSetValue are direct semantic operations. Keyboard events target the app PID. Cmd+A uses the focused editor's writable accessibility selection range where supported. Coordinate mouse input requires the target app in front; `activate_app` makes that focus change explicit and confirms it. Click/drag/scroll validate the foreground process, unchanged window geometry and the app that receives the point; the topmost window-list entry is not the click target because Notification Center and the Dock keep full-screen click-through windows. A hit on a window's own background counts as that window. Background tabs and minimized windows share bounds with the window in front, so they take no coordinate input. An action that opens UI (AXPress, AXPick, AXShowMenu) and times out while the app shows a sheet, modal or menu (`cannotComplete`) counts as dispatched; any other action that times out fails as uncertain. A menu-item press that leaves the item's own menu open with the app highlighting that item (Calendar on macOS 26) is completed with Return. Sheets, including Go to Folder and out-of-process open and save panels whose contents report no window, count as the window they are attached to. Menu item labels carry their shortcuts (for example โ‡งโŒ˜N). Some apps ignore background keyboard events. `dispatched:true,verified:false` means only dispatch succeeded; the agent must observe again to establish the outcome. No foreground activation fallback is hidden inside the helper. Native key codes currently assume the US layout for shortcuts; literal text uses Unicode events. Unsupported shortcuts fail explicitly. -`type_text` and `input_sequence` target a text control (text fields and areas, terminals, combo boxes, date fields, or anything with an editable text selection), or a container whose focused editor already holds keyboard focus, such as a web composer; keystrokes are sent only while focus is in a text input. `input_sequence` executes 1โ€“32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly. +`type_text` and `input_sequence` target a text control (text fields and areas, terminals, combo boxes, date and time pickers, or anything with an editable text selection), or a container whose focused editor already holds keyboard focus, such as a web composer; keystrokes are sent only while focus is in a text input. `input_sequence` executes 1โ€“32 key/text steps against one such element. It validates every step before dispatch and checks the same editor, window, and secure-input ancestry between steps. Enter can continue into another step when the same editor and window retain focus. Actual focus loss, Tab, or navigation shortcuts end the sequence before later input. Explicit `activateFirst: true` activates the target app inside the batch before input; it never enables automatic refocusing between steps. Partial execution reports completed steps and an error; callers must observe before retrying. Every mutation is followed by fresh state under the same authorization and queue entry; `observeAfter` optionally requests a screenshot. Observation failure is reported separately so the mutation is never replayed. The model receives a compact, bounded projection of the native snapshot; omitted nodes are reported explicitly. ## Cancellation diff --git a/apps/desktop/native/computer-use/tests/Fixture.swift b/apps/desktop/native/computer-use/tests/Fixture.swift index 9e934a8ee93..6445f5bbbfb 100644 --- a/apps/desktop/native/computer-use/tests/Fixture.swift +++ b/apps/desktop/native/computer-use/tests/Fixture.swift @@ -13,6 +13,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { var count = 0 let counter = NSTextField(labelWithString: "Count: 0") let scrollCounter = NSTextField(labelWithString: "Scroll: 0") + let timeLabel = NSTextField(labelWithString: "Time: 10:15") func applicationDidFinishLaunching(_ notification: Notification) { NSEvent.addLocalMonitorForEvents(matching: [.leftMouseDown, .leftMouseDragged, .leftMouseUp, .scrollWheel]) { event in print("fixture-event type=\(event.type.rawValue) window=\(event.windowNumber) x=\(event.locationInWindow.x) y=\(event.locationInWindow.y)"); fflush(stdout) @@ -31,6 +32,8 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { let secure = NSSecureTextField(string: "fixture-only-secret"); secure.setAccessibilityIdentifier("fixture-password"); secure.setAccessibilityLabel("Fixture password") let button = NSButton(title: "Increment", target: self, action: #selector(increment)); button.setAccessibilityIdentifier("fixture-increment") let slider = NSSlider(value: 25, minValue: 0, maxValue: 100, target: nil, action: nil); slider.setAccessibilityLabel("Fixture slider") + let time = NSDatePicker(); time.datePickerStyle = .textField; time.datePickerElements = .hourMinute; time.setAccessibilityLabel("Fixture time") + time.dateValue = Calendar.current.date(bySettingHour: 10, minute: 15, second: 0, of: Date()) ?? Date(); time.target = self; time.action = #selector(timeChanged(_:)) let marker = MarkerView(); marker.setAccessibilityElement(false) marker.translatesAutoresizingMaskIntoConstraints = false let scroll = NSScrollView(); scroll.hasVerticalScroller = true @@ -40,7 +43,7 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { scroll.documentView = document scroll.contentView.postsBoundsChangedNotifications = true NotificationCenter.default.addObserver(self, selector: #selector(scrolled(_:)), name: NSView.boundsDidChangeNotification, object: scroll.contentView) - for view in [field, secure, button, counter, slider, marker, scrollCounter, scroll] { stack.addArrangedSubview(view) } + for view in [field, secure, button, counter, slider, time, timeLabel, marker, scrollCounter, scroll] { stack.addArrangedSubview(view) } window.contentView!.addSubview(stack) NSLayoutConstraint.activate([stack.leadingAnchor.constraint(equalTo: window.contentView!.leadingAnchor, constant: 30), stack.trailingAnchor.constraint(equalTo: window.contentView!.trailingAnchor, constant: -30), stack.topAnchor.constraint(equalTo: window.contentView!.topAnchor, constant: 30), field.widthAnchor.constraint(equalTo: stack.widthAnchor), secure.widthAnchor.constraint(equalTo: stack.widthAnchor), slider.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.widthAnchor.constraint(equalTo: stack.widthAnchor), scroll.heightAnchor.constraint(equalToConstant: 230), marker.widthAnchor.constraint(equalToConstant: 100), marker.heightAnchor.constraint(equalToConstant: 70)]) window.makeKeyAndOrderFront(nil) @@ -49,6 +52,10 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { print("fixture-ready"); fflush(stdout) } @objc func increment() { count += 1; counter.stringValue = "Count: \(count)" } + @objc func timeChanged(_ picker: NSDatePicker) { + let time = Calendar.current.dateComponents([.hour, .minute], from: picker.dateValue) + timeLabel.stringValue = "Time: \(time.hour ?? 0):\(String(format: "%02d", time.minute ?? 0))" + } @objc func scrolled(_ notification: Notification) { guard let clip = notification.object as? NSClipView else { return } scrollCounter.stringValue = "Scroll: \(Int(clip.bounds.origin.y))" diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index 0a3bacb4eab..f6ddf624952 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -119,6 +119,9 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5): assert error['code'] == 'unsupported_action' and error['dispatchState'] == 'not_started', error assert find(state(), role='AXSlider')['value'] == slider['value'], 'Text reached a control that does not take text' snapshot = state() + snapshot = mutate('click', snapshot, **coordinate(snapshot, find(snapshot, role='AXDateTimeArea'), 0.15)) + snapshot = mutate('type_text', snapshot, elementId=find(snapshot, role='AXDateTimeArea')['elementId'], text='3') + assert any(node.get('value', node.get('label', '')) == 'Time: 3:15' for node in snapshot['nodes']), 'Typing did not reach the date picker' snapshot = mutate('click', snapshot, elementId=find(snapshot, role='AXTextField', label='Fixture editable text')['elementId']) snapshot = mutate('type_text', snapshot, elementId=find(snapshot, role='AXWindow')['elementId'], text=' window') assert find(snapshot, role='AXTextField', label='Fixture editable text')['value'].endswith(' window'), 'Window target did not type into its focused editor' @@ -127,7 +130,7 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5): button = find(snapshot, role='AXButton', label='Increment') snapshot = mutate('perform_action', snapshot, elementId=button['elementId'], accessibilityAction='AXPress') assert any(f'Count: {baseline + 2}' in node.get('value', '') or f'Count: {baseline + 2}' in node.get('label', '') for node in snapshot['nodes']), 'Explicit AX action did not increment' - print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, a window target typing into its focused editor, a title-bar click; every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) + print('PASS: real fixture state, secure suppression, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click; every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) finally: if fixture: fixture.terminate()