diff --git a/.gitignore b/.gitignore index ab74cb98..973bb7a4 100644 --- a/.gitignore +++ b/.gitignore @@ -116,16 +116,13 @@ dmypy.json # Pyre type checker .pyre/ -# Pycharm -.idea -.idea/* -.idea/AutoControl.iml -.idea/misc.xml -.idea/workspace.xml +# Pycharm — the rule is `.idea/` at the top of this file. Five more spellings +# accumulated here because the files were already tracked, and .gitignore has +# no effect on a tracked file however many ways you name it. They were +# untracked with `git rm --cached -r .idea`, so one rule is enough now. .claude/settings.local.json /.claude/ /.claude -/.idea # Local test/smoke artifacts .test-tmp/ diff --git a/.idea/AutoControl.iml b/.idea/AutoControl.iml deleted file mode 100644 index d8b726ac..00000000 --- a/.idea/AutoControl.iml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/discord.xml b/.idea/discord.xml deleted file mode 100644 index 912db825..00000000 --- a/.idea/discord.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 5d6d9bd2..00000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2da..00000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 82f64c00..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 49cadaf8..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 50980db1..00000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,685 +0,0 @@ - - - - - - - - - - - - - - { - "lastFilter": { - "state": "OPEN", - "assignee": "JE-Chen" - } -} - - - - - - - - - { - "selectedUrlAndAccountId": { - "url": "https://github.com/Integration-Automation/AutoControlGUI.git", - "accountId": "a99e3205-1b0b-4362-b014-5d2163fb0c3a" - }, - "editorReviewEnabled": false -} - - - - - - - - - - - - - { - "associatedIndex": 4 -} - - - - - - - { - "keyToString": { - "DefaultHtmlFileTemplate": "HTML File", - "Python.auto_control_keyboard.executor": "Run", - "Python.auto_control_mouse.executor": "Run", - "Python.calculator.executor": "Run", - "Python.callback_test.executor": "Run", - "Python.create_project_test.executor": "Run", - "Python.critical_exit_test.executor": "Run", - "Python.executor_one_file.executor": "Run", - "Python.get_pixel_test.executor": "Run", - "Python.keyboard_is_press_test.executor": "Run", - "Python.keyboard_type_test.executor": "Run", - "Python.main_widget.executor": "Run", - "Python.main_window.executor": "Run", - "Python.mouse_test.executor": "Run", - "Python.record_test.executor": "Run", - "Python.screen_test.executor": "Run", - "Python.screenshot_test.executor": "Run", - "Python.test.executor": "Run", - "Python.video_recording.executor": "Run", - "Python.win32_screen.executor": "Run", - "RunOnceActivity.OpenProjectViewOnStart": "true", - "RunOnceActivity.ShowReadmeOnStart": "true", - "RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252": "true", - "RunOnceActivity.git.unshallow": "true", - "RunOnceActivity.typescript.service.memoryLimit.init": "true", - "WebServerToolWindowFactoryState": "false", - "codeWithMe.voiceChat.enabledByDefault": "false", - "com.intellij.ml.llm.matterhorn.ej.ui.settings.DefaultAutoModeForALLUsers.v1": "true", - "com.intellij.ml.llm.matterhorn.ej.ui.settings.DefaultModelSelectionForGA.v1": "true", - "git-widget-placeholder": "dev", - "ignore.virus.scanning.warn.message": "true", - "junie.onboarding.icon.badge.shown": "true", - "last_opened_file_path": "D:/Codes/AutoControlGUI", - "node.js.detected.package.eslint": "true", - "node.js.detected.package.tslint": "true", - "node.js.selected.package.eslint": "(autodetect)", - "node.js.selected.package.tslint": "(autodetect)", - "nodejs_package_manager_path": "npm", - "settings.editor.selected.configurable": "discord-application", - "to.speed.mode.migration.done": "true", - "vue.rearranger.settings.migration": "true" - } -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - C:\Users\user\AppData\Roaming\Subversion - - - - - 1629079247175 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 06d0aa0b..aa0064d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,12 +14,120 @@ only when documented here with a migration path. - Portable `autocontrol.failure-bundle/v1` diagnostic archives and CLI command. - Public API lifecycle, capability matrix, security policy, coverage and type checking configuration. +- Unicode text entry by key injection: `type_unicode_keys`, `type_unicode_text`, + `plan_unicode_keys`, `unicode_keys_supported` (commands + `AC_type_unicode_keys` / `AC_type_unicode_text`, MCP tools + `ac_type_unicode_keys` / `ac_type_unicode_text`), on Windows backend + primitives `press_unicode` / `release_unicode` / `type_unicode_unit`. +- Cross-word OCR matching helpers `find_spans` / `group_lines`. +- `monitor_layout.grab_logical` / `logical_virtual_rect` / `logical_scale` / + `needs_rescale` — screen capture in the coordinate space the mouse uses. +- `find_image` / `find_image_multi` accept `all_screens` and `screen_region`. +- `AutoControlFlatTemplateException` (a subclass of `AutoControlScreenException`) + for a template with too little variation to locate. +- Accessibility search scoping and matching: `window_title` on + `list_accessibility_elements` / `find_accessibility_element` / + `click_accessibility_element` / `control_get_state`, a `contains` substring + mode with exact-name ranking, `find_accessibility_elements`, + `accessibility_status`, `control_get_state`, and `rank_by_name` (commands + `AC_a11y_find_all` / `AC_control_get_state`, MCP `ac_a11y_find_all` / + `ac_control_get_state`). The accessibility GUI tab gains a window filter. +- `AccessibilityElement.enabled`. +- `stop_record_timeline` (`AC_stop_record_timeline`, + `ac_record_stop_timeline`): the recording as press *and* release, wheel + movement and `delta_ms`, ready for `replay_timeline`. +- `utils/input_reach`: `input_desktop_available`, `input_reaches_system` + (`AC_input_reachable`, `ac_input_reachable`) — whether input this process + sends can actually arrive. The second probe presses F13 to find out. +- `utils/keyboard_layout`: `char_table`, `layout_char_table`, `vk_to_char`, + `foreground_keyboard_layout` — which character each key produces on the + active layout, with a US fallback. +- Window management gains the primitives it was missing: + `minimize_window_by_title`, `foreground_window`, `window_rect` and + `move_window_by_title` (`AC_minimize_window`, `AC_foreground_window`, + `AC_window_rect`, `AC_move_window`; `ac_minimize_window`, + `ac_foreground_window`, `ac_window_rect`). `list_windows` takes + `titled_only`, and `move_window_by_title` keeps the window's current size + when width/height are omitted. +- `utils/url_canon` reaches its delivery surfaces: `canonicalize_url`, + `normalize_url`, `urls_equal`, `build_query` and `parse_query` are exported + from the facade, with `AC_canonicalize_url` / `AC_normalize_url` / + `AC_urls_equal`, the matching `ac_*` MCP tools, and three Script Builder + specs. The module and its tests already existed; only the wiring is new. + +### Removed + +- **Breaking — `je_auto_control.windows.listener` is gone**, with its + `Win32KeyboardListener` and `Win32MouseListener` classes. Recording moved to + `windows/record/win32_input_hook.py`, after which nothing in the package or + the test suite referenced them. +- **Breaking — `je_auto_control.utils.clipboard.clipboard_image` is gone.** Its + two functions were duplicates of the ones in + `je_auto_control.utils.clipboard.clipboard`, under identical names but with a + different `set_clipboard_image` signature, so importing the wrong module + failed at runtime and only for one of the two argument types. Import from + `je_auto_control.utils.clipboard` (or the top-level facade) instead; the + surviving function accepts both PNG bytes and a file path. ### Changed +- `set_clipboard_image` accepts PNG bytes **or** a path to any Pillow-readable + image, and `get_clipboard_image` / `set_clipboard_image` are now exported + from `je_auto_control.utils.clipboard` and the top-level facade, with + `AC_clipboard_get_image` / `AC_clipboard_set_image` commands. They were + previously reachable only through MCP and the GUI, not `execute_action`. +- **Breaking — `close_window_by_title` / `AC_close_window` / `ac_close_window` + now actually close the window** (they post `WM_CLOSE`). They previously + *minimised* it: the Win32 call underneath is named `CloseWindow` but + minimises, and the wrapper inherited both the call and the wrong promise, so + every caller asking to close a window silently got a minimise instead. The + old behaviour is available unchanged as `minimize_window_by_title` / + `AC_minimize_window` / `ac_minimize_window`. +- `focus_window` restores a window that is minimised before bringing it to the + front — focusing a minimised window used to do nothing visible. A maximised + window is left maximised (`SW_RESTORE` would have un-maximised it). +- `show_window_by_title` no longer calls `SetForegroundWindow` after `SW_HIDE`; + hiding a window and then pulling it forward are contradictory. - Releases are prepared from version tags and use PyPI Trusted Publishing. - The USB/IP server binds `127.0.0.1` by default (least-privilege). Exporting the attached device to the LAN now requires an explicit `host="0.0.0.0"`. +- `write` no longer raises on a character missing from the virtual-key table + where the backend can inject Unicode; it types that character instead. +- `find_text_matches` returns runs of consecutive word boxes, so a target split + across boxes now matches. Results are merged boxes covering the whole run + (union rectangle, minimum confidence) rather than one box per word. +- `find_image` / `find_image_multi` search every monitor by default and return + virtual-desktop coordinates, which are negative when a monitor sits left of or + above the primary. Pass `all_screens=False` for the previous primary-only + behaviour. +- `match_template` / `match_template_all` capture every monitor and return + screen coordinates. A hit found inside a `region` previously came back in + region-local coordinates; it is now offset by the region's origin. Matches + against a caller-supplied `haystack` are unchanged (image-local). +- `match_template` / `match_template_all` refuse an almost-single-colour + template instead of returning an arbitrary position. +- `element_matches` accepts a friendly role name (`"button"`) as well as the + raw `"ControlType_50000"` the Windows backend reports. +- `AccessibilityBackend.list_elements` takes `window_title`; in-tree backends + accept it, and the facade only forwards it when set, so an out-of-tree + backend keeps working until someone asks for scoping. +- `AccessibilityElement.to_dict()` gains an `enabled` key. +- The Windows recorder captures through one low-level hook + (`Win32InputHook`) instead of the two listeners. `record` / `stop_record` + keep their behaviour and return shape. +- An unscoped `list_accessibility_elements` walks one top-level window at a + time in z-order, node by node, and stops at `max_results`, instead of one + uninterruptible `FindAll` over the whole desktop. Results are therefore + ordered front-most window first, and a small `max_results` no longer + reaches windows further back. +- The UIAutomation object is created from `CUIAutomation8` as + `IUIAutomation2` with a bounded `ConnectionTimeout` where available, so an + application that never answers UIA can no longer stall a search for a + minute. Falls back to `CUIAutomation` / `IUIAutomation` otherwise. +- `find_accessibility_elements` / `AC_a11y_find_all` / `ac_a11y_find_all`: + `max_results` now caps the matches returned (default 50) and the new + `scan_limit` caps how many elements are examined (default 1500). Callers + that passed `max_results` expecting a scan bound should pass `scan_limit`. ### Deprecated @@ -28,6 +136,30 @@ only when documented here with a migration path. ### Fixed +- `write` failing a whole string on the first character outside the 192-entry + virtual-key table — on a US layout that includes `, . / : ? ! _ + @ %` and + every CJK character, so URLs and non-English text could not be typed at all. +- OCR locating text that the engine split across word boxes (`Save As`, + `另存新檔`), which previously reported "not found" for text plainly on screen. +- Template matching never finding a target on a second monitor, and returning + coordinates offset by the physical-vs-logical pixel difference on a mixed-DPI + desktop (measured ~116 px) and by the virtual-desktop origin. +- Template images failing to load from a path containing non-ASCII characters + (`cv2.imread` returns `None` there, which surfaced as "could not read image"). +- `list_windows` handing back `LP_c_long` pointer objects instead of integer + hwnds, so `int(hwnd)` raised `ValueError` and a listed window could not be + used in any follow-up Win32 call. The `EnumWindows` callback declared its + hwnd as `POINTER(c_int)`; it is now `HWND`, and every Win32 prototype in + `windows_window_manage` declares `argtypes`/`restype` so a 64-bit handle is + not truncated to 32 bits. This also un-breaks the `ac_list_windows` MCP tool, + whose handler called `int(hwnd)`. +- Accessibility listing truncating to `max_results` *before* filtering, so an + element past the cap could never be found however specific the filter. +- `control_get_value` returning a password field's value when a custom-drawn + control puts plaintext in ValuePattern instead of masking it. +- The recorder leaking one thread per session: its listener pumped + `GetMessage` once and `stop_record` never woke it, so the thread stayed + blocked forever. - macOS cursor position and omitted-coordinate clicks on Retina / HiDPI displays (pixel-vs-point display-height mismatch). - Remote-desktop relay hang on Linux + CPython 3.14 when one paired peer diff --git a/CLAUDE.md b/CLAUDE.md index 8bf015a4..23638098 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,236 +2,149 @@ ## Project Overview -AutoControl (`je_auto_control`) is a cross-platform Python GUI automation framework supporting Windows (Win32 API), macOS (pyobjc/Quartz), and Linux (X11). It provides mouse/keyboard control, image recognition, screen capture, action scripting, and report generation through a unified API. +AutoControl (`je_auto_control`) is a cross-platform GUI automation framework: mouse and keyboard control, image recognition, OCR, accessibility-tree and VLM element location, action scripting, and report generation behind one API. Backends: Windows (Win32 ctypes), macOS (pyobjc/Quartz), Linux X11 (python-Xlib), Linux Wayland (libei / ydotool), Android (adb), iOS (WebDriverAgent). -- **Package name**: `je_auto_control` -- **Python**: >= 3.10 -- **License**: MIT -- **Author**: JE-Chen +- **Package**: `je_auto_control` · **Python** ≥ 3.10 · **License**: MIT · **Author**: JE-Chen +- **[architecture_explore.md](architecture_explore.md)** is the per-module map — read it before changing structure; it lists all 308 `utils/` subpackages, every GUI tab, and file-level tables for the large subsystems. -## Architecture & Design Patterns +## Architecture -### Strategy Pattern — Platform Abstraction +| Pattern | Where | Contract | +| --- | --- | --- | +| Strategy | `wrapper/platform_wrapper.py` | Detects the OS and imports exactly one backend. New platform = new backend package, no wrapper change. | +| Facade | `je_auto_control/__init__.py` | Re-exports every public name. `api/core.py` is the small versioned façade for new integrations. | +| Command | `utils/executor/action_executor.py` | `event_dict` maps `AC_*` names to callables; `flow_control.py` adds block commands (loop / branch / try / macro / variables). | +| Observer | `utils/callback/`, `utils/observer/`, `utils/triggers/` | Post-action callbacks; screen- and event-driven firing. | +| Template Method | `utils/generate_report/` | HTML / JSON / XML share collect → format → write. | +| Backend seam | `backends/` under `accessibility`, `ocr`, `vision`, `llm`, `agent`, `hotkey`, `usb`, `usbip` | Abstract base + concrete impls + null fallback, so dependency-free environments still import. | -`wrapper/platform_wrapper.py` auto-detects the OS and loads the correct backend. All wrapper modules (`auto_control_mouse.py`, `auto_control_keyboard.py`, etc.) delegate to the platform-specific implementation. New platform support is added by implementing the backend interface — no wrapper changes needed. +Layering: entry points (`cli.py`, `gui/`, socket / REST / MCP servers) → executor → `utils/` (308 headless subpackages) → `wrapper/` → per-OS backend. -### Facade Pattern — Unified API Surface - -`je_auto_control/__init__.py` re-exports all public functions from wrapper and utility modules, providing a single entry point. Users import only `je_auto_control` and access all features. - -### Command Pattern — JSON Action Executor - -`utils/executor/action_executor.py` maps string command names (e.g., `AC_click_mouse`) to callable functions. JSON action files define sequences of commands with parameters, enabling recording, serialization, and replay of automation flows. - -### Observer Pattern — Callback Executor +## Development Commands -`utils/callback/callback_function_executor.py` allows registering callback functions that fire after automation actions complete, supporting event-driven chaining. +```bash +pip install -r dev_requirements.txt # dev deps +pip install -e .[gui] # + GUI extra +python -m pytest test/unit_test/headless # headless unit tests +python -m pytest test/integrated_test/ # cross-module workflows +python -m build # build +``` -### Template Method — Report Generation +`pyproject.toml` pins `python_files = ["test_*.py"]` on purpose: the `*_test.py` files under `test/unit_test/` are manual demo scripts whose module bodies drive the real mouse and keyboard on import. Never loosen that setting. -`utils/generate_report/` provides HTML, JSON, and XML report generators sharing a common structure: collect test records, format output, write file. Each format implements its own rendering. +## Feature Delivery Rules -## Directory Structure +### Every feature ships both a headless API and a GUI surface -``` -je_auto_control/ -├── wrapper/ # Platform-agnostic API (Strategy consumers) -├── windows/ # Win32 backend (ctypes) -├── osx/ # macOS backend (pyobjc/Quartz) -├── linux_with_x11/ # Linux X11 backend (python-Xlib) -├── gui/ # PySide6 GUI application -└── utils/ - ├── executor/ # JSON action executor (Command pattern) - ├── callback/ # Callback executor (Observer pattern) - ├── cv2_utils/ # OpenCV: screenshot, template matching, video - ├── socket_server/ # TCP server for remote automation - ├── shell_process/ # Shell command manager - ├── generate_report/ # HTML/JSON/XML report generators - ├── test_record/ # Test action recording - ├── json/ # JSON action file I/O - ├── project/ # Project scaffolding - ├── package_manager/ # Dynamic package loading - ├── logging/ # Logging - └── exception/ # Custom exceptions -``` +No feature is complete unless it can be driven entirely without the GUI **and** has a GUI affordance: -## Development Commands +- **Headless core in `utils/` or `wrapper/`** — all business logic in a module with zero `PySide6` imports. +- **Re-export from the facade** — add public names to `je_auto_control/__init__.py` and its `__all__`. +- **Executor command** — wire an `AC_*` command into `utils/executor/action_executor.py`, so the feature works from JSON action files, the socket server, the scheduler, and the script builder without Python glue. +- **GUI tab is a thin wrapper** — the Qt widget only translates user input into calls on the headless core; no business logic that would be unreachable headlessly. +- **Tab commands live in the Actions menu, not in-tab buttons** — a tab keeps only inputs, tables, and result views. Core tabs declare `(label_key, handler)` pairs at registration in `gui/main_widget.py`; feature tabs expose `menu_actions()` returning the same shape. Script Builder and Remote Desktop are exempt (interactive panels). `test/unit_test/headless/test_actions_menu_gui.py` fails CI for a tab without either hook. +- **The top-level package stays Qt-free** — `import je_auto_control` MUST NOT import `PySide6`; the GUI loads lazily inside `start_autocontrol_gui()`. Verify: `import sys, je_auto_control; assert not any("PySide6" in m for m in sys.modules)`. +- **Tests cover the headless path** — at least one test in `test/unit_test/` exercising the non-GUI API with no Qt imports. -```bash -# Install dependencies -pip install -r dev_requirements.txt +Inherently interactive features (region picking, template cropping) may stay GUI-only, but must accept programmatic equivalents (e.g. `screenshot(screen_region=[...])`) so scripts replay the same effect headlessly. -# Install with GUI support -pip install -e .[gui] +### `architecture_explore.md` is updated with every change -# Run unit tests -python -m pytest test/unit_test/ +The map is only useful while it matches the tree, so **update it in the same change that moves the code**. Required when you add / remove / rename / move a module or subpackage, change what a module is responsible for (the map quotes the docstring's first line — update both), or add or remove an `AC_*` command, GUI tab, platform backend, entry point, extension point, server surface, or `__all__` name. -# Run integration tests -python -m pytest test/integrated_test/ +- **Measure, never estimate** — every count in the document is measured. Re-run rather than adjust by hand: -# Build package -python -m build -``` + ```bash + python -c "from je_auto_control.utils.executor.action_executor import executor as e; print(len(e.known_commands()))" # AC_* commands + python -c "import je_auto_control as ac; print(len(ac.__all__))" # public API + python -c "from je_auto_control.utils.mcp_server.tools import build_default_tool_registry as b; print(len(b()))" # MCP tools + ``` -## Feature Delivery Rules + Module and line counts come from walking the tree with `ast`; recompute §1, the affected §5.4 theme totals, and the §8 size appendix together so they stay consistent. -### Every feature must ship both a headless API and a GUI surface +- A new `utils/` subpackage needs a row in **exactly one** §5.4 theme table — the tables partition all 308 subpackages; appearing twice or not at all is a defect. +- A new subsystem over ~1,000 lines also needs a file-level table in §5.4.17. +- Keep the header's scan date, version, and branch current. +- `README.md` and both translations under `README/` cite the same figures (command / subpackage / tab / MCP-tool / example counts) — update all three alongside the map. +- **`test/unit_test/headless/test_doc_counts.py` enforces this and fails CI on a mismatch.** It re-measures the command, MCP-tool, `utils/` subpackage and `examples/` counts and compares them against every place the four documents quote them, so code and docs have to move in the same commit. If you reword a sentence that holds one of those numbers, update the test's pattern — it fails loudly when a citation disappears rather than passing on a document it can no longer read. The GUI tab count is guarded the same way but from `test_actions_menu_gui.py`, whose subprocess probe already builds the widget that count needs. -No feature is complete unless it can be driven entirely without the GUI **and** has a corresponding GUI affordance. Concretely: +### Outstanding work goes in `Progress.md` -- **Headless core in `utils/` or `wrapper/`**: all business logic lives in a module with zero `PySide6` imports. Users must be able to `import je_auto_control` and call the feature without ever instantiating a Qt class. -- **Re-export from the package facade**: add the public functions / classes to `je_auto_control/__init__.py` and its `__all__` so `import je_auto_control as ac; ac.(...)` works out of the box. -- **Executor command coverage**: wire an `AC_*` command into `utils/executor/action_executor.py` so the feature is usable from JSON action files, the socket server, the scheduler, and the visual script builder — all without Python glue. -- **GUI tab or control is a thin wrapper**: the Qt widget must only translate user input into calls on the headless core. It must not contain business logic that would be unreachable headlessly. -- **Tab commands live in the Actions menu, not in-tab buttons**: the main window is menu-driven. A tab keeps only its inputs, tables, and result/status views; its commands surface through the window-level **Actions** menu. Core tabs declare `(label_key, handler)` pairs at registration in `gui/main_widget.py`; feature tabs expose a `menu_actions()` method returning the same shape. Script Builder and Remote Desktop are the only exempt tabs (interactive panel layouts). `test/unit_test/headless/test_actions_menu_gui.py` guards this contract — a new tab without registry actions or a `menu_actions()` hook fails CI. -- **The top-level package stays Qt-free**: `import je_auto_control` MUST NOT import `PySide6`. The GUI entry point is loaded lazily inside `start_autocontrol_gui()`. Verify with: +Anything agreed but not done — deferred follow-ups, known gaps, half-delivered features, decisions waiting on the maintainer — is recorded in [Progress.md](Progress.md), not left in chat history or buried in a commit message. - ```python - import sys, je_auto_control # noqa - assert not any("PySide6" in m for m in sys.modules) - ``` +- **Write the entry when you defer the work**, in the same change that created the gap. Each entry states its status (`TODO` / `WIP` / `BLOCKED` / `DECIDE`), what is missing, and where in the tree. +- **Open items only.** Delete the entry when the work lands; shipped work is described in `WHATS_NEW.md` and compatibility changes in `CHANGELOG.md`. `Progress.md` is not a changelog. +- A feature that reaches only some of the delivery surfaces above belongs here until the rest land. -- **Tests cover the headless path**: at least one unit test in `test/unit_test/` must exercise the feature through its non-GUI API with no Qt imports. +## Coding Standards -Features that are inherently interactive (e.g. region picking with the mouse, template cropping) still count as GUI-only — but they must accept programmatic equivalents (e.g. `screenshot(screen_region=[...])` with explicit coordinates) so scripts can replay the same effect headlessly. +### Project-specific rules -## Coding Standards +- **Exception hierarchy is flat by design** — every framework error derives from `AutoControlException` so containment boundaries (executor, background poll loops, request handlers, GUI slots) can catch the family in one `except`. Never add a sibling inheriting `Exception` directly; it silently escapes every boundary. Assertion failures (`AutoControlAssertionException`) must keep propagating through `raise_on_error=False`. +- **Fail fast** — raise the specific typed exception at the point of failure; do not swallow errors. +- **Validate at boundaries** — user input, file content, network data, and JSON action commands. Reject unknown command names; `realpath` and bound user-supplied paths. +- **Least privilege** — servers bind `127.0.0.1` by default; `0.0.0.0` needs an explicit, documented opt-in. +- **No `print()` in library code** (`je_auto_control/` outside `gui/` stdout tooling) — use `autocontrol_logger`. +- **No `assert` for runtime checks** outside tests — it is stripped under `-O`. +- **Lazy imports for optional and platform-specific dependencies** — never import all backends unconditionally. +- **Release platform resources** (GDI handles, Quartz event sources, X display, OpenCV writers) in `finally` / `__exit__`; use `with` everywhere it applies. +- **Thread safety** — state shared between the socket server, recording threads, and the callback executor is guarded by `threading.Lock` / `queue.Queue`. +- **Reuse screen captures** when running several searches against the same frame; avoid per-event allocations in mouse/keyboard dispatch. +- **Action lists and loaded config are read-only** once loaded. +- **Pin dependency versions**, including transitive ones that can change return shapes (`opencv-python` is bounded `<6` for exactly this reason). Review new dependencies for known vulnerabilities. +- Common logic belongs in `wrapper/` or `utils/`, never duplicated across platform backends. -### Security First +### Limits enforced by CI -- **Input validation**: Validate all external inputs (user input, file content, network data, JSON action commands) at system boundaries. Sanitize file paths to prevent path traversal. Never trust data from TCP socket clients without validation. -- **Injection prevention**: When executing shell commands (`shell_process`), never construct command strings from unsanitized input. Use parameterized approaches or allowlists. -- **Deserialization safety**: JSON action files and socket server payloads must be validated against expected schemas before execution. Reject unknown command names. -- **No secrets in code**: Never commit credentials, API keys, tokens, or `.env` files. Keep secrets out of logs and reports. -- **Principle of least privilege**: Socket server should bind to localhost by default. Document security implications of exposing to network. -- **Dependency awareness**: Pin dependency versions. Review transitive dependencies for known vulnerabilities. +Cyclomatic complexity ≤ 10 · cognitive complexity ≤ 15 · function ≤ 75 lines · parameters ≤ 7 · nesting ≤ 4 · file ≤ 750 lines · line ≤ 120 chars · no duplicated block ≥ 10 lines. -### Performance Best Practices +Docstrings on every public module, class, and function (one-line summary minimum; type hints replace parameter-type prose). Type hints on all public signatures. Import order stdlib → third-party → first-party; no wildcard imports outside the `__init__.py` façade. -- **Lazy imports**: Platform-specific backends are loaded only for the current OS — do not import all backends unconditionally. -- **Avoid redundant screenshots**: Image recognition operations should reuse screen captures when performing multiple searches on the same frame. -- **Buffer management**: Screen recording and video capture must properly release resources (file handles, codec buffers) in `finally` blocks or context managers. -- **Thread safety**: Socket server and recording threads must use proper synchronization. Avoid shared mutable state without locks. -- **Minimize allocations in hot paths**: Mouse/keyboard event dispatch should avoid unnecessary object creation per event. +### Automated verification -### Software Engineering Principles +Run before every commit and fix all new findings: -- **SOLID**: Each module has a single responsibility. Platform backends are open for extension (new OS) without modifying wrappers. Depend on abstractions (wrapper API), not concrete implementations (Win32/X11/Quartz). -- **DRY**: Common logic belongs in `wrapper/` or `utils/`, not duplicated across platform backends. -- **YAGNI**: Do not add speculative features. Implement what is needed now. -- **Fail fast**: Raise clear, specific exceptions (`AutoControlMouseException`, `AutoControlKeyboardException`, etc.) at the point of failure. Do not silently swallow errors. -- **Immutable data where possible**: Action lists and configuration should be treated as read-only once loaded. +```bash +pip install ruff pylint bandit radon +ruff check je_auto_control/ +pylint je_auto_control/ +bandit -c pyproject.toml -r je_auto_control/ # uses [tool.bandit] excludes/skips +radon cc je_auto_control/ -a -nc # flags functions with CC >= C (>10) +``` -### Code Style +These tools own the generic rules (bare `except`, mutable defaults, unused names, dead code, `eval`/`exec`, `shell=True`, `pickle` on untrusted data, weak hashes, missing `timeout=`, hardcoded secrets, naming). Fix what they report rather than restating them here. -- Follow PEP 8. -- Use type hints for all public function signatures. -- Keep functions focused and short — one function, one task. -- Prefer composition over inheritance for extending functionality. -- Remove dead code immediately — no commented-out blocks, no unused imports, no unreachable branches. +Suppressions need an inline justification — `# noqa: # reason: ` or `# nosec B404 # reason: `. Blanket file- or module-level suppressions are forbidden. ## Commit Conventions -- Write concise commit messages focused on **why**, not what. +- Concise messages focused on **why**, not what. Imperative mood: `Add image threshold parameter validation`, `Fix mouse scroll direction on macOS`, `Remove deprecated screen capture fallback`. - **No AI attribution anywhere.** Not in commit messages, pull request titles or bodies, issue and review comments, release notes, code comments, or documentation. Specifically forbidden: `Co-Authored-By:` lines naming an assistant, "Generated with …" / "Created by …" footers, tool badges, links to assistant products, emoji markers standing in for the same thing, and any statement that code was AI-generated. Everything published under this repo reads as the author's own work. -- Use imperative mood: "Add feature", "Fix bug", "Remove unused code". -- Examples: - - `Add image threshold parameter validation` - - `Fix mouse scroll direction on macOS` - - `Remove deprecated screen capture fallback` ## Testing -- **Unit tests**: `test/unit_test/` — test individual functions in isolation. -- **Integration tests**: `test/integrated_test/` — test cross-module workflows. -- **Manual tests**: `test/manual_test/` — require human verification (GUI, visual). -- **GUI tests**: `test/gui_test/` — PySide6 interface tests. -- All tests must pass before merging. Ensure cross-platform compatibility. +- `test/unit_test/headless/` — headless unit tests, the CI gate. `test/unit_test/flow_control/` — executor flow control. +- `test/integrated_test/` — cross-module workflows. `test/gui_test/` — PySide6 interface. `test/manual_test/` — human verification. +- No `time.sleep` > 1s in unit tests; use fakes or event signals. Tests must not depend on execution order. +- All tests pass before merging; keep cross-platform compatibility. +- **Queued Qt `deleteLater()` work is flushed after every test** by the autouse + fixture in `test/unit_test/headless/conftest.py`. Do not remove it, and keep + any new Qt test directory covered the same way. `deleteLater()` is a no-op + until an event loop runs, and most GUI test modules never run one — so the + widget, plus any helper thread or timer it started at construction, survives + until some *later* test pumps events and is destroyed inside that unrelated + test. This is not theoretical: `test_admin_console_thumbnails_gui.py` leaked + seven `AdminConsoleTab`s this way, and they detonated inside the nested modal + `exec()` of `test_usb_acl_prompt.py`, killing the interpreter with rc + 3221226505 (0xC0000409) — a `__fastfail`, so no traceback, no faulthandler + output, and nothing after it in the suite ran. Note the failure is invisible + to CI: `test_usb_acl_prompt.py` needs the optional `webrtc` extra (`av`, + `aiortc`), which CI does not install, so CI skips it and only developers with + that extra installed see the crash. ## Key Conventions -- All public API functions are exported from `je_auto_control/__init__.py` and listed in `__all__`. -- JSON action command names use `AC_` prefix (e.g., `AC_click_mouse`). -- Platform backends follow naming: `{platform}_{function}.py` (e.g., `win32_ctype_mouse_control.py`). -- Virtual key mappings are in `core/utils/*_vk.py` per platform. - -## Static Analysis Compliance (SonarQube / Codacy / Pylint / Bandit) - -All code must satisfy the following rules so automated scanners (SonarQube, Codacy, Pylint, Bandit, Radon, Prospector) report zero new issues. - -### Complexity & Size Limits - -- **Cyclomatic complexity** per function ≤ 10. Refactor with early returns, extracted helpers, or lookup dicts when exceeded. -- **Cognitive complexity** per function ≤ 15 (SonarQube `python:S3776`). -- **Function length** ≤ 75 lines. Long procedural flows must be split into named helpers. -- **Parameter count** ≤ 7 (`python:S107`). Group related parameters into a dataclass or config object when exceeded. -- **Nesting depth** ≤ 4 (`python:S134`). Flatten with guard clauses. -- **File length** ≤ 750 lines. Split large modules along responsibility lines. -- **Identical branches**: `if`/`elif`/`else` branches must not have identical bodies (`python:S3923`). -- **Duplicated code**: no duplicated blocks ≥ 10 lines across the project (Sonar default). Extract to a shared helper. - -### Bug & Correctness Rules - -- **No bare `except:`** — always catch a specific exception type (`python:S5754`, Bandit `B001`). -- **No empty `except` blocks** (`python:S2737`). At minimum log the error or re-raise. -- **Preserve exception chain**: inside `except`, raising a new exception must use `raise NewError(...) from exc` (`python:S5655`). -- **No mutable default arguments** (`python:S5727`): never use `def f(x=[])` or `{}`. Use `None` + lazy init. -- **No unused imports, variables, parameters, or assignments** (`python:S1481`, `S1854`, `S1172`). Remove them; do not rename to `_unused`. -- **No dead / unreachable code** (`python:S1763`). -- **No commented-out code blocks** (`python:S125`) — delete instead; git history preserves it. -- **No `TODO`/`FIXME`/`XXX`** without an issue-tracker reference in the same comment (`python:S1135`). -- **No `print()`** in library code (`je_auto_control/` outside `gui/` stdout tooling). Use the project logger. -- **No `assert` for runtime checks** in non-test code (Bandit `B101`) — `assert` is stripped with `-O`. Raise explicit exceptions. -- **String formatting**: prefer f-strings over `%` or `.format()` for readability; never interpolate untrusted data into shell/SQL. -- **Equality with `None`/`True`/`False`**: use `is` / `is not`, never `==` (`python:S2589`). -- **Boolean simplification**: no `if cond: return True else: return False` — return the expression directly (`python:S1126`). -- **Identical expressions on both sides** of `and`/`or`/`==`/`!=` are forbidden (`python:S1764`). - -### Security Rules (Bandit / Sonar Security Hotspots) - -- **No `eval`, `exec`, `compile`** on any runtime-sourced string (Bandit `B307`, `B102`). -- **No `pickle`, `marshal`, `shelve`, `dill`** on data from disk, network, or user input (Bandit `B301`, `B302`). Use JSON with schema validation. -- **No `subprocess` with `shell=True`** or string-built command lines (Bandit `B602`, `B605`). Pass argv lists and validate against allowlists. -- **No `os.system`, `os.popen`, `commands.*`** (Bandit `B605`, `B607`). -- **No insecure hash** (`md5`, `sha1`) for security purposes (Bandit `B303`, `B324`). Use `hashlib.sha256` or better. -- **No `tempfile.mktemp`** — use `NamedTemporaryFile` / `mkstemp` (Bandit `B306`). -- **No hardcoded passwords, tokens, or secrets** (Bandit `B105`–`B107`, `python:S2068`). -- **No `yaml.load`** without `SafeLoader` (Bandit `B506`). -- **`requests`/`urllib` calls** must set explicit `timeout=` (`python:S5332`, Bandit `B113`). -- **No `ssl._create_unverified_context` / `verify=False`** (Bandit `B501`). -- **Path traversal**: validate and `os.path.realpath` user-supplied paths before I/O. -- **Socket binds** must default to `127.0.0.1`; `0.0.0.0` requires an explicit, documented opt-in. - -### Resource & Concurrency - -- **Always use `with`** for files, sockets, locks, and OpenCV `VideoCapture`/`VideoWriter` (`python:S5720`). No manual `close()` in normal flow. -- **Release platform resources** (GDI handles, Quartz event sources, X display) in `finally` or `__exit__`. -- **Thread-safety**: shared mutable state between the socket server, recording thread, and callback executor must be guarded by `threading.Lock` / `queue.Queue`. - -### Style & Naming - -- **snake_case** for functions, methods, variables, modules; **PascalCase** for classes; **UPPER_SNAKE_CASE** for module-level constants (`python:S117`, Pylint `C0103`). -- **Max line length**: 120 chars (`python:S103`). -- **Docstrings** on every public module, class, and function (`python:S1720`, Pylint `C0114`–`C0116`) — one-line summary minimum; type hints replace parameter-type prose. -- **Import order**: stdlib → third-party → first-party, separated by blank lines; no wildcard imports except in `__init__.py` façade (`python:S2208`). -- **No `global`** statements outside module initialization (`python:S2208`). - -### Test Hygiene - -- Tests must avoid `assert` against object identity of mutable literals and must not depend on execution order (Pylint / Sonar `python:S5914`). -- No `time.sleep` > 1s in unit tests; use fakes / event signals. - -### Automated Verification - -Run before every commit; fix all new findings: - -```bash -pip install ruff pylint bandit radon -ruff check je_auto_control/ -pylint je_auto_control/ -bandit -c pyproject.toml -r je_auto_control/ # uses [tool.bandit] excludes/skips -radon cc je_auto_control/ -a -nc # flags functions with CC >= C (>10) -``` - -If a rule must be suppressed, add an inline justification: `# noqa: # reason: ` or `# nosec B404 # reason: `. Blanket suppressions at file/module level are forbidden. +- Public API is exported from `je_auto_control/__init__.py` and listed in `__all__`. +- JSON action command names use the `AC_` prefix (e.g. `AC_click_mouse`); MCP tools use `ac_`. +- Platform backends are named `{platform}_{function}.py` (e.g. `win32_ctype_mouse_control.py`). +- Virtual key mappings live in `core/utils/*_vk.py` per platform. diff --git a/Progress.md b/Progress.md new file mode 100644 index 00000000..7088a115 --- /dev/null +++ b/Progress.md @@ -0,0 +1,29 @@ +# Progress + +**只記未完成的事。** 已出貨的內容寫進 [WHATS_NEW.md](WHATS_NEW.md)，相容性變更寫進 +[CHANGELOG.md](CHANGELOG.md)；完成的項目從本檔移除，不累積歷史。 + +狀態標記： + +| 標記 | 意思 | +| --- | --- | +| `TODO` | 已決定要做，尚未開始 | +| `WIP` | 進行中，工作樹已有部分成果 | +| `BLOCKED` | 卡在外部條件（硬體、第三方、上游套件） | +| `DECIDE` | 需要維護者拍板才能往下走 | + +--- + +## [TODO] `windows_backend.py` 915 行，超過 750 行上限 + +`je_auto_control/utils/accessibility/backends/windows_backend.py` 目前 915 行，超出 +`CLAUDE.md` §Limits 的 750 行。拆出 `windows_query.py`（170）與 `windows_state.py`（98）之後 +仍然超標——這個檔在拆之前就已經是 772 行，後續補視窗限定搜尋與控制項模式又長回來。 + +- **注意**：目前**沒有任何 CI job 在檢查行數與複雜度**（`quality.yml` 只有 ruff 與 bandit）， + 所以這條上限實際上靠自律；`action_executor.py` 8,021 行、`_factories.py` 8,866 行同樣超標。 +- **待決**：是要真的拆這個檔、把上限改成符合現況的數字，還是把這條規則的適用範圍寫清楚。 + +--- + +（目前沒有其他待辦。） diff --git a/README.md b/README.md index 1df11e07..676e2b75 100644 --- a/README.md +++ b/README.md @@ -5,599 +5,212 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) [![Documentation](https://readthedocs.org/projects/autocontrol/badge/?version=latest)](https://autocontrol.readthedocs.io/en/latest/?badge=latest) -**AutoControl** is a cross-platform Python GUI automation framework providing mouse control, keyboard input, image recognition, screen capture, action scripting, and report generation — all through a unified API that works on Windows, macOS, and Linux (X11). +**AutoControl** is a cross-platform GUI automation framework for Python. It drives the +mouse and keyboard, finds things on screen (template matching, OCR, the OS accessibility +tree, or a vision model), records and replays flows, and runs them from JSON action +files — on Windows, macOS, Linux (X11 and Wayland), Android, and iOS. -**[繁體中文](README/README_zh-TW.md)** | **[简体中文](README/README_zh-CN.md)** +Every capability ships three ways: a **Python API**, an **`AC_*` action command** usable +from JSON files / CLI / servers, and a **GUI tab**. Nothing is GUI-only. ---- - -## Table of Contents - -- [What's New](#whats-new) -- [Features](#features) -- [Architecture](#architecture) -- [Installation](#installation) -- [Requirements](#requirements) -- [Quick Start](#quick-start) - - [Mouse Control](#mouse-control) - - [Keyboard Control](#keyboard-control) - - [Image Recognition](#image-recognition) - - [Accessibility Element Finder](#accessibility-element-finder) - - [AI Element Locator (VLM)](#ai-element-locator-vlm) - - [OCR (Text on Screen)](#ocr-text-on-screen) - - [LLM Action Planner](#llm-action-planner) - - [Runtime Variables & Control Flow](#runtime-variables--control-flow) - - [Remote Desktop](#remote-desktop) - - [Clipboard](#clipboard) - - [Screenshot](#screenshot) - - [Action Recording & Playback](#action-recording--playback) - - [JSON Action Scripting](#json-action-scripting) - - [MCP Server (Use AutoControl from Claude)](#mcp-server-use-autocontrol-from-claude) - - [Scheduler (Interval & Cron)](#scheduler-interval--cron) - - [Global Hotkey Daemon](#global-hotkey-daemon) - - [Event Triggers](#event-triggers) - - [Run History](#run-history) - - [Report Generation](#report-generation) - - [Observability (Prometheus / OpenTelemetry)](#observability-prometheus--opentelemetry) - - [Remote Automation (Socket / REST)](#remote-automation-socket--rest) - - [Plugin Loader](#plugin-loader) - - [Shell Command Execution](#shell-command-execution) - - [Screen Recording](#screen-recording) - - [Callback Executor](#callback-executor) - - [Package Manager](#package-manager) - - [Project Management](#project-management) - - [Window Management](#window-management) - - [GUI Application](#gui-application) -- [Command-Line Interface](#command-line-interface) -- [Platform Support](#platform-support) -- [Development](#development) -- [License](#license) +**[繁體中文](README/README_zh-TW.md)** · **[简体中文](README/README_zh-CN.md)** --- -## What's New - -**Latest (2026-07-18) — cross-platform reliability hardening.** A full-project runtime audit fixed execution-time defects across the macOS / Windows / Linux / Wayland backends, the executor, and the remote-desktop / USB stacks — correct Retina cursor math, a relay hang on CPython 3.14, `AC_expect_poll` / `AC_parallel` robustness, localhost-by-default USB/IP, and typed exceptions preserved at I/O boundaries — each covered by a headless regression test. No API changes. - -All per-release notes are in **[WHATS_NEW.md](WHATS_NEW.md)**. - -## Features - -- **QA / Test Framework** — assertion DSL (`assert_text` / `_image` / `_pixel` / `_window` + audio/video assertions), data-driven execution (CSV / JSON / SQLite / Excel → `AC_for_each_row`), a scored `run_suite` with setup/teardown/tags, JUnit + Allure report output, flaky-test detection with auto-quarantine, accessibility / i18n auditing (missing labels, WCAG contrast, truncation), and a parallel mobile device matrix. See [What's new (2026-06)](WHATS_NEW.md) -- **Automation toolkit** — human-like mouse motion + typing, VLM / variable / duration assertions, reusable macros + in-process parallel blocks, composite + cron triggers, read-into-a-variable commands (OCR / shell / file / HTTP / time / random), variable transforms, scroll-to-find, region colour stats, QR reading, per-window capture / layout save-restore / snap, screenshot annotation, desktop notifications, action-file signing + encryption, recoverable (recycle-bin) deletion, and Recording-Editor undo. See [What's new (2026-06-17)](WHATS_NEW.md) -- **Mouse Automation** — move, click, press, release, drag, and scroll with precise coordinate control -- **Keyboard Automation** — press/release individual keys, type strings, hotkey combinations, key state detection -- **Image Recognition** — locate UI elements on screen using OpenCV template matching with configurable threshold -- **Accessibility Element Finder** — query the OS accessibility tree (Windows UIA / macOS AX) to locate buttons, menus, and controls by name/role -- **AI Element Locator (VLM)** — describe a UI element in plain language and let a vision-language model (Anthropic / OpenAI) find its screen coordinates -- **OCR** — extract text from screen regions through three pluggable backends (Tesseract for ASCII, EasyOCR for CJK without an external binary, PaddleOCR for highest-quality Chinese / Japanese / Korean). Single unified API + canonical language codes; backend chosen by `backend=` kwarg, `AUTOCONTROL_OCR_BACKEND` env var, or auto-detection. Wait for, click, or locate rendered text; regex search and full-region dump -- **LLM Action Planner** — translate a plain-language description into a validated `AC_*` action list using Claude -- **Runtime Variables & Control Flow** — `${var}` substitution at execution time, plus `AC_set_var` / `AC_inc_var` / `AC_if_var` / `AC_for_each` / `AC_loop` / `AC_while_var` / `AC_retry` / `AC_try` for data-driven scripts. `AC_while_var` loops while a variable comparison holds (re-checked each iteration, `max_iter` safety cap). `AC_try` adds try/catch/finally: when `body` fails it runs the `catch` recovery branch instead of aborting, always runs `finally`, exposes the error to `error_var`, and can `reraise` after cleanup (loop `break`/`continue` still propagate through it) -- **Remote Desktop** — stream this machine's screen and accept remote input over a token-authenticated TCP protocol, *or* connect to another machine and view + control it (host + viewer GUIs included). Optional TLS (HTTPS-grade encryption), WebSocket transport (ws:// + wss:// for browser / firewall-friendly clients), persistent 9-digit Host ID, host→viewer audio streaming, bidirectional clipboard sync (text + image), and chunked file transfer (drag-drop + progress bar; arbitrary destination path; no size cap). Plus folder sync (additive mirror — local deletions never propagate) and a self-hosted coturn TURN config bundle generator (turnserver.conf + systemd unit + docker-compose + README). **AnyDesk-style popout**: when the viewer authenticates, the live remote desktop opens in its own resizable top-level window so the control panel stays uncluttered. The Remote Desktop tabs are wrapped in `QScrollArea` so the panel stays usable on small windows and stretches edge-to-edge on 4K displays. Driveable headlessly via `je_auto_control` and over MCP through the new `ac_remote_*` tools -- **Driver-level input backends (opt-in)** — for games / apps that ignore SendInput (Win) or XTest (Linux): **Interception driver backend** for Windows (HID-layer keyboard / mouse injection via Oblita's WHQL-signed driver, opt-in via `JE_AUTOCONTROL_WIN32_BACKEND=interception`), **uinput backend** for Linux (kernel `/dev/uinput` synthetic HID device, opt-in via `JE_AUTOCONTROL_LINUX_BACKEND=uinput`), and **ViGEm virtual gamepad** for Windows games that read controllers (virtual Xbox 360 pad with friendly button / dpad / stick / trigger API, exposed as `AC_gamepad_*` executor commands and `ac_gamepad_*` MCP tools). All three fall back gracefully when the driver isn't installed, so existing deployments keep working unchanged -- **Clipboard** — read/write system clipboard text on Windows, macOS, and Linux -- **Screenshot & Screen Recording** — capture full screen or regions as images, record screen to video (AVI/MP4) -- **Action Recording & Playback** — record mouse/keyboard events and replay them -- **JSON-Based Action Scripting** — define and execute automation flows using JSON action files (dry-run + step debug) -- **Scheduler** — run scripts on an interval or cron expression; jobs persist across restarts -- **Global Hotkey Daemon** — bind OS-level hotkeys to action scripts on all three desktops: Windows (`RegisterHotKey`), macOS (`CGEventTap`, needs Accessibility permission), and Linux X11 (`XGrabKey` with NumLock / CapsLock variant masking). Wayland hotkeys are still compositor-dependent (each session bus exposes a different shortcut portal); a Wayland session can still drive AutoControl via the new Wayland input backend (see [What's new (2026-05)](WHATS_NEW.md)). Same `bind()` / `start()` API across platforms; the Strategy-pattern dispatch in `backends/` auto-picks the right backend at start time -- **Event Triggers** — fire scripts when an image appears, a window opens, a pixel changes, or a file is modified -- **Run History** — SQLite-backed run log across scheduler / triggers / hotkeys / REST with auto error-screenshot artifacts -- **Report Generation** — export test records as HTML, JSON, or XML reports with success/failure status -- **MCP Server** — JSON-RPC 2.0 Model Context Protocol server (stdio + HTTP/SSE) so Claude Desktop / Claude Code / custom tool-use loops can drive AutoControl. ~100 tools, full protocol coverage (resources, prompts, sampling, roots, logging, progress, cancellation, elicitation), bearer-token auth + TLS, audit log, rate limit, plugin hot-reload, CI fake backend. New in this release: `ac_remote_host_start` / `ac_remote_host_stop` / `ac_remote_host_status` / `ac_remote_viewer_connect` / `ac_remote_viewer_disconnect` / `ac_remote_viewer_status` / `ac_remote_viewer_send_input` wrap the same singleton remote-desktop registry the GUI uses, so a model can spin up a host, open a viewer to another machine, and forward mouse / keyboard / type / hotkey actions through the active session -- **Remote Automation** — TCP socket server **and** hardened REST API: bearer-token auth, per-IP rate limit + lockout, SQLite audit hook, Prometheus `/metrics`, OpenAPI-style endpoint table (`/health`, `/screen_size`, `/sessions`, `/screenshot`, `/execute`, `/audit/list`, `/audit/verify`, `/inspector/recent`, `/usb/devices`, `/diagnose`, ...), and a vanilla-JS browser dashboard at `/dashboard` (any phone with HTTP reach can monitor the host) -- **Plugin Loader** — drop `.py` files exposing `AC_*` callables into a directory and register them as executor commands at runtime -- **Shell Integration** — execute shell commands within automation workflows with async output capture -- **Callback Executor** — trigger automation functions with callback hooks for chaining operations -- **Dynamic Package Loading** — extend the executor at runtime by importing external Python packages -- **Project & Template Management** — scaffold automation projects with keyword/executor directory structure -- **Window Management** — send keyboard/mouse events directly to specific windows (Windows/Linux) -- **GUI Application** — built-in PySide6 graphical interface with live language switching (English / 繁體中文 / 简体中文 / 日本語) -- **CLI Runner** — `python -m je_auto_control.cli run|list-jobs|start-server|start-rest` -- **Cross-Platform** — unified API across Windows, macOS, Linux (X11 + Wayland), Android (adb + uiautomator2), and iOS (WebDriverAgent / facebook-wda) -- **Screenshot PII redaction** — `RedactionEngine` blurs emails / credit cards / SSNs / phones / secure-text fields / forced regions before screenshots leave the host (VLM upload, audit log, REST). Policy via env var `JE_AUTOCONTROL_REDACTION=off|moderate|strict` or per-call -- **Multi-Host Admin Console** — register N AutoControl REST endpoints in one address book, poll them in parallel for health/sessions/jobs, broadcast actions to all of them. Persisted to `~/.je_auto_control/admin_hosts.json` (mode 0600 on POSIX). Bad-token hosts surface as unhealthy with the actual HTTP error -- **Tamper-Evident Audit Log** — SQLite events table with SHA-256 hash chain (`prev_hash` + `row_hash` per row); editing any past row breaks the chain. `verify_chain()` walks rows top-down and reports the first broken link. Legacy tables get backfilled at startup ("trust on first use") -- **WebRTC Packet Inspector** — process-global rolling window of `StatsSnapshot` samples (default 600 / ~10 min @ 1Hz) fed by the existing WebRTC stats pollers. Per-metric `last/min/max/avg/p95` for RTT, FPS, bitrate, packet loss, jitter -- **USB Device Enumeration** — read-only cross-platform device listing. Tries pyusb (libusb) first; falls back to platform-specific (Windows `Get-PnpDevice`, macOS `system_profiler`, Linux `/sys/bus/usb/devices`). Phase 2 passthrough builds on this (see below) -- **System Diagnostics** — single-command "is everything OK?" probe across platform, optional deps, executor command count, audit chain, screenshot, mouse, disk space, REST registry. CLI exits 0 if all green / 1 otherwise; REST `/diagnose`; severity-tagged GUI tab -- **Stable API & Failure Bundles** — versioned, lazy `je_auto_control.api` façade for new integrations (`execute_action`, `generate_code`, `run_diagnostics`, failure bundles) with a documented [lifecycle policy](docs/API_LIFECYCLE.md). Portable `autocontrol.failure-bundle/v1` diagnostic ZIPs: manifest + redacted context/events/log tail, optional screenshot and diagnostics, best-effort collectors, atomic write. CLI `je_auto_control failure-bundle out.zip`; `codegen --failure-bundle` wraps generated pytest in automatic failure diagnostics -- **USB Hotplug Events** — polling-based hotplug watcher (`UsbHotplugWatcher`) with bounded ring buffer + sequence-numbered events; `GET /usb/events?since=N` lets late subscribers catch up. GUI auto-refresh toggle on the USB tab. -- **OpenAPI 3.1 + Swagger UI** — `GET /openapi.json` (auth-gated, generated from the live route table) + `GET /docs` (browser Swagger UI with bearer token bar). Drift test in CI catches new routes added without metadata. -- **Configuration Bundle** — single-file JSON export/import of user config (admin hosts, address book, trusted viewers, known hosts, host service, IDs). Atomic write with `.bak.` backups; CLI `python -m je_auto_control.utils.config_bundle export|import`; `POST /config/{export,import}`; export/import commands on the REST API tab's Actions menu. -- **USB Passthrough (opt-in)** — let a remote viewer use a USB device physically attached to the host, over a WebRTC `usb` DataChannel. Wire-level protocol (11 opcodes incl. `RESUME`, CREDIT-based flow control, 16 KiB payload cap with EOF fragmentation for oversize transfers). All eight original open questions resolved: reliable-ordered channel, LIST-over-channel (ACL-filtered), per-claim credits, Linux kernel-driver detach/reattach, and ACL **HMAC-SHA256 integrity** (fail-closed on tamper; pluggable key — Windows DPAPI or passphrase vault). **Backends:** `LibusbBackend` (production), `WinusbBackend` (ctypes) and `IokitBackend` (native IOKit enumeration + libusb transfers) — Windows/macOS *hardware-unverified*; `default_passthrough_backend()` picks per-OS. Viewer-side blocking client (`control/bulk/interrupt_transfer`, `list_devices`, `resume`); in-process `UsbLoopback` so one machine can share + use a device through the full stack. **Wired into WebRTC** host/viewer (`viewer.usb_client()`) plus claim **resume tokens** that survive a reconnect. Persistent ACL (default deny, mode 0600) with host-side prompt dialog, abuse **rate-limit / lockout**, and tamper-evident audit integration. Five driving surfaces: AnyDesk-style **GUI panel** (share + ACL allow/block + local/remote use), `AC_usb_*` executor commands (JSON / socket / scheduler), **REST** `/usb/...`, first-class **MCP** `ac_usb_*` tools, and the Python API. Default off — opt-in via `enable_usb_passthrough(True)` or `JE_AUTOCONTROL_USB_PASSTHROUGH=1`; default-on still pending Phase 2e external security sign-off + real-hardware verification. -- **Observability (Prometheus + OpenTelemetry)** — stdlib-only `Counter` / `Gauge` / `Histogram` registry with a tiny built-in HTTP exporter on `/metrics`, plus an OpenTelemetry-compatible tracer that upgrades to real OTel spans when the SDK is installed. The executor and agent loop emit `autocontrol_action_calls_total{action,outcome}`, `autocontrol_action_duration_seconds`, and `autocontrol_agent_steps_total{tool,outcome}` automatically — drop the URL into a Prometheus scrape config and you have a Grafana dashboard with zero per-script wiring. - ---- - -## Architecture - -The runtime is layered: **client surfaces** (CLI, GUI, MCP/REST/socket -servers) sit on top of the **headless API** (`wrapper/` + `utils/`), -which resolves to a **per-OS backend** chosen at import time by -`wrapper/platform_wrapper.py`. The package façade -(`je_auto_control/__init__.py`) re-exports every public name so users -need only `import je_auto_control` regardless of which surface or -backend they hit. - -```mermaid -flowchart LR - subgraph Clients["Client Surfaces"] - direction TB - Claude[["Claude Desktop /
Claude Code"]] - APIUser[["Custom Anthropic /
OpenAI tool loops"]] - HTTPClient[["HTTP / SSE clients"]] - TCPClient[["Socket / REST clients"]] - Browser[["Browser
(/dashboard · /docs)"]] - GUIUser[["PySide6 GUI"]] - CLIUser[["python -m
je_auto_control[.cli]"]] - Library[["Library users
(import je_auto_control)"]] - end - - subgraph Transports["Transports & Servers"] - direction TB - Stdio["MCP stdio
JSON-RPC 2.0"] - HTTPMCP["MCP HTTP /
SSE + auth + TLS"] - REST["REST server :9939
bearer auth · rate-limit ·
OpenAPI · /metrics · /dashboard"] - Socket["Socket server
:9938"] - WebRTC["WebRTC sessions
(remote desktop ·
files · audio · USB)"] - end - - subgraph MCP["mcp_server/"] - direction TB - Dispatcher["MCPServer
(JSON-RPC dispatcher)"] - Tools["tools/
~90 ac_* + aliases"] - Resources["resources/
files · history ·
commands · screen-live"] - Prompts["prompts/
built-in templates"] - Context["context · audit ·
rate-limit · log-bridge"] - FakeBE["fake_backend
(CI smoke)"] - end - - subgraph Core["Headless Core (wrapper/ + utils/)"] - direction TB - Wrapper["wrapper/
mouse · keyboard · screen ·
image · record · window"] - Executor["executor/
AC_* JSON action engine"] - Vision["vision/ · ocr/ ·
accessibility/"] - Recorder["scheduler/ · triggers/ ·
hotkey/ · plugin_loader/
run_history/"] - IOUtils["clipboard/ · cv2_utils/ ·
shell_process/ · json/"] - end - - subgraph Ops["Operations Layer (utils/)"] - direction TB - Admin["admin/
multi-host poll +
broadcast"] - Audit["remote_desktop/
audit_log
(SHA-256 chain)"] - Inspector["remote_desktop/
webrtc_inspector"] - Diag["diagnostics/
self-test"] - ConfigB["config_bundle/
export/import"] - end - - subgraph USB["USB"] - direction TB - UsbEnum["usb/
list + hotplug events"] - UsbPass["usb/passthrough/
session · client · ACL(HMAC) ·
libusb · WinUSB · IOKit ·
loopback · webrtc channel · commands"] - end - - subgraph Remote["Remote Desktop (utils/remote_desktop/)"] - direction TB - RDHost["host · webrtc_host ·
signaling · multi_viewer"] - RDFiles["webrtc_files · file_sync ·
clipboard_sync · audio"] - RDTrust["trust_list · fingerprint ·
turn_config · lan_discovery"] - end - - subgraph Backends["Per-OS Backends"] - direction TB - Win["windows/
Win32 ctypes"] - Mac["osx/
pyobjc · Quartz"] - X11["linux_with_x11/
python-Xlib"] - end - - Claude --> Stdio - APIUser --> Stdio - HTTPClient --> HTTPMCP - TCPClient --> Socket - TCPClient --> REST - Browser --> REST - - Stdio --> Dispatcher - HTTPMCP --> Dispatcher - Dispatcher --> Tools - Dispatcher --> Resources - Dispatcher --> Prompts - Dispatcher -.- Context - Tools -.optional.-> FakeBE - - Tools --> Wrapper - Tools --> Executor - Tools --> Vision - Tools --> Recorder - Tools --> IOUtils - Resources --> Recorder - Resources --> Wrapper - - REST --> Executor - REST --> Ops - REST --> USB - Socket --> Executor - WebRTC --> Remote - WebRTC --> UsbPass - - GUIUser --> Wrapper - GUIUser --> Recorder - GUIUser --> Ops - GUIUser --> USB - GUIUser --> Remote - CLIUser --> Executor - Library --> Wrapper - Library --> Executor - Library --> Ops - - Admin --> REST - Inspector -.- WebRTC - Audit -.- REST - Audit -.- USB - UsbPass --> Backends - - Wrapper --> Backends - Vision -.- Wrapper - Recorder -.- Executor -``` - -``` -je_auto_control/ -├── wrapper/ # Platform-agnostic API layer -│ ├── platform_wrapper.py # Auto-detects OS and loads the correct backend -│ ├── auto_control_mouse.py # Mouse operations -│ ├── auto_control_keyboard.py# Keyboard operations -│ ├── auto_control_image.py # Image recognition (OpenCV template matching) -│ ├── auto_control_screen.py # Screenshot, screen size, pixel color -│ ├── auto_control_window.py # Cross-platform window manager facade -│ └── auto_control_record.py # Action recording/playback -├── windows/ # Windows-specific backend (Win32 API / ctypes) -├── osx/ # macOS-specific backend (pyobjc / Quartz) -├── linux_with_x11/ # Linux-specific backend (python-Xlib) -├── gui/ # PySide6 GUI application -└── utils/ - ├── mcp_server/ # MCP server (stdio + HTTP/SSE) — server, tools/, resources, prompts, audit, rate_limit, fake_backend, plugin_watcher - ├── executor/ # JSON action executor engine - ├── callback/ # Callback function executor - ├── cv2_utils/ # OpenCV screenshot, template matching, video recording - ├── accessibility/ # UIA (Windows) / AX (macOS) element finder - ├── vision/ # VLM-based locator (Anthropic / OpenAI backends) - ├── ocr/ # Tesseract-backed text locator - ├── clipboard/ # Cross-platform clipboard (text + image) - ├── llm/ # Plain-language → AC_* action planner - ├── scheduler/ # Interval + cron scheduler - ├── hotkey/ # Global hotkey daemon - ├── triggers/ # Image/window/pixel/file triggers - ├── run_history/ # SQLite run log + error-screenshot artifacts - ├── rest_api/ # Stdlib HTTP/REST server — auth · audit · rate-limit · OpenAPI · /metrics · dashboard · Swagger UI - ├── admin/ # Multi-host AdminConsoleClient (poll + broadcast) - ├── diagnostics/ # System self-test runner + CLI - ├── config_bundle/ # Single-file user-config export / import - ├── usb/ # Cross-platform enumeration, hotplug events, passthrough/{protocol, session, viewer client, loopback, webrtc channel, ACL+HMAC, descriptor, key providers, commands, libusb / WinUSB / IOKit} - ├── remote_desktop/ # WebRTC host + viewer, signalling, multi-viewer, file/clipboard/audio sync, audit log (hash chain), trust list, TURN config, mDNS discovery, WebRTC stats inspector - ├── plugin_loader/ # Dynamic AC_* plugin discovery - ├── socket_server/ # TCP socket server for remote automation - ├── shell_process/ # Shell command manager - ├── generate_report/ # HTML / JSON / XML report generators - ├── test_record/ # Test action recording - ├── script_vars/ # Script variable interpolation - ├── watcher/ # Mouse / pixel / log watchers (Live HUD) - ├── recording_edit/ # Trim, filter, re-scale recorded actions - ├── json/ # JSON action file read/write - ├── project/ # Project scaffolding & templates - ├── package_manager/ # Dynamic package loading - ├── logging/ # Logging - └── exception/ # Custom exception classes -``` - -The `platform_wrapper.py` module automatically detects the current operating system and imports the corresponding backend, so all wrapper functions work identically regardless of platform. +## Why AutoControl + +- **One API, six platforms.** `wrapper/platform_wrapper.py` picks the backend at import + time; your script does not change between Windows, macOS, X11, and Wayland. +- **Scriptable without Python.** 767 `AC_*` commands cover the whole feature set, so a + JSON file can do anything the library can — including loops, branches, try/catch, + macros, and variables. +- **Headless by default.** `import je_auto_control` never loads Qt. The GUI is an + optional extra that wraps the same headless core. +- **Locate things four ways.** Template matching, OCR, the accessibility tree, and a + vision-language model — composable through anchor locators and self-healing fallbacks. +- **Light dependency floor.** The REST server, JSON Schema validator, JWT, TOTP, + WebSocket framing, ACME client, USB/IP protocol, and Prometheus metrics are all + standard-library implementations. Heavy things are opt-in extras. --- ## Installation -### Basic Installation - ```bash -pip install je_auto_control +pip install je_auto_control # core +pip install je_auto_control[gui] # + PySide6 desktop app ``` -### With GUI Support (PySide6) +Optional extras, installed only when you need them: -```bash -pip install je_auto_control[gui] -``` - -### Linux Prerequisites +| Extra | Enables | +|---|---| +| `gui` | PySide6 desktop application (48 tabs) | +| `webrtc` | WebRTC remote desktop, USB passthrough (`aiortc`, `av`) | +| `signaling` | Standalone signaling / rendezvous server (`fastapi`, `uvicorn`) | +| `discovery` | mDNS / Zeroconf LAN host discovery | +| `pdf` / `office` | PDF and Excel / Word / PowerPoint reading | +| `fuzzy` / `locale` | `rapidfuzz` matching, `babel` locale parsing | +| `s3` / `audio` | S3 artifact store, system volume control | -On Linux, install the following system packages before installing: +**Requirements:** Python ≥ 3.10. On Linux, install build prerequisites first: ```bash sudo apt-get install cmake libssl-dev ``` ---- - -## Requirements - -- **Python** >= 3.10 -- **pip** >= 19.3 - -### Dependencies - -| Package | Purpose | -|---|---| -| `je_open_cv` | Image recognition (OpenCV template matching) | -| `pillow` | Screenshot capture | -| `mss` | Fast multi-monitor screenshot | -| `pyobjc` | macOS backend (auto-installed on macOS) | -| `python-Xlib` | Linux X11 backend (auto-installed on Linux) | -| `PySide6` | GUI application (optional, install with `[gui]`) | -| `qt-material` | GUI theme (optional, install with `[gui]`) | -| `uiautomation` | Windows accessibility backend (optional, loaded on demand) | -| `pytesseract` + Tesseract | OCR engine (optional, loaded on demand) | -| `anthropic` | VLM locator — Anthropic backend (optional, loaded on demand) | -| `openai` | VLM locator — OpenAI backend (optional, loaded on demand) | - -See [Third_Party_License.md](Third_Party_License.md) for a full list of -third-party components and their licenses. +OCR, VLM, and LLM backends (`pytesseract`, `easyocr`, `paddleocr`, `anthropic`, +`openai`) are loaded on demand — install whichever you actually use. --- -## Quick Start - -Looking for copy-pasteable end-to-end scripts instead of API snippets? -The [`examples/`](examples/) directory has 17 self-contained programs -covering screenshot + click, OCR, the headless scheduler, remote -desktop, the agent loop, observability, recording / replay, runtime -variables, window management, hotkeys, image triggers, HTML reports, -the MCP stdio bridge, the REST API, the secrets vault, and plugin -loading. - -### Mouse Control - -```python -import je_auto_control - -# Get current mouse position -x, y = je_auto_control.get_mouse_position() -print(f"Mouse at: ({x}, {y})") - -# Move mouse to coordinates -je_auto_control.set_mouse_position(500, 300) - -# Left click at current position (use key name) -je_auto_control.click_mouse("mouse_left") +## 60-second quick start -# Right click at specific coordinates -je_auto_control.click_mouse("mouse_right", x=800, y=400) - -# Scroll down -je_auto_control.mouse_scroll(scroll_value=5) -``` - -### Keyboard Control - -```python -import je_auto_control - -# Press and release a single key -je_auto_control.type_keyboard("a") - -# Type a whole string character by character -je_auto_control.write("Hello World") - -# Hotkey combination (e.g., Ctrl+C) -je_auto_control.hotkey(["ctrl_l", "c"]) - -# Check if a key is currently pressed -is_pressed = je_auto_control.check_key_is_press("shift_l") -``` - -### Image Recognition - -```python -import je_auto_control - -# Find all occurrences of an image on screen -positions = je_auto_control.locate_all_image("button.png", detect_threshold=0.9) -# Returns: [[x1, y1, x2, y2], ...] - -# Find a single image and get its center coordinates -cx, cy = je_auto_control.locate_image_center("icon.png", detect_threshold=0.85) -print(f"Found at: ({cx}, {cy})") - -# Find an image and automatically click it -je_auto_control.locate_and_click("submit_button.png", mouse_keycode="mouse_left") -``` - -### Accessibility Element Finder - -Query the OS accessibility tree to locate controls by name, role, or app. -Works on Windows (UIA, via `uiautomation`) and macOS (AX). - -```python -import je_auto_control - -# List all visible buttons in the Calculator app -elements = je_auto_control.list_accessibility_elements(app_name="Calculator") - -# Find a specific element -ok = je_auto_control.find_accessibility_element(name="OK", role="Button") -if ok is not None: - print(ok.bounds, ok.center) - -# Click it directly -je_auto_control.click_accessibility_element(name="OK", app_name="Calculator") -``` - -Raises `AccessibilityNotAvailableError` if no accessibility backend is -installed for the current platform. - -### AI Element Locator (VLM) - -When template matching and accessibility both fail, describe the element -in plain language and let a vision-language model find its coordinates. - -```python -import je_auto_control - -# Uses Anthropic by default if ANTHROPIC_API_KEY is set, else OpenAI. -x, y = je_auto_control.locate_by_description("the green Submit button") - -# Or click it in one shot -je_auto_control.click_by_description( - "the cookie-banner 'Accept all' button", - screen_region=[0, 800, 1920, 1080], # optional crop -) -``` - -Configuration (environment variables only — keys are never persisted or -logged): - -| Variable | Effect | -|---|---| -| `ANTHROPIC_API_KEY` | Enables the Anthropic backend | -| `OPENAI_API_KEY` | Enables the OpenAI backend | -| `AUTOCONTROL_VLM_BACKEND` | `anthropic` or `openai` to force a backend | -| `AUTOCONTROL_VLM_MODEL` | Override the default model (e.g. `claude-opus-4-7`, `gpt-4o-mini`) | - -Raises `VLMNotAvailableError` if neither SDK is installed or no API key -is set. - -### OCR (Text on Screen) +**1. As a Python library** ```python import je_auto_control as ac -# Locate all matches of a piece of text -matches = ac.find_text_matches("Submit") - -# Center of the first match, or None -cx, cy = ac.locate_text_center("Submit") +ac.set_mouse_position(500, 300) +ac.click_mouse("mouse_left") +ac.write("Hello World") +ac.hotkey(["ctrl_l", "s"]) -# Click text in one call -ac.click_text("Submit") - -# Block until text appears (or timeout) -ac.wait_for_text("Loading complete", timeout=15.0) +x, y = ac.locate_image_center("save_button.png", detect_threshold=0.9) +ac.click_text("Submit") # OCR +ac.click_accessibility_element(name="OK") # accessibility tree +ac.click_by_description("the green Submit button") # vision model +ac.screenshot("shot.png", screen_region=[0, 0, 800, 600]) ``` -Backend selection — set ``AUTOCONTROL_OCR_BACKEND=tesseract|easyocr|paddleocr`` -or pass ``backend=`` per call; otherwise auto-detection picks the first -one that imports: +**2. As a JSON action file** — `flow.json` -```python -ac.find_text_matches("登入", lang="chi_tra", backend="easyocr") -ac.click_text("Sign in", backend="tesseract") +```json +[ + ["AC_set_var", {"name": "user", "value": "alice"}], + ["AC_locate_and_click", {"image": "login.png", "mouse_keycode": "mouse_left"}], + ["AC_write", {"write_string": "${user}"}], + ["AC_retry", {"max_attempts": 3, "body": [ + ["AC_wait_text", {"target": "Welcome", "timeout": 10}] + ]}], + ["AC_assert_text", {"text": "Welcome"}], + ["AC_generate_html_report", {"html_name": "report"}] +] ``` -If Tesseract is not on `PATH`, point at it explicitly: - -```python -ac.set_tesseract_cmd(r"C:\Program Files\Tesseract-OCR\tesseract.exe") +```bash +je_auto_control run flow.json --var user=bob +je_auto_control run flow.json --dry-run # list the steps without touching the mouse ``` -Backend install paths and the canonical lang-code table are in -[docs/source/Eng/doc/ocr_backends/ocr_backends_doc.rst](docs/source/Eng/doc/ocr_backends/ocr_backends_doc.rst) -(or the [繁體中文](docs/source/Zh/doc/ocr_backends/ocr_backends_doc.rst) -version). - -Dump every recognised text record in a region (or full screen), or -search by regex when the text varies: - -```python -import je_auto_control as ac +**3. As a desktop app** -# Every hit in a region as TextMatch records (text, bounding box, confidence) -for match in ac.read_text_in_region(region=[0, 0, 800, 600]): - print(match.text, match.center, match.confidence) - -# Regex — accepts a pattern string or a compiled re.Pattern -for match in ac.find_text_regex(r"Order#\d+"): - print(match.text, match.center) +```bash +pip install je_auto_control[gui] +python -m je_auto_control # or: je_auto_control.start_autocontrol_gui() ``` -GUI: **OCR Reader** tab. +Record a flow, edit it in the visual Script Builder, and save it as the same JSON +format the CLI runs. -### LLM Action Planner +--- -Translate plain-language descriptions into validated `AC_*` action lists -using an LLM (Anthropic Claude by default). Output is leniently parsed -(strips code fences, extracts the first JSON array from prose) and then -validated by the same schema the executor uses, so the result can be -piped straight into `execute_action`: +## Capability overview -```python -import je_auto_control as ac -from je_auto_control.utils.executor.action_executor import executor +Every row works headlessly. "GUI tab" is where the same feature surfaces in the +desktop app; tab commands live in the window's **Actions** menu. -actions = ac.plan_actions( - "click the Submit button, then type 'done' and save", - known_commands=executor.known_commands(), -) -executor.execute_action(actions) +| Capability | Python API | `AC_*` command | GUI tab | +|---|---|---|---| +| Mouse | `click_mouse`, `set_mouse_position`, `mouse_scroll` | `AC_click_mouse` | Auto Click | +| Keyboard | `write`, `hotkey`, `type_keyboard` | `AC_write`, `AC_hotkey` | Auto Click | +| Screen & pixels | `screenshot`, `screen_size`, `get_pixel` | `AC_screenshot` | Screenshot | +| Image matching | `locate_image_center`, `locate_and_click` | `AC_locate_and_click` | Image Detect | +| OCR text | `click_text`, `wait_for_text`, `read_text_in_region` | `AC_click_text`, `AC_wait_text` | OCR Reader | +| Accessibility tree | `find_accessibility_element`, `click_accessibility_element` | `AC_a11y_find`, `AC_a11y_click` | Accessibility | +| Vision-model locator | `locate_by_description`, `click_by_description` | `AC_vlm_locate`, `AC_vlm_click` | VLM | +| Anchor locator | — | `AC_anchor_click`, `AC_anchor_locate` | — | +| Self-healing locators | `self_heal_click`, `self_heal_locate` | `AC_self_heal_click` | Self-Healing | +| Natural-language planner | `plan_actions`, `run_from_description` | `AC_llm_plan` | LLM Planner | +| Computer-use agent | `AgentLoop`, `run_agent` | `AC_run_agent` | Computer Use | +| Record & replay | `record`, `stop_record` | `AC_record`, `AC_stop_record` | Record | +| JSON scripting | `execute_action`, `execute_files` | all 767 commands | Script, Script Builder | +| Variables & flow control | `execute_action_with_vars` | `AC_set_var`, `AC_loop`, `AC_for_each`, `AC_try`, `AC_retry` | Variables | +| Data-driven runs | — | `AC_for_each_row` (CSV / JSON / SQLite / Excel) | Data Sources | +| Assertions | `assert_text`, `assert_image` | `AC_assert_text` + 20 more | Assertions | +| Test suites | `run_suite` | `AC_run_suite` | Test Suites | +| Scheduler (interval + cron) | `default_scheduler` | — | Scheduler | +| Global hotkeys | `default_hotkey_daemon` | — | Hotkeys | +| Event triggers | `default_trigger_engine` | `AC_email_trigger_add` | Triggers, Webhooks, Email | +| Window management *(Windows)* | `list_windows`, `focus_window` | `AC_focus_window`, `AC_snap_window` | Window Manager | +| Clipboard (text + image) | `get_clipboard`, `set_clipboard`, `get_clipboard_image`, `set_clipboard_image` | `AC_clipboard_get`, `AC_clipboard_set`, `AC_clipboard_get_image`, `AC_clipboard_set_image` | — | +| Remote desktop | `RemoteDesktopHost`, `RemoteDesktopViewer` | `AC_start_remote_host`, `AC_remote_connect` | Remote Desktop | +| USB enumeration & passthrough | `list_usb_devices`, `enable_usb_passthrough` | `AC_usb_*` (16 commands) | USB Devices, USB Share | +| Secrets vault | `default_secret_manager` | `AC_secret_set` + `${secrets.NAME}` | Secrets | +| Reports (HTML / JSON / XML) | `generate_html_report` | `AC_generate_html_report` | Report | +| Run history | — | — | Run History | +| Metrics & tracing | `default_metric_registry`, `render_metrics_text` | — | — | +| Diagnostics | `run_diagnostics` | `AC_diagnose` | Diagnostics | +| Test-code generation | `generate_code` | — | — | + +Beyond this table, `utils/` holds 308 headless packages covering assertions, resilience, +data quality, i18n auditing, redaction, governance, observability, and more. The full +per-module map is in **[architecture_explore.md](architecture_explore.md)**. -# Or in a single call: -ac.run_from_description("open Notepad and type hello", executor=executor) -``` +--- -| Variable | Effect | -|---|---| -| `ANTHROPIC_API_KEY` | Enables the Anthropic backend | -| `AUTOCONTROL_LLM_BACKEND` | `anthropic` to force a backend | -| `AUTOCONTROL_LLM_MODEL` | Override the default model (e.g. `claude-opus-4-7`) | +## Command-line interface -GUI: **LLM Planner** tab — description box and action-list preview; -*Plan* (`QThread`-backed) and *Run plan* live in the window's Actions menu. +```bash +je_auto_control run script.json [--var name=value] [--dry-run] +je_auto_control validate script.json # alias: lint +je_auto_control fmt script.json [--check] +je_auto_control list-commands [--filter mouse] [--json] +je_auto_control record out.json [--duration 5] +je_auto_control codegen script.json --target pytest -o test_flow.py +je_auto_control failure-bundle failure.zip --error "login timed out" +je_auto_control list-jobs +je_auto_control start-server --port 9938 # TCP socket server +je_auto_control start-rest --port 9939 # REST API +je_auto_control version +``` + +`--var name=value` is parsed as JSON when possible (`count=10` becomes an int), +otherwise kept as a string. The legacy `python -m je_auto_control -e file.json` +entry point still works. -### Runtime Variables & Control Flow +--- -The executor resolves `${var}` placeholders **per command call** rather -than pre-flattening, so nested `body` / `then` / `else` lists keep their -placeholders and re-bind on every iteration. Combined with new mutation -commands, scripts can drive themselves from data without Python glue: +## Servers and integrations -```json -[ - ["AC_set_var", {"name": "items", "value": ["alpha", "beta"]}], - ["AC_set_var", {"name": "i", "value": 0}], - ["AC_for_each", { - "items": "${items}", "as": "name", - "body": [ - ["AC_inc_var", {"name": "i"}], - ["AC_if_var", { - "name": "i", "op": "ge", "value": 2, - "then": [["AC_break"]], "else": [] - }] - ] - }] -] -``` +| Surface | Start it with | Notes | +|---|---|---| +| **MCP server** | `je_auto_control_mcp` (stdio) or `AC_start_mcp_http_server` | 670 tools for Claude Desktop / Claude Code / custom tool loops. Bearer auth, TLS, audit log, rate limit, plugin hot-reload, CI fake backend. | +| **REST API** | `je_auto_control start-rest` | Bearer token, per-IP rate limit + lockout, SQLite audit hook, `/metrics`, `/openapi.json`, `/docs` Swagger UI, `/dashboard`. | +| **TCP socket server** | `je_auto_control start-server` | Newline-framed JSON action lists. Binds `127.0.0.1` by default. | +| **pytest plugin** | installed automatically | Fixtures plus a Gherkin step library for pytest-bdd / behave. | +| **Language server** | `python -m autocontrol_lsp.server` | Completion and diagnostics for `AC_*` action JSON, generated from the live command table. | +| **Remote desktop** | `RemoteDesktopHost` / GUI | TCP, WebSocket, or WebRTC; TOTP, trust list, TURN config, file/clipboard/audio sync. | -`AC_if_var` operators: `eq`, `ne`, `lt`, `le`, `gt`, `ge`, `contains`, -`startswith`, `endswith`. GUI: **Variables** tab — live view of -`executor.variables`; single-set, JSON seed, and clear-all run from the -window's Actions menu. +All servers bind to `127.0.0.1` unless you opt in explicitly. -### Remote Desktop +### How the remote-desktop wire protocol works -Stream this machine's screen and accept remote input, **or** view and -control another machine. The wire format is a length-prefixed framing -on raw TCP (no extra deps), starting with an HMAC-SHA256 -challenge / response handshake; viewers that fail auth are dropped -before they can see a frame. JPEG frames are produced at the configured -FPS / quality and broadcast to authenticated viewers via a shared -latest-frame slot, so a slow viewer drops frames instead of blocking -the rest. Viewer input is JSON, validated against an allowlist, and -applied through the existing wrappers. +Worth knowing before you expose a host, and described nowhere else in the +docs. The default transport is length-prefixed framing over raw TCP — no extra +dependencies — and it opens with an **HMAC-SHA256 challenge/response +handshake**: a viewer that fails auth is dropped before it is sent a single +frame. JPEG frames are encoded at the configured FPS and quality and handed to +authenticated viewers through a shared *latest-frame slot*, so a slow viewer +drops frames instead of stalling the rest. Viewer input arrives as JSON and is +**validated against an allow-list** of actions before being applied through the +ordinary input wrappers, so a viewer cannot invent new operations. ```python # Be remoted — start a host and hand the token + port to whoever views you from je_auto_control import RemoteDesktopHost host = RemoteDesktopHost(token="hunter2", bind="127.0.0.1", - port=0, fps=10, quality=70) + port=0, fps=10, quality=70) host.start() print("listening on", host.port, "viewers:", host.connected_clients) ``` @@ -606,866 +219,87 @@ print("listening on", host.port, "viewers:", host.connected_clients) # Control another machine — connect a viewer and send input from je_auto_control import RemoteDesktopViewer viewer = RemoteDesktopViewer(host="10.0.0.5", port=51234, token="hunter2", - on_frame=lambda jpeg: ...) + on_frame=lambda jpeg: ...) viewer.connect() viewer.send_input({"action": "mouse_move", "x": 100, "y": 200}) -viewer.send_input({"action": "type", "text": "hello"}) viewer.disconnect() ``` -GUI: **Remote Desktop** tab opens to the **Quick Connect** screen -(AnyDesk-style) by default — huge Host ID on one side, a single input -that accepts `host:port`, `ws://`, `wss://`, or a 9-digit Host ID on -the other, with *Connect* and *Start hosting* as the two primary -buttons. Recent connections are remembered across sessions. Advanced -per-transport sub-tabs (legacy TCP / WS host + viewer, WebRTC host + -viewer with manual SDP / custom codecs / TLS pinning) stay one click -away. WebRTC sub-tabs lazy-load so a stock install without the -`[webrtc]` extra still opens the tab. - -> ⚠️ Anyone with the host:port and token gets full mouse / keyboard -> control of the host machine. Default bind is `127.0.0.1`; expose -> externally only via SSH tunnel or TLS front-end. The token is the -> only line of defence — treat it like a password. - -**Quick Connect headless API.** The transport coordinator that backs -the GUI input box is also exported, so scripts can dispatch the same -way: - -```python -from je_auto_control import parse_remote_desktop_target -parse_remote_desktop_target("192.168.1.10:5555") -# ConnectTarget(kind='tcp', host='192.168.1.10', port=5555, ...) -parse_remote_desktop_target("ws://hub:8765/desk") -# ConnectTarget(kind='ws', host='hub', port=8765, path='/desk') -parse_remote_desktop_target("123-456-789") -# ConnectTarget(kind='webrtc_id', host_id='123456789') -``` - -**Connection approval + view-only mode.** Optional callback gates -every incoming session AnyDesk-style. Returning `"view_only"` admits -the viewer but drops their `INPUT` messages; returning a falsy value -(or raising) sends `AUTH_FAIL` "rejected by host": - -```python -from je_auto_control import RemoteDesktopHost, PendingViewer - -def gate(p: PendingViewer) -> str: - if p.address[0].startswith("10."): - return "view_only" - return "full" # or True - -host = RemoteDesktopHost(token="tok", on_pending_viewer=gate) -``` - -**IP allowlist (CIDR + exact IPs).** Reject peers outside the -configured ranges *before* TLS / auth runs, so attackers can't probe -further: - -```python -host = RemoteDesktopHost( - token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"], -) -``` - -**One-time share codes** — extra tokens that self-destruct on first -successful auth, ideal for client-support workflows: - -```python -host = RemoteDesktopHost(token="tok", single_use_tokens=["abc123"]) -host.add_single_use_token("9k4ndx") # rotate at runtime -host.revoke_single_use_token("abc123") # cancel before it's used -``` - -**TOTP 2FA (RFC 6238, stdlib only).** Layer a 6-digit OTP on top of -the token; host accepts ±1 step of clock drift: - -```python -from je_auto_control.utils.remote_desktop.totp import ( - generate_secret, generate_code, provisioning_uri, -) -secret = generate_secret() -print(provisioning_uri(secret, account="alice")) # otpauth:// URI for QR - -host = RemoteDesktopHost(token="tok", totp_secret=secret) -viewer = RemoteDesktopViewer( - host=..., token="tok", totp_code=generate_code(secret), -) -``` - -**Multi-monitor selection.** Capture one specific monitor instead of -the combined virtual desktop: - -```python -from je_auto_control import list_host_monitors, RemoteDesktopHost -print(list_host_monitors()) -# [{'index': 0, 'is_combined': True, ...}, -# {'index': 1, 'left': 0, 'top': 0, ...}, -# {'index': 2, 'left': 1920, ...}] -host = RemoteDesktopHost(token="tok", monitor_index=1) -``` - -**Remote cursor overlay.** Host broadcasts cursor position at 30 Hz -(deduped on still desktops); the viewer's popup window draws an arrow -on top of the JPEG stream so you can see exactly where the host's -pointer is. Disable via `enable_cursor_broadcast=False`. - -**Multi-viewer collaborative cursors + chat.** Two new message types -(`CHAT` and `CURSOR` with `viewer_id`). Use a `MultiViewerHost` to -relay one viewer's pointer to the others; pair with the chat channel -for ad-hoc text between operators: - -```python -host = RemoteDesktopHost( - token="tok", on_chat=lambda sender, text: print(sender, ":", text), -) -host.broadcast_chat("session starts in 30s") -host.broadcast_viewer_cursor("alice", 200, 300) - -viewer = RemoteDesktopViewer( - host=..., on_chat=lambda s, t: ..., - on_viewer_cursor=lambda vid, x, y: ..., -) -viewer.send_chat("ack") -``` - -**Relative mouse mode (FPS / CAD).** New input action that sends -deltas instead of absolute coordinates: - -```python -viewer.send_input({"action": "mouse_move_relative", "dx": 5, "dy": -3}) -``` - -**Motion-aware capture.** The capture loop now hashes each encoded -JPEG; identical frames are skipped, so a static desktop produces -~zero bandwidth. New viewers are seeded with the latest frame on auth -so they never see a black popup. - -**Live stats** (FPS / kbps / totals over a 3-second window): - -```python -viewer.stats() -# {'fps': 24.3, 'kbps': 4801.2, 'frames': 720.0, 'bytes': 1.8e7, 'uptime': 30.2} -``` - -**JPEG sequence recorder (no PyAV needed).** TCP-path session -capture: each frame written to disk plus `manifest.json` so it can -be replayed at original cadence: - -```python -from je_auto_control.utils.remote_desktop.jpeg_recorder import ( - JpegSequenceRecorder, -) -rec = JpegSequenceRecorder("~/recordings/2026-05-23") -rec.start() -viewer = RemoteDesktopViewer(host=..., on_frame=rec.record_frame) -# ... session ... -rec.stop() # writes manifest.json next to the .jpg files -``` - -**TCP relay (WebRTC fallback).** When P2P fails (strict NAT, mobile -CGNAT, hotel Wi-Fi), both peers connect outbound to a relay and -exchange a shared 32-byte session ID; the relay pipes bytes between -them. Same module ships an `encode_handshake(role, session_id)` -helper for clients: - -```python -from je_auto_control.utils.remote_desktop.relay import RelayServer -relay = RelayServer(bind="0.0.0.0", port=9000) # NOSONAR # public relay -relay.start() -``` - -**Service installer (unattended host).** `python -m -je_auto_control.utils.remote_desktop.host_service ...` -exposes `configure` / `init` / `run` plus per-platform installers: -`install-windows-service` / `uninstall-windows-service` (pywin32), -`generate-launchd` / `uninstall-launchd`, `generate-systemd` / -`uninstall-systemd`. - -**Encrypted transports + alternate protocols.** Pass an `ssl_context` -to either `RemoteDesktopHost` or `RemoteDesktopViewer` to wrap every -connection in TLS. For firewall-friendly access, use the in-tree -WebSocket variants (no extra deps) — same protocol, RFC 6455 framing, -and `wss://` if you also pass `ssl_context`: - -```python -from je_auto_control import ( - WebSocketDesktopHost, WebSocketDesktopViewer, -) -host = WebSocketDesktopHost(token="hunter2", ssl_context=server_ctx) -viewer = WebSocketDesktopViewer( - host="example.com", port=443, token="hunter2", - ssl_context=client_ctx, expected_host_id="123456789", -) -``` - -**Persistent Host ID.** Every host owns a stable 9-digit numeric ID -(persisted at `~/.je_auto_control/remote_host_id`), announced in -`AUTH_OK` and verifiable via the viewer's `expected_host_id`: - -```python -print(host.host_id) # e.g. "123456789" -viewer = RemoteDesktopViewer( - host=..., port=..., token=..., - expected_host_id="123456789", # AuthenticationError on mismatch -) -``` - -**Audio streaming (host → viewer).** Optional `sounddevice` dep; opt -in with an `AudioCaptureConfig` on the host, attach an `AudioPlayer` -(or your own callback) on the viewer: - -```python -from je_auto_control.utils.remote_desktop import AudioCaptureConfig -host = RemoteDesktopHost( - token="tok", - audio_config=AudioCaptureConfig(enabled=True), # default mic -) -# Or pick a loopback / monitor device: -# audio_config=AudioCaptureConfig(enabled=True, device=12) - -from je_auto_control.utils.remote_desktop import AudioPlayer -player = AudioPlayer(); player.start() -viewer = RemoteDesktopViewer(host=..., on_audio=player.play) -``` - -**Clipboard sync (text + image, bidirectional).** Explicit per-call — -no auto-poll loops. Image clipboard works on Windows (CF_DIB via -ctypes) and Linux (`xclip -t image/png`); macOS get is supported via -Pillow ImageGrab, set requires PyObjC. - -```python -viewer.send_clipboard_text("hello") -viewer.send_clipboard_image(open("logo.png", "rb").read()) -host.broadcast_clipboard_text("greetings") -``` - -**File transfer with progress.** Bidirectional, chunked, arbitrary -destination path, no size cap; the GUI viewer also accepts drag-drop: - -```python -viewer.send_file( - "local.bin", "/tmp/uploaded.bin", - on_progress=lambda tid, done, total: print(done, total), -) -host.send_file_to_viewers("local.bin", "/tmp/from_host.bin") -``` - -> ⚠️ Path is unrestricted and there is no aggregate size limit. -> Anyone with the token can write any file to any location and can -> fill the disk — keep "trusted token holders == trusted users" in -> mind, or wrap with your own `FileReceiver` subclass that vets -> destination paths. - -### Clipboard - -```python -import je_auto_control as ac -ac.set_clipboard("hello") -text = ac.get_clipboard() -``` - -Backends: Windows (Win32 via `ctypes`), macOS (`pbcopy`/`pbpaste`), -Linux (`xclip` or `xsel`). - -### Screenshot - -```python -import je_auto_control - -# Take a full-screen screenshot and save to file -je_auto_control.pil_screenshot("screenshot.png") - -# Take a screenshot of a specific region [x1, y1, x2, y2] -je_auto_control.pil_screenshot("region.png", screen_region=[100, 100, 500, 400]) - -# Get screen resolution -width, height = je_auto_control.screen_size() - -# Get pixel color at coordinates -color = je_auto_control.get_pixel(500, 300) -``` - -### Action Recording & Playback - -```python -import je_auto_control -import time - -# Start recording mouse and keyboard events -je_auto_control.record() - -time.sleep(10) # Record for 10 seconds - -# Stop recording and get the action list -actions = je_auto_control.stop_record() - -# Clean up the recording before replay: collapse runs of consecutive -# mouse-move samples into their final position (often shrinks a raw -# recording by an order of magnitude without changing replay behaviour) -actions = je_auto_control.dedupe_moves(actions) - -# Replay the recorded actions -je_auto_control.execute_action(actions) -``` - -> Non-destructive recording editors (all return a new list): `dedupe_moves` (collapse mouse-move runs), `merge_sleeps` (sum consecutive `AC_sleep` runs), `trim_actions`, `insert_action`, `remove_action`, `filter_actions`, `adjust_delays` (scale `AC_sleep` delays), `scale_coordinates` (replay at a different resolution). Exposed over MCP as `ac_dedupe_moves` / `ac_merge_sleeps` / `ac_trim_actions` / `ac_adjust_delays` / `ac_scale_coordinates`. - -### JSON Action Scripting - -Create a JSON action file (`actions.json`): - -```json -[ - ["AC_set_mouse_position", {"x": 500, "y": 300}], - ["AC_click_mouse", {"mouse_keycode": "mouse_left"}], - ["AC_write", {"write_string": "Hello from AutoControl"}], - ["AC_screenshot", {"file_path": "result.png"}], - ["AC_hotkey", {"key_code_list": ["ctrl_l", "s"]}] -] -``` - -Execute it: - -```python -import je_auto_control - -# Execute from file -je_auto_control.execute_action(je_auto_control.read_action_json("actions.json")) - -# Or execute from a list directly -je_auto_control.execute_action([ - ["AC_set_mouse_position", {"x": 100, "y": 200}], - ["AC_click_mouse", {"mouse_keycode": "mouse_left"}] -]) -``` - -**Available action commands:** - -| Category | Commands | -|---|---| -| Mouse | `AC_click_mouse`, `AC_set_mouse_position`, `AC_get_mouse_position`, `AC_get_mouse_table`, `AC_press_mouse`, `AC_release_mouse`, `AC_mouse_scroll`, `AC_mouse_left`, `AC_mouse_right`, `AC_mouse_middle` | -| Keyboard | `AC_type_keyboard`, `AC_press_keyboard_key`, `AC_release_keyboard_key`, `AC_write`, `AC_hotkey`, `AC_check_key_is_press`, `AC_get_keyboard_keys_table` | -| Image | `AC_locate_all_image`, `AC_locate_image_center`, `AC_locate_and_click` | -| Screen | `AC_screen_size`, `AC_screenshot` | -| Accessibility | `AC_a11y_list`, `AC_a11y_find`, `AC_a11y_click` | -| VLM (AI Locator) | `AC_vlm_locate`, `AC_vlm_click` | -| OCR | `AC_locate_text`, `AC_click_text`, `AC_wait_text`, `AC_read_text_in_region`, `AC_find_text_regex` | -| LLM planner | `AC_llm_plan`, `AC_llm_run` | -| Clipboard | `AC_clipboard_get`, `AC_clipboard_set` | -| Window | `AC_list_windows`, `AC_focus_window`, `AC_wait_window`, `AC_close_window` | -| Flow control | `AC_loop`, `AC_break`, `AC_continue`, `AC_if_image_found`, `AC_if_pixel`, `AC_if_var`, `AC_while_image`, `AC_while_var`, `AC_for_each`, `AC_wait_image`, `AC_wait_pixel`, `AC_sleep`, `AC_retry`, `AC_try` | -| Variables | `AC_set_var`, `AC_get_var`, `AC_inc_var` | -| Remote desktop | `AC_start_remote_host`, `AC_stop_remote_host`, `AC_remote_host_status`, `AC_remote_connect`, `AC_remote_disconnect`, `AC_remote_viewer_status`, `AC_remote_send_input` | -| Record | `AC_record`, `AC_stop_record`, `AC_set_record_enable` | -| Report | `AC_generate_html`, `AC_generate_json`, `AC_generate_xml`, `AC_generate_html_report`, `AC_generate_json_report`, `AC_generate_xml_report` | -| Run history | `AC_history_list`, `AC_history_clear` | -| Project | `AC_create_project` | -| Shell | `AC_shell_command` | -| Process | `AC_execute_process` | -| Executor | `AC_execute_action`, `AC_execute_files`, `AC_add_package_to_executor`, `AC_add_package_to_callback_executor` | -| MCP server | `AC_start_mcp_server`, `AC_start_mcp_http_server` | - -### MCP Server (Use AutoControl from Claude) - -Expose AutoControl as a Model Context Protocol server so any -MCP-compatible client (Claude Desktop, Claude Code, custom Anthropic -/ OpenAI tool-use loops) can drive the host machine. Stdlib-only — -JSON-RPC 2.0 over stdio or HTTP+SSE. - -**Register with Claude Code:** - -```bash -claude mcp add autocontrol -- python -m je_auto_control.utils.mcp_server -``` - -**Register with Claude Desktop** (`claude_desktop_config.json`): - -```json -{ - "mcpServers": { - "autocontrol": { - "command": "python", - "args": ["-m", "je_auto_control.utils.mcp_server"] - } - } -} -``` - -**Start programmatically:** - -```python -import je_auto_control as ac - -# Stdio (blocks until stdin closes) -ac.start_mcp_stdio_server() - -# Or HTTP / SSE with bearer-token auth + optional TLS -ac.start_mcp_http_server(host="127.0.0.1", port=9940, - auth_token="hunter2") -``` - -**Inspect the catalogue without starting the server:** - -```bash -je_auto_control_mcp --list-tools -je_auto_control_mcp --list-tools --read-only -je_auto_control_mcp --list-resources -je_auto_control_mcp --list-prompts -``` - -**What ships:** - -| Surface | Coverage | -|---|---| -| Tools (~90) | mouse · keyboard · drag · screen / multi-monitor · screenshot-as-image · diff · OCR · image · windows (move/min/max/restore/...) · clipboard text+image · process / shell · recording · screen recording · scheduler / triggers / hotkeys · accessibility tree · VLM locator · executor · history | -| Aliases | `click`, `type`, `screenshot`, `find_image`, `drag`, `shell`, `wait_image`, ... — toggle with `JE_AUTOCONTROL_MCP_ALIASES=0` | -| Resources | `autocontrol://files/`, `autocontrol://history`, `autocontrol://commands`, `autocontrol://screen/live` (with `resources/subscribe`) | -| Prompts | `automate_ui_task`, `record_and_generalize`, `compare_screenshots`, `find_widget`, `explain_action_file` | -| Protocol | tools / resources / prompts / sampling / roots / logging / progress / cancellation / list_changed / elicitation | -| Transports | stdio, HTTP `POST /mcp`, SSE streaming when `Accept: text/event-stream` | -| Safety | tool annotations · `JE_AUTOCONTROL_MCP_READONLY` · `JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE` · audit log · token-bucket rate limiter · auto-screenshot on error | -| Ops | bearer-token auth · TLS via `ssl_context` · `PluginWatcher` hot-reload · `JE_AUTOCONTROL_FAKE_BACKEND=1` for CI | - -See [docs/source/Eng/doc/mcp_server/mcp_server_doc.rst](docs/source/Eng/doc/mcp_server/mcp_server_doc.rst) -for the full reference (or the -[繁體中文](docs/source/Zh/doc/mcp_server/mcp_server_doc.rst) version). - -> ⚠️ The MCP server can move the mouse, send keystrokes, capture the -> screen, and execute arbitrary `AC_*` actions. Only register it with -> MCP clients you trust. HTTP defaults to `127.0.0.1`; binding to -> `0.0.0.0` requires explicit reason and **must** be paired with -> `auth_token` plus `ssl_context`. - -### Scheduler (Interval & Cron) - -```python -import je_auto_control as ac - -# Interval job — run every 30 seconds -job = ac.default_scheduler.add_job( - script_path="scripts/poll.json", interval_seconds=30, repeat=True, -) - -# Cron job — 09:00 on weekdays (minute hour dom month dow) -cron_job = ac.default_scheduler.add_cron_job( - script_path="scripts/daily.json", cron_expression="0 9 * * 1-5", -) - -ac.default_scheduler.start() -``` - -Both flavours coexist; `job.is_cron` tells them apart. - -### Global Hotkey Daemon - -Bind OS-level hotkeys to action JSON scripts. Cross-platform — Windows -uses `RegisterHotKey`, macOS uses `CGEventTap` (requires Accessibility -permission), Linux X11 uses `XGrabKey` (Wayland not supported). The -same call sites work everywhere; the daemon picks the backend at -`start()` time. +Narrow who may connect at all with an IP allow-list (CIDR ranges or exact +addresses); peers outside it are rejected during the handshake: ```python -from je_auto_control import default_hotkey_daemon - -default_hotkey_daemon.bind("ctrl+alt+1", "scripts/greet.json") -default_hotkey_daemon.start() +RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) ``` -### Event Triggers - -Poll-based triggers that fire a script when a condition becomes true: - -```python -from je_auto_control import ( - default_trigger_engine, ImageAppearsTrigger, - WindowAppearsTrigger, PixelColorTrigger, FilePathTrigger, -) - -default_trigger_engine.add(ImageAppearsTrigger( - trigger_id="", script_path="scripts/click_ok.json", - image_path="templates/ok_button.png", threshold=0.85, repeat=True, -)) -default_trigger_engine.start() -``` - -### Run History - -Every run from the scheduler, trigger engine, hotkey daemon, REST API, -and manual GUI replay is recorded to `~/.je_auto_control/history.db`. -Errors automatically attach a screenshot under -`~/.je_auto_control/artifacts/run_{id}_{ms}.png` for post-mortem. - -```python -from je_auto_control import default_history_store - -for run in default_history_store.list_runs(limit=20): - print(run.id, run.source, run.status, run.artifact_path) -``` - -The GUI **Run History** tab shows the runs table with -double-click-to-open on the artifact column; filter, refresh, and -clear run from the window's Actions menu. - -### Report Generation - -```python -import je_auto_control - -# Enable test recording first -je_auto_control.test_record_instance.set_record_enable(True) - -# ... perform automation actions ... -je_auto_control.set_mouse_position(100, 200) -je_auto_control.click_mouse("mouse_left") - -# Generate reports -je_auto_control.generate_html_report("test_report") # -> test_report.html -je_auto_control.generate_json_report("test_report") # -> test_report.json -je_auto_control.generate_xml_report("test_report") # -> test_report.xml - -# Or get report content as string -html_string = je_auto_control.generate_html() -json_string = je_auto_control.generate_json() -xml_string = je_auto_control.generate_xml() -``` - -Reports include: function name, parameters, timestamp, and exception info (if any) for each recorded action. HTML reports display successful actions in cyan and failed actions in red. - -### Observability (Prometheus / OpenTelemetry) - -Stdlib-only metric primitives plus an OpenTelemetry-compatible tracer -fallback. The executor and agent loop emit call counts and latency -histograms automatically — no per-script wiring required. - -```python -import je_auto_control as ac - -# Expose /metrics on http://127.0.0.1:9090 for Prometheus to scrape. -exporter = ac.default_metrics_exporter() -exporter.start() - -# Add your own metric — same shapes as prometheus_client. -counter = ac.default_metric_registry().register(ac.MetricCounter( - "myapp_widgets_built_total", "widgets built", - label_names=("kind",), -)) -counter.inc(labels={"kind": "blue"}) - -# Wrap a callable in a span — no-op until opentelemetry-api is installed. -@ac.traced("my_pipeline.process_one") -def process_one(item): ... -``` - -Built-in metrics are listed in -[docs/source/Eng/doc/observability/observability_doc.rst](docs/source/Eng/doc/observability/observability_doc.rst) -(or the [繁體中文](docs/source/Zh/doc/observability/observability_doc.rst) -version). - -### Remote Automation (Socket / REST) - -Two servers are available — a raw TCP socket and a stdlib HTTP/REST -server. Both default to `127.0.0.1`; binding to `0.0.0.0` is an explicit, -documented opt-in. - -```python -import je_auto_control as ac - -# TCP socket server (default: 127.0.0.1:9938) -ac.start_autocontrol_socket_server(host="127.0.0.1", port=9938) - -# REST API server (default: 127.0.0.1:9939) -ac.start_rest_api_server(host="127.0.0.1", port=9939) -# Endpoints: -# GET /health liveness probe -# GET /jobs scheduler job list -# POST /execute body: {"actions": [...]} -``` - -Client example: - -```python -import socket -import json - -sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -sock.connect(("localhost", 9938)) - -# Send an automation command -command = json.dumps([ - ["AC_set_mouse_position", {"x": 500, "y": 300}], - ["AC_click_mouse", {"mouse_keycode": "mouse_left"}] -]) -sock.sendall(command.encode("utf-8")) - -# Receive response -response = sock.recv(8192).decode("utf-8") -print(response) -sock.close() -``` - -### Plugin Loader - -Drop `.py` files defining top-level `AC_*` callables into a directory, -then register them as executor commands at runtime: - -```python -from je_auto_control import ( - load_plugin_directory, register_plugin_commands, -) - -commands = load_plugin_directory("./my_plugins") -register_plugin_commands(commands) - -# Now usable from any JSON action script: -# [["AC_greet", {"name": "world"}]] -``` - -> **Warning:** Plugin files execute arbitrary Python on load. Only load -> from directories you control. - -### Shell Command Execution - -```python -import je_auto_control - -# Using the default shell manager -je_auto_control.default_shell_manager.exec_shell("echo Hello") -je_auto_control.default_shell_manager.pull_text() # Print captured output - -# Or create a custom ShellManager -shell = je_auto_control.ShellManager(shell_encoding="utf-8") -shell.exec_shell("ls -la") -shell.pull_text() -shell.exit_program() -``` - -### Screen Recording - -```python -import je_auto_control -import time - -# Method 1: ScreenRecorder (manages multiple recordings) -recorder = je_auto_control.ScreenRecorder() -recorder.start_new_record( - recorder_name="my_recording", - path_and_filename="output.avi", - codec="XVID", - frame_per_sec=30, - resolution=(1920, 1080) -) -time.sleep(10) -recorder.stop_record("my_recording") - -# Method 2: RecordingThread (simple single recording, outputs MP4) -recording = je_auto_control.RecordingThread(video_name="my_video", fps=20) -recording.start() -time.sleep(10) -recording.stop() -``` - -### Callback Executor - -Execute an automation function and trigger a callback upon completion: - -```python -import je_auto_control - -def my_callback(): - print("Action completed!") - -# Execute set_mouse_position then call my_callback -je_auto_control.callback_executor.callback_function( - trigger_function_name="AC_set_mouse_position", - callback_function=my_callback, - x=500, y=300 -) - -# With callback parameters -def on_done(message): - print(f"Done: {message}") - -je_auto_control.callback_executor.callback_function( - trigger_function_name="AC_click_mouse", - callback_function=on_done, - callback_function_param={"message": "Click finished"}, - callback_param_method="kwargs", - mouse_keycode="mouse_left" -) -``` - -### Package Manager - -Dynamically load external Python packages into the executor at runtime: - -```python -import je_auto_control - -# Add all functions/classes from a package to the executor -je_auto_control.package_manager.add_package_to_executor("os") - -# Now you can use os functions in JSON action scripts: -# ["os_getcwd", {}] -# ["os_listdir", {"path": "."}] -``` - -### Project Management - -Scaffold a project directory structure with template files: - -```python -import je_auto_control - -# Create a project structure -je_auto_control.create_project_dir(project_path="./my_project", parent_name="AutoControl") - -# This creates: -# my_project/ -# └── AutoControl/ -# ├── keyword/ -# │ ├── keyword1.json # Template action file -# │ ├── keyword2.json # Template action file -# │ └── bad_keyword_1.json # Error handling template -# └── executor/ -# ├── executor_one_file.py # Execute single file example -# ├── executor_folder.py # Execute folder example -# └── executor_bad_file.py # Error handling example -``` - -### Window Management - -Send events directly to specific windows (Windows and Linux only): - -```python -import je_auto_control - -# Send keyboard event to a window by title -je_auto_control.send_key_event_to_window("Notepad", keycode="a") - -# Send mouse event to a window handle -je_auto_control.send_mouse_event_to_window(window_handle, mouse_keycode="mouse_left", x=100, y=50) -``` - -### GUI Application - -Launch the built-in graphical interface (requires `[gui]` extra): - -```python -import je_auto_control -je_auto_control.start_autocontrol_gui() -``` - -Or from the command line: - -```bash -python -m je_auto_control -``` - -The main window is menu-driven: tabs hold only their inputs, tables, -and result views, and every tab's commands live in the window-level -**Actions** menu, which rebuilds for the active tab. **View → Tabs** -shows or hides any of the ~48 registered tabs, grouped by category -(Core / Editing / Detection & Vision / Automation Engines / System); -the default layout opens with just Record, Script Builder, and Remote -Desktop. **View → Text Size** offers auto/preset font scaling, and the -**Language** menu (English / 繁體中文 / 简体中文 / 日本語) retranslates -the whole window live. - --- -## Command-Line Interface - -AutoControl can be used directly from the command line: - -```bash -# Execute a single action file -python -m je_auto_control -e actions.json - -# Execute all action files in a directory -python -m je_auto_control -d ./action_files/ +## Platform support -# Execute a JSON string directly -python -m je_auto_control --execute_str '[["AC_screenshot", {"file_path": "test.png"}]]' +| Platform | Backend | Input | Screen capture | Recording | Window management | +|---|---|:---:|:---:|:---:|:---:| +| Windows 10 / 11 | Win32 ctypes (+ optional Interception driver) | ✅ | ✅ | ✅ | ✅ | +| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | +| Linux X11 | python-Xlib (+ optional `uinput`) | ✅ | ✅ | ✅ | ❌ | +| Linux Wayland | libei, or ydotool / wtype / grim | ✅ | ✅ | ❌ | ❌ | +| Android | adb + uiautomator2 | ✅ | ✅ | — | — | +| iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | -# Create a project template -python -m je_auto_control -c ./my_project -``` - -A richer subcommand CLI built on the headless APIs: - -```bash -# Run a script, optionally with variables, and/or a dry-run -python -m je_auto_control.cli run script.json -python -m je_auto_control.cli run script.json --var name=alice --dry-run - -# List scheduler jobs -python -m je_auto_control.cli list-jobs - -# Start the socket or REST server -python -m je_auto_control.cli start-server --port 9938 -python -m je_auto_control.cli start-rest --port 9939 -``` - -`--var name=value` is parsed as JSON when possible (so `count=10` becomes -an int), otherwise treated as a string. +Wayland forbids global input recording for unprivileged clients — set +`JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` to record on an X11 session. Window +management is currently Windows-only and raises a clear `NotImplementedError` +elsewhere. Opt-in driver-level backends (`JE_AUTOCONTROL_WIN32_BACKEND=interception`, +`JE_AUTOCONTROL_LINUX_BACKEND=uinput`, ViGEm virtual gamepad) exist for apps that +ignore synthetic input, and fall back silently when the driver is absent. --- -## Platform Support +## Documentation and examples -| Platform | Status | Backend | Notes | -|---|---|---|---| -| Windows 10 / 11 | Supported | Win32 API (ctypes) | Full feature support | -| macOS 10.15+ | Supported | pyobjc / Quartz | Action recording not available; `send_key_event_to_window` / `send_mouse_event_to_window` not supported | -| Linux (X11) | Supported | python-Xlib | Full feature support | -| Linux (Wayland) | Not supported | — | May be added in a future release | -| Raspberry Pi 3B / 4B | Supported | python-Xlib | Runs on X11 | +| Resource | What's in it | +|---|---| +| [`examples/`](examples/) | 27 self-contained scripts: screenshot + click, OCR, scheduler, remote desktop, agent loop, observability, recording, variables, hotkeys, triggers, reports, MCP, REST, secrets, plugins, computer use, Wayland, cross-host DAGs, chat-ops, pytest/BDD, anchor locators. | +| [Read the Docs](https://autocontrol.readthedocs.io/en/latest/) | Full API reference, English and 中文. | +| [architecture_explore.md](architecture_explore.md) | Every module's responsibility, layer by layer. | +| [docs/CAPABILITY_MATRIX.md](docs/CAPABILITY_MATRIX.md) | Capability × platform matrix. | +| [docs/API_LIFECYCLE.md](docs/API_LIFECYCLE.md) | Stable-API and deprecation policy. | +| [WHATS_NEW.md](WHATS_NEW.md) | Per-release notes. | +| [CHANGELOG.md](CHANGELOG.md) | Compatibility changelog. | +| [SECURITY.md](SECURITY.md) | Security policy and reporting. | --- ## Development -### Setting Up - ```bash git clone https://github.com/Intergration-Automation-Testing/AutoControl.git cd AutoControl pip install -r dev_requirements.txt +uv sync # or: reproducible install from the committed uv.lock ``` -Reproducible installs use the committed `uv.lock`: - ```bash -uv sync # install pinned versions across the whole dep tree -uv lock --upgrade # refresh after editing pyproject.toml -``` +python -m pytest test/unit_test/headless # headless unit tests +python -m pytest test/integrated_test/ # cross-module workflows -### Running Tests - -```bash -# Unit tests -python -m pytest test/unit_test/ - -# Integration tests -python -m pytest test/integrated_test/ +ruff check je_auto_control/ +pylint je_auto_control/ +bandit -c pyproject.toml -r je_auto_control/ ``` -### Project Links - -- [Capability and platform matrix](docs/CAPABILITY_MATRIX.md) -- [Public API lifecycle and deprecation policy](docs/API_LIFECYCLE.md) -- [Security policy](SECURITY.md) -- [Compatibility changelog](CHANGELOG.md) - -- **Homepage**: https://github.com/Intergration-Automation-Testing/AutoControl -- **Documentation**: https://autocontrol.readthedocs.io/en/latest/ -- **PyPI**: https://pypi.org/project/je_auto_control/ +Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md) and +[CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md). Two rules the CI enforces: `import +je_auto_control` must never pull in PySide6, and every feature needs both a headless +API and a GUI surface. --- ## License [MIT License](LICENSE) © JE-Chen. -See [Third_Party_License.md](Third_Party_License.md) for the licenses of -bundled and optional third-party dependencies. +See [Third_Party_License.md](Third_Party_License.md) for the licenses of bundled and +optional third-party components. + +- **Homepage**: https://github.com/Intergration-Automation-Testing/AutoControl +- **PyPI**: https://pypi.org/project/je_auto_control/ +- **Documentation**: https://autocontrol.readthedocs.io/en/latest/ diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index be26df10..d23c68ca 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -3,1339 +3,289 @@ [![PyPI](https://img.shields.io/pypi/v/je_auto_control)](https://pypi.org/project/je_auto_control/) [![Python](https://img.shields.io/pypi/pyversions/je_auto_control)](https://pypi.org/project/je_auto_control/) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](../LICENSE) +[![Documentation](https://readthedocs.org/projects/autocontrol/badge/?version=latest)](https://autocontrol.readthedocs.io/en/latest/?badge=latest) -**AutoControl** 是一个跨平台的 Python GUI 自动化框架，提供鼠标控制、键盘输入、图像识别、屏幕捕获、脚本执行与报告生成等功能 — 通过统一的 API 在 Windows、macOS 和 Linux (X11) 上运行。 +**AutoControl** 是一套跨平台的 Python GUI 自动化框架。它能驱动鼠标与键盘、在画面上找到目标 +（模板匹配、OCR、操作系统无障碍树，或视觉模型）、录制与回放操作流程，并以 JSON 动作文件执行—— +支持 Windows、macOS、Linux（X11 与 Wayland）、Android 与 iOS。 -**[English](../README.md)** | **[繁體中文](README_zh-TW.md)** +每项能力都以三种形式提供：**Python API**、可在 JSON 文件／CLI／服务器使用的 **`AC_*` 动作命令**， +以及 **GUI 标签页**。没有任何功能只存在于 GUI。 ---- - -## 目录 - -- [本次更新](#本次更新) -- [功能特性](#功能特性) -- [架构](#架构) -- [安装](#安装) -- [系统要求](#系统要求) -- [快速开始](#快速开始) - - [鼠标控制](#鼠标控制) - - [键盘控制](#键盘控制) - - [图像识别](#图像识别) - - [Accessibility 元件搜索](#accessibility-元件搜索) - - [AI 元件定位（VLM）](#ai-元件定位vlm) - - [OCR 屏幕文字识别](#ocr-屏幕文字识别) - - [LLM 动作规划器](#llm-动作规划器) - - [运行期变量与流程控制](#运行期变量与流程控制) - - [远程桌面](#远程桌面) - - [剪贴板](#剪贴板) - - [截图](#截图) - - [动作录制与回放](#动作录制与回放) - - [JSON 脚本执行器](#json-脚本执行器) - - [MCP 服务器（让 Claude 使用 AutoControl）](#mcp-服务器让-claude-使用-autocontrol) - - [调度器（Interval & Cron）](#调度器interval--cron) - - [全局热键](#全局热键) - - [事件触发器](#事件触发器) - - [执行历史](#执行历史) - - [报告生成](#报告生成) - - [可观测性（Prometheus / OpenTelemetry）](#可观测性prometheus--opentelemetry) - - [远程自动化（Socket / REST）](#远程自动化socket--rest) - - [插件加载器](#插件加载器) - - [Shell 命令执行](#shell-命令执行) - - [屏幕录制](#屏幕录制) - - [回调执行器](#回调执行器) - - [包管理器](#包管理器) - - [项目管理](#项目管理) - - [窗口管理](#窗口管理) - - [GUI 应用程序](#gui-应用程序) -- [命令行界面](#命令行界面) -- [平台支持](#平台支持) -- [开发](#开发) -- [许可证](#许可证) +**[English](../README.md)** · **[繁體中文](README_zh-TW.md)** --- -## 本次更新 - -**最新（2026-07-18）— 跨平台稳定性强化。** 一次全项目运行期审计修正了 macOS／Windows／Linux／Wayland 各后端、执行器,以及远程桌面／USB 堆栈的运行期缺陷——包含正确的 Retina 光标坐标运算、CPython 3.14 上的中继卡死、`AC_expect_poll`／`AC_parallel` 的健壮性、USB/IP 默认绑定本机,以及在 I/O 边界保留类型化异常——每项均有 headless 回归测试覆盖。无 API 变更。 - -各版本更新说明详见 **[WHATS_NEW.md](../WHATS_NEW.md)**。 - -## 功能特性 - -- **QA / 测试框架** — 断言 DSL（`assert_text` / `_image` / `_pixel` / `_window` / `_clipboard` / `_process` / `_file` / `_http` 加上音频/视频断言,以及 `assert_all` / `assert_any` / `assert_eventually` 组合器）、数据驱动执行（CSV / JSON / SQLite / Excel → `AC_for_each_row`）、具 setup/teardown/标签的计分 `run_suite`、JUnit + Allure 报告输出、不稳定测试检测与自动隔离、无障碍 / i18n 审计（缺失标签、WCAG 对比度、截断），以及并行的移动设备矩阵。详见 [本次更新 (2026-06)](WHATS_NEW_zh-CN.md) -- **鼠标自动化** — 移动、点击、按下、释放、拖拽、滚动，支持精确坐标控制 -- **键盘自动化** — 按下/释放单一按键、输入字符串、组合键、按键状态检测 -- **图像识别** — 使用 OpenCV 模板匹配在屏幕上定位 UI 元素，支持可配置的检测阈值 -- **Accessibility 元件搜索** — 通过操作系统无障碍树（Windows UIA / macOS AX）按名称/角色定位按钮、菜单、控件 -- **AI 元件定位（VLM）** — 用自然语言描述 UI 元素，由视觉语言模型（Anthropic / OpenAI）返回屏幕坐标 -- **OCR** — 三个可插拔后端（Tesseract 用于 ASCII、EasyOCR 无外部可执行文件且支持 CJK、PaddleOCR 中／日／韩质量最高），统一 API 与标准语言代码；后端由 `backend=` 参数、`AUTOCONTROL_OCR_BACKEND` 环境变量或自动探测决定。可搜索、点击或等待文字出现；支持 regex 搜索与整块区域 dump -- **LLM 动作规划器** — 用 Claude 把自然语言描述翻译成验证过的 `AC_*` 动作清单 -- **运行期变量与流程控制** — 执行时 `${var}` 替换，加上 `AC_set_var` / `AC_inc_var` / `AC_if_var` / `AC_for_each` / `AC_loop` / `AC_while_var` / `AC_retry` / `AC_try` 让脚本数据驱动。`AC_while_var` 在变量比较成立时持续循环（每轮重新判断，`max_iter` 安全上限）；`AC_try` 提供 try/catch/finally：`body` 失败时改走 `catch` 恢复分支而非中止、`finally` 必定执行、错误通过 `error_var` 暴露、可在清理后 `reraise`（循环 `break`/`continue` 仍能穿透） -- **远程桌面** — 用 token 认证的 TCP 协议串流本机画面并接收输入，**或** 连接到他机观看与控制（host + viewer GUI 内置）。可选 TLS（HTTPS 级加密）、WebSocket 传输（``ws://`` + ``wss://``，穿墙／浏览器友好）、持久化 9 位数 Host ID、host→viewer 音频串流、双向剪贴板同步（文字 + 图片）、分块文件传输（拖放 + 进度条；任意目的路径；无大小上限）。另含文件夹同步（增量镜像 — 本地删除不会传出去）与自建 coturn TURN 配置包生成器（turnserver.conf + systemd unit + docker-compose + README）。**AnyDesk 风格弹出窗口**：viewer 认证成功后远程桌面会开在独立的可调整大小顶层窗口，控制面板保持简洁；Remote Desktop 子分页外层包了 `QScrollArea`，小窗口下可滚动、4K 屏幕下会铺满。同时支持 headless API 与 MCP 工具 (`ac_remote_*`) 直接驱动 -- **驱动级输入后端（可选）** — 针对忽略 SendInput（Win）或 XTest（Linux）的游戏／应用：**Interception driver 后端**（Windows，HID 层鍵鼠注入，使用 Oblita WHQL-signed driver，通过 `JE_AUTOCONTROL_WIN32_BACKEND=interception` 启用）、**uinput 后端**（Linux，kernel `/dev/uinput` 合成 HID 设备，通过 `JE_AUTOCONTROL_LINUX_BACKEND=uinput` 启用），以及 **ViGEm 虚拟手柄**（Windows，针对只认手柄的游戏，提供虚拟 Xbox 360 手柄 + 友善的 button / dpad / stick / trigger API，并暴露为 `AC_gamepad_*` 执行器命令与 `ac_gamepad_*` MCP 工具）。三者在 driver 没装时都会优雅 fallback，不影响既有部署 -- **剪贴板** — 于 Windows / macOS / Linux 读写系统剪贴板文本 -- **截图与屏幕录制** — 捕获全屏或指定区域为图片，录制屏幕为视频（AVI/MP4） -- **动作录制与回放** — 录制鼠标/键盘事件并重新播放 -- **JSON 脚本执行** — 使用 JSON 动作文件定义并执行自动化流程（支持 dry-run 与逐步调试） -- **调度器** — 以 interval 或 cron 表达式执行脚本，两类调度可同时存在 -- **全局热键** — 跨平台绑定 OS 热键到 action 脚本：Windows (`RegisterHotKey`)、macOS (`CGEventTap`，需 Accessibility 权限)、Linux X11 (`XGrabKey`，含 NumLock / CapsLock 变体掩码)。Wayland 不支持。三个平台共享同一个 API；`backends/` 在 `start()` 时自动挑后端 -- **事件触发器** — 检测到图像出现、窗口出现、像素变化或文件变动时自动执行脚本 -- **执行历史** — 使用 SQLite 记录 scheduler / triggers / hotkeys / REST 的执行结果；错误时自动附带截图 -- **报告生成** — 将测试记录导出为 HTML、JSON 或 XML 报告，包含成功/失败状态 -- **MCP 服务器** — JSON-RPC 2.0 Model Context Protocol 服务（stdio + HTTP/SSE），让 Claude Desktop / Claude Code / 自定义 tool-use 循环直接驱动 AutoControl。约 100 个工具,完整协议支持(resources、prompts、sampling、roots、logging、progress、cancellation、elicitation),Bearer token 验证 + TLS、审计 log、rate limit、plugin 热加载、CI fake backend。**本次新增** `ac_remote_host_start` / `ac_remote_host_stop` / `ac_remote_host_status` / `ac_remote_viewer_connect` / `ac_remote_viewer_disconnect` / `ac_remote_viewer_status` / `ac_remote_viewer_send_input` 包装 GUI 远程桌面分页所用的 process-global registry，模型可以直接启动 host、连线 viewer、转发 mouse／keyboard／type／hotkey 动作 -- **远程自动化** — TCP Socket 服务器 **加上** 强化版 REST API：bearer token 认证、per-IP 速率限制 + 失败锁定、SQLite 审计 hook、Prometheus `/metrics`、完整端点列表（`/health`、`/screen_size`、`/sessions`、`/screenshot`、`/execute`、`/audit/list`、`/audit/verify`、`/inspector/recent`、`/usb/devices`、`/diagnose`、…），以及 vanilla-JS 的浏览器 dashboard `/dashboard`（任何能 HTTP 连到主机的手机都能监控） -- **插件加载器** — 将定义 `AC_*` 可调用对象的 `.py` 文件放入目录，运行时即可注册为 executor 命令 -- **Shell 集成** — 在自动化流程中执行 Shell 命令，支持异步输出捕获 -- **回调执行器** — 触发自动化函数后自动调用回调函数，实现操作串联 -- **动态包加载** — 在运行时导入外部 Python 包，扩展执行器功能 -- **项目与模板管理** — 快速创建包含 keyword/executor 目录结构的自动化项目 -- **窗口管理** — 直接将键盘/鼠标事件发送至指定窗口（Windows/Linux） -- **GUI 应用程序** — 内置 PySide6 图形界面，支持即时切换语言（English / 繁體中文 / 简体中文 / 日本語） -- **CLI 运行器** — `python -m je_auto_control.cli run|list-jobs|start-server|start-rest` -- **跨平台** — 统一 API，支持 Windows、macOS、Linux（X11 + Wayland）、Android（adb + uiautomator2）、iOS（WebDriverAgent / facebook-wda） -- **截屏 PII 脱敏** — `RedactionEngine` 在截屏上传 VLM、写入 audit log 或通过 REST 返回前，把 email / 信用卡号 / SSN / 电话 / secure-text 字段 / 强制区域模糊掉。通过环境变量 `JE_AUTOCONTROL_REDACTION=off|moderate|strict` 或逐次调用指定策略 -- **多主机管理控制台** — 在一份通讯录中注册 N 个远程 AutoControl REST 端点，并行轮询 health/sessions/jobs，把同一份动作清单广播给全部主机。储存于 `~/.je_auto_control/admin_hosts.json`（POSIX 上模式 0600）。Token 错误的主机会以实际 HTTP 错误显示为不健康 -- **可检测篡改的审计日志** — SQLite events 表加上 SHA-256 哈希链（每条记录含 `prev_hash` + `row_hash`）；修改任何过去记录都会打断哈希链。`verify_chain()` 自顶向下走访并报告第一个断点。既有数据表会在启动时回填（"初次使用即信任"） -- **WebRTC 包监测** — 由既有 WebRTC stats 轮询喂入的进程级 `StatsSnapshot` 滚动窗口（默认 600 条 / 1 Hz 约 10 分钟）。对 RTT、FPS、bitrate、丢包率、jitter 各回 `last/min/max/avg/p95` -- **USB 设备列举** — 只读的跨平台 USB 设备列举。优先尝试 pyusb（libusb）；若无则退回平台特定命令（Windows `Get-PnpDevice`、macOS `system_profiler`、Linux `/sys/bus/usb/devices`）。第二阶段 passthrough 构建于此（见下） -- **系统诊断** — 一键"目前正常吗？"探测：平台、可选依赖包、executor 命令数、审计链、截图、鼠标、磁盘空间、REST registry。CLI 全绿 exit 0／否则 1；REST `/diagnose`；按严重度上色的 GUI 分页 -- **稳定 API 与失败诊断包** — 给新集成用的版本化、延迟加载 `je_auto_control.api` 门面(`execute_action`、`generate_code`、`run_diagnostics`、failure bundles),附[生命周期政策](../docs/API_LIFECYCLE.md)。便携式 `autocontrol.failure-bundle/v1` 诊断 ZIP:manifest + 已脱敏的 context/events/log 尾段、可选截图与诊断、best-effort 收集器、原子写入。CLI `je_auto_control failure-bundle out.zip`;`codegen --failure-bundle` 让生成的 pytest 自动包上失败诊断 -- **USB Hotplug 事件** — 轮询式 hotplug 监测（`UsbHotplugWatcher`），含 bounded ring buffer 与带序号的事件；`GET /usb/events?since=N` 让晚加入的订阅者补上进度。USB 分页有自动刷新切换钮。 -- **OpenAPI 3.1 + Swagger UI** — `GET /openapi.json`（auth-gated，从活的路由表生成）+ `GET /docs`（浏览器版 Swagger UI 含 bearer token 栏）。CI 上有 drift 测试，新加路由忘记写 metadata 会被拦下。 -- **配置包导出／导入** — 单一 JSON 文件，导出／导入用户配置（admin hosts、address book、trusted viewers、known hosts、host service、IDs）。原子写入加 `.bak.<时间戳>` 备份；CLI `python -m je_auto_control.utils.config_bundle export|import`；`POST /config/{export,import}`；REST API 分页的导出／导入命令位于窗口的 Actions 菜单。 -- **USB Passthrough（需主动启用）** — 让远端 viewer 使用实体插在 host 上的 USB 设备，走 WebRTC `usb` DataChannel。Wire-level 协议（11 个 opcode 含 `RESUME`、CREDIT 流量控制、16 KiB payload 上限，超量传输以 EOF 分片）。八个原始未决问题全部解决：可靠有序 channel、LIST 走 channel（ACL 过滤）、per-claim credit、Linux kernel driver detach/reattach、ACL **HMAC-SHA256 完整性**（篡改 fail-closed；密钥可插拔 — Windows DPAPI 或 passphrase vault）。**Backend：**`LibusbBackend`（production）、`WinusbBackend`（ctypes）、`IokitBackend`（原生 IOKit 列举 + libusb 传输）— Windows/macOS *硬件未验证*；`default_passthrough_backend()` 依 OS 自动挑。Viewer 端阻塞式 client（`control/bulk/interrupt_transfer`、`list_devices`、`resume`）；in-process `UsbLoopback` 让同机可走完整堆栈 share+use。**已接入 WebRTC** host/viewer（`viewer.usb_client()`）并含断线可续租的 **resume token**。持久化 ACL（默认 deny、mode 0600），含 host 端 prompt 对话框、滥用 **rate-limit / lockout** 与可检测篡改审计整合。五个驱动面：AnyDesk 风 **GUI 面板**（分享 + ACL 允许/封锁 + 本机/远端使用）、`AC_usb_*` executor 命令（JSON / socket / 调度器）、**REST** `/usb/...`、一级 **MCP** `ac_usb_*` 工具、以及 Python API。默认 off — 用 `enable_usb_passthrough(True)` 或 `JE_AUTOCONTROL_USB_PASSTHROUGH=1` 启用；默认启用仍待 Phase 2e 外部安全签核 + 实机硬件验证。 - ---- - -## 架构 - -运行时是分层的:**客户端接口**(CLI、GUI、MCP/REST/Socket 服务 -器)位于最上层,下面是**无头 API**(`wrapper/` + `utils/`),最后 -解析到 `wrapper/platform_wrapper.py` 在 import 时选定的**操作系统 -后端**。包 façade(`je_auto_control/__init__.py`)会 re-export 所 -有公开名称,使用者只需要 `import je_auto_control`,无论用哪个接口 -或后端都一样。 - -```mermaid -flowchart LR - subgraph Clients["客户端接口"] - direction TB - Claude[["Claude Desktop /
Claude Code"]] - APIUser[["自定义 Anthropic /
OpenAI tool-use 循环"]] - HTTPClient[["HTTP / SSE clients"]] - TCPClient[["Socket / REST clients"]] - Browser[["浏览器
(/dashboard · /docs)"]] - GUIUser[["PySide6 GUI"]] - CLIUser[["python -m
je_auto_control[.cli]"]] - Library[["Library 使用者
(import je_auto_control)"]] - end - - subgraph Transports["传输与服务器"] - direction TB - Stdio["MCP stdio
JSON-RPC 2.0"] - HTTPMCP["MCP HTTP /
SSE + auth + TLS"] - REST["REST 服务器 :9939
bearer auth · rate-limit ·
OpenAPI · /metrics · /dashboard"] - Socket["Socket 服务器
:9938"] - WebRTC["WebRTC sessions
(远程桌面 ·
文件 · 音频 · USB)"] - end - - subgraph MCP["mcp_server/"] - direction TB - Dispatcher["MCPServer
(JSON-RPC dispatcher)"] - Tools["tools/
~90 ac_* + 别名"] - Resources["resources/
files · history ·
commands · screen-live"] - Prompts["prompts/
内置模板"] - Context["context · audit ·
rate-limit · log-bridge"] - FakeBE["fake_backend
(CI 烟雾测试)"] - end - - subgraph Core["无头核心 (wrapper/ + utils/)"] - direction TB - Wrapper["wrapper/
鼠标 · 键盘 · 屏幕 ·
图像 · 录制 · 窗口"] - Executor["executor/
AC_* JSON 动作引擎"] - Vision["vision/ · ocr/ ·
accessibility/"] - Recorder["scheduler/ · triggers/ ·
hotkey/ · plugin_loader/
run_history/"] - IOUtils["clipboard/ · cv2_utils/ ·
shell_process/ · json/"] - end - - subgraph Ops["运维层 (utils/)"] - direction TB - Admin["admin/
多主机轮询 +
广播"] - Audit["remote_desktop/
audit_log
(SHA-256 链)"] - Inspector["remote_desktop/
webrtc_inspector"] - Diag["diagnostics/
自我诊断"] - ConfigB["config_bundle/
导出/导入"] - end - - subgraph USB["USB"] - direction TB - UsbEnum["usb/
列举 + hotplug"] - UsbPass["usb/passthrough/
session · client · ACL(HMAC) ·
libusb · WinUSB · IOKit ·
loopback · webrtc channel · commands"] - end - - subgraph Remote["远程桌面 (utils/remote_desktop/)"] - direction TB - RDHost["host · webrtc_host ·
signaling · multi_viewer"] - RDFiles["webrtc_files · file_sync ·
clipboard_sync · audio"] - RDTrust["trust_list · fingerprint ·
turn_config · lan_discovery"] - end - - subgraph Backends["操作系统后端"] - direction TB - Win["windows/
Win32 ctypes"] - Mac["osx/
pyobjc · Quartz"] - X11["linux_with_x11/
python-Xlib"] - end - - Claude --> Stdio - APIUser --> Stdio - HTTPClient --> HTTPMCP - TCPClient --> Socket - TCPClient --> REST - Browser --> REST - - Stdio --> Dispatcher - HTTPMCP --> Dispatcher - Dispatcher --> Tools - Dispatcher --> Resources - Dispatcher --> Prompts - Dispatcher -.- Context - Tools -.可选.-> FakeBE - - Tools --> Wrapper - Tools --> Executor - Tools --> Vision - Tools --> Recorder - Tools --> IOUtils - Resources --> Recorder - Resources --> Wrapper - - REST --> Executor - REST --> Ops - REST --> USB - Socket --> Executor - WebRTC --> Remote - WebRTC --> UsbPass - - GUIUser --> Wrapper - GUIUser --> Recorder - GUIUser --> Ops - GUIUser --> USB - GUIUser --> Remote - CLIUser --> Executor - Library --> Wrapper - Library --> Executor - Library --> Ops - - Admin --> REST - Inspector -.- WebRTC - Audit -.- REST - Audit -.- USB - UsbPass --> Backends +## 为什么选择 AutoControl - Wrapper --> Backends - Vision -.- Wrapper - Recorder -.- Executor -``` - -``` -je_auto_control/ -├── wrapper/ # 平台无关 API 层 -│ ├── platform_wrapper.py # 自动检测操作系统并加载对应后端 -│ ├── auto_control_mouse.py # 鼠标操作 -│ ├── auto_control_keyboard.py# 键盘操作 -│ ├── auto_control_image.py # 图像识别（OpenCV 模板匹配） -│ ├── auto_control_screen.py # 截图、屏幕大小、像素颜色 -│ ├── auto_control_window.py # 跨平台窗口管理 facade -│ └── auto_control_record.py # 动作录制/回放 -├── windows/ # Windows 专用后端（Win32 API / ctypes） -├── osx/ # macOS 专用后端（pyobjc / Quartz） -├── linux_with_x11/ # Linux 专用后端（python-Xlib） -├── gui/ # PySide6 GUI 应用程序 -└── utils/ - ├── mcp_server/ # MCP 服务器（stdio + HTTP/SSE）— server / tools / resources / prompts / audit / rate_limit / fake_backend / plugin_watcher - ├── executor/ # JSON 动作执行引擎 - ├── callback/ # 回调函数执行器 - ├── cv2_utils/ # OpenCV 截图、模板匹配、视频录制 - ├── accessibility/ # UIA (Windows) / AX (macOS) 元件搜索 - ├── vision/ # VLM 元件定位（Anthropic / OpenAI） - ├── ocr/ # Tesseract 文字定位 - ├── clipboard/ # 跨平台剪贴板（文字 + 图像） - ├── llm/ # 自然语言 → AC_* 动作规划器 - ├── scheduler/ # Interval + cron 调度器 - ├── hotkey/ # 全局热键守护进程 - ├── triggers/ # 图像/窗口/像素/文件 触发器 - ├── run_history/ # SQLite 执行记录 + 错误截图 - ├── rest_api/ # 纯 stdlib HTTP/REST 服务器 — auth · audit · rate-limit · OpenAPI · /metrics · dashboard · Swagger UI - ├── admin/ # 多主机 AdminConsoleClient（轮询 + 广播） - ├── diagnostics/ # 系统自我诊断 + CLI - ├── config_bundle/ # 单文件用户配置导出／导入 - ├── usb/ # 跨平台列举、hotplug 事件、passthrough/{protocol, session, viewer client, loopback, webrtc channel, ACL+HMAC, descriptor, key providers, commands, libusb / WinUSB / IOKit} - ├── remote_desktop/ # WebRTC host + viewer、signalling、multi-viewer、文件／剪贴板／音频同步、审计日志（哈希链）、信任列表、TURN 配置、mDNS 发现、WebRTC stats inspector - ├── plugin_loader/ # 动态 AC_* 插件搜索与注册 - ├── socket_server/ # TCP Socket 服务器（远程自动化） - ├── shell_process/ # Shell 命令管理器 - ├── generate_report/ # HTML / JSON / XML 报告生成器 - ├── test_record/ # 测试动作记录 - ├── script_vars/ # 脚本变量插值 - ├── watcher/ # 鼠标 / 像素 / log 监视器（Live HUD） - ├── recording_edit/ # 录制内容的裁剪、过滤、缩放 - ├── json/ # JSON 动作文件读写 - ├── project/ # 项目创建与模板 - ├── package_manager/ # 动态包加载 - ├── logging/ # 日志记录 - └── exception/ # 自定义异常类 -``` - -`platform_wrapper.py` 模块会自动检测当前的操作系统并导入对应的后端，因此所有 wrapper 函数在不同平台上的行为完全一致。 +- **一套 API，六个平台。** `wrapper/platform_wrapper.py` 在导入时挑选后端；同一份脚本在 + Windows、macOS、X11 与 Wayland 上都不需要改写。 +- **不写 Python 也能脚本化。** 767 个 `AC_*` 命令覆盖全部功能，因此一个 JSON 文件能做到库 + 能做的任何事——包含循环、分支、try/catch、宏与变量。 +- **默认无头运行。** `import je_auto_control` 绝不会加载 Qt。GUI 是可选包，包在同一个无头内核之外。 +- **四种定位方式。** 模板匹配、OCR、无障碍树、视觉语言模型——可通过锚点定位器与自愈回退串接组合。 +- **依赖基线轻量。** REST 服务器、JSON Schema 校验、JWT、TOTP、WebSocket 帧、ACME 客户端、 + USB/IP 协议与 Prometheus 指标全部以标准库实现；较重的依赖都是可选项。 --- ## 安装 -### 基本安装 - ```bash -pip install je_auto_control +pip install je_auto_control # 内核 +pip install je_auto_control[gui] # 加上 PySide6 桌面应用 ``` -### 安装 GUI 支持（PySide6） +按需安装的可选组件： -```bash -pip install je_auto_control[gui] -``` - -### Linux 前置要求 +| Extra | 启用的功能 | +|---|---| +| `gui` | PySide6 桌面应用（48 个标签页） | +| `webrtc` | WebRTC 远程桌面、USB 直通（`aiortc`、`av`） | +| `signaling` | 独立的信令／rendezvous 服务器（`fastapi`、`uvicorn`） | +| `discovery` | mDNS / Zeroconf 局域网主机发现 | +| `pdf` / `office` | PDF 与 Excel／Word／PowerPoint 读取 | +| `fuzzy` / `locale` | `rapidfuzz` 模糊匹配、`babel` 区域解析 | +| `s3` / `audio` | S3 制品存储、系统音量控制 | -在 Linux 上安装前，请先安装以下系统包： +**系统需求：** Python ≥ 3.10。Linux 请先安装构建依赖： ```bash sudo apt-get install cmake libssl-dev ``` ---- - -## 系统要求 - -- **Python** >= 3.10 -- **pip** >= 19.3 - -### 依赖包 - -| 包 | 用途 | -|---|---| -| `je_open_cv` | 图像识别（OpenCV 模板匹配） | -| `pillow` | 截图捕获 | -| `mss` | 快速多屏幕截图 | -| `pyobjc` | macOS 后端（在 macOS 上自动安装） | -| `python-Xlib` | Linux X11 后端（在 Linux 上自动安装） | -| `PySide6` | GUI 应用程序（可选，使用 `[gui]` 安装） | -| `qt-material` | GUI 主题（可选，使用 `[gui]` 安装） | -| `uiautomation` | Windows Accessibility 后端（可选，首次使用时加载） | -| `pytesseract` + Tesseract | OCR 文字识别（可选，首次使用时加载） | -| `anthropic` | VLM 定位 — Anthropic 后端（可选，首次使用时加载） | -| `openai` | VLM 定位 — OpenAI 后端（可选，首次使用时加载） | - -完整第三方依赖及其许可证请见 [Third_Party_License.md](../Third_Party_License.md)。 +OCR、VLM 与 LLM 后端（`pytesseract`、`easyocr`、`paddleocr`、`anthropic`、`openai`） +都是按需加载——只装你实际会用到的。 --- -## 快速开始 - -想要可以直接复制粘贴的完整脚本而不只是 API 片段？ -[`examples/`](../examples/) 目录收录 17 个独立示例：截屏+点击、OCR、 -调度器、远程桌面、agent loop、可观测性、录制/回放、运行期变量、 -窗口管理、热键、图像触发器、HTML 报告、MCP stdio bridge、REST API、 -secret vault，以及插件加载。 +## 60 秒上手 -### 鼠标控制 - -```python -import je_auto_control - -# 获取当前鼠标位置 -x, y = je_auto_control.get_mouse_position() -print(f"鼠标位置: ({x}, {y})") - -# 移动鼠标到指定坐标 -je_auto_control.set_mouse_position(500, 300) - -# 在当前位置左键点击（使用按键名称） -je_auto_control.click_mouse("mouse_left") - -# 在指定坐标右键点击 -je_auto_control.click_mouse("mouse_right", x=800, y=400) - -# 向下滚动 -je_auto_control.mouse_scroll(scroll_value=5) -``` - -### 键盘控制 - -```python -import je_auto_control - -# 按下并释放单一按键 -je_auto_control.type_keyboard("a") - -# 逐字输入整个字符串 -je_auto_control.write("Hello World") - -# 组合键（例如 Ctrl+C） -je_auto_control.hotkey(["ctrl_l", "c"]) - -# 检查某个按键是否正在被按下 -is_pressed = je_auto_control.check_key_is_press("shift_l") -``` - -### 图像识别 - -```python -import je_auto_control - -# 在屏幕上找出所有匹配的图像 -positions = je_auto_control.locate_all_image("button.png", detect_threshold=0.9) -# 返回: [[x1, y1, x2, y2], ...] - -# 找出单一图像并获取其中心坐标 -cx, cy = je_auto_control.locate_image_center("icon.png", detect_threshold=0.85) -print(f"找到位置: ({cx}, {cy})") - -# 找出图像并自动点击 -je_auto_control.locate_and_click("submit_button.png", mouse_keycode="mouse_left") -``` - -### Accessibility 元件搜索 - -通过操作系统无障碍树按名称/角色/App 搜索控件（Windows UIA，via -`uiautomation`；macOS AX）。 - -```python -import je_auto_control - -# 列出 Calculator 中所有可见按钮 -elements = je_auto_control.list_accessibility_elements(app_name="Calculator") - -# 查找特定元件 -ok = je_auto_control.find_accessibility_element(name="OK", role="Button") -if ok is not None: - print(ok.bounds, ok.center) - -# 一步定位并点击 -je_auto_control.click_accessibility_element(name="OK", app_name="Calculator") -``` - -当前平台无可用后端时会抛出 `AccessibilityNotAvailableError`。 - -### AI 元件定位（VLM） - -当模板匹配与 Accessibility 都失效时，可用自然语言描述元件，交给视觉 -语言模型返回坐标。 - -```python -import je_auto_control - -# 默认优先 Anthropic（若已设置 ANTHROPIC_API_KEY），否则使用 OpenAI -x, y = je_auto_control.locate_by_description("绿色的 Submit 按钮") - -# 一步定位并点击 -je_auto_control.click_by_description( - "Cookie 横幅上的『全部接受』按钮", - screen_region=[0, 800, 1920, 1080], # 可选：只在该区域内搜索 -) -``` - -配置（仅从环境变量读取 — 密钥不会写入代码或日志）： - -| 变量 | 作用 | -|---|---| -| `ANTHROPIC_API_KEY` | 启用 Anthropic 后端 | -| `OPENAI_API_KEY` | 启用 OpenAI 后端 | -| `AUTOCONTROL_VLM_BACKEND` | 强制指定 `anthropic` 或 `openai` | -| `AUTOCONTROL_VLM_MODEL` | 覆盖默认模型（如 `claude-opus-4-7`、`gpt-4o-mini`） | - -若两个 SDK 均未安装或未设置 API key，会抛出 `VLMNotAvailableError`。 - -### OCR 屏幕文字识别 +**1. 作为 Python 库** ```python import je_auto_control as ac -# 查找所有匹配的文字位置 -matches = ac.find_text_matches("Submit") - -# 获取第一个匹配的中心坐标（找不到返回 None） -cx, cy = ac.locate_text_center("Submit") - -# 一步定位并点击 -ac.click_text("Submit") +ac.set_mouse_position(500, 300) +ac.click_mouse("mouse_left") +ac.write("Hello World") +ac.hotkey(["ctrl_l", "s"]) -# 等待文字出现（或 timeout） -ac.wait_for_text("加载完成", timeout=15.0) +x, y = ac.locate_image_center("save_button.png", detect_threshold=0.9) +ac.click_text("Submit") # OCR +ac.click_accessibility_element(name="OK") # 无障碍树 +ac.click_by_description("the green Submit button") # 视觉模型 +ac.screenshot("shot.png", screen_region=[0, 0, 800, 600]) ``` -选择后端 — 设置 ``AUTOCONTROL_OCR_BACKEND=tesseract|easyocr|paddleocr`` -或在调用时传入 ``backend=``；都不设置时会自动挑第一个 import 成功的： +**2. 作为 JSON 动作文件** — `flow.json` -```python -ac.find_text_matches("登录", lang="chi_sim", backend="easyocr") -ac.click_text("Sign in", backend="tesseract") +```json +[ + ["AC_set_var", {"name": "user", "value": "alice"}], + ["AC_locate_and_click", {"image": "login.png", "mouse_keycode": "mouse_left"}], + ["AC_write", {"write_string": "${user}"}], + ["AC_retry", {"max_attempts": 3, "body": [ + ["AC_wait_text", {"target": "Welcome", "timeout": 10}] + ]}], + ["AC_assert_text", {"text": "Welcome"}], + ["AC_generate_html_report", {"html_name": "report"}] +] ``` -若 Tesseract 不在 `PATH` 中，可手动指定路径： - -```python -ac.set_tesseract_cmd(r"C:\Program Files\Tesseract-OCR\tesseract.exe") +```bash +je_auto_control run flow.json --var user=bob +je_auto_control run flow.json --dry-run # 只列出步骤，不会真的动鼠标 ``` -各后端安装路径与标准语言代码表见 -[docs/source/Eng/doc/ocr_backends/ocr_backends_doc.rst](../docs/source/Eng/doc/ocr_backends/ocr_backends_doc.rst) -或[繁体中文版本](../docs/source/Zh/doc/ocr_backends/ocr_backends_doc.rst)。 - -把区域（或整屏）内所有识别到的文字 dump 出来，或用 regex 搜索变动内容： - -```python -import je_auto_control as ac +**3. 作为桌面应用** -# TextMatch 列表，含文字、边界框、置信度 -for match in ac.read_text_in_region(region=[0, 0, 800, 600]): - print(match.text, match.center, match.confidence) - -# Regex（接受字符串或 compiled re.Pattern） -for match in ac.find_text_regex(r"Order#\d+"): - print(match.text, match.center) +```bash +pip install je_auto_control[gui] +python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui() ``` -GUI：**OCR Reader** 分页。 +录制一段流程、在可视化 Script Builder 里编辑，然后存成 CLI 能直接执行的同一种 JSON 格式。 -### LLM 动作规划器 +--- -把自然语言描述交给 LLM（默认 Anthropic Claude），翻译成验证过的 `AC_*` 动作清单。输出采用宽松解析（剥 code fence、从散文中抽出第一个 JSON array），再用 executor 同样的 schema 验证，所以结果可以直接喂给 `execute_action`： +## 能力总览 -```python -import je_auto_control as ac -from je_auto_control.utils.executor.action_executor import executor +每一行都能无头执行。“GUI 标签页”是同一功能在桌面应用中的位置；标签页的命令都放在窗口的 +**Actions** 菜单里。 -actions = ac.plan_actions( - "点击 Submit 按钮，然后输入 'done' 并保存", - known_commands=executor.known_commands(), -) -executor.execute_action(actions) +| 能力 | Python API | `AC_*` 命令 | GUI 标签页 | +|---|---|---|---| +| 鼠标 | `click_mouse`、`set_mouse_position`、`mouse_scroll` | `AC_click_mouse` | Auto Click | +| 键盘 | `write`、`hotkey`、`type_keyboard` | `AC_write`、`AC_hotkey` | Auto Click | +| 屏幕与像素 | `screenshot`、`screen_size`、`get_pixel` | `AC_screenshot` | Screenshot | +| 图像匹配 | `locate_image_center`、`locate_and_click` | `AC_locate_and_click` | Image Detect | +| OCR 文字 | `click_text`、`wait_for_text`、`read_text_in_region` | `AC_click_text`、`AC_wait_text` | OCR Reader | +| 无障碍树 | `find_accessibility_element`、`click_accessibility_element` | `AC_a11y_find`、`AC_a11y_click` | Accessibility | +| 视觉模型定位 | `locate_by_description`、`click_by_description` | `AC_vlm_locate`、`AC_vlm_click` | VLM | +| 锚点定位 | — | `AC_anchor_click`、`AC_anchor_locate` | — | +| 自愈定位器 | `self_heal_click`、`self_heal_locate` | `AC_self_heal_click` | Self-Healing | +| 自然语言规划 | `plan_actions`、`run_from_description` | `AC_llm_plan` | LLM Planner | +| Computer-use agent | `AgentLoop`、`run_agent` | `AC_run_agent` | Computer Use | +| 录制与回放 | `record`、`stop_record` | `AC_record`、`AC_stop_record` | Record | +| JSON 脚本 | `execute_action`、`execute_files` | 全部 767 个命令 | Script、Script Builder | +| 变量与流程控制 | `execute_action_with_vars` | `AC_set_var`、`AC_loop`、`AC_for_each`、`AC_try`、`AC_retry` | Variables | +| 数据驱动执行 | — | `AC_for_each_row`（CSV／JSON／SQLite／Excel） | Data Sources | +| 断言 | `assert_text`、`assert_image` | `AC_assert_text` 等 21 个 | Assertions | +| 测试套件 | `run_suite` | `AC_run_suite` | Test Suites | +| 调度（间隔 + cron） | `default_scheduler` | — | Scheduler | +| 全局热键 | `default_hotkey_daemon` | — | Hotkeys | +| 事件触发 | `default_trigger_engine` | `AC_email_trigger_add` | Triggers、Webhooks、Email | +| 窗口管理 *(仅 Windows)* | `list_windows`、`focus_window` | `AC_focus_window`、`AC_snap_window` | Window Manager | +| 剪贴板（文本 + 图片） | `get_clipboard`、`set_clipboard`、`get_clipboard_image`、`set_clipboard_image` | `AC_clipboard_get`、`AC_clipboard_set`、`AC_clipboard_get_image`、`AC_clipboard_set_image` | — | +| 远程桌面 | `RemoteDesktopHost`、`RemoteDesktopViewer` | `AC_start_remote_host`、`AC_remote_connect` | Remote Desktop | +| USB 枚举与直通 | `list_usb_devices`、`enable_usb_passthrough` | `AC_usb_*`（16 个命令） | USB Devices、USB Share | +| 密钥保险库 | `default_secret_manager` | `AC_secret_set` + `${secrets.NAME}` | Secrets | +| 报表（HTML／JSON／XML） | `generate_html_report` | `AC_generate_html_report` | Report | +| 运行历史 | — | — | Run History | +| 指标与追踪 | `default_metric_registry`、`render_metrics_text` | — | — | +| 系统诊断 | `run_diagnostics` | `AC_diagnose` | Diagnostics | +| 测试代码生成 | `generate_code` | — | — | + +除了这张表，`utils/` 下还有 308 个无头包，覆盖断言、韧性、数据质量、i18n 审计、脱敏、 +治理、可观测性等等。完整的逐模块地图在 **[architecture_explore.md](../architecture_explore.md)**。 -# 或者一行做完： -ac.run_from_description("打开记事本并输入 hello", executor=executor) -``` +--- -| 变量 | 效果 | -|---|---| -| `ANTHROPIC_API_KEY` | 启用 Anthropic 后端 | -| `AUTOCONTROL_LLM_BACKEND` | 强制指定 `anthropic` | -| `AUTOCONTROL_LLM_MODEL` | 覆盖默认模型（如 `claude-opus-4-7`） | +## 命令行界面 -GUI：**LLM Planner** 分页 — 描述输入框与指令清单预览；*Plan*（`QThread` 后台执行）与 *Run plan* 位于窗口的 Actions 菜单。 +```bash +je_auto_control run script.json [--var name=value] [--dry-run] +je_auto_control validate script.json # 别名：lint +je_auto_control fmt script.json [--check] +je_auto_control list-commands [--filter mouse] [--json] +je_auto_control record out.json [--duration 5] +je_auto_control codegen script.json --target pytest -o test_flow.py +je_auto_control failure-bundle failure.zip --error "login timed out" +je_auto_control list-jobs +je_auto_control start-server --port 9938 # TCP socket 服务器 +je_auto_control start-rest --port 9939 # REST API +je_auto_control version +``` + +`--var name=value` 会尽量以 JSON 解析（`count=10` 会变成整数），否则视为字符串。 +旧版 `python -m je_auto_control -e file.json` 入口仍然可用。 -### 运行期变量与流程控制 +--- -executor 改成「每次调用」才解析 `${var}` placeholder（不会事先展平），所以嵌套的 `body` / `then` / `else` 列表会保留 placeholder，每次重复执行时重新绑定。配合新的变量修改命令，脚本可以数据驱动而不需要 Python 黏合： +## 服务器与集成 -```json -[ - ["AC_set_var", {"name": "items", "value": ["alpha", "beta"]}], - ["AC_set_var", {"name": "i", "value": 0}], - ["AC_for_each", { - "items": "${items}", "as": "name", - "body": [ - ["AC_inc_var", {"name": "i"}], - ["AC_if_var", { - "name": "i", "op": "ge", "value": 2, - "then": [["AC_break"]], "else": [] - }] - ] - }] -] -``` +| 接口 | 启动方式 | 说明 | +|---|---|---| +| **MCP 服务器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 670 个工具，供 Claude Desktop／Claude Code／自定义 tool loop 使用。Bearer 认证、TLS、审计日志、限流、插件热重载、CI 假后端。 | +| **REST API** | `je_auto_control start-rest` | Bearer token、按 IP 限流与锁定、SQLite 审计 hook、`/metrics`、`/openapi.json`、`/docs` Swagger UI、`/dashboard`。 | +| **TCP socket 服务器** | `je_auto_control start-server` | 以换行分隔的 JSON 动作列表。默认绑定 `127.0.0.1`。 | +| **pytest 插件** | 安装后自动生效 | 提供 fixture 与供 pytest-bdd／behave 使用的 Gherkin step library。 | +| **语言服务器** | `python -m autocontrol_lsp.server` | 为 `AC_*` 动作 JSON 提供补全与诊断，命令清单直接取自运行期的命令表。 | +| **远程桌面** | `RemoteDesktopHost` 或 GUI | TCP、WebSocket 或 WebRTC；TOTP、信任列表、TURN 配置、文件／剪贴板／音频同步。 | -`AC_if_var` 比较运算符：`eq`、`ne`、`lt`、`le`、`gt`、`ge`、`contains`、`startswith`、`endswith`。GUI：**Variables** 分页 — 实时查看 `executor.variables`，支持单条设置、JSON 批量 seed、清空。 +除非明确指定，所有服务器都绑定在 `127.0.0.1`。 -### 远程桌面 +### 远程桌面的线路协议 -把本机画面串流给别人看 / 控制，**或** 观看并控制别人的机器。协议是 raw TCP 上的长度前缀框架（不引入额外依赖），先做一轮 HMAC-SHA256 challenge / response 认证；认证失败的 viewer 在看到任何画面前就被踢掉。JPEG frame 按照配置的 FPS / 质量产生，通过共享 latest-frame slot 广播给通过认证的 viewers，慢的 viewer 只会丢 frame 而不会卡其他人。Viewer 输入消息是 JSON，host 端用允许列表验证后才通过既有 wrapper 派发。 +把主机开放出去之前值得先了解，而且这一段在其他文档里都没有写。默认传输是**裸 +TCP 上的长度前缀分帧**（不需要额外依赖），连接一开始就是 **HMAC-SHA256 的 +challenge／response 握手**：认证不通过的观看端在拿到任何一帧之前就会被断开。 +JPEG 帧按配置的 FPS 与质量编码，再通过一个共享的**最新帧槽**发给已认证的观看 +端——所以慢的观看端是**丢帧**，不会把其他人一起卡住。观看端发来的输入是 JSON， +会先比对**动作允许列表**才交给既有的输入包装层执行，观看端无法自己发明新的操作。 ```python -# 被远程 — 启动 host 把 token + port 给对方 +# 让别人连进来——启动一个主机，把 token 与 port 给对方 from je_auto_control import RemoteDesktopHost host = RemoteDesktopHost(token="hunter2", bind="127.0.0.1", - port=0, fps=10, quality=70) + port=0, fps=10, quality=70) host.start() print("listening on", host.port, "viewers:", host.connected_clients) ``` ```python -# 控制他机 — 连接 viewer 并发送输入 +# 控制另一台机器——连上去并发送输入 from je_auto_control import RemoteDesktopViewer viewer = RemoteDesktopViewer(host="10.0.0.5", port=51234, token="hunter2", - on_frame=lambda jpeg: ...) + on_frame=lambda jpeg: ...) viewer.connect() viewer.send_input({"action": "mouse_move", "x": 100, "y": 200}) -viewer.send_input({"action": "type", "text": "hello"}) viewer.disconnect() ``` -GUI：**Remote Desktop** 分页默认打开的是 **快速连线**（AnyDesk 风格）— 一边是超大本机 Host ID，另一边一个输入框接受 `host:port`、`ws://`、`wss://` 或 9 位数字 Host ID，搭配 *连接* 与 *开始被远程* 两个主要按钮。近期连线会跨 session 记住。进阶的逐传输子分页（既有 TCP / WS host + viewer、WebRTC host + viewer 含手动 SDP / 自定义编码器 / TLS pinning）仍只差一个 click。WebRTC 子分页采延迟载入，没装 `[webrtc]` extra 也能正常开启整个分页。 - -> ⚠️ 取得 host:port 与 token 的人，等同拥有本机完整鼠标 / 键盘控制权。默认仅绑 `127.0.0.1`；要对外暴露请务必搭配 SSH tunnel 或 TLS 前端。Token 是唯一防线 — 请当作密码保管。 - -**快速连线的 headless API**。撑起 GUI 输入框的 transport coordinator 也对外开放，脚本可以走同样的解析路径： - -```python -from je_auto_control import parse_remote_desktop_target -parse_remote_desktop_target("192.168.1.10:5555") -# ConnectTarget(kind='tcp', host='192.168.1.10', port=5555, ...) -parse_remote_desktop_target("ws://hub:8765/desk") -# ConnectTarget(kind='ws', host='hub', port=8765, path='/desk') -parse_remote_desktop_target("123-456-789") -# ConnectTarget(kind='webrtc_id', host_id='123456789') -``` - -**连接审批 + 仅检视模式**。可选 callback 守住每一个 incoming session，AnyDesk 风格。返回 `"view_only"` admit 但丢掉 viewer 的 `INPUT`；返回 falsy（或 raise）就送 `AUTH_FAIL "rejected by host"`： - -```python -from je_auto_control import RemoteDesktopHost, PendingViewer - -def gate(p: PendingViewer) -> str: - if p.address[0].startswith("10."): - return "view_only" - return "full" # 或 True - -host = RemoteDesktopHost(token="tok", on_pending_viewer=gate) -``` - -**IP 白名单（CIDR + 单一 IP）**。在 TLS / auth 之前就拒绝范围外的对端，攻击者连探测都不行： - -```python -host = RemoteDesktopHost( - token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"], -) -``` - -**一次性分享码** — 额外的 token，认证成功一次后自毁；客服支援流程很好用： - -```python -host = RemoteDesktopHost(token="tok", single_use_tokens=["abc123"]) -host.add_single_use_token("9k4ndx") # 运行时加 -host.revoke_single_use_token("abc123") # 还没被用就先撤销 -``` - -**TOTP 2FA（RFC 6238，纯 stdlib）**。在 token 之上加一层 6 位数字 OTP；host 接受 ±1 时间步的 clock drift： - -```python -from je_auto_control.utils.remote_desktop.totp import ( - generate_secret, generate_code, provisioning_uri, -) -secret = generate_secret() -print(provisioning_uri(secret, account="alice")) # 给 QR code 用的 otpauth:// URI - -host = RemoteDesktopHost(token="tok", totp_secret=secret) -viewer = RemoteDesktopViewer( - host=..., token="tok", totp_code=generate_code(secret), -) -``` - -**多屏幕选择**。指定某一屏幕截取，而非合并虚拟桌面： - -```python -from je_auto_control import list_host_monitors, RemoteDesktopHost -print(list_host_monitors()) -# [{'index': 0, 'is_combined': True, ...}, -# {'index': 1, ...}, -# {'index': 2, ...}] -host = RemoteDesktopHost(token="tok", monitor_index=1) -``` - -**远程光标 overlay**。host 每秒 30 Hz 广播 cursor 位置（静止桌面去重）；viewer 的弹出窗口会在 JPEG 流上叠一个箭头，看得到 host 鼠标位置。可用 `enable_cursor_broadcast=False` 关掉。 - -**多 viewer 协作光标 + 文字 chat**。两个新 message type（`CHAT` 与 `CURSOR` 带 `viewer_id`）。搭配 `MultiViewerHost` 把一个 viewer 的指针 echo 给其他人；chat channel 给操作者之间临时对话用： - -```python -host = RemoteDesktopHost( - token="tok", on_chat=lambda sender, text: print(sender, ":", text), -) -host.broadcast_chat("session starts in 30s") -host.broadcast_viewer_cursor("alice", 200, 300) - -viewer = RemoteDesktopViewer( - host=..., on_chat=lambda s, t: ..., - on_viewer_cursor=lambda vid, x, y: ..., -) -viewer.send_chat("ack") -``` - -**相对鼠标模式（FPS / CAD）**。新输入 action 送 delta 而非绝对坐标： - -```python -viewer.send_input({"action": "mouse_move_relative", "dx": 5, "dy": -3}) -``` - -**动态截取**。capture loop 会 hash 每张编码后的 JPEG；重复 frame 直接跳过，所以静止桌面几乎零带宽。新 viewer 在 auth 后立即拿到最新 frame，不会看到一片黑。 - -**即时统计**（FPS / kbps / 累计 — 3 秒滑动窗口）： - -```python -viewer.stats() -# {'fps': 24.3, 'kbps': 4801.2, 'frames': 720.0, 'bytes': 1.8e7, 'uptime': 30.2} -``` - -**JPEG 序列录影（不需要 PyAV）**。TCP path 的 session 录影：每张 frame 写到磁盘，再加一份 `manifest.json` 让播放器可以原速重放： - -```python -from je_auto_control.utils.remote_desktop.jpeg_recorder import ( - JpegSequenceRecorder, -) -rec = JpegSequenceRecorder("~/recordings/2026-05-23") -rec.start() -viewer = RemoteDesktopViewer(host=..., on_frame=rec.record_frame) -# ... session ... -rec.stop() # 在 .jpg 旁边写出 manifest.json -``` - -**TCP relay（WebRTC fallback）**。当 P2P 失败（严格 NAT、移动 CGNAT、酒店 Wi-Fi），两端都向 relay 主动连线、交换一个 32-byte session ID，relay 在中间互转 bytes。同一模块附 `encode_handshake(role, session_id)` 给 client 用： - -```python -from je_auto_control.utils.remote_desktop.relay import RelayServer -relay = RelayServer(bind="0.0.0.0", port=9000) # NOSONAR # 对外 relay -relay.start() -``` - -**服务安装器（无人值守 host）**。`python -m je_auto_control.utils.remote_desktop.host_service ...` 提供 `configure` / `init` / `run`，以及每个平台的安装命令：`install-windows-service` / `uninstall-windows-service`（需 pywin32）、`generate-launchd` / `uninstall-launchd`、`generate-systemd` / `uninstall-systemd`。 - -**加密传输与替代协议**：传 `ssl_context` 给 `RemoteDesktopHost` 或 `RemoteDesktopViewer` 即套上 TLS。要穿墙／给浏览器接，用内置的 WebSocket 版本（无额外依赖），加 `ssl_context` 即 `wss://`： - -```python -from je_auto_control import ( - WebSocketDesktopHost, WebSocketDesktopViewer, -) -host = WebSocketDesktopHost(token="hunter2", ssl_context=server_ctx) -viewer = WebSocketDesktopViewer( - host="example.com", port=443, token="hunter2", - ssl_context=client_ctx, expected_host_id="123456789", -) -``` - -**持久化 Host ID**：每台 host 有稳定的 9 位数字 ID（存在 `~/.je_auto_control/remote_host_id`），在 `AUTH_OK` 中声明，viewer 通过 `expected_host_id` 验证： - -```python -print(host.host_id) # 例如 "123456789" -viewer = RemoteDesktopViewer( - host=..., port=..., token=..., - expected_host_id="123456789", # 不一致就抛 AuthenticationError -) -``` - -**音频串流（host → viewer）**：可选 `sounddevice` 依赖；host 用 `AudioCaptureConfig` 开启，viewer 端接 `AudioPlayer`（或自己的 callback）： - -```python -from je_auto_control.utils.remote_desktop import AudioCaptureConfig -host = RemoteDesktopHost( - token="tok", - audio_config=AudioCaptureConfig(enabled=True), # 默认 mic -) -# 或指定 loopback / monitor 设备: -# audio_config=AudioCaptureConfig(enabled=True, device=12) - -from je_auto_control.utils.remote_desktop import AudioPlayer -player = AudioPlayer(); player.start() -viewer = RemoteDesktopViewer(host=..., on_audio=player.play) -``` - -**剪贴板同步（文字 + 图片，双向）**：明确调用，没有自动 polling 循环。图片剪贴板在 Windows（CF_DIB via ctypes）和 Linux（`xclip -t image/png`）支持；macOS get 走 Pillow ImageGrab、set 暂时需要 PyObjC。 - -```python -viewer.send_clipboard_text("hello") -viewer.send_clipboard_image(open("logo.png", "rb").read()) -host.broadcast_clipboard_text("greetings") -``` - -**文件传输 + 进度**：双向、分块、目的路径任意、无大小上限；GUI viewer 还可以拖放： - -```python -viewer.send_file( - "local.bin", "/tmp/uploaded.bin", - on_progress=lambda tid, done, total: print(done, total), -) -host.send_file_to_viewers("local.bin", "/tmp/from_host.bin") -``` - -> ⚠️ 路径无限制、大小无上限。任何拿到 token 的人都能把任意文件写到任意位置，也能塞满磁盘 — 必须等同信任 token 持有者，或自己继承 `FileReceiver` 在 `handle_begin` 内验证 dest_path。 - -### 剪贴板 - -```python -import je_auto_control as ac -ac.set_clipboard("hello") -text = ac.get_clipboard() -``` - -后端：Windows（Win32 + ctypes）、macOS（`pbcopy`/`pbpaste`）、Linux -（`xclip` 或 `xsel`）。 - -### 截图 - -```python -import je_auto_control - -# 捕获全屏截图并保存 -je_auto_control.pil_screenshot("screenshot.png") - -# 捕获指定区域的截图 [x1, y1, x2, y2] -je_auto_control.pil_screenshot("region.png", screen_region=[100, 100, 500, 400]) - -# 获取屏幕分辨率 -width, height = je_auto_control.screen_size() - -# 获取指定坐标的像素颜色 -color = je_auto_control.get_pixel(500, 300) -``` - -### 动作录制与回放 - -```python -import je_auto_control -import time - -# 开始录制鼠标和键盘事件 -je_auto_control.record() - -time.sleep(10) # 录制 10 秒 - -# 停止录制并获取动作列表 -actions = je_auto_control.stop_record() - -# 回放前先清理录制内容：把连续的鼠标移动采样压缩成最后位置 -#（通常能把原始录制缩小一个数量级，且不改变回放行为） -actions = je_auto_control.dedupe_moves(actions) - -# 重新播放录制的动作 -je_auto_control.execute_action(actions) -``` - -> 非破坏式录制编辑器（均返回新的 list）：`dedupe_moves`（压缩鼠标移动）、`merge_sleeps`（合并连续 `AC_sleep`）、`trim_actions`、`insert_action`、`remove_action`、`filter_actions`、`adjust_delays`（缩放 `AC_sleep` 延迟）、`scale_coordinates`（以不同分辨率回放）。通过 MCP 暴露为 `ac_dedupe_moves` / `ac_merge_sleeps` / `ac_trim_actions` / `ac_adjust_delays` / `ac_scale_coordinates`。 - -### JSON 脚本执行器 - -创建 JSON 动作文件（`actions.json`）： - -```json -[ - ["AC_set_mouse_position", {"x": 500, "y": 300}], - ["AC_click_mouse", {"mouse_keycode": "mouse_left"}], - ["AC_write", {"write_string": "Hello from AutoControl"}], - ["AC_screenshot", {"file_path": "result.png"}], - ["AC_hotkey", {"key_code_list": ["ctrl_l", "s"]}] -] -``` - -执行方式： - -```python -import je_auto_control - -# 从文件执行 -je_auto_control.execute_action(je_auto_control.read_action_json("actions.json")) - -# 或直接从列表执行 -je_auto_control.execute_action([ - ["AC_set_mouse_position", {"x": 100, "y": 200}], - ["AC_click_mouse", {"mouse_keycode": "mouse_left"}] -]) -``` - -**可用的动作命令：** - -| 类别 | 命令 | -|---|---| -| 鼠标 | `AC_click_mouse`, `AC_set_mouse_position`, `AC_get_mouse_position`, `AC_get_mouse_table`, `AC_press_mouse`, `AC_release_mouse`, `AC_mouse_scroll`, `AC_mouse_left`, `AC_mouse_right`, `AC_mouse_middle` | -| 键盘 | `AC_type_keyboard`, `AC_press_keyboard_key`, `AC_release_keyboard_key`, `AC_write`, `AC_hotkey`, `AC_check_key_is_press`, `AC_get_keyboard_keys_table` | -| 图像 | `AC_locate_all_image`, `AC_locate_image_center`, `AC_locate_and_click` | -| 屏幕 | `AC_screen_size`, `AC_screenshot` | -| Accessibility | `AC_a11y_list`, `AC_a11y_find`, `AC_a11y_click` | -| VLM（AI 定位） | `AC_vlm_locate`, `AC_vlm_click` | -| OCR | `AC_locate_text`, `AC_click_text`, `AC_wait_text`, `AC_read_text_in_region`, `AC_find_text_regex` | -| LLM 规划器 | `AC_llm_plan`, `AC_llm_run` | -| 剪贴板 | `AC_clipboard_get`, `AC_clipboard_set` | -| 窗口 | `AC_list_windows`, `AC_focus_window`, `AC_wait_window`, `AC_close_window` | -| 流程控制 | `AC_loop`, `AC_break`, `AC_continue`, `AC_if_image_found`, `AC_if_pixel`, `AC_if_var`, `AC_while_image`, `AC_while_var`, `AC_for_each`, `AC_wait_image`, `AC_wait_pixel`, `AC_sleep`, `AC_retry`, `AC_try` | -| 变量 | `AC_set_var`, `AC_get_var`, `AC_inc_var` | -| 远程桌面 | `AC_start_remote_host`, `AC_stop_remote_host`, `AC_remote_host_status`, `AC_remote_connect`, `AC_remote_disconnect`, `AC_remote_viewer_status`, `AC_remote_send_input` | -| 录制 | `AC_record`, `AC_stop_record`, `AC_set_record_enable` | -| 报告 | `AC_generate_html`, `AC_generate_json`, `AC_generate_xml`, `AC_generate_html_report`, `AC_generate_json_report`, `AC_generate_xml_report` | -| 执行记录 | `AC_history_list`, `AC_history_clear` | -| 项目 | `AC_create_project` | -| Shell | `AC_shell_command` | -| 进程 | `AC_execute_process` | -| 执行器 | `AC_execute_action`, `AC_execute_files`, `AC_add_package_to_executor`, `AC_add_package_to_callback_executor` | -| MCP 服务器 | `AC_start_mcp_server`, `AC_start_mcp_http_server` | - -### MCP 服务器（让 Claude 使用 AutoControl） - -把 AutoControl 包装成 Model Context Protocol 服务,任何支持 MCP 的 -client(Claude Desktop、Claude Code、自定义 Anthropic / OpenAI tool-use -循环)都能驱动本机桌面。纯 stdlib — JSON-RPC 2.0 走 stdio 或 HTTP+ -SSE。 - -**注册到 Claude Code:** - -```bash -claude mcp add autocontrol -- python -m je_auto_control.utils.mcp_server -``` - -**注册到 Claude Desktop**(`claude_desktop_config.json`): - -```json -{ - "mcpServers": { - "autocontrol": { - "command": "python", - "args": ["-m", "je_auto_control.utils.mcp_server"] - } - } -} -``` - -**程序启动:** - -```python -import je_auto_control as ac - -# Stdio(会阻塞直到 stdin 关闭) -ac.start_mcp_stdio_server() - -# 或 HTTP / SSE,带 Bearer token 验证 + 可选 TLS -ac.start_mcp_http_server(host="127.0.0.1", port=9940, - auth_token="hunter2") -``` - -**不启动服务器、只看目录:** - -```bash -je_auto_control_mcp --list-tools -je_auto_control_mcp --list-tools --read-only -je_auto_control_mcp --list-resources -je_auto_control_mcp --list-prompts -``` - -**功能总览:** - -| 面向 | 涵盖 | -|---|---| -| 工具(约 90 个) | 鼠标 · 键盘 · drag · 屏幕 / 多屏 · 截图回 image · diff · OCR · 图像 · 窗口(move/min/max/restore/...) · 剪贴板文字+图像 · 进程 / shell · 动作录制 · 屏幕录像 · scheduler / triggers / hotkeys · accessibility tree · VLM · executor · history | -| 别名 | `click`、`type`、`screenshot`、`find_image`、`drag`、`shell`、`wait_image`...,以 `JE_AUTOCONTROL_MCP_ALIASES=0` 关闭 | -| Resources | `autocontrol://files/`、`autocontrol://history`、`autocontrol://commands`、`autocontrol://screen/live`(支持 `resources/subscribe`)| -| Prompts | `automate_ui_task`、`record_and_generalize`、`compare_screenshots`、`find_widget`、`explain_action_file` | -| 协议 | tools / resources / prompts / sampling / roots / logging / progress / cancellation / list_changed / elicitation | -| 传输 | stdio、HTTP `POST /mcp`、`Accept: text/event-stream` 时走 SSE 流 | -| 安全 | 工具注解 · `JE_AUTOCONTROL_MCP_READONLY` · `JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE` · 审计 log · token-bucket rate limiter · 工具失败自动截图 | -| 部署 | Bearer token 验证 · 通过 `ssl_context` 启用 TLS · `PluginWatcher` 热加载 · `JE_AUTOCONTROL_FAKE_BACKEND=1` 给 CI | - -完整参考请见 [docs/source/Zh/doc/mcp_server/mcp_server_doc.rst](../docs/source/Zh/doc/mcp_server/mcp_server_doc.rst) -(英文版本在 [docs/source/Eng/doc/mcp_server/mcp_server_doc.rst](../docs/source/Eng/doc/mcp_server/mcp_server_doc.rst))。 - -> ⚠️ MCP 服务器可以移动鼠标、发送键盘事件、截图、执行任意 `AC_*` -> 动作。请只注册给可信任的 client。HTTP 默认绑 `127.0.0.1`,要对外 -> 必须有明确理由,**并且**搭配 `auth_token` 与 `ssl_context`。 - -### 调度器（Interval & Cron） - -```python -import je_auto_control as ac - -# Interval：每 30 秒执行一次 -job = ac.default_scheduler.add_job( - script_path="scripts/poll.json", interval_seconds=30, repeat=True, -) - -# Cron：周一到周五 09:00（字段为 minute hour dom month dow） -cron_job = ac.default_scheduler.add_cron_job( - script_path="scripts/daily.json", cron_expression="0 9 * * 1-5", -) - -ac.default_scheduler.start() -``` - -两种调度可同时存在，通过 `job.is_cron` 区分类型。 - -### 全局热键 - -将 OS 热键绑定到 action JSON 脚本。跨平台 — Windows 用 -`RegisterHotKey`、macOS 用 `CGEventTap`（需要 Accessibility 权限）、 -Linux X11 用 `XGrabKey`（不支持 Wayland）。三个平台同一个 API；daemon -在 `start()` 时自动挑后端。 - -```python -from je_auto_control import default_hotkey_daemon - -default_hotkey_daemon.bind("ctrl+alt+1", "scripts/greet.json") -default_hotkey_daemon.start() -``` - -### 事件触发器 - -轮询式触发器，检测到条件成立时自动执行脚本： - -```python -from je_auto_control import ( - default_trigger_engine, ImageAppearsTrigger, - WindowAppearsTrigger, PixelColorTrigger, FilePathTrigger, -) - -default_trigger_engine.add(ImageAppearsTrigger( - trigger_id="", script_path="scripts/click_ok.json", - image_path="templates/ok_button.png", threshold=0.85, repeat=True, -)) -default_trigger_engine.start() -``` - -### 执行历史 - -调度器、触发器、热键、REST API 及 GUI 手动回放的每一次执行都会写入 -`~/.je_auto_control/history.db`。错误时会自动在 -`~/.je_auto_control/artifacts/run_{id}_{ms}.png` 附上截图以便排查。 - -```python -from je_auto_control import default_history_store - -for run in default_history_store.list_runs(limit=20): - print(run.id, run.source, run.status, run.artifact_path) -``` - -GUI **执行历史** 标签页显示运行记录表格，可双击截图列打开附件；筛选 / -刷新 / 清除命令位于窗口的 Actions 菜单。 - -### 报告生成 - -```python -import je_auto_control - -# 先启用测试记录 -je_auto_control.test_record_instance.set_record_enable(True) - -# ... 执行自动化动作 ... -je_auto_control.set_mouse_position(100, 200) -je_auto_control.click_mouse("mouse_left") - -# 生成报告 -je_auto_control.generate_html_report("test_report") # -> test_report.html -je_auto_control.generate_json_report("test_report") # -> test_report.json -je_auto_control.generate_xml_report("test_report") # -> test_report.xml - -# 或获取报告内容为字符串 -html_string = je_auto_control.generate_html() -json_string = je_auto_control.generate_json() -xml_string = je_auto_control.generate_xml() -``` - -报告内容包含：每个记录动作的函数名称、参数、时间戳及异常信息（如有）。HTML 报告中成功的动作以青色显示，失败的动作以红色显示。 - -### 可观测性（Prometheus / OpenTelemetry） - -纯标准库的 metric 原语加上 OpenTelemetry 兼容 tracer， -executor 与 agent loop 会自动发送调用次数与延迟分布 metric， -不用手动 instrument。 +也可以用 IP 允许列表（CIDR 网段或具体地址）限制谁能连进来，列表外的对端在握手 +阶段就会被拒绝： ```python -import je_auto_control as ac - -# 在 http://127.0.0.1:9090 开放 /metrics，给 Prometheus scrape。 -exporter = ac.default_metrics_exporter() -exporter.start() - -# 自定义 metric — 形状与 prometheus_client 相同。 -counter = ac.default_metric_registry().register(ac.MetricCounter( - "myapp_widgets_built_total", "widgets built", - label_names=("kind",), -)) -counter.inc(labels={"kind": "blue"}) - -# 把 callable 包进 span — 未安装 opentelemetry-api 时为 no-op。 -@ac.traced("my_pipeline.process_one") -def process_one(item): ... -``` - -内建 metric 清单见 -[docs/source/Eng/doc/observability/observability_doc.rst](../docs/source/Eng/doc/observability/observability_doc.rst) -或[繁体中文版](../docs/source/Zh/doc/observability/observability_doc.rst)。 - -### 远程自动化（Socket / REST） - -提供两种服务器：原始 TCP socket 与纯 stdlib HTTP/REST。默认均绑定 -`127.0.0.1`，绑定到 `0.0.0.0` 需显式指定。 - -```python -import je_auto_control as ac - -# TCP Socket 服务器（默认：127.0.0.1:9938） -ac.start_autocontrol_socket_server(host="127.0.0.1", port=9938) - -# REST API 服务器（默认：127.0.0.1:9939） -ac.start_rest_api_server(host="127.0.0.1", port=9939) -# 端点： -# GET /health 存活检查 -# GET /jobs 列出调度任务 -# POST /execute body: {"actions": [...]} -``` - -### 插件加载器 - -将定义顶层 `AC_*` 可调用对象的 `.py` 文件放入一个目录，运行时即可注 -册为 executor 命令： - -```python -from je_auto_control import ( - load_plugin_directory, register_plugin_commands, -) - -commands = load_plugin_directory("./my_plugins") -register_plugin_commands(commands) - -# 之后任何 JSON 脚本都能使用： -# [["AC_greet", {"name": "world"}]] -``` - -> **警告：** 插件文件会直接执行任意 Python，请仅加载自己信任的目录。 - -### Shell 命令执行 - -```python -import je_auto_control - -# 使用默认的 Shell 管理器 -je_auto_control.default_shell_manager.exec_shell("echo Hello") -je_auto_control.default_shell_manager.pull_text() # 输出捕获的结果 - -# 或创建自定义的 ShellManager -shell = je_auto_control.ShellManager(shell_encoding="utf-8") -shell.exec_shell("ls -la") -shell.pull_text() -shell.exit_program() +RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) ``` -### 屏幕录制 - -```python -import je_auto_control -import time - -# 方法一：ScreenRecorder（管理多个录像） -recorder = je_auto_control.ScreenRecorder() -recorder.start_new_record( - recorder_name="my_recording", - path_and_filename="output.avi", - codec="XVID", - frame_per_sec=30, - resolution=(1920, 1080) -) -time.sleep(10) -recorder.stop_record("my_recording") - -# 方法二：RecordingThread（简易单一录像，输出 MP4） -recording = je_auto_control.RecordingThread(video_name="my_video", fps=20) -recording.start() -time.sleep(10) -recording.stop() -``` - -### 回调执行器 - -执行自动化函数后自动触发回调函数： - -```python -import je_auto_control - -def my_callback(): - print("动作完成！") - -# 执行 set_mouse_position 后调用 my_callback -je_auto_control.callback_executor.callback_function( - trigger_function_name="AC_set_mouse_position", - callback_function=my_callback, - x=500, y=300 -) - -# 带有参数的回调 -def on_done(message): - print(f"完成: {message}") - -je_auto_control.callback_executor.callback_function( - trigger_function_name="AC_click_mouse", - callback_function=on_done, - callback_function_param={"message": "点击完成"}, - callback_param_method="kwargs", - mouse_keycode="mouse_left" -) -``` - -### 包管理器 - -在运行时动态加载外部 Python 包到执行器中： - -```python -import je_auto_control - -# 将包的所有函数/类加入执行器 -je_auto_control.package_manager.add_package_to_executor("os") - -# 现在可以在 JSON 动作脚本中使用 os 函数： -# ["os_getcwd", {}] -# ["os_listdir", {"path": "."}] -``` - -### 项目管理 - -快速创建包含模板文件的项目目录结构： - -```python -import je_auto_control - -# 创建项目结构 -je_auto_control.create_project_dir(project_path="./my_project", parent_name="AutoControl") - -# 会创建以下结构： -# my_project/ -# └── AutoControl/ -# ├── keyword/ -# │ ├── keyword1.json # 模板动作文件 -# │ ├── keyword2.json # 模板动作文件 -# │ └── bad_keyword_1.json # 错误处理模板 -# └── executor/ -# ├── executor_one_file.py # 执行单一文件示例 -# ├── executor_folder.py # 执行文件夹示例 -# └── executor_bad_file.py # 错误处理示例 -``` - -### 窗口管理 - -直接将事件发送至指定窗口（仅限 Windows 和 Linux）： - -```python -import je_auto_control - -# 通过窗口标题发送键盘事件 -je_auto_control.send_key_event_to_window("Notepad", keycode="a") - -# 通过窗口 handle 发送鼠标事件 -je_auto_control.send_mouse_event_to_window(window_handle, mouse_keycode="mouse_left", x=100, y=50) -``` - -### GUI 应用程序 - -启动内置图形界面（需安装 `[gui]` 扩展）： - -```python -import je_auto_control -je_auto_control.start_autocontrol_gui() -``` - -或通过命令行： - -```bash -python -m je_auto_control -``` - -主窗口采用菜单驱动设计：标签页只保留输入字段、表格与结果视图，每个 -标签页的命令都集中在窗口级的 **Actions** 菜单，会随当前标签页动态重建。 -**View → Tabs** 可按分类（核心 / 编辑 / 检测与视觉 / 自动化引擎 / 系统） -显示或隐藏约 48 个已注册标签页；默认布局只打开录制、脚本构建器与远程 -桌面三个标签页。**View → Text Size** 提供自动／预设字号，**Language** -菜单（English / 繁體中文 / 简体中文 / 日本語）可即时切换整个窗口的语言。 - --- -## 命令行界面 - -AutoControl 可直接从命令行使用： - -```bash -# 执行单一动作文件 -python -m je_auto_control -e actions.json - -# 执行目录中所有动作文件 -python -m je_auto_control -d ./action_files/ - -# 直接执行 JSON 字符串 -python -m je_auto_control --execute_str '[["AC_screenshot", {"file_path": "test.png"}]]' - -# 创建项目模板 -python -m je_auto_control -c ./my_project -``` - -另外还有以 headless API 为基础的子命令 CLI： - -```bash -# 执行脚本（可带变量或 dry-run） -python -m je_auto_control.cli run script.json -python -m je_auto_control.cli run script.json --var name=alice --dry-run - -# 列出调度任务 -python -m je_auto_control.cli list-jobs - -# 启动 Socket / REST 服务器 -python -m je_auto_control.cli start-server --port 9938 -python -m je_auto_control.cli start-rest --port 9939 -``` +## 平台支持 -`--var name=value` 优先以 JSON 解析（`count=10` 会变成 int），解析失败 -则视为字符串。 +| 平台 | 后端 | 输入 | 屏幕捕获 | 录制 | 窗口管理 | +|---|---|:---:|:---:|:---:|:---:| +| Windows 10 / 11 | Win32 ctypes（可选 Interception 驱动） | ✅ | ✅ | ✅ | ✅ | +| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | +| Linux X11 | python-Xlib（可选 `uinput`） | ✅ | ✅ | ✅ | ❌ | +| Linux Wayland | libei，或 ydotool／wtype／grim | ✅ | ✅ | ❌ | ❌ | +| Android | adb + uiautomator2 | ✅ | ✅ | — | — | +| iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | + +Wayland 禁止非特权客户端进行全局输入录制——若要录制，请设置 +`JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` 并在 X11 会话下运行。窗口管理目前仅 +Windows 有实现，其他平台会抛出明确的 `NotImplementedError`。对于会忽略合成输入的应用， +可选用驱动层后端（`JE_AUTOCONTROL_WIN32_BACKEND=interception`、 +`JE_AUTOCONTROL_LINUX_BACKEND=uinput`、ViGEm 虚拟手柄）；驱动未安装时会自动回退到原有行为。 --- -## 平台支持 +## 文档与示例 -| 平台 | 状态 | 后端 | 备注 | -|---|---|---|---| -| Windows 10 / 11 | 支持 | Win32 API (ctypes) | 完整功能支持 | -| macOS 10.15+ | 支持 | pyobjc / Quartz | 不支持动作录制；不支持 `send_key_event_to_window` / `send_mouse_event_to_window` | -| Linux（X11） | 支持 | python-Xlib | 完整功能支持 | -| Linux（Wayland） | 暂不支持 | — | 未来版本可能加入支持 | -| Raspberry Pi 3B / 4B | 支持 | python-Xlib | 在 X11 上运行 | +| 资源 | 内容 | +|---|---| +| [`examples/`](../examples/) | 27 个自包含脚本：截图点击、OCR、调度器、远程桌面、agent loop、可观测性、录制、变量、热键、触发器、报表、MCP、REST、密钥、插件、computer use、Wayland、跨主机 DAG、chat-ops、pytest/BDD、锚点定位。 | +| [Read the Docs](https://autocontrol.readthedocs.io/en/latest/) | 完整 API 参考，含英文与中文。 | +| [architecture_explore.md](../architecture_explore.md) | 逐层记录每个模块的职责。 | +| [docs/CAPABILITY_MATRIX.md](../docs/CAPABILITY_MATRIX.md) | 能力 × 平台对照矩阵。 | +| [docs/API_LIFECYCLE.md](../docs/API_LIFECYCLE.md) | 稳定 API 与弃用策略。 | +| [WHATS_NEW.md](../WHATS_NEW.md) | 各版本更新说明。 | +| [CHANGELOG.md](../CHANGELOG.md) | 兼容性变更记录。 | +| [SECURITY.md](../SECURITY.md) | 安全策略与报告方式。 | --- ## 开发 -### 环境配置 - ```bash git clone https://github.com/Intergration-Automation-Testing/AutoControl.git cd AutoControl pip install -r dev_requirements.txt +uv sync # 或：以已提交的 uv.lock 做可重现安装 ``` -可复现的安装走已 commit 的 `uv.lock`： - -```bash -uv sync # 依锁文件同步整条依赖链 -uv lock --upgrade # 编辑 pyproject.toml 后重新锁 -``` - -### 运行测试 - ```bash -# 单元测试 -python -m pytest test/unit_test/ +python -m pytest test/unit_test/headless # 无头单元测试 +python -m pytest test/integrated_test/ # 跨模块流程测试 -# 集成测试 -python -m pytest test/integrated_test/ +ruff check je_auto_control/ +pylint je_auto_control/ +bandit -c pyproject.toml -r je_auto_control/ ``` -### 项目链接 - -- **主页**: https://github.com/Intergration-Automation-Testing/AutoControl -- **文档**: https://autocontrol.readthedocs.io/en/latest/ -- **PyPI**: https://pypi.org/project/je_auto_control/ +欢迎贡献——请见 [CONTRIBUTING.md](../CONTRIBUTING.md) 与 +[CODE_OF_CONDUCT.md](../CODE_OF_CONDUCT.md)。CI 会强制两条规则：`import je_auto_control` +绝不能加载 PySide6；每个功能都必须同时具备无头 API 与 GUI 界面。 --- -## 许可证 +## 许可 [MIT License](../LICENSE) © JE-Chen。 -第三方依赖的许可证请见 -[Third_Party_License.md](../Third_Party_License.md)。 +内含与可选第三方组件的许可请见 [Third_Party_License.md](../Third_Party_License.md)。 + +- **主页**：https://github.com/Intergration-Automation-Testing/AutoControl +- **PyPI**：https://pypi.org/project/je_auto_control/ +- **文档**：https://autocontrol.readthedocs.io/en/latest/ diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index a5fbdd54..d75feed8 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -3,1339 +3,290 @@ [![PyPI](https://img.shields.io/pypi/v/je_auto_control)](https://pypi.org/project/je_auto_control/) [![Python](https://img.shields.io/pypi/pyversions/je_auto_control)](https://pypi.org/project/je_auto_control/) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](../LICENSE) +[![Documentation](https://readthedocs.org/projects/autocontrol/badge/?version=latest)](https://autocontrol.readthedocs.io/en/latest/?badge=latest) -**AutoControl** 是一個跨平台的 Python GUI 自動化框架，提供滑鼠控制、鍵盤輸入、圖像辨識、螢幕擷取、腳本執行與報告產生等功能 — 透過統一的 API 在 Windows、macOS 和 Linux (X11) 上運作。 +**AutoControl** 是一套跨平台的 Python GUI 自動化框架。它能驅動滑鼠與鍵盤、在畫面上找到目標 +（樣板比對、OCR、作業系統無障礙樹，或視覺模型）、錄製與重播操作流程，並以 JSON 動作檔執行—— +支援 Windows、macOS、Linux（X11 與 Wayland）、Android 與 iOS。 -**[English](../README.md)** | **[简体中文](README_zh-CN.md)** +每項能力都以三種形式提供：**Python API**、可在 JSON 檔／CLI／伺服器使用的 **`AC_*` 動作指令**， +以及 **GUI 分頁**。沒有任何功能只存在於 GUI。 ---- - -## 目錄 - -- [本次更新](#本次更新) -- [功能特色](#功能特色) -- [架構](#架構) -- [安裝](#安裝) -- [系統需求](#系統需求) -- [快速開始](#快速開始) - - [滑鼠控制](#滑鼠控制) - - [鍵盤控制](#鍵盤控制) - - [圖像辨識](#圖像辨識) - - [Accessibility 元件搜尋](#accessibility-元件搜尋) - - [AI 元件定位（VLM）](#ai-元件定位vlm) - - [OCR 螢幕文字辨識](#ocr-螢幕文字辨識) - - [LLM 動作規劃器](#llm-動作規劃器) - - [執行期變數與流程控制](#執行期變數與流程控制) - - [遠端桌面](#遠端桌面) - - [剪貼簿](#剪貼簿) - - [截圖](#截圖) - - [動作錄製與回放](#動作錄製與回放) - - [JSON 腳本執行器](#json-腳本執行器) - - [MCP 伺服器（讓 Claude 使用 AutoControl）](#mcp-伺服器讓-claude-使用-autocontrol) - - [排程器（Interval & Cron）](#排程器interval--cron) - - [全域熱鍵](#全域熱鍵) - - [事件觸發器](#事件觸發器) - - [執行歷史](#執行歷史) - - [報告產生](#報告產生) - - [可觀測性（Prometheus / OpenTelemetry）](#可觀測性prometheus--opentelemetry) - - [遠端自動化（Socket / REST）](#遠端自動化socket--rest) - - [外掛載入器](#外掛載入器) - - [Shell 命令執行](#shell-命令執行) - - [螢幕錄製](#螢幕錄製) - - [回呼執行器](#回呼執行器) - - [套件管理器](#套件管理器) - - [專案管理](#專案管理) - - [視窗管理](#視窗管理) - - [GUI 應用程式](#gui-應用程式) -- [命令列介面](#命令列介面) -- [平台支援](#平台支援) -- [開發](#開發) -- [授權條款](#授權條款) +**[English](../README.md)** · **[简体中文](README_zh-CN.md)** --- -## 本次更新 - -**最新（2026-07-18）— 跨平台穩定性強化。** 一次全專案執行期稽核修正了 macOS／Windows／Linux／Wayland 各後端、執行器,以及遠端桌面／USB 堆疊的執行期缺陷——包含正確的 Retina 游標座標運算、CPython 3.14 上的中繼卡死、`AC_expect_poll`／`AC_parallel` 的健全性、USB/IP 預設綁定本機,以及在 I/O 邊界保留型別化例外——每項皆有 headless 回歸測試涵蓋。無 API 變更。 - -各版本更新說明詳見 **[WHATS_NEW.md](../WHATS_NEW.md)**。 - -## 功能特色 - -- **QA / 測試框架** — 斷言 DSL（`assert_text` / `_image` / `_pixel` / `_window` / `_clipboard` / `_process` / `_file` / `_http` 加上音訊/影片斷言,以及 `assert_all` / `assert_any` / `assert_eventually` 組合器）、資料驅動執行（CSV / JSON / SQLite / Excel → `AC_for_each_row`）、具 setup/teardown/標籤的計分 `run_suite`、JUnit + Allure 報告輸出、不穩定測試偵測與自動隔離、無障礙 / i18n 稽核（缺漏標籤、WCAG 對比度、截斷），以及並行的行動裝置矩陣。詳見 [本次更新 (2026-06)](WHATS_NEW_zh-TW.md) -- **滑鼠自動化** — 移動、點擊、按下、釋放、拖曳、滾動，支援精確座標控制 -- **鍵盤自動化** — 按下/釋放單一按鍵、輸入字串、組合鍵、按鍵狀態偵測 -- **圖像辨識** — 使用 OpenCV 模板匹配在螢幕上定位 UI 元素，支援可設定的偵測閾值 -- **Accessibility 元件搜尋** — 透過作業系統無障礙樹（Windows UIA / macOS AX）依名稱/角色定位按鈕、選單、控制項 -- **AI 元件定位（VLM）** — 用自然語言描述 UI 元素，交由視覺語言模型（Anthropic / OpenAI）取得螢幕座標 -- **OCR** — 三個可插拔後端（Tesseract 用於 ASCII、EasyOCR 不需外部執行檔且支援 CJK、PaddleOCR 中／日／韓品質最佳），統一 API 與標準語言代碼；後端由 `backend=` 參數、`AUTOCONTROL_OCR_BACKEND` 環境變數或自動偵測決定。可搜尋、點擊或等待文字出現；支援 regex 搜尋與整塊區域 dump -- **LLM 動作規劃器** — 用 Claude 把自然語言描述翻譯成驗證過的 `AC_*` 動作清單 -- **執行期變數與流程控制** — 執行時 `${var}` 取代，加上 `AC_set_var` / `AC_inc_var` / `AC_if_var` / `AC_for_each` / `AC_loop` / `AC_while_var` / `AC_retry` / `AC_try` 讓腳本資料驅動。`AC_while_var` 在變數比較成立時持續迴圈（每輪重新判斷，`max_iter` 安全上限）；`AC_try` 提供 try/catch/finally：`body` 失敗時改走 `catch` 復原分支而非中止、`finally` 必定執行、錯誤透過 `error_var` 暴露、可在清理後 `reraise`（迴圈 `break`/`continue` 仍能穿透） -- **遠端桌面** — 用 token 認證的 TCP 協定串流本機畫面並接收輸入，**或** 連線到他機觀看與控制（host + viewer GUI 皆內建）。可選 TLS（HTTPS 級加密）、WebSocket 傳輸（``ws://`` + ``wss://``，穿牆／瀏覽器友善）、持久化 9 位數 Host ID、host→viewer 音訊串流、雙向剪貼簿同步（文字 + 圖片）、分塊檔案傳輸（拖放 + 進度條；任意目的路徑；無大小上限）。另含資料夾同步（增量鏡像 — 本地刪除不會傳出去）與自架 coturn TURN 設定包產生器（turnserver.conf + systemd unit + docker-compose + README）。**AnyDesk 風格彈出視窗**：viewer 認證成功後遠端桌面會開在獨立的可調整大小頂層視窗，控制面板維持簡潔；Remote Desktop 子分頁外層包了 `QScrollArea`，小視窗下可捲動、4K 螢幕下會延展到整寬。同時可由 headless API 與 MCP 工具（`ac_remote_*`）直接驅動 -- **驅動層輸入後端（可選）** — 針對忽略 SendInput（Win）或 XTest（Linux）的遊戲／應用:**Interception driver 後端**(Windows,HID 層鍵鼠注入,使用 Oblita WHQL-signed driver,以 `JE_AUTOCONTROL_WIN32_BACKEND=interception` 啟用)、**uinput 後端**(Linux,kernel `/dev/uinput` 合成 HID 裝置,以 `JE_AUTOCONTROL_LINUX_BACKEND=uinput` 啟用),以及 **ViGEm 虛擬手把**(Windows,針對只認手把的遊戲,提供虛擬 Xbox 360 手把 + 友善的 button / dpad / stick / trigger API,並暴露為 `AC_gamepad_*` 執行器指令與 `ac_gamepad_*` MCP 工具)。三者在 driver 沒裝時都會優雅 fallback,不影響既有部署 -- **剪貼簿** — 於 Windows / macOS / Linux 讀寫系統剪貼簿文字 -- **截圖與螢幕錄製** — 擷取全螢幕或指定區域為圖片，錄製螢幕為影片（AVI/MP4） -- **動作錄製與回放** — 錄製滑鼠/鍵盤事件並重新播放 -- **JSON 腳本執行** — 使用 JSON 動作檔案定義並執行自動化流程（支援 dry-run 與逐步除錯） -- **排程器** — 以 interval 或 cron 表示式執行腳本，interval 與 cron job 可同時存在 -- **全域熱鍵** — 跨平台綁定 OS 熱鍵到 action 腳本：Windows (`RegisterHotKey`)、macOS (`CGEventTap`，需 Accessibility 權限)、Linux X11 (`XGrabKey`，含 NumLock / CapsLock 變體遮罩)。Wayland 不支援。三個平台共用同一個 API；`backends/` 在 `start()` 時自動挑後端 -- **事件觸發器** — 偵測到影像出現、視窗出現、像素變化或檔案變動時自動執行腳本 -- **執行歷史** — 以 SQLite 紀錄 scheduler / triggers / hotkeys / REST 的執行結果；錯誤時自動附上截圖 -- **報告產生** — 將測試紀錄匯出為 HTML、JSON 或 XML 報告，包含成功/失敗狀態 -- **MCP 伺服器** — JSON-RPC 2.0 Model Context Protocol 服務（stdio + HTTP/SSE），讓 Claude Desktop / Claude Code / 自訂 tool-use 迴圈直接驅動 AutoControl。約 100 個工具,完整協定支援(resources、prompts、sampling、roots、logging、progress、cancellation、elicitation),Bearer token 驗證 + TLS、稽核 log、rate limit、plugin 熱重載、CI fake backend。**本次新增** `ac_remote_host_start` / `ac_remote_host_stop` / `ac_remote_host_status` / `ac_remote_viewer_connect` / `ac_remote_viewer_disconnect` / `ac_remote_viewer_status` / `ac_remote_viewer_send_input` 包裝 GUI 遠端桌面分頁所用的 process-global registry,模型可以直接啟動 host、連線 viewer、轉送 mouse／keyboard／type／hotkey 動作 -- **遠端自動化** — TCP Socket 伺服器 **加上** 強化版 REST API：bearer token 認證、per-IP 速率限制 + 失敗鎖定、SQLite 稽核 hook、Prometheus `/metrics`、完整端點清單（`/health`、`/screen_size`、`/sessions`、`/screenshot`、`/execute`、`/audit/list`、`/audit/verify`、`/inspector/recent`、`/usb/devices`、`/diagnose`、…），以及 vanilla-JS 的瀏覽器 dashboard `/dashboard`（任何能 HTTP 連到主機的手機都能監看） -- **外掛載入器** — 將定義 `AC_*` 可呼叫物的 `.py` 檔放入目錄，執行時即可註冊成 executor 指令 -- **Shell 整合** — 在自動化流程中執行 Shell 命令，支援非同步輸出擷取 -- **回呼執行器** — 觸發自動化函式後自動呼叫回呼函式，實現操作串接 -- **動態套件載入** — 在執行時匯入外部 Python 套件，擴充執行器功能 -- **專案與範本管理** — 快速建立包含 keyword/executor 目錄結構的自動化專案 -- **視窗管理** — 直接將鍵盤/滑鼠事件送至指定視窗（Windows/Linux） -- **GUI 應用程式** — 內建 PySide6 圖形介面，支援即時切換語系（English / 繁體中文 / 简体中文 / 日本語） -- **CLI 執行介面** — `python -m je_auto_control.cli run|list-jobs|start-server|start-rest` -- **跨平台** — 統一 API，支援 Windows、macOS、Linux（X11 + Wayland）、Android（adb + uiautomator2）、iOS（WebDriverAgent / facebook-wda） -- **截圖 PII 遮罩** — `RedactionEngine` 在截圖上傳 VLM、寫入 audit log 或經由 REST 回傳前，把 email / 信用卡號 / SSN / 電話 / secure-text 欄位 / 強制區域模糊掉。透過環境變數 `JE_AUTOCONTROL_REDACTION=off|moderate|strict` 或逐次呼叫指定政策 -- **多主機管理主控台** — 在一份通訊錄中註冊 N 個遠端 AutoControl REST 端點，並行輪詢 health/sessions/jobs，把同一份動作清單廣播給全部主機。儲存於 `~/.je_auto_control/admin_hosts.json`（POSIX 上模式 0600）。Token 錯誤的主機會以實際 HTTP 錯誤呈現為不健康 -- **可偵測竄改的稽核紀錄** — SQLite events 表加上 SHA-256 雜湊鏈（每筆紀錄含 `prev_hash` + `row_hash`）；修改任何過去紀錄都會打斷雜湊鏈。`verify_chain()` 由上往下走訪並回報第一個斷點。既有資料表會在啟動時回填（「初次使用即信任」） -- **WebRTC 封包監測** — 由既有 WebRTC stats 輪詢餵入的程序級 `StatsSnapshot` 滾動視窗（預設 600 筆 / 1 Hz 約 10 分鐘）。對 RTT、FPS、bitrate、封包遺失、jitter 各回 `last/min/max/avg/p95` -- **USB 裝置列舉** — 唯讀的跨平台 USB 裝置列舉。優先嘗試 pyusb（libusb）；若無則退回平台特定指令（Windows `Get-PnpDevice`、macOS `system_profiler`、Linux `/sys/bus/usb/devices`）。第二階段 passthrough 建構於此（見下） -- **系統診斷** — 一鍵「目前正常嗎？」探測：平台、選用相依套件、executor 指令數、稽核鏈、截圖、滑鼠、硬碟空間、REST registry。CLI 全綠 exit 0／否則 1；REST `/diagnose`；依嚴重度上色的 GUI 分頁 -- **穩定 API 與失敗診斷包** — 給新整合用的版本化、延遲載入 `je_auto_control.api` 門面(`execute_action`、`generate_code`、`run_diagnostics`、failure bundles),附[生命週期政策](../docs/API_LIFECYCLE.md)。可攜式 `autocontrol.failure-bundle/v1` 診斷 ZIP:manifest + 已遮罩的 context/events/log 尾段、可選截圖與診斷、best-effort 收集器、原子寫入。CLI `je_auto_control failure-bundle out.zip`;`codegen --failure-bundle` 讓產生的 pytest 自動包上失敗診斷 -- **USB Hotplug 事件** — 輪詢式 hotplug 監測（`UsbHotplugWatcher`），含 bounded ring buffer 與帶序號的事件；`GET /usb/events?since=N` 讓晚加入的訂閱者補上進度。USB 分頁有自動更新切換鈕。 -- **OpenAPI 3.1 + Swagger UI** — `GET /openapi.json`（auth-gated，從活的路由表生成）+ `GET /docs`（瀏覽器版 Swagger UI 含 bearer token 列）。CI 上有 drift 測試，新加路由忘記寫 metadata 會被擋下。 -- **設定包匯出／匯入** — 單一 JSON 檔，匯出／匯入使用者設定（admin hosts、address book、trusted viewers、known hosts、host service、IDs）。原子寫入加 `.bak.<時間戳>` 備份；CLI `python -m je_auto_control.utils.config_bundle export|import`；`POST /config/{export,import}`；REST API 分頁的匯出／匯入指令位於視窗的 Actions 選單。 -- **USB Passthrough（需主動啟用）** — 讓遠端 viewer 使用實體插在 host 上的 USB 裝置，走 WebRTC `usb` DataChannel。Wire-level 協定（11 個 opcode 含 `RESUME`、CREDIT 流量控制、16 KiB payload 上限，超量傳輸以 EOF 分片）。八個原始未決問題全部解決：可靠有序 channel、LIST 走 channel（ACL 過濾）、per-claim credit、Linux kernel driver detach/reattach、ACL **HMAC-SHA256 完整性**（竄改 fail-closed；金鑰可插拔 — Windows DPAPI 或 passphrase vault）。**Backend：**`LibusbBackend`（production）、`WinusbBackend`（ctypes）、`IokitBackend`（原生 IOKit 列舉 + libusb 傳輸）— Windows/macOS *硬體未驗證*；`default_passthrough_backend()` 依 OS 自動挑。Viewer 端阻塞式 client（`control/bulk/interrupt_transfer`、`list_devices`、`resume`）；in-process `UsbLoopback` 讓同機可走完整堆疊 share+use。**已接入 WebRTC** host/viewer（`viewer.usb_client()`）並含斷線可續租的 **resume token**。持久化 ACL（預設 deny、mode 0600），含 host 端 prompt 對話框、濫用 **rate-limit / lockout** 與可偵測竄改稽核整合。五個驅動面：AnyDesk 風 **GUI 面板**（分享 + ACL 允許/封鎖 + 本機/遠端使用）、`AC_usb_*` executor 指令（JSON / socket / 排程器）、**REST** `/usb/...`、一級 **MCP** `ac_usb_*` 工具、以及 Python API。預設 off — 用 `enable_usb_passthrough(True)` 或 `JE_AUTOCONTROL_USB_PASSTHROUGH=1` 開啟；預設啟用仍待 Phase 2e 外部安全簽核 + 實機硬體驗證。 - ---- - -## 架構 - -執行階段是分層的：**客戶端介面**(CLI、GUI、MCP/REST/Socket 伺服 -器)位於最上層,底下是**無頭 API**(`wrapper/` + `utils/`),最後 -解析到 `wrapper/platform_wrapper.py` 在 import 時挑選的**作業系統 -後端**。套件 façade(`je_auto_control/__init__.py`)會 re-export 所 -有公開名稱,使用者只需要 `import je_auto_control`,不論用哪個介面或 -後端都一樣。 - -```mermaid -flowchart LR - subgraph Clients["客戶端介面"] - direction TB - Claude[["Claude Desktop /
Claude Code"]] - APIUser[["自訂 Anthropic /
OpenAI tool-use 迴圈"]] - HTTPClient[["HTTP / SSE clients"]] - TCPClient[["Socket / REST clients"]] - Browser[["瀏覽器
(/dashboard · /docs)"]] - GUIUser[["PySide6 GUI"]] - CLIUser[["python -m
je_auto_control[.cli]"]] - Library[["Library 使用者
(import je_auto_control)"]] - end - - subgraph Transports["傳輸與伺服器"] - direction TB - Stdio["MCP stdio
JSON-RPC 2.0"] - HTTPMCP["MCP HTTP /
SSE + auth + TLS"] - REST["REST 伺服器 :9939
bearer auth · rate-limit ·
OpenAPI · /metrics · /dashboard"] - Socket["Socket 伺服器
:9938"] - WebRTC["WebRTC sessions
(遠端桌面 ·
檔案 · 音訊 · USB)"] - end - - subgraph MCP["mcp_server/"] - direction TB - Dispatcher["MCPServer
(JSON-RPC dispatcher)"] - Tools["tools/
~90 ac_* + 別名"] - Resources["resources/
files · history ·
commands · screen-live"] - Prompts["prompts/
內建範本"] - Context["context · audit ·
rate-limit · log-bridge"] - FakeBE["fake_backend
(CI 煙霧測試)"] - end - - subgraph Core["無頭核心 (wrapper/ + utils/)"] - direction TB - Wrapper["wrapper/
滑鼠 · 鍵盤 · 螢幕 ·
影像 · 錄製 · 視窗"] - Executor["executor/
AC_* JSON 動作引擎"] - Vision["vision/ · ocr/ ·
accessibility/"] - Recorder["scheduler/ · triggers/ ·
hotkey/ · plugin_loader/
run_history/"] - IOUtils["clipboard/ · cv2_utils/ ·
shell_process/ · json/"] - end - - subgraph Ops["維運層 (utils/)"] - direction TB - Admin["admin/
多主機輪詢 +
廣播"] - Audit["remote_desktop/
audit_log
(SHA-256 鏈)"] - Inspector["remote_desktop/
webrtc_inspector"] - Diag["diagnostics/
自我診斷"] - ConfigB["config_bundle/
匯出/匯入"] - end - - subgraph USB["USB"] - direction TB - UsbEnum["usb/
列舉 + hotplug"] - UsbPass["usb/passthrough/
session · client · ACL(HMAC) ·
libusb · WinUSB · IOKit ·
loopback · webrtc channel · commands"] - end - - subgraph Remote["遠端桌面 (utils/remote_desktop/)"] - direction TB - RDHost["host · webrtc_host ·
signaling · multi_viewer"] - RDFiles["webrtc_files · file_sync ·
clipboard_sync · audio"] - RDTrust["trust_list · fingerprint ·
turn_config · lan_discovery"] - end - - subgraph Backends["作業系統後端"] - direction TB - Win["windows/
Win32 ctypes"] - Mac["osx/
pyobjc · Quartz"] - X11["linux_with_x11/
python-Xlib"] - end - - Claude --> Stdio - APIUser --> Stdio - HTTPClient --> HTTPMCP - TCPClient --> Socket - TCPClient --> REST - Browser --> REST - - Stdio --> Dispatcher - HTTPMCP --> Dispatcher - Dispatcher --> Tools - Dispatcher --> Resources - Dispatcher --> Prompts - Dispatcher -.- Context - Tools -.選用.-> FakeBE - - Tools --> Wrapper - Tools --> Executor - Tools --> Vision - Tools --> Recorder - Tools --> IOUtils - Resources --> Recorder - Resources --> Wrapper - - REST --> Executor - REST --> Ops - REST --> USB - Socket --> Executor - WebRTC --> Remote - WebRTC --> UsbPass - - GUIUser --> Wrapper - GUIUser --> Recorder - GUIUser --> Ops - GUIUser --> USB - GUIUser --> Remote - CLIUser --> Executor - Library --> Wrapper - Library --> Executor - Library --> Ops - - Admin --> REST - Inspector -.- WebRTC - Audit -.- REST - Audit -.- USB - UsbPass --> Backends +## 為什麼選擇 AutoControl - Wrapper --> Backends - Vision -.- Wrapper - Recorder -.- Executor -``` - -``` -je_auto_control/ -├── wrapper/ # 平台無關 API 層 -│ ├── platform_wrapper.py # 自動偵測作業系統並載入對應後端 -│ ├── auto_control_mouse.py # 滑鼠操作 -│ ├── auto_control_keyboard.py# 鍵盤操作 -│ ├── auto_control_image.py # 圖像辨識（OpenCV 模板匹配） -│ ├── auto_control_screen.py # 截圖、螢幕大小、像素顏色 -│ ├── auto_control_window.py # 跨平台視窗管理 facade -│ └── auto_control_record.py # 動作錄製/回放 -├── windows/ # Windows 專用後端（Win32 API / ctypes） -├── osx/ # macOS 專用後端（pyobjc / Quartz） -├── linux_with_x11/ # Linux 專用後端（python-Xlib） -├── gui/ # PySide6 GUI 應用程式 -└── utils/ - ├── mcp_server/ # MCP 伺服器（stdio + HTTP/SSE）— server / tools / resources / prompts / audit / rate_limit / fake_backend / plugin_watcher - ├── executor/ # JSON 動作執行引擎 - ├── callback/ # 回呼函式執行器 - ├── cv2_utils/ # OpenCV 截圖、模板匹配、影片錄製 - ├── accessibility/ # UIA (Windows) / AX (macOS) 元件搜尋 - ├── vision/ # VLM 元件定位（Anthropic / OpenAI） - ├── ocr/ # Tesseract 文字定位 - ├── clipboard/ # 跨平台剪貼簿（文字 + 圖像） - ├── llm/ # 自然語言 → AC_* 動作規劃器 - ├── scheduler/ # Interval + cron 排程器 - ├── hotkey/ # 全域熱鍵守護程序 - ├── triggers/ # 影像/視窗/像素/檔案 觸發器 - ├── run_history/ # SQLite 執行紀錄 + 錯誤截圖 - ├── rest_api/ # 純 stdlib HTTP/REST 伺服器 — auth · audit · rate-limit · OpenAPI · /metrics · dashboard · Swagger UI - ├── admin/ # 多主機 AdminConsoleClient（輪詢 + 廣播） - ├── diagnostics/ # 系統自我診斷 + CLI - ├── config_bundle/ # 單檔使用者設定匯出／匯入 - ├── usb/ # 跨平台列舉、hotplug 事件、passthrough/{protocol, session, viewer client, loopback, webrtc channel, ACL+HMAC, descriptor, key providers, commands, libusb / WinUSB / IOKit} - ├── remote_desktop/ # WebRTC host + viewer、signalling、multi-viewer、檔案／剪貼簿／音訊同步、稽核紀錄（雜湊鏈）、信任清單、TURN 設定、mDNS 發現、WebRTC stats inspector - ├── plugin_loader/ # 動態 AC_* 外掛搜尋與註冊 - ├── socket_server/ # TCP Socket 伺服器（遠端自動化） - ├── shell_process/ # Shell 命令管理器 - ├── generate_report/ # HTML / JSON / XML 報告產生器 - ├── test_record/ # 測試動作紀錄 - ├── script_vars/ # 腳本變數插值 - ├── watcher/ # 滑鼠 / 像素 / log 監看器（Live HUD） - ├── recording_edit/ # 錄製內容的修剪、過濾、縮放 - ├── json/ # JSON 動作檔案讀寫 - ├── project/ # 專案建立與範本 - ├── package_manager/ # 動態套件載入 - ├── logging/ # 日誌紀錄 - └── exception/ # 自訂例外類別 -``` - -`platform_wrapper.py` 模組會自動偵測目前的作業系統並匯入對應的後端，因此所有 wrapper 函式在不同平台上的行為完全一致。 +- **一套 API，六個平台。** `wrapper/platform_wrapper.py` 在匯入時挑選後端；同一份腳本在 + Windows、macOS、X11 與 Wayland 上都不需要改寫。 +- **不寫 Python 也能腳本化。** 767 個 `AC_*` 指令涵蓋全部功能，因此一個 JSON 檔能做到函式庫 + 能做的任何事——包含迴圈、分支、try/catch、巨集與變數。 +- **預設無頭執行。** `import je_auto_control` 絕不會載入 Qt。GUI 是選用套件，包在同一個無頭核心之外。 +- **四種定位方式。** 樣板比對、OCR、無障礙樹、視覺語言模型——可透過錨點定位器與自癒後備串接組合。 +- **相依基線輕薄。** REST 伺服器、JSON Schema 驗證、JWT、TOTP、WebSocket 框架、ACME 用戶端、 + USB/IP 協定與 Prometheus 指標全部以標準庫實作；較重的相依都是選用。 --- ## 安裝 -### 基本安裝 - ```bash -pip install je_auto_control +pip install je_auto_control # 核心 +pip install je_auto_control[gui] # 加上 PySide6 桌面應用程式 ``` -### 安裝 GUI 支援（PySide6） +需要時才安裝的選用套件： -```bash -pip install je_auto_control[gui] -``` - -### Linux 前置需求 +| Extra | 啟用的功能 | +|---|---| +| `gui` | PySide6 桌面應用程式（48 個分頁） | +| `webrtc` | WebRTC 遠端桌面、USB 直通（`aiortc`、`av`） | +| `signaling` | 獨立的訊令／rendezvous 伺服器（`fastapi`、`uvicorn`） | +| `discovery` | mDNS / Zeroconf 區網主機探索 | +| `pdf` / `office` | PDF 與 Excel／Word／PowerPoint 讀取 | +| `fuzzy` / `locale` | `rapidfuzz` 模糊比對、`babel` 地區解析 | +| `s3` / `audio` | S3 產出物儲存、系統音量控制 | -在 Linux 上安裝前，請先安裝以下系統套件： +**系統需求：** Python ≥ 3.10。Linux 請先安裝建置前置套件： ```bash sudo apt-get install cmake libssl-dev ``` ---- - -## 系統需求 - -- **Python** >= 3.10 -- **pip** >= 19.3 - -### 相依套件 - -| 套件 | 用途 | -|---|---| -| `je_open_cv` | 圖像辨識（OpenCV 模板匹配） | -| `pillow` | 截圖擷取 | -| `mss` | 快速多螢幕截圖 | -| `pyobjc` | macOS 後端（在 macOS 上自動安裝） | -| `python-Xlib` | Linux X11 後端（在 Linux 上自動安裝） | -| `PySide6` | GUI 應用程式（選用，使用 `[gui]` 安裝） | -| `qt-material` | GUI 主題（選用，使用 `[gui]` 安裝） | -| `uiautomation` | Windows Accessibility 後端（選用，首次使用時載入） | -| `pytesseract` + Tesseract | OCR 文字辨識（選用，首次使用時載入） | -| `anthropic` | VLM 定位 — Anthropic 後端（選用，首次使用時載入） | -| `openai` | VLM 定位 — OpenAI 後端（選用，首次使用時載入） | - -完整第三方相依套件與授權資訊請見 [Third_Party_License.md](../Third_Party_License.md)。 +OCR、VLM 與 LLM 後端（`pytesseract`、`easyocr`、`paddleocr`、`anthropic`、`openai`） +都是按需載入——只裝你實際會用到的。 --- -## 快速開始 - -想要可以直接複製貼上的完整腳本而不只是 API 片段？ -[`examples/`](../examples/) 資料夾收錄 17 支獨立範例：截圖+點擊、OCR、 -排程器、遠端桌面、agent loop、可觀測性、錄製/回放、執行期變數、 -視窗管理、熱鍵、影像觸發、HTML 報告、MCP stdio bridge、REST API、 -secret vault，以及外掛載入。 +## 60 秒上手 -### 滑鼠控制 - -```python -import je_auto_control - -# 取得目前滑鼠位置 -x, y = je_auto_control.get_mouse_position() -print(f"滑鼠位置: ({x}, {y})") - -# 移動滑鼠到指定座標 -je_auto_control.set_mouse_position(500, 300) - -# 在目前位置左鍵點擊（使用按鍵名稱） -je_auto_control.click_mouse("mouse_left") - -# 在指定座標右鍵點擊 -je_auto_control.click_mouse("mouse_right", x=800, y=400) - -# 向下滾動 -je_auto_control.mouse_scroll(scroll_value=5) -``` - -### 鍵盤控制 - -```python -import je_auto_control - -# 按下並釋放單一按鍵 -je_auto_control.type_keyboard("a") - -# 逐字輸入整個字串 -je_auto_control.write("Hello World") - -# 組合鍵（例如 Ctrl+C） -je_auto_control.hotkey(["ctrl_l", "c"]) - -# 檢查某個按鍵是否正在被按下 -is_pressed = je_auto_control.check_key_is_press("shift_l") -``` - -### 圖像辨識 - -```python -import je_auto_control - -# 在螢幕上找出所有符合的圖像 -positions = je_auto_control.locate_all_image("button.png", detect_threshold=0.9) -# 回傳: [[x1, y1, x2, y2], ...] - -# 找出單一圖像並取得其中心座標 -cx, cy = je_auto_control.locate_image_center("icon.png", detect_threshold=0.85) -print(f"找到位置: ({cx}, {cy})") - -# 找出圖像並自動點擊 -je_auto_control.locate_and_click("submit_button.png", mouse_keycode="mouse_left") -``` - -### Accessibility 元件搜尋 - -透過作業系統無障礙樹依名稱/角色/App 搜尋控制項（Windows UIA，via -`uiautomation`；macOS AX）。 - -```python -import je_auto_control - -# 列出 Calculator 中所有可見按鈕 -elements = je_auto_control.list_accessibility_elements(app_name="Calculator") - -# 搜尋特定元件 -ok = je_auto_control.find_accessibility_element(name="OK", role="Button") -if ok is not None: - print(ok.bounds, ok.center) - -# 一步定位並點擊 -je_auto_control.click_accessibility_element(name="OK", app_name="Calculator") -``` - -若當前平台無可用後端，會拋出 `AccessibilityNotAvailableError`。 - -### AI 元件定位（VLM） - -當模板匹配與 Accessibility 都失效時，可用自然語言描述元件，交給視覺 -語言模型取得座標。 - -```python -import je_auto_control - -# 預設偏好 Anthropic（若有設定 ANTHROPIC_API_KEY），否則用 OpenAI -x, y = je_auto_control.locate_by_description("綠色的 Submit 按鈕") - -# 一次定位並點擊 -je_auto_control.click_by_description( - "Cookie 橫幅中的『全部接受』按鈕", - screen_region=[0, 800, 1920, 1080], # 可選：只在此區域找 -) -``` - -設定（僅從環境變數讀取 — 金鑰不會被寫入程式碼或日誌）： - -| 變數 | 作用 | -|---|---| -| `ANTHROPIC_API_KEY` | 啟用 Anthropic 後端 | -| `OPENAI_API_KEY` | 啟用 OpenAI 後端 | -| `AUTOCONTROL_VLM_BACKEND` | 強制指定 `anthropic` 或 `openai` | -| `AUTOCONTROL_VLM_MODEL` | 覆寫預設模型（如 `claude-opus-4-7`、`gpt-4o-mini`） | - -若兩個 SDK 皆未安裝或未設定 API key，會拋出 `VLMNotAvailableError`。 - -### OCR 螢幕文字辨識 +**1. 當成 Python 函式庫** ```python import je_auto_control as ac -# 找出所有吻合的文字位置 -matches = ac.find_text_matches("Submit") - -# 取得第一個吻合位置的中心座標（找不到則回傳 None） -cx, cy = ac.locate_text_center("Submit") - -# 一步定位並點擊 -ac.click_text("Submit") +ac.set_mouse_position(500, 300) +ac.click_mouse("mouse_left") +ac.write("Hello World") +ac.hotkey(["ctrl_l", "s"]) -# 等待文字出現（或 timeout） -ac.wait_for_text("載入完成", timeout=15.0) +x, y = ac.locate_image_center("save_button.png", detect_threshold=0.9) +ac.click_text("Submit") # OCR +ac.click_accessibility_element(name="OK") # 無障礙樹 +ac.click_by_description("the green Submit button") # 視覺模型 +ac.screenshot("shot.png", screen_region=[0, 0, 800, 600]) ``` -選擇後端 — 設定 ``AUTOCONTROL_OCR_BACKEND=tesseract|easyocr|paddleocr`` -或在呼叫時傳入 ``backend=``；都不設定時會自動挑第一個 import 成功的： +**2. 當成 JSON 動作檔** — `flow.json` -```python -ac.find_text_matches("登入", lang="chi_tra", backend="easyocr") -ac.click_text("Sign in", backend="tesseract") +```json +[ + ["AC_set_var", {"name": "user", "value": "alice"}], + ["AC_locate_and_click", {"image": "login.png", "mouse_keycode": "mouse_left"}], + ["AC_write", {"write_string": "${user}"}], + ["AC_retry", {"max_attempts": 3, "body": [ + ["AC_wait_text", {"target": "Welcome", "timeout": 10}] + ]}], + ["AC_assert_text", {"text": "Welcome"}], + ["AC_generate_html_report", {"html_name": "report"}] +] ``` -若 Tesseract 不在 `PATH` 中，可手動指定路徑： - -```python -ac.set_tesseract_cmd(r"C:\Program Files\Tesseract-OCR\tesseract.exe") +```bash +je_auto_control run flow.json --var user=bob +je_auto_control run flow.json --dry-run # 只列出步驟，不會真的動滑鼠 ``` -各後端安裝路徑與標準語言代碼表見 -[docs/source/Eng/doc/ocr_backends/ocr_backends_doc.rst](../docs/source/Eng/doc/ocr_backends/ocr_backends_doc.rst) -或[繁體中文版本](../docs/source/Zh/doc/ocr_backends/ocr_backends_doc.rst)。 - -把區域（或整螢幕）內所有辨識到的文字 dump 出來，或用 regex 搜尋變動內容： - -```python -import je_auto_control as ac +**3. 當成桌面應用程式** -# TextMatch 列表，含文字、邊界框、信心度 -for match in ac.read_text_in_region(region=[0, 0, 800, 600]): - print(match.text, match.center, match.confidence) - -# Regex（接受字串或 compiled re.Pattern） -for match in ac.find_text_regex(r"Order#\d+"): - print(match.text, match.center) +```bash +pip install je_auto_control[gui] +python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui() ``` -GUI：**OCR Reader** 分頁。 +錄製一段流程、在視覺化 Script Builder 裡編輯，然後存成 CLI 能直接執行的同一種 JSON 格式。 -### LLM 動作規劃器 +--- -把自然語言描述交給 LLM（預設 Anthropic Claude），翻譯成驗證過的 `AC_*` 動作清單。輸出採寬鬆解析（會剝 code fence、從散文中抽出第一個 JSON array），再用 executor 同樣的 schema 驗證，所以結果可以直接餵給 `execute_action`： +## 能力總覽 -```python -import je_auto_control as ac -from je_auto_control.utils.executor.action_executor import executor +每一列都能無頭執行。「GUI 分頁」是同一功能在桌面應用中的位置；分頁的指令都放在視窗的 +**Actions** 選單裡。 -actions = ac.plan_actions( - "點擊 Submit 按鈕，然後輸入 'done' 並儲存", - known_commands=executor.known_commands(), -) -executor.execute_action(actions) +| 能力 | Python API | `AC_*` 指令 | GUI 分頁 | +|---|---|---|---| +| 滑鼠 | `click_mouse`、`set_mouse_position`、`mouse_scroll` | `AC_click_mouse` | Auto Click | +| 鍵盤 | `write`、`hotkey`、`type_keyboard` | `AC_write`、`AC_hotkey` | Auto Click | +| 螢幕與像素 | `screenshot`、`screen_size`、`get_pixel` | `AC_screenshot` | Screenshot | +| 影像比對 | `locate_image_center`、`locate_and_click` | `AC_locate_and_click` | Image Detect | +| OCR 文字 | `click_text`、`wait_for_text`、`read_text_in_region` | `AC_click_text`、`AC_wait_text` | OCR Reader | +| 無障礙樹 | `find_accessibility_element`、`click_accessibility_element` | `AC_a11y_find`、`AC_a11y_click` | Accessibility | +| 視覺模型定位 | `locate_by_description`、`click_by_description` | `AC_vlm_locate`、`AC_vlm_click` | VLM | +| 錨點定位 | — | `AC_anchor_click`、`AC_anchor_locate` | — | +| 自癒定位器 | `self_heal_click`、`self_heal_locate` | `AC_self_heal_click` | Self-Healing | +| 自然語言規劃 | `plan_actions`、`run_from_description` | `AC_llm_plan` | LLM Planner | +| Computer-use agent | `AgentLoop`、`run_agent` | `AC_run_agent` | Computer Use | +| 錄製與重播 | `record`、`stop_record` | `AC_record`、`AC_stop_record` | Record | +| JSON 腳本 | `execute_action`、`execute_files` | 全部 767 個指令 | Script、Script Builder | +| 變數與流程控制 | `execute_action_with_vars` | `AC_set_var`、`AC_loop`、`AC_for_each`、`AC_try`、`AC_retry` | Variables | +| 資料驅動執行 | — | `AC_for_each_row`（CSV／JSON／SQLite／Excel） | Data Sources | +| 斷言 | `assert_text`、`assert_image` | `AC_assert_text` 等 21 個 | Assertions | +| 測試套件 | `run_suite` | `AC_run_suite` | Test Suites | +| 排程（間隔 + cron） | `default_scheduler` | — | Scheduler | +| 全域熱鍵 | `default_hotkey_daemon` | — | Hotkeys | +| 事件觸發 | `default_trigger_engine` | `AC_email_trigger_add` | Triggers、Webhooks、Email | +| 視窗管理 *(僅 Windows)* | `list_windows`、`focus_window` | `AC_focus_window`、`AC_snap_window` | Window Manager | +| 剪貼簿（文字 + 影像） | `get_clipboard`、`set_clipboard`、`get_clipboard_image`、`set_clipboard_image` | `AC_clipboard_get`、`AC_clipboard_set`、`AC_clipboard_get_image`、`AC_clipboard_set_image` | — | +| 遠端桌面 | `RemoteDesktopHost`、`RemoteDesktopViewer` | `AC_start_remote_host`、`AC_remote_connect` | Remote Desktop | +| USB 列舉與直通 | `list_usb_devices`、`enable_usb_passthrough` | `AC_usb_*`（16 個指令） | USB Devices、USB Share | +| 機密保險庫 | `default_secret_manager` | `AC_secret_set` + `${secrets.NAME}` | Secrets | +| 報表（HTML／JSON／XML） | `generate_html_report` | `AC_generate_html_report` | Report | +| 執行歷史 | — | — | Run History | +| 指標與追蹤 | `default_metric_registry`、`render_metrics_text` | — | — | +| 系統診斷 | `run_diagnostics` | `AC_diagnose` | Diagnostics | +| 測試碼產生 | `generate_code` | — | — | + +除了這張表，`utils/` 底下還有 308 個無頭套件，涵蓋斷言、韌性、資料品質、i18n 稽核、遮蔽、 +治理、可觀測性等等。完整的逐模組地圖在 **[architecture_explore.md](../architecture_explore.md)**。 -# 或者一行做完： -ac.run_from_description("開記事本，輸入 hello", executor=executor) -``` +--- -| 變數 | 效果 | -|---|---| -| `ANTHROPIC_API_KEY` | 啟用 Anthropic 後端 | -| `AUTOCONTROL_LLM_BACKEND` | 強制指定 `anthropic` | -| `AUTOCONTROL_LLM_MODEL` | 覆寫預設模型（如 `claude-opus-4-7`） | +## 命令列介面 -GUI：**LLM Planner** 分頁 — 描述輸入框與指令清單預覽；*Plan*（`QThread` 背景執行）與 *Run plan* 位於視窗的 Actions 選單。 +```bash +je_auto_control run script.json [--var name=value] [--dry-run] +je_auto_control validate script.json # 別名：lint +je_auto_control fmt script.json [--check] +je_auto_control list-commands [--filter mouse] [--json] +je_auto_control record out.json [--duration 5] +je_auto_control codegen script.json --target pytest -o test_flow.py +je_auto_control failure-bundle failure.zip --error "login timed out" +je_auto_control list-jobs +je_auto_control start-server --port 9938 # TCP socket 伺服器 +je_auto_control start-rest --port 9939 # REST API +je_auto_control version +``` + +`--var name=value` 會盡量以 JSON 解析（`count=10` 會變成整數），否則視為字串。 +舊版 `python -m je_auto_control -e file.json` 進入點仍然可用。 -### 執行期變數與流程控制 +--- -executor 改成「每次呼叫」才解析 `${var}` placeholder（不會事先攤平），所以巢狀的 `body` / `then` / `else` 清單會保留 placeholder，每次重複執行時重新繫結。搭配新的變數修改指令，腳本可以資料驅動而不需要 Python 黏合： +## 伺服器與整合 -```json -[ - ["AC_set_var", {"name": "items", "value": ["alpha", "beta"]}], - ["AC_set_var", {"name": "i", "value": 0}], - ["AC_for_each", { - "items": "${items}", "as": "name", - "body": [ - ["AC_inc_var", {"name": "i"}], - ["AC_if_var", { - "name": "i", "op": "ge", "value": 2, - "then": [["AC_break"]], "else": [] - }] - ] - }] -] -``` +| 介面 | 啟動方式 | 說明 | +|---|---|---| +| **MCP 伺服器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 670 個工具，供 Claude Desktop／Claude Code／自訂 tool loop 使用。Bearer 驗證、TLS、稽核記錄、限流、外掛熱重載、CI 假後端。 | +| **REST API** | `je_auto_control start-rest` | Bearer token、逐 IP 限流與鎖定、SQLite 稽核 hook、`/metrics`、`/openapi.json`、`/docs` Swagger UI、`/dashboard`。 | +| **TCP socket 伺服器** | `je_auto_control start-server` | 以換行分隔的 JSON 動作清單。預設綁 `127.0.0.1`。 | +| **pytest 外掛** | 安裝後自動生效 | 提供 fixture 與供 pytest-bdd／behave 使用的 Gherkin step library。 | +| **語言伺服器** | `python -m autocontrol_lsp.server` | 為 `AC_*` 動作 JSON 提供補全與診斷，指令清單直接取自執行期的指令表。 | +| **遠端桌面** | `RemoteDesktopHost` 或 GUI | TCP、WebSocket 或 WebRTC；TOTP、信任清單、TURN 設定、檔案／剪貼簿／音訊同步。 | -`AC_if_var` 比較運算子：`eq`、`ne`、`lt`、`le`、`gt`、`ge`、`contains`、`startswith`、`endswith`。GUI：**Variables** 分頁 — 即時檢視 `executor.variables`，可單筆設定、JSON 整批 seed、清空。 +除非明確指定，所有伺服器都綁在 `127.0.0.1`。 -### 遠端桌面 +### 遠端桌面的線路協定 -把本機畫面串流給別人看／控制，**或** 觀看並控制別人的機器。協定是 raw TCP 上的長度前綴框架（沒有額外相依），先做一輪 HMAC-SHA256 challenge / response 認證；認證失敗的 viewer 在看到任何畫面前就被踢掉。JPEG frame 依設定的 FPS / 品質產生，透過共享 latest-frame slot 廣播給通過認證的 viewers，慢的 viewer 只會掉 frame 而不會卡其他人。Viewer 輸入訊息是 JSON，host 端用允許清單驗證後才透過既有 wrapper 派送。 +把主機開出去之前值得先知道，而且這段在其他文件裡都沒有寫。預設傳輸是**裸 TCP +上的長度前綴分幀**（不需要額外相依），連線一開始就是 **HMAC-SHA256 的 +challenge／response 握手**：驗證沒過的觀看端在拿到任何一張畫面之前就會被斷掉。 +JPEG 影格依設定的 FPS 與品質編碼，再透過一個共用的**最新影格槽**發給已驗證的 +觀看端——所以慢的觀看端是**掉影格**，不會把其他人一起卡住。觀看端送來的輸入是 +JSON，會先比對**動作允許清單**才交給既有的輸入包裝層執行，觀看端無法自己發明 +新的操作。 ```python -# 被遠端 — 啟動 host 把 token + port 給對方 +# 讓別人連進來——開一個主機，把 token 與 port 給對方 from je_auto_control import RemoteDesktopHost host = RemoteDesktopHost(token="hunter2", bind="127.0.0.1", - port=0, fps=10, quality=70) + port=0, fps=10, quality=70) host.start() print("listening on", host.port, "viewers:", host.connected_clients) ``` ```python -# 控制他機 — 連線 viewer 並送出輸入 +# 控制另一台機器——連上去並送輸入 from je_auto_control import RemoteDesktopViewer viewer = RemoteDesktopViewer(host="10.0.0.5", port=51234, token="hunter2", - on_frame=lambda jpeg: ...) + on_frame=lambda jpeg: ...) viewer.connect() viewer.send_input({"action": "mouse_move", "x": 100, "y": 200}) -viewer.send_input({"action": "type", "text": "hello"}) viewer.disconnect() ``` -GUI：**Remote Desktop** 分頁預設打開的是 **快速連線**（AnyDesk 風格）— 一邊是超大本機 Host ID，另一邊是一個輸入框接受 `host:port`、`ws://`、`wss://` 或 9 位數 Host ID，搭配 *連線* 與 *開始被遠端* 兩個主要按鈕。近期連線會跨 session 記住。進階的逐傳輸子分頁（既有 TCP / WS host + viewer、WebRTC host + viewer 含手動 SDP / 自訂編碼器 / TLS pinning）仍只差一個 click。WebRTC 子分頁採延遲載入，沒裝 `[webrtc]` extra 也能正常開啟整個分頁。 - -> ⚠️ 取得 host:port 與 token 的人，等同擁有本機完整滑鼠 / 鍵盤控制權。預設只綁 `127.0.0.1`；要對外暴露請務必搭配 SSH tunnel 或 TLS 前端。Token 是唯一防線 — 請當作密碼來保管。 - -**快速連線的 headless API**。撐起 GUI 輸入框的 transport coordinator 也對外開放，腳本可以走同樣的解析路徑： - -```python -from je_auto_control import parse_remote_desktop_target -parse_remote_desktop_target("192.168.1.10:5555") -# ConnectTarget(kind='tcp', host='192.168.1.10', port=5555, ...) -parse_remote_desktop_target("ws://hub:8765/desk") -# ConnectTarget(kind='ws', host='hub', port=8765, path='/desk') -parse_remote_desktop_target("123-456-789") -# ConnectTarget(kind='webrtc_id', host_id='123456789') -``` - -**連線審批 + 僅檢視模式**。可選的 callback 守住每一個 incoming session，AnyDesk 風格。回傳 `"view_only"` admit 但丟掉 viewer 的 `INPUT`；回傳 falsy（或 raise）就送 `AUTH_FAIL "rejected by host"`： - -```python -from je_auto_control import RemoteDesktopHost, PendingViewer - -def gate(p: PendingViewer) -> str: - if p.address[0].startswith("10."): - return "view_only" - return "full" # 或 True - -host = RemoteDesktopHost(token="tok", on_pending_viewer=gate) -``` - -**IP 白名單（CIDR + 單一 IP）**。在 TLS / auth 之前就拒絕範圍外的對端，攻擊者連探測都不行： - -```python -host = RemoteDesktopHost( - token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"], -) -``` - -**一次性分享碼** — 額外的 token，認證成功一次後自毀；客服支援流程很好用： - -```python -host = RemoteDesktopHost(token="tok", single_use_tokens=["abc123"]) -host.add_single_use_token("9k4ndx") # 運行時加 -host.revoke_single_use_token("abc123") # 還沒被用就先撤銷 -``` - -**TOTP 2FA（RFC 6238，純 stdlib）**。在 token 之上加一層 6 位數 OTP；host 接受 ±1 時間步的 clock drift： - -```python -from je_auto_control.utils.remote_desktop.totp import ( - generate_secret, generate_code, provisioning_uri, -) -secret = generate_secret() -print(provisioning_uri(secret, account="alice")) # 給 QR code 用的 otpauth:// URI - -host = RemoteDesktopHost(token="tok", totp_secret=secret) -viewer = RemoteDesktopViewer( - host=..., token="tok", totp_code=generate_code(secret), -) -``` - -**多螢幕選擇**。指定某一個螢幕擷取，而非合併虛擬桌面： - -```python -from je_auto_control import list_host_monitors, RemoteDesktopHost -print(list_host_monitors()) -# [{'index': 0, 'is_combined': True, ...}, -# {'index': 1, ...}, -# {'index': 2, ...}] -host = RemoteDesktopHost(token="tok", monitor_index=1) -``` - -**遠端游標 overlay**。host 每秒 30 Hz 廣播 cursor 位置（靜止桌面去重）；viewer 的彈出視窗會在 JPEG 串流上疊一個箭頭，看得到 host 滑鼠位置。可用 `enable_cursor_broadcast=False` 關掉。 - -**多 viewer 協作游標 + 文字 chat**。兩個新 message type（`CHAT` 與 `CURSOR` 帶 `viewer_id`）。搭配 `MultiViewerHost` 把一個 viewer 的指標 echo 給其他人；chat channel 給操作者之間臨時對話用： - -```python -host = RemoteDesktopHost( - token="tok", on_chat=lambda sender, text: print(sender, ":", text), -) -host.broadcast_chat("session starts in 30s") -host.broadcast_viewer_cursor("alice", 200, 300) - -viewer = RemoteDesktopViewer( - host=..., on_chat=lambda s, t: ..., - on_viewer_cursor=lambda vid, x, y: ..., -) -viewer.send_chat("ack") -``` - -**相對滑鼠模式（FPS / CAD）**。新輸入 action 送 delta 而非絕對座標： - -```python -viewer.send_input({"action": "mouse_move_relative", "dx": 5, "dy": -3}) -``` - -**動態擷取**。capture loop 會 hash 每張編碼後的 JPEG；重複 frame 直接跳過，所以靜止桌面幾乎零頻寬。新 viewer 在 auth 後立即拿到最新 frame，不會看到一片黑。 - -**即時統計**（FPS / kbps / 累計 — 3 秒滑動視窗）： - -```python -viewer.stats() -# {'fps': 24.3, 'kbps': 4801.2, 'frames': 720.0, 'bytes': 1.8e7, 'uptime': 30.2} -``` - -**JPEG 序列錄影（不需要 PyAV）**。TCP path 的 session 錄影：每張 frame 寫到磁碟，再加一份 `manifest.json` 讓播放器可以原速重放： - -```python -from je_auto_control.utils.remote_desktop.jpeg_recorder import ( - JpegSequenceRecorder, -) -rec = JpegSequenceRecorder("~/recordings/2026-05-23") -rec.start() -viewer = RemoteDesktopViewer(host=..., on_frame=rec.record_frame) -# ... session ... -rec.stop() # 在 .jpg 旁邊寫出 manifest.json -``` - -**TCP relay（WebRTC fallback）**。當 P2P 失敗（嚴格 NAT、行動電信 CGNAT、旅館 Wi-Fi）兩端都向 relay 主動連線、交換一個 32-byte session ID，relay 在中間互轉 bytes。同一模組附 `encode_handshake(role, session_id)` 給 client 用： - -```python -from je_auto_control.utils.remote_desktop.relay import RelayServer -relay = RelayServer(bind="0.0.0.0", port=9000) # NOSONAR # 對外 relay -relay.start() -``` - -**服務安裝器（無人值守 host）**。`python -m je_auto_control.utils.remote_desktop.host_service ...` 提供 `configure` / `init` / `run`，以及每個平台的安裝指令：`install-windows-service` / `uninstall-windows-service`（需 pywin32）、`generate-launchd` / `uninstall-launchd`、`generate-systemd` / `uninstall-systemd`。 - -**加密傳輸與替代協定**：傳 `ssl_context` 給 `RemoteDesktopHost` 或 `RemoteDesktopViewer` 即套上 TLS。要穿牆／給瀏覽器接，用內建的 WebSocket 版本（無額外相依），加 `ssl_context` 就變 `wss://`： - -```python -from je_auto_control import ( - WebSocketDesktopHost, WebSocketDesktopViewer, -) -host = WebSocketDesktopHost(token="hunter2", ssl_context=server_ctx) -viewer = WebSocketDesktopViewer( - host="example.com", port=443, token="hunter2", - ssl_context=client_ctx, expected_host_id="123456789", -) -``` - -**持久化 Host ID**：每台 host 有穩定的 9 位數字 ID（存在 `~/.je_auto_control/remote_host_id`），在 `AUTH_OK` 中宣告，viewer 透過 `expected_host_id` 驗證： - -```python -print(host.host_id) # 例如 "123456789" -viewer = RemoteDesktopViewer( - host=..., port=..., token=..., - expected_host_id="123456789", # 不一致就拋 AuthenticationError -) -``` - -**音訊串流（host → viewer）**：選用 `sounddevice` 相依；host 用 `AudioCaptureConfig` 開啟，viewer 端接 `AudioPlayer`（或自己的 callback）： - -```python -from je_auto_control.utils.remote_desktop import AudioCaptureConfig -host = RemoteDesktopHost( - token="tok", - audio_config=AudioCaptureConfig(enabled=True), # 預設 mic -) -# 或指定 loopback / monitor 裝置： -# audio_config=AudioCaptureConfig(enabled=True, device=12) - -from je_auto_control.utils.remote_desktop import AudioPlayer -player = AudioPlayer(); player.start() -viewer = RemoteDesktopViewer(host=..., on_audio=player.play) -``` - -**剪貼簿同步（文字 + 圖片，雙向）**：明確呼叫，沒有自動 polling 迴圈。圖片剪貼簿在 Windows（CF_DIB via ctypes）跟 Linux（`xclip -t image/png`）支援；macOS get 走 Pillow ImageGrab、set 暫時需要 PyObjC。 - -```python -viewer.send_clipboard_text("hello") -viewer.send_clipboard_image(open("logo.png", "rb").read()) -host.broadcast_clipboard_text("greetings") -``` - -**檔案傳輸 + 進度**：雙向、分塊、目的路徑任意、無大小上限；GUI viewer 還可以拖放： - -```python -viewer.send_file( - "local.bin", "/tmp/uploaded.bin", - on_progress=lambda tid, done, total: print(done, total), -) -host.send_file_to_viewers("local.bin", "/tmp/from_host.bin") -``` - -> ⚠️ 路徑無限制、大小無上限。任何拿到 token 的人都能把任意檔案寫到任意位置，也能塞滿磁碟 — 必須等同信任 token 持有者，或自己繼承 `FileReceiver` 在 `handle_begin` 內驗證 dest_path。 - -### 剪貼簿 - -```python -import je_auto_control as ac -ac.set_clipboard("hello") -text = ac.get_clipboard() -``` - -後端：Windows（Win32 + ctypes）、macOS（`pbcopy`/`pbpaste`）、Linux -（`xclip` 或 `xsel`）。 - -### 截圖 - -```python -import je_auto_control - -# 擷取全螢幕截圖並儲存 -je_auto_control.pil_screenshot("screenshot.png") - -# 擷取指定區域的截圖 [x1, y1, x2, y2] -je_auto_control.pil_screenshot("region.png", screen_region=[100, 100, 500, 400]) - -# 取得螢幕解析度 -width, height = je_auto_control.screen_size() - -# 取得指定座標的像素顏色 -color = je_auto_control.get_pixel(500, 300) -``` - -### 動作錄製與回放 - -```python -import je_auto_control -import time - -# 開始錄製滑鼠和鍵盤事件 -je_auto_control.record() - -time.sleep(10) # 錄製 10 秒 - -# 停止錄製並取得動作列表 -actions = je_auto_control.stop_record() - -# 回放前先清理錄製內容：把連續的滑鼠移動取樣壓縮成最後位置 -#（通常能把原始錄製縮小一個數量級，且不改變回放行為） -actions = je_auto_control.dedupe_moves(actions) - -# 重新播放錄製的動作 -je_auto_control.execute_action(actions) -``` - -> 非破壞式錄製編輯器（皆回傳新的 list）：`dedupe_moves`（壓縮滑鼠移動）、`merge_sleeps`（合併連續 `AC_sleep`）、`trim_actions`、`insert_action`、`remove_action`、`filter_actions`、`adjust_delays`（縮放 `AC_sleep` 延遲）、`scale_coordinates`（以不同解析度回放）。透過 MCP 暴露為 `ac_dedupe_moves` / `ac_merge_sleeps` / `ac_trim_actions` / `ac_adjust_delays` / `ac_scale_coordinates`。 - -### JSON 腳本執行器 - -建立 JSON 動作檔案（`actions.json`）： - -```json -[ - ["AC_set_mouse_position", {"x": 500, "y": 300}], - ["AC_click_mouse", {"mouse_keycode": "mouse_left"}], - ["AC_write", {"write_string": "Hello from AutoControl"}], - ["AC_screenshot", {"file_path": "result.png"}], - ["AC_hotkey", {"key_code_list": ["ctrl_l", "s"]}] -] -``` - -執行方式： - -```python -import je_auto_control - -# 從檔案執行 -je_auto_control.execute_action(je_auto_control.read_action_json("actions.json")) - -# 或直接從列表執行 -je_auto_control.execute_action([ - ["AC_set_mouse_position", {"x": 100, "y": 200}], - ["AC_click_mouse", {"mouse_keycode": "mouse_left"}] -]) -``` - -**可用的動作命令：** - -| 類別 | 命令 | -|---|---| -| 滑鼠 | `AC_click_mouse`, `AC_set_mouse_position`, `AC_get_mouse_position`, `AC_get_mouse_table`, `AC_press_mouse`, `AC_release_mouse`, `AC_mouse_scroll`, `AC_mouse_left`, `AC_mouse_right`, `AC_mouse_middle` | -| 鍵盤 | `AC_type_keyboard`, `AC_press_keyboard_key`, `AC_release_keyboard_key`, `AC_write`, `AC_hotkey`, `AC_check_key_is_press`, `AC_get_keyboard_keys_table` | -| 圖像 | `AC_locate_all_image`, `AC_locate_image_center`, `AC_locate_and_click` | -| 螢幕 | `AC_screen_size`, `AC_screenshot` | -| Accessibility | `AC_a11y_list`, `AC_a11y_find`, `AC_a11y_click` | -| VLM（AI 定位） | `AC_vlm_locate`, `AC_vlm_click` | -| OCR | `AC_locate_text`, `AC_click_text`, `AC_wait_text`, `AC_read_text_in_region`, `AC_find_text_regex` | -| LLM 規劃器 | `AC_llm_plan`, `AC_llm_run` | -| 剪貼簿 | `AC_clipboard_get`, `AC_clipboard_set` | -| 視窗 | `AC_list_windows`, `AC_focus_window`, `AC_wait_window`, `AC_close_window` | -| 流程控制 | `AC_loop`, `AC_break`, `AC_continue`, `AC_if_image_found`, `AC_if_pixel`, `AC_if_var`, `AC_while_image`, `AC_while_var`, `AC_for_each`, `AC_wait_image`, `AC_wait_pixel`, `AC_sleep`, `AC_retry`, `AC_try` | -| 變數 | `AC_set_var`, `AC_get_var`, `AC_inc_var` | -| 遠端桌面 | `AC_start_remote_host`, `AC_stop_remote_host`, `AC_remote_host_status`, `AC_remote_connect`, `AC_remote_disconnect`, `AC_remote_viewer_status`, `AC_remote_send_input` | -| 錄製 | `AC_record`, `AC_stop_record`, `AC_set_record_enable` | -| 報告 | `AC_generate_html`, `AC_generate_json`, `AC_generate_xml`, `AC_generate_html_report`, `AC_generate_json_report`, `AC_generate_xml_report` | -| 執行紀錄 | `AC_history_list`, `AC_history_clear` | -| 專案 | `AC_create_project` | -| Shell | `AC_shell_command` | -| 程序 | `AC_execute_process` | -| 執行器 | `AC_execute_action`, `AC_execute_files`, `AC_add_package_to_executor`, `AC_add_package_to_callback_executor` | -| MCP 伺服器 | `AC_start_mcp_server`, `AC_start_mcp_http_server` | - -### MCP 伺服器（讓 Claude 使用 AutoControl） - -把 AutoControl 包裝成 Model Context Protocol 服務,任何支援 MCP 的 -client(Claude Desktop、Claude Code、自訂 Anthropic / OpenAI tool-use -迴圈)都能驅動本機桌面。純 stdlib — JSON-RPC 2.0 走 stdio 或 HTTP+ -SSE。 - -**註冊到 Claude Code:** - -```bash -claude mcp add autocontrol -- python -m je_auto_control.utils.mcp_server -``` - -**註冊到 Claude Desktop**(`claude_desktop_config.json`): - -```json -{ - "mcpServers": { - "autocontrol": { - "command": "python", - "args": ["-m", "je_auto_control.utils.mcp_server"] - } - } -} -``` - -**程式啟動:** - -```python -import je_auto_control as ac - -# Stdio(會阻塞直到 stdin 關閉) -ac.start_mcp_stdio_server() - -# 或 HTTP / SSE,含 Bearer token 驗證 + 可選 TLS -ac.start_mcp_http_server(host="127.0.0.1", port=9940, - auth_token="hunter2") -``` - -**不啟動伺服器、只看目錄:** - -```bash -je_auto_control_mcp --list-tools -je_auto_control_mcp --list-tools --read-only -je_auto_control_mcp --list-resources -je_auto_control_mcp --list-prompts -``` - -**功能總覽:** - -| 面向 | 涵蓋 | -|---|---| -| 工具(約 90 個) | 滑鼠 · 鍵盤 · drag · 螢幕 / 多螢幕 · 截圖回 image · diff · OCR · 影像 · 視窗(move/min/max/restore/...) · 剪貼簿文字+圖像 · 程序 / shell · 動作錄製 · 螢幕錄影 · scheduler / triggers / hotkeys · accessibility tree · VLM · executor · history | -| 別名 | `click`、`type`、`screenshot`、`find_image`、`drag`、`shell`、`wait_image`...,以 `JE_AUTOCONTROL_MCP_ALIASES=0` 關閉 | -| Resources | `autocontrol://files/`、`autocontrol://history`、`autocontrol://commands`、`autocontrol://screen/live`(支援 `resources/subscribe`)| -| Prompts | `automate_ui_task`、`record_and_generalize`、`compare_screenshots`、`find_widget`、`explain_action_file` | -| 協定 | tools / resources / prompts / sampling / roots / logging / progress / cancellation / list_changed / elicitation | -| 傳輸 | stdio、HTTP `POST /mcp`、`Accept: text/event-stream` 時走 SSE 串流 | -| 安全 | 工具註記 · `JE_AUTOCONTROL_MCP_READONLY` · `JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE` · 稽核 log · token-bucket rate limiter · 工具失敗自動截圖 | -| 部署 | Bearer token 驗證 · 透過 `ssl_context` 啟用 TLS · `PluginWatcher` 熱重載 · `JE_AUTOCONTROL_FAKE_BACKEND=1` 給 CI | - -完整參考請見 [docs/source/Zh/doc/mcp_server/mcp_server_doc.rst](../docs/source/Zh/doc/mcp_server/mcp_server_doc.rst) -(英文版本在 [docs/source/Eng/doc/mcp_server/mcp_server_doc.rst](../docs/source/Eng/doc/mcp_server/mcp_server_doc.rst))。 - -> ⚠️ MCP 伺服器可以移動滑鼠、送鍵盤事件、截圖、執行任意 `AC_*` 動 -> 作。請只註冊給可信任的 client。HTTP 預設綁 `127.0.0.1`,要對外 -> 必須要有明確理由,**並且**搭配 `auth_token` 與 `ssl_context`。 - -### 排程器（Interval & Cron） - -```python -import je_auto_control as ac - -# Interval：每 30 秒執行一次 -job = ac.default_scheduler.add_job( - script_path="scripts/poll.json", interval_seconds=30, repeat=True, -) - -# Cron：週一到週五 09:00（欄位為 minute hour dom month dow） -cron_job = ac.default_scheduler.add_cron_job( - script_path="scripts/daily.json", cron_expression="0 9 * * 1-5", -) - -ac.default_scheduler.start() -``` - -兩種排程可同時存在，可由 `job.is_cron` 判斷類型。 - -### 全域熱鍵 - -將 OS 熱鍵綁定到 action JSON 腳本。跨平台 — Windows 用 -`RegisterHotKey`、macOS 用 `CGEventTap`（需要 Accessibility 權限）、 -Linux X11 用 `XGrabKey`（不支援 Wayland）。呼叫端三個平台一樣， -daemon 在 `start()` 時自動挑後端。 - -```python -from je_auto_control import default_hotkey_daemon - -default_hotkey_daemon.bind("ctrl+alt+1", "scripts/greet.json") -default_hotkey_daemon.start() -``` - -### 事件觸發器 - -輪詢式觸發器，偵測到條件成立時自動執行腳本： - -```python -from je_auto_control import ( - default_trigger_engine, ImageAppearsTrigger, - WindowAppearsTrigger, PixelColorTrigger, FilePathTrigger, -) - -default_trigger_engine.add(ImageAppearsTrigger( - trigger_id="", script_path="scripts/click_ok.json", - image_path="templates/ok_button.png", threshold=0.85, repeat=True, -)) -default_trigger_engine.start() -``` - -### 執行歷史 - -排程器、觸發器、熱鍵、REST API 與 GUI 手動回放的每一次執行都會被寫入 -`~/.je_auto_control/history.db`。錯誤時會自動在 -`~/.je_auto_control/artifacts/run_{id}_{ms}.png` 附上截圖以便除錯。 - -```python -from je_auto_control import default_history_store - -for run in default_history_store.list_runs(limit=20): - print(run.id, run.source, run.status, run.artifact_path) -``` - -GUI **執行歷史** 分頁顯示執行紀錄表格，可雙擊截圖欄位開啟附件；篩選 / -更新 / 清除指令位於視窗的 Actions 選單。 - -### 報告產生 - -```python -import je_auto_control - -# 先啟用測試紀錄 -je_auto_control.test_record_instance.set_record_enable(True) - -# ... 執行自動化動作 ... -je_auto_control.set_mouse_position(100, 200) -je_auto_control.click_mouse("mouse_left") - -# 產生報告 -je_auto_control.generate_html_report("test_report") # -> test_report.html -je_auto_control.generate_json_report("test_report") # -> test_report.json -je_auto_control.generate_xml_report("test_report") # -> test_report.xml - -# 或取得報告內容為字串 -html_string = je_auto_control.generate_html() -json_string = je_auto_control.generate_json() -xml_string = je_auto_control.generate_xml() -``` - -報告內容包含：每個紀錄動作的函式名稱、參數、時間戳記及例外資訊（如有）。HTML 報告中成功的動作以青色顯示，失敗的動作以紅色顯示。 - -### 可觀測性（Prometheus / OpenTelemetry） - -純標準函式庫的 metric 元件加上 OpenTelemetry 相容 tracer， -executor 與 agent loop 都會自動發送呼叫次數與延遲分布 metric， -不用手動 instrument。 +也可以用 IP 允許清單（CIDR 網段或個別位址）限制誰連得進來，清單外的對端在握手 +階段就會被拒絕： ```python -import je_auto_control as ac - -# 在 http://127.0.0.1:9090 開放 /metrics，給 Prometheus scrape。 -exporter = ac.default_metrics_exporter() -exporter.start() - -# 自訂 metric — 形狀與 prometheus_client 相同。 -counter = ac.default_metric_registry().register(ac.MetricCounter( - "myapp_widgets_built_total", "widgets built", - label_names=("kind",), -)) -counter.inc(labels={"kind": "blue"}) - -# 把 callable 包進 span — 未安裝 opentelemetry-api 時為 no-op。 -@ac.traced("my_pipeline.process_one") -def process_one(item): ... -``` - -內建 metric 清單見 -[docs/source/Eng/doc/observability/observability_doc.rst](../docs/source/Eng/doc/observability/observability_doc.rst) -或[繁體中文版本](../docs/source/Zh/doc/observability/observability_doc.rst)。 - -### 遠端自動化（Socket / REST） - -提供兩種伺服器：原始 TCP socket 與純 stdlib HTTP/REST。預設均綁定 -`127.0.0.1`，綁定到 `0.0.0.0` 須明確指定。 - -```python -import je_auto_control as ac - -# TCP Socket 伺服器（預設：127.0.0.1:9938） -ac.start_autocontrol_socket_server(host="127.0.0.1", port=9938) - -# REST API 伺服器（預設：127.0.0.1:9939） -ac.start_rest_api_server(host="127.0.0.1", port=9939) -# 端點： -# GET /health 存活檢查 -# GET /jobs 列出排程工作 -# POST /execute body: {"actions": [...]} -``` - -### 外掛載入器 - -將定義頂層 `AC_*` 可呼叫物的 `.py` 檔放進一個目錄，執行時即可註冊成 -executor 指令： - -```python -from je_auto_control import ( - load_plugin_directory, register_plugin_commands, -) - -commands = load_plugin_directory("./my_plugins") -register_plugin_commands(commands) - -# 之後任何 JSON 腳本都能使用： -# [["AC_greet", {"name": "world"}]] -``` - -> **警告：** 外掛檔案會直接執行任意 Python，請僅載入自己信任的目錄。 - -### Shell 命令執行 - -```python -import je_auto_control - -# 使用預設的 Shell 管理器 -je_auto_control.default_shell_manager.exec_shell("echo Hello") -je_auto_control.default_shell_manager.pull_text() # 輸出擷取的結果 - -# 或建立自訂的 ShellManager -shell = je_auto_control.ShellManager(shell_encoding="utf-8") -shell.exec_shell("ls -la") -shell.pull_text() -shell.exit_program() +RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) ``` -### 螢幕錄製 - -```python -import je_auto_control -import time - -# 方法一：ScreenRecorder（管理多個錄影） -recorder = je_auto_control.ScreenRecorder() -recorder.start_new_record( - recorder_name="my_recording", - path_and_filename="output.avi", - codec="XVID", - frame_per_sec=30, - resolution=(1920, 1080) -) -time.sleep(10) -recorder.stop_record("my_recording") - -# 方法二：RecordingThread（簡易單一錄影，輸出 MP4） -recording = je_auto_control.RecordingThread(video_name="my_video", fps=20) -recording.start() -time.sleep(10) -recording.stop() -``` - -### 回呼執行器 - -執行自動化函式後自動觸發回呼函式： - -```python -import je_auto_control - -def my_callback(): - print("動作完成！") - -# 執行 set_mouse_position 後呼叫 my_callback -je_auto_control.callback_executor.callback_function( - trigger_function_name="AC_set_mouse_position", - callback_function=my_callback, - x=500, y=300 -) - -# 帶有參數的回呼 -def on_done(message): - print(f"完成: {message}") - -je_auto_control.callback_executor.callback_function( - trigger_function_name="AC_click_mouse", - callback_function=on_done, - callback_function_param={"message": "點擊完成"}, - callback_param_method="kwargs", - mouse_keycode="mouse_left" -) -``` - -### 套件管理器 - -在執行時動態載入外部 Python 套件到執行器中： - -```python -import je_auto_control - -# 將套件的所有函式/類別加入執行器 -je_auto_control.package_manager.add_package_to_executor("os") - -# 現在可以在 JSON 動作腳本中使用 os 函式： -# ["os_getcwd", {}] -# ["os_listdir", {"path": "."}] -``` - -### 專案管理 - -快速建立包含範本檔案的專案目錄結構： - -```python -import je_auto_control - -# 建立專案結構 -je_auto_control.create_project_dir(project_path="./my_project", parent_name="AutoControl") - -# 會建立以下結構： -# my_project/ -# └── AutoControl/ -# ├── keyword/ -# │ ├── keyword1.json # 範本動作檔案 -# │ ├── keyword2.json # 範本動作檔案 -# │ └── bad_keyword_1.json # 錯誤處理範本 -# └── executor/ -# ├── executor_one_file.py # 執行單一檔案範例 -# ├── executor_folder.py # 執行資料夾範例 -# └── executor_bad_file.py # 錯誤處理範例 -``` - -### 視窗管理 - -直接將事件送至指定視窗（僅限 Windows 和 Linux）： - -```python -import je_auto_control - -# 透過視窗標題送出鍵盤事件 -je_auto_control.send_key_event_to_window("Notepad", keycode="a") - -# 透過視窗 handle 送出滑鼠事件 -je_auto_control.send_mouse_event_to_window(window_handle, mouse_keycode="mouse_left", x=100, y=50) -``` - -### GUI 應用程式 - -啟動內建圖形介面（需安裝 `[gui]` 擴充）： - -```python -import je_auto_control -je_auto_control.start_autocontrol_gui() -``` - -或透過命令列： - -```bash -python -m je_auto_control -``` - -主視窗採選單驅動設計：分頁只保留輸入欄位、表格與結果檢視，每個分頁的 -指令都集中在視窗層級的 **Actions** 選單，會隨當前分頁動態重建。 -**View → Tabs** 可依分類（核心 / 編輯 / 偵測與視覺 / 自動化引擎 / 系統） -顯示或隱藏約 48 個已註冊分頁；預設版面只開啟錄製、腳本建構器與遠端桌面 -三個分頁。**View → Text Size** 提供自動／預設字級，**Language** 選單 -（English / 繁體中文 / 简体中文 / 日本語）可即時切換整個視窗的語系。 - --- -## 命令列介面 - -AutoControl 可直接從命令列使用： - -```bash -# 執行單一動作檔案 -python -m je_auto_control -e actions.json - -# 執行目錄中所有動作檔案 -python -m je_auto_control -d ./action_files/ - -# 直接執行 JSON 字串 -python -m je_auto_control --execute_str '[["AC_screenshot", {"file_path": "test.png"}]]' - -# 建立專案範本 -python -m je_auto_control -c ./my_project -``` - -另外還有以 headless API 為基礎的子命令 CLI： - -```bash -# 執行腳本（可帶變數或 dry-run） -python -m je_auto_control.cli run script.json -python -m je_auto_control.cli run script.json --var name=alice --dry-run - -# 列出排程工作 -python -m je_auto_control.cli list-jobs - -# 啟動 Socket / REST 伺服器 -python -m je_auto_control.cli start-server --port 9938 -python -m je_auto_control.cli start-rest --port 9939 -``` +## 平台支援 -`--var name=value` 會優先以 JSON 解析（`count=10` 會變成 int），失敗 -則視為字串。 +| 平台 | 後端 | 輸入 | 螢幕擷取 | 錄製 | 視窗管理 | +|---|---|:---:|:---:|:---:|:---:| +| Windows 10 / 11 | Win32 ctypes（可選 Interception 驅動） | ✅ | ✅ | ✅ | ✅ | +| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | +| Linux X11 | python-Xlib（可選 `uinput`） | ✅ | ✅ | ✅ | ❌ | +| Linux Wayland | libei，或 ydotool／wtype／grim | ✅ | ✅ | ❌ | ❌ | +| Android | adb + uiautomator2 | ✅ | ✅ | — | — | +| iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | + +Wayland 禁止非特權用戶端進行全域輸入錄製——若要錄製，請設定 +`JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` 並在 X11 session 下執行。視窗管理目前僅 +Windows 有實作，其他平台會拋出明確的 `NotImplementedError`。對於會忽略合成輸入的應用程式， +可選用驅動層後端（`JE_AUTOCONTROL_WIN32_BACKEND=interception`、 +`JE_AUTOCONTROL_LINUX_BACKEND=uinput`、ViGEm 虛擬手把）；驅動未安裝時會自動退回原本行為。 --- -## 平台支援 +## 文件與範例 -| 平台 | 狀態 | 後端 | 備註 | -|---|---|---|---| -| Windows 10 / 11 | 支援 | Win32 API (ctypes) | 完整功能支援 | -| macOS 10.15+ | 支援 | pyobjc / Quartz | 不支援動作錄製；不支援 `send_key_event_to_window` / `send_mouse_event_to_window` | -| Linux（X11） | 支援 | python-Xlib | 完整功能支援 | -| Linux（Wayland） | 尚未支援 | — | 未來版本可能加入支援 | -| Raspberry Pi 3B / 4B | 支援 | python-Xlib | 在 X11 上運行 | +| 資源 | 內容 | +|---|---| +| [`examples/`](../examples/) | 27 個自足腳本：截圖點擊、OCR、排程器、遠端桌面、agent loop、可觀測性、錄製、變數、熱鍵、觸發器、報表、MCP、REST、機密、外掛、computer use、Wayland、跨主機 DAG、chat-ops、pytest/BDD、錨點定位。 | +| [Read the Docs](https://autocontrol.readthedocs.io/en/latest/) | 完整 API 參考，含英文與中文。 | +| [architecture_explore.md](../architecture_explore.md) | 逐層記錄每個模組的職責。 | +| [docs/CAPABILITY_MATRIX.md](../docs/CAPABILITY_MATRIX.md) | 能力 × 平台對照矩陣。 | +| [docs/API_LIFECYCLE.md](../docs/API_LIFECYCLE.md) | 穩定 API 與棄用政策。 | +| [WHATS_NEW.md](../WHATS_NEW.md) | 各版本更新說明。 | +| [CHANGELOG.md](../CHANGELOG.md) | 相容性變更記錄。 | +| [SECURITY.md](../SECURITY.md) | 安全政策與回報方式。 | --- ## 開發 -### 環境設定 - ```bash git clone https://github.com/Intergration-Automation-Testing/AutoControl.git cd AutoControl pip install -r dev_requirements.txt +uv sync # 或：以已提交的 uv.lock 做可重現安裝 ``` -可重現的安裝走已 commit 的 `uv.lock`： - -```bash -uv sync # 依鎖檔同步整條相依鏈 -uv lock --upgrade # 編輯 pyproject.toml 後重新鎖 -``` - -### 執行測試 - ```bash -# 單元測試 -python -m pytest test/unit_test/ +python -m pytest test/unit_test/headless # 無頭單元測試 +python -m pytest test/integrated_test/ # 跨模組流程測試 -# 整合測試 -python -m pytest test/integrated_test/ +ruff check je_auto_control/ +pylint je_auto_control/ +bandit -c pyproject.toml -r je_auto_control/ ``` -### 專案連結 - -- **首頁**: https://github.com/Intergration-Automation-Testing/AutoControl -- **文件**: https://autocontrol.readthedocs.io/en/latest/ -- **PyPI**: https://pypi.org/project/je_auto_control/ +歡迎貢獻——請見 [CONTRIBUTING.md](../CONTRIBUTING.md) 與 +[CODE_OF_CONDUCT.md](../CODE_OF_CONDUCT.md)。CI 會強制兩條規則：`import je_auto_control` +絕不能載入 PySide6；每個功能都必須同時具備無頭 API 與 GUI 介面。 --- -## 授權條款 +## 授權 [MIT License](../LICENSE) © JE-Chen。 -第三方相依套件之授權請見 -[Third_Party_License.md](../Third_Party_License.md)。 +內含與選用第三方元件的授權請見 [Third_Party_License.md](../Third_Party_License.md)。 + +- **首頁**：https://github.com/Intergration-Automation-Testing/AutoControl +- **PyPI**：https://pypi.org/project/je_auto_control/ +- **文件**：https://autocontrol.readthedocs.io/en/latest/ diff --git a/WHATS_NEW.md b/WHATS_NEW.md index c878728c..fe09f4fc 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,226 @@ # What's New — AutoControl +## What's new (2026-08-15) + +### Text Entry and On-Screen Location That Match What You See + +Three defects that made automation miss silently rather than fail loudly: text +that could not be typed, targets that could not be found, and coordinates that +were subtly wrong. All three produced "sometimes it works" behaviour, which is +harder to diagnose than a crash. + +- **Type anything, without the clipboard** (`type_unicode_keys`, + `type_unicode_text`, `AC_type_unicode_keys`, `AC_type_unicode_text`, + `ac_type_unicode_keys`, `ac_type_unicode_text`): `write` typed through the + 192-entry virtual-key table and *raised* on the first character outside it — + which on a US layout means `, . / : ? ! _ + @ %` as well as all CJK, so a URL + or a Chinese sentence failed as a whole string. The Windows backend gains + `press_unicode` / `release_unicode` / `type_unicode_unit` built on the + `KEYEVENTF_UNICODE` flag its `KeyboardInput` already understood, and `write` + now falls back to that route per character instead of raising. The existing + clipboard-paste `type_unicode` stays, but is no longer the only option: it + overwrites the user's clipboard and is refused by inputs that block paste, so + key injection is the default where a backend supports it. +- **Find text the engine split across boxes** (`find_spans`, `group_lines`): OCR + backends box one *word* at a time, so `Save As` and `另存新檔` had no single + box to compare against and `find_text_matches` reported nothing for text + plainly on screen. Matching now scans runs of consecutive boxes on a line — + grouped by vertical overlap, so it also works for backends that report no line + ids — and returns the shortest run that spells the target, with the union + rectangle and the weakest word's confidence. A one-box run is the old + behaviour, so nothing that matched before stops matching. +- **Capture in the coordinate space the mouse uses** (`grab_logical`, + `logical_virtual_rect`, `logical_scale`, `needs_rescale`): `find_image` and + `find_image_multi` captured through `ImageGrab.grab()`, which sees only the + primary monitor — a target on a second display could never be found. The + full-desktop capture has the opposite problem: it returns *physical* pixels + while a DPI-unaware process clicks in *logical* ones, so on a mixed-DPI + desktop (3840 physical against 3456 logical) a point read off the frame lands + up to ~116 px away. `monitor_layout.grab_logical` is now the single capture + primitive behind both OCR and template matching: it covers every monitor, + rescales into logical pixels, and reports the virtual-desktop origin to add to + a hit — which is negative whenever a monitor sits left of or above the + primary. Template matches are translated by that origin, so a located box is + directly clickable. `find_image` / `find_image_multi` also gained + `all_screens` and `screen_region`. +- **`visual_match` reports coordinates you can act on.** The scored matcher had + the same three problems plus one of its own: it captured through + `pil_screenshot` (primary monitor, physical pixels), and a hit found inside a + `region` was returned in *region-local* coordinates — so clicking a match was + wrong by the region's own offset. It now grabs through `grab_logical` and adds + the origin to every hit; a caller-supplied `haystack` is still its own + coordinate space, as it must be. Two more traps closed: a template that is + almost a single colour now raises `AutoControlFlatTemplateException` instead + of saturating the score map at 1.0 and "finding" the target at an arbitrary + position, and templates load through `imdecode` so a **non-ASCII path** no + longer reads as a corrupt file. Everything downstream of `_haystack_gray` + (`barcode`, `edge_lines`, `edge_match`) inherits the corrected capture. +- **Accessibility search that can actually find the control** (`window_title` + on `list_accessibility_elements` / `find_accessibility_element`, new + `find_accessibility_elements`, `contains`, `accessibility_status`, + `control_get_state`; commands `AC_a11y_find_all` / `AC_control_get_state`, + MCP `ac_a11y_find_all` / `ac_control_get_state`). Four things stood between + the API and a real target: the name had to match **exactly**, so a label + carrying an accelerator (`Save(&S)`) or trailing padding never matched; + `role="button"` never matched either, because the Windows backend reports the + raw `ControlType_50000` and nothing translated it; `max_results` truncated the + list *before* filtering, so an element past the cap could not be found however + specific the filter; and there was no way to search one window. Scoping is the + one that matters most for speed — measured on a busy desktop, walking + everything is 2,085 elements in **61 s** against 135 elements in **0.14 s** for + a single window. `contains` matching ranks an exact name first, so "OK" offers + the `OK` button before `OK and close`. `control_get_state` answers what pixels + cannot — a field's text scrolled out of view, a checkbox's true state, a + slider's exact number — in one call, with an absent key meaning "no such + state" rather than "empty"; password fields report only that they are password + fields, on both `get_value` and `get_state`, because UIA's masking is a + convention a custom-drawn control can ignore. Elements now also carry + `enabled`: a disabled control looks clickable and silently swallows the click. + Conversion cost is gone too — properties come back through one + `FindAllBuildCache` call instead of one cross-process read per property, and + the per-element `OpenProcess` for the app name is cached (converting 500 + elements: 0.02 s). + + **A desktop-wide search went from 61 s to about 2 s**, which took three + separate fixes because there were three separate causes: + + 1. *The root.* One `FindAll` from the desktop walks every window's subtree and + cannot be interrupted. An unscoped listing now takes **one top-level window + at a time in z-order**, so it can stop as soon as it has enough — and the + window the user is looking at is searched first. + 2. *The walk.* Even per window, `FindAll` is atomic: one 34,507-element window + took 10.3 s to answer a request for 200. The walk is now node by node + through `ControlViewWalker` with a cache request, so asking for 200 + elements costs 200 elements of work (0.036 s for 50, 0.114 s for 200, + 0.486 s for 1,000). + 3. *The provider.* UIA waits on the application itself. A full-screen game + that never answers made a single `ElementFromHandle` block for **60 s** — + and it was not detectable in advance: the window pumps messages, replies to + `WM_GETOBJECT`, and `IsHungAppWindow` says it is fine. The automation object + now comes from `CUIAutomation8` as `IUIAutomation2` with + `ConnectionTimeout` bounded, which brings that same call to 1.0 s. + + Measured end to end afterwards: 50 elements 0.20 s, 200 in 1.29 s, 1,000 in + 1.90 s, 3,000 in 3.87 s. Naming a window is still an order of magnitude + better (0.03 s) and remains the advice. + + `find_*` also separates `max_results` (how many matches to return) from + `scan_limit` (how many elements to examine) — one number cannot mean both, and + conflating them turns "up to 40 buttons" into "only look at the first 40 + elements on the desktop". + +### Telling You When Your Input Is Going Nowhere + +Sent input can be discarded before it reaches anything while the send call still +reports success — the caller is told "clicked (500, 300)", nothing happens, and +no error exists anywhere. `utils/input_reach` (`input_desktop_available`, +`input_reaches_system`, `AC_input_reachable`, `ac_input_reachable`) answers the +question directly. + +Two causes needing two checks. A locked workstation is detectable for free by +asking for the input desktop. Input *filtering* is not: measured on a machine +with an anti-cheat game in front, `SendInput` succeeds, `GetAsyncKeyState` never +sees the key, `OpenInputDesktop` reports everything fine, and the game's +integrity level is the same *Medium* as ours — so neither a privilege comparison +nor any cheap query can tell. The only honest test is to send a key and look, +which is why that probe is a diagnostic (it presses F13, which nothing binds) +rather than a gate in front of every action. + +### A Recording That Can Actually Be Replayed + +`record` captured presses and nothing else, which is not enough to reproduce a +session — and the gap was silent, because the recording looked fine until it was +played back. Three things were missing and one was leaking: + +- **Releases.** A press-only log cannot tell a drag from a click, and a modifier + held across several actions cannot be reconstructed. Verified before the + change: five press-and-release pairs produced five events. +- **The wheel.** `WM_MOUSEWHEEL` was not handled at all, so scrolling vanished. + Its `mouseData` high word is a *signed* notch count — read unsigned, a scroll + down becomes a scroll up by 65,534 notches. +- **Timing.** Without timestamps every step replays at once and no real + interface keeps up. `utils/input_macro` already had `replay_timeline` waiting + for `delta_ms` events that nothing produced. +- **A leaked thread per recording.** The listener pumped `GetMessage` once and + `stop_record` never woke it, so each record cycle left a thread blocked + forever. Verified: the thread was still alive after `stop_record` returned. + +`Win32InputHook` replaces both listeners with one hook that records press *and* +release, wheel deltas and a monotonic timestamp, pumps messages properly, and +exits on `WM_QUIT` when stopped. `stop_record_timeline` (`AC_stop_record_timeline`, +`ac_record_stop_timeline`) returns those events with `delta_ms`, ready for +`replay_timeline`. `stop_record` is untouched and still returns the historical +press-only queue, so existing callers keep working. + +New `utils/keyboard_layout` answers the other half: which character a key +produces. Punctuation differs per layout, so a hard-coded US table mislabels +every punctuation key on a German or Nordic keyboard. It asks the **foreground +window's** layout (the user types into what is in front, not into this process) +and only translates **after** recording — `ToUnicodeEx` mutates dead-key +composition state, so calling it mid-typing corrupts the character being +composed. + +### One Clipboard Image API Instead of Two + +`utils/clipboard` carried two `get_clipboard_image` / `set_clipboard_image` +pairs under identical names — one in `clipboard.py` taking PNG bytes, one in +`clipboard_image.py` taking a file path. Both had live callers, so importing +the wrong module failed at runtime, and only for whichever argument type you +passed. There is one pair now: `set_clipboard_image` accepts **either** PNG +bytes or a path, `clipboard_image.py` is gone, and both functions are exported +from the subpackage and the facade with `AC_clipboard_get_image` / +`AC_clipboard_set_image` commands — previously they were reachable only from +MCP and the GUI, never from `execute_action`. + +`windows/listener/` went with it: `Win32KeyboardListener` and +`Win32MouseListener` had no callers left once recording moved to +`win32_input_hook.py`. + +### Window Handles You Can Actually Use + +Window management listed windows but could not really operate on them. + +- **Handles are integers again.** The `EnumWindows` callback declared its hwnd + as `POINTER(c_int)`, so every handle came back as an `LP_c_long` object; + `int(hwnd)` on one raises `ValueError`. The list was readable and otherwise + useless — you could not focus, move or measure anything it returned, and the + `ac_list_windows` MCP tool raised outright because its handler called + `int(hwnd)`. Every Win32 prototype in `windows_window_manage` now declares + `argtypes` / `restype`, which also stops a 64-bit handle being truncated to + 32 bits. +- **`close_window_by_title` closes.** It used to minimise, because Win32's + `CloseWindow()` minimises despite its name and the wrapper passed that + through. It now posts `WM_CLOSE` — the same thing the window's own close + button does, so the application still gets to run its save prompts. + `minimize_window_by_title` keeps the old behaviour under an honest name. +- **New primitives**: `foreground_window` (what the user is working in), + `window_rect` (screen rectangle, negative coordinates and all), + `move_window_by_title` (omit width/height to reposition without resizing) and + `list_windows(titled_only=True)` — on this desktop that is 17 windows rather + than 34, the rest being shell and helper surfaces. +- **`focus_window` restores a minimised window** before raising it; focusing a + minimised window previously did nothing you could see. A maximised window + stays maximised. + +### URL Canonicalisation, Reachable From Every Surface + +`utils/url_canon` (RFC 3986 canonicalisation, normalisation and query helpers) +had a working headless core and passing tests, but none of its delivery +surfaces were connected, so it could only be reached by importing the submodule +directly. + +- **`canonicalize_url`, `normalize_url`, `urls_equal`, `build_query`, + `parse_query`** are now re-exported from the facade and listed in `__all__`. +- **`AC_canonicalize_url`, `AC_normalize_url`, `AC_urls_equal`** wire the same + functions into the executor, so they work from JSON action files, the socket + server, the scheduler and the Script Builder without Python glue; the + matching **`ac_canonicalize_url` / `ac_normalize_url` / `ac_urls_equal`** MCP + tools and three Script Builder `CommandSpec`s come with them. + +Comparing two URLs for "the same page" is the actual use: `urls_equal` ignores +query order and the fragment, so `?b=1&a=2` and `?a=2&b=1#top` match, which is +what a navigation assertion needs and what plain string comparison gets wrong. + ## What's new (2026-07-18) ### Cross-Platform Reliability Hardening diff --git a/architecture_explore.md b/architecture_explore.md new file mode 100644 index 00000000..52cb1a4e --- /dev/null +++ b/architecture_explore.md @@ -0,0 +1,1019 @@ +# AutoControl 架構全覽（architecture_explore） + +> 本文件為全專案架構掃描結果，逐層記錄每個模組的職責。 +> +> **掃描方法**：以 AST 走訪 `je_auto_control/`、`autocontrol-lsp/`、`autocontrol_driver/`、`AutoControl/`、`exe/`、`benchmarks/`， +> 擷取每個模組的 docstring 與頂層公開名稱；統計數字取自實際檔案，非估算。 +> 指令數與公開 API 數以 `executor.known_commands()` 與 `je_auto_control.__all__` 在工作樹上實測取得。 +> +> **掃描時間**：2026-08-16　**版本**：`pyproject.toml` version `0.0.195`　**分支**：`feat/desktop-automation-gaps` + +--- + +## 1. 專案定位與規模 + +`je_auto_control` 是一套跨平台 GUI 自動化框架，涵蓋 Windows（Win32 API）、macOS（pyobjc/Quartz）、 +Linux X11（python-Xlib）與 Linux Wayland（libei / ydotool），並延伸到 Android（ADB / uiautomator2）與 +iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、螢幕擷取、動作腳本化與報表產生； +在此之上長出了 AI agent、遠端桌面、USB 直通、MCP／REST／TCP 伺服器、可觀測性與測試治理等子系統。 + +| 指標 | 數值 | +| --- | ---: | +| Python 模組總數（含周邊子專案） | 998 | +| 程式碼總行數 | 133,534 | +| `je_auto_control/utils/` 子套件數 | 308 | +| `AC_*` 動作指令數（`known_commands()` 實測） | 767 | +| 套件門面 `__all__` 公開名稱數 | 1,221 | +| GUI 分頁數（`main_widget` 註冊） | 48 | +| MCP 工具數（`build_default_tool_registry()` 實測） | 670 | +| `test_*.py` 測試檔／測試函式 | 458 / 4,319 | +| 範例腳本 | 27 | + +**技術基線**：Python ≥ 3.10、MIT 授權、必要相依只有 `je_open_cv`／`opencv-python`／`pillow`／`mss`／ +`defusedxml`／`cryptography`（加上各平台專屬的 pyobjc、python-Xlib）；其餘全部是選用 extras +（`gui`、`webrtc`、`signaling`、`discovery`、`pdf`、`office`、`fuzzy`、`s3`、`locale`、`audio`）。 +大量子系統刻意只用標準庫實作（REST 伺服器、JSON Schema、JWT、TOTP、WebSocket 框架、ACME、 +USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 + +--- + +## 2. 分層架構 + +``` +┌──────────────────────────────────────────────────────────────────────────┐ +│ 介面層 Entry Points │ +│ cli.py (je_auto_control 指令) │ __main__.py (argparse) │ gui/ (PySide6) │ +│ utils/socket_server (TCP) │ utils/rest_api (HTTP) │ utils/mcp_server │ +│ utils/chatops (Slack) │ utils/pytest_plugin │ autocontrol-lsp │ +└───────────────────────────────┬──────────────────────────────────────────┘ + │ 全部只呼叫下面這一層，不含業務邏輯 +┌───────────────────────────────▼──────────────────────────────────────────┐ +│ 執行核心 Execution Core │ +│ utils/executor/action_executor.py ── Executor.event_dict（767 個 AC_*） │ +│ utils/executor/flow_control.py ── 34 個區塊指令（迴圈/分支/try/巨集） │ +│ utils/script_vars ── ${var} 插值 │ utils/json ── action 檔 I/O │ +└───────────────────────────────┬──────────────────────────────────────────┘ + │ +┌───────────────────────────────▼──────────────────────────────────────────┐ +│ 能力層 utils/（308 個子套件，全部無 Qt 相依） │ +│ 影像辨識 │ OCR │ 無障礙樹 │ 定位自癒 │ AI/Agent │ 遠端桌面 │ USB │ +│ 報表觀測 │ 資料 │ 安全 │ 韌性 │ 系統整合 │ 排程觸發 │ 網路協定 │ +└───────────────────────────────┬──────────────────────────────────────────┘ + │ +┌───────────────────────────────▼──────────────────────────────────────────┐ +│ 平台無關 API wrapper/ │ +│ auto_control_mouse / keyboard / screen / image / record / window │ +│ platform_wrapper.py ── 依 sys.platform 載入唯一後端 │ +└───────────────────────────────┬──────────────────────────────────────────┘ + │ +┌───────────────────────────────▼──────────────────────────────────────────┐ +│ 平台後端 Platform Backends（僅載入當前 OS） │ +│ windows/ (ctypes Win32 + Interception 驅動) │ +│ osx/ (pyobjc Quartz) │ +│ linux_with_x11/ (python-Xlib + 選用 uinput) │ +│ linux_wayland/ (libei ctypes + ydotool/wtype/grim CLI) │ +│ android/ (ADB + uiautomator2) │ ios/ (WebDriverAgent) │ +└──────────────────────────────────────────────────────────────────────────┘ +``` + +**三條不可違反的架構約束**（由 CLAUDE.md 定義、CI 測試強制）： + +1. **`import je_auto_control` 絕不載入 PySide6**。GUI 進入點在 `start_autocontrol_gui()` 內延遲匯入。 + （已於工作樹實測確認：匯入門面後 `sys.modules` 無任何 PySide6。） +2. **每個功能都必須同時有無頭 API 與 GUI 介面**。業務邏輯一律住在 `utils/` 或 `wrapper/`， + Qt widget 只是把使用者輸入翻譯成對無頭核心的呼叫。 +3. **每個功能都要接上 `AC_*` 指令**，讓它自動可從 JSON 動作檔、TCP 伺服器、排程器、 + MCP 伺服器與視覺化腳本編輯器使用，不需要寫任何 Python 膠水碼。 + +--- + +## 3. 核心設計模式 + +| 模式 | 落點 | 說明 | +| --- | --- | --- | +| **Strategy** | `wrapper/platform_wrapper.py` | 依 `sys.platform` 只匯入當前 OS 的 `keyboard`／`mouse`／`screen`／`recorder` 實作；Linux 再細分 Wayland／X11，Wayland 後端不可用時自動退回 XWayland 並記警告。新增平台不需要改 wrapper。 | +| **Facade** | `je_auto_control/__init__.py` | 把 1,200 個公開名稱集中再匯出，使用者只 `import je_auto_control`。`api/core.py` 另提供一個小而穩定的版本化門面（7 個名稱）給新整合使用。 | +| **Command** | `utils/executor/action_executor.py` | `Executor.event_dict` 是字串 → callable 的分派表；JSON 動作檔即指令序列，因此可錄製、序列化、重播、簽章。 | +| **Observer** | `utils/callback/`、`utils/observer/`、`utils/triggers/` | 動作完成後觸發回呼；畫面出現／消失／變化與外部事件（webhook／IMAP／檔案）驅動腳本。 | +| **Template Method** | `utils/generate_report/` | HTML／JSON／XML 三個產生器共用「收集紀錄 → 格式化 → 寫檔」骨架，各自實作渲染。 | +| **Adapter / Backend seam** | `accessibility/backends/`、`ocr/backends/`、`vision/backends/`、`llm/backends/`、`agent/backends/`、`hotkey/backends/`、`usb/passthrough/*_backend.py`、`usbip/backend.py` | 每個外部能力都有抽象基底 + 具體實作 + null fallback，讓無相依環境仍可載入與測試。 | +| **Registry / Singleton** | `remote_desktop/registry.py`、`rest_api/rest_registry.py`、`profiler`、`run_history`、`secrets` | 行程級單例，讓 `AC_*` 指令能操作長生命週期的伺服器與狀態。 | + +--- + +## 4. 三條主要執行路徑 + +**A. JSON 動作腳本（最主要路徑）** + +``` +action.json ─► utils/json/json_file.read_action_json + ─► utils/executor/action_schema.validate_actions （結構驗證，拒絕未知指令） + ─► Executor.execute_action + ├─ _resolve_runtime_args ── ${var}/${secrets.*} 插值（巢狀 body 延後解析） + ├─ flow_control BLOCK_COMMANDS ── AC_loop / AC_if_* / AC_try / AC_retry … + └─ event_dict[name](**args) ── 呼叫 wrapper 或 utils 的實作 + ─► default_profiler.measure + observability 指標 + ─► 執行紀錄 dict（每個動作一筆，重複動作加序號後綴） +``` + +錯誤處理原則：`AutoControlException` 家族在此被「收納」成紀錄而非中止整份腳本； +但 `AutoControlAssertionException`（`AC_assert_*` 失敗）即使在 `raise_on_error=False` 下仍會往上拋， +確保斷言不會被靜默吃掉。`execute_files` 會先呼叫 `require_signed_actions` 驗簽。 + +**B. 錄製 → 重播 → 產碼** + +``` +wrapper/auto_control_record.record ─► 平台 listener（win32/x11/osx） + ─► stop_record / record_to_json ─► action list + ─► utils/semantic_recording.enrich （加語義錨點，可換機重播） + ─► utils/recording_edit （裁切／過濾／縮放） + ─► utils/codegen （產生 pytest / python / robot 程式碼） +``` + +**C. 遠端與外部驅動** + +``` +TCP socket_server ─┐ +REST rest_api ─┤ +MCP mcp_server ─┼─► execute_action（同一個全域 Executor 實例） +ChatOps chatops ─┤ +Scheduler/Triggers─┘ +``` + +所有伺服器預設綁 `127.0.0.1`；REST 有 Bearer token + 限流，MCP 有稽核與限流， +socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 + +--- + +## 5. 逐層模組清單 + +### 5.1 套件入口與公開介面 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `je_auto_control/__init__.py` | 1,927 | **套件門面**。集中匯入並再匯出 1,200 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | +| `je_auto_control/__main__.py` | 71 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | +| `je_auto_control/cli.py` | 327 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | +| `je_auto_control/api/__init__.py` | 23 | 版本化整合進入點。 | +| `je_auto_control/api/core.py` | 20 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約只針對這一面。 | +| `je_auto_control/utils/deprecation.py` | 36 | 公開 API 的一致性棄用警告。 | +| `je_auto_control/utils/http_headers.py` | 33 | 入站 HTTP 標頭的共用防禦式解析。 | + +### 5.2 wrapper 抽象層 + +平台無關 API，所有上層（executor、GUI、伺服器）只呼叫這裡，不直接碰後端。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `wrapper/platform_wrapper.py` | 60 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；載入失敗直接拋 `AutoControlException`（fail fast）。 | +| `wrapper/_platform_windows.py` | 326 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | +| `wrapper/_platform_osx.py` | 150 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | +| `wrapper/_platform_linux.py` | 268 | X11 後端組裝（python-Xlib + 選用 uinput）。 | +| `wrapper/_platform_wayland.py` | 58 | Wayland 後端組裝（libei／ydotool／grim）。 | +| `wrapper/auto_control_mouse.py` | 304 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | +| `wrapper/auto_control_keyboard.py` | 228 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。 | +| `wrapper/auto_control_screen.py` | 98 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | +| `wrapper/auto_control_image.py` | 83 | 影像 API：`locate_all_image`、`locate_image_center`、`locate_and_click`。 | +| `wrapper/auto_control_record.py` | 76 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | +| `wrapper/auto_control_window.py` | 94 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態（目前僅 Windows 實作）。 | + +### 5.3 平台後端 + +#### Windows（`windows/`，26 檔／1,939 行） + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `core/utils/win32_ctype_input.py` | 73 | `SendInput` 的 ctypes 結構定義與送出。 | +| `core/utils/win32_vk.py` | 188 | Windows 虛擬鍵碼對照表。 | +| `core/utils/win32_keypress_check.py` | 21 | `GetAsyncKeyState` 按鍵狀態查詢。 | +| `mouse/win32_ctype_mouse_control.py` | 220 | 滑鼠事件產生（含多螢幕絕對座標換算）。 | +| `keyboard/win32_ctype_keyboard_control.py` | 55 | 鍵盤事件產生。 | +| `record/win32_input_hook.py` | 223 | 單一一組低階鍵鼠 hook（`WH_KEYBOARD_LL`／`WH_MOUSE_LL`）＋訊息迴圈，產生帶時間戳的事件時間軸；停止時以 `PostThreadMessageW(WM_QUIT)` 收掉執行緒，不會每錄一次就漏一條。 | +| `record/win32_record.py` | 126 | 把 `win32_input_hook` 的時間軸轉成 action list（含按鍵放開、滾輪與間隔）。 | +| `screen/win32_screen.py` | 53 | 螢幕尺寸與像素讀取。 | +| `window/windows_window_manage.py` | 208 | 視窗列舉／聚焦／關閉／最小化／幾何（`auto_control_window` 的實作）。**每支 Win32 函式都明寫 argtypes/restype**，並持有自己的 user32 handle，避免把原型外溢到別的模組；hwnd 一律是 int。 | +| `message/window_message.py` | 97 | 直接對視窗送 `WM_*` 訊息（背景輸入）。 | +| `interception/_dll.py` | 231 | `interception.dll` 的延遲 ctypes 載入與結構定義。 | +| `interception/keyboard.py` | 71 | 經 Interception 驅動的鍵盤輸入（繞過部分反自動化偵測）。 | +| `interception/mouse.py` | 161 | 經 Interception 驅動的滑鼠輸入。 | + +#### macOS（`osx/`，17 檔／771 行） + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `core/utils/osx_vk.py` | 114 | macOS 虛擬鍵碼表。 | +| `mouse/osx_mouse.py` | 137 | Quartz `CGEvent` 滑鼠事件。 | +| `keyboard/osx_keyboard.py` | 129 | Quartz 鍵盤事件。 | +| `keyboard/osx_keyboard_check.py` | 24 | 按鍵狀態查詢。 | +| `listener/osx_listener.py` | 96 | `CGEventTap` 監聽。 | +| `record/osx_record.py` | 52 | 錄製（CLI `record` 於 macOS 明確不支援）。 | +| `screen/osx_screen.py` | 143 | 螢幕擷取與尺寸（含 Retina 座標處理）。 | +| `pid/pid_control.py` | 64 | 以 PID 操作應用程式。 | + +#### Linux X11（`linux_with_x11/`，19 檔／1,189 行） + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `core/utils/x11_linux_display.py` | 14 | 共用 `Xlib.display.Display` 實例。 | +| `core/utils/x11_linux_vk.py` | 197 | X11 keysym 對照表。 | +| `mouse/x11_linux_mouse_control.py` | 133 | XTest 滑鼠事件。 | +| `keyboard/x11_linux_keyboard_control.py` | 85 | XTest 鍵盤事件。 | +| `listener/x11_linux_listener.py` | 195 | XRecord 監聽。 | +| `record/x11_linux_record.py` | 73 | 錄製。 | +| `screen/x11_linux_screen.py` | 62 | 螢幕尺寸與擷取。 | +| `uinput/_device.py` | 234 | `/dev/uinput` 封裝（核心層輸入，選用）。 | +| `uinput/keyboard.py` | 33 | uinput 鍵盤後端，介面與 X11 版一致。 | +| `uinput/mouse.py` | 116 | uinput 滑鼠後端。 | + +#### Linux Wayland（`linux_wayland/`，10 檔／1,093 行） + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `_detect.py` | 66 | Wayland session 偵測與 CLI 工具探測。 | +| `_select_input.py` | 50 | 決定使用原生 libei 或 CLI shim。 | +| `libei.py` | 243 | libei（Wayland HID 層輸入模擬）的 ctypes 綁定。 | +| `mouse.py` | 168 | 經 ydotool 的滑鼠後端。 | +| `keyboard.py` | 149 | 經 ydotool + wtype 的鍵盤後端。 | +| `keymap.py` | 156 | 友善鍵名 → evdev key code。 | +| `screen.py` | 136 | 經 grim + wlr-randr 的螢幕後端。 | +| `listener.py` / `record.py` | 49 / 35 | 監聽與錄製 stub（Wayland 限制）。 | + +#### 行動裝置 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `android/adb_client.py` | 184 | `adb` CLI 的薄封裝。 | +| `android/client.py` | 92 | `uiautomator2.Device` 的延遲封裝。 | +| `android/find.py` | 105 | uiautomator2 widget 樹的元素查詢。 | +| `ios/client.py` | 95 | `facebook-wda`（WebDriverAgent）封裝。 | +| `ios/find.py` | 87 | XCUITest 無障礙查詢。 | +| `ios/input.py` | 47 | iOS 觸控與按鍵原語。 | +| `ios/screen.py` | 33 | iOS 裝置螢幕擷取與尺寸。 | + +### 5.4 能力層 `utils/`（308 個子套件） + +以下依主題分組。每個子套件都是獨立可匯入的無頭模組，不含任何 Qt 相依。 + + +### 5.4.1 執行引擎與腳本資產 + +> 24 個套件、約 12,443 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/action_lint/` | 332 | action 檔 linter 與 JSON Schema 產生器（CI 用 `python -m` 進入點） | +| `utils/action_signing/` | 232 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | +| `utils/checkpoint/` | 117 | 流程檢查點與續跑，讓長 action list 具持久性 | +| `utils/codegen/` | 159 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | +| `utils/dag/` | 478 | 跨主機 DAG 編排器（圖模型 + runner） | +| `utils/decision_table/` | 105 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | +| `utils/deterministic/` | 98 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | +| `utils/executor/` | 8,931 | **核心**。`Executor` 指令分派表（767 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | +| `utils/flow_debugger/` | 138 | action list 的單步除錯器與追蹤器 | +| `utils/input_macro/` | 129 | 定時輸入事件重播與宣告式輸入序列 DSL | +| `utils/json/` | 75 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | +| `utils/json_store/` | 63 | JSON 字典檔持久化的共用小工具（內部管線） | +| `utils/loop_guard/` | 142 | 機械式卡死迴圈偵測（agent loop 用） | +| `utils/plugin_loader/` | 87 | 掃描外部 Python 外掛目錄並註冊其 `AC_` callable | +| `utils/plugin_sdk/` | 70 | 外掛 SDK：透過 entry points 發佈／載入第三方 `AC_*` 指令 | +| `utils/project/` | 186 | 專案腳手架：建立目錄結構與範本 action 檔 | +| `utils/recording_edit/` | 152 | 不重錄的前提下裁切／過濾／縮放已錄製的 action list | +| `utils/saga/` | 95 | Saga 協調器：失敗時以 LIFO 補償動作回滾 | +| `utils/script_vars/` | 193 | 執行期變數作用域與 `${var}` / `${secrets.*}` 插值 | +| `utils/skill_library/` | 118 | 具名可重用 action 序列（skill）的持久化倉庫 | +| `utils/state_machine/` | 183 | 宣告式有限狀態機驅動 action JSON | +| `utils/stubs/` | 239 | 為 `AC_*` 指令面產生型別 stub | +| `utils/test_record/` | 65 | 全域測試紀錄單例，記錄每個動作的參數與例外 | +| `utils/work_queue/` | 176 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | + +### 5.4.2 框架基礎設施 + +> 12 個套件、約 1,848 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/callback/` | 201 | Observer 模式：`callback_executor` 以字串名觸發功能，執行後呼叫回呼 | +| `utils/config_bundle/` | 402 | 使用者設定的單檔匯出／匯入 | +| `utils/critical_exit/` | 98 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | +| `utils/diagnostics/` | 272 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | +| `utils/exception/` | 186 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | +| `utils/failure_bundle/` | 189 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | +| `utils/file_process/` | 27 | 目錄檔案列舉（`execute_dir` 的後端） | +| `utils/logging/` | 72 | `autocontrol_logger` 單例 + 輪替檔案 handler | +| `utils/package_manager/` | 99 | 動態載入套件並把 executor 注入其中 | +| `utils/path_guard/` | 101 | 命令列傳入路徑的正規化與邊界檢查（防路徑穿越） | +| `utils/shell_process/` | 161 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | +| `utils/start_exe/` | 40 | 啟動另一個執行檔行程 | + +### 5.4.3 排程、觸發與背景監看 + +> 11 個套件、約 3,574 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/hotkey/` | 734 | 全域熱鍵守護行程，把 OS 層熱鍵綁到 action 檔（Win/macOS/X11 三後端） | +| `utils/idle_keepawake/` | 214 | 偵測使用者閒置時間並在無人值守執行期間阻止系統睡眠 | +| `utils/lock_session/` | 165 | 鎖定工作站、等待解鎖並分類鎖定狀態轉換 | +| `utils/observer/` | 222 | 反應式畫面觀察者，在出現／消失／變化時觸發 | +| `utils/recurrence/` | 326 | RFC 5545 重複規則解析與發生時間展開 | +| `utils/scheduler/` | 355 | 間隔式與 cron 式的 action JSON 排程器 | +| `utils/session_guard/` | 64 | 驅動輸入前先偵測工作階段是否已鎖定／非互動 | +| `utils/triggers/` | 1,150 | 事件驅動觸發引擎：影像／視窗／像素／檔案／webhook／IMAP 郵件 | +| `utils/voice/` | 89 | 語音指令路由：把辨識到的語句對應到 `AC_*` action list | +| `utils/watchdog/` | 175 | 背景彈窗／中斷看門狗，供無人值守自動化 | +| `utils/watcher/` | 80 | 無頭輪詢原語：滑鼠位置、像素顏色、log tail | + +### 5.4.4 輸入模擬與動作品質 + +> 20 個套件、約 2,325 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/act_in_view/` | 78 | 先把目標捲進視野，待其可操作後再動作 | +| `utils/act_modes/` | 69 | actionability 閘門之上的 trial／force 動作模式 | +| `utils/action_effect/` | 112 | 判定一個動作是否真的產生效果，並歸因到目標區域 | +| `utils/action_grounding/` | 82 | 動作前的接地守衛（邊界檢查 + 吸附到元素） | +| `utils/actionability/` | 165 | 動作前就緒閘門（可見 + 穩定 + 啟用 + 未被遮擋） | +| `utils/ensure_state/` | 74 | 冪等地把控制項／設定帶到期望狀態 | +| `utils/field_entry/` | 78 | 清空再輸入的欄位填寫慣用法（Playwright `fill`） | +| `utils/gamepad/` | 313 | 虛擬遊戲手把後端（Windows ViGEmBus 驅動） | +| `utils/humanize/` | 186 | 擬人輸入：貝茲曲線滑鼠路徑 + 抖動打字節奏 | +| `utils/ime_state/` | 146 | 讀取即時 IME 組字／轉換狀態，確保 CJK 輸入安全 | +| `utils/key_hold/` | 109 | 按住按鍵一段時間，或以固定頻率自動重複 | +| `utils/modifier_state/` | 78 | 跨一組動作按住修飾鍵，並保證安全釋放 | +| `utils/mouse_path/` | 94 | 多路徑點滑鼠手勢（沿折線移動或拖曳） | +| `utils/mouse_relative/` | 61 | 相對位移滑鼠移動 | +| `utils/postcondition/` | 140 | 宣告式的動作預期結果規格，對照畫面驗證 | +| `utils/step_repair/` | 116 | 失敗／無效動作的修復策略（自我修正迴圈） | +| `utils/table_grid_fill/` | 143 | 以 OCR 文字填滿格線表格，取得可定址的表格 | +| `utils/input_reach/` | 111 | 送出去的輸入到不到得了：桌面鎖定查詢（免費）＋ 實際送一個 F13 確認沒有被過濾（有副作用，只給診斷用） | +| `utils/keyboard_layout/` | 148 | 向系統問「這個鍵盤配置下每個鍵印出什麼字」（`ToUnicodeEx`），問不到退回 US 對照表 | +| `utils/text_unicode/` | 135 | 輸入任意 Unicode（emoji／CJK／重音字）：優先送字元按鍵事件，不支援時退回剪貼簿貼上 | +| `utils/tween_drag/` | 97 | 沿曲線的緩動插值拖曳 | +| `utils/verify_field/` | 114 | 打字後讀回欄位，確認內容確實落地 | + +### 5.4.5 影像辨識與畫面分析 + +> 37 個套件、約 4,572 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/annotate/` | 109 | 截圖標註：畫框、highlight、箭頭、標籤 | +| `utils/barcode/` | 55 | 一維條碼（EAN／UPC）解碼，解碼器可注入 | +| `utils/color_match/` | 105 | 在 HSV 通道上做顏色感知的樣板比對 | +| `utils/color_region/` | 81 | 以顏色定位畫面區域（遮罩 + 連通元件） | +| `utils/color_stats/` | 91 | 區域顏色統計：平均色與主色 | +| `utils/coordinate_space/` | 86 | 模型網格座標與實體像素之間的座標空間對映 | +| `utils/cv2_utils/` | 355 | OpenCV 基礎層：截圖（mss/Pillow）、樣板比對（走 `grab_logical`，涵蓋所有螢幕）、螢幕錄影、影片錄製、連通元件 | +| `utils/edge_lines/` | 122 | 以 Hough 轉換偵測線條／格線／分隔線 | +| `utils/edge_match/` | 114 | 邊緣形狀（Chamfer／距離轉換）樣板比對 | +| `utils/feature_match/` | 131 | ORB 特徵比對：在旋轉／縮放／主題變更下定位樣板 | +| `utils/hsv_segment/` | 93 | HSV 色彩空間分割（抗光照的顏色遮罩 + blob 框） | +| `utils/icon_classify/` | 115 | 從像素形狀判斷一個框是哪一類元件 | +| `utils/image_dedup/` | 85 | 感知雜湊影像去重（Pillow aHash/dHash） | +| `utils/image_quality/` | 79 | 在 OCR／比對前評分影像品質（銳利度／對比／亮度） | +| `utils/img_histogram/` | 101 | 顏色直方圖指紋與變化偵測（抗光照） | +| `utils/marks_layout/` | 126 | Set-of-Marks 標籤的不重疊排版與可讀配色 | +| `utils/match_autothresh/` | 107 | Otsu 自動門檻，免去手動調 `min_score` | +| `utils/match_ensemble/` | 65 | 多樣板共識比對（多張參考圖投票到同一位置） | +| `utils/match_stability/` | 70 | 比對前的靜止閘門與跨影格的比對持續性 | +| `utils/match_trust/` | 138 | 樣板比對可信度評分（次峰比 + peak-to-sidelobe） | +| `utils/monitor_layout/` | 290 | 多螢幕／虛擬桌面幾何（在哪個螢幕、位置、重映射）＋ `logical_frame` 以滑鼠座標空間擷取畫面 | +| `utils/motion_regions/` | 75 | 兩影格間的局部變化／活動偵測（absdiff） | +| `utils/perceptual_diff/` | 102 | 感知式（YIQ）影像差異，抑制反鋸齒邊緣誤報 | +| `utils/preprocess/` | 187 | OCR／比對前的影像前處理（灰階、二值化、去傾斜…） | +| `utils/qr/` | 55 | 從影像或螢幕區域解碼 QR code（OpenCV） | +| `utils/rotated_match/` | 147 | 容忍旋轉與縮放的樣板比對（尺度空間 × 角度掃描） | +| `utils/saliency/` | 109 | 頻譜殘差視覺顯著性：顯著圖與排序後的顯著區域 | +| `utils/scale_detect/` | 86 | 偵測樣板實際渲染的顯示縮放／視覺 DPI | +| `utils/screen_grid/` | 145 | 供 VLM 接地用的粗粒度標號網格（點 ↔ 格對映） | +| `utils/set_of_marks/` | 152 | Set-of-Marks 疊圖：為畫面元素編號供 VLM 指認 | +| `utils/shape_locator/` | 107 | 以邊緣／輪廓偵測定位元件（矩形／形狀，免樣板） | +| `utils/ssim/` | 142 | 結構相似度比較：感知分數 + 變化區域 | +| `utils/subpixel_match/` | 103 | 以二次曲面擬合做次像素級比對精修 | +| `utils/theme_normalize/` | 94 | 主題無關的影像正規化，讓亮色樣板能配對深色模式 | +| `utils/video_report/` | 135 | 影片步驟疊圖報告：把截圖加字幕串成操作導覽影片 | +| `utils/visual_match/` | 427 | 會回傳信心值的樣板比對（分數、多尺度、find-all + NMS）；擷取走 `grab_logical`，命中座標已加回虛擬桌面原點，單色樣板直接拒收 | +| `utils/visual_regression/` | 218 | 桌面 GUI 的視覺回歸測試（黃金圖比對） | + +### 5.4.6 OCR 與文字理解 + +> 19 個套件、約 3,098 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/bidi_check/` | 118 | 雙向文字 QA（bidi 控制碼、巢狀平衡、Trojan-source 掃描） | +| `utils/column_layout/` | 152 | 從垂直空白推斷欄位，處理無框線表格 | +| `utils/confusables/` | 114 | 易混淆／同形字偵測（Unicode 欺騙骨架） | +| `utils/form_fields/` | 130 | 多方向關聯表單標籤與值，並讀取核取方塊狀態 | +| `utils/fuzzy/` | 96 | 模糊字串比對與去重（預設 difflib，有 rapidfuzz 則優先） | +| `utils/grid_locator/` | 73 | 以 (row, column) 從邊界框定址表格／網格儲存格 | +| `utils/guardrail/` | 110 | 針對畫面／OCR 文字的啟發式 prompt-injection 防護 | +| `utils/heading_segment/` | 71 | 判定 OCR 行是標題或內文，建出文件大綱 | +| `utils/near_dup/` | 107 | 近似重複文字偵測（SimHash／MinHash） | +| `utils/ocr/` | 1,105 | OCR 引擎門面 + 三個後端（Tesseract／EasyOCR／PaddleOCR）、版面結構化與跨詞比對（`text_span`） | +| `utils/pii_text/` | 100 | 自由文字中的 PII 偵測與遮蔽（email／電話／SSN／卡號／IP／IBAN） | +| `utils/readability/` | 139 | 可讀性評分（Flesch、Flesch-Kincaid、Gunning Fog、SMOG、ARI） | +| `utils/reading_flow/` | 121 | 以遞迴 XY-cut 推導欄位感知的閱讀順序 | +| `utils/search_index/` | 142 | 記憶體內 BM25／TF-IDF 全文檢索 | +| `utils/text_blocks/` | 90 | 把 OCR 行組成段落與項目符號／編號清單 | +| `utils/text_diff/` | 150 | unified diff 產生、套用與三方合併 | +| `utils/text_normalize/` | 65 | Unicode 正規化與 slug 產生 | +| `utils/text_regions/` | 159 | 免模型的畫面文字區域偵測（MSER）：區域與行 | +| `utils/text_similarity/` | 167 | 字串距離度量（文字比對用） | + +### 5.4.7 無障礙樹與原生控制項 + +> 16 個套件、約 3,287 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/a11y_audit/` | 358 | 以無障礙樹 + OCR 進行無障礙與 i18n 稽核 | +| `utils/accessibility/` | 2,390 | 跨平台無障礙樹定位與錄製；Windows UIA／macOS AX／null 三後端。支援限定視窗（換搜尋起點，不是過濾）、逐節點可中斷走訪、`IUIAutomation2` 連線逾時、名稱子字串比對與排序、`control_get_state` 一次讀完值／勾選／選取／數值（密碼欄位不回內容） | +| `utils/ax_events/` | 31 | 反應式 UIA 事件等待（focus-changed） | +| `utils/ax_props/` | 46 | 讀取豐富 UIA 屬性（enabled／offscreen／help／status／快捷鍵） | +| `utils/ax_text/` | 104 | 透過 UIA TextPattern 取得原生文字（讀取／尋找／選取／屬性） | +| `utils/ax_tree_walk/` | 120 | 可讀、可定址的無障礙樹後處理（角色名 + 節點路徑） | +| `utils/contrast_map/` | 122 | 取樣實際顏色以評定畫面文字的可讀性（WCAG） | +| `utils/control_patterns/` | 90 | 延伸 UIA 控制項模式動作（Expand／Select／Range／Scroll） | +| `utils/cvd_simulate/` | 127 | 模擬色覺缺陷並標示在該狀況下會撞色的顏色 | +| `utils/element_repository/` | 107 | 原生 UI 元素的具名定位器倉庫（object repository） | +| `utils/focus_order/` | 97 | 鍵盤焦點順序：預期 Tab 序列、WCAG 稽核與設定焦點 | +| `utils/legacy_accessible/` | 47 | MSAA 橋接，處理 UIA 無法建模的舊控制項 | +| `utils/selection_view/` | 59 | 容器選取狀態與檢視切換（Selection／MultipleView 模式） | +| `utils/table_pattern/` | 67 | 原生表格的表頭與儲存格定址（UIA TablePattern／GridItem） | +| `utils/transform_window/` | 72 | 以 UIA Transform／Window 模式移動、調整大小與視窗狀態 | +| `utils/virtualized/` | 45 | 實體化虛擬化清單／網格中的離屏項目 | + +### 5.4.8 元素定位、自我修復與智慧等待 + +> 23 個套件、約 4,044 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/ab_locator/` | 339 | A/B 定位器框架：同時競速 N 種策略並記錄各自勝率 | +| `utils/adaptive_timeout/` | 86 | 由觀測到的步驟耗時推導等待逾時，而非硬猜 | +| `utils/anchor_locator/` | 440 | 錨點定位器：以空間關係組合 影像／OCR／VLM／a11y 四種來源 | +| `utils/app_idle/` | 110 | 等應用程式不再忙碌，再驅動下一步 | +| `utils/change_localize/` | 82 | 把畫面變化歸因到實際改變的元素框 | +| `utils/critic_features/` | 87 | 每步的 critic 特徵集合與規則式步驟評分 | +| `utils/element_diff/` | 90 | 跨影格的幾何感知元素比對（穩定 ID、移動追蹤） | +| `utils/element_parse/` | 108 | 融合並排序畫面元素框（IoU、合併、多來源融合、閱讀順序） | +| `utils/element_proposal/` | 88 | 免樣板、免模型地從原始像素提出乾淨元素清單 | +| `utils/element_scoring/` | 107 | 加權候選評分（角色 + 名稱相似度 + 鄰近度 + 啟用狀態） | +| `utils/expect_poll/` | 139 | 反覆取值直到符合條件（Playwright `expect.poll` 風格） | +| `utils/grounding_consensus/` | 129 | 對同一目標的多個接地提案做自我一致性投票 | +| `utils/heal_analytics/` | 79 | 自癒事件記錄的分析（治癒率、脆弱定位器） | +| `utils/locator_chain/` | 114 | 可組合／可過濾的候選定位器（chained-locator 慣用法） | +| `utils/locator_repair/` | 119 | 自癒回寫：把修正後的定位器持久化 | +| `utils/observation/` | 94 | 供 VLM／agent 接地用的 token 預算內、帶索引的 a11y 文字觀察 | +| `utils/observation_delta/` | 105 | token 預算內的觀察差異：兩個 UI 影格之間變了什麼 | +| `utils/screen_state/` | 145 | 語義畫面狀態：快照／差異與結構化畫面描述 | +| `utils/scroll_find/` | 86 | 捲動直到目標影像／文字可見 | +| `utils/self_healing/` | 345 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | +| `utils/semantic_recording/` | 427 | 為錄製內容加上語義錨點，支援換機重播與自癒重播 | +| `utils/settle_detector/` | 78 | 以純函式介面判定 UI 是否已靜止 | +| `utils/smart_waits/` | 647 | 智慧等待：以影格差異取代 `time.sleep` | + +### 5.4.9 AI / Agent / LLM + +> 13 個套件、約 19,764 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/a2a/` | 94 | A2A（agent-to-agent）agent card 產生 | +| `utils/agent/` | 1,258 | 閉環 Computer-Use Agent 主迴圈 + Anthropic／OpenAI／Computer-Use 三後端 | +| `utils/agent_memory/` | 148 | agent 的持久化情節記憶（goal → trajectory → outcome） | +| `utils/agent_replay/` | 65 | 可攜的 agent 軌跡追蹤（記錄 observation→action 並重播） | +| `utils/agent_trace/` | 131 | agent 可觀測性：OpenTelemetry GenAI 慣例的 LLM span | +| `utils/cost_telemetry/` | 295 | 每次呼叫的 LLM 成本遙測：token 數 + 估算美金 | +| `utils/cua_action/` | 129 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | +| `utils/llm/` | 363 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | +| `utils/mcp_registry/` | 94 | MCP registry `server.json` 資訊清單產生（可被發現） | +| `utils/mcp_server/` | 16,441 | **無頭 MCP 伺服器**（16K LOC，預設註冊 670 個工具＝651 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/tool_use_schema/` | 182 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | +| `utils/trajectory_eval/` | 108 | agent 軌跡評估：依評分規準為一次執行打分 | +| `utils/vision/` | 456 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | + +### 5.4.10 遠端桌面與 USB + +> 6 個套件、約 17,622 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/admin/` | 329 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | +| `utils/config_sync/` | 247 | 透過訊令伺服器做跨機器設定同步 | +| `utils/device_matrix/` | 140 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | +| `utils/remote_desktop/` | 11,726 | **遠端桌面子系統**（51 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | +| `utils/usb/` | 4,255 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | +| `utils/usbip/` | 925 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | + +### 5.4.11 伺服器、網路協定與外部整合 + +> 24 個套件、約 5,889 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/acme_v2/` | 589 | 完整 ACME v2 用戶端（RFC 8555），不依賴 certbot | +| `utils/chatops/` | 632 | Chat-ops bot：接收 Slack／Discord／webhook 的 slash 指令並路由到動作 | +| `utils/cookie_jar/` | 105 | RFC 6265 cookie jar | +| `utils/email_send/` | 118 | SMTP 寄信（email 觸發器的發送端搭檔） | +| `utils/events/` | 84 | 對外 CloudEvents 發送（執行生命週期事件） | +| `utils/http_cassette/` | 112 | 錄製／重播 HTTP 互動，做離線決定性 API 測試 | +| `utils/http_client/` | 134 | 零依賴 HTTP(S) 用戶端，供 action 步驟呼叫 API | +| `utils/http_conditional/` | 89 | 條件式 HTTP 請求與快取驗證器 | +| `utils/http_content/` | 105 | HTTP 內容協商與回應解壓縮 | +| `utils/http_problem/` | 118 | RFC 9457 problem+json 解析 | +| `utils/jwt/` | 174 | JWT（HMAC 家族）編碼、解碼與 claim 驗證 | +| `utils/link_header/` | 114 | RFC 8288 Link header 解析與分頁 | +| `utils/multipart/` | 141 | multipart/form-data 建構與解析 | +| `utils/notify/` | 97 | 跨平台桌面通知 | +| `utils/notify_channels/` | 102 | 對外聊天／webhook 通知（Slack／Discord／Teams／raw） | +| `utils/otp/` | 39 | TOTP 一次性密碼產生（自動化 2FA 登入） | +| `utils/outbox/` | 94 | 交易式 outbox，保證至少一次的事件投遞 | +| `utils/pytest_plugin/` | 377 | pytest 外掛 + BDD step library（`pytest11` entry point） | +| `utils/rest_api/` | 1,693 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | +| `utils/socket_server/` | 133 | 執行 action JSON 的執行緒式 TCP 指令伺服器（預設綁 127.0.0.1） | +| `utils/sse_client/` | 114 | Server-Sent Events 用戶端解析 | +| `utils/tls_acme/` | 445 | TLS 自動化：HTTP-01 挑戰伺服器、金鑰／CSR、自動續期 | +| `utils/url_canon/` | 117 | RFC 3986 URL 正規化與查詢字串工具 | +| `utils/webrunner_bridge/` | 163 | 把 action JSON 橋接到 WebRunner（`je_web_runner`） | + +### 5.4.12 報表、可觀測性與測試治理 + +> 34 個套件、約 6,942 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/anomaly/` | 109 | 單一序列異常偵測 | +| `utils/approval/` | 104 | Approval testing：以核可基準線驗證產出物 | +| `utils/assertion/` | 866 | 斷言 DSL：畫面狀態驗證 + 組合子 | +| `utils/baggage/` | 113 | W3C Baggage 傳遞 | +| `utils/canonical_log/` | 92 | canonical log line 與結構化 JSON 日誌 | +| `utils/ci_annotations/` | 64 | 由執行結果輸出 CI 工作流程註記（GitHub Actions） | +| `utils/compliance/` | 138 | 合規：把治理證據對應到 SOC2／ISO 27001 控制項 | +| `utils/failure_hooks/` | 400 | 失敗 → 工單自動化：開 Jira／Linear／GitHub issue | +| `utils/failure_signature/` | 76 | 把錯誤訊息正規化成穩定的 SHA-256 失敗簽章並分群 | +| `utils/flake_cluster/` | 105 | 以共同失敗 Jaccard 相似度為易碎測試分群 | +| `utils/flakiness/` | 152 | 以執行歷史分析不穩定測試 | +| `utils/generate_report/` | 311 | HTML／JSON／XML 三種報表產生器（Template Method） | +| `utils/media_assert/` | 235 | 媒體斷言：音訊活動與影片動態檢查 | +| `utils/observability/` | 665 | Prometheus 格式指標 + OpenTelemetry 相容 trace + `/metrics` 匯出伺服器 | +| `utils/otlp_export/` | 83 | OTLP/JSON span 匯出 | +| `utils/percentiles/` | 105 | 可合併的串流延遲摘要與精確百分位數 | +| `utils/process_doc/` | 87 | 由錄製的 action list 產生逐步 SOP 文件 | +| `utils/process_mining/` | 112 | 流程探勘：從動作日誌挖掘可自動化的候選 | +| `utils/profiler/` | 425 | 逐動作效能剖析器 + 資源剖析器 | +| `utils/quarantine/` | 192 | 易碎測試隔離區，讓套件執行器跳過已知不穩定案例 | +| `utils/run_diff/` | 125 | 兩次執行軌跡的差異（LCS 對齊：新增／移除／狀態翻轉／退化） | +| `utils/run_history/` | 349 | 執行歷史儲存與產出物管理 | +| `utils/sarif/` | 136 | 以 SARIF 2.1.0 匯出發現項，供 GitHub／Azure code scanning | +| `utils/slo/` | 114 | SLO 評估：SLI、錯誤預算與多視窗燃燒率告警 | +| `utils/smoothing/` | 69 | 數列移動平均平滑 | +| `utils/soft_assert/` | 64 | 軟斷言：累積檢查並在區塊結束時一次拋出 | +| `utils/stats/` | 215 | 描述統計與 A/B 顯著性檢定（純標準庫） | +| `utils/step_timeline/` | 83 | 每次執行的步驟瀑布圖與瓶頸（關鍵路徑）步驟排名 | +| `utils/test_select/` | 125 | 以執行歷史做風險導向的測試選取 | +| `utils/test_shard/` | 89 | 以耗時為權重的套件切分與分片結果合併 | +| `utils/test_suite/` | 446 | QA 套件編排：把扁平 action list 評分為測試案例 + CI 報表 | +| `utils/time_travel/` | 384 | 錄製 session 的時光回溯除錯（控制器 + 播放器） | +| `utils/timeseries/` | 145 | 時間序列轉換（rate／降採樣／重採樣） | +| `utils/trace_context/` | 164 | W3C Trace Context 傳遞 | + +### 5.4.13 資料來源、結構驗證與 i18n + +> 24 個套件、約 3,937 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/checksum/` | 134 | 檢查碼演算法：Luhn、Verhoeff、Damm、ISO 7064 MOD 97-10 | +| `utils/config_schema/` | 111 | 型別化設定結構驗證 | +| `utils/data_drift/` | 127 | 分布漂移偵測 | +| `utils/data_profile/` | 123 | 資料剖析與結構推斷 | +| `utils/data_quality/` | 187 | 資料品質：列結構驗證、欄位擷取、遮蔽 | +| `utils/data_source/` | 182 | 資料驅動執行：從 CSV／JSON／SQLite／Excel 載入資料列 | +| `utils/dataset_diff/` | 91 | 表格資料列差異比對（CDC 風格） | +| `utils/gettext_catalog/` | 298 | GNU gettext 目錄 I/O（解析 .po、編譯／讀取 .mo、訊息查詢） | +| `utils/i18n_test/` | 132 | 國際化／在地化測試輔助 | +| `utils/json_contract/` | 137 | JSON 契約／快照比對：`match_json`、`diff_json`、`snapshot_json` | +| `utils/json_patch/` | 314 | JSON Pointer（6901）、JSON Patch（6902）與 Merge Patch（7386） | +| `utils/json_schema/` | 376 | JSON Schema（Draft 2020-12 子集）驗證 | +| `utils/jsonpath/` | 181 | 精簡 JSONPath 查詢 | +| `utils/list_format/` | 74 | 地區感知清單格式化（CLDR 風格的「A、B 和 C」） | +| `utils/locale_collation/` | 130 | 地區感知字串排序（決定性多層排序鍵） | +| `utils/locale_parse/` | 70 | 地區感知數字／貨幣／日期解析與格式化（選用 babel） | +| `utils/message_format/` | 238 | ICU-lite MessageFormat（plural／select／selectordinal） | +| `utils/office/` | 164 | Office 文件無頭讀寫（Excel／Word／PowerPoint） | +| `utils/pdf/` | 89 | PDF 讀取與斷言（選用 pypdf 後端） | +| `utils/referential/` | 77 | 跨資料集的參照完整性檢查 | +| `utils/schema_compat/` | 164 | JSON Schema 相容性分級 | +| `utils/sql/` | 76 | 對 SQLite 的臨時唯讀 SQL 查詢 | +| `utils/test_data/` | 207 | 帶種子的合成測試資料產生（純標準庫） | +| `utils/xml/` | 255 | XML 檔讀寫與結構變更（`defusedxml`） | + +### 5.4.14 安全、機密與合規 + +> 13 個套件、約 2,290 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/config_redaction/` | 77 | 設定結構與 log 字串的機密遮蔽 | +| `utils/egress/` | 116 | 無頭 HTTP 用戶端的網路外連允許清單守衛 | +| `utils/governance/` | 202 | 治理：maker-checker 核准閘門與即時憑證租約 | +| `utils/license_policy/` | 141 | 以 SBOM 元件評估 SPDX 授權允許／拒絕政策 | +| `utils/provenance/` | 106 | SLSA 建置來源證明（in-toto v1） | +| `utils/rbac/` | 274 | 角色型存取控制與逐使用者稽核歸因 | +| `utils/redaction/` | 461 | 截圖遮蔽層：規則偵測 + 政策 + 協調器（上傳 VLM 前先遮） | +| `utils/sbom/` | 110 | SBOM（CycloneDX）產生 | +| `utils/secret_ref/` | 128 | URI scheme 形式的值參照解析 | +| `utils/secrets/` | 253 | 加密機密儲存庫，供 `${secrets.NAME}` 解析 | +| `utils/secrets_scan/` | 100 | 掃描 action JSON／資料中應入庫卻硬編碼的機密 | +| `utils/vex/` | 132 | OpenVEX 陳述撰寫與漏洞分類處置 | +| `utils/vuln_scan/` | 190 | 以 OSV 比對 SBOM 元件的漏洞（純標準庫） | + +### 5.4.15 韌性、流量控制與設定 + +> 14 個套件、約 1,734 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/artifact_store/` | 116 | S3 相容產出物儲存（報表／截圖／錄影） | +| `utils/assets/` | 157 | 環境範圍的型別化資產／設定儲存（UiPath Assets 風格） | +| `utils/bulkhead/` | 136 | Bulkhead 併發隔離 + 伺服器限流標頭解析 | +| `utils/chaos/` | 155 | 決定性混沌實驗（穩態假說 + 故障注入） | +| `utils/dedup_window/` | 65 | 時間視窗內的訊息去重 | +| `utils/dotenv/` | 103 | `.env` 檔解析與序列化 | +| `utils/feature_flags/` | 175 | 功能旗標評估，含目標規則與決定性灰度 | +| `utils/idempotency/` | 116 | 冪等鍵儲存與已存回應重放 | +| `utils/layered_config/` | 112 | 分層設定解析 | +| `utils/optimistic/` | 107 | 樂觀併發的版本化儲存 | +| `utils/rate_limit/` | 164 | 用戶端限流：token bucket、滑動視窗、throttle | +| `utils/resilience/` | 112 | 韌性原語：退避重試與斷路器 | +| `utils/retry_budget/` | 149 | 重試預算：以牆鐘期限與 full jitter 約束重試 | +| `utils/sequence_gap/` | 67 | 逐串流的序號缺口偵測 | + +### 5.4.16 系統、視窗與剪貼簿 + +> 16 個套件、約 2,411 行。 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `utils/clipboard/` | 353 | 跨平台無頭剪貼簿存取（文字 + 影像）。`set_clipboard_image` 同時接受 PNG 位元組與檔案路徑——先前這個名字在本子套件裡有**兩份不同簽章的實作**（`clipboard.py` 吃 bytes、`clipboard_image.py` 吃路徑），匯錯來源只會在執行期才炸，已合併成一支 | +| `utils/clipboard_files/` | 119 | 剪貼簿檔案清單（CF_HDROP）：純 DROPFILES 封裝 + Win32 存取 | +| `utils/clipboard_formats/` | 152 | 檢視與分類剪貼簿可用格式（純分類／差異 + Win32 列舉） | +| `utils/clipboard_history/` | 111 | 剪貼簿歷史：環形緩衝 + 背景輪詢器 | +| `utils/clipboard_rich_formats/` | 279 | 豐富剪貼簿格式 — RTF 與 CSV/TSV 編解碼 + Windows 存取 | +| `utils/file_assoc/` | 94 | 解析哪個應用程式被註冊來開啟某副檔名 | +| `utils/file_dialog/` | 62 | 驅動原生檔案 開啟／儲存／資料夾選擇 對話框 | +| `utils/file_drop/` | 98 | 以 WM_DROPFILES 把檔案拖放到視窗 | +| `utils/rich_clipboard/` | 151 | 豐富剪貼簿格式 — HTML（CF_HTML）建構／解析／存取 | +| `utils/shell_open/` | 99 | 以預設應用開啟檔案，或以預設瀏覽器開啟 URL | +| `utils/system_volume/` | 196 | 讀取與控制系統主音量與靜音狀態 | +| `utils/trash/` | 90 | 把檔案移到系統資源回收筒（可復原刪除） | +| `utils/window_capture/` | 249 | 逐視窗截圖、視窗版面儲存／還原、貼齊與排列 | +| `utils/window_geometry/` | 83 | 視窗客戶區幾何（外框內縮、client→screen 對映） | +| `utils/window_layout/` | 136 | 視窗拼貼／版面規劃器（左右半、四象限、網格、層疊） | +| `utils/window_zorder/` | 78 | 視窗 z 序控制（最上層／移到最前／送到最後） | + +### 5.4.17 大型子系統的檔案級剖析 + +上表以子套件為單位；以下把行數最大的幾個子系統展開到檔案層。 + +#### `utils/executor/`（8,811 行）— 執行核心 + +| 檔案 | 行數 | 職責 | +| --- | ---: | --- | +| `action_executor.py` | 8,042 | `Executor` 類別與 `event_dict` 分派表（767 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | +| `flow_control.py` | 757 | 34 個區塊指令：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`、`AC_*_to_var`）／`AC_assert_var`。`LoopBreak`／`LoopContinue` 以例外實作。 | +| `action_schema.py` | 94 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。 | +| `mouse_aliases.py` | 40 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | + +#### `utils/mcp_server/`（16,441 行，670 個工具）— 最大子系統 + +| 檔案 | 行數 | 職責 | +| --- | ---: | --- | +| `tools/_factories.py` | 8,739 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | +| `tools/_handlers.py` | 4,651 | 把 MCP 工具呼叫橋接到 AutoControl 無頭 API 的 adapter。 | +| `server.py` | 995 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器。 | +| `http_transport.py` | 323 | MCP 的 HTTP 傳輸。 | +| `resources.py` | 304 | MCP resource 提供者。 | +| `prompts.py` | 221 | MCP prompt 目錄。 | +| `fake_backend.py` | 185 | CI／無頭測試用的記憶體內假後端。 | +| `plugin_watcher.py` | 150 | 檔案變更時熱重載外掛工具的背景 watcher。 | +| `tools/_base.py` | 147 | 工具註冊表的共用型別與輔助。 | +| `tools/_validation.py` | 107 | MCP 工具用到的 JSON Schema 子集驗證器。 | +| `tools/plugin_tools.py` | 90 | 把外掛載入的 `AC_*` callable 包成 `MCPTool`。 | +| `log_bridge.py` | 91 | 把 Python logging 記錄橋接成 MCP `notifications/message`。 | +| `audit.py` | 79 | MCP 工具呼叫稽核記錄。 | +| `context.py` | 72 | 傳給 opt-in 工具處理器的每次呼叫上下文。 | +| `rate_limit.py` | 49 | 工具呼叫的 token bucket 限流。 | +| `__main__.py` | 88 | `je_auto_control_mcp` console script 進入點。 | + +#### `utils/remote_desktop/`（11,726 行／51 檔） + +三條傳輸路徑並存：**TCP**（JPEG 影格）、**WebSocket**（同協定換傳輸）、**WebRTC**（aiortc 視訊 + DataChannel）。 + +| 檔案 | 行數 | 職責 | +| --- | ---: | --- | +| `host.py` | 1,319 | TCP 主機：串流 JPEG 影格並套用檢視端輸入。 | +| `webrtc_host.py` | 996 | WebRTC 主機：串流螢幕視訊並接受檢視端輸入。 | +| `webrtc_viewer.py` | 639 | WebRTC 檢視端：接收視訊並送出輸入。 | +| `viewer.py` | 624 | TCP 檢視端。 | +| `host_service.py` | 543 | 無頭 WebRTC 主機執行器 + 多平台服務安裝器。 | +| `registry.py` | 371 | `AC_remote_*` 指令使用的行程級單例。 | +| `webrtc_transport.py` | 357 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | +| `multi_viewer.py` | 315 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | +| `signaling_server.py` | 298 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | +| `audit_log.py` | 284 | SQLite 雜湊鏈稽核記錄。 | +| `ws_protocol.py` | 278 | 最小 RFC 6455 WebSocket 框架與握手。 | +| `file_transfer.py` | 274 | 分塊檔案傳輸。 | +| `relay.py` | 271 | NAT 穿透失敗時的 TCP 中繼。 | +| `fingerprint.py` | 251 | TOFU 主機指紋驗證。 | +| `turn_config.py` | 235 | coturn 設定產生器。 | +| `presence.py` | 222 | 多檢視者的執行緒安全在場註冊表。 | +| `jpeg_recorder_encrypted.py` | 218 | AES-GCM 加密版 session 錄影。 | +| `address_book.py` | 210 | 檢視端的主機通訊錄。 | +| `audio.py` / `webrtc_audio.py` / `webrtc_mic.py` | 206 / 190 / 152 | 音訊擷取播放、音訊軌、麥克風上行。 | +| `webrtc_files.py` | 206 | 專屬 DataChannel 的分塊檔案傳輸。 | +| `lan_discovery.py` | 190 | mDNS／Zeroconf 區網探索。 | +| `video_codec.py` | 183 | TCP／WS 路徑的可插拔視訊編解碼。 | +| `hw_codec.py` | 170 | 硬體 H.264 編碼偵測與啟用。 | +| `webrtc_stats.py` | 164 | 把 aiortc 的 `RTCStats` 報告輪詢成精簡 dict。 | +| `webrtc_inspector.py` | 139 | 行程級的 `StatsSnapshot` 滾動視窗。 | +| `adaptive_bitrate.py` | 149 | 依統計調整主機擷取 FPS。 | +| `connect_coordinator.py` | 150 | 由使用者輸入的目標決定該用哪條傳輸。 | +| `signaling_client.py` | 146 | 純標準庫的訊令用戶端。 | +| `trust_list.py` | 145 | 自動接受的檢視端信任清單。 | +| `input_dispatch.py` | 134 | 在主機端套用輸入訊息。 | +| `session_recorder.py` | 130 | 以 PyAV 把 WebRTC 影格錄成 mp4。 | +| `totp.py` | 130 | RFC 6238 TOTP（零外部相依）。 | +| `file_sync.py` | 127 | 輪詢式資料夾鏡像。 | +| `transport.py` | 124 | 可插拔的型別化訊息傳輸。 | +| `protocol.py` | 97 | 長度前綴的 TCP 框架。 | +| `resume_tokens.py` / `session_quality_cache.py` / `rate_limit.py` | 95 / 86 / 85 | 快速重連 token、每 session 品質快取、檢視端限流。 | +| `host_id.py` / `viewer_id.py` | 82 / 78 | 主機與檢視端的持久身分。 | +| `permissions.py` / `clipboard_sync.py` / `wake_on_lan.py` / `session_actions.py` / `auth.py` | 65 / 73 / 57 / 41 / 29 | 逐 session 權限、剪貼簿同步、WOL、SAS 注入與螢幕遮蔽、HMAC 挑戰回應。 | +| `ws_host.py` / `ws_viewer.py` / `jpeg_recorder.py` | 41 / 30 / 139 | WebSocket 傳輸變體與 TCP 路徑錄影。 | + +#### `utils/usb/`（4,255 行）與 `utils/usbip/`（925 行） + +| 檔案 | 行數 | 職責 | +| --- | ---: | --- | +| `usb/passthrough/session.py` | 595 | 逐 peer 的 USB 直通 session。 | +| `usb/passthrough/viewer_client.py` | 561 | 檢視端的直通協定用戶端。 | +| `usb/passthrough/backend.py` | 464 | 後端 ABC + libusb 實作。 | +| `usb/passthrough/winusb_backend.py` | 458 | Windows WinUSB 後端（ctypes）。 | +| `usb/passthrough/acl.py` | 433 | 逐裝置 ACL。 | +| `usb/passthrough/iokit_backend.py` | 222 | macOS IOKit 後端。 | +| `usb/passthrough/webrtc_channel.py` | 168 | 把直通協定橋到 WebRTC `usb` DataChannel。 | +| `usb/passthrough/loopback.py` | 158 | 行程內 loopback 傳輸（測試用）。 | +| `usb/passthrough/protocol.py` | 133 | 線路框格式。 | +| `usb/passthrough/descriptor.py` | 133 | USB 標準裝置描述元解析。 | +| `usb/passthrough/key_provider.py` | 123 | ACL 的可插拔 HMAC 金鑰來源。 | +| `usb/passthrough/commands.py` | 151 | 無頭直通指令（單一真實來源）。 | +| `usb/usb_devices.py` | 286 | 跨平台 USB 裝置列舉。 | +| `usb/usb_watcher.py` | 214 | 輪詢式 USB 熱插拔監看。 | +| `usbip/protocol.py` | 331 | USB/IP 線路格式封裝／解析。 | +| `usbip/server.py` | 236 | USB/IP 主機端 TCP 伺服器。 | +| `usbip/libusb_backend.py` | 209 | 以 PyUSB／libusb 執行 URB 的正式後端。 | +| `usbip/backend.py` | 88 | 可插拔 URB 執行後端。 | + +#### `utils/rest_api/`（1,693 行） + +| 檔案 | 行數 | 職責 | +| --- | ---: | --- | +| `rest_server.py` | 468 | HTTP 前端主體。 | +| `rest_handlers.py` | 451 | 端點實作。 | +| `rest_openapi.py` | 406 | 走訪路由表產生 OpenAPI 3.1 規格。 | +| `rest_auth.py` | 144 | Bearer token 驗證 + 逐 client 限流閘門。 | +| `rest_metrics.py` | 76 | Prometheus 曝露端點。 | +| `rest_registry.py` | 76 | 保存執行中 REST 伺服器的行程級單例。 | +| `__main__.py` | 57 | `python -m je_auto_control.utils.rest_api` 進入點。 | + +#### 其他多檔子套件 + +| 子套件 | 檔案組成 | +| --- | --- | +| `accessibility/` | `accessibility_api.py`（公開 API）、`element.py`（dataclass）、`tree.py`（遞迴樹傾印）、`recorder.py`（輪詢式事件錄製）、`backends/`：`base.py` 330 行抽象、`windows_backend.py` 915 行（comtypes UIA）、`windows_query.py` 170 行（UIA 搜尋起點、可中斷走訪、快取請求、NULL COM 指標判定與 `UIA_ERRORS`）、`windows_state.py` 98 行（控制項狀態讀取與密碼欄位判定）、`macos_backend.py` 125 行（pyobjc AX）、`null_backend.py` fallback | +| `agent/` | `agent_loop.py`、`computer_use.py`、`backends/`：`anthropic.py`、`anthropic_computer_use.py`（435 行）、`openai.py`、`base.py` | +| `ocr/` | `ocr_engine.py`（門面）、`structure.py`（版面）、`backends/`：`tesseract_backend.py`、`easyocr_backend.py`、`paddleocr_backend.py`、`base.py` | +| `vision/` | `vlm_api.py`、`backends/`：`anthropic_backend.py`、`openai_backend.py`、`null_backend.py`、`_parse.py`、`base.py` | +| `llm/` | `planner.py`、`backends/`：`anthropic_backend.py`、`null_backend.py`、`base.py` | +| `hotkey/` | `hotkey_daemon.py`、`backends/`：`windows_backend.py`（RegisterHotKey + 訊息幫浦）、`linux_backend.py`（XGrabKey）、`macos_backend.py`（CGEventTap）、`base.py` | +| `observability/` | `metrics.py`（334 行 Prometheus 原語）、`tracing.py`（OTel 相容 + no-op fallback）、`exporter.py`（`/metrics` 伺服器） | +| `triggers/` | `trigger_engine.py`（輪詢引擎）、`webhook_server.py`（HTTP 推送）、`email_trigger.py`（IMAP 輪詢） | +| `chatops/` | `router.py`（傳輸無關指令路由）、`slack_bot.py`（Slack adapter）、`handlers.py`（內建處理器） | +| `redaction/` | `rules.py`（偵測器）、`policies.py`（政策）、`engine.py`（協調器） | +| `failure_hooks/` | `backends.py`（Jira／Linear／GitHub）、`manager.py`（扇出）、`report.py`（資料類別） | +| `test_suite/` | `runner.py`、`result.py`、`reports.py` | +| `semantic_recording/` | `enrich.py`（加錨點）、`replay.py`（換機重播）、`self_healing.py`（自癒重播） | +| `tls_acme/` | `challenge.py`、`keys.py`、`renewal.py` | +| `pytest_plugin/` | `plugin.py`（pytest11 進入點）、`keywords.py`、`bdd_steps.py`（Gherkin） | +| `cv2_utils/` | `screenshot.py`、`template_detection.py`、`screen_record.py`、`video_recording.py`、`blobs.py` | +| `action_lint/` | `linter.py`、`schema.py`、`__main__.py`（CI 使用） | +| `time_travel/` | `controller.py`、`player.py` | +| `dag/` | `graph.py`、`runner.py` | +| `run_history/` | `history_store.py`、`artifact_manager.py` | +| `self_healing/` | `locator.py`、`heal_log.py` | +| `ab_locator/` | `runner.py`、`store.py` | +| `cost_telemetry/` | `pricing.py`、`store.py` | +| `governance/` | `governance.py`、`credential_broker.py` | +| `profiler/` | `profiler.py`、`resource_profiler.py` | +| `script_vars/` | `interpolate.py`、`scope.py` | +| `humanize/` | `motion.py`（貝茲路徑）、`typing.py`（抖動節奏） | +| `assertion/` | `assertions.py`、`combinators.py` | +| `scheduler/` | `scheduler.py`、`cron.py` | +| `stubs/` | `generator.py`、`__main__.py` | +| `config_bundle/` | `config_bundle.py`、`__main__.py` | +| `diagnostics/` | `diagnostics.py`、`__main__.py` | +| `xml/` | `xml_file/xml_file.py`、`change_xml_structure/change_xml_structure.py` | +| `generate_report/` | `generate_html_report.py`、`generate_json_report.py`、`generate_xml_report.py` | + +### 5.5 GUI 層（`gui/`，84 檔／26,367 行） + +GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-material），且刻意保持「薄」： +每個分頁只把使用者輸入翻譯成對 `utils/` 無頭核心的呼叫。 + +#### 骨架 + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `gui/__init__.py` | 24 | `start_autocontrol_gui()`：**唯一**會延遲匯入 PySide6 的地方，維持頂層套件 Qt-free。 | +| `main_window.py` | 290 | `QMainWindow`：選單列（File／Actions／View／…）、可關閉分頁、即時語言切換、字級預設、qt-material 主題。分頁分為 core／editing／detection／automation／system 五類。 | +| `main_widget.py` | 780 | 擁有 `QTabWidget`，註冊 48 個分頁，並暴露 show/hide/list API 給選單列。核心分頁在註冊時直接宣告 `(label_key, handler)` 動作對。 | +| `_auto_click_tab.py` | 270 | 自動點擊分頁的 mixin 建構器。 | +| `_report_tab.py` | 81 | 報表分頁 mixin。 | +| `_i18n_helpers.py` | 67 | 需要即時語言切換的分頁共用的翻譯註冊 mixin。 | +| `language_wrapper/` | 4,977 | 四語系字典（英／日／簡中／繁中）+ `multi_language_wrapper` 執行期切換器與監聽註冊表。 | +| `selector/` | 183 | 拖曳選取螢幕區域的半透明全螢幕覆蓋層與樣板裁切工具（互動式，但都有對應的程式化 API）。 | + +> **分頁指令一律走 Actions 選單**：分頁本身只放輸入、表格與結果檢視，指令由視窗層選單暴露。 +> 核心分頁在 `main_widget.py` 註冊時宣告動作；功能分頁實作 `menu_actions()`（目前 40 個檔案有此 hook）。 +> `test/unit_test/headless/test_actions_menu_gui.py` 會守住這個契約——沒有動作宣告的新分頁會讓 CI 失敗。 + +#### 48 個分頁 + +| 分頁 | 模組 | 行數 | 職責 | +| --- | --- | ---: | --- | +| auto_click | `_auto_click_tab.py` | 270 | 自動點擊：座標、間隔、熱鍵、`write`、捲動。 | +| screenshot | `main_widget.py` 內建 | — | 截圖、選區、螢幕尺寸、取像素色。 | +| image_detect | `main_widget.py` 內建 | — | 樣板裁切、定位、定位全部、定位並點擊。 | +| record | `main_widget.py` 內建 | — | 錄製／停止／回放／存檔／載入。 | +| script_builder | `script_builder/` | 5,708 | **視覺化腳本編輯器**：`command_schema.py`（4,924 行 `AC_*` 參數綱要）、`step_model.py`（步驟模型與 AC JSON 序列化）、`step_list_view.py`（含巢狀 body 的樹狀檢視）、`step_form_view.py`（綱要驅動表單）、`builder_tab.py`。 | +| flow_editor | `flow_editor/` | 490 | 節點式流程圖檢視：`layout.py`（純 Python 佈局演算法，可單測）、`scene.py`（Qt 場景繪製）、`tab.py`。 | +| script | `main_widget.py` 內建 | — | 載入／執行單檔或整個目錄、內建編輯器執行。 | +| recording_editor | `recording_editor_tab.py` | 244 | 裁切、過濾、重新縮放錄製內容。 | +| variables | `variables_tab.py` | 166 | 檢視、灌入、清除 executor 執行期作用域。 | +| secrets | `secrets_tab.py` | 188 | 解鎖保險庫並管理 `${secrets.NAME}`。 | +| vlm | `vlm_tab.py` | 112 | 用文字描述 UI 元素，交由模型定位。 | +| self_healing | `self_healing_tab.py` | 195 | 自癒定位器管理與治癒記錄。 | +| ocr_reader | `ocr_tab.py` | 170 | 傾印區域文字或以 regex 搜尋。 | +| accessibility | `accessibility_tab.py` | 131 | 瀏覽 OS UI 樹並依 role/name 點擊。 | +| live_hud | `live_hud_tab.py` | 99 | 即時 HUD：滑鼠位置、游標下像素色、log tail。 | +| llm_planner | `llm_planner_tab.py` | 182 | 自然語言描述 → 預覽 → 執行。 | +| computer_use | `computer_use_tab.py` | 166 | 從 GUI 啟動 Anthropic 閉環 agent。 | +| scheduler | `scheduler_tab.py` | 138 | 註冊間隔式 action JSON 執行。 | +| hotkeys | `hotkeys_tab.py` | 130 | 將全域熱鍵綁到 action 檔。 | +| triggers | `triggers_tab.py` | 321 | 影像／視窗／像素／檔案事件監看。 | +| webhooks | `webhooks_tab.py` | 210 | 將 HTTP 請求綁到 action 腳本。 | +| email_triggers | `email_triggers_tab.py` | 224 | 將 IMAP 信箱綁到 action 腳本。 | +| test_suite | `test_suite_tab.py` | 163 | 執行 QA 套件規格並管理易碎隔離區。 | +| assertions | `assertions_tab.py` | 130 | 執行單一畫面狀態斷言並顯示通過／失敗。 | +| data_source | `data_source_tab.py` | 136 | 預覽無頭資料層載入的資料列。 | +| flakiness | `flakiness_tab.py` | 117 | 依執行歷史排序間歇失敗的腳本。 | +| a11y_audit | `a11y_audit_tab.py` | 114 | 從即時樹找出無障礙／i18n 缺陷。 | +| device_matrix | `device_matrix_tab.py` | 111 | 一份 action list 跨多裝置平行執行。 | +| media_checks | `media_checks_tab.py` | 116 | 音訊活動與影片動態斷言。 | +| run_history | `run_history_tab.py` + `run_history_timeline.py` | 462 | 瀏覽過去的排程／觸發／熱鍵執行，含自訂時間軸元件。 | +| profiler | `profiler_tab.py` | 131 | 視覺化逐動作耗時熱點。 | +| window_manager | `window_tab.py` | 128 | 列出、聚焦、關閉視窗。 | +| plugins | `plugins_tab.py` | 83 | 從使用者目錄載入額外 `AC_` 指令。 | +| webrunner | `webrunner_tab.py` | 188 | 從 GUI 驅動 `je_web_runner`。 | +| dag_runner | `dag_tab.py` | 188 | 編輯、驗證、執行跨主機 DAG。 | +| chatops | `chatops_tab.py` | 108 | 在接上 Slack 前先本機測試 slash 指令。 | +| trace_replay | `trace_replay_tab.py` | 187 | 拖曳捲動時光回溯錄製內容。 | +| remote_desktop | `remote_desktop/`（16 檔） | 6,240 | 見下。 | +| presence | `presence_tab.py` | 152 | 多檢視者在場名單。 | +| rest_api | `rest_api_tab.py` | 198 | 啟停 HTTP 前端並顯示 URL 與 token。 | +| admin_console | `admin_console_tab.py` | 313 | 管理多個遠端 AutoControl REST 端點。 | +| audit_log | `audit_log_tab.py` | 192 | 瀏覽並驗證防竄改雜湊鏈。 | +| inspector | `inspector_tab.py` | 121 | WebRTC 檢測器：即時摘要與近期統計取樣。 | +| usb_devices | `usb_devices_tab.py` | 121 | 唯讀列舉 + 熱插拔監看控制。 | +| usb_browser | `usb_browser_tab.py` | 309 | 檢視端 USB 裝置瀏覽器。 | +| usb_share | `usb_passthrough_panel.py` + `usb_passthrough_prompt.py` | 704 | AnyDesk 風格 USB 直通面板與主機端 ACL 授權對話框。 | +| diagnostics | `diagnostics_tab.py` | 91 | 執行子系統檢查並顯示結果。 | +| report | `_report_tab.py` | 81 | 產生 HTML／JSON／XML 報表。 | + +#### 遠端桌面 GUI（`gui/remote_desktop/`，16 檔／6,240 行） + +| 模組 | 行數 | 職責 | +| --- | ---: | --- | +| `webrtc_panel.py` | 2,556 | WebRTC 子分頁主體。 | +| `webrtc_dialogs.py` | 845 | WebRTC GUI 用的自訂對話框與清單元件。 | +| `connection_screen.py` | 673 | Quick Connect —— AnyDesk 風格單畫面入口。 | +| `viewer_panel.py` | 543 | 「控制另一台機器」子分頁。 | +| `host_panel.py` | 335 | 「分享這台機器」子分頁。 | +| `frame_display.py` | 229 | 繪製 JPEG 影格並發出遠端輸入事件的元件。 | +| `webrtc_workers.py` | 196 | 訊令流程的背景 `QThread` worker。 | +| `tab.py` | 166 | 外層容器分頁。 | +| `_helpers.py` | 149 | 面板共用輔助。 | +| `remote_screen_window.py` | 141 | 檢視端的彈出視窗。 | +| `tray_icon.py` | 99 | WebRTC 主機的系統匣圖示。 | +| `annotation_overlay.py` | 89 | 主機端標註的透明最上層覆蓋。 | +| `sparkline.py` | 78 | WebRTC 統計面板的迷你走勢圖。 | +| `blanking_overlay.py` | 72 | 遠端連線期間的隱私遮蔽全螢幕覆蓋。 | +| `viewer_screen_window.py` | 47 | 顯示連入檢視端分享畫面的彈出視窗。 | + +### 5.6 周邊子專案與資產 + +| 目錄 | 內容 | 職責 | +| --- | --- | --- | +| `autocontrol-lsp/` | 8 檔／752 行 | **AC_* action JSON 的語言伺服器**：`server.py`（JSON-RPC over stdio）、`handlers.py`（純函式 LSP 處理器，易單測）、`diagnostics.py`（診斷清單）、`documents.py`（記憶體文件庫）、`commands.py`（從 executor 取得所有指令，自動同步）。 | +| `autocontrol_driver/` | 1 檔 | 產生 AutoControl driver 的小工具。 | +| `AutoControl/executor/` | 3 檔 | 由 `create_project_dir` 產生的專案範本示範（單檔／整個目錄／錯誤檔案三種執行方式）。 | +| `exe/start_autocontrol_gui.py` | 4 行 | 打包成執行檔用的 GUI 啟動器。 | +| `benchmarks/core_latency.py` | 32 行 | 對穩定無頭進入點的可重複煙霧基準測試。 | +| `examples/` | 27 個腳本 | 從截圖點擊、OCR、排程、遠端桌面、agent loop、可觀測性，一路到 computer-use、Wayland、跨主機 DAG、chatops、pytest/BDD、anchor locator。 | +| `browser-extension/` | manifest v3 擴充 | 瀏覽器端配合元件（background／content script／popup）。 | +| `docker/` | Dockerfile ×2 + compose | 無頭容器與帶 XFCE 桌面的容器。 | +| `k8s/helm/` | Helm chart | Kubernetes 部署。 | +| `ci_templates/.gitlab-ci.yml` | — | 供使用者專案複製的 GitLab CI 範本。 | +| `docs/` | Sphinx（`API`／`Eng`／`Zh`／`getting_started`） | Read the Docs 文件。 | +| `architecture_diagram/` | drawio + png | 既有的架構圖原始檔。 | +| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`test_source` | 460 個 `test_*.py`／4,333 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 則守住文件引用的數字與實測值一致。 | + +--- + +## 6. 擴充點 + +| 想加什麼 | 該動哪裡 | 不需要動什麼 | +| --- | --- | --- | +| **新平台後端** | 新增 `je_auto_control//` 實作 backend 介面，並在 `wrapper/platform_wrapper.py` 加一個分支 | 所有 wrapper 模組與上層 | +| **新 `AC_*` 指令** | 在 `utils/` 寫無頭實作 → 加進 `Executor.event_dict` → 加進 `gui/script_builder/command_schema.py` | executor 分派邏輯本身 | +| **執行期外掛指令** | `add_command_to_executor({"AC_x": fn})`，或用 `utils/plugin_loader`（掃描目錄）／`utils/plugin_sdk`（entry points） | 核心程式碼 | +| **新 GUI 分頁** | 在 `gui/` 新增 widget（只做 UI 翻譯）→ 在 `main_widget.py` `_add_tab` 註冊 → 提供 `menu_actions()` | 主視窗選單建構邏輯 | +| **新 OCR／VLM／LLM／a11y 後端** | 在對應 `backends/` 實作 base 協定 | 呼叫端 | +| **新報表格式** | 仿 `generate_report/` 既有三者的骨架新增產生器 | 執行紀錄收集 | +| **新 MCP 工具** | 在 `mcp_server/tools/_factories.py` 加工廠、`_handlers.py` 加 adapter | 傳輸層 | + +--- + +## 7. 品質閘門與工程約束 + +**CI 工作流程**（`.github/workflows/`）： + +| 檔案 | 用途 | +| --- | --- | +| `dev.yml` | 開發分支測試。 | +| `stable.yml` | 合併到 main 後版本遞增並上傳 PyPI（使用 `PYPI_API_TOKEN`）。 | +| `release.yml` | 發佈流程（上傳步驟目前關閉）。 | +| `quality.yml` | 靜態分析與型別檢查。 | +| `platform-smoke.yml` | 跨平台煙霧測試。 | +| `docker.yml` | 容器映像建置。 | +| `action-json-lint.yml` | 用 `python -m je_auto_control.utils.action_lint` 檢查 action JSON。 | + +**設定基線**（`pyproject.toml`）： + +- **pytest**：`testpaths` 限定 `test/unit_test/headless` 與 `test/unit_test/flow_control`；`--strict-markers --strict-config`。 +- **coverage**：`fail_under = 35`（實測基線，CI 只確保不退步），排除 `gui/` 與 `language_wrapper/`。 +- **mypy**：只對穩定 API 面把關；`follow_imports = "silent"`，numpy stub 以 `follow_imports_for_stubs` 略過。 +- **bandit**：排除 `test`／`docs`／`language_wrapper`（翻譯字典會誤觸 B105），只跳過 B101。 + +**程式碼硬約束**（CLAUDE.md）：循環複雜度 ≤ 10、認知複雜度 ≤ 15、函式 ≤ 75 行、參數 ≤ 7、 +巢狀 ≤ 4、檔案 ≤ 750 行、行寬 ≤ 120；禁止裸 `except`、可變預設參數、`eval`/`exec`、 +`shell=True`、`pickle` 反序列化外部資料、函式庫程式碼中的 `print` 與 `assert`； +socket 預設綁 `127.0.0.1`；資源一律用 `with`。 + +**例外設計**（`utils/exception/exceptions.py`）：所有框架錯誤都繼承 `AutoControlException`， +讓 executor、背景輪詢迴圈、請求處理器與 GUI slot 這四種收納邊界能用單一 `except` 攔住整個家族。 +**不可**新增直接繼承 `Exception` 的兄弟類別——那會靜默逃出每一道邊界。 + +--- + +## 8. 附錄：各層規模 + +| 層／子系統 | 檔案數 | 行數 | +| --- | ---: | ---: | +| `gui/` | 84 | 26,367 | +| `utils/mcp_server/` | 18 | 16,441 | +| `utils/remote_desktop/` | 51 | 11,726 | +| `utils/executor/` | 5 | 8,811 | +| `utils/usb/` | 17 | 4,255 | +| `je_auto_control/`（頂層 3 檔） | 3 | 2,325 | +| `utils/accessibility/` | 12 | 2,390 | +| `wrapper/` | 12 | 1,747 | +| `utils/rest_api/` | 8 | 1,693 | +| `windows/` | 26 | 1,939 | +| `utils/agent/` | 8 | 1,258 | +| `linux_with_x11/` | 19 | 1,189 | +| `utils/triggers/` | 4 | 1,150 | +| `linux_wayland/` | 10 | 1,093 | +| `utils/ocr/` | 9 | 1,105 | +| `utils/usbip/` | 5 | 925 | +| `utils/assertion/` | 3 | 866 | +| `osx/` | 17 | 771 | +| `autocontrol-lsp/` | 8 | 752 | +| `utils/hotkey/` | 7 | 734 | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 667 | 46,047 | +| **總計** | **989** | **132,598** | + diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 84c51200..f6c48481 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -30,6 +30,8 @@ AutoControlRecordException from je_auto_control.utils.exception.exceptions import \ AutoControlScreenException +from je_auto_control.utils.exception.exceptions import \ + AutoControlFlatTemplateException from je_auto_control.utils.exception.exceptions import ImageNotFoundException from je_auto_control.utils.executor.action_executor import \ add_command_to_executor @@ -41,12 +43,13 @@ from je_auto_control.utils.executor.action_executor import executor # Accessibility (headless) from je_auto_control.utils.accessibility import ( - AccessibilityElement, AccessibilityNotAvailableError, + AccessibilityElement, accessibility_status, AccessibilityNotAvailableError, AccessibilityRecorder, AXRecorderEvent, AXTreeNode, - click_accessibility_element, control_get_value, control_invoke, + click_accessibility_element, control_get_state, control_get_value, + control_invoke, control_set_value, control_toggle, dump_accessibility_tree, - find_accessibility_element, list_accessibility_elements, - read_control_table, + find_accessibility_element, find_accessibility_elements, + list_accessibility_elements, read_control_table, ) # Extended UIA control patterns (Expand / Select / Range / Scroll) from je_auto_control.utils.control_patterns import ( @@ -230,7 +233,7 @@ ) # Clipboard (headless) from je_auto_control.utils.clipboard.clipboard import ( - get_clipboard, set_clipboard, + get_clipboard, get_clipboard_image, set_clipboard, set_clipboard_image, ) # Hotkey daemon (headless) from je_auto_control.utils.hotkey.hotkey_daemon import ( @@ -402,9 +405,10 @@ from je_auto_control.utils.mouse_relative import ( move_mouse_relative, relative_target, ) -# Type arbitrary Unicode (emoji / CJK) via the clipboard +# Type arbitrary Unicode (emoji / CJK) by key injection or the clipboard from je_auto_control.utils.text_unicode import ( - plan_paste, type_unicode, unicode_code_units, + plan_paste, plan_unicode_keys, type_unicode, type_unicode_keys, + type_unicode_text, unicode_code_units, unicode_keys_supported, ) # Hold modifier keys across a group of actions (release-on-error) from je_auto_control.utils.modifier_state import ( @@ -532,7 +536,8 @@ ) # Multi-monitor / virtual-desktop geometry (which monitor, where, remapping) from je_auto_control.utils.monitor_layout import ( - Monitor, enumerate_monitors, monitor_at_point, monitor_for_window, + Monitor, enumerate_monitors, grab_logical, logical_scale, + logical_virtual_rect, monitor_at_point, monitor_for_window, needs_rescale, primary_monitor, remap_point, to_local, to_virtual, virtual_bounds, ) # Pre-action readiness gate (visible + stable + enabled + not-occluded) @@ -695,6 +700,10 @@ from je_auto_control.utils.image_dedup import ( average_hash, dedupe_images, dhash, hamming_distance, images_similar, ) +# RFC 3986 URL canonicalisation / normalisation and query helpers +from je_auto_control.utils.url_canon import ( + build_query, canonicalize_url, normalize_url, parse_query, urls_equal, +) # Locale-aware number/currency/date parsing & formatting (optional babel) from je_auto_control.utils.locale_parse import ( format_currency, format_date, format_decimal, parse_decimal, parse_number, @@ -918,6 +927,16 @@ locate_text_center, read_text_in_region, set_tesseract_cmd, wait_for_text, ) +# Group OCR word boxes into lines / runs (engines box one word at a time) +from je_auto_control.utils.ocr.text_span import find_spans, group_lines +# Whether input this process sends can actually arrive +from je_auto_control.utils.input_reach import ( + input_desktop_available, input_reaches_system, +) +# What character each key produces on the active keyboard layout +from je_auto_control.utils.keyboard_layout import ( + char_table, foreground_keyboard_layout, layout_char_table, vk_to_char, +) # LLM action planner (headless) from je_auto_control.utils.llm import ( LLMBackend, LLMNotAvailableError, LLMPlanError, @@ -1241,6 +1260,7 @@ # record from je_auto_control.wrapper.auto_control_record import record from je_auto_control.wrapper.auto_control_record import stop_record +from je_auto_control.wrapper.auto_control_record import stop_record_timeline from je_auto_control.wrapper.auto_control_record import record_to_json # Screen wrappers from je_auto_control.wrapper.auto_control_screen import screen_size @@ -1248,8 +1268,9 @@ from je_auto_control.wrapper.auto_control_screen import get_pixel # Cross-platform window manager (headless) from je_auto_control.wrapper.auto_control_window import ( - close_window_by_title, find_window, focus_window, list_windows, - show_window_by_title, wait_for_window, + close_window_by_title, find_window, focus_window, foreground_window, + list_windows, minimize_window_by_title, move_window_by_title, + show_window_by_title, wait_for_window, window_rect, ) # Windows-only modules (ctypes.WINFUNCTYPE / Win32 API) — gated so # ``import je_auto_control`` keeps working on macOS / Linux. Kept last @@ -1277,9 +1298,11 @@ def start_autocontrol_gui(*args, **kwargs): "screen_size", "screenshot", "locate_all_image", "locate_image_center", "locate_and_click", "CriticalExit", "AutoControlException", "AutoControlKeyboardException", "AutoControlMouseException", "AutoControlCantFindKeyException", - "AutoControlScreenException", "ImageNotFoundException", "AutoControlJsonActionException", + "AutoControlScreenException", "AutoControlFlatTemplateException", + "ImageNotFoundException", "AutoControlJsonActionException", "AutoControlRecordException", "AutoControlActionNullException", "AutoControlActionException", "record", - "stop_record", "read_action_json", "write_action_json", "format_action_json", + "stop_record", "stop_record_timeline", + "read_action_json", "write_action_json", "format_action_json", "execute_action", "execute_files", "executor", "execute_action_with_vars", "record_to_json", "generate_code", "generate_code_file", "http_request", "query_sqlite", @@ -1289,7 +1312,9 @@ def start_autocontrol_gui(*args, **kwargs): # OCR "TextMatch", "find_text_matches", "locate_text_center", "wait_for_text", "click_text", "set_tesseract_cmd", "read_text_in_region", - "find_text_regex", + "find_text_regex", "find_spans", "group_lines", + "char_table", "foreground_keyboard_layout", "layout_char_table", + "vk_to_char", "input_desktop_available", "input_reaches_system", # Recording editor "trim_actions", "insert_action", "remove_action", "filter_actions", "adjust_delays", "scale_coordinates", "dedupe_moves", "merge_sleeps", @@ -1303,8 +1328,11 @@ def start_autocontrol_gui(*args, **kwargs): # Window manager "list_windows", "find_window", "focus_window", "wait_for_window", "close_window_by_title", "show_window_by_title", + "minimize_window_by_title", "foreground_window", "window_rect", + "move_window_by_title", # Clipboard "get_clipboard", "set_clipboard", + "get_clipboard_image", "set_clipboard_image", # Hotkey daemon "HotkeyDaemon", "HotkeyBinding", "default_hotkey_daemon", "PopupWatchdog", "WatchdogRule", "default_popup_watchdog", @@ -1401,8 +1429,12 @@ def start_autocontrol_gui(*args, **kwargs): "move_mouse_relative", "relative_target", "type_unicode", + "type_unicode_keys", + "type_unicode_text", "plan_paste", + "plan_unicode_keys", "unicode_code_units", + "unicode_keys_supported", "hold_modifiers", "plan_with_modifiers", "cluster_grid", @@ -1512,6 +1544,10 @@ def start_autocontrol_gui(*args, **kwargs): "to_local", "to_virtual", "virtual_bounds", + "grab_logical", + "logical_virtual_rect", + "logical_scale", + "needs_rescale", "wait_actionable", "act_when_ready", "ActionabilityReport", @@ -1612,6 +1648,8 @@ def start_autocontrol_gui(*args, **kwargs): "set_default_store", "average_hash", "dedupe_images", "dhash", "hamming_distance", "images_similar", + "build_query", "canonicalize_url", "normalize_url", "parse_query", + "urls_equal", "format_currency", "format_date", "format_decimal", "parse_decimal", "parse_number", "VoiceCommand", "VoiceRouter", "default_voice_router", @@ -1743,11 +1781,13 @@ def start_autocontrol_gui(*args, **kwargs): "HistoryStore", "RunRecord", "default_history_store", # Accessibility "AccessibilityElement", "AccessibilityNotAvailableError", + "accessibility_status", "AccessibilityRecorder", "AXRecorderEvent", "AXTreeNode", "click_accessibility_element", "dump_accessibility_tree", - "find_accessibility_element", "list_accessibility_elements", - "control_get_value", "control_set_value", "control_invoke", - "control_toggle", "read_control_table", + "find_accessibility_element", "find_accessibility_elements", + "list_accessibility_elements", + "control_get_state", "control_get_value", "control_set_value", + "control_invoke", "control_toggle", "read_control_table", "expand_control", "collapse_control", "control_expand_state", "select_control_item", "control_range", "set_control_range", "scroll_control_into_view", diff --git a/je_auto_control/gui/accessibility_tab.py b/je_auto_control/gui/accessibility_tab.py index 5a8e0919..cb2ad23b 100644 --- a/je_auto_control/gui/accessibility_tab.py +++ b/je_auto_control/gui/accessibility_tab.py @@ -33,6 +33,7 @@ def __init__(self, parent: Optional[QWidget] = None) -> None: super().__init__(parent) self._tr_init() self._app_filter = QLineEdit() + self._window_filter = QLineEdit() self._name_filter = QLineEdit() self._table = QTableWidget(0, _COLUMN_COUNT) self._table.setEditTriggers(QAbstractItemView.NoEditTriggers) @@ -49,6 +50,7 @@ def retranslate(self) -> None: TranslatableMixin.retranslate(self) self._apply_table_headers() self._app_filter.setPlaceholderText(_t("a11y_app_placeholder")) + self._window_filter.setPlaceholderText(_t("a11y_window_placeholder")) self._name_filter.setPlaceholderText(_t("a11y_name_placeholder")) def _apply_table_headers(self) -> None: @@ -66,6 +68,9 @@ def _build_layout(self) -> None: row.addWidget(self._tr(QLabel(), "a11y_app_label")) self._app_filter.setPlaceholderText(_t("a11y_app_placeholder")) row.addWidget(self._app_filter, stretch=1) + row.addWidget(self._tr(QLabel(), "a11y_window_label")) + self._window_filter.setPlaceholderText(_t("a11y_window_placeholder")) + row.addWidget(self._window_filter, stretch=1) row.addWidget(self._tr(QLabel(), "a11y_name_label")) self._name_filter.setPlaceholderText(_t("a11y_name_placeholder")) row.addWidget(self._name_filter, stretch=1) @@ -82,8 +87,13 @@ def menu_actions(self) -> list: def _refresh(self) -> None: app = self._app_filter.text().strip() or None + window = self._window_filter.text().strip() or None try: - elements = list_accessibility_elements(app_name=app) + # Scoping to one window is not just a filter: the desktop tree is + # orders of magnitude larger, so this is both faster and less + # ambiguous than listing everything and filtering by name. + elements = list_accessibility_elements(app_name=app, + window_title=window) except AccessibilityNotAvailableError as error: self._status.setText(str(error)) self._table.setRowCount(0) diff --git a/je_auto_control/gui/language_wrapper/english.py b/je_auto_control/gui/language_wrapper/english.py index 77be9539..e6b4cb67 100644 --- a/je_auto_control/gui/language_wrapper/english.py +++ b/je_auto_control/gui/language_wrapper/english.py @@ -952,6 +952,8 @@ # Accessibility Tab "a11y_app_label": "App:", "a11y_app_placeholder": "e.g. Calculator", + "a11y_window_label": "Window:", + "a11y_window_placeholder": "part of the window title", "a11y_name_label": "Name contains:", "a11y_name_placeholder": "partial match", "a11y_refresh": "Refresh", diff --git a/je_auto_control/gui/language_wrapper/japanese.py b/je_auto_control/gui/language_wrapper/japanese.py index 1f683e96..970d3bb5 100644 --- a/je_auto_control/gui/language_wrapper/japanese.py +++ b/je_auto_control/gui/language_wrapper/japanese.py @@ -841,6 +841,8 @@ # Accessibility Tab "a11y_app_label": "アプリ:", "a11y_app_placeholder": "例: 電卓", + "a11y_window_label": "ウィンドウ:", + "a11y_window_placeholder": "ウィンドウタイトルの一部", "a11y_name_label": "名前に含む:", "a11y_name_placeholder": "部分一致", "a11y_refresh": "更新", diff --git a/je_auto_control/gui/language_wrapper/simplified_chinese.py b/je_auto_control/gui/language_wrapper/simplified_chinese.py index 94c5c5a3..a0733ef8 100644 --- a/je_auto_control/gui/language_wrapper/simplified_chinese.py +++ b/je_auto_control/gui/language_wrapper/simplified_chinese.py @@ -830,6 +830,8 @@ # Accessibility Tab "a11y_app_label": "应用:", "a11y_app_placeholder": "例如：计算器", + "a11y_window_label": "窗口：", + "a11y_window_placeholder": "窗口标题的一部分", "a11y_name_label": "名称包含:", "a11y_name_placeholder": "部分匹配", "a11y_refresh": "刷新", diff --git a/je_auto_control/gui/language_wrapper/traditional_chinese.py b/je_auto_control/gui/language_wrapper/traditional_chinese.py index af9881b0..c106b490 100644 --- a/je_auto_control/gui/language_wrapper/traditional_chinese.py +++ b/je_auto_control/gui/language_wrapper/traditional_chinese.py @@ -831,6 +831,8 @@ # Accessibility Tab "a11y_app_label": "應用程式：", "a11y_app_placeholder": "例如：小算盤", + "a11y_window_label": "視窗：", + "a11y_window_placeholder": "視窗標題的一部分", "a11y_name_label": "名稱包含：", "a11y_name_placeholder": "部分比對", "a11y_refresh": "重新整理", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 283cc834..f250aa11 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -204,6 +204,24 @@ def _add_keyboard_specs(specs: List[CommandSpec]) -> None: ), description="Enter any Unicode text via clipboard paste (write can't).", )) + specs.append(CommandSpec( + "AC_type_unicode_keys", "Keyboard", "Type Unicode (key events)", + fields=( + FieldSpec("text", FieldType.STRING, placeholder="café 🚀 値"), + ), + description="Enter any Unicode text as key events; leaves the clipboard " + "untouched. Needs a backend that supports it (Windows).", + )) + specs.append(CommandSpec( + "AC_type_unicode_text", "Keyboard", "Type Unicode (best route)", + fields=( + FieldSpec("text", FieldType.STRING, placeholder="café 🚀 値"), + FieldSpec("modifier", FieldType.STRING, optional=True, + default="ctrl", placeholder="ctrl | command"), + ), + description="Enter any Unicode text: key events where available, " + "clipboard paste otherwise.", + )) specs.append(CommandSpec( "AC_with_modifiers", "Keyboard", "With Modifiers Held", fields=( @@ -1063,6 +1081,32 @@ def _add_window_specs(specs: List[CommandSpec]) -> None: specs.append(CommandSpec( "AC_close_window", "Window", "Close Window", fields=(FieldSpec("title_substring", FieldType.STRING),), + description="Ask the first matching window to close (posts WM_CLOSE).", + )) + specs.append(CommandSpec( + "AC_minimize_window", "Window", "Minimize Window", + fields=(FieldSpec("title_substring", FieldType.STRING),), + description="Minimise the first matching window.", + )) + specs.append(CommandSpec( + "AC_foreground_window", "Window", "Foreground Window", + description="The window the user is currently working in.", + )) + specs.append(CommandSpec( + "AC_window_rect", "Window", "Window Rectangle", + fields=(FieldSpec("title_substring", FieldType.STRING),), + description="Screen rectangle (left, top, right, bottom) of a window.", + )) + specs.append(CommandSpec( + "AC_move_window", "Window", "Move Window by Title", + fields=( + FieldSpec("title_substring", FieldType.STRING), + FieldSpec("x", FieldType.INT), + FieldSpec("y", FieldType.INT), + FieldSpec("width", FieldType.INT, optional=True), + FieldSpec("height", FieldType.INT, optional=True), + ), + description="Move, and optionally resize, the first matching window.", )) specs.append(CommandSpec( "AC_drop_files", "Window", "Drop Files onto Window", @@ -1828,6 +1872,17 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: "AC_clipboard_formats", "Data", "List Clipboard Formats", description="Enumerate + classify the clipboard's formats (Windows).", )) + specs.append(CommandSpec( + "AC_clipboard_get_image", "Data", "Clipboard: Save Image", + fields=(FieldSpec("path", FieldType.FILE_PATH, + placeholder="clipboard.png"),), + description="Save the clipboard's image to a PNG file, if it has one.", + )) + specs.append(CommandSpec( + "AC_clipboard_set_image", "Data", "Clipboard: Set Image", + fields=(FieldSpec("path", FieldType.FILE_PATH),), + description="Put an image file onto the clipboard.", + )) specs.append(CommandSpec( "AC_classify_formats", "Data", "Classify Clipboard Formats", fields=(FieldSpec("formats", FieldType.STRING, @@ -2232,6 +2287,40 @@ def _add_misc_specs(specs: List[CommandSpec]) -> None: ), description="Collapse near-duplicate images by perceptual hash.", )) + specs.append(CommandSpec( + "AC_canonicalize_url", "Data", "URL: Canonicalize", + fields=( + FieldSpec("url", FieldType.STRING, + placeholder="HTTP://Example.COM:80/a/../b?b=2&a=1#frag"), + ), + description="Canonical form of a URL, for equality and de-duplication.", + )) + specs.append(CommandSpec( + "AC_normalize_url", "Data", "URL: Normalize", + fields=( + FieldSpec("url", FieldType.STRING, + placeholder="https://EXAMPLE.com:443/p%2fx/"), + FieldSpec("sort_query", FieldType.BOOL, optional=True, + default=False), + FieldSpec("drop_fragment", FieldType.BOOL, optional=True, + default=False), + ), + description="RFC 3986 syntax-based normalisation of a URL.", + )) + specs.append(CommandSpec( + "AC_urls_equal", "Data", "URL: Compare", + fields=( + # https, because nothing in this example depends on the scheme — + # it shows that query order and the fragment are ignored. The + # canonicalize placeholder above keeps http on purpose: it needs + # port 80 to demonstrate default-port removal. + FieldSpec("first", FieldType.STRING, + placeholder="https://x.com/a?b=1&a=2"), + FieldSpec("second", FieldType.STRING, + placeholder="https://x.com/a?a=2&b=1#top"), + ), + description="Whether two URLs are equivalent after canonicalisation.", + )) specs.append(CommandSpec( "AC_parse_decimal", "Data", "Locale: Parse Decimal", fields=( diff --git a/je_auto_control/utils/accessibility/__init__.py b/je_auto_control/utils/accessibility/__init__.py index e6bf4378..0f23c190 100644 --- a/je_auto_control/utils/accessibility/__init__.py +++ b/je_auto_control/utils/accessibility/__init__.py @@ -1,10 +1,11 @@ """Cross-platform accessibility-tree widget location + recording.""" from je_auto_control.utils.accessibility.accessibility_api import ( - AccessibilityElement, AccessibilityNotAvailableError, AXTreeNode, - click_accessibility_element, control_get_value, control_invoke, + AccessibilityElement, accessibility_status, AccessibilityNotAvailableError, AXTreeNode, + click_accessibility_element, control_get_state, control_get_value, + control_invoke, control_set_value, control_toggle, dump_accessibility_tree, - find_accessibility_element, list_accessibility_elements, - read_control_table, + find_accessibility_element, find_accessibility_elements, + list_accessibility_elements, read_control_table, ) from je_auto_control.utils.accessibility.recorder import ( AXRecorderEvent, AccessibilityRecorder, @@ -15,11 +16,13 @@ __all__ = [ - "AccessibilityElement", "AccessibilityNotAvailableError", + "AccessibilityElement", "accessibility_status", "AccessibilityNotAvailableError", "AccessibilityRecorder", "AXRecorderEvent", "AXTreeNode", "AXTreeWalker", "click_accessibility_element", "count_nodes", "dump_accessibility_tree", "find_accessibility_element", - "list_accessibility_elements", "max_depth", - "control_get_value", "control_set_value", "control_invoke", + "find_accessibility_elements", "list_accessibility_elements", + "max_depth", + "control_get_state", "control_get_value", "control_set_value", + "control_invoke", "control_toggle", "read_control_table", ] diff --git a/je_auto_control/utils/accessibility/accessibility_api.py b/je_auto_control/utils/accessibility/accessibility_api.py index 39816b13..06d5933f 100644 --- a/je_auto_control/utils/accessibility/accessibility_api.py +++ b/je_auto_control/utils/accessibility/accessibility_api.py @@ -4,47 +4,116 @@ coordinates. The backend is chosen by :func:`get_backend` per platform and can be swapped out in tests via ``reset_backend_cache``. """ -from typing import List, Optional +from typing import Any, Dict, List, Optional, Tuple from je_auto_control.utils.accessibility.backends import get_backend from je_auto_control.utils.accessibility.element import ( AccessibilityElement, AccessibilityNotAvailableError, element_matches, + rank_by_name, ) from je_auto_control.utils.accessibility.tree import AXTreeNode +def accessibility_status() -> Tuple[bool, str]: + """``(usable, reason)`` for the platform's accessibility backend. + + A predicate rather than the backend object, so callers can report "not + available, and here is why" without reaching into the backend layer or + provoking an exception to find out. + """ + backend = get_backend() + if getattr(backend, "available", False): + return True, f"{backend.name} backend ready" + return False, getattr(backend, "_reason", "no accessibility backend available") + + def list_accessibility_elements(app_name: Optional[str] = None, max_results: int = 200, + window_title: Optional[str] = None, ) -> List[AccessibilityElement]: - """Return a flat list of accessibility elements, optionally filtered.""" + """Return a flat list of accessibility elements, optionally filtered. + + ``window_title`` scopes the search to one window by a case-insensitive + substring of its title — far fewer nodes to walk, and far less ambiguity + than searching the whole desktop. + """ + # Only forwarded when actually requested: a backend written before scoping + # existed keeps working for every call that does not ask for it, and fails + # loudly at the point someone does. + extra = {"window_title": window_title} if window_title else {} return get_backend().list_elements( - app_name=app_name, max_results=int(max_results), + app_name=app_name, max_results=int(max_results), **extra, ) +SCAN_LIMIT = 1500 + + +def find_accessibility_elements(name: Optional[str] = None, + role: Optional[str] = None, + app_name: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False, + max_results: int = 50, + scan_limit: int = SCAN_LIMIT, + ) -> List[AccessibilityElement]: + """Matching elements, best name match first. + + ``max_results`` caps the **matches returned**; ``scan_limit`` caps how many + elements are examined to find them. They are separate on purpose — one + number cannot mean both, and conflating them quietly turns "give me up to + 40 buttons" into "only look at the first 40 elements on the desktop". + + Without ``window_title`` the scan covers the front-most windows up to + ``scan_limit`` and stops, so a target further back is not found. Name the + window for a search that is both complete and far faster. + + With ``contains`` the name is matched as a case-insensitive substring and + the results are ranked so an exact name wins: searching "OK" offers the + ``OK`` button ahead of ``OK and close``. + """ + found = [element for element in list_accessibility_elements( + app_name=app_name, max_results=scan_limit, window_title=window_title) + if element_matches(element, name=name, role=role, app_name=app_name, + contains=contains)] + if contains and name: + found = rank_by_name(found, name) + return found[:max(0, int(max_results))] + + def find_accessibility_element(name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False, ) -> Optional[AccessibilityElement]: """First element matching all provided filters, or ``None``.""" - for element in list_accessibility_elements(app_name=app_name): - if element_matches(element, name=name, role=role, app_name=app_name): - return element - return None + found = find_accessibility_elements( + name=name, role=role, app_name=app_name, window_title=window_title, + contains=contains) + return found[0] if found else None def click_accessibility_element(name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False, ) -> bool: """Click the center of the first element matching the filters. Returns ``True`` on success, ``False`` if nothing matched. Raises :class:`AccessibilityNotAvailableError` if the platform backend is missing. + + Deliberately moves the real pointer rather than calling the Invoke pattern: + Invoke does not require the control to be on screen, and so skips the + hover / focus / drag state an application uses to decide whether a human + actually clicked it. """ element = find_accessibility_element( - name=name, role=role, app_name=app_name, + name=name, role=role, app_name=app_name, window_title=window_title, + contains=contains, ) if element is None: return False @@ -93,12 +162,52 @@ def dump_accessibility_tree(app_name: Optional[str] = None, ) +def _scope_kwargs(window_title: Optional[str], contains: bool) -> Dict[str, Any]: + """The narrowing arguments, omitted entirely when nothing was asked for.""" + extra: Dict[str, Any] = {} + if window_title: + extra["window_title"] = window_title + if contains: + extra["contains"] = True + return extra + + def control_get_value(name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, - automation_id: Optional[str] = None) -> Optional[str]: - """Read a control's value (e.g. a textbox/combo), or None if not found.""" + automation_id: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False) -> Optional[str]: + """Read a control's value (e.g. a textbox/combo), or None if not found. + + ``window_title`` narrows the search to one window, which disambiguates + controls that share a name across applications. + """ + # Forwarded only when asked for, so a backend written before scoping + # existed keeps working for every call that does not use it. + extra = _scope_kwargs(window_title, contains) return get_backend().get_value( - name=name, role=role, app_name=app_name, automation_id=automation_id) + name=name, role=role, app_name=app_name, automation_id=automation_id, + **extra) + + +def control_get_state(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False, + ) -> Optional[Dict[str, Any]]: + """Read everything the control currently holds, in one call. + + ``value`` (+ ``read_only``), ``toggle``, ``selected``, ``number`` — only the + keys the control actually supports, so an absent key means "no such state" + rather than "empty". This is the half pixels cannot answer: text scrolled + out of view, a checkbox's true state, a slider's exact number. Password + fields report only ``{"password": True}``. + """ + extra = _scope_kwargs(window_title, contains) + return get_backend().get_state( + name=name, role=role, app_name=app_name, automation_id=automation_id, + **extra) def control_set_value(value: str, name: Optional[str] = None, diff --git a/je_auto_control/utils/accessibility/backends/base.py b/je_auto_control/utils/accessibility/backends/base.py index 3b1f6637..4fe19e19 100644 --- a/je_auto_control/utils/accessibility/backends/base.py +++ b/je_auto_control/utils/accessibility/backends/base.py @@ -20,15 +20,30 @@ class AccessibilityBackend: def list_elements(self, app_name: Optional[str] = None, max_results: int = 200, + window_title: Optional[str] = None, ) -> List[AccessibilityElement]: + """Flat list of elements, optionally scoped. + + ``window_title`` is a case-insensitive substring of a window's title. + Scoping to one window is not just filtering: the desktop tree holds + orders of magnitude more nodes, so searching a single window is both + much faster and much less ambiguous. Backends that cannot scope may + ignore it. + """ raise NotImplementedError # --- control patterns (object-level actions) --------------------------- def get_value(self, name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, - automation_id: Optional[str] = None) -> Optional[str]: - """Return the matched control's value text, or None if not found.""" + automation_id: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False) -> Optional[str]: + """Return the matched control's value text, or None if not found. + + ``window_title`` / ``contains`` narrow which control is meant, the + same way :meth:`get_state` does — the two reads stay a pair. + """ self._unsupported("get_value", name, role, app_name, automation_id) def set_value(self, value: str, name: Optional[str] = None, @@ -184,6 +199,21 @@ def get_properties(self, name: Optional[str] = None, """ self._unsupported("get_properties", name, role, app_name, automation_id) + def get_state(self, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False, + ) -> Optional[Dict[str, Any]]: + """Return what the matched control currently holds, or None. + + The keys present depend on the control: ``value`` (+ ``read_only``), + ``toggle`` (``on`` / ``off`` / ``mixed``), ``selected``, ``number``. A + key is absent when the control does not support it — distinct from the + value being empty. A password field reports only ``{"password": True}``. + """ + self._unsupported("get_state", name, role, app_name, automation_id) + # --- table headers + cell addressing (TablePattern / GridItemPattern) --- def get_table_headers(self, name: Optional[str] = None, diff --git a/je_auto_control/utils/accessibility/backends/macos_backend.py b/je_auto_control/utils/accessibility/backends/macos_backend.py index 20086c98..e11235bd 100644 --- a/je_auto_control/utils/accessibility/backends/macos_backend.py +++ b/je_auto_control/utils/accessibility/backends/macos_backend.py @@ -34,7 +34,12 @@ def __init__(self) -> None: def list_elements(self, app_name: Optional[str] = None, max_results: int = 200, + window_title: Optional[str] = None, ) -> List[AccessibilityElement]: + # Accepted for signature parity and ignored: this AX walk is already + # per-application, and returning nothing would be worse than returning + # the application's elements unscoped. + del window_title if not self.available: raise AccessibilityNotAvailableError( "pyobjc (ApplicationServices, AppKit) is required for " diff --git a/je_auto_control/utils/accessibility/backends/null_backend.py b/je_auto_control/utils/accessibility/backends/null_backend.py index 691e3b0e..2b7ecc7a 100644 --- a/je_auto_control/utils/accessibility/backends/null_backend.py +++ b/je_auto_control/utils/accessibility/backends/null_backend.py @@ -20,5 +20,6 @@ def __init__(self, reason: str = "no accessibility backend available"): def list_elements(self, app_name: Optional[str] = None, max_results: int = 200, + window_title: Optional[str] = None, ) -> List[AccessibilityElement]: raise AccessibilityNotAvailableError(self._reason) diff --git a/je_auto_control/utils/accessibility/backends/windows_backend.py b/je_auto_control/utils/accessibility/backends/windows_backend.py index 67598676..a264b1b9 100644 --- a/je_auto_control/utils/accessibility/backends/windows_backend.py +++ b/je_auto_control/utils/accessibility/backends/windows_backend.py @@ -8,6 +8,7 @@ Only ``is_control_element=True`` nodes are surfaced to avoid millions of decorative text children. """ +import functools from typing import Any, Dict, List, Optional from je_auto_control.utils.accessibility.backends.base import ( @@ -16,9 +17,14 @@ from je_auto_control.utils.accessibility.element import ( AccessibilityElement, AccessibilityNotAvailableError, element_matches, ) +from je_auto_control.utils.accessibility.backends.windows_query import ( + UIA_ERRORS, search_roots, walk_elements, +) +from je_auto_control.utils.accessibility.backends.windows_state import ( + is_password, read_state, +) from je_auto_control.utils.logging.logging_instance import autocontrol_logger -_TREE_SCOPE_DESCENDANTS = 4 _UIA_IS_CONTROL_ELEMENT_PROPERTY = 30016 _UIA_NAME_PROPERTY = 30005 _UIA_VALUE_PATTERN_ID = 10002 @@ -40,6 +46,18 @@ _UIA_SELECTION_PATTERN_ID = 10001 _UIA_MULTIPLEVIEW_PATTERN_ID = 10008 _UIA_AUTOMATIONID_PROPERTY = 30011 +# How much further to walk than we keep when an app_name filter is on: most +# elements in a window belong to that window's app, so a small factor is +# plenty, and an unbounded search would walk everything to return nothing. +_FILTER_OVERSCAN = 4 +# How many elements a single-control search may examine before giving up. +# Unbounded, a target that is not there walks every window on the desktop and +# costs ~60 s to answer "no". The two bounds encode intent: naming a window +# says "it is in here, find it", so that search is allowed to go deep — a +# browser window holds thousands of nodes and a real target can sit well past +# any small cap. Not naming one says "look around", and stays cheap. +_FIND_SCAN_LIMIT = 1500 +_FIND_SCAN_LIMIT_SCOPED = 20000 _EXPAND_STATES = {0: "collapsed", 1: "expanded", 2: "partial", 3: "leaf"} _WINDOW_VISUAL_STATES = {"normal": 0, "maximized": 1, "minimized": 2} _WINDOW_INTERACTION_STATES = { @@ -56,6 +74,37 @@ def _is_available() -> bool: return False +# ``CUIAutomation8`` is the only class that hands out ``IUIAutomation2``, which +# is the only way to bound how long UIA waits for an application's provider. +# It matters: a full-screen game that never answers UIA made a single +# ``ElementFromHandle`` block for **60 seconds** here, poisoning every +# desktop-wide search. With the connection timeout set, the same call is 1.0 s. +_CLSID_CUIAUTOMATION8 = "{e22ad333-b25f-460c-83d0-0581107395c9}" +_CLSID_CUIAUTOMATION = "{ff48dba4-60ef-4201-aa87-54103eef594e}" +# Only the connect step is tightened. A provider that cannot even connect within +# a second is not going to answer; how long a legitimate *query* may take is a +# different question, so ``TransactionTimeout`` keeps its default. +_CONNECTION_TIMEOUT_MS = 1000 + + +def _create_automation(uia_module): + """The UIAutomation object, with a bounded provider-connect wait if possible.""" + from comtypes import CoCreateInstance, GUID + interface = getattr(uia_module, "IUIAutomation2", None) + if interface is not None: + try: + automation = CoCreateInstance(GUID(_CLSID_CUIAUTOMATION8), + interface=interface) + automation.ConnectionTimeout = _CONNECTION_TIMEOUT_MS + return automation + except (OSError, AttributeError, ValueError) as error: + autocontrol_logger.info( + "UIAutomation2 unavailable, provider waits are unbounded: %r", + error) + return CoCreateInstance(GUID(_CLSID_CUIAUTOMATION), + interface=uia_module.IUIAutomation) + + class WindowsAccessibilityBackend(AccessibilityBackend): """UIAutomation-based flat element listing.""" @@ -77,66 +126,125 @@ def _ensure_automation(self): "install it with: pip install comtypes", ) import comtypes.client # noqa: F401 - from comtypes import CoCreateInstance, GUID try: uia_module = comtypes.client.GetModule("UIAutomationCore.dll") except OSError as error: raise AccessibilityNotAvailableError( f"UIAutomationCore.dll unavailable: {error!r}", ) from error - automation = CoCreateInstance( - GUID("{ff48dba4-60ef-4201-aa87-54103eef594e}"), - interface=uia_module.IUIAutomation, - ) + automation = _create_automation(uia_module) self._automation = automation self._uia_module = uia_module return automation + def _collect_from(self, automation, root, app_name, wanted, + results: List[AccessibilityElement]) -> None: + """Append this root's elements to ``results``, stopping at ``wanted``. + + An ``app_name`` filter needs more elements walked than kept, so the walk + gets room to keep looking — but still a bound, because an unmatched + filter would otherwise walk an entire subtree to return nothing. + """ + budget = wanted - len(results) + if budget <= 0: + return + if app_name is not None: + budget *= _FILTER_OVERSCAN + try: + for raw in walk_elements(automation, root, budget): + if len(results) >= wanted: + return + element = _convert_uia(raw, cached=True) + if element is None: + continue + if app_name is not None and element.app_name != app_name: + continue + results.append(element) + except UIA_ERRORS as error: + # One unresponsive window must not lose the whole listing. + autocontrol_logger.info("UIA walk skipped a root: %r", error) + def list_elements(self, app_name: Optional[str] = None, max_results: int = 200, + window_title: Optional[str] = None, ) -> List[AccessibilityElement]: automation = self._ensure_automation() - try: - root = automation.GetRootElement() - condition = automation.CreatePropertyCondition( - _UIA_IS_CONTROL_ELEMENT_PROPERTY, True, - ) - found = root.FindAll(_TREE_SCOPE_DESCENDANTS, condition) - except (OSError, AttributeError) as error: - autocontrol_logger.error("UIA FindAll failed: %r", error) - return [] + wanted = max(0, int(max_results)) results: List[AccessibilityElement] = [] - count = min(max(0, int(max_results)), int(found.Length or 0)) - for idx in range(count): - element = _convert_uia(found.GetElement(idx)) - if element is None: - continue - if app_name is not None and element.app_name != app_name: - continue - results.append(element) + # One window at a time, walked node by node, so the search stops as soon + # as there are enough elements. Both halves matter: a desktop-rooted + # FindAll cost ~61 s, and even per window a single FindAll is atomic — + # one 34,507-element window took 10.3 s to answer a request for 200. + # + # The budget is checked *before* pulling the next window, not after: + # obtaining a window's root element is itself a cross-process call, and + # against a hung application it blocks for a minute. Fetching one more + # root only to discover the results were already complete cost exactly + # that. + roots = search_roots(automation, window_title) + while len(results) < wanted: + try: + root = next(roots) + except StopIteration: + break + if window_title is None: + # Searching a window's descendants does not include the window + # element itself, which the desktop-rooted walk did return. + window_element = _convert_uia(root) + if window_element is not None and ( + app_name is None or window_element.app_name == app_name): + results.append(window_element) + self._collect_from(automation, root, app_name, wanted, results) return results - def _find_raw(self, name, role, app_name, automation_id): - """Re-walk the tree and return the first matching raw UIA element.""" + def _raw_matches(self, raw, filters, cached: bool) -> bool: + """Whether this element satisfies the caller's filters.""" + element = _convert_uia(raw, cached=cached) + if element is None: + return False + automation_id = filters.get("automation_id") + if automation_id is not None and element.native_id != automation_id: + return False + return element_matches(element, name=filters.get("name"), + role=filters.get("role"), + app_name=filters.get("app_name"), + contains=bool(filters.get("contains"))) + + def _find_raw(self, name, role, app_name, automation_id, + window_title=None, contains=False): + """Return the first matching raw UIA element, or ``None``. + + Every control-pattern call lands here, so this is the hot path for + reading or acting on one control. It walks window by window and node by + node and **stops at the first match**: a target in the front window is + found in milliseconds. The obvious implementation — one + ``FindAll(TreeScope_Descendants)`` from the desktop — cannot stop, and + measured ~61 s on a busy desktop whether or not the target was the very + first element. + + Naming ``window_title`` is still much better: it skips straight to that + window instead of hoping the target is near the front. + """ automation = self._ensure_automation() - try: - root = automation.GetRootElement() - condition = automation.CreatePropertyCondition( - _UIA_IS_CONTROL_ELEMENT_PROPERTY, True, - ) - found = root.FindAll(_TREE_SCOPE_DESCENDANTS, condition) - except (OSError, AttributeError) as error: - autocontrol_logger.error("UIA FindAll failed: %r", error) - return None - for idx in range(int(found.Length or 0)): - raw = found.GetElement(idx) - element = _convert_uia(raw) - if element is None: - continue - if automation_id is not None and element.native_id != automation_id: - continue - if element_matches(element, name=name, role=role, app_name=app_name): - return raw + filters = {"name": name, "role": role, "app_name": app_name, + "automation_id": automation_id, "contains": contains} + budget = (_FIND_SCAN_LIMIT_SCOPED if window_title + else _FIND_SCAN_LIMIT) + try: + for root in search_roots(automation, window_title): + if budget <= 0: + break + # A window can itself be the target; the desktop-rooted walk + # used to return window elements too. + if window_title is None and self._raw_matches(root, filters, + cached=False): + return root + for raw in walk_elements(automation, root, budget): + budget -= 1 + if self._raw_matches(raw, filters, cached=True): + return raw + except UIA_ERRORS as error: + autocontrol_logger.error("UIA element search failed: %r", error) return None def _pattern(self, raw, pattern_id, interface_name): @@ -151,8 +259,16 @@ def _pattern(self, raw, pattern_id, interface_name): return None def get_value(self, name=None, role=None, app_name=None, - automation_id=None) -> Optional[str]: - raw = self._find_raw(name, role, app_name, automation_id) + automation_id=None, window_title=None, + contains=False) -> Optional[str]: + raw = self._find_raw(name, role, app_name, automation_id, + window_title=window_title, contains=contains) + if raw is not None and is_password(raw): + # UIA is supposed to mask a password field's value, but a + # custom-drawn control can put the plaintext in ValuePattern + # anyway. Never hand that back — callers log and forward values. + autocontrol_logger.info("get_value refused: password field") + return None pattern = self._pattern(raw, _UIA_VALUE_PATTERN_ID, "IUIAutomationValuePattern") if raw else None if pattern is None: @@ -319,6 +435,13 @@ def get_properties(self, name=None, role=None, app_name=None, return None return _read_properties(raw) + def get_state(self, name=None, role=None, app_name=None, + automation_id=None, window_title=None, + contains=False) -> Optional[Dict[str, Any]]: + raw = self._find_raw(name, role, app_name, automation_id, + window_title=window_title, contains=contains) + return None if not raw else read_state(raw) + def move_element(self, x=0.0, y=0.0, name=None, role=None, app_name=None, automation_id=None): return self._invoke_pattern_method( @@ -728,13 +851,23 @@ def _read_properties(raw) -> Dict[str, Any]: return properties -def _convert_uia(raw) -> Optional[AccessibilityElement]: +def _convert_uia(raw, cached: bool = False) -> Optional[AccessibilityElement]: + """Convert one UIA element. ``cached`` reads the pre-fetched properties. + + Every ``Current*`` read is a cross-process call into the application that + owns the window, so converting a few thousand elements one property at a + time is the whole cost of a desktop-wide listing. Elements returned by + ``FindAllBuildCache`` carry their properties already, and reading those + costs nothing. + """ + prefix = "Cached" if cached else "Current" try: - name = str(raw.CurrentName or "") - control_type = int(raw.CurrentControlType or 0) - rect = raw.CurrentBoundingRectangle - process_id = int(raw.CurrentProcessId or 0) - automation_id = str(raw.CurrentAutomationId or "") + name = str(getattr(raw, prefix + "Name") or "") + control_type = int(getattr(raw, prefix + "ControlType") or 0) + rect = getattr(raw, prefix + "BoundingRectangle") + process_id = int(getattr(raw, prefix + "ProcessId") or 0) + automation_id = str(getattr(raw, prefix + "AutomationId") or "") + enabled = bool(getattr(raw, prefix + "IsEnabled")) except (OSError, AttributeError): return None width = max(0, int(rect.right - rect.left)) @@ -745,10 +878,20 @@ def _convert_uia(raw) -> Optional[AccessibilityElement]: app_name=_process_name(process_id), process_id=process_id, native_id=automation_id, + enabled=enabled, ) +@functools.lru_cache(maxsize=256) def _process_name(process_id: int) -> str: + """Executable name for a pid. + + Cached because a desktop listing asks for the same handful of pids + thousands of times, and each miss is an ``OpenProcess`` / + ``QueryFullProcessImageNameW`` / ``CloseHandle`` round trip. Windows does + recycle pids, so a very long-lived session could in principle read a stale + name here; it only labels ``app_name``, and the cache is bounded. + """ if process_id <= 0: return "" try: diff --git a/je_auto_control/utils/accessibility/backends/windows_query.py b/je_auto_control/utils/accessibility/backends/windows_query.py new file mode 100644 index 00000000..adedb02a --- /dev/null +++ b/je_auto_control/utils/accessibility/backends/windows_query.py @@ -0,0 +1,170 @@ +"""Where to start a UIA search, and how to fetch its properties in bulk. + +Two things decide what a desktop-wide accessibility search costs: + +* **The root.** ``TreeScope_Descendants`` from the desktop walks every top-level + window's whole subtree, cross-process. Measured on a busy desktop: 2,085 + elements in 61 s, against 135 elements in 0.14 s starting from one window. So + scoping to a window is not a filter applied afterwards — it is a different + search. +* **The properties.** Every ``Current*`` read is another cross-process call, so + reading five properties from a few thousand elements is thousands of round + trips. The walk asks for them with a cache request, after which reading them + is free (measured: 500 elements converted in 0.02 s). +* **The provider.** UIA waits on the *application* to answer. A full-screen game + that never does made one ``ElementFromHandle`` block for 60 s; bounding + ``IUIAutomation2.ConnectionTimeout`` brings the same call to 1.0 s. + +Imports no ``PySide6``. +""" +from typing import Any, Iterator, Optional, Tuple + +from je_auto_control.utils.accessibility.element import ( + AccessibilityNotAvailableError, +) + +TREE_SCOPE_DESCENDANTS = 4 + + +def _uia_errors() -> Tuple[type, ...]: + """Exception types a UIA call can raise. + + ``comtypes`` reports provider failures as ``COMError``, which inherits from + ``Exception`` and from none of the usual suspects — so an ``except + (OSError, AttributeError)`` around a UIA call does not actually contain it. + A window that closes mid-walk, or an application that stops responding, + surfaces exactly that way. + """ + errors: list = [OSError, AttributeError, ValueError] + try: + from _ctypes import COMError + except ImportError: # non-Windows + return tuple(errors) + errors.append(COMError) + return tuple(errors) + + +UIA_ERRORS = _uia_errors() + +UIA_NAME_PROPERTY = 30005 +UIA_AUTOMATIONID_PROPERTY = 30011 +UIA_BOUNDINGRECTANGLE_PROPERTY = 30001 +UIA_PROCESSID_PROPERTY = 30002 +UIA_CONTROLTYPE_PROPERTY = 30003 +UIA_IS_ENABLED_PROPERTY = 30010 + +# Everything the element conversion reads, fetched in one bulk call. +CACHED_PROPERTIES = ( + UIA_NAME_PROPERTY, UIA_CONTROLTYPE_PROPERTY, + UIA_BOUNDINGRECTANGLE_PROPERTY, UIA_PROCESSID_PROPERTY, + UIA_AUTOMATIONID_PROPERTY, UIA_IS_ENABLED_PROPERTY, +) + + +def search_root(automation, window_title: Optional[str]): + """The element to search from: one window by title substring, else the desktop.""" + if not window_title: + return automation.GetRootElement() + from je_auto_control.windows.window.windows_window_manage import ( + get_all_window_hwnd, + ) + needle = window_title.strip().lower() + for hwnd, title in get_all_window_hwnd(): + if needle in (title or "").lower(): + element = automation.ElementFromHandle(hwnd) + if element is not None: + return element + raise AccessibilityNotAvailableError( + f"no visible window title contains {window_title!r}") + + +def search_roots(automation, window_title: Optional[str]) -> Iterator[Any]: + """The roots to search, one at a time, front-most window first. + + Unscoped, this yields each visible top-level window rather than the desktop + root — same coverage, but the caller can stop once it has enough. That is + the whole difference between usable and not: reaching the default 200 + elements costs 0.22 s this way, against 61 s for the single desktop walk, + which cannot be interrupted once started. ``EnumWindows`` returns z-order, + so the window the user is actually looking at is searched first. + """ + if window_title: + yield search_root(automation, window_title) + return + from je_auto_control.windows.window.windows_window_manage import ( + get_all_window_hwnd, + ) + for hwnd, _title in get_all_window_hwnd(): + try: + element = automation.ElementFromHandle(hwnd) + except UIA_ERRORS: + continue # window closed between enumerate and use + if not _is_null(element): + yield element + + +def cache_request(automation): + """A cache request for every property the element conversion reads.""" + request = automation.CreateCacheRequest() + for prop in CACHED_PROPERTIES: + request.AddProperty(prop) + return request + + +def _is_null(element) -> bool: + """Whether a walker returned "no such element". + + ``comtypes`` hands back a **wrapper around a NULL pointer**, not ``None``, + so ``child is not None`` is true at the end of a sibling list and the caller + collects a phantom element that raises ``ValueError: NULL COM pointer + access`` the moment anything reads it. Truthiness is the check that works. + """ + return element is None or not bool(element) + + +def _children(walker, node, request, limit: int) -> list: + """Up to ``limit`` control-view children of ``node``, properties cached. + + Capped because the caller can never need more siblings from one node than + its whole remaining budget — and a node with tens of thousands of children + would otherwise cost that many calls before the walk could descend. + """ + out: list = [] + if limit <= 0: + return out + try: + child = walker.GetFirstChildElementBuildCache(node, request) + except UIA_ERRORS: + return out + while not _is_null(child) and len(out) < limit: + out.append(child) + try: + child = walker.GetNextSiblingElementBuildCache(child, request) + except UIA_ERRORS: + break + return out + + +def walk_elements(automation, root, limit: int) -> Iterator[Any]: + """Yield up to ``limit`` control elements under ``root``, in reading order. + + The point is that it can be **stopped**. ``FindAll`` is one call that + returns after walking everything; this walks node by node, so asking for 200 + elements costs 200 elements' worth of work no matter how large the window is + (measured on this host: 0.036 s for 50, 0.114 s for 200, 0.486 s for 1,000, + against 10.3 s for one atomic 34,507-element walk). + + Properties come back cached, so the caller reads ``Cached*``. + """ + request = cache_request(automation) + walker = automation.ControlViewWalker + remaining = max(0, int(limit)) + # Depth-first, pre-order: children pushed reversed so they pop left to right. + stack = list(reversed(_children(walker, root, request, remaining))) + while stack and remaining > 0: + node = stack.pop() + yield node + remaining -= 1 + if remaining <= 0: + return + stack.extend(reversed(_children(walker, node, request, remaining))) diff --git a/je_auto_control/utils/accessibility/backends/windows_state.py b/je_auto_control/utils/accessibility/backends/windows_state.py new file mode 100644 index 00000000..75a92189 --- /dev/null +++ b/je_auto_control/utils/accessibility/backends/windows_state.py @@ -0,0 +1,98 @@ +"""Read what a Windows UIA control currently holds, safely. + +This is the half that pixels cannot answer: text scrolled out of view, a +checkbox's true state, a slider's exact number. Values are read through +``GetCurrentPropertyValue`` rather than ``QueryInterface`` on each pattern — +only the value is wanted, not the pattern object. + +Two rules shape everything here: + +* **Ask whether the pattern exists before reading it.** An unsupported pattern + answers with a *default* (empty string, ``0``), which reads as "the value is + empty" instead of "this control has no value" — the more misleading of the + two, because a caller will act on it. +* **Never return a password field's contents.** UIA is supposed to mask them, + but that is a convention a custom-drawn control can ignore, and the caller + may well be forwarding what it reads somewhere else. + +Imports no ``PySide6``. +""" +from typing import Any, Dict + +_UIA_IS_PASSWORD_PROPERTY = 30019 +_UIA_VALUE_IS_READONLY_PROPERTY = 30046 +_UIA_LEGACY_VALUE_PROPERTY = 30093 + +# ``key -> (is-this-pattern-available id, value id)`` +_STATE_READS = ( + ("value", 30029, 30045), # IsValuePatternAvailable, Value.Value + ("toggle", 30041, 30086), # IsTogglePatternAvailable, ToggleState + ("selected", 30036, 30079), # IsSelectionItemPatternAvailable, IsSelected + ("number", 30034, 30047), # IsRangeValuePatternAvailable, RangeValue +) + +TOGGLE_STATES = {0: "off", 1: "on", 2: "mixed"} + + +def _prop(raw, property_id: int) -> Any: + """Read one UIA property; ``None`` when it cannot be read.""" + try: + return raw.GetCurrentPropertyValue(property_id) + except (OSError, AttributeError, ValueError): + return None + + +def is_password(raw) -> bool: + """Whether the element is a password field. Unreadable counts as yes.""" + value = _prop(raw, _UIA_IS_PASSWORD_PROPERTY) + return True if value is None else bool(value) + + +def _supported_values(raw) -> Dict[str, Any]: + """Raw values for every pattern the control actually supports.""" + state: Dict[str, Any] = {} + for key, available_id, value_id in _STATE_READS: + if not _prop(raw, available_id): + continue + value = _prop(raw, value_id) + if value is not None: + state[key] = value + return state + + +def _normalise_value(raw, state: Dict[str, Any]) -> None: + """Coerce ``value`` to text, or fall back to the legacy accessible value.""" + if "value" in state: + state["value"] = str(state["value"] or "") + state["read_only"] = bool(_prop(raw, _UIA_VALUE_IS_READONLY_PROPERTY)) + return + legacy = _prop(raw, _UIA_LEGACY_VALUE_PROPERTY) + if legacy: + state["value"] = str(legacy) + + +def _normalise_rest(state: Dict[str, Any]) -> None: + """Turn the remaining raw enum / numeric values into plain Python ones.""" + if "toggle" in state: + code = state["toggle"] + state["toggle"] = TOGGLE_STATES.get(int(code or 0), str(code)) + if "selected" in state: + state["selected"] = bool(state["selected"]) + if "number" in state: + state["number"] = float(state["number"]) + + +def read_state(raw) -> Dict[str, Any]: + """``{value?, read_only?, toggle?, selected?, number?}`` for one element. + + A key is absent when the control does not support it. Password fields + report only ``{"password": True}``. + """ + if is_password(raw): + # nosec B105 # reason: a flag marking the field AS a password, + # returned deliberately in place of its contents — not a credential. + return {"password": True} # nosec B105 + state = _supported_values(raw) + _normalise_value(raw, state) + _normalise_rest(state) + return state diff --git a/je_auto_control/utils/accessibility/element.py b/je_auto_control/utils/accessibility/element.py index 3d286ae0..020b7e10 100644 --- a/je_auto_control/utils/accessibility/element.py +++ b/je_auto_control/utils/accessibility/element.py @@ -1,6 +1,6 @@ """Shared dataclasses and exceptions for the accessibility API.""" from dataclasses import dataclass -from typing import Optional, Tuple +from typing import List, Optional, Tuple @dataclass(frozen=True) @@ -16,6 +16,9 @@ class AccessibilityElement: app_name: str = "" process_id: int = 0 native_id: str = "" + # A disabled control looks clickable and silently ignores the click, so + # this is worth carrying on every element rather than asking per element. + enabled: bool = True @property def center(self) -> Tuple[int, int]: @@ -27,7 +30,7 @@ def to_dict(self) -> dict: "name": self.name, "role": self.role, "bounds": list(self.bounds), "app_name": self.app_name, "process_id": self.process_id, - "native_id": self.native_id, + "native_id": self.native_id, "enabled": self.enabled, "center": list(self.center), } @@ -36,15 +39,58 @@ class AccessibilityNotAvailableError(RuntimeError): """Raised when the platform backend cannot be initialised.""" +def _role_matches(actual: str, wanted: str) -> bool: + """Compare roles by either spelling. + + The Windows backend reports the raw UIA role (``ControlType_50000``) on + purpose — translation is a separate step — but nobody filtering a search + types that. Without accepting the friendly name, ``role="button"`` matches + nothing on Windows and reports it as "not found" rather than as a mistake. + """ + if actual.lower() == wanted.lower(): + return True + from je_auto_control.utils.ax_tree_walk.ax_tree_walk import humanize_role + return humanize_role(actual).lower() == humanize_role(wanted).lower() + + def element_matches(element: AccessibilityElement, name: Optional[str] = None, role: Optional[str] = None, - app_name: Optional[str] = None) -> bool: - """Return True if ``element`` matches all non-None filters.""" - if name is not None and element.name != name: - return False - if role is not None and element.role.lower() != role.lower(): + app_name: Optional[str] = None, + contains: bool = False) -> bool: + """Return True if ``element`` matches all non-None filters. + + ``contains`` compares ``name`` case-insensitively as a *substring*. Real + interfaces label controls with accelerator markers and trailing padding + (``Save(&S)``, ``OK ``), so an exact comparison misses a large share of + genuine targets — but exact stays the default, because a caller that + already holds a full name must keep getting exactly that element. + """ + if name is not None: + if contains: + if name.strip().lower() not in element.name.lower(): + return False + elif element.name != name: + return False + if role is not None and not _role_matches(element.role, role): return False if app_name is not None and element.app_name != app_name: return False return True + + +def rank_by_name(elements: List[AccessibilityElement], + name: str) -> List[AccessibilityElement]: + """Order substring hits so an exact name wins, then reading order. + + Searching "OK" should offer the ``OK`` button before ``OK and close``; + without the ordering the caller clicks whichever the tree happened to + enumerate first, which is neither stable nor what was asked for. + """ + needle = (name or "").strip().lower() + + def _key(element: AccessibilityElement): + left, top, _width, _height = element.bounds + return (element.name.strip().lower() != needle, top, left) + + return sorted(elements, key=_key) diff --git a/je_auto_control/utils/clipboard/__init__.py b/je_auto_control/utils/clipboard/__init__.py index 004d6132..7aabcb19 100644 --- a/je_auto_control/utils/clipboard/__init__.py +++ b/je_auto_control/utils/clipboard/__init__.py @@ -1,4 +1,9 @@ """Cross-platform headless clipboard access.""" -from je_auto_control.utils.clipboard.clipboard import get_clipboard, set_clipboard +from je_auto_control.utils.clipboard.clipboard import ( + get_clipboard, get_clipboard_image, set_clipboard, set_clipboard_image, +) -__all__ = ["get_clipboard", "set_clipboard"] +__all__ = [ + "get_clipboard", "get_clipboard_image", + "set_clipboard", "set_clipboard_image", +] diff --git a/je_auto_control/utils/clipboard/clipboard.py b/je_auto_control/utils/clipboard/clipboard.py index 1357b048..18ecce03 100644 --- a/je_auto_control/utils/clipboard/clipboard.py +++ b/je_auto_control/utils/clipboard/clipboard.py @@ -10,11 +10,12 @@ All functions raise ``RuntimeError`` if the platform backend is missing so callers can degrade gracefully. """ +import os import shutil import subprocess # nosec B404 # reason: required for pbcopy/pbpaste/xclip/xsel import sys from io import BytesIO -from typing import Optional +from typing import Optional, Union _OPEN_CLIPBOARD_FAILED = "OpenClipboard failed" @@ -50,19 +51,48 @@ def get_clipboard_image() -> Optional[bytes]: return _linux_get_image() -def set_clipboard_image(png_bytes: bytes) -> None: - """Place a PNG image (as bytes) onto the clipboard.""" - if not isinstance(png_bytes, (bytes, bytearray)): - raise TypeError("set_clipboard_image expects bytes") - if not png_bytes: - raise ValueError("png_bytes is empty") +def _as_png_bytes(image: Union[bytes, bytearray, str, os.PathLike]) -> bytes: + """PNG bytes for either raw bytes or a path to any Pillow-readable file.""" + if isinstance(image, (bytes, bytearray)): + if not image: + raise ValueError("image bytes are empty") + return bytes(image) + if isinstance(image, (str, os.PathLike)): + safe_path = os.path.realpath(os.fspath(image)) + if not os.path.isfile(safe_path): + raise FileNotFoundError(f"image not found: {safe_path}") + try: + from PIL import Image # noqa: PLC0415 lazy import + except ImportError as error: + raise RuntimeError( + "Pillow is required for clipboard image support" + ) from error + buffer = BytesIO() + with Image.open(safe_path) as opened: + opened.convert("RGB").save(buffer, format="PNG") + return buffer.getvalue() + raise TypeError("set_clipboard_image expects PNG bytes or a file path") + + +def set_clipboard_image( + image: Union[bytes, bytearray, str, os.PathLike]) -> None: + """Place an image on the clipboard, from PNG bytes **or** a file path. + + Both are accepted because both callers are real: the remote-desktop viewer + already holds decoded bytes, while the MCP tool and script callers name a + file. This package used to carry two different ``set_clipboard_image`` + functions under the same name — one per signature, in this module and in + ``clipboard_image.py`` — so importing the wrong one failed at runtime, and + only for whichever argument type you happened to pass. + """ + png_bytes = _as_png_bytes(image) if sys.platform.startswith("win"): - _win_set_image(bytes(png_bytes)) + _win_set_image(png_bytes) return if sys.platform == "darwin": - _mac_set_image(bytes(png_bytes)) + _mac_set_image(png_bytes) return - _linux_set_image(bytes(png_bytes)) + _linux_set_image(png_bytes) # === Windows backend ========================================================= diff --git a/je_auto_control/utils/clipboard/clipboard_image.py b/je_auto_control/utils/clipboard/clipboard_image.py deleted file mode 100644 index 0789989b..00000000 --- a/je_auto_control/utils/clipboard/clipboard_image.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Headless image clipboard helpers. - -Reads work via Pillow's ``ImageGrab.grabclipboard`` on every supported -platform (Windows, macOS, Linux with xclip). Writes are -Windows-only today: macOS and Linux callers receive a clear -``NotImplementedError`` so the higher-level MCP tool can surface the -limitation in its result instead of crashing. -""" -import io -import os -import sys -from typing import Optional - - -def get_clipboard_image() -> Optional[bytes]: - """Return the current clipboard image as PNG bytes, or ``None`` if empty.""" - from PIL import ImageGrab - try: - image = ImageGrab.grabclipboard() - except (OSError, NotImplementedError): - return None - if image is None: - return None - if isinstance(image, list): - # On macOS / Linux the clipboard may carry file paths instead of an image. - return None - buffer = io.BytesIO() - image.save(buffer, format="PNG") - return buffer.getvalue() - - -def set_clipboard_image(image_path: str) -> None: - """Place ``image_path`` (any Pillow-readable file) onto the clipboard.""" - safe_path = os.path.realpath(os.fspath(image_path)) - if not os.path.isfile(safe_path): - raise FileNotFoundError(f"image not found: {safe_path}") - if sys.platform.startswith("win"): - _win_set_image(safe_path) - return - raise NotImplementedError( - f"set_clipboard_image is currently only implemented on Windows " - f"(got {sys.platform})" - ) - - -def _win_set_image(path: str) -> None: - """Win32 implementation: copy a Pillow-rendered DIB onto the clipboard.""" - import ctypes - from ctypes import wintypes - from PIL import Image - - image = Image.open(path).convert("RGB") - buffer = io.BytesIO() - image.save(buffer, format="BMP") - # Strip the 14-byte BITMAPFILEHEADER — clipboard wants raw DIB. - dib_payload = buffer.getvalue()[14:] - - user32 = ctypes.WinDLL("user32", use_last_error=True) - kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) - cf_dib = 8 - gmem_moveable = 0x0002 - - user32.OpenClipboard.argtypes = [wintypes.HWND] - user32.OpenClipboard.restype = wintypes.BOOL - user32.EmptyClipboard.restype = wintypes.BOOL - user32.SetClipboardData.argtypes = [wintypes.UINT, wintypes.HANDLE] - user32.SetClipboardData.restype = wintypes.HANDLE - user32.CloseClipboard.restype = wintypes.BOOL - kernel32.GlobalAlloc.argtypes = [wintypes.UINT, ctypes.c_size_t] - kernel32.GlobalAlloc.restype = wintypes.HGLOBAL - kernel32.GlobalLock.argtypes = [wintypes.HGLOBAL] - kernel32.GlobalLock.restype = ctypes.c_void_p - kernel32.GlobalUnlock.argtypes = [wintypes.HGLOBAL] - - handle = kernel32.GlobalAlloc(gmem_moveable, len(dib_payload)) - if not handle: - raise RuntimeError("GlobalAlloc failed for clipboard image") - pointer = kernel32.GlobalLock(handle) - if not pointer: - raise RuntimeError("GlobalLock failed for clipboard image") - ctypes.memmove(pointer, dib_payload, len(dib_payload)) - kernel32.GlobalUnlock(handle) - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - user32.EmptyClipboard() - if not user32.SetClipboardData(cf_dib, handle): - raise RuntimeError("SetClipboardData failed for clipboard image") - finally: - user32.CloseClipboard() - - -__all__ = ["get_clipboard_image", "set_clipboard_image"] diff --git a/je_auto_control/utils/cv2_utils/template_detection.py b/je_auto_control/utils/cv2_utils/template_detection.py index 66fda465..9c151460 100644 --- a/je_auto_control/utils/cv2_utils/template_detection.py +++ b/je_auto_control/utils/cv2_utils/template_detection.py @@ -1,9 +1,48 @@ -from typing import List -from PIL import ImageGrab +"""Locate a template image on screen, in the coordinates the mouse takes. + +Captures through :func:`monitor_layout.grab_logical` rather than +``ImageGrab.grab()``: the plain grab sees only the primary monitor, so a target +on a second display could never be found — the search reported "not found" for +something plainly on screen. The shared grab also returns the frame in logical +pixels, so a hit read off it can be clicked directly on a mixed-DPI desktop. +""" +from typing import Any, List, Optional, Sequence, Tuple + from je_open_cv import template_detection +from je_auto_control.utils.monitor_layout.logical_frame import grab_logical + + +def _shift(box: Sequence[int], origin: Tuple[int, int]) -> List[int]: + """Translate an image-local ``(x1, y1, x2, y2)`` box into screen coordinates.""" + origin_x, origin_y = origin + return [int(value) + (origin_x if index % 2 == 0 else origin_y) + for index, value in enumerate(box)] + + +def _shift_result(result: Any, origin: Tuple[int, int], multi: bool) -> Any: + """Translate a ``(found, box…)`` result, passing other shapes through. -def find_image(image, detect_threshold: float = 1.0, draw_image: bool = False) -> List[int]: + ``je_open_cv`` returns ``(found, box)`` normally but a different shape when + asked to draw markers (and, on the multi path, an image-first tuple). Only + the coordinate-bearing shape is translated; anything else is returned + untouched rather than guessed at. + """ + if not (isinstance(result, (tuple, list)) and len(result) >= 2 + and isinstance(result[0], bool)): + return result + found, boxes = result[0], result[1] + if not found or not boxes: + return result + moved = ([_shift(box, origin) for box in boxes] if multi + else _shift(boxes, origin)) + return [found, moved, *result[2:]] + + +def find_image(image: Any, detect_threshold: float = 1.0, + draw_image: bool = False, + all_screens: bool = True, + screen_region: Optional[Sequence[int]] = None) -> List[Any]: """ Find a single image on the screen using template detection. 使用模板匹配在螢幕上尋找單一影像 @@ -11,21 +50,25 @@ def find_image(image, detect_threshold: float = 1.0, draw_image: bool = False) - :param image: Template image 模板影像 (要尋找的影像) :param detect_threshold: Detection precision (0.0 ~ 1.0, 1.0 = 完全相同) :param draw_image: Whether to draw detection markers 是否在回傳影像上標記偵測結果 - :return: List[int] [x, y] 座標位置 + :param all_screens: Search every monitor 是否搜尋所有螢幕 + :param screen_region: Limit the search to (x, y, width, height) 限定搜尋範圍 + :return: [found, [x1, y1, x2, y2]] 座標為螢幕座標 """ - # 擷取螢幕畫面 Capture screen - grab_image = ImageGrab.grab() - - # 使用模板匹配 Find object - return template_detection.find_object( + grab_image, origin_x, origin_y = grab_logical( + screen_region, all_screens=all_screens) + result = template_detection.find_object( image=grab_image, template=image, detect_threshold=detect_threshold, draw_image=draw_image ) + return _shift_result(result, (origin_x, origin_y), multi=False) -def find_image_multi(image, detect_threshold: float = 1.0, draw_image: bool = False) -> List[List[int]]: +def find_image_multi(image: Any, detect_threshold: float = 1.0, + draw_image: bool = False, + all_screens: bool = True, + screen_region: Optional[Sequence[int]] = None) -> List[Any]: """ Find multiple occurrences of an image on the screen using template detection. 使用模板匹配在螢幕上尋找多個影像 @@ -33,15 +76,16 @@ def find_image_multi(image, detect_threshold: float = 1.0, draw_image: bool = Fa :param image: Template image 模板影像 (要尋找的影像) :param detect_threshold: Detection precision (0.0 ~ 1.0, 1.0 = 完全相同) :param draw_image: Whether to draw detection markers 是否在回傳影像上標記偵測結果 - :return: List[List[int]] 多個座標位置 [[x1, y1], [x2, y2], ...] + :param all_screens: Search every monitor 是否搜尋所有螢幕 + :param screen_region: Limit the search to (x, y, width, height) 限定搜尋範圍 + :return: [found, [[x1, y1, x2, y2], ...]] 座標為螢幕座標 """ - # 擷取螢幕畫面 Capture screen - grab_image = ImageGrab.grab() - - # 使用模板匹配 Find multiple objects - return template_detection.find_multi_object( + grab_image, origin_x, origin_y = grab_logical( + screen_region, all_screens=all_screens) + result = template_detection.find_multi_object( image=grab_image, template=image, detect_threshold=detect_threshold, draw_image=draw_image - ) \ No newline at end of file + ) + return _shift_result(result, (origin_x, origin_y), multi=True) diff --git a/je_auto_control/utils/exception/exceptions.py b/je_auto_control/utils/exception/exceptions.py index a5e8891c..ff700e48 100644 --- a/je_auto_control/utils/exception/exceptions.py +++ b/je_auto_control/utils/exception/exceptions.py @@ -38,6 +38,16 @@ class ImageNotFoundException(AutoControlException): pass +class AutoControlFlatTemplateException(AutoControlScreenException): + """Template has (almost) no variation, so normalised correlation degenerates. + + A subclass rather than a sibling, so existing ``except + AutoControlScreenException`` handlers keep catching it; callers that want to + tell the user what to do about it (crop something with a pattern in it) can + catch this one specifically. + """ + + # Record diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 065e714e..fb919e02 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -81,12 +81,27 @@ from je_auto_control.wrapper.auto_control_record import record, stop_record from je_auto_control.wrapper.auto_control_screen import screenshot, screen_size from je_auto_control.wrapper.auto_control_window import ( - close_window_by_title, focus_window, list_windows, wait_for_window, + close_window_by_title, focus_window, foreground_window, list_windows, + minimize_window_by_title, move_window_by_title, wait_for_window, + window_rect, ) +def _as_bool(value: Any) -> bool: + """Coerce a JSON-action flag to bool, accepting the usual string spellings. + + A value read from a JSON action file, a CLI argument or an MCP call can + arrive as the string ``"false"``, which is truthy — so a plain ``bool()`` + would turn "off" into "on". + """ + if isinstance(value, str): + return value.strip().lower() in ("1", "true", "yes", "on") + return bool(value) + + def _a11y_list_as_dicts(app_name: Optional[str] = None, - max_results: int = 200) -> List[dict]: + max_results: int = 200, + window_title: Optional[str] = None) -> List[dict]: """Executor adapter: list accessibility elements as plain dicts.""" from je_auto_control.utils.accessibility.accessibility_api import ( list_accessibility_elements, @@ -95,20 +110,42 @@ def _a11y_list_as_dicts(app_name: Optional[str] = None, element.to_dict() for element in list_accessibility_elements( app_name=app_name, max_results=int(max_results), + window_title=window_title, ) ] def _a11y_find_as_dict(name: Optional[str] = None, role: Optional[str] = None, - app_name: Optional[str] = None) -> Optional[dict]: + app_name: Optional[str] = None, + window_title: Optional[str] = None, + contains: Any = False) -> Optional[dict]: """Executor adapter: find an accessibility element, return its dict.""" element = find_accessibility_element( - name=name, role=role, app_name=app_name, + name=name, role=role, app_name=app_name, window_title=window_title, + contains=_as_bool(contains), ) return None if element is None else element.to_dict() +def _a11y_find_all_as_dicts(name: Optional[str] = None, + role: Optional[str] = None, + app_name: Optional[str] = None, + window_title: Optional[str] = None, + contains: Any = False, + max_results: int = 50, + scan_limit: int = 1500) -> List[dict]: + """Executor adapter: every matching accessibility element, best match first.""" + from je_auto_control.utils.accessibility.accessibility_api import ( + find_accessibility_elements, + ) + return [element.to_dict() for element in find_accessibility_elements( + name=name, role=role, app_name=app_name, window_title=window_title, + contains=_as_bool(contains), max_results=int(max_results), + scan_limit=int(scan_limit), + )] + + def _vlm_locate_as_list(description: str, screen_region: Optional[List[int]] = None, model: Optional[str] = None) -> Optional[List[int]]: @@ -2373,6 +2410,15 @@ def _control_get_value(name: Optional[str] = None, role: Optional[str] = None, automation_id=automation_id) +def _control_get_state(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Optional[dict]: + """Adapter: read everything a native control currently holds, in one call.""" + from je_auto_control.utils.accessibility import control_get_state + return control_get_state(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + def _control_set_value(value: str, name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, automation_id: Optional[str] = None) -> bool: @@ -3916,12 +3962,43 @@ def _move_mouse_relative(dx: Any, dy: Any) -> Dict[str, Any]: return move_mouse_relative(int(dx), int(dy)) +def _stop_record_timeline() -> List[dict]: + """Adapter: stop recording and return the full replayable timeline.""" + from je_auto_control.wrapper.auto_control_record import ( + stop_record_timeline, + ) + return stop_record_timeline() + + +def _input_reachable() -> Dict[str, Any]: + """Adapter: can this process actually drive the machine right now?""" + from je_auto_control.utils.input_reach import ( + input_desktop_available, input_reaches_system, + ) + desktop = input_desktop_available() + # Only worth sending the probe key when the desktop can take it. + return {"desktop_available": desktop, + "reaches_system": input_reaches_system() if desktop else False} + + def _type_unicode(text: str, modifier: str = "ctrl") -> Dict[str, Any]: """Adapter: enter arbitrary Unicode text via clipboard paste.""" from je_auto_control.utils.text_unicode import type_unicode return type_unicode(text, modifier=modifier) +def _type_unicode_keys(text: str) -> Dict[str, Any]: + """Adapter: enter arbitrary Unicode text as key events, sparing the clipboard.""" + from je_auto_control.utils.text_unicode import type_unicode_keys + return type_unicode_keys(text) + + +def _type_unicode_text(text: str, modifier: str = "ctrl") -> Dict[str, Any]: + """Adapter: enter arbitrary Unicode text by the best route this platform has.""" + from je_auto_control.utils.text_unicode import type_unicode_text + return type_unicode_text(text, modifier=modifier) + + def _grid_cell(boxes: Any, row: Any, col: Any, row_tolerance: Any = 10) -> Dict[str, Any]: """Adapter: address a grid cell by (row, col) from a JSON list of boxes.""" @@ -6545,6 +6622,61 @@ def _dedupe_images(paths: Any, max_distance: int = 5) -> Dict[str, Any]: max_distance=max_distance)} +def _clipboard_get_image(path: str) -> Dict[str, Any]: + """Adapter: save the clipboard's image to ``path``. ``{saved: bool}``.""" + import os + from je_auto_control.utils.clipboard.clipboard import get_clipboard_image + payload = get_clipboard_image() + if not payload: + return {"saved": False, "path": None} + safe_path = os.path.realpath(os.fspath(path)) + with open(safe_path, "wb") as handle: + handle.write(payload) + return {"saved": True, "path": safe_path} + + +def _clipboard_set_image(path: str) -> Dict[str, Any]: + """Adapter: put an image file on the clipboard.""" + from je_auto_control.utils.clipboard.clipboard import set_clipboard_image + set_clipboard_image(path) + return {"ok": True} + + +def _foreground_window() -> Dict[str, Any]: + """Adapter: the window the user is currently working in.""" + hit = foreground_window() + if hit is None: + return {"hwnd": 0, "title": ""} + return {"hwnd": hit[0], "title": hit[1]} + + +def _window_rect(title_substring: str, + case_sensitive: bool = False) -> Dict[str, Any]: + """Adapter: a window's screen rectangle as ``{rect: [l, t, r, b]}``.""" + rect = window_rect(title_substring, bool(case_sensitive)) + return {"rect": list(rect) if rect is not None else None} + + +def _canonicalize_url(url: str) -> Dict[str, Any]: + """Adapter: opinionated canonical form of a URL, for equality checks.""" + from je_auto_control.utils.url_canon import canonicalize_url + return {"url": canonicalize_url(url)} + + +def _normalize_url(url: str, sort_query: bool = False, + drop_fragment: bool = False) -> Dict[str, Any]: + """Adapter: RFC 3986 syntax-based normalisation of a URL.""" + from je_auto_control.utils.url_canon import normalize_url + return {"url": normalize_url(url, sort_query=bool(sort_query), + drop_fragment=bool(drop_fragment))} + + +def _urls_equal(first: str, second: str) -> Dict[str, Any]: + """Adapter: whether two URLs are equivalent after canonicalisation.""" + from je_auto_control.utils.url_canon import urls_equal + return {"equal": urls_equal(first, second)} + + def _parse_decimal(text: str, locale: str = "en_US") -> Dict[str, Any]: """Adapter: parse a locale-formatted decimal string to a float.""" from je_auto_control.utils.locale_parse import parse_decimal @@ -6900,6 +7032,7 @@ def __init__(self): # Record 錄製 "AC_record": record, "AC_stop_record": stop_record, + "AC_stop_record_timeline": _stop_record_timeline, # Executor 執行器 "AC_execute_action": self.execute_action, @@ -6928,10 +7061,16 @@ def __init__(self): "AC_focus_window": focus_window, "AC_wait_window": wait_for_window, "AC_close_window": close_window_by_title, + "AC_minimize_window": minimize_window_by_title, + "AC_foreground_window": _foreground_window, + "AC_window_rect": _window_rect, + "AC_move_window": move_window_by_title, # Clipboard "AC_clipboard_get": get_clipboard, "AC_clipboard_set": set_clipboard, + "AC_clipboard_get_image": _clipboard_get_image, + "AC_clipboard_set_image": _clipboard_set_image, # Run history "AC_history_list": _history_list_as_dicts, @@ -6972,6 +7111,7 @@ def __init__(self): # Accessibility-tree widget location "AC_a11y_list": _a11y_list_as_dicts, "AC_a11y_find": _a11y_find_as_dict, + "AC_a11y_find_all": _a11y_find_all_as_dicts, "AC_a11y_click": click_accessibility_element, "AC_a11y_dump": _a11y_dump, "AC_walk_tree": _walk_tree, @@ -6980,6 +7120,7 @@ def __init__(self): "AC_audit_focus_order": _audit_focus_order, "AC_focus_control": _focus_control, "AC_control_get_value": _control_get_value, + "AC_control_get_state": _control_get_state, "AC_control_set_value": _control_set_value, "AC_control_invoke": _control_invoke, "AC_control_toggle": _control_toggle, @@ -7224,7 +7365,10 @@ def __init__(self): "AC_set_field_text": _set_field_text, "AC_hold_key": _hold_key, "AC_move_mouse_relative": _move_mouse_relative, + "AC_input_reachable": _input_reachable, "AC_type_unicode": _type_unicode, + "AC_type_unicode_keys": _type_unicode_keys, + "AC_type_unicode_text": _type_unicode_text, "AC_with_modifiers": _with_modifiers, "AC_grid_cell": _grid_cell, "AC_match_template": _match_template, @@ -7389,6 +7533,9 @@ def __init__(self): "AC_s3_delete": _s3_delete, "AC_image_hash": _image_hash, "AC_dedupe_images": _dedupe_images, + "AC_canonicalize_url": _canonicalize_url, + "AC_normalize_url": _normalize_url, + "AC_urls_equal": _urls_equal, "AC_parse_decimal": _parse_decimal, "AC_parse_number": _parse_number, "AC_format_decimal": _format_decimal, diff --git a/je_auto_control/utils/input_reach/__init__.py b/je_auto_control/utils/input_reach/__init__.py new file mode 100644 index 00000000..34bdd91e --- /dev/null +++ b/je_auto_control/utils/input_reach/__init__.py @@ -0,0 +1,6 @@ +"""Whether input this process sends can actually arrive.""" +from je_auto_control.utils.input_reach.input_reach import ( + PROBE_VK, input_desktop_available, input_reaches_system, +) + +__all__ = ["PROBE_VK", "input_desktop_available", "input_reaches_system"] diff --git a/je_auto_control/utils/input_reach/input_reach.py b/je_auto_control/utils/input_reach/input_reach.py new file mode 100644 index 00000000..b525a9b0 --- /dev/null +++ b/je_auto_control/utils/input_reach/input_reach.py @@ -0,0 +1,105 @@ +"""Will the input I send actually arrive? + +Sent input can be discarded before it reaches anything, and the send call still +reports success. That is the worst failure this library has: the caller is told +"clicked (500, 300)", nothing happened, and there is no error anywhere. Two +causes, which need two different checks: + +* **The workstation is locked**, or a UAC secure desktop is up. Detected by + asking for the input desktop — a cheap, side-effect-free query. +* **Something is filtering injected input.** Anti-cheat in a foreground game + does this. Measured on such a machine: after ``SendInput`` even + ``GetAsyncKeyState`` does not see the key. Nothing cheap detects it — + ``OpenInputDesktop`` succeeds, and the game's integrity level is the same + *Medium* as ours, so a privilege comparison says everything is fine. The only + honest test is to send a key and look. + +So :func:`input_desktop_available` is free and safe to call often, while +:func:`input_reaches_system` **sends a real keystroke** and belongs on +diagnostics, not in front of every action. It uses F13, which effectively +nothing binds. + +Both answer ``True`` when they cannot tell: these report a problem, and a probe +that fails must not itself become one. Imports no ``PySide6``. +""" +import sys +import time +from typing import Optional, Tuple + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +# F13. Real keyboards stop at F12, so nothing is listening for it. +PROBE_VK = 0x7C +_DESKTOP_SWITCHDESKTOP = 0x0100 +_KEY_DOWN_MASK = 0x8000 + +# The desktop query is cheap but input primitives ask constantly; a short cache +# keeps that free without hiding a lock for long. +DESKTOP_CACHE_SEC = 2.0 +_desktop_cache: Optional[Tuple[float, bool]] = None + + +def input_desktop_available() -> bool: + """Whether the input desktop can receive events (not locked / secure desktop).""" + global _desktop_cache + if not sys.platform.startswith("win"): + return True + now = time.monotonic() + if _desktop_cache and now - _desktop_cache[0] < DESKTOP_CACHE_SEC: + return _desktop_cache[1] + available = True + try: + import ctypes + user32 = ctypes.windll.user32 + user32.OpenInputDesktop.restype = ctypes.c_void_p + user32.OpenInputDesktop.argtypes = [ + ctypes.c_uint, ctypes.c_bool, ctypes.c_uint] + handle = user32.OpenInputDesktop(0, False, _DESKTOP_SWITCHDESKTOP) + available = bool(handle) + if handle: + user32.CloseDesktop(ctypes.c_void_p(handle)) + except (OSError, AttributeError, ValueError) as error: + autocontrol_logger.info("input desktop probe failed: %r", error) + available = True + _desktop_cache = (now, available) + return available + + +def input_reaches_system(vk: int = PROBE_VK) -> bool: + """Send an inert key and check the system saw it. + + **This injects a real keystroke.** Call it from a diagnostic, not before + every action. ``True`` when the key was observed, or when the check itself + could not run. + """ + if not sys.platform.startswith("win"): + return True + try: + import ctypes + user32 = ctypes.windll.user32 + user32.GetAsyncKeyState.argtypes = [ctypes.c_int] + user32.GetAsyncKeyState.restype = ctypes.c_short + from je_auto_control.wrapper.auto_control_keyboard import ( + press_keyboard_key, release_keyboard_key, + ) + except (OSError, ImportError, AttributeError) as error: + autocontrol_logger.info("input reach probe unavailable: %r", error) + return True + try: + user32.GetAsyncKeyState(int(vk)) # clear the sticky "was pressed" bit + press_keyboard_key(int(vk)) + try: + time.sleep(0.02) + seen = bool(user32.GetAsyncKeyState(int(vk)) & _KEY_DOWN_MASK) + finally: + # Always release: a probe that leaves a key down is worse than no + # probe, and the caller cannot see that it happened. + release_keyboard_key(int(vk)) + except Exception as error: # noqa: BLE001 # reason: a probe must not raise + autocontrol_logger.info("input reach probe failed: %r", error) + return True + if not seen: + autocontrol_logger.warning( + "injected input is not reaching the system; something is " + "filtering it (anti-cheat in a foreground game does this)") + return seen diff --git a/je_auto_control/utils/keyboard_layout/__init__.py b/je_auto_control/utils/keyboard_layout/__init__.py new file mode 100644 index 00000000..c6729755 --- /dev/null +++ b/je_auto_control/utils/keyboard_layout/__init__.py @@ -0,0 +1,8 @@ +"""What character each key produces on the active keyboard layout.""" +from je_auto_control.utils.keyboard_layout.keyboard_layout import ( + US_PRINTABLE_VK, char_table, foreground_keyboard_layout, + layout_char_table, vk_to_char, +) + +__all__ = ["US_PRINTABLE_VK", "char_table", "foreground_keyboard_layout", + "layout_char_table", "vk_to_char"] diff --git a/je_auto_control/utils/keyboard_layout/keyboard_layout.py b/je_auto_control/utils/keyboard_layout/keyboard_layout.py new file mode 100644 index 00000000..32660e22 --- /dev/null +++ b/je_auto_control/utils/keyboard_layout/keyboard_layout.py @@ -0,0 +1,140 @@ +"""Ask the system what character each key produces on the active layout. + +A recorded session stores *virtual key codes*, but a replay — and anything that +shows the user what was recorded — needs characters. The mapping is not fixed: +letters and digits agree across Latin layouts, punctuation does not, so a +hard-coded US table mislabels every punctuation key on a German, French or +Nordic keyboard. + +Two pieces of timing make this correct rather than nearly correct: + +* **Ask about the foreground window's layout, not this thread's.** The user is + typing into whatever is in front; this process's own thread can be on a + completely different layout. +* **Translate after recording, never during.** ``ToUnicodeEx`` mutates the + keyboard's dead-key composition state, so calling it while someone is typing + corrupts the character they are half-way through composing. Record key codes, + translate once at the end, then flush the state. + +Falls back to the US table where the OS cannot answer, and returns an empty +mapping off Windows. Imports no ``PySide6``. +""" +import sys +from typing import Dict, Optional, Tuple + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +# Virtual key code -> (unshifted, shifted) on a **US** layout. Only the fallback +# for when the OS will not answer; letters and digits are layout-independent +# anyway, punctuation is what actually differs. +US_PRINTABLE_VK: Dict[int, Tuple[str, str]] = { + **{vk: (chr(vk).lower(), chr(vk)) for vk in range(0x41, 0x5B)}, # A-Z + **{vk: (chr(vk), shifted) for vk, shifted + in zip(range(0x30, 0x3A), ")!@#$%^&*(")}, # 0-9 + **{vk: (chr(0x30 + vk - 0x60),) * 2 for vk in range(0x60, 0x6A)}, # numpad + 0x20: (" ", " "), + 0xBA: (";", ":"), 0xBB: ("=", "+"), 0xBC: (",", "<"), 0xBD: ("-", "_"), + 0xBE: (".", ">"), 0xBF: ("/", "?"), 0xC0: ("`", "~"), + 0xDB: ("[", "{"), 0xDC: ("\\", "|"), 0xDD: ("]", "}"), 0xDE: ("'", '"'), + 0x6A: ("*", "*"), 0x6B: ("+", "+"), 0x6D: ("-", "-"), + 0x6E: (".", "."), 0x6F: ("/", "/"), +} + +_VK_SHIFT = 0x10 +_VK_SPACE = 0x20 +_MAPVK_VK_TO_VSC = 0 +_LAYOUT_CACHE: Dict[int, Dict[int, Tuple[str, str]]] = {} + + +def foreground_keyboard_layout() -> Optional[int]: + """The layout handle the **foreground** window's thread is using.""" + if not sys.platform.startswith("win"): + return None + try: + import ctypes + user32 = ctypes.windll.user32 + window = user32.GetForegroundWindow() + thread_id = user32.GetWindowThreadProcessId(window, None) if window else 0 + return int(user32.GetKeyboardLayout(thread_id)) + except (OSError, AttributeError, ValueError) as error: + autocontrol_logger.info("keyboard layout probe failed: %r", error) + return None + + +def _translator(user32, layout: int): + """Return ``translate(vk, shifted) -> str`` for one layout.""" + import ctypes + from ctypes import wintypes + user32.ToUnicodeEx.argtypes = [ + wintypes.UINT, wintypes.UINT, ctypes.c_char * 256, wintypes.LPWSTR, + ctypes.c_int, wintypes.UINT, wintypes.HKL] + user32.ToUnicodeEx.restype = ctypes.c_int + user32.MapVirtualKeyExW.argtypes = [ + wintypes.UINT, wintypes.UINT, wintypes.HKL] + user32.MapVirtualKeyExW.restype = wintypes.UINT + buffer = ctypes.create_unicode_buffer(8) + + def _translate(vk: int, shifted: bool) -> str: + state = (ctypes.c_char * 256)() + if shifted: + state[_VK_SHIFT] = b"\x80" + scan = user32.MapVirtualKeyExW(vk, _MAPVK_VK_TO_VSC, layout) + count = user32.ToUnicodeEx(vk, scan, state, buffer, 8, 0, layout) + if count == -1: + # A dead key. Call again to clear it out of the composition buffer, + # then report it as untranslatable rather than as its accent. + user32.ToUnicodeEx(vk, scan, state, buffer, 8, 0, layout) + return "" + return buffer.value[:count] if count > 0 else "" + + return _translate + + +def _build_table(translate) -> Dict[int, Tuple[str, str]]: + """Translate every candidate key, keeping only the printable results.""" + table: Dict[int, Tuple[str, str]] = {} + for vk in US_PRINTABLE_VK: + plain, shifted = translate(vk, False), translate(vk, True) + if len(plain) == 1 and plain.isprintable(): + usable = len(shifted) == 1 and shifted.isprintable() + table[vk] = (plain, shifted if usable else plain) + translate(_VK_SPACE, False) # flush any dead-key state left behind + return table + + +def layout_char_table(layout: Optional[int] = None + ) -> Dict[int, Tuple[str, str]]: + """``{vk: (unshifted, shifted)}`` for ``layout`` (default: the foreground one). + + Empty off Windows or when the OS will not answer, so callers can fall back + to :data:`US_PRINTABLE_VK`. + """ + if layout is None: + layout = foreground_keyboard_layout() + if layout is None or not sys.platform.startswith("win"): + return {} + if layout in _LAYOUT_CACHE: + return _LAYOUT_CACHE[layout] + try: + import ctypes + table = _build_table(_translator(ctypes.windll.user32, layout)) + except (OSError, AttributeError, ValueError) as error: + autocontrol_logger.info("layout table build failed: %r", error) + return {} + _LAYOUT_CACHE[layout] = table + return table + + +def char_table(layout: Optional[int] = None) -> Dict[int, Tuple[str, str]]: + """The layout's table over the US fallback, so every known key is covered.""" + return {**US_PRINTABLE_VK, **layout_char_table(layout)} + + +def vk_to_char(vk: int, shifted: bool = False, + table: Optional[Dict[int, Tuple[str, str]]] = None + ) -> Optional[str]: + """The character this key produces, or ``None`` if it produces none.""" + pair = (char_table() if table is None else table).get(int(vk)) + if pair is None: + return None + return pair[1] if shifted else pair[0] diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index a5130fc9..9105f38a 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -280,7 +280,9 @@ def window_tools() -> List[MCPTool]: ), MCPTool( name="ac_close_window", - description="Minimise the first window matching title_substring.", + description=("Ask the first window matching title_substring to " + "close (posts WM_CLOSE). Use ac_minimize_window to " + "only minimise it."), input_schema=schema({ "title_substring": {"type": "string"}, "case_sensitive": {"type": "boolean"}, @@ -288,6 +290,36 @@ def window_tools() -> List[MCPTool]: handler=h.close_window, annotations=DESTRUCTIVE, ), + MCPTool( + name="ac_minimize_window", + description="Minimise the first window matching title_substring.", + input_schema=schema({ + "title_substring": {"type": "string"}, + "case_sensitive": {"type": "boolean"}, + }, required=["title_substring"]), + handler=h.minimize_window, + annotations=NON_DESTRUCTIVE, + ), + MCPTool( + name="ac_foreground_window", + description=("The window the user is currently working in, as " + "{hwnd, title}; hwnd is 0 when there is none."), + input_schema=schema({}), + handler=h.foreground_window, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_window_rect", + description=("Screen rectangle of the first matching window as " + "{rect: [left, top, right, bottom]}, or {rect: null}. " + "Values can be negative on a multi-monitor desktop."), + input_schema=schema({ + "title_substring": {"type": "string"}, + "case_sensitive": {"type": "boolean"}, + }, required=["title_substring"]), + handler=h.window_rect, + annotations=READ_ONLY, + ), MCPTool( name="ac_window_move", description=("Move and resize the first matching window to " @@ -472,6 +504,19 @@ def recording_tools() -> List[MCPTool]: handler=h.record_stop, annotations=SIDE_EFFECT_ONLY, ), + MCPTool( + name="ac_record_stop_timeline", + description=("Stop the active recorder and return the full " + "event timeline: presses *and* releases, wheel " + "movement, and the real gap before each event as " + "delta_ms. ac_record_stop reports presses only, " + "which cannot reproduce a drag, a scroll, or the " + "original pacing. Feed these to " + "ac_replay_timeline."), + input_schema=schema({}), + handler=h.record_stop_timeline, + annotations=SIDE_EFFECT_ONLY, + ), MCPTool( name="ac_read_action_file", description="Read a JSON action file from disk and return its parsed contents.", @@ -610,6 +655,7 @@ def semantic_locator_tools() -> List[MCPTool]: input_schema=schema({ "app_name": {"type": "string"}, "max_results": {"type": "integer"}, + "window_title": {"type": "string"}, }), handler=h.a11y_list, annotations=READ_ONLY, @@ -617,15 +663,44 @@ def semantic_locator_tools() -> List[MCPTool]: MCPTool( name="ac_a11y_find", description=("Find the first accessibility element matching name " - "/ role / app_name. Returns null when nothing matches."), + "/ role / app_name. 'window_title' scopes the search " + "to one window (substring of its title), which is " + "much faster and less ambiguous than the whole " + "desktop. 'contains' matches the name as a " + "case-insensitive substring, ranking an exact name " + "first — real labels carry accelerators like " + "'Save(&S)'. Returns null when nothing matches."), input_schema=schema({ "name": {"type": "string"}, "role": {"type": "string"}, "app_name": {"type": "string"}, + "window_title": {"type": "string"}, + "contains": {"type": "boolean"}, }), handler=h.a11y_find, annotations=READ_ONLY, ), + MCPTool( + name="ac_a11y_find_all", + description=("Every accessibility element matching the filters, " + "best name match first. Same options as " + "ac_a11y_find, plus 'max_results' (how many " + "matches to return) and 'scan_limit' (how many " + "elements to examine looking for them) — without " + "a window_title the scan covers the front-most " + "windows only."), + input_schema=schema({ + "name": {"type": "string"}, + "role": {"type": "string"}, + "app_name": {"type": "string"}, + "window_title": {"type": "string"}, + "contains": {"type": "boolean"}, + "max_results": {"type": "integer"}, + "scan_limit": {"type": "integer"}, + }), + handler=h.a11y_find_all, + annotations=READ_ONLY, + ), MCPTool( name="ac_a11y_click", description=("Click the centre of the first accessibility " @@ -1063,6 +1138,18 @@ def a11y_control_tools() -> List[MCPTool]: handler=h.control_get_value, annotations=READ_ONLY, ), + MCPTool( + name="ac_control_get_state", + description=("Read everything a native control currently holds " + "in one call: value (+read_only), toggle " + "(on/off/mixed), selected, number. A key is absent " + "when the control has no such state, which is not " + "the same as the value being empty. Password " + "fields report only {password: true}."), + input_schema=schema(dict(_M)), + handler=h.control_get_state, + annotations=READ_ONLY, + ), MCPTool( name="ac_control_set_value", description=("Set a native control's value directly (no per-key " @@ -5262,6 +5349,40 @@ def text_unicode_tools() -> List[MCPTool]: handler=h.type_unicode, annotations=SIDE_EFFECT_ONLY, ), + MCPTool( + name="ac_input_reachable", + description=("Check whether input this process sends can arrive: " + "{desktop_available, reaches_system}. A locked " + "workstation, or anti-cheat filtering in a " + "foreground game, makes every click and keystroke " + "silently do nothing while still reporting success. " + "Sends one inert keystroke (F13) to find out."), + input_schema=schema({}), + handler=h.input_reachable, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_type_unicode_keys", + description=("Enter arbitrary Unicode 'text' as character-carrying " + "key events, leaving the clipboard untouched. Needs a " + "backend that supports it (Windows). " + "Returns {ops, plan, method, code_units}."), + input_schema=schema({"text": {"type": "string"}}, required=["text"]), + handler=h.type_unicode_keys, + annotations=SIDE_EFFECT_ONLY, + ), + MCPTool( + name="ac_type_unicode_text", + description=("Enter arbitrary Unicode 'text' by the best route this " + "platform offers: key events where available, clipboard " + "paste otherwise. 'modifier' is the paste key used by " + "the fallback. Returns {ops, plan, method, code_units}."), + input_schema=schema( + {"text": {"type": "string"}, "modifier": {"type": "string"}}, + required=["text"]), + handler=h.type_unicode_text, + annotations=SIDE_EFFECT_ONLY, + ), ] @@ -5757,6 +5878,44 @@ def image_dedup_tools() -> List[MCPTool]: ] +def url_canon_tools() -> List[MCPTool]: + _URL = {"type": "string"} + return [ + MCPTool( + name="ac_canonicalize_url", + description=("Canonical form of a URL for equality / de-duplication " + "(lower-cases scheme and host, drops a default port " + "and the fragment, collapses '.'/'..', sorts the " + "query). Returns {url}."), + input_schema=schema({"url": _URL}, ["url"]), + handler=h.canonicalize_url, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_normalize_url", + description=("RFC 3986 syntax-based normalisation of a URL, " + "preserving query order and fragment unless asked " + "otherwise. Returns {url}."), + input_schema=schema( + {"url": _URL, + "sort_query": {"type": "boolean"}, + "drop_fragment": {"type": "boolean"}}, ["url"]), + handler=h.normalize_url, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_urls_equal", + description=("Whether two URLs are equivalent after " + "canonicalisation (query order and fragment are " + "ignored). Returns {equal}."), + input_schema=schema({"first": _URL, "second": _URL}, + ["first", "second"]), + handler=h.urls_equal, + annotations=READ_ONLY, + ), + ] + + def locale_tools() -> List[MCPTool]: _LOC = {"type": "string"} return [ @@ -8707,6 +8866,7 @@ def media_assert_tools() -> List[MCPTool]: credential_lease_tools, egress_tools, approval_testing_tools, trajectory_eval_tools, compliance_tools, agent_trace_tools, video_report_tools, fuzzy_tools, artifact_store_tools, image_dedup_tools, + url_canon_tools, locale_tools, voice_tools, coordinate_space_tools, loop_guard_tools, process_mining_tools, asset_tools, events_tools, notify_channel_tools, jsonpath_tools, json_schema_tools, vuln_scan_tools, vex_tools, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 7af7bba3..76f2d695 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -424,6 +424,33 @@ def close_window(title_substring: str, case_sensitive=case_sensitive)) +def minimize_window(title_substring: str, + case_sensitive: bool = False) -> bool: + from je_auto_control.wrapper.auto_control_window import ( + minimize_window_by_title, + ) + return bool(minimize_window_by_title(title_substring, + case_sensitive=case_sensitive)) + + +def foreground_window() -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + foreground_window as _front, + ) + hit = _front() + return {"hwnd": 0, "title": ""} if hit is None else {"hwnd": hit[0], + "title": hit[1]} + + +def window_rect(title_substring: str, + case_sensitive: bool = False) -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + window_rect as _rect, + ) + rect = _rect(title_substring, case_sensitive=case_sensitive) + return {"rect": list(rect) if rect is not None else None} + + def _resolve_window_hwnd(title_substring: str, case_sensitive: bool) -> int: from je_auto_control.wrapper.auto_control_window import find_window @@ -845,7 +872,7 @@ def set_clipboard(text: str) -> str: def get_clipboard_image() -> List[MCPContent]: """Return the clipboard image as a base64 PNG content block.""" - from je_auto_control.utils.clipboard.clipboard_image import ( + from je_auto_control.utils.clipboard.clipboard import ( get_clipboard_image as _read, ) payload = _read() @@ -856,11 +883,10 @@ def get_clipboard_image() -> List[MCPContent]: def set_clipboard_image(image_path: str) -> str: - from je_auto_control.utils.clipboard.clipboard_image import ( + from je_auto_control.utils.clipboard.clipboard import ( set_clipboard_image as _write, ) - safe_path = os.path.realpath(os.fspath(image_path)) - _write(safe_path) + _write(os.path.realpath(os.fspath(image_path))) return "ok" @@ -912,6 +938,13 @@ def record_stop() -> List[Any]: return stop_record() or [] +def record_stop_timeline() -> List[Any]: + from je_auto_control.wrapper.auto_control_record import ( + stop_record_timeline, + ) + return stop_record_timeline() or [] + + def read_action_file(file_path: str) -> List[Any]: from je_auto_control.utils.json.json_file import read_action_json safe_path = os.path.realpath(os.fspath(file_path)) @@ -962,27 +995,44 @@ def merge_sleeps(actions: List[Any]) -> List[Any]: # === Semantic locators (a11y / VLM) ========================================= def a11y_list(app_name: Optional[str] = None, - max_results: int = 100) -> List[Dict[str, Any]]: + max_results: int = 100, + window_title: Optional[str] = None) -> List[Dict[str, Any]]: from je_auto_control.utils.accessibility.accessibility_api import ( list_accessibility_elements, ) return [element.to_dict() for element in list_accessibility_elements( app_name=app_name, max_results=int(max_results), + window_title=window_title, )] def a11y_find(name: Optional[str] = None, role: Optional[str] = None, - app_name: Optional[str] = None) -> Optional[Dict[str, Any]]: - from je_auto_control.utils.accessibility.accessibility_api import ( - find_accessibility_element, - ) - element = find_accessibility_element(name=name, role=role, - app_name=app_name) + app_name: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False) -> Optional[Dict[str, Any]]: + from je_auto_control.utils.accessibility import accessibility_api as api + element = api.find_accessibility_element( + name=name, role=role, app_name=app_name, window_title=window_title, + contains=bool(contains)) return None if element is None else element.to_dict() +def a11y_find_all(name: Optional[str] = None, + role: Optional[str] = None, + app_name: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False, + max_results: int = 50, + scan_limit: int = 1500) -> List[Dict[str, Any]]: + from je_auto_control.utils.accessibility import accessibility_api as api + return [element.to_dict() for element in api.find_accessibility_elements( + name=name, role=role, app_name=app_name, window_title=window_title, + contains=bool(contains), max_results=int(max_results), + scan_limit=int(scan_limit))] + + def a11y_click(name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None) -> bool: @@ -1000,6 +1050,13 @@ def control_get_value(name=None, role=None, app_name=None, automation_id=automation_id) +def control_get_state(name=None, role=None, app_name=None, + automation_id=None): + from je_auto_control.utils.accessibility import control_get_state as _g + return _g(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + def control_set_value(value, name=None, role=None, app_name=None, automation_id=None): from je_auto_control.utils.accessibility import control_set_value as _s @@ -1862,6 +1919,22 @@ def dedupe_images(paths, max_distance=5): return {"unique": _dedupe(paths, max_distance=max_distance)} +def canonicalize_url(url): + from je_auto_control.utils.url_canon import canonicalize_url as _canon + return {"url": _canon(url)} + + +def normalize_url(url, sort_query=False, drop_fragment=False): + from je_auto_control.utils.url_canon import normalize_url as _norm + return {"url": _norm(url, sort_query=bool(sort_query), + drop_fragment=bool(drop_fragment))} + + +def urls_equal(first, second): + from je_auto_control.utils.url_canon import urls_equal as _equal + return {"equal": _equal(first, second)} + + def parse_decimal(text, locale="en_US"): from je_auto_control.utils.locale_parse import parse_decimal as _parse return {"value": _parse(text, locale)} @@ -2503,11 +2576,28 @@ def move_mouse_relative(dx, dy): return _move_mouse_relative(dx, dy) +def input_reachable(): + from je_auto_control.utils.executor.action_executor import ( + _input_reachable, + ) + return _input_reachable() + + def type_unicode(text, modifier="ctrl"): from je_auto_control.utils.executor.action_executor import _type_unicode return _type_unicode(text, modifier) +def type_unicode_keys(text): + from je_auto_control.utils.executor.action_executor import _type_unicode_keys + return _type_unicode_keys(text) + + +def type_unicode_text(text, modifier="ctrl"): + from je_auto_control.utils.executor.action_executor import _type_unicode_text + return _type_unicode_text(text, modifier) + + def with_modifiers(modifiers, actions): from je_auto_control.utils.executor.action_executor import _with_modifiers return _with_modifiers(modifiers, actions) diff --git a/je_auto_control/utils/monitor_layout/__init__.py b/je_auto_control/utils/monitor_layout/__init__.py index 0dcdd9a0..d1c70e40 100644 --- a/je_auto_control/utils/monitor_layout/__init__.py +++ b/je_auto_control/utils/monitor_layout/__init__.py @@ -1,9 +1,13 @@ """Multi-monitor / virtual-desktop geometry (which monitor, where, remapping).""" +from je_auto_control.utils.monitor_layout.logical_frame import ( + grab_logical, logical_scale, logical_virtual_rect, needs_rescale, +) from je_auto_control.utils.monitor_layout.monitor_layout import ( Monitor, enumerate_monitors, monitor_at_point, monitor_for_window, primary_monitor, remap_point, to_local, to_virtual, virtual_bounds, ) -__all__ = ["Monitor", "enumerate_monitors", "monitor_at_point", - "monitor_for_window", "primary_monitor", "remap_point", "to_local", +__all__ = ["Monitor", "enumerate_monitors", "grab_logical", "logical_scale", + "logical_virtual_rect", "monitor_at_point", "monitor_for_window", + "needs_rescale", "primary_monitor", "remap_point", "to_local", "to_virtual", "virtual_bounds"] diff --git a/je_auto_control/utils/monitor_layout/logical_frame.py b/je_auto_control/utils/monitor_layout/logical_frame.py new file mode 100644 index 00000000..3fd3de7b --- /dev/null +++ b/je_auto_control/utils/monitor_layout/logical_frame.py @@ -0,0 +1,119 @@ +"""Capture a frame whose pixels map 1:1 onto the coordinates the mouse takes. + +Two things quietly disagree on Windows once a second monitor is attached: + +* ``ImageGrab.grab()`` sees **only the primary monitor**, so anything located + from it can never be on the second one — the search does not fail, it just + never finds. +* ``ImageGrab.grab(all_screens=True)`` makes itself DPI-aware first and returns + **physical** pixels, while a DPI-unaware process (and therefore + ``GetSystemMetrics`` and every mouse API) works in **logical** pixels. On a + mixed-DPI desktop the two differ — a 1920×1080 monitor beside a 1920×1080 one + scaled to 125% is 3840 physical but 3456 logical wide — so a point read off + the capture lands somewhere else when clicked. Measured on such a desktop the + drift reaches ~116 px, which reads as "sometimes misses" rather than "broken". + +Both are the same requirement: one pixel in the frame must be one coordinate for +the mouse. ``grab_logical`` captures the whole virtual desktop and scales it back +into the logical space, reporting the origin to add to any hit — the virtual +desktop starts at negative coordinates whenever a monitor sits left of or above +the primary one. + +The arithmetic (:func:`needs_rescale`, :func:`logical_scale`) is pure and +unit-testable; the OS reader and the grabber are both injectable. Imports no +``PySide6``. +""" +import sys +from typing import Any, Callable, Optional, Sequence, Tuple + +Rect = Tuple[int, int, int, int] +MetricsReader = Callable[[int], int] + +# GetSystemMetrics indices for the virtual desktop, in logical pixels. +SM_XVIRTUALSCREEN = 76 +SM_YVIRTUALSCREEN = 77 +SM_CXVIRTUALSCREEN = 78 +SM_CYVIRTUALSCREEN = 79 + + +def _system_metrics(index: int) -> int: + """Read one ``GetSystemMetrics`` value; 0 off Windows.""" + if not sys.platform.startswith("win"): + return 0 + import ctypes + return int(ctypes.windll.user32.GetSystemMetrics(index)) + + +def logical_virtual_rect(metrics: Optional[MetricsReader] = None) -> Optional[Rect]: + """Virtual desktop as ``(x, y, width, height)`` in mouse coordinates. + + ``None`` where the platform cannot report it, so callers skip the rescale + rather than guess. + """ + reader = metrics or _system_metrics + try: + rect = (reader(SM_XVIRTUALSCREEN), reader(SM_YVIRTUALSCREEN), + reader(SM_CXVIRTUALSCREEN), reader(SM_CYVIRTUALSCREEN)) + except (OSError, AttributeError, ValueError): + return None + return rect if rect[2] > 0 and rect[3] > 0 else None + + +def logical_scale(physical: Tuple[int, int], + logical: Tuple[int, int]) -> Tuple[float, float]: + """Physical-to-logical pixel ratio per axis.""" + width = physical[0] / logical[0] if logical[0] else 1.0 + height = physical[1] / logical[1] if logical[1] else 1.0 + return width, height + + +def needs_rescale(physical: Tuple[int, int], logical: Tuple[int, int]) -> bool: + """Whether a capture is in a different pixel space from the mouse.""" + return bool(logical[0] and logical[1]) and tuple(physical) != tuple(logical) + + +def _load_image_grab(): + """Import Pillow's ``ImageGrab`` lazily (optional dependency at runtime).""" + from PIL import ImageGrab + return ImageGrab + + +def _resample(): + """Pillow's high-quality downscale filter, across Pillow versions.""" + from PIL import Image + return getattr(getattr(Image, "Resampling", Image), "LANCZOS") + + +def grab_logical(region: Optional[Sequence[int]] = None, *, + all_screens: bool = True, + grabber: Optional[Callable[..., Any]] = None, + metrics: Optional[MetricsReader] = None) -> Tuple[Any, int, int]: + """Capture the screen in mouse-coordinate space. + + :param region: ``(x, y, width, height)`` in mouse coordinates, or ``None`` + for everything. + :param all_screens: include monitors beyond the primary one. + :param grabber: ``ImageGrab``-shaped object, for tests. + :param metrics: ``GetSystemMetrics``-shaped reader, for tests. + :return: ``(image, origin_x, origin_y)`` — add the origin to any hit found in + the image to get a coordinate the mouse can be sent to. + """ + image_grab = grabber or _load_image_grab() + if region is None and not all_screens: + # The primary-only grab is already in logical pixels and starts at (0, 0). + return image_grab.grab(), 0, 0 + + image = image_grab.grab(all_screens=True) + rect = logical_virtual_rect(metrics) + origin_x, origin_y = (rect[0], rect[1]) if rect else (0, 0) + if rect and needs_rescale((image.width, image.height), (rect[2], rect[3])): + image = image.resize((rect[2], rect[3]), _resample()) + if region is None: + return image, origin_x, origin_y + + # Crop on the rescaled frame, never through ImageGrab's bbox: that crop + # happens in physical pixels and would cut the wrong place on a scaled screen. + left, top, width, height = (int(value) for value in region) + image = image.crop((left - origin_x, top - origin_y, + left - origin_x + width, top - origin_y + height)) + return image, left, top diff --git a/je_auto_control/utils/ocr/__init__.py b/je_auto_control/utils/ocr/__init__.py index ba3c00c5..d264b0ab 100644 --- a/je_auto_control/utils/ocr/__init__.py +++ b/je_auto_control/utils/ocr/__init__.py @@ -3,8 +3,9 @@ TextMatch, click_text, find_text_matches, locate_text_center, set_tesseract_cmd, wait_for_text, ) +from je_auto_control.utils.ocr.text_span import find_spans, group_lines __all__ = [ - "TextMatch", "click_text", "find_text_matches", "locate_text_center", - "set_tesseract_cmd", "wait_for_text", + "TextMatch", "click_text", "find_spans", "find_text_matches", "group_lines", + "locate_text_center", "set_tesseract_cmd", "wait_for_text", ] diff --git a/je_auto_control/utils/ocr/ocr_engine.py b/je_auto_control/utils/ocr/ocr_engine.py index 0fa1eb9b..0d13b3c2 100644 --- a/je_auto_control/utils/ocr/ocr_engine.py +++ b/je_auto_control/utils/ocr/ocr_engine.py @@ -16,9 +16,11 @@ from je_auto_control.utils.exception.exceptions import AutoControlActionException from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.monitor_layout.logical_frame import grab_logical from je_auto_control.utils.ocr.backends import ( OCRBackend, OCRBackendNotAvailableError, get_backend, ) +from je_auto_control.utils.ocr.text_span import find_spans _image_grab = None @@ -66,36 +68,15 @@ def set_tesseract_cmd(path: str) -> None: backend.set_cmd(path) -def _virtual_screen_origin() -> Tuple[int, int]: - """Return (x, y) origin of the virtual desktop in screen coordinates. +def _grab(region: Optional[Sequence[int]]): + """Capture in mouse-coordinate space; returns ``(image, offset_x, offset_y)``. - On Windows with multiple monitors, the virtual screen can start at - negative coordinates (e.g. a monitor positioned above or to the left - of the primary). ``ImageGrab.grab(all_screens=True)`` captures the - full virtual screen with its top-left at (0, 0) of the captured - image — meaning image-local coords differ from screen coords by the - virtual-screen origin. Without compensating, OCR-derived - coordinates can't be clicked on directly. + Delegated to :func:`monitor_layout.grab_logical` so the OCR and template + paths cannot drift apart: it handles both the negative virtual-desktop + origin and the physical-vs-logical pixel mismatch on a mixed-DPI desktop, + either of which silently shifts every coordinate this module reports. """ - try: - import ctypes - user32 = ctypes.windll.user32 - return int(user32.GetSystemMetrics(76)), int(user32.GetSystemMetrics(77)) - except Exception: - return 0, 0 - - -def _grab(region: Optional[Sequence[int]]): - image_grab = _load_image_grab() - if region is None: - # Full virtual screen capture — origin may be negative on - # multi-monitor setups, so report it as the coord offset to add - # to image-local matches. - vx, vy = _virtual_screen_origin() - return image_grab.grab(all_screens=True), vx, vy - x, y, w, h = region - bbox = (int(x), int(y), int(x) + int(w), int(y) + int(h)) - return image_grab.grab(bbox=bbox, all_screens=True), int(x), int(y) + return grab_logical(region, grabber=_load_image_grab()) def _resolve(backend: Optional[Union[str, OCRBackend]]) -> OCRBackend: @@ -125,11 +106,27 @@ def find_text_matches(target: str, text=m.text, x=m.x + offset_x, y=m.y + offset_y, width=m.width, height=m.height, confidence=m.confidence, ) for m in matches] - - needle = target if case_sensitive else target.lower() - return [m for m in shifted - if (m.text if case_sensitive else m.text.lower()) == needle - or needle in (m.text if case_sensitive else m.text.lower())] + # Engines box one *word* at a time, so a target crossing a word boundary + # ("Save As", "另存新檔") has no single box to match against. Search runs of + # consecutive boxes on a line; a run of one is the old per-box behaviour. + spans = find_spans(shifted, target, case_sensitive=case_sensitive) + return [_merge_span(span) for span in spans] + + +def _merge_span(span: List[TextMatch]) -> TextMatch: + """Collapse a run of word boxes into the single box they cover.""" + if len(span) == 1: + return span[0] + left = min(m.x for m in span) + top = min(m.y for m in span) + right = max(m.x + m.width for m in span) + bottom = max(m.y + m.height for m in span) + return TextMatch( + text=" ".join(m.text for m in span), + x=left, y=top, width=right - left, height=bottom - top, + # The weakest word bounds how much the whole reading can be trusted. + confidence=min(m.confidence for m in span), + ) def read_text_in_region(region: Optional[Sequence[int]] = None, diff --git a/je_auto_control/utils/ocr/text_span.py b/je_auto_control/utils/ocr/text_span.py new file mode 100644 index 00000000..9e4607e6 --- /dev/null +++ b/je_auto_control/utils/ocr/text_span.py @@ -0,0 +1,121 @@ +"""Match a target that OCR split across several word boxes on one line. + +Engines return one box per *word*, so ``另存新檔`` comes back as ``另存`` + +``新檔`` and ``Save As`` as ``Save`` + ``As``. Comparing the target against one +box at a time therefore misses every target that crosses a word boundary — which +is most real menu items and button labels, and the failure is silent: the caller +just gets "text not found" for text plainly on the screen. + +The logic here is pure and backend-agnostic. Boxes are grouped into lines by +vertical overlap rather than by an engine-supplied line id, because not every +backend reports one. Within a line, the shortest run of consecutive boxes whose +concatenation contains the target wins. Whitespace is dropped from both sides +before comparing: where the engine chooses to split is arbitrary, so it must not +decide whether a match exists. Imports no ``PySide6``. +""" +import re +from typing import Any, List, Optional, Sequence, Tuple + +# How far apart two boxes' vertical centres may be, as a fraction of the shorter +# box's height, and still count as the same line. Generous enough for mixed font +# sizes in one row (an icon label next to a heading), tight enough that the row +# above does not bleed in. +LINE_TOLERANCE = 0.6 + +# Give up extending a run once it is this many characters longer than the target +# — beyond that the run cannot become a *shortest* match, and a long line would +# otherwise cost O(words²) concatenations. +MAX_OVERSHOOT = 40 + +_WHITESPACE = re.compile(r"\s+") + + +def normalize(text: str, case_sensitive: bool = False) -> str: + """Strip all whitespace (and case, by default) for comparison.""" + stripped = _WHITESPACE.sub("", text or "") + return stripped if case_sensitive else stripped.lower() + + +def same_line(first: Any, second: Any, + tolerance: float = LINE_TOLERANCE) -> bool: + """Whether two boxes sit on the same text line, by vertical overlap.""" + first_centre = first.y + first.height / 2.0 + second_centre = second.y + second.height / 2.0 + shorter = max(1.0, float(min(first.height, second.height))) + return abs(first_centre - second_centre) <= tolerance * shorter + + +def group_lines(boxes: Sequence[Any], + tolerance: float = LINE_TOLERANCE) -> List[List[Any]]: + """Group boxes into lines, each sorted left to right.""" + lines: List[List[Any]] = [] + for box in sorted(boxes, key=lambda item: (item.y, item.x)): + for line in lines: + if same_line(line[-1], box, tolerance): + line.append(box) + break + else: + lines.append([box]) + return [sorted(line, key=lambda item: item.x) for line in lines] + + +def _joined(line: Sequence[Any], start: int, end: int, + case_sensitive: bool) -> str: + """Concatenated, normalized text of ``line[start:end + 1]``.""" + return "".join(normalize(box.text, case_sensitive) + for box in line[start:end + 1]) + + +def _shrink_left(line: Sequence[Any], start: int, end: int, needle: str, + case_sensitive: bool) -> int: + """Advance ``start`` while the run still contains ``needle``. + + Without this the run is merely *a* match, not the *shortest* one: searching + "Save As" in ``File | Save | As`` would report the whole line and click its + centre — on "Save" if you are lucky, on nothing if you are not. + """ + while start < end and needle in _joined(line, start + 1, end, case_sensitive): + start += 1 + return start + + +def _next_span(line: Sequence[Any], index: int, needle: str, + case_sensitive: bool) -> Optional[Tuple[int, int]]: + """First ``(start, end)`` run at or after ``index`` that spells ``needle``.""" + accumulated = "" + for end in range(index, len(line)): + accumulated += normalize(line[end].text, case_sensitive) + if needle in accumulated: + return _shrink_left(line, index, end, needle, case_sensitive), end + if len(accumulated) > len(needle) + MAX_OVERSHOOT: + # This start can no longer produce a shortest match; drop the + # leftmost box and keep scanning instead of restarting the line. + index += 1 + accumulated = _joined(line, index, end, case_sensitive) + return None + + +def find_spans(boxes: Sequence[Any], target: str, + case_sensitive: bool = False, + tolerance: float = LINE_TOLERANCE) -> List[List[Any]]: + """Return every run of consecutive same-line boxes that spells ``target``. + + Runs are minimal, and a single box that already contains the target comes + back as a one-box run — so this is a superset of per-box matching. + """ + needle = normalize(target, case_sensitive) + if not needle: + return [] + spans: List[List[Any]] = [] + for line in group_lines(boxes, tolerance): + index = 0 + while index < len(line): + found = _next_span(line, index, needle, case_sensitive) + if found is None: + break + start, end = found + spans.append(list(line[start:end + 1])) + # Resume after the run so one hit is not reported again from a + # later start inside it. + index = end + 1 + return spans diff --git a/je_auto_control/utils/text_unicode/__init__.py b/je_auto_control/utils/text_unicode/__init__.py index e94ea2ce..28599bd2 100644 --- a/je_auto_control/utils/text_unicode/__init__.py +++ b/je_auto_control/utils/text_unicode/__init__.py @@ -1,6 +1,9 @@ -"""Type arbitrary Unicode (emoji / CJK / accented) via the clipboard.""" +"""Type arbitrary Unicode (emoji / CJK / accented) by key injection or clipboard.""" from je_auto_control.utils.text_unicode.text_unicode import ( - plan_paste, type_unicode, unicode_code_units, + plan_paste, plan_unicode_keys, type_unicode, type_unicode_keys, + type_unicode_text, unicode_code_units, unicode_keys_supported, ) -__all__ = ["plan_paste", "type_unicode", "unicode_code_units"] +__all__ = ["plan_paste", "plan_unicode_keys", "type_unicode", + "type_unicode_keys", "type_unicode_text", "unicode_code_units", + "unicode_keys_supported"] diff --git a/je_auto_control/utils/text_unicode/text_unicode.py b/je_auto_control/utils/text_unicode/text_unicode.py index de7edf2a..9e915857 100644 --- a/je_auto_control/utils/text_unicode/text_unicode.py +++ b/je_auto_control/utils/text_unicode/text_unicode.py @@ -1,15 +1,23 @@ -"""Type arbitrary Unicode (emoji / CJK / accented) via the clipboard. +"""Type arbitrary Unicode (emoji / CJK / accented) by key injection or clipboard. ``write`` types through the platform virtual-key table and *raises* on any -character outside it — emoji, CJK, many accented letters — so non-ASCII text -entry is impossible through the normal path. The reliable, cross-platform way to -enter arbitrary Unicode is to put it on the clipboard and paste it. - -:func:`plan_paste` builds the deterministic op-plan and :func:`unicode_code_units` -splits text into UTF-16 code units (for a backend that can do -``KEYEVENTF_UNICODE``); both are pure and unit-testable. :func:`type_unicode` -dispatches the paste plan through an injectable ``sink`` so it is tested without -touching the real clipboard. Imports no ``PySide6``. +character outside it — emoji, CJK, many accented letters, and on a US table even +``, . / : ? ! _ + @ %`` — so non-ASCII and most punctuation are unreachable +through the normal path. + +Two ways out, and the difference matters: + +* **Key injection** (:func:`type_unicode_keys`) sends each UTF-16 code unit as a + character-carrying key event. Nothing else on the machine changes, so it is the + default wherever the backend supports it (Windows ``KEYEVENTF_UNICODE``). +* **Clipboard paste** (:func:`type_unicode`) works everywhere but **overwrites + whatever the user had on the clipboard**, and fails outright in fields that + block paste (many password and licence-key inputs). + +:func:`plan_paste`, :func:`plan_unicode_keys` and :func:`unicode_code_units` are +pure and unit-testable; the typing entry points dispatch through an injectable +``sink`` so they are tested without touching the real clipboard or keyboard. +Imports no ``PySide6``. """ from typing import Any, Callable, Dict, List, Optional @@ -36,6 +44,25 @@ def plan_paste(text: str, *, modifier: str = "ctrl") -> List[Dict[str, Any]]: {"op": "hotkey", "keys": [modifier, "v"]}] +def plan_unicode_keys(text: str) -> List[Dict[str, Any]]: + """Return the op-plan to enter ``text`` as character-carrying key events. + + One op per UTF-16 code unit, so a character above U+FFFF becomes the two + surrogates the platform layer has to send separately. + """ + return [{"op": "unicode_unit", "unit": unit} + for unit in unicode_code_units(text)] + + +def unicode_keys_supported() -> bool: + """Whether this platform's keyboard backend can inject Unicode directly.""" + try: + from je_auto_control.wrapper.platform_wrapper import keyboard + except (ImportError, AttributeError): + return False + return callable(getattr(keyboard, "type_unicode_unit", None)) + + def _default_sink(event: Dict[str, Any]) -> None: """Default dispatch: drive the real clipboard / keyboard backend.""" op = event["op"] @@ -45,6 +72,9 @@ def _default_sink(event: Dict[str, Any]) -> None: elif op == "hotkey": from je_auto_control.wrapper.auto_control_keyboard import hotkey hotkey(list(event["keys"])) + elif op == "unicode_unit": + from je_auto_control.wrapper.platform_wrapper import keyboard + keyboard.type_unicode_unit(int(event["unit"])) def type_unicode(text: str, *, modifier: str = "ctrl", @@ -53,10 +83,44 @@ def type_unicode(text: str, *, modifier: str = "ctrl", ``modifier`` is the platform paste key (``"ctrl"``; use ``"command"`` on macOS). Returns the dispatched plan plus the UTF-16 code-unit count. + + Prefer :func:`type_unicode_text` unless the clipboard route is wanted + deliberately — this one replaces the user's clipboard contents. """ plan = plan_paste(text, modifier=modifier) + return _dispatch(plan, text, sink, "paste") + + +def type_unicode_keys(text: str, *, + sink: Optional[Sink] = None) -> Dict[str, Any]: + """Enter ``text`` as character-carrying key events, leaving the clipboard alone. + + Requires a backend exposing ``type_unicode_unit`` (Windows today); callers + that need a guaranteed route on every platform should use + :func:`type_unicode_text`. + """ + plan = plan_unicode_keys(text) + return _dispatch(plan, text, sink, "keys") + + +def type_unicode_text(text: str, *, modifier: str = "ctrl", + sink: Optional[Sink] = None) -> Dict[str, Any]: + """Enter ``text`` by the best route this platform offers. + + Key injection when the backend supports it, clipboard paste otherwise. The + returned ``method`` says which one ran, because the two are not equivalent: + paste clobbers the clipboard and is refused by some inputs. + """ + if unicode_keys_supported(): + return type_unicode_keys(text, sink=sink) + return type_unicode(text, modifier=modifier, sink=sink) + + +def _dispatch(plan: List[Dict[str, Any]], text: str, + sink: Optional[Sink], method: str) -> Dict[str, Any]: + """Run ``plan`` through ``sink`` and describe what was dispatched.""" dispatch = sink or _default_sink for event in plan: dispatch(event) - return {"ops": len(plan), "plan": plan, + return {"ops": len(plan), "plan": plan, "method": method, "code_units": len(unicode_code_units(text))} diff --git a/je_auto_control/utils/url_canon/__init__.py b/je_auto_control/utils/url_canon/__init__.py new file mode 100644 index 00000000..e038ca5d --- /dev/null +++ b/je_auto_control/utils/url_canon/__init__.py @@ -0,0 +1,9 @@ +"""RFC 3986 URL canonicalisation, normalisation and query helpers.""" +from je_auto_control.utils.url_canon.url_canon import ( + build_query, canonicalize_url, normalize_url, parse_query, urls_equal, +) + +__all__ = [ + "build_query", "canonicalize_url", "normalize_url", "parse_query", + "urls_equal", +] diff --git a/je_auto_control/utils/url_canon/url_canon.py b/je_auto_control/utils/url_canon/url_canon.py new file mode 100644 index 00000000..9889280d --- /dev/null +++ b/je_auto_control/utils/url_canon/url_canon.py @@ -0,0 +1,106 @@ +"""RFC 3986 URL canonicalisation, normalisation and query helpers. + +The ``egress`` policy only lowercases a URL's hostname for matching and +``http_client`` passes raw URLs straight to ``urllib``. Nothing lowercases the +scheme, removes a default port, collapses ``.``/``..`` path segments, normalises +percent-encoding case, or offers query build/parse/sort or URL-equality — the +primitives every crawler, cache key, allowlist match and link comparison needs. + +Pure standard library (``urllib.parse`` + ``posixpath``); imports no ``PySide6``. +Every function is pure (URL in, URL/bool/list out), so it is fully deterministic +in CI. +""" +import posixpath +import re +from typing import List, Mapping, Sequence, Tuple, Union +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +_DEFAULT_PORTS = {"http": 80, "https": 443, "ftp": 21, "ws": 80, "wss": 443} +_PERCENT = re.compile(r"%[0-9a-fA-F]{2}") +QueryPairs = Sequence[Tuple[str, str]] + + +def _normalize_percent(text: str) -> str: + """Upper-case the hex digits of every percent-escape (RFC 3986 §6.2.2.1).""" + return _PERCENT.sub(lambda match: match.group(0).upper(), text) + + +def _normalize_path(path: str) -> str: + """Collapse ``.``/``..`` segments and guarantee a leading slash.""" + if not path: + return "/" + collapsed = posixpath.normpath(path) + if path.endswith("/") and not collapsed.endswith("/"): + collapsed += "/" + if not collapsed.startswith("/"): + collapsed = "/" + collapsed + return _normalize_percent(collapsed) + + +def _build_netloc(parts: "urlsplit", host: str, scheme: str, + strip_default_port: bool) -> str: + """Reassemble the authority, dropping a redundant default port.""" + userinfo = "" + if parts.username: + userinfo = parts.username + if parts.password: + userinfo += ":" + parts.password + userinfo += "@" + port = parts.port + if (port is not None and strip_default_port + and _DEFAULT_PORTS.get(scheme) == port): + port = None + netloc = userinfo + host + if port is not None: + netloc += f":{port}" + return netloc + + +def normalize_url(url: str, *, sort_query: bool = False, + strip_default_port: bool = True, + strip_fragment: bool = False) -> str: + """Return a normalised form of ``url`` (RFC 3986 syntax-based).""" + parts = urlsplit((url or "").strip()) + scheme = parts.scheme.lower() + host = (parts.hostname or "").lower() + netloc = _build_netloc(parts, host, scheme, strip_default_port) + path = _normalize_path(parts.path) if netloc or parts.path else parts.path + query = parts.query + if query: + pairs = parse_qsl(query, keep_blank_values=True) + if sort_query: + pairs = sorted(pairs) + query = _normalize_percent(urlencode(pairs)) + fragment = "" if strip_fragment else parts.fragment + return urlunsplit((scheme, netloc, path, query, fragment)) + + +def canonicalize_url(url: str) -> str: + """Return the opinionated canonical form for equality / de-duplication. + + Sorts the query, drops the default port and the fragment on top of + :func:`normalize_url`. + """ + return normalize_url(url, sort_query=True, strip_default_port=True, + strip_fragment=True) + + +def urls_equal(first: str, second: str) -> bool: + """Whether two URLs are equivalent after canonicalisation.""" + return canonicalize_url(first) == canonicalize_url(second) + + +def build_query(params: Union[Mapping[str, object], QueryPairs], *, + sort: bool = False, doseq: bool = True) -> str: + """Encode a mapping or pair-list into a query string.""" + items: List[Tuple[str, object]] = (list(params.items()) + if isinstance(params, Mapping) + else list(params)) + if sort: + items = sorted(items, key=lambda kv: (kv[0], str(kv[1]))) + return urlencode(items, doseq=doseq) + + +def parse_query(query: str, *, keep_blank: bool = True) -> List[Tuple[str, str]]: + """Parse a query string into an ordered list of ``(key, value)`` pairs.""" + return parse_qsl(query or "", keep_blank_values=keep_blank) diff --git a/je_auto_control/utils/visual_match/visual_match.py b/je_auto_control/utils/visual_match/visual_match.py index d02e3223..b63c3b48 100644 --- a/je_auto_control/utils/visual_match/visual_match.py +++ b/je_auto_control/utils/visual_match/visual_match.py @@ -16,7 +16,9 @@ from dataclasses import asdict, dataclass from typing import Any, Dict, List, Optional, Sequence -from je_auto_control.utils.exception.exceptions import AutoControlScreenException +from je_auto_control.utils.exception.exceptions import ( + AutoControlFlatTemplateException, AutoControlScreenException, +) # cv2 method name -> the OpenCV constant is resolved lazily in _method(). _METHOD_NAMES = ("ccoeff_normed", "ccorr_normed", "sqdiff_normed") @@ -98,7 +100,7 @@ def _to_gray(source: ImageSource): if hasattr(source, "shape"): array = np.asarray(source) elif isinstance(source, (str, bytes)) or hasattr(source, "__fspath__"): - array = cv2.imread(str(source), cv2.IMREAD_COLOR) + array = _imread(str(source), cv2.IMREAD_COLOR) if array is None: raise ValueError(f"could not read image: {source!r}") is_bgr = True @@ -110,14 +112,74 @@ def _to_gray(source: ImageSource): def _grab_gray(region: Optional[Sequence[int]]): - from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot - image = pil_screenshot(screen_region=list(region) if region else None) - return _to_gray(image) + return _grab_gray_with_origin(region)[0] + + +def _grab_gray_with_origin(region: Optional[Sequence[int]]): + """``(gray screen, origin_x, origin_y)`` in the coordinate space of the mouse. + + Goes through ``monitor_layout.grab_logical`` rather than a plain screenshot: + that one sees only the primary monitor (so a target on a second display is + never found) and returns physical pixels on a scaled desktop (so a matched + point is clicked in the wrong place). The origin is what to add to an + image-local hit — it is negative when a monitor sits left of or above the + primary one. + """ + from je_auto_control.utils.monitor_layout.logical_frame import grab_logical + image, origin_x, origin_y = grab_logical(region) + return _to_gray(image), origin_x, origin_y def _haystack_gray(haystack: Optional[ImageSource], region: Optional[Sequence[int]]): - return _to_gray(haystack) if haystack is not None else _grab_gray(region) + return _haystack_gray_with_origin(haystack, region)[0] + + +def _haystack_gray_with_origin(haystack: Optional[ImageSource], + region: Optional[Sequence[int]]): + """``(gray haystack, origin_x, origin_y)``. + + A caller-supplied ``haystack`` is its own coordinate space, so its origin is + ``(0, 0)`` — only a real screen grab carries a screen offset. + """ + if haystack is not None: + return _to_gray(haystack), 0, 0 + return _grab_gray_with_origin(region) + + +def _imread(path: str, flags: int): + """Read an image file, tolerating non-ASCII paths. + + ``cv2.imread`` goes through the C locale on Windows and simply returns + ``None`` for a path containing non-ASCII characters — indistinguishable + from a corrupt file, and it hits anyone whose user name or folder is not + Latin. Decoding the bytes ourselves sidesteps the filename entirely. + """ + import cv2 + import numpy as np + try: + with open(path, "rb") as handle: + buffer = np.frombuffer(handle.read(), dtype=np.uint8) + except OSError as error: + raise ValueError(f"could not read image: {path!r}") from error + return cv2.imdecode(buffer, flags) + + +# A template with (almost) no variation breaks normalised correlation: the +# denominator is the template's variance, so as it approaches zero the whole +# score map saturates at 1.0 and the matcher "finds" the target at an arbitrary +# position. Failing outright is far better than clicking somewhere random — +# users hit this by cropping the flat inside of a button. +FLAT_TEMPLATE_STD = 1.0 + + +def _reject_flat_template(template) -> None: + """Raise if ``template`` is too uniform for normalised correlation.""" + import numpy as np + if float(np.asarray(template, dtype=np.float64).std()) < FLAT_TEMPLATE_STD: + raise AutoControlFlatTemplateException( + "template is almost a single colour, so normalised correlation " + "cannot locate it; crop a region with a pattern or text in it") def _resize(template, scale: float): @@ -164,7 +226,8 @@ def match_template(template: ImageSource, *, haystack: Optional[ImageSource] = N """ import cv2 tmpl = _to_gray(template) - hay = _haystack_gray(haystack, region) + _reject_flat_template(tmpl) + hay, origin_x, origin_y = _haystack_gray_with_origin(haystack, region) metric = _method(method) best: Optional[Match] = None for scale in scales: @@ -174,8 +237,9 @@ def match_template(template: ImageSource, *, haystack: Optional[ImageSource] = N _, max_val, _, max_loc = cv2.minMaxLoc(cv2.matchTemplate(hay, scaled, metric)) if max_val >= min_score and (best is None or max_val > best.score): - best = Match(int(max_loc[0]), int(max_loc[1]), scaled.shape[1], - scaled.shape[0], round(float(max_val), 4), float(scale)) + best = Match(int(max_loc[0]) + origin_x, int(max_loc[1]) + origin_y, + scaled.shape[1], scaled.shape[0], + round(float(max_val), 4), float(scale)) return best @@ -204,7 +268,8 @@ def _nms(matches: List[Match], iou_threshold: float) -> List[Match]: def _select_candidates(result, min_score: float, width: int, height: int, - max_results: int) -> List[Match]: + max_results: int, origin: Sequence[int] = (0, 0), + ) -> List[Match]: """Candidate matches >= ``min_score``, capped to the top scorers. ``best_matches`` calls ``match_template_all(min_score=-1.0)``; since @@ -221,7 +286,8 @@ def _select_candidates(result, min_score: float, width: int, height: int, if scores.size > cap: top = np.argpartition(scores, scores.size - cap)[-cap:] xs, ys, scores = xs[top], ys[top], scores[top] - return [Match(int(x), int(y), width, height, round(float(s), 4), 1.0) + return [Match(int(x) + int(origin[0]), int(y) + int(origin[1]), + width, height, round(float(s), 4), 1.0) for x, y, s in zip(xs, ys, scores)] @@ -239,11 +305,14 @@ def match_template_all(template: ImageSource, *, """ import cv2 tmpl = _to_gray(template) - hay = _haystack_gray(haystack, region) + _reject_flat_template(tmpl) + hay, origin_x, origin_y = _haystack_gray_with_origin(haystack, region) height, width = tmpl.shape[:2] + if height > hay.shape[0] or width > hay.shape[1]: + return [] result = cv2.matchTemplate(hay, tmpl, cv2.TM_CCOEFF_NORMED) candidates = _select_candidates(result, min_score, width, height, - max_results) + max_results, (origin_x, origin_y)) return _nms(candidates, float(nms_iou))[:int(max_results)] @@ -267,7 +336,7 @@ def _load_unchanged(source: ImageSource): if hasattr(source, "shape"): return np.asarray(source), False if isinstance(source, (str, bytes)) or hasattr(source, "__fspath__"): - array = cv2.imread(str(source), cv2.IMREAD_UNCHANGED) + array = _imread(str(source), cv2.IMREAD_UNCHANGED) if array is None: raise ValueError(f"could not read image: {source!r}") return array, True diff --git a/je_auto_control/windows/keyboard/win32_ctype_keyboard_control.py b/je_auto_control/windows/keyboard/win32_ctype_keyboard_control.py index 8aac4b6a..c3372009 100644 --- a/je_auto_control/windows/keyboard/win32_ctype_keyboard_control.py +++ b/je_auto_control/windows/keyboard/win32_ctype_keyboard_control.py @@ -13,6 +13,7 @@ from je_auto_control.windows.core.utils.win32_ctype_input import SendInput from je_auto_control.windows.core.utils.win32_ctype_input import ctypes from je_auto_control.windows.core.utils.win32_vk import WIN32_EventF_KEYUP +from je_auto_control.windows.core.utils.win32_vk import WIN32_EventF_UNICODE def press_key(keycode: int) -> None: @@ -37,6 +38,48 @@ def release_key(keycode: int) -> None: SendInput(1, ctypes.byref(keyboard), ctypes.sizeof(keyboard)) +def press_unicode(code_unit: int) -> None: + """ + 送出一個 UTF-16 碼位的按下事件（與鍵盤配置無關） + Send a key-down carrying one UTF-16 code unit, independent of the layout + + ``KEYEVENTF_UNICODE`` makes ``wScan`` a character rather than a scan code, so + this reaches characters with no key on the current layout — punctuation the + virtual-key table omits, CJK, accented letters, emoji. ``KeyboardInput`` + leaves ``wScan`` untouched when the flag is set. + + :param code_unit: UTF-16 碼位 UTF-16 code unit (surrogates sent separately) + """ + keyboard = Input(type=Keyboard, + ki=KeyboardInput(wVk=0, wScan=code_unit, + dwFlags=WIN32_EventF_UNICODE)) + SendInput(1, ctypes.byref(keyboard), ctypes.sizeof(keyboard)) + + +def release_unicode(code_unit: int) -> None: + """ + 送出一個 UTF-16 碼位的放開事件 + Send the matching key-up for one UTF-16 code unit + + :param code_unit: UTF-16 碼位 UTF-16 code unit + """ + keyboard = Input(type=Keyboard, + ki=KeyboardInput(wVk=0, wScan=code_unit, + dwFlags=WIN32_EventF_UNICODE | WIN32_EventF_KEYUP)) + SendInput(1, ctypes.byref(keyboard), ctypes.sizeof(keyboard)) + + +def type_unicode_unit(code_unit: int) -> None: + """ + 送出一個 UTF-16 碼位（按下再放開） + Type one UTF-16 code unit (press then release) + + :param code_unit: UTF-16 碼位 UTF-16 code unit + """ + press_unicode(code_unit) + release_unicode(code_unit) + + def send_key_event_to_window(window: str, keycode: int) -> None: """ 將鍵盤事件送到指定視窗 diff --git a/je_auto_control/windows/listener/__init__.py b/je_auto_control/windows/listener/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/je_auto_control/windows/listener/win32_keyboard_listener.py b/je_auto_control/windows/listener/win32_keyboard_listener.py deleted file mode 100644 index 83107160..00000000 --- a/je_auto_control/windows/listener/win32_keyboard_listener.py +++ /dev/null @@ -1,118 +0,0 @@ -import sys -from ctypes import windll, WINFUNCTYPE, c_int, POINTER, c_void_p, byref -from ctypes.wintypes import MSG -from threading import Thread -from queue import Queue -from typing import Optional - -from je_auto_control.utils.exception.exception_tags import windows_import_error_message -from je_auto_control.utils.exception.exceptions import AutoControlException - -# 僅允許在 Windows 平台使用 Only allow on Windows platform -if sys.platform not in ["win32", "cygwin", "msys"]: - raise AutoControlException(windows_import_error_message) - -_user32 = windll.user32 -_kernel32 = windll.kernel32 -_wm_keydown: int = 0x100 - - -def _get_function_pointer(function) -> WINFUNCTYPE: - """ - 將 Python 函式轉換成 Win32 API 可用的函式指標 - Convert Python function to Win32-compatible function pointer - """ - win_function = WINFUNCTYPE(c_int, c_int, c_int, POINTER(c_void_p)) - return win_function(function) - - -class Win32KeyboardListener(Thread): - """ - Win32KeyboardListener - Windows 鍵盤事件監聽器 - - 使用 SetWindowsHookExA 設置鍵盤 hook - - 將鍵盤事件記錄到 Queue - """ - - def __init__(self): - super().__init__() - self.daemon = True - self.hooked: Optional[int] = None - self.record_queue: Optional[Queue] = None - self.record_flag: bool = False - self.hook_event_code_int: int = 13 # WH_KEYBOARD_LL - - def _set_win32_hook(self, point) -> bool: - """ - 設置鍵盤 hook - Set keyboard hook - """ - self.hooked = _user32.SetWindowsHookExA( - self.hook_event_code_int, - point, - 0, - 0 - ) - return bool(self.hooked) - - def _remove_win32_hook_proc(self) -> None: - """ - 移除鍵盤 hook - Remove keyboard hook - """ - if self.hooked: - _user32.UnhookWindowsHookEx(self.hooked) - self.hooked = None - - def _win32_hook_proc(self, code, w_param, l_param): - """ - 鍵盤事件處理函式 - Keyboard hook procedure - """ - if w_param != _wm_keydown: - return _user32.CallNextHookEx(self.hooked, code, w_param, l_param) - - if self.record_flag and self.record_queue is not None: - # 將 l_param 轉換成 keycode - temp = hex(l_param[0] & 0xFFFFFFFF) - self.record_queue.put(("AC_type_keyboard", int(temp, 16))) - - return _user32.CallNextHookEx(self.hooked, code, w_param, l_param) - - def _start_listener(self) -> None: - """ - 啟動鍵盤監聽 - Start keyboard listener - """ - self._pointer = _get_function_pointer(self._win32_hook_proc) - if not self._set_win32_hook(self._pointer): - raise AutoControlException("Failed to set keyboard hook") - - message = MSG() - # 進入訊息迴圈 Enter message loop - _user32.GetMessageA(byref(message), 0, 0, 0) # NOSONAR S5655 false positive — MSG is a ctypes Structure - - def record(self, want_to_record_queue: Queue) -> None: - """ - 開始紀錄鍵盤事件 - Start recording keyboard events - """ - self.record_flag = True - self.record_queue = want_to_record_queue - self.start() - - def stop_record(self) -> Queue: - """ - 停止紀錄並移除 hook - Stop recording and remove hook - """ - self.record_flag = False - self._remove_win32_hook_proc() - return self.record_queue - - def run(self) -> None: - """ - Thread 執行入口 - Thread run entry - """ - self._start_listener() \ No newline at end of file diff --git a/je_auto_control/windows/listener/win32_mouse_listener.py b/je_auto_control/windows/listener/win32_mouse_listener.py deleted file mode 100644 index 345bfa0d..00000000 --- a/je_auto_control/windows/listener/win32_mouse_listener.py +++ /dev/null @@ -1,127 +0,0 @@ -import sys -from ctypes import windll, WINFUNCTYPE, c_int, POINTER, c_void_p, byref -from ctypes.wintypes import MSG -from threading import Thread -from queue import Queue -from typing import Optional - -from je_auto_control.utils.exception.exception_tags import windows_import_error_message -from je_auto_control.utils.exception.exceptions import AutoControlException -from je_auto_control.windows.mouse.win32_ctype_mouse_control import position - -# 僅允許在 Windows 平台使用 Only allow on Windows platform -if sys.platform not in ["win32", "cygwin", "msys"]: - raise AutoControlException(windows_import_error_message) - -_user32 = windll.user32 -_kernel32 = windll.kernel32 - -# 滑鼠按鍵事件 Mouse button events -WM_LBUTTONDOWN = 0x0201 -WM_RBUTTONDOWN = 0x0204 -WM_MBUTTONDOWN = 0x0207 -_wm_mouse_key_code = [WM_LBUTTONDOWN, WM_RBUTTONDOWN, WM_MBUTTONDOWN] - - -def _get_function_pointer(function) -> WINFUNCTYPE: - """ - 將 Python 函式轉換成 Win32 API 可用的函式指標 - Convert Python function to Win32-compatible function pointer - """ - win_function = WINFUNCTYPE(c_int, c_int, c_int, POINTER(c_void_p)) - return win_function(function) - - -class Win32MouseListener(Thread): - """ - Win32MouseListener - Windows 滑鼠事件監聽器 - - 使用 SetWindowsHookExA 設置滑鼠 hook - - 將滑鼠事件記錄到 Queue - """ - - def __init__(self): - super().__init__() - self.daemon = True - self.hooked: Optional[int] = None - self.record_queue: Optional[Queue] = None - self.record_flag: bool = False - self.hook_event_code_int: int = 14 # WH_MOUSE_LL - - def _set_win32_hook(self, point) -> bool: - """ - 設置滑鼠 hook - Set mouse hook - """ - self.hooked = _user32.SetWindowsHookExA( - self.hook_event_code_int, - point, - 0, - 0 - ) - return bool(self.hooked) - - def _remove_win32_hook_proc(self) -> None: - """ - 移除滑鼠 hook - Remove mouse hook - """ - if self.hooked: - _user32.UnhookWindowsHookEx(self.hooked) - self.hooked = None - - def _win32_hook_proc(self, code, w_param, l_param): - """ - 滑鼠事件處理函式 - Mouse hook procedure - """ - if w_param not in _wm_mouse_key_code: - return _user32.CallNextHookEx(self.hooked, code, w_param, l_param) - - if self.record_flag and self.record_queue is not None: - x, y = position() - if w_param == WM_LBUTTONDOWN: - self.record_queue.put(("AC_mouse_left", x, y)) - elif w_param == WM_RBUTTONDOWN: - self.record_queue.put(("AC_mouse_right", x, y)) - elif w_param == WM_MBUTTONDOWN: - self.record_queue.put(("AC_mouse_middle", x, y)) - - return _user32.CallNextHookEx(self.hooked, code, w_param, l_param) - - def _start_listener(self) -> None: - """ - 啟動滑鼠監聽 - Start mouse listener - """ - self._pointer = _get_function_pointer(self._win32_hook_proc) - if not self._set_win32_hook(self._pointer): - raise AutoControlException("Failed to set mouse hook") - - message = MSG() - _user32.GetMessageA(byref(message), 0, 0, 0) # NOSONAR S5655 false positive — MSG is a ctypes Structure - - def record(self, want_to_record_queue: Queue) -> None: - """ - 開始紀錄滑鼠事件 - Start recording mouse events - """ - self.record_flag = True - self.record_queue = want_to_record_queue - self.start() - - def stop_record(self) -> Queue: - """ - 停止紀錄並移除 hook - Stop recording and remove hook - """ - self.record_flag = False - self._remove_win32_hook_proc() - return self.record_queue - - def run(self) -> None: - """ - Thread 執行入口 - Thread run entry - """ - self._start_listener() \ No newline at end of file diff --git a/je_auto_control/windows/record/win32_input_hook.py b/je_auto_control/windows/record/win32_input_hook.py new file mode 100644 index 00000000..44877253 --- /dev/null +++ b/je_auto_control/windows/record/win32_input_hook.py @@ -0,0 +1,223 @@ +"""Low-level Windows keyboard/mouse capture that a replay can actually reproduce. + +Recording only "what was pressed" is not enough to play a session back. Three +things decide whether the replay matches what the user did: + +* **Releases.** Without key-up / button-up, a drag is indistinguishable from a + click, and a modifier held across several actions cannot be reconstructed. +* **The wheel.** Scrolling is invisible to a listener that only watches buttons. +* **Timing.** With no timestamps every step replays at once, and real interfaces + never keep up — the window that was supposed to open has not opened yet. + +So the hook records press *and* release, wheel deltas, and a monotonic timestamp +per event. :func:`timeline` turns those into the ``delta_ms`` events that +:func:`je_auto_control.utils.input_macro.replay_timeline` already knows how to +play back. + +The hook must be installed on a thread that pumps messages, and the callback is +invoked on that same thread, so install / pump / uninstall all live in one +thread here. Stopping posts ``WM_QUIT`` to it — the thread cannot be left +blocked in ``GetMessage``, or every record cycle leaks one. + +**Everything typed while recording is captured, passwords included.** Callers +must treat the result as sensitive. +""" +import ctypes +import sys +import threading +import time +from ctypes import wintypes +from typing import Any, Dict, List, Optional + +from je_auto_control.utils.exception.exception_tags import ( + windows_import_error_message, +) +from je_auto_control.utils.exception.exceptions import ( + AutoControlException, AutoControlRecordException, +) +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +if sys.platform not in ["win32", "cygwin", "msys"]: + raise AutoControlException(windows_import_error_message) + +WH_KEYBOARD_LL = 13 +WH_MOUSE_LL = 14 +WM_QUIT = 0x0012 + +_KEY_DOWN = (0x0100, 0x0104) # WM_KEYDOWN, WM_SYSKEYDOWN +_KEY_UP = (0x0101, 0x0105) # WM_KEYUP, WM_SYSKEYUP +_WM_MOUSEWHEEL = 0x020A +_WHEEL_NOTCH = 120 # one detent, per Win32 + +_MOUSE_BUTTONS = { + 0x0201: ("mouse_down", "left"), 0x0202: ("mouse_up", "left"), + 0x0204: ("mouse_down", "right"), 0x0205: ("mouse_up", "right"), + 0x0207: ("mouse_down", "middle"), 0x0208: ("mouse_up", "middle"), +} + +# A hook that is left installed keeps growing this list forever. Recording is +# bounded so a forgotten session cannot consume the process. +MAX_EVENTS = 20000 + + +class _KBDLLHOOKSTRUCT(ctypes.Structure): + _fields_ = [("vkCode", wintypes.DWORD), ("scanCode", wintypes.DWORD), + ("flags", wintypes.DWORD), ("time", wintypes.DWORD), + ("extra", ctypes.POINTER(wintypes.ULONG))] + + +class _MSLLHOOKSTRUCT(ctypes.Structure): + _fields_ = [("pt", wintypes.POINT), ("mouseData", wintypes.DWORD), + ("flags", wintypes.DWORD), ("time", wintypes.DWORD), + ("extra", ctypes.POINTER(wintypes.ULONG))] + + +class Win32InputHook: + """Captures keyboard and mouse events, with releases, wheel and timing.""" + + def __init__(self, max_events: int = MAX_EVENTS) -> None: + self.events: List[Dict[str, Any]] = [] + self.started = time.monotonic() + self.error: Optional[str] = None + self.max_events = int(max_events) + self._thread_id = 0 + self._ready = threading.Event() + self._hooks: List[Any] = [] + # WINFUNCTYPE callbacks must stay referenced: once collected, the OS + # still calls that address and the process dies. + self._procs: List[Any] = [] + + # -- public ------------------------------------------------------------ + def start(self) -> None: + """Install the hooks and begin recording. Raises if they cannot be set.""" + threading.Thread(target=self._run, daemon=True).start() + self._ready.wait(timeout=5.0) + if self.error: + raise AutoControlRecordException(self.error) + + def stop(self) -> List[Dict[str, Any]]: + """Stop recording and return the raw events.""" + if self._thread_id: + try: + ctypes.windll.user32.PostThreadMessageW( + self._thread_id, WM_QUIT, 0, 0) + except OSError as error: + autocontrol_logger.error("recorder stop failed: %r", error) + return self.events + + # -- hook thread ------------------------------------------------------- + def _run(self) -> None: + user32 = ctypes.windll.user32 + try: + self._install(user32) + except OSError as error: + autocontrol_logger.error("recorder start failed: %r", error) + self.error = "could not install the keyboard/mouse hook" + self._unhook(user32) + self._ready.set() + return + self._ready.set() + try: + message = wintypes.MSG() + while user32.GetMessageW(ctypes.byref(message), None, 0, 0) > 0: + user32.TranslateMessage(ctypes.byref(message)) + user32.DispatchMessageW(ctypes.byref(message)) + finally: + self._unhook(user32) + + def _install(self, user32) -> None: + self._thread_id = ctypes.windll.kernel32.GetCurrentThreadId() + proto = ctypes.WINFUNCTYPE( + ctypes.c_ssize_t, ctypes.c_int, wintypes.WPARAM, wintypes.LPARAM) + # Declare the signatures: a hook handle is 64-bit and the default + # c_int return would truncate it. + user32.SetWindowsHookExW.argtypes = [ + ctypes.c_int, proto, wintypes.HINSTANCE, wintypes.DWORD] + user32.SetWindowsHookExW.restype = ctypes.c_void_p + user32.CallNextHookEx.argtypes = [ + ctypes.c_void_p, ctypes.c_int, wintypes.WPARAM, wintypes.LPARAM] + user32.CallNextHookEx.restype = ctypes.c_ssize_t + user32.UnhookWindowsHookEx.argtypes = [ctypes.c_void_p] + + self._procs = [proto(self._keyboard_proc(user32)), + proto(self._mouse_proc(user32))] + for hook_id, proc in zip((WH_KEYBOARD_LL, WH_MOUSE_LL), self._procs): + handle = user32.SetWindowsHookExW(hook_id, proc, None, 0) + if not handle: + raise OSError(f"SetWindowsHookExW failed for {hook_id}") + self._hooks.append(handle) + + def _unhook(self, user32) -> None: + for handle in self._hooks: + try: + user32.UnhookWindowsHookEx(handle) + except OSError as error: + autocontrol_logger.info("unhook failed: %r", error) + self._hooks = [] + + def _put(self, event: Dict[str, Any]) -> None: + """Record one event, stopping the hook once the cap is reached.""" + if len(self.events) >= self.max_events: + self.stop() + return + event["time"] = time.monotonic() + self.events.append(event) + + def _keyboard_proc(self, user32): + def _proc(code, w_param, l_param): + # Called by the OS on this thread: an exception must not escape, + # and it must not be slow — all desktop input queues behind it. + try: + if code >= 0: + data = ctypes.cast( + l_param, ctypes.POINTER(_KBDLLHOOKSTRUCT)).contents + if w_param in _KEY_DOWN: + self._put({"op": "key_down", "vk": int(data.vkCode)}) + elif w_param in _KEY_UP: + self._put({"op": "key_up", "vk": int(data.vkCode)}) + except (OSError, ValueError, AttributeError): + pass + return user32.CallNextHookEx(None, code, w_param, l_param) + + return _proc + + def _mouse_proc(self, user32): + def _proc(code, w_param, l_param): + try: + if code >= 0: + self._mouse_event(l_param, int(w_param)) + except (OSError, ValueError, AttributeError): + pass + return user32.CallNextHookEx(None, code, w_param, l_param) + + return _proc + + def _mouse_event(self, l_param, message: int) -> None: + data = ctypes.cast(l_param, ctypes.POINTER(_MSLLHOOKSTRUCT)).contents + button = _MOUSE_BUTTONS.get(message) + if button is not None: + self._put({"op": button[0], "button": button[1], + "x": int(data.pt.x), "y": int(data.pt.y)}) + elif message == _WM_MOUSEWHEEL: + # The high word of mouseData is a signed notch count times 120. + raw = ctypes.c_short((int(data.mouseData) >> 16) & 0xFFFF).value + self._put({"op": "scroll", "delta": raw // _WHEEL_NOTCH, + "x": int(data.pt.x), "y": int(data.pt.y)}) + + +def timeline(events: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Convert raw events to ``delta_ms`` form for ``replay_timeline``. + + The first event has no gap before it; every later one carries the real pause + that preceded it, which is what makes a replay track the original pace. + """ + out: List[Dict[str, Any]] = [] + previous: Optional[float] = None + for event in events: + moment = float(event.get("time", 0.0)) + item = {key: value for key, value in event.items() if key != "time"} + item["delta_ms"] = 0 if previous is None else max( + 0, int((moment - previous) * 1000)) + out.append(item) + previous = moment + return out diff --git a/je_auto_control/windows/record/win32_record.py b/je_auto_control/windows/record/win32_record.py index 989c51f5..5892e8c6 100644 --- a/je_auto_control/windows/record/win32_record.py +++ b/je_auto_control/windows/record/win32_record.py @@ -1,5 +1,5 @@ import sys -from typing import Optional +from typing import Any, Dict, List, Optional from queue import Queue from je_auto_control.utils.exception.exception_tags import windows_import_error_message @@ -8,8 +8,13 @@ if sys.platform not in ["win32", "cygwin", "msys"]: raise AutoControlException(windows_import_error_message) -from je_auto_control.windows.listener.win32_keyboard_listener import Win32KeyboardListener -from je_auto_control.windows.listener.win32_mouse_listener import Win32MouseListener +from je_auto_control.windows.record.win32_input_hook import ( + Win32InputHook, timeline, +) + +# Legacy queue entries: the down-event half, shaped as executor commands. +_LEGACY_MOUSE_COMMAND = {"left": "AC_mouse_left", "right": "AC_mouse_right", + "middle": "AC_mouse_middle"} class Win32Recorder: @@ -18,78 +23,103 @@ class Win32Recorder: Windows 錄製器 - 可同時錄製滑鼠與鍵盤事件 - 可選擇只錄製滑鼠或鍵盤 + + Capture runs through :class:`Win32InputHook`, which records releases, wheel + movement and timing as well as presses. ``stop_record`` still returns the + historical down-events-only queue so existing callers are unaffected; use + ``stop_record_timeline`` for everything needed to reproduce the session. """ def __init__(self): - self.mouse_record_listener: Optional[Win32MouseListener] = None - self.keyboard_record_listener: Optional[Win32KeyboardListener] = None + self.hook: Optional[Win32InputHook] = None self.record_queue: Optional[Queue] = None self.result_queue: Optional[Queue] = None + self._kinds: tuple = ("keyboard", "mouse") + + def _start(self, kinds: tuple) -> None: + self._kinds = kinds + self.hook = Win32InputHook() + self.record_queue = Queue() + self.hook.start() + + def _stop(self) -> List[Dict[str, Any]]: + if self.hook is None: + return [] + events = [event for event in self.hook.stop() + if self._wanted(event)] + self.hook = None + return events + + def _wanted(self, event: Dict[str, Any]) -> bool: + keyboard = event.get("op", "").startswith("key_") + return ("keyboard" if keyboard else "mouse") in self._kinds + + def _as_queue(self, events: List[Dict[str, Any]]) -> Queue: + """The historical shape: one executor command per press, no releases.""" + queue: Queue = Queue() + for event in events: + operation = event.get("op") + if operation == "key_down": + queue.put(("AC_type_keyboard", int(event.get("vk", 0)))) + elif operation == "mouse_down": + command = _LEGACY_MOUSE_COMMAND.get(event.get("button", "")) + if command: + queue.put((command, int(event.get("x", 0)), + int(event.get("y", 0)))) + self.record_queue = None + self.result_queue = queue + return queue def record(self) -> None: """ 開始錄製滑鼠與鍵盤事件 Start recording both mouse and keyboard events """ - self.mouse_record_listener = Win32MouseListener() - self.keyboard_record_listener = Win32KeyboardListener() - self.record_queue = Queue() - self.mouse_record_listener.record(self.record_queue) - self.keyboard_record_listener.record(self.record_queue) + self._start(("keyboard", "mouse")) def stop_record(self) -> Queue: """ 停止錄製並回傳事件 Stop recording and return recorded events """ - mouse_queue = self.mouse_record_listener.stop_record() if self.mouse_record_listener else Queue() - keyboard_queue = self.keyboard_record_listener.stop_record() if self.keyboard_record_listener else Queue() + return self._as_queue(self._stop()) - # 合併兩個 Queue 的內容 Merge both queues - self.result_queue = Queue() - while not mouse_queue.empty(): - self.result_queue.put(mouse_queue.get()) - while not keyboard_queue.empty(): - self.result_queue.put(keyboard_queue.get()) + def stop_record_timeline(self) -> List[Dict[str, Any]]: + """ + 停止錄製並回傳含放開、滾輪與間隔時間的完整事件 + Stop recording and return press *and* release, wheel and ``delta_ms`` - self.record_queue = None - return self.result_queue + Ready for :func:`je_auto_control.utils.input_macro.replay_timeline`. + """ + return timeline(self._stop()) def record_mouse(self) -> None: """ 開始錄製滑鼠事件 Start recording mouse events """ - self.mouse_record_listener = Win32MouseListener() - self.record_queue = Queue() - self.mouse_record_listener.record(self.record_queue) + self._start(("mouse",)) def stop_record_mouse(self) -> Queue: """ 停止錄製滑鼠事件並回傳結果 Stop recording mouse events and return results """ - self.result_queue = self.mouse_record_listener.stop_record() if self.mouse_record_listener else Queue() - self.record_queue = None - return self.result_queue + return self._as_queue(self._stop()) def record_keyboard(self) -> None: """ 開始錄製鍵盤事件 Start recording keyboard events """ - self.keyboard_record_listener = Win32KeyboardListener() - self.record_queue = Queue() - self.keyboard_record_listener.record(self.record_queue) + self._start(("keyboard",)) def stop_record_keyboard(self) -> Queue: """ 停止錄製鍵盤事件並回傳結果 Stop recording keyboard events and return results """ - self.result_queue = self.keyboard_record_listener.stop_record() if self.keyboard_record_listener else Queue() - self.record_queue = None - return self.result_queue + return self._as_queue(self._stop()) # 全域錄製器實例 Global recorder instance diff --git a/je_auto_control/windows/window/windows_window_manage.py b/je_auto_control/windows/window/windows_window_manage.py index 80fb1aad..d922fdb2 100644 --- a/je_auto_control/windows/window/windows_window_manage.py +++ b/je_auto_control/windows/window/windows_window_manage.py @@ -1,62 +1,165 @@ -from ctypes import WINFUNCTYPE, c_bool, c_int, POINTER, create_unicode_buffer -from typing import List, Tuple, Optional - -from je_auto_control.windows.core.utils.win32_ctype_input import user32 - -# Win32 API 函式指標 Win32 API function pointers -EnumWindows = user32.EnumWindows -EnumWindowsProc = WINFUNCTYPE(c_bool, POINTER(c_int), POINTER(c_int)) -GetWindowText = user32.GetWindowTextW -GetWindowTextLength = user32.GetWindowTextLengthW -IsWindowVisible = user32.IsWindowVisible -FindWindowW = user32.FindWindowW -CloseWindow = user32.CloseWindow -DestroyWindow = user32.DestroyWindow +""" +Windows 視窗管理（Win32 ctypes） +Windows window management (Win32 ctypes) +""" +import ctypes +from ctypes import WINFUNCTYPE, byref, create_unicode_buffer, wintypes +from typing import List, Optional, Tuple + +# 相容用途：舊版本從這個模組匯出共用的 user32。 +# Compatibility: older code imported the shared user32 from this module. +from je_auto_control.windows.core.utils.win32_ctype_input import user32 # noqa: F401 + +# 這個模組刻意持有自己的 user32 handle，而不是共用上面那個：底下每個函式都要 +# 設 argtypes/restype，而那是設在**函式物件**上的，共用同一個 handle 會讓設定 +# 外溢到別的模組。例如 `utils/window_capture/` 用它自己的 RECT 呼叫 +# `GetWindowRect`，若被這裡的 argtypes 綁死就會直接壞掉。 +# +# This module deliberately owns its own user32 handle: argtypes/restype live on +# the function objects, so sharing one would leak these prototypes into other +# callers (`utils/window_capture/` passes its own RECT to GetWindowRect). +_user32 = ctypes.WinDLL("user32", use_last_error=True) + +# HWND 是指標寬度的 handle。ctypes 預設把參數與回傳值當成 c_int，在 64 位元 +# Windows 上會截斷成 32 位元——與 `OpenProcess` 那個經典陷阱同一類。每個函式 +# 都明寫 argtypes/restype，不要依賴預設值。 +# +# An HWND is a pointer-width handle. ctypes defaults to c_int, which truncates +# it to 32 bits on 64-bit Windows. Every prototype below is explicit. +EnumWindowsProc = WINFUNCTYPE(wintypes.BOOL, wintypes.HWND, wintypes.LPARAM) + +_user32.EnumWindows.argtypes = [EnumWindowsProc, wintypes.LPARAM] +_user32.EnumWindows.restype = wintypes.BOOL +_user32.GetWindowTextW.argtypes = [wintypes.HWND, wintypes.LPWSTR, ctypes.c_int] +_user32.GetWindowTextW.restype = ctypes.c_int +_user32.GetWindowTextLengthW.argtypes = [wintypes.HWND] +_user32.GetWindowTextLengthW.restype = ctypes.c_int +_user32.IsWindowVisible.argtypes = [wintypes.HWND] +_user32.IsWindowVisible.restype = wintypes.BOOL +_user32.FindWindowW.argtypes = [wintypes.LPCWSTR, wintypes.LPCWSTR] +_user32.FindWindowW.restype = wintypes.HWND +_user32.CloseWindow.argtypes = [wintypes.HWND] +_user32.CloseWindow.restype = wintypes.BOOL +_user32.DestroyWindow.argtypes = [wintypes.HWND] +_user32.DestroyWindow.restype = wintypes.BOOL +_user32.PostMessageW.argtypes = [wintypes.HWND, wintypes.UINT, + wintypes.WPARAM, wintypes.LPARAM] +_user32.PostMessageW.restype = wintypes.BOOL +_user32.SetForegroundWindow.argtypes = [wintypes.HWND] +_user32.SetForegroundWindow.restype = wintypes.BOOL +_user32.GetForegroundWindow.argtypes = [] +_user32.GetForegroundWindow.restype = wintypes.HWND +_user32.GetWindowRect.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.RECT)] +_user32.GetWindowRect.restype = wintypes.BOOL +_user32.SetWindowPos.argtypes = [wintypes.HWND, wintypes.HWND, ctypes.c_int, + ctypes.c_int, ctypes.c_int, ctypes.c_int, + wintypes.UINT] +_user32.SetWindowPos.restype = wintypes.BOOL +_user32.ShowWindow.argtypes = [wintypes.HWND, ctypes.c_int] +_user32.ShowWindow.restype = wintypes.BOOL +_user32.MoveWindow.argtypes = [wintypes.HWND, ctypes.c_int, ctypes.c_int, + ctypes.c_int, ctypes.c_int, wintypes.BOOL] +_user32.MoveWindow.restype = wintypes.BOOL +_user32.IsIconic.argtypes = [wintypes.HWND] +_user32.IsIconic.restype = wintypes.BOOL + +WM_CLOSE = 0x0010 +SW_RESTORE = 9 def get_all_window_hwnd() -> List[Tuple[int, str]]: """ - 列舉所有可見視窗 - Enumerate all visible windows + 列舉所有可見視窗，依 z-order（最前面的在最前） + Enumerate all visible windows, front-most first + + hwnd 一律回傳 **int**。先前的回呼把 hwnd 宣告成 `POINTER(c_int)`，於是每個 + handle 都成了 `LP_c_long` 物件：`int(hwnd)` 會丟 `ValueError`，也沒辦法拿去 + 跟其他 Win32 呼叫組合，等於這份清單只能看不能用。 :return: [(hwnd, window_title), ...] """ window_info: List[Tuple[int, str]] = [] - def _foreach_window(hwnd: int, l_param: int) -> bool: - if IsWindowVisible(hwnd): - length = GetWindowTextLength(hwnd) + def _foreach_window(hwnd, _l_param) -> bool: + if _user32.IsWindowVisible(hwnd): + length = _user32.GetWindowTextLengthW(hwnd) buff = create_unicode_buffer(length + 1) - GetWindowText(hwnd, buff, length + 1) - window_info.append((hwnd, buff.value)) + _user32.GetWindowTextW(hwnd, buff, length + 1) + window_info.append((int(hwnd), buff.value)) return True - EnumWindows(EnumWindowsProc(_foreach_window), 0) + _user32.EnumWindows(EnumWindowsProc(_foreach_window), 0) return window_info -def get_one_window_hwnd(window_class: Optional[str], window_name: Optional[str]) -> int: +def get_one_window_hwnd(window_class: Optional[str], + window_name: Optional[str]) -> int: """ 取得指定視窗的 HWND Get window handle by class name and/or window title """ - return FindWindowW(window_class, window_name) + return int(_user32.FindWindowW(window_class, window_name) or 0) + + +def get_foreground_window() -> int: + """ + 目前的前景視窗 HWND，沒有就回 0 + The foreground window's HWND, or 0 when there is none + """ + return int(_user32.GetForegroundWindow() or 0) + + +def get_window_rect(hwnd: int) -> Optional[Tuple[int, int, int, int]]: + """ + 視窗在螢幕座標上的矩形，查不到回 None + The window's screen-coordinate rectangle, or None if unavailable + + :return: (left, top, right, bottom) + """ + rect = wintypes.RECT() + if not _user32.GetWindowRect(hwnd, byref(rect)): + return None + return (int(rect.left), int(rect.top), int(rect.right), int(rect.bottom)) + + +def is_window_minimized(hwnd: int) -> bool: + """ + 視窗目前是否被最小化 + Whether the window is currently minimized + """ + return bool(_user32.IsIconic(hwnd)) def close_window(hwnd: int) -> bool: """ - 嘗試關閉視窗 (最小化) - Attempt to close (minimize) a window + 請視窗關閉（送出 WM_CLOSE） + Ask a window to close (posts WM_CLOSE) + + **行為變更**：本函式過去呼叫 Win32 的 `CloseWindow()`，而那支 API 其實是把 + 視窗**最小化**，不是關閉——這是 Win32 有名的命名陷阱。舊行為已改名為 + :func:`minimize_window`。要關掉別的行程的視窗只能送 WM_CLOSE， + `DestroyWindow` 無法銷毀不屬於本執行緒的視窗。 + """ + return bool(_user32.PostMessageW(hwnd, WM_CLOSE, 0, 0)) + + +def minimize_window(hwnd: int) -> bool: + """ + 最小化視窗 + Minimize a window + + 包的是 Win32 `CloseWindow()`——名字寫 close，做的是 minimize。 + Wraps Win32 `CloseWindow()`, which minimizes despite its name. """ - return bool(CloseWindow(hwnd)) + return bool(_user32.CloseWindow(hwnd)) def destroy_window(hwnd: int) -> bool: """ - 銷毀視窗 - Destroy a window + 銷毀視窗（只對本執行緒自己的視窗有效） + Destroy a window (only works on windows owned by this thread) """ - return bool(DestroyWindow(hwnd)) + return bool(_user32.DestroyWindow(hwnd)) def set_foreground_window(hwnd: int) -> None: @@ -64,7 +167,7 @@ def set_foreground_window(hwnd: int) -> None: 設定視窗為前景視窗 Set window to foreground """ - user32.SetForegroundWindow(hwnd) + _user32.SetForegroundWindow(hwnd) def set_window_position(hwnd: int, position: int) -> None: @@ -72,9 +175,10 @@ def set_window_position(hwnd: int, position: int) -> None: 設定視窗位置 (僅改變 Z-order，不改變大小與座標) Set window position (only Z-order, no resize or move) """ - SWP_NO_SIZE = 0x0001 - SWP_NO_MOVE = 0x0002 - user32.SetWindowPos(hwnd, position, 0, 0, 0, 0, SWP_NO_MOVE | SWP_NO_SIZE) + swp_no_size = 0x0001 + swp_no_move = 0x0002 + _user32.SetWindowPos(hwnd, position, 0, 0, 0, 0, + swp_no_move | swp_no_size) def show_window(hwnd: int, cmd_show: int) -> None: @@ -86,8 +190,11 @@ def show_window(hwnd: int, cmd_show: int) -> None: """ if cmd_show < 0 or cmd_show > 11: # Win32 ShowWindow 常見範圍 cmd_show = 1 # 預設為 Normal - user32.ShowWindow(hwnd, cmd_show) - user32.SetForegroundWindow(hwnd) + _user32.ShowWindow(hwnd, cmd_show) + # 隱藏之後不該再把它拉到前景，那是自相矛盾的一組動作。 + # Do not pull a window forward right after hiding it. + if cmd_show != 0: + _user32.SetForegroundWindow(hwnd) def move_window(hwnd: int, x: int, y: int, width: int, height: int, @@ -96,6 +203,6 @@ def move_window(hwnd: int, x: int, y: int, width: int, height: int, 搬移與調整視窗大小 (一次設定座標與寬高) Move and resize a window in one call. """ - return bool(user32.MoveWindow(int(hwnd), int(x), int(y), + return bool(_user32.MoveWindow(int(hwnd), int(x), int(y), int(width), int(height), - c_bool(bool(repaint)))) \ No newline at end of file + bool(repaint))) diff --git a/je_auto_control/wrapper/auto_control_keyboard.py b/je_auto_control/wrapper/auto_control_keyboard.py index 89b11cd5..859cc646 100644 --- a/je_auto_control/wrapper/auto_control_keyboard.py +++ b/je_auto_control/wrapper/auto_control_keyboard.py @@ -10,6 +10,7 @@ ) from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.test_record.record_test_class import record_action_to_list +from je_auto_control.utils.text_unicode.text_unicode import unicode_code_units from je_auto_control.wrapper.platform_wrapper import keyboard, keyboard_keys_table, keyboard_check def get_keyboard_keys_table() -> dict: @@ -128,11 +129,40 @@ def check_key_is_press(keycode: Union[int, str]) -> Optional[bool]: return None +# Whitespace that means a *key*, not a character. Sent as a Unicode code point +# these are silently dropped by most applications — a newline especially, which +# turns a multi-line `write` into one run-on line with nothing reported. +WRITE_CONTROL_KEYS = {"\n": "return", "\r": "return", "\t": "tab", + "\b": "back"} + + +def _write_char_via_unicode(single_char: str) -> bool: + """ + 以 Unicode 事件輸入單一字元 (鍵盤對應表沒有的字元) + Type one character the virtual-key table has no entry for + + :param single_char: 單一字元 One character + :return: 是否成功送出 Whether the backend could send it + """ + type_unicode_unit = getattr(keyboard, "type_unicode_unit", None) + if not callable(type_unicode_unit): + return False + for unit in unicode_code_units(single_char): + type_unicode_unit(unit) + return True + + def write(write_string: str, is_shift: bool = False) -> Optional[str]: """ 模擬輸入整個字串 Type a whole string + The virtual-key table covers barely 192 keys, so a literal reading of it + cannot type ``, . / : ? ! _ + @ %`` on a US layout, nor any CJK or accented + character. Characters it lacks fall back to Unicode key events where the + backend supports them, and only raise where it does not — otherwise a single + comma fails the whole string. + :param write_string: 要輸入的字串 String to type :param is_shift: 是否同時按下 Shift :return: 輸入的字串 @@ -142,15 +172,21 @@ def write(write_string: str, is_shift: bool = False) -> Optional[str]: record_write_chars = [] for single_char in write_string: key = keyboard_keys_table.get(single_char) - if key is not None: + control_key = WRITE_CONTROL_KEYS.get(single_char) + if control_key is not None and control_key in keyboard_keys_table: + # Before the table lookup: a newline must press Enter, not type + # U+000A and not fall through to the space fallback below. + type_keyboard(control_key, is_shift, skip_record=True) + elif key is not None: type_keyboard(key, is_shift, skip_record=True) - record_write_chars.append(single_char) + elif _write_char_via_unicode(single_char): + pass elif single_char.isspace(): type_keyboard("space", is_shift, skip_record=True) - record_write_chars.append(single_char) else: autocontrol_logger.error(f"write failed: {keyboard_write_cant_find_error_message}, char={single_char}") raise AutoControlKeyboardException(keyboard_write_cant_find_error_message) + record_write_chars.append(single_char) result = "".join(record_write_chars) record_action_to_list("write", {"write_string": write_string, "is_shift": is_shift}) diff --git a/je_auto_control/wrapper/auto_control_record.py b/je_auto_control/wrapper/auto_control_record.py index bdb40602..d161d3b7 100644 --- a/je_auto_control/wrapper/auto_control_record.py +++ b/je_auto_control/wrapper/auto_control_record.py @@ -52,6 +52,37 @@ def stop_record() -> list: autocontrol_logger.error(f"stop_record, failed: {repr(error)}") +def stop_record_timeline() -> list: + """ + 停止錄製並回傳含放開、滾輪與間隔時間的完整事件 + Stop recording and return press *and* release, wheel, and ``delta_ms`` + + :func:`stop_record` reports only what was pressed, which is not enough to + reproduce a session: a drag looks like a click, scrolling is missing, and + every step replays at once. These events feed + :func:`je_auto_control.utils.input_macro.replay_timeline` directly. + + Returns an empty list on a platform whose recorder cannot supply it. + """ + autocontrol_logger.info("stop_record_timeline") + try: + if sys.platform == "darwin": + raise AutoControlException(macos_record_error_message) + collect = getattr(recorder, "stop_record_timeline", None) + if collect is None: + autocontrol_logger.error( + "stop_record_timeline: this recorder has no timeline support") + return [] + events = collect() + record_action_to_list("stop_record_timeline", None) + return list(events) + except (OSError, RuntimeError, AttributeError, TypeError, ValueError, + AutoControlException) as error: + record_action_to_list("stop_record_timeline", None, repr(error)) + autocontrol_logger.error(f"stop_record_timeline, failed: {repr(error)}") + return [] + + def record_to_json(output_path: str, *, stop_event: threading.Event, timeout: Optional[float] = None) -> list: """ diff --git a/je_auto_control/wrapper/auto_control_window.py b/je_auto_control/wrapper/auto_control_window.py index dcd4fcb2..9469f727 100644 --- a/je_auto_control/wrapper/auto_control_window.py +++ b/je_auto_control/wrapper/auto_control_window.py @@ -20,11 +20,20 @@ def _require_windows() -> None: ) -def list_windows() -> List[Tuple[int, str]]: - """Return a list of ``(hwnd, title)`` for every visible top-level window.""" +def list_windows(titled_only: bool = False) -> List[Tuple[int, str]]: + """Return ``(hwnd, title)`` for every visible top-level window, front-most + first. + + ``hwnd`` is a plain ``int``, so it composes with any other Win32 call. + Most visible windows have no title (shell and helper surfaces); pass + ``titled_only`` for just the ones a user would recognise. + """ _require_windows() from je_auto_control.windows.window import windows_window_manage as wm - return wm.get_all_window_hwnd() + found = wm.get_all_window_hwnd() + if titled_only: + return [(hwnd, title) for hwnd, title in found if title.strip()] + return found def find_window(title_substring: str, @@ -48,6 +57,11 @@ def focus_window(title_substring: str, case_sensitive: bool = False) -> int: ) hwnd, title = hit from je_auto_control.windows.window import windows_window_manage as wm + # A minimized window stays invisible however often you foreground it, so + # restore it first — but only when it really is minimized: SW_RESTORE on a + # maximized window un-maximizes it, which is not what "focus" should do. + if wm.is_window_minimized(hwnd): + wm.show_window(hwnd, wm.SW_RESTORE) wm.set_foreground_window(hwnd) autocontrol_logger.info("focused window hwnd=%s title=%r", hwnd, title) return hwnd @@ -72,7 +86,12 @@ def wait_for_window(title_substring: str, def close_window_by_title(title_substring: str, case_sensitive: bool = False) -> bool: - """Minimise the first matching window.""" + """Ask the first matching window to close. ``False`` if nothing matched. + + **Behaviour change**: this used to *minimise* the window, because the Win32 + call underneath is named ``CloseWindow`` but minimises. Use + :func:`minimize_window_by_title` for the old behaviour. + """ _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: @@ -81,6 +100,68 @@ def close_window_by_title(title_substring: str, case_sensitive: bool = False) -> return wm.close_window(hit[0]) +def minimize_window_by_title(title_substring: str, + case_sensitive: bool = False) -> bool: + """Minimise the first matching window. ``False`` if nothing matched.""" + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + return False + from je_auto_control.windows.window import windows_window_manage as wm + return wm.minimize_window(hit[0]) + + +def foreground_window() -> Optional[Tuple[int, str]]: + """The window the user is currently working in, or ``None``.""" + _require_windows() + from je_auto_control.windows.window import windows_window_manage as wm + hwnd = wm.get_foreground_window() + if not hwnd: + return None + titles = dict(wm.get_all_window_hwnd()) + return hwnd, titles.get(hwnd, "") + + +def window_rect(title_substring: str, + case_sensitive: bool = False, + ) -> Optional[Tuple[int, int, int, int]]: + """``(left, top, right, bottom)`` of the first matching window. + + Screen coordinates, so on a multi-monitor desktop the values can be + negative for a monitor left of or above the primary one. + """ + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + return None + from je_auto_control.windows.window import windows_window_manage as wm + return wm.get_window_rect(hit[0]) + + +def move_window_by_title(title_substring: str, x: int, y: int, + width: Optional[int] = None, + height: Optional[int] = None, + case_sensitive: bool = False) -> bool: + """Move (and optionally resize) the first matching window. + + Omitting ``width`` / ``height`` keeps the window's current size, so a plain + reposition does not have to restate dimensions the caller has to look up. + """ + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + return False + from je_auto_control.windows.window import windows_window_manage as wm + if width is None or height is None: + rect = wm.get_window_rect(hit[0]) + if rect is None: + return False + left, top, right, bottom = rect + width = right - left if width is None else width + height = bottom - top if height is None else height + return wm.move_window(hit[0], int(x), int(y), int(width), int(height)) + + def show_window_by_title(title_substring: str, cmd_show: int = 1, case_sensitive: bool = False) -> bool: """Show or restore a window (``cmd_show`` follows Win32 ShowWindow).""" diff --git a/test/unit_test/headless/conftest.py b/test/unit_test/headless/conftest.py new file mode 100644 index 00000000..998928e1 --- /dev/null +++ b/test/unit_test/headless/conftest.py @@ -0,0 +1,46 @@ +"""Shared teardown for the headless suite.""" +import sys + +import pytest + + +def _live_qt_application(): + """The running QApplication, or None. + + Looks in ``sys.modules`` rather than importing PySide6: most of this suite + is Qt-free, and importing Qt to ask whether Qt is in use would load it into + every one of those runs. + """ + widgets = sys.modules.get("PySide6.QtWidgets") + if widgets is None: + return None + return widgets.QApplication.instance() + + +@pytest.fixture(autouse=True) +def flush_qt_deferred_deletes(): + """Run Qt's queued ``deleteLater()`` work at the end of every test. + + ``deleteLater()`` does nothing until an event loop runs, and almost no GUI + test module here runs one. Left queued, a widget — and any helper thread or + timer it started at construction — survives until some *later* test pumps + events, and is then destroyed inside that unrelated test. + + That is not hypothetical: seven ``AdminConsoleTab``s queued by + ``test_admin_console_thumbnails_gui.py`` were destroyed inside the nested + modal ``exec()`` of ``test_usb_acl_prompt.py``, killing the interpreter with + rc 3221226505 (0xC0000409, a ``__fastfail``). There was no traceback, + faulthandler could not see it, and the ~500 tests after it never ran. + + Flushing here makes each test clean up after itself, so no test module has + to remember. Being autouse, this is set up before any test-local fixture + and therefore torn down *after* it — the module's own ``deleteLater()`` + calls have already been made by the time this runs. + """ + yield + app = _live_qt_application() + if app is None: + return + from PySide6.QtCore import QCoreApplication, QEvent + QCoreApplication.sendPostedEvents(None, QEvent.Type.DeferredDelete) + app.processEvents() diff --git a/test/unit_test/headless/test_accessibility.py b/test/unit_test/headless/test_accessibility.py index a8728ecb..c8b7c856 100644 --- a/test/unit_test/headless/test_accessibility.py +++ b/test/unit_test/headless/test_accessibility.py @@ -28,8 +28,10 @@ def __init__(self, elements: List[AccessibilityElement]) -> None: def list_elements(self, app_name: Optional[str] = None, max_results: int = 200, + window_title: Optional[str] = None, ) -> List[AccessibilityElement]: - self.last_args = {"app_name": app_name, "max_results": max_results} + self.last_args = {"app_name": app_name, "max_results": max_results, + "window_title": window_title} if app_name is None: return list(self._elements) return [e for e in self._elements if e.app_name == app_name] @@ -117,7 +119,7 @@ def test_list_elements_passes_filters_through(fake_backend): result = list_accessibility_elements(app_name="Calculator", max_results=50) assert fake_backend.last_args == { - "app_name": "Calculator", "max_results": 50, + "app_name": "Calculator", "max_results": 50, "window_title": None, } assert len(result) == 2 assert all(e.app_name == "Calculator" for e in result) @@ -137,12 +139,101 @@ def test_find_element_returns_none_when_no_match(fake_backend): ) is None +# --- substring matching and ranking ---------------------------------------- + +def test_exact_name_match_is_still_the_default(fake_backend): + # Callers holding a full name must keep getting exactly that element. + assert find_accessibility_element(name="OK") is not None + assert find_accessibility_element(name="O") is None + + +def test_contains_finds_names_with_accelerators_and_padding(): + # Real labels carry accelerator markers and trailing spaces, so an exact + # comparison misses a large share of genuine targets. + padded = AccessibilityElement(name="Save(&S) ", role="Button", + bounds=(0, 0, 10, 10), app_name="Editor") + assert element_matches(padded, name="save", contains=True) + assert not element_matches(padded, name="save") + + +def test_contains_ranks_an_exact_name_first(): + from je_auto_control.utils.accessibility.element import rank_by_name + elements = [ + AccessibilityElement(name="OK and close", role="Button", + bounds=(0, 0, 10, 10)), + AccessibilityElement(name="OK", role="Button", bounds=(0, 50, 10, 10)), + ] + # ...even though the exact one is lower down the screen + assert [e.name for e in rank_by_name(elements, "OK")] == [ + "OK", "OK and close"] + + +def test_role_filter_accepts_the_friendly_name(): + # The Windows backend reports the raw UIA role on purpose, but nobody + # filtering a search types "ControlType_50000" — and a role that never + # matches reads as "element not found". + raw = AccessibilityElement(name="OK", role="ControlType_50000", + bounds=(0, 0, 10, 10)) + assert element_matches(raw, role="button") + assert element_matches(raw, role="ControlType_50000") + assert not element_matches(raw, role="edit") + # a backend that already reports friendly roles keeps working + friendly = AccessibilityElement(name="OK", role="Button", + bounds=(0, 0, 10, 10)) + assert element_matches(friendly, role="button") + assert element_matches(friendly, role="ControlType_50000") + + +def test_find_all_returns_every_hit_best_first(fake_backend): + from je_auto_control.utils.accessibility.accessibility_api import ( + find_accessibility_elements, + ) + found = find_accessibility_elements(name="o", contains=True) + assert [e.name for e in found] == sorted( + (e.name for e in found), key=lambda n: n.strip().lower() != "o") + + +def test_find_caps_matches_and_scan_separately(fake_backend): + # One number cannot mean both: using the match cap as the scan cap turns + # "up to 2 buttons" into "only look at the first 2 elements". + from je_auto_control.utils.accessibility.accessibility_api import ( + find_accessibility_elements, + ) + found = find_accessibility_elements(name="o", contains=True, max_results=1, + scan_limit=500) + assert len(found) <= 1 + assert fake_backend.last_args["max_results"] == 500 # the scan, not the cap + + +def test_window_title_is_forwarded_to_the_backend(fake_backend): + list_accessibility_elements(window_title="Untitled - Notepad") + assert fake_backend.last_args["window_title"] == "Untitled - Notepad" + + +def test_window_title_is_omitted_when_not_requested(fake_backend): + # Left out entirely so a backend written before scoping existed keeps + # working for every call that does not ask for it. + list_accessibility_elements() + assert fake_backend.last_args["window_title"] is None + + def test_executor_registers_a11y_commands(): from je_auto_control.utils.executor.action_executor import executor commands = executor.known_commands() assert "AC_a11y_list" in commands assert "AC_a11y_find" in commands + assert "AC_a11y_find_all" in commands assert "AC_a11y_click" in commands + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_a11y_find", "ac_a11y_find_all"} <= names + + +def test_executor_flag_coercion_treats_string_false_as_false(): + # A JSON action file / CLI / MCP call can deliver "false", which is truthy. + from je_auto_control.utils.executor.action_executor import _as_bool + assert _as_bool("false") is False and _as_bool("0") is False + assert _as_bool("true") is True and _as_bool(True) is True def test_package_facade_exports_accessibility_api(): diff --git a/test/unit_test/headless/test_actions_menu_gui.py b/test/unit_test/headless/test_actions_menu_gui.py index ae705fdf..92013fd3 100644 --- a/test/unit_test/headless/test_actions_menu_gui.py +++ b/test/unit_test/headless/test_actions_menu_gui.py @@ -9,6 +9,8 @@ """ import json import os +import pathlib +import re import subprocess import sys @@ -48,6 +50,9 @@ def entry_actions(entry): "record_menu_matches": False, "variables_menu_matches": False, "variables_has_actions": False, + # Reported from here because this probe already pays for the Qt startup + # the count needs; a second subprocess just to count tabs is not worth it. + "tab_count": len(widget._tab_entries), } for entry in widget._tab_entries: @@ -121,3 +126,30 @@ def test_hook_tab_actions_reach_the_menu(report): assert report["variables_has_actions"], ( "hook-based tab should surface its actions" ) + + +# The other documented counts are guarded by test_doc_counts.py. The tab count +# lives here instead because counting tabs means constructing the widget, which +# means Qt — and this module already runs it in a subprocess for that reason. +TAB_CITATIONS = ( + ("README.md", r"\((\d+) tabs\)"), + ("README/README_zh-CN.md", r"（(\d+) 个标签页）"), + ("README/README_zh-TW.md", r"（(\d+) 個分頁）"), +) + + +@pytest.mark.parametrize("doc,pattern", TAB_CITATIONS) +def test_documented_tab_count_matches_the_widget(report, doc, pattern): + root = pathlib.Path(__file__).resolve().parents[3] + text = (root / doc).read_text(encoding="utf-8") + found = re.findall(pattern, text) + assert found, ( + f"{doc}: no longer states the tab count in the expected form " + f"({pattern!r}). If the wording changed on purpose, update this " + f"pattern; the count itself must stay in the document." + ) + for quoted in found: + assert int(quoted) == report["tab_count"], ( + f"{doc} says {quoted} GUI tabs, the widget builds " + f"{report['tab_count']}. Update all three READMEs." + ) diff --git a/test/unit_test/headless/test_admin_console_thumbnails_gui.py b/test/unit_test/headless/test_admin_console_thumbnails_gui.py index cc24d3c1..7985e615 100644 --- a/test/unit_test/headless/test_admin_console_thumbnails_gui.py +++ b/test/unit_test/headless/test_admin_console_thumbnails_gui.py @@ -38,6 +38,8 @@ def populated_admin_tab(qapp, tmp_path, monkeypatch): from je_auto_control.gui.admin_console_tab import AdminConsoleTab tab = AdminConsoleTab() yield tab, client + # The queued deletion is flushed by the autouse fixture in conftest.py — + # see there for why leaving it queued once killed the interpreter. tab.deleteLater() diff --git a/test/unit_test/headless/test_clipboard_image.py b/test/unit_test/headless/test_clipboard_image.py new file mode 100644 index 00000000..ccb586fe --- /dev/null +++ b/test/unit_test/headless/test_clipboard_image.py @@ -0,0 +1,102 @@ +"""Headless tests for the merged clipboard image API. No Qt. + +``set_clipboard_image`` used to exist twice under the same name in this +package — ``clipboard.py`` took PNG bytes, ``clipboard_image.py`` took a path — +so importing the wrong module failed only at runtime, and only for whichever +argument type the caller happened to pass. There is one function now, and it +accepts both. +""" +import io +import sys + +import pytest + +from je_auto_control.utils.clipboard import clipboard as cb + +pytest.importorskip("PIL", exc_type=ImportError) + + +def _png(size=(6, 4), colour=(10, 20, 30)) -> bytes: + from PIL import Image + buffer = io.BytesIO() + Image.new("RGB", size, colour).save(buffer, format="PNG") + return buffer.getvalue() + + +# --- the conversion, tested without touching the real clipboard ------------ + +def test_bytes_pass_straight_through(): + payload = _png() + assert cb._as_png_bytes(payload) == payload + assert cb._as_png_bytes(bytearray(payload)) == payload + + +def test_a_path_is_read_and_re_encoded_as_png(tmp_path): + from PIL import Image + source = tmp_path / "shot.bmp" # deliberately not a PNG + Image.new("RGB", (9, 7), (1, 2, 3)).save(source) + out = cb._as_png_bytes(str(source)) + assert out.startswith(b"\x89PNG\r\n\x1a\n") + assert Image.open(io.BytesIO(out)).size == (9, 7) + + +def test_path_objects_work_too(tmp_path): + from PIL import Image + source = tmp_path / "shot.png" + Image.new("RGB", (5, 5), (9, 9, 9)).save(source) + assert cb._as_png_bytes(source).startswith(b"\x89PNG") + + +def test_empty_bytes_are_rejected(): + with pytest.raises(ValueError): + cb._as_png_bytes(b"") + + +def test_a_missing_file_is_rejected_by_name(tmp_path): + with pytest.raises(FileNotFoundError): + cb._as_png_bytes(str(tmp_path / "absent.png")) + + +def test_a_directory_is_not_mistaken_for_an_image(tmp_path): + """`realpath` + `isfile`, so a directory fails here rather than in Pillow.""" + with pytest.raises(FileNotFoundError): + cb._as_png_bytes(str(tmp_path)) + + +@pytest.mark.parametrize("bad", [123, None, 4.5, ["a"]]) +def test_other_types_are_rejected(bad): + with pytest.raises(TypeError): + cb._as_png_bytes(bad) + + +# --- the real clipboard ---------------------------------------------------- + +@pytest.mark.skipif(not sys.platform.startswith("win"), + reason="clipboard writes need the Windows backend here") +def test_round_trip_accepts_both_argument_forms(tmp_path): + from PIL import Image + source = tmp_path / "in.png" + Image.new("RGB", (24, 16), (200, 40, 90)).save(source) + + cb.set_clipboard_image(str(source)) # the MCP / script form + from_path = cb.get_clipboard_image() + assert from_path and Image.open(io.BytesIO(from_path)).size == (24, 16) + + cb.set_clipboard_image(_png((8, 8))) # the remote-desktop form + from_bytes = cb.get_clipboard_image() + assert from_bytes and Image.open(io.BytesIO(from_bytes)).size == (8, 8) + + +def test_the_duplicate_module_is_gone(): + """A stale import of the old module must fail loudly, not resurrect it.""" + with pytest.raises(ImportError): + __import__("je_auto_control.utils.clipboard.clipboard_image") + + +def test_subpackage_exports_the_image_helpers(): + """They existed but were unreachable from the package or the facade.""" + import je_auto_control as ac + from je_auto_control.utils import clipboard as pkg + for name in ("get_clipboard_image", "set_clipboard_image"): + assert name in pkg.__all__ and hasattr(pkg, name) + assert name in ac.__all__ and hasattr(ac, name) diff --git a/test/unit_test/headless/test_doc_counts.py b/test/unit_test/headless/test_doc_counts.py new file mode 100644 index 00000000..9b78f429 --- /dev/null +++ b/test/unit_test/headless/test_doc_counts.py @@ -0,0 +1,100 @@ +"""The counts quoted in the docs must match the tree. No Qt. + +CLAUDE.md requires `architecture_explore.md` and the three READMEs to be +updated with every change, and says every count in them is measured. Nothing +enforced it, and it drifted in practice: wiring `utils/url_canon` added three +commands and three MCP tools while every document kept quoting the old totals. +That was caught by hand afterwards, not by any check. + +A mismatch here means one of two things, and the message says which: +the number moved and the docs were not updated, or the sentence holding the +number was reworded and this test's pattern needs to follow it. +""" +import pathlib +import re +from typing import Callable, List, Tuple + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parents[3] + + +def _commands() -> int: + from je_auto_control.utils.executor.action_executor import executor + return len(executor.known_commands()) + + +def _mcp_tools() -> int: + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + # Pinned, not defaulted: the registry consults + # JE_AUTOCONTROL_MCP_READONLY and JE_AUTOCONTROL_MCP_ALIASES, so a + # developer with either set in their shell would otherwise see this guard + # fail on a documentation file that is perfectly correct. + return len(build_default_tool_registry(read_only=False, aliases=True)) + + +def _utils_subpackages() -> int: + utils = ROOT / "je_auto_control" / "utils" + return sum(1 for p in utils.iterdir() + if p.is_dir() and (p / "__init__.py").exists()) + + +def _examples() -> int: + return len(list((ROOT / "examples").glob("*.py"))) + + +# (doc, regex with one capture group, what it counts, how to measure it) +CITATIONS: List[Tuple[str, str, str, Callable[[], int]]] = [ + ("architecture_explore.md", + r"`AC_\*` 動作指令數（`known_commands\(\)` 實測） \| (\d+) \|", + "AC_* commands", _commands), + ("architecture_explore.md", + r"MCP 工具數（`build_default_tool_registry\(\)` 實測） \| (\d+) \|", + "MCP tools", _mcp_tools), + ("architecture_explore.md", + r"### 5\.4 能力層 `utils/`（(\d+) 個子套件）", + "utils/ subpackages", _utils_subpackages), + ("README.md", r"(\d+) `AC_\*` commands", "AC_* commands", _commands), + ("README.md", r"all (\d+) commands", "AC_* commands", _commands), + ("README.md", r"(\d+) tools for", "MCP tools", _mcp_tools), + ("README.md", r"(\d+) self-contained scripts", "examples", _examples), + ("README/README_zh-CN.md", r"(\d+) 个 `AC_\*` 命令", + "AC_* commands", _commands), + ("README/README_zh-CN.md", r"全部 (\d+) 个命令", "AC_* commands", _commands), + ("README/README_zh-CN.md", r"(\d+) 个工具", "MCP tools", _mcp_tools), + ("README/README_zh-TW.md", r"(\d+) 個 `AC_\*` 指令", + "AC_* commands", _commands), + ("README/README_zh-TW.md", r"全部 (\d+) 個指令", "AC_* commands", _commands), + ("README/README_zh-TW.md", r"(\d+) 個工具", "MCP tools", _mcp_tools), + ("CLAUDE.md", r"(\d+) `utils/` subpackages", + "utils/ subpackages", _utils_subpackages), + ("CLAUDE.md", r"\((\d+) headless subpackages\)", + "utils/ subpackages", _utils_subpackages), + ("CLAUDE.md", r"partition all (\d+) subpackages", + "utils/ subpackages", _utils_subpackages), +] + + +@pytest.mark.parametrize("doc,pattern,label,measure", CITATIONS, + ids=[f"{d}:{lbl}" for d, _p, lbl, _m in CITATIONS]) +def test_quoted_count_matches_the_tree(doc, pattern, label, measure): + text = (ROOT / doc).read_text(encoding="utf-8") + found = re.findall(pattern, text) + assert found, ( + f"{doc}: no longer states the {label} count in the expected form " + f"({pattern!r}). If the wording changed on purpose, update this " + f"test's pattern; the count itself must stay in the document." + ) + actual = measure() + for quoted in found: + assert int(quoted) == actual, ( + f"{doc} says {quoted} {label}, the tree has {actual}. " + f"Re-measure and update every document that quotes it " + f"(architecture_explore.md, README.md and both translations)." + ) + + +def test_every_citation_target_exists(): + """A renamed doc must not silently switch this guard off.""" + for doc, _pattern, _label, _measure in CITATIONS: + assert (ROOT / doc).is_file(), f"missing documentation file: {doc}" diff --git a/test/unit_test/headless/test_input_hook_timeline.py b/test/unit_test/headless/test_input_hook_timeline.py new file mode 100644 index 00000000..afdcb885 --- /dev/null +++ b/test/unit_test/headless/test_input_hook_timeline.py @@ -0,0 +1,215 @@ +"""Headless tests for the recorder's replayable timeline. No Qt, no real input.""" +import sys + +import pytest + +import je_auto_control as ac + +if not sys.platform.startswith("win"): # pragma: no cover + pytest.skip("Windows recorder", allow_module_level=True) + +from je_auto_control.windows.record.win32_input_hook import ( # noqa: E402 + timeline, +) +from je_auto_control.windows.record.win32_record import Win32Recorder # noqa: E402 + + +def _events(): + """A press, a release, and a scroll — the three things replay needs.""" + return [ + {"op": "key_down", "vk": 65, "time": 10.0}, + {"op": "key_up", "vk": 65, "time": 10.25}, + {"op": "scroll", "delta": -3, "x": 5, "y": 6, "time": 10.75}, + ] + + +def test_timeline_turns_timestamps_into_gaps(): + # Without the gaps every step replays at once and no real interface keeps up. + out = timeline(_events()) + assert [event["delta_ms"] for event in out] == [0, 250, 500] + assert "time" not in out[0] + + +def test_timeline_keeps_releases_and_wheel(): + # The whole point: a press-only recording cannot tell a drag from a click, + # and loses scrolling entirely. + out = timeline(_events()) + assert [event["op"] for event in out] == ["key_down", "key_up", "scroll"] + assert out[2]["delta"] == -3 + + +def test_timeline_of_nothing_is_empty(): + assert timeline([]) == [] + + +def test_timeline_never_reports_a_negative_gap(): + # Monotonic time should not go backwards, but a clamped gap is better than + # a replay that tries to sleep a negative amount. + out = timeline([{"op": "key_down", "vk": 1, "time": 5.0}, + {"op": "key_up", "vk": 1, "time": 4.0}]) + assert out[1]["delta_ms"] == 0 + + +class _FakeHook: + def __init__(self, events): + self._events = events + + def start(self): + pass + + def stop(self): + return list(self._events) + + +def _recorder(monkeypatch, events): + recorder = Win32Recorder() + monkeypatch.setattr( + "je_auto_control.windows.record.win32_record.Win32InputHook", + lambda *a, **k: _FakeHook(events)) + return recorder + + +def test_legacy_stop_record_still_returns_press_only_commands(monkeypatch): + # Existing callers feed this straight into the executor; it must not change. + recorder = _recorder(monkeypatch, [ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "key_up", "vk": 65, "time": 1.1}, + {"op": "mouse_down", "button": "left", "x": 7, "y": 8, "time": 1.2}, + {"op": "mouse_up", "button": "left", "x": 7, "y": 8, "time": 1.3}, + ]) + recorder.record() + assert list(recorder.stop_record().queue) == [ + ("AC_type_keyboard", 65), ("AC_mouse_left", 7, 8)] + + +def test_timeline_stop_returns_everything(monkeypatch): + recorder = _recorder(monkeypatch, _events()) + recorder.record() + out = recorder.stop_record_timeline() + assert [event["op"] for event in out] == ["key_down", "key_up", "scroll"] + + +def test_keyboard_only_recording_drops_mouse_events(monkeypatch): + recorder = _recorder(monkeypatch, [ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "mouse_down", "button": "left", "x": 1, "y": 2, "time": 1.1}, + ]) + recorder.record_keyboard() + assert [e["op"] for e in recorder.stop_record_timeline()] == ["key_down"] + + +def test_mouse_only_recording_drops_key_events(monkeypatch): + recorder = _recorder(monkeypatch, [ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "scroll", "delta": 1, "time": 1.1}, + ]) + recorder.record_mouse() + assert [e["op"] for e in recorder.stop_record_timeline()] == ["scroll"] + + +def test_stopping_without_recording_is_not_an_error(): + assert Win32Recorder().stop_record_timeline() == [] + + +# --- hook decoding -------------------------------------------------------- +# +# Driven by handing the callbacks fabricated structures, so it needs no real +# input and no particular window in front. That matters here: a game with +# anti-cheat in the foreground blocks both injected input and low-level hooks, +# which makes any live test of this silently return nothing. + +class _FakeUser32: + @staticmethod + def CallNextHookEx(*_args): # noqa: N802 (Win32 naming) + return 0 + + +def _hook(): + from je_auto_control.windows.record.win32_input_hook import Win32InputHook + return Win32InputHook() + + +def _key_param(vk): + import ctypes + from je_auto_control.windows.record import win32_input_hook as hook + data = hook._KBDLLHOOKSTRUCT(vkCode=vk) + return ctypes.cast(ctypes.pointer(data), ctypes.c_void_p).value, data + + +def _mouse_param(x, y, mouse_data=0): + import ctypes + from ctypes import wintypes + from je_auto_control.windows.record import win32_input_hook as hook + data = hook._MSLLHOOKSTRUCT(pt=wintypes.POINT(x, y), mouseData=mouse_data) + return ctypes.cast(ctypes.pointer(data), ctypes.c_void_p).value, data + + +def test_hook_records_key_press_and_release(): + recorder = _hook() + proc = recorder._keyboard_proc(_FakeUser32) + param, _keep = _key_param(65) + proc(0, 0x0100, param) # WM_KEYDOWN + proc(0, 0x0101, param) # WM_KEYUP + assert [e["op"] for e in recorder.events] == ["key_down", "key_up"] + assert recorder.events[0]["vk"] == 65 + assert all("time" in e for e in recorder.events) + + +def test_hook_records_system_keys_too(): + # Alt combinations arrive as WM_SYSKEYDOWN, not WM_KEYDOWN. + recorder = _hook() + proc = recorder._keyboard_proc(_FakeUser32) + param, _keep = _key_param(0x12) + proc(0, 0x0104, param) + assert [e["op"] for e in recorder.events] == ["key_down"] + + +def test_hook_records_button_release_not_just_press(): + # Without the release a drag is indistinguishable from a click. + recorder = _hook() + proc = recorder._mouse_proc(_FakeUser32) + down, _a = _mouse_param(10, 20) + up, _b = _mouse_param(90, 60) + proc(0, 0x0201, down) # WM_LBUTTONDOWN + proc(0, 0x0202, up) # WM_LBUTTONUP + assert [(e["op"], e["x"], e["y"]) for e in recorder.events] == [ + ("mouse_down", 10, 20), ("mouse_up", 90, 60)] + + +def test_hook_decodes_a_negative_wheel_delta(): + # mouseData's high word is a *signed* notch count times 120; reading it + # unsigned turns a scroll down into a scroll up by 65,536 notches. + recorder = _hook() + proc = recorder._mouse_proc(_FakeUser32) + param, _keep = _mouse_param(5, 6, mouse_data=(-240 & 0xFFFF) << 16) + proc(0, 0x020A, param) # WM_MOUSEWHEEL + assert recorder.events[0]["op"] == "scroll" + assert recorder.events[0]["delta"] == -2 + + +def test_hook_ignores_negative_codes_and_unknown_messages(): + recorder = _hook() + proc = recorder._mouse_proc(_FakeUser32) + param, _keep = _mouse_param(1, 2) + proc(-1, 0x0201, param) # must pass through untouched + proc(0, 0x0200, param) # WM_MOUSEMOVE: too noisy to record + assert recorder.events == [] + + +def test_hook_stops_itself_at_the_event_cap(): + # A forgotten recording must not grow without bound. + recorder = _hook() + recorder.max_events = 3 + proc = recorder._keyboard_proc(_FakeUser32) + param, _keep = _key_param(65) + for _ in range(10): + proc(0, 0x0100, param) + assert len(recorder.events) == 3 + + +def test_wiring(): + assert "AC_stop_record_timeline" in set(ac.executor.known_commands()) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + assert "ac_record_stop_timeline" in {t.name for t in build_default_tool_registry()} + assert hasattr(ac, "stop_record_timeline") + assert "stop_record_timeline" in ac.__all__ diff --git a/test/unit_test/headless/test_input_reach.py b/test/unit_test/headless/test_input_reach.py new file mode 100644 index 00000000..1c6f6c1a --- /dev/null +++ b/test/unit_test/headless/test_input_reach.py @@ -0,0 +1,77 @@ +"""Headless tests for the "will my input arrive?" probes. No Qt.""" +import time + +import pytest + +import je_auto_control as ac +from je_auto_control.utils.input_reach import input_reach as reach + + +@pytest.fixture(autouse=True) +def _clear_cache(): + reach._desktop_cache = None + yield + reach._desktop_cache = None + + +def test_desktop_probe_is_cached_then_expires(monkeypatch): + # Input primitives ask constantly, so it must be cached — but not for so + # long that an unlocked machine still reports locked. + monkeypatch.setattr(reach.sys, "platform", "win32") + reach._desktop_cache = (time.monotonic(), False) + assert reach.input_desktop_available() is False + reach._desktop_cache = (time.monotonic() - reach.DESKTOP_CACHE_SEC - 1, False) + assert reach.input_desktop_available() is not False + + +def test_desktop_probe_is_true_off_windows(monkeypatch): + monkeypatch.setattr(reach.sys, "platform", "linux") + assert reach.input_desktop_available() is True + + +def test_reach_probe_is_true_off_windows(monkeypatch): + monkeypatch.setattr(reach.sys, "platform", "linux") + assert reach.input_reaches_system() is True + + +def test_reach_probe_uses_a_key_nothing_listens_for(): + # F13: real keyboards stop at F12, so the probe cannot trigger anything. + assert reach.PROBE_VK == 0x7C + + +def test_executor_adapter_skips_the_keystroke_when_the_desktop_is_locked(monkeypatch): + # No point sending a probe key at a lock screen, and no point pretending + # the answer is yes. + from je_auto_control.utils.executor import action_executor + + import je_auto_control.utils.input_reach as package + + sent = [] + # The adapter imports these from the package namespace, so that is what has + # to be patched — patching the module underneath would let the real probe + # run and actually press a key. + monkeypatch.setattr(package, "input_desktop_available", lambda: False) + monkeypatch.setattr(package, "input_reaches_system", + lambda *a, **k: sent.append(1) or True) + result = action_executor._input_reachable() + assert result == {"desktop_available": False, "reaches_system": False} + assert sent == [] + + +def test_executor_adapter_probes_when_the_desktop_is_available(monkeypatch): + from je_auto_control.utils.executor import action_executor + + import je_auto_control.utils.input_reach as package + + monkeypatch.setattr(package, "input_desktop_available", lambda: True) + monkeypatch.setattr(package, "input_reaches_system", lambda *a, **k: False) + assert action_executor._input_reachable() == { + "desktop_available": True, "reaches_system": False} + + +def test_wiring(): + assert "AC_input_reachable" in set(ac.executor.known_commands()) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + assert "ac_input_reachable" in {t.name for t in build_default_tool_registry()} + for attr in ("input_desktop_available", "input_reaches_system"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_keyboard_layout.py b/test/unit_test/headless/test_keyboard_layout.py new file mode 100644 index 00000000..7496b1ee --- /dev/null +++ b/test/unit_test/headless/test_keyboard_layout.py @@ -0,0 +1,44 @@ +"""Headless tests for key-to-character translation. No Qt, no real keyboard.""" +import je_auto_control as ac +from je_auto_control.utils.keyboard_layout import keyboard_layout as kl + + +def test_us_fallback_covers_letters_digits_and_punctuation(): + # Letters and digits are the same on every Latin layout; punctuation is + # exactly what differs, which is why the OS is asked first. + assert kl.US_PRINTABLE_VK[0x41] == ("a", "A") + assert kl.US_PRINTABLE_VK[0x31] == ("1", "!") + assert kl.US_PRINTABLE_VK[0xBC] == (",", "<") + + +def test_char_table_prefers_the_layout_over_the_fallback(monkeypatch): + # A layout that reports a different punctuation key must win, or every + # comma recorded on a non-US keyboard is mislabelled. + monkeypatch.setattr(kl, "layout_char_table", lambda layout=None: {0xBC: (";", ":")}) + table = kl.char_table() + assert table[0xBC] == (";", ":") + assert table[0x41] == ("a", "A") # untouched keys still come from US + + +def test_char_table_falls_back_when_the_os_says_nothing(monkeypatch): + monkeypatch.setattr(kl, "layout_char_table", lambda layout=None: {}) + assert kl.char_table()[0xBC] == (",", "<") + + +def test_vk_to_char_picks_the_shifted_half(): + table = {0x41: ("a", "A")} + assert kl.vk_to_char(0x41, False, table) == "a" + assert kl.vk_to_char(0x41, True, table) == "A" + assert kl.vk_to_char(0x99, False, table) is None + + +def test_layout_table_is_empty_off_windows(monkeypatch): + monkeypatch.setattr(kl.sys, "platform", "linux") + assert kl.layout_char_table(12345) == {} + assert kl.foreground_keyboard_layout() is None + + +def test_facade_exports(): + for attr in ("char_table", "vk_to_char", "layout_char_table", + "foreground_keyboard_layout"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_logical_frame.py b/test/unit_test/headless/test_logical_frame.py new file mode 100644 index 00000000..1a05af03 --- /dev/null +++ b/test/unit_test/headless/test_logical_frame.py @@ -0,0 +1,106 @@ +"""Headless tests for capturing in mouse-coordinate space. No Qt, no screen.""" +from je_auto_control.utils.monitor_layout import ( + grab_logical, logical_scale, logical_virtual_rect, needs_rescale, +) + + +class FakeImage: + """Minimal PIL-Image stand-in recording the resize / crop it was asked for.""" + + def __init__(self, width, height): + self.width = width + self.height = height + self.resized_to = None + self.cropped_to = None + + @property + def size(self): + return self.width, self.height + + def resize(self, size, _resample=None): + out = FakeImage(*size) + out.resized_to = size + return out + + def crop(self, box): + out = FakeImage(box[2] - box[0], box[3] - box[1]) + out.cropped_to = box + return out + + +class FakeGrab: + """``ImageGrab`` stand-in: separate sizes for the primary and full desktop.""" + + def __init__(self, primary=(1920, 1080), full=(3840, 1244)): + self._primary = primary + self._full = full + self.calls = [] + + def grab(self, *, all_screens=False, **_kwargs): + self.calls.append(all_screens) + return FakeImage(*(self._full if all_screens else self._primary)) + + +def _metrics(x=0, y=-164, width=3456, height=1244): + """A mixed-DPI desktop: logical 3456 wide against a 3840 physical capture.""" + return lambda index: {76: x, 77: y, 78: width, 79: height}[index] + + +def test_logical_scale_and_needs_rescale_are_pure(): + assert logical_scale((3840, 1244), (3456, 1244)) == (3840 / 3456, 1.0) + assert needs_rescale((3840, 1244), (3456, 1244)) + assert not needs_rescale((3456, 1244), (3456, 1244)) + # nothing to compare against -> do not rescale on a guess + assert not needs_rescale((100, 100), (0, 0)) + + +def test_logical_virtual_rect_reports_a_negative_origin(): + # a monitor above the primary one puts the desktop origin at a negative y + assert logical_virtual_rect(_metrics()) == (0, -164, 3456, 1244) + + +def test_logical_virtual_rect_is_none_when_unreported(): + assert logical_virtual_rect(_metrics(width=0, height=0)) is None + + +def test_grab_logical_rescales_a_physical_capture_to_mouse_space(): + # unscaled, a point read off the 3840-wide capture lands ~116 px away from + # where it was clicked on this desktop + grabber = FakeGrab() + image, origin_x, origin_y = grab_logical(grabber=grabber, metrics=_metrics()) + assert image.size == (3456, 1244) + assert (origin_x, origin_y) == (0, -164) + assert grabber.calls == [True] + + +def test_grab_logical_leaves_a_matching_capture_alone(): + grabber = FakeGrab(full=(3456, 1244)) + image, _x, _y = grab_logical(grabber=grabber, metrics=_metrics()) + assert image.resized_to is None + + +def test_grab_logical_primary_only_skips_the_virtual_desktop(): + grabber = FakeGrab() + image, origin_x, origin_y = grab_logical( + all_screens=False, grabber=grabber, metrics=_metrics()) + assert image.size == (1920, 1080) + assert (origin_x, origin_y) == (0, 0) + assert grabber.calls == [False] + + +def test_grab_logical_crops_after_rescaling_not_before(): + # cropping through ImageGrab's bbox happens in physical pixels and would cut + # the wrong place on a scaled screen, so the crop must be on the rescaled frame + grabber = FakeGrab() + image, origin_x, origin_y = grab_logical( + (100, 0, 600, 400), grabber=grabber, metrics=_metrics()) + assert image.cropped_to == (100, 164, 700, 564) + assert image.size == (600, 400) + assert (origin_x, origin_y) == (100, 0) + + +def test_grab_logical_region_origin_is_the_region_corner(): + grabber = FakeGrab() + _image, origin_x, origin_y = grab_logical( + (10, -50, 100, 100), grabber=grabber, metrics=_metrics()) + assert (origin_x, origin_y) == (10, -50) diff --git a/test/unit_test/headless/test_mcp_server.py b/test/unit_test/headless/test_mcp_server.py index 0753e429..a9c8ab58 100644 --- a/test/unit_test/headless/test_mcp_server.py +++ b/test/unit_test/headless/test_mcp_server.py @@ -1037,7 +1037,7 @@ def test_clipboard_image_tools_present_in_default_registry(): def test_get_clipboard_image_returns_text_block_when_empty(monkeypatch): """When the clipboard has no image, return a clear text fallback.""" import je_auto_control.utils.mcp_server.tools._handlers as handlers - import je_auto_control.utils.clipboard.clipboard_image as image_clip + import je_auto_control.utils.clipboard.clipboard as image_clip monkeypatch.setattr(image_clip, "get_clipboard_image", lambda: None) result = handlers.get_clipboard_image() assert result[0].type == "text" @@ -1046,7 +1046,7 @@ def test_get_clipboard_image_returns_text_block_when_empty(monkeypatch): def test_get_clipboard_image_returns_image_block_when_set(monkeypatch): import je_auto_control.utils.mcp_server.tools._handlers as handlers - import je_auto_control.utils.clipboard.clipboard_image as image_clip + import je_auto_control.utils.clipboard.clipboard as image_clip monkeypatch.setattr(image_clip, "get_clipboard_image", lambda: b"\x89PNG\r\n\x1a\n") result = handlers.get_clipboard_image() diff --git a/test/unit_test/headless/test_ocr_text_span.py b/test/unit_test/headless/test_ocr_text_span.py new file mode 100644 index 00000000..685b5388 --- /dev/null +++ b/test/unit_test/headless/test_ocr_text_span.py @@ -0,0 +1,93 @@ +"""Headless tests for cross-word OCR matching. No Qt, no screen.""" +from dataclasses import dataclass + +from je_auto_control.utils.ocr.text_span import ( + LINE_TOLERANCE, find_spans, group_lines, normalize, same_line, +) + + +@dataclass +class Box: + """Stand-in for one OCR word box.""" + text: str + x: int + y: int + width: int = 40 + height: int = 20 + + +def _line(*words, y=100, step=50): + return [Box(text=word, x=10 + index * step, y=y) + for index, word in enumerate(words)] + + +def test_normalize_drops_whitespace_and_case(): + # where the engine splits a phrase is arbitrary, so spacing must not decide + # whether a target matches + assert normalize(" Save As ") == "saveas" + assert normalize("Save As", case_sensitive=True) == "SaveAs" + assert normalize("") == "" + + +def test_same_line_uses_vertical_overlap_not_exact_y(): + assert same_line(Box("a", 0, 100), Box("b", 60, 104)) + # a mixed-size row still counts as one line + assert same_line(Box("a", 0, 100, height=20), Box("B", 60, 98, height=30)) + assert not same_line(Box("a", 0, 100), Box("b", 0, 140)) + + +def test_group_lines_sorts_each_line_left_to_right(): + boxes = [Box("As", 60, 100), Box("Save", 10, 100), Box("File", 10, 200)] + lines = group_lines(boxes) + assert [[b.text for b in line] for line in lines] == [["Save", "As"], ["File"]] + + +def test_find_spans_matches_across_word_boxes(): + # the whole point: "Save As" is two boxes, and per-box matching never sees it + spans = find_spans(_line("File", "Save", "As", "Later"), "Save As") + assert [[b.text for b in span] for span in spans] == [["Save", "As"]] + + +def test_find_spans_matches_cjk_split_by_the_engine(): + spans = find_spans(_line("另存", "新檔"), "另存新檔") + assert [b.text for b in spans[0]] == ["另存", "新檔"] + + +def test_find_spans_returns_a_single_box_hit_too(): + # a superset of per-box matching, so nothing that used to match stops + spans = find_spans(_line("Cancel", "OK"), "OK") + assert [[b.text for b in span] for span in spans] == [["OK"]] + + +def test_find_spans_prefers_the_shortest_run(): + spans = find_spans(_line("Save", "As", "PDF"), "Save As") + assert [b.text for b in spans[0]] == ["Save", "As"] + + +def test_find_spans_does_not_join_across_lines(): + boxes = [Box("Save", 10, 100), Box("As", 10, 200)] + assert find_spans(boxes, "Save As") == [] + + +def test_find_spans_reports_each_occurrence_once(): + boxes = _line("OK", "OK") + assert len(find_spans(boxes, "OK")) == 2 + + +def test_find_spans_is_case_insensitive_by_default(): + assert find_spans(_line("SAVE"), "save") + assert not find_spans(_line("SAVE"), "save", case_sensitive=True) + + +def test_find_spans_ignores_an_empty_target(): + assert find_spans(_line("Save"), " ") == [] + + +def test_find_spans_stops_extending_a_hopeless_run(): + # a long line must not cost a concatenation per (start, end) pair + boxes = _line(*[f"w{i}" for i in range(200)]) + assert find_spans(boxes, "nothing-here") == [] + + +def test_line_tolerance_is_a_fraction_of_height(): + assert 0 < LINE_TOLERANCE < 1 diff --git a/test/unit_test/headless/test_text_unicode_batch.py b/test/unit_test/headless/test_text_unicode_batch.py index f2c80141..e75f27c7 100644 --- a/test/unit_test/headless/test_text_unicode_batch.py +++ b/test/unit_test/headless/test_text_unicode_batch.py @@ -1,8 +1,10 @@ -"""Headless tests for Unicode text entry via clipboard. No Qt.""" +"""Headless tests for Unicode text entry by key events or clipboard. No Qt.""" import je_auto_control as ac from je_auto_control.utils.text_unicode import ( - plan_paste, type_unicode, unicode_code_units, + plan_paste, plan_unicode_keys, type_unicode, type_unicode_keys, + type_unicode_text, unicode_code_units, ) +from je_auto_control.utils.text_unicode import text_unicode as tu def test_code_units_ascii_and_bmp(): @@ -32,6 +34,100 @@ def test_type_unicode_dispatches_plan(): assert result["ops"] == 2 and result["code_units"] == 7 +# --- key injection --------------------------------------------------------- + +def test_plan_unicode_keys_is_one_op_per_code_unit(): + assert plan_unicode_keys("a値") == [ + {"op": "unicode_unit", "unit": 97}, + {"op": "unicode_unit", "unit": 0x5024}, + ] + # astral characters need both surrogates sent separately + assert plan_unicode_keys("🚀") == [ + {"op": "unicode_unit", "unit": 0xD83D}, + {"op": "unicode_unit", "unit": 0xDE80}, + ] + assert plan_unicode_keys("") == [] + + +def test_type_unicode_keys_never_touches_the_clipboard(): + # the whole point of the key route: the user's clipboard survives + events = [] + result = type_unicode_keys("café 🚀", sink=events.append) + assert {e["op"] for e in events} == {"unicode_unit"} + assert result["method"] == "keys" + assert result["ops"] == result["code_units"] == 7 + + +def test_type_unicode_text_picks_keys_when_the_backend_supports_them(monkeypatch): + monkeypatch.setattr(tu, "unicode_keys_supported", lambda: True) + events = [] + assert type_unicode_text("値", sink=events.append)["method"] == "keys" + assert [e["op"] for e in events] == ["unicode_unit"] + + +def test_type_unicode_text_falls_back_to_paste_without_a_backend(monkeypatch): + monkeypatch.setattr(tu, "unicode_keys_supported", lambda: False) + events = [] + assert type_unicode_text("値", sink=events.append)["method"] == "paste" + assert [e["op"] for e in events] == ["set_clipboard", "hotkey"] + + +def test_write_falls_back_to_unicode_for_characters_outside_the_key_table(): + # `write` used to raise on the first character missing from the 192-key + # table — which on a US layout includes `, . / : ? ! _ + @ %` and every CJK + # character, so a URL or a Chinese sentence failed as a whole. + from je_auto_control.wrapper import auto_control_keyboard as kb + + sent = [] + + class _Backend: + @staticmethod + def type_unicode_unit(unit): + sent.append(unit) + + original = kb.keyboard + try: + kb.keyboard = _Backend + assert kb._write_char_via_unicode("，") is True + finally: + kb.keyboard = original + assert sent == [0xFF0C] + + +def test_write_sends_newline_and_tab_as_keys_not_characters(): + # U+000A through the Unicode route is dropped by most applications, which + # silently collapses a multi-line write into one line + from je_auto_control.wrapper import auto_control_keyboard as kb + + assert kb.WRITE_CONTROL_KEYS["\n"] == "return" + assert kb.WRITE_CONTROL_KEYS["\r"] == "return" + assert kb.WRITE_CONTROL_KEYS["\t"] == "tab" + # Each control character needs *some* key route on this platform, or write() + # falls through to the space fallback and silently turns a newline into a + # space. Asserting one spelling is wrong: Windows names backspace "back", + # X11 names it "backspace" and maps the raw "\b" instead — and write() + # already copes, because it checks the name against the table before using + # it. So assert the property that matters, not the spelling. + for char, name in kb.WRITE_CONTROL_KEYS.items(): + assert (name in kb.keyboard_keys_table + or char in kb.keyboard_keys_table), ( + f"{char!r} has no key route on this platform: neither {name!r} nor " + f"the raw character is in keyboard_keys_table, so write() would " + f"fall through to the space fallback" + ) + + +def test_write_reports_no_unicode_route_when_the_backend_lacks_one(): + from je_auto_control.wrapper import auto_control_keyboard as kb + + original = kb.keyboard + try: + kb.keyboard = object() # a backend with no unicode entry point + assert kb._write_char_via_unicode("，") is False + finally: + kb.keyboard = original + + # --- wiring --------------------------------------------------------------- def test_executor_adapter_dispatches_via_sink(): @@ -43,16 +139,19 @@ def test_executor_adapter_dispatches_via_sink(): def test_wiring(): - known = ac.executor.known_commands() - assert "AC_type_unicode" in set(known) + wanted = {"AC_type_unicode", "AC_type_unicode_keys", "AC_type_unicode_text"} + assert wanted <= set(ac.executor.known_commands()) from je_auto_control.utils.mcp_server.tools import build_default_tool_registry names = {t.name for t in build_default_tool_registry()} - assert "ac_type_unicode" in names + assert {"ac_type_unicode", "ac_type_unicode_keys", + "ac_type_unicode_text"} <= names from je_auto_control.gui.script_builder.command_schema import _build_specs specs = {s.command for s in _build_specs()} - assert "AC_type_unicode" in specs + assert wanted <= specs def test_facade_exports(): - for attr in ("type_unicode", "plan_paste", "unicode_code_units"): + for attr in ("type_unicode", "type_unicode_keys", "type_unicode_text", + "plan_paste", "plan_unicode_keys", "unicode_code_units", + "unicode_keys_supported"): assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_url_canon_batch.py b/test/unit_test/headless/test_url_canon_batch.py new file mode 100644 index 00000000..34e455a6 --- /dev/null +++ b/test/unit_test/headless/test_url_canon_batch.py @@ -0,0 +1,71 @@ +"""Headless tests for RFC 3986 URL canonicalisation. No Qt.""" +import je_auto_control as ac +from je_auto_control.utils.url_canon import ( + build_query, canonicalize_url, normalize_url, parse_query, urls_equal, +) + + +def test_canonicalize_lowercases_and_collapses(): + assert canonicalize_url("HTTP://Example.COM:80/a/./b/../c?b=2&a=1#frag") == \ + "http://example.com/a/c?a=1&b=2" + + +def test_normalize_default_port_and_percent_case(): + assert normalize_url("https://EXAMPLE.com:443/p%2fx/") == \ + "https://example.com/p%2Fx/" + + +def test_empty_path_becomes_root(): + assert normalize_url("http://host") == "http://host/" + + +def test_dot_segments_beyond_root_clamped(): + assert canonicalize_url("http://h/../../x") == "http://h/x" + + +def test_urls_equal_ignores_query_order_and_fragment(): + assert urls_equal("http://x.com/a?b=1&a=2", + "http://x.com/a?a=2&b=1#top") is True + assert urls_equal("http://x.com/a", "http://x.com/b") is False + + +def test_build_and_parse_query(): + assert build_query({"q": "hello world", "p": 2}, sort=True) == \ + "p=2&q=hello+world" + assert parse_query("a=1&b=&c=3") == [("a", "1"), ("b", ""), ("c", "3")] + assert parse_query("a=1&b=", keep_blank=False) == [("a", "1")] + + +def test_userinfo_preserved(): + assert normalize_url("http://User:Pass@Host.COM/") == \ + "http://User:Pass@host.com/" + + +# --- wiring --------------------------------------------------------------- + +def test_executor_round_trip(): + rec = ac.execute_action([[ + "AC_canonicalize_url", {"url": "HTTP://A.com:80/x/../y"}]]) + out = next(v for v in rec.values() if isinstance(v, dict)) + assert out["url"] == "http://a.com/y" + rec2 = ac.execute_action([[ + "AC_urls_equal", + {"first": "http://x/a?b=1&a=2", "second": "http://x/a?a=2&b=1"}]]) + assert next(v for v in rec2.values() if isinstance(v, dict))["equal"] is True + + +def test_wiring(): + known = ac.executor.known_commands() + assert {"AC_canonicalize_url", "AC_urls_equal"} <= set(known) + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_canonicalize_url", "ac_urls_equal"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_canonicalize_url", "AC_urls_equal"} <= specs + + +def test_facade_exports(): + for attr in ("canonicalize_url", "normalize_url", "urls_equal", + "build_query", "parse_query"): + assert hasattr(ac, attr) and attr in ac.__all__ diff --git a/test/unit_test/headless/test_visual_match_batch.py b/test/unit_test/headless/test_visual_match_batch.py index d47988ac..5b7f2a47 100644 --- a/test/unit_test/headless/test_visual_match_batch.py +++ b/test/unit_test/headless/test_visual_match_batch.py @@ -67,6 +67,70 @@ def test_to_dict_has_center(): assert data["center"] == [60, 40] and data["score"] == pytest.approx(1.0) +# --- screen coordinates ---------------------------------------------------- + +def _stub_grab(monkeypatch, image, origin): + """Replace the screen grab with a fixed frame at a fixed screen origin.""" + from je_auto_control.utils.visual_match import visual_match as vm + monkeypatch.setattr(vm, "_grab_gray_with_origin", + lambda region: (image, origin[0], origin[1])) + + +def test_grabbed_matches_are_translated_to_screen_coordinates(monkeypatch): + # The virtual desktop starts at a negative origin whenever a monitor sits + # left of or above the primary one; an untranslated hit is unclickable. + _stub_grab(monkeypatch, _haystack(), (-100, -164)) + match = match_template(_patch(), min_score=0.9) + assert (match.x, match.y) == (50 - 100, 30 - 164) + assert match.center == [60 - 100, 40 - 164] + + +def test_match_all_translates_every_hit(monkeypatch): + _stub_grab(monkeypatch, _haystack(), (10, 20)) + hits = match_template_all(_patch(), min_score=0.9) + assert sorted((m.x, m.y) for m in hits) == [(60, 50), (130, 50)] + + +def test_supplied_haystack_keeps_its_own_coordinates(monkeypatch): + # An injected image is its own space — adding a screen origin to it would + # be wrong, and would break every synthetic-array test above. + _stub_grab(monkeypatch, _haystack(), (999, 999)) + match = match_template(_patch(), haystack=_haystack(), min_score=0.9) + assert (match.x, match.y) == (50, 30) + + +# --- degenerate templates -------------------------------------------------- + +def test_flat_template_is_refused_rather_than_matched_anywhere(): + # Normalised correlation divides by the template's variance, so a single + # colour saturates the whole score map at 1.0 and "finds" the target at an + # arbitrary position — worse than failing, because the caller clicks there. + flat = np.full((20, 20), 128, dtype=np.uint8) + with pytest.raises(ac.AutoControlScreenException): + match_template(flat, haystack=_haystack()) + with pytest.raises(ac.AutoControlScreenException): + match_template_all(flat, haystack=_haystack()) + + +def test_template_larger_than_haystack_finds_nothing(): + big = np.tile(np.arange(0, 250, 1, dtype=np.uint8), (250, 1)) + assert match_template_all(big, haystack=_haystack()) == [] + + +# --- file loading ---------------------------------------------------------- + +def test_template_loads_from_a_non_ascii_path(tmp_path): + # cv2.imread goes through the C locale on Windows and returns None for a + # path with non-ASCII characters — indistinguishable from a corrupt file. + import cv2 + ascii_path = tmp_path / "ascii.png" + cv2.imwrite(str(ascii_path), _patch()) + target = tmp_path / "樣板圖.png" + target.write_bytes(ascii_path.read_bytes()) + match = match_template(str(target), haystack=_haystack(), min_score=0.9) + assert match is not None and (match.x, match.y) == (50, 30) + + # --- wiring --------------------------------------------------------------- def test_wiring(): diff --git a/test/unit_test/headless/test_window_manage.py b/test/unit_test/headless/test_window_manage.py new file mode 100644 index 00000000..3042a734 --- /dev/null +++ b/test/unit_test/headless/test_window_manage.py @@ -0,0 +1,129 @@ +"""Headless tests for window management. No Qt. + +The wrapper imports ``windows_window_manage`` inside each function, so patching +attributes on that module intercepts the Win32 layer and keeps these tests off +the real desktop. +""" +import sys + +import pytest + +from je_auto_control.wrapper import auto_control_window as w + +_WINDOWS = sys.platform in ("win32", "cygwin", "msys") +pytestmark = pytest.mark.skipif(not _WINDOWS, + reason="window management is Windows-only") + + +@pytest.fixture() +def wm(monkeypatch): + """The Win32 backend module, with every call stubbed out.""" + from je_auto_control.windows.window import windows_window_manage as module + calls = [] + monkeypatch.setattr(module, "get_all_window_hwnd", + lambda: [(11, "Editor"), (12, " "), (13, "Browser")]) + monkeypatch.setattr(module, "get_window_rect", lambda h: (10, 20, 110, 220)) + monkeypatch.setattr(module, "get_foreground_window", lambda: 13) + for name in ("close_window", "minimize_window"): + monkeypatch.setattr(module, name, + lambda h, _n=name: calls.append((_n, h)) or True) + monkeypatch.setattr(module, "move_window", + lambda h, x, y, cx, cy, *a: calls.append( + ("move", h, x, y, cx, cy)) or True) + module.calls = calls + return module + + +# --- hwnd type ------------------------------------------------------------- + +def test_enumerated_hwnds_are_plain_ints(): + """The whole point of the callback prototype fix. + + A ``POINTER(c_int)`` declaration handed back ``LP_c_long`` objects, and + ``int(hwnd)`` on one raises ValueError — the list was unusable for any + follow-up Win32 call. + """ + for hwnd, _title in w.list_windows(): + assert isinstance(hwnd, int) + assert int(hwnd) == hwnd + + +def test_real_foreground_window_is_int_or_none(): + hit = w.foreground_window() + assert hit is None or isinstance(hit[0], int) + + +# --- listing --------------------------------------------------------------- + +def test_titled_only_drops_blank_titles(wm): + assert w.list_windows(titled_only=True) == [(11, "Editor"), (13, "Browser")] + assert len(w.list_windows()) == 3 + + +def test_find_window_is_case_insensitive_by_default(wm): + assert w.find_window("editor") == (11, "Editor") + assert w.find_window("editor", case_sensitive=True) is None + + +# --- close vs minimise ----------------------------------------------------- + +def test_close_and_minimize_reach_different_backend_calls(wm): + """They were the same call, so 'close' silently only minimised.""" + assert w.close_window_by_title("Editor") is True + assert w.minimize_window_by_title("Editor") is True + assert wm.calls == [("close_window", 11), ("minimize_window", 11)] + + +def test_close_and_minimize_report_a_miss(wm): + assert w.close_window_by_title("nothing here") is False + assert w.minimize_window_by_title("nothing here") is False + assert wm.calls == [] + + +# --- geometry -------------------------------------------------------------- + +def test_window_rect_returns_the_backend_rect(wm): + assert w.window_rect("Browser") == (10, 20, 110, 220) + assert w.window_rect("nothing here") is None + + +def test_move_keeps_current_size_when_width_height_omitted(wm): + assert w.move_window_by_title("Editor", 5, 6) is True + # rect (10, 20, 110, 220) -> 100 x 200, carried over rather than zeroed. + assert wm.calls == [("move", 11, 5, 6, 100, 200)] + + +def test_move_uses_explicit_size_when_given(wm): + w.move_window_by_title("Editor", 5, 6, width=42, height=43) + assert wm.calls == [("move", 11, 5, 6, 42, 43)] + + +def test_foreground_window_pairs_hwnd_with_title(wm): + assert w.foreground_window() == (13, "Browser") + + +def test_foreground_window_is_none_when_backend_reports_zero(wm, monkeypatch): + monkeypatch.setattr(wm, "get_foreground_window", lambda: 0) + assert w.foreground_window() is None + + +# --- show_window ----------------------------------------------------------- + +def test_hiding_a_window_does_not_pull_it_to_the_foreground(monkeypatch): + """Hide-then-foreground is self-contradicting, and it used to do both.""" + from je_auto_control.windows.window import windows_window_manage as module + seen = [] + + class _FakeUser32: + def ShowWindow(self, hwnd, cmd): # noqa: N802 # Win32 name + seen.append(("show", hwnd, cmd)) + + def SetForegroundWindow(self, hwnd): # noqa: N802 # Win32 name + seen.append(("front", hwnd)) + + monkeypatch.setattr(module, "_user32", _FakeUser32()) + module.show_window(7, 0) # SW_HIDE + assert seen == [("show", 7, 0)] + seen.clear() + module.show_window(7, 3) # SW_MAXIMIZE + assert seen == [("show", 7, 3), ("front", 7)]