From fb26620169ab0bba81a36fa66f9b46308b1156ae Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 25 Jun 2026 02:33:59 +0800 Subject: [PATCH 1/3] Add virtualized: realize off-screen items in virtualized lists/grids Long lists/grids/trees only materialize visible rows, so an off-screen row has no accessibility element at all - list/read_table/select can't see it and scroll_control_into_view can't help because it doesn't exist yet. realize_item locates the item by property (ItemContainerPattern) and realizes it (VirtualizedItemPattern) so it becomes a real element. Extends the backend ABC + Windows UIA backend via the same fake-backend seam as control_patterns. --- WHATS_NEW.md | 6 ++ .../doc/new_features/v195_features_doc.rst | 48 ++++++++++ .../Zh/doc/new_features/v195_features_doc.rst | 41 +++++++++ je_auto_control/__init__.py | 3 + .../gui/script_builder/command_schema.py | 13 +++ .../utils/accessibility/backends/base.py | 17 ++++ .../accessibility/backends/windows_backend.py | 34 +++++++ .../utils/executor/action_executor.py | 15 ++++ .../utils/mcp_server/tools/_factories.py | 21 +++++ .../utils/mcp_server/tools/_handlers.py | 7 ++ je_auto_control/utils/virtualized/__init__.py | 4 + .../utils/virtualized/virtualized.py | 39 ++++++++ .../headless/test_virtualized_batch.py | 90 +++++++++++++++++++ 13 files changed, 338 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v195_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v195_features_doc.rst create mode 100644 je_auto_control/utils/virtualized/__init__.py create mode 100644 je_auto_control/utils/virtualized/virtualized.py create mode 100644 test/unit_test/headless/test_virtualized_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 0587166d..364d9264 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-25) — Realize Off-Screen Items in Virtualized Lists / Grids + +Reach a row that isn't scrolled into view yet — the "element not found in a long list" fix. Full reference: [`docs/source/Eng/doc/new_features/v195_features_doc.rst`](docs/source/Eng/doc/new_features/v195_features_doc.rst). + +- **`realize_item`** (`AC_realize_item`): long lists / data grids / trees only materialize visible rows, so an off-screen row has no accessibility element at all — `list_accessibility_elements` / `read_control_table` / `select_control_item` can't see it, and `scroll_control_into_view` can't help because the element doesn't exist yet. This locates the item by property (UIA `ItemContainerPattern.FindItemByProperty`) and realizes it (`VirtualizedItemPattern.Realize`) so it becomes a real, clickable element. Match `by` name (default) or `automation_id`; locate the container by name/role/app. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA in the Windows backend). No `PySide6`. + ## What's new (2026-06-25) — Per-Run Step Timeline (waterfall + bottleneck steps) Read why *this* run was slow — a step waterfall and its bottlenecks. Full reference: [`docs/source/Eng/doc/new_features/v194_features_doc.rst`](docs/source/Eng/doc/new_features/v194_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v195_features_doc.rst b/docs/source/Eng/doc/new_features/v195_features_doc.rst new file mode 100644 index 00000000..43a8d486 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v195_features_doc.rst @@ -0,0 +1,48 @@ +Realize Off-Screen Items in Virtualized Lists / Grids +===================================================== + +Long lists, data grids and trees (WPF / WinUI / File Explorer / virtual +treeviews) only materialize the rows that are scrolled into view — a row that is +off-screen has **no** accessibility element at all. So +``list_accessibility_elements`` / ``read_control_table`` / ``select_control_item`` +simply cannot see it, and ``scroll_control_into_view`` can't help because the +target element does not exist yet. This is the classic "element not found in a +long list" wall. + +``realize_item`` closes that gap: it locates the item inside its container by +property (UI Automation ``ItemContainerPattern.FindItemByProperty``) and realizes +it (``VirtualizedItemPattern.Realize``) so it materializes as a real element you +can then click or read. + +It is a thin dispatch onto the injectable ``accessibility.backends.get_backend()`` +seam (the same seam the rest of the accessibility module uses) — headless-testable +on any platform by injecting a fake backend; the real UIA calls live in the +Windows backend. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import realize_item, click_accessibility_element + + # Bring a far-down row into existence, then act on it: + row = realize_item("Order 5000", container_name="Orders") + if row is not None: + click_accessibility_element(name=row.name) # now a real element + + realize_item("row-42", by="automation_id", container_name="DataGrid") + +``item_name`` is matched against the item's Name (``by="name"``, default) or its +AutomationId (``by="automation_id"``). The container is located by +``container_name`` / ``container_role`` / ``app_name`` / ``automation_id`` (the +same matchers as the other native-control actions). Returns the realized +``AccessibilityElement``, or ``None`` if the container or item isn't found. + +Executor commands +----------------- + +``AC_realize_item`` (``item_name`` / ``by`` / ``container_name`` / +``container_role`` / ``app_name`` / ``automation_id``) returns +``{found, element}``. It is exposed as the read-only ``ac_realize_item`` MCP tool +and as a Script Builder command under **Native UI**. diff --git a/docs/source/Zh/doc/new_features/v195_features_doc.rst b/docs/source/Zh/doc/new_features/v195_features_doc.rst new file mode 100644 index 00000000..ee1cbc8e --- /dev/null +++ b/docs/source/Zh/doc/new_features/v195_features_doc.rst @@ -0,0 +1,41 @@ +實體化虛擬化清單 / 格線中的離畫面項目 +====================================== + +長清單、資料格線與樹(WPF / WinUI / 檔案總管 / 虛擬化 treeview)只會實體化已捲入視野的列—— +離畫面的列**完全沒有**無障礙元素。因此 ``list_accessibility_elements`` / +``read_control_table`` / ``select_control_item`` 根本看不到它,而 ``scroll_control_into_view`` +也幫不上忙,因為目標元素根本還不存在。這就是經典的「長清單裡找不到元素」的牆。 + +``realize_item`` 補上這個缺口:它以屬性在容器內定位該項目(UI Automation +``ItemContainerPattern.FindItemByProperty``)並將其實體化(``VirtualizedItemPattern.Realize``), +使其成為一個真正、可點擊或可讀取的元素。 + +它是對可注入的 ``accessibility.backends.get_backend()`` 接縫的薄分派(與無障礙模組其餘部分相同的 +接縫)——可在任何平台透過注入 fake backend 進行無頭測試;真正的 UIA 呼叫位於 Windows 後端。 +不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import realize_item, click_accessibility_element + + # 讓一個很下方的列「存在」,然後對它操作: + row = realize_item("Order 5000", container_name="Orders") + if row is not None: + click_accessibility_element(name=row.name) # 現在是真正的元素 + + realize_item("row-42", by="automation_id", container_name="DataGrid") + +``item_name`` 會比對項目的 Name(``by="name"``,預設)或其 AutomationId +(``by="automation_id"``)。容器以 ``container_name`` / ``container_role`` / ``app_name`` / +``automation_id`` 定位(與其他原生控制動作相同的比對方式)。回傳實體化後的 +``AccessibilityElement``,若找不到容器或項目則回傳 ``None``。 + +執行器指令 +---------- + +``AC_realize_item``(``item_name`` / ``by`` / ``container_name`` / ``container_role`` / +``app_name`` / ``automation_id``)回傳 ``{found, element}``。以唯讀 ``ac_realize_item`` MCP +工具及 Script Builder 指令(位於 **Native UI** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 81bc5b59..4113e55c 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -66,6 +66,8 @@ from je_auto_control.utils.focus_order import ( audit_focus_order, focus_control, is_interactive_role, tab_order, ) +# Realize off-screen items in virtualized lists / grids (UIA VirtualizedItem) +from je_auto_control.utils.virtualized import realize_item # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1666,6 +1668,7 @@ def start_autocontrol_gui(*args, **kwargs): "control_type_name", "humanize_role", "humanize_tree", "assign_node_paths", "find_by_path", "is_interactive_role", "tab_order", "audit_focus_order", "focus_control", + "realize_item", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index d4d5bce4..c407d839 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1608,6 +1608,19 @@ def _add_native_control_specs(specs: List[CommandSpec]) -> None: fields=fields, description="Scroll a control into view (ScrollItemPattern).", )) + specs.append(CommandSpec( + "AC_realize_item", "Native UI", "Realize Virtualized Item", + fields=( + FieldSpec("item_name", FieldType.STRING), + FieldSpec("by", FieldType.ENUM, optional=True, default="name", + choices=("name", "automation_id")), + FieldSpec("container_name", FieldType.STRING, optional=True), + FieldSpec("container_role", FieldType.STRING, optional=True), + FieldSpec("app_name", FieldType.STRING, optional=True), + FieldSpec("automation_id", FieldType.STRING, optional=True), + ), + description="Realize an off-screen item in a virtualized list/grid.", + )) specs.append(CommandSpec( "AC_get_control_text", "Native UI", "Get Control Text", fields=fields, diff --git a/je_auto_control/utils/accessibility/backends/base.py b/je_auto_control/utils/accessibility/backends/base.py index 82d5798c..2f40c8af 100644 --- a/je_auto_control/utils/accessibility/backends/base.py +++ b/je_auto_control/utils/accessibility/backends/base.py @@ -131,6 +131,23 @@ def set_focus(self, name: Optional[str] = None, role: Optional[str] = None, """Set keyboard focus on the matched control (SetFocus); True on success.""" self._unsupported("set_focus") + # --- virtualized items (realize off-screen list / grid items) ----------- + + def find_virtual_item(self, item_name: Optional[str] = None, by: str = "name", + container_name: Optional[str] = None, + container_role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[AccessibilityElement]: + """Find a (possibly virtualized) item inside a container and realize it. + + Long virtualized lists / grids only materialize visible rows; this locates + the item by property (``ItemContainerPattern``) and realizes it + (``VirtualizedItemPattern``) so it exists as a real element. Returns the + realized element, or None if the container or item isn't found. + """ + self._unsupported("find_virtual_item") + def _unsupported(self, operation: str): """Raise a clear error for an action this backend can't perform.""" raise AccessibilityNotAvailableError( diff --git a/je_auto_control/utils/accessibility/backends/windows_backend.py b/je_auto_control/utils/accessibility/backends/windows_backend.py index cbfcba60..0d0594fc 100644 --- a/je_auto_control/utils/accessibility/backends/windows_backend.py +++ b/je_auto_control/utils/accessibility/backends/windows_backend.py @@ -30,6 +30,9 @@ _UIA_RANGEVALUE_PATTERN_ID = 10003 _UIA_SCROLLITEM_PATTERN_ID = 10017 _UIA_TEXT_PATTERN_ID = 10014 +_UIA_ITEMCONTAINER_PATTERN_ID = 10019 +_UIA_VIRTUALIZEDITEM_PATTERN_ID = 10020 +_UIA_AUTOMATIONID_PROPERTY = 30011 _EXPAND_STATES = {0: "collapsed", 1: "expanded", 2: "partial", 3: "leaf"} @@ -264,6 +267,37 @@ def get_range(self, name=None, role=None, app_name=None, except (OSError, AttributeError, ValueError, TypeError): return None + def _realize(self, raw) -> None: + """Realize a virtualized element so it materializes (VirtualizedItemPattern).""" + pattern = self._pattern(raw, _UIA_VIRTUALIZEDITEM_PATTERN_ID, + "IUIAutomationVirtualizedItemPattern") + if pattern is None: + return + try: + pattern.Realize() + except (OSError, AttributeError): + pass + + def find_virtual_item(self, item_name=None, by="name", container_name=None, + container_role=None, app_name=None, automation_id=None): + container = self._find_raw(container_name, container_role, app_name, + automation_id) + pattern = self._pattern(container, _UIA_ITEMCONTAINER_PATTERN_ID, + "IUIAutomationItemContainerPattern" + ) if container else None + if pattern is None: + return None + property_id = (_UIA_AUTOMATIONID_PROPERTY if by == "automation_id" + else _UIA_NAME_PROPERTY) + try: + found = pattern.FindItemByProperty(None, property_id, item_name) + except (OSError, AttributeError, ValueError): + return None + if not found: + return None + self._realize(found) + return _convert_uia(found) + def _text_pattern(self, name, role, app_name, automation_id): """Find a control and return its IUIAutomationTextPattern, or None.""" raw = self._find_raw(name, role, app_name, automation_id) diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index edff6eda..be846d81 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2462,6 +2462,20 @@ def _scroll_control_into_view(name: Optional[str] = None, role: Optional[str] = automation_id=automation_id) +def _realize_item(item_name: str, by: str = "name", + container_name: Optional[str] = None, + container_role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Dict[str, Any]: + """Adapter: find + realize a virtualized list/grid item (VirtualizedItem).""" + from je_auto_control.utils.virtualized import realize_item + element = realize_item(item_name, by=str(by), container_name=container_name, + container_role=container_role, app_name=app_name, + automation_id=automation_id) + return {"found": element is not None, + "element": element.to_dict() if element else None} + + def _get_control_text(name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, automation_id: Optional[str] = None) -> Dict[str, Any]: @@ -6405,6 +6419,7 @@ def __init__(self): "AC_control_range": _control_range, "AC_set_control_range": _set_control_range, "AC_scroll_control_into_view": _scroll_control_into_view, + "AC_realize_item": _realize_item, "AC_get_control_text": _get_control_text, "AC_get_selected_text": _get_selected_text, "AC_get_visible_text": _get_visible_text, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index d32d15ae..fb805544 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1157,6 +1157,27 @@ def a11y_control_tools() -> List[MCPTool]: handler=h.scroll_control_into_view, annotations=DESTRUCTIVE, ), + MCPTool( + name="ac_realize_item", + description=("Find and REALIZE an off-screen item in a virtualized " + "list/grid (ItemContainer + VirtualizedItem patterns) so " + "it materializes as a real element — rows that aren't " + "scrolled into view have no element until realized. " + "'item_name' matched by 'name' (default) or " + "'automation_id'; the container by container_name/" + "container_role/app_name/automation_id. Returns " + "{found, element}."), + input_schema=schema({ + "item_name": {"type": "string"}, + "by": {"type": "string", "enum": ["name", "automation_id"]}, + "container_name": {"type": "string"}, + "container_role": {"type": "string"}, + "app_name": {"type": "string"}, + "automation_id": {"type": "string"}}, + required=["item_name"]), + handler=h.realize_item, + annotations=READ_ONLY, + ), MCPTool( name="ac_get_control_text", description=("Read a control's full text via TextPattern: " diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index ab6d91cc..e9caff10 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -795,6 +795,13 @@ def get_control_text(name=None, role=None, app_name=None, automation_id=None): return _get_control_text(name, role, app_name, automation_id) +def realize_item(item_name, by="name", container_name=None, container_role=None, + app_name=None, automation_id=None): + from je_auto_control.utils.executor.action_executor import _realize_item + return _realize_item(item_name, by, container_name, container_role, + app_name, automation_id) + + def get_selected_text(name=None, role=None, app_name=None, automation_id=None): from je_auto_control.utils.executor.action_executor import _get_selected_text return _get_selected_text(name, role, app_name, automation_id) diff --git a/je_auto_control/utils/virtualized/__init__.py b/je_auto_control/utils/virtualized/__init__.py new file mode 100644 index 00000000..8a010951 --- /dev/null +++ b/je_auto_control/utils/virtualized/__init__.py @@ -0,0 +1,4 @@ +"""Realize off-screen items in virtualized lists / grids (ItemContainer + VirtualizedItem).""" +from je_auto_control.utils.virtualized.virtualized import realize_item + +__all__ = ["realize_item"] diff --git a/je_auto_control/utils/virtualized/virtualized.py b/je_auto_control/utils/virtualized/virtualized.py new file mode 100644 index 00000000..7615ea34 --- /dev/null +++ b/je_auto_control/utils/virtualized/virtualized.py @@ -0,0 +1,39 @@ +"""Realize off-screen items in virtualized lists / grids / trees. + +Long lists, data grids and trees (WPF / WinUI / Explorer / virtual treeviews) +only materialize the rows that are scrolled into view — a row that is off-screen +has *no* accessibility element at all, so ``list_accessibility_elements`` / +``read_control_table`` / ``select_control_item`` simply cannot see it, and +``scroll_control_into_view`` can't help because the target doesn't exist yet. + +``realize_item`` closes that gap: it locates the item inside its container by +property (UI Automation ``ItemContainerPattern``) and realizes it +(``VirtualizedItemPattern``) so it materializes as a real element you can then +click / read. It is a thin dispatch onto the injectable +``accessibility.backends.get_backend()`` seam — headless-testable on any platform +by injecting a fake backend; the real UIA calls live in the Windows backend. +Imports no ``PySide6``. +""" +from typing import Optional + +from je_auto_control.utils.accessibility.element import AccessibilityElement + + +def realize_item(item_name: str, *, by: str = "name", + container_name: Optional[str] = None, + container_role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[AccessibilityElement]: + """Find and realize a virtualized item, returning its element (or None). + + ``item_name`` is matched against the item's Name (``by="name"``) or its + AutomationId (``by="automation_id"``). The container is located by + ``container_name`` / ``container_role`` / ``app_name`` / ``automation_id`` + (the same matchers as the other native-control actions). + """ + from je_auto_control.utils.accessibility.backends import get_backend + return get_backend().find_virtual_item( + item_name, by=by, container_name=container_name, + container_role=container_role, app_name=app_name, + automation_id=automation_id) diff --git a/test/unit_test/headless/test_virtualized_batch.py b/test/unit_test/headless/test_virtualized_batch.py new file mode 100644 index 00000000..b01c6a65 --- /dev/null +++ b/test/unit_test/headless/test_virtualized_batch.py @@ -0,0 +1,90 @@ +"""Headless tests for realizing virtualized list/grid items (fake backend seam).""" +import je_auto_control as ac +from je_auto_control.utils.accessibility.backends import base as backend_base +from je_auto_control.utils.accessibility.element import AccessibilityElement +from je_auto_control.utils.virtualized import realize_item + + +class _FakeBackend(backend_base.AccessibilityBackend): + name = "fake" + available = True + + def __init__(self, items=None): + # items: {name: AccessibilityElement} the container can realize + self.items = items or {} + self.calls = [] + + def find_virtual_item(self, item_name=None, by="name", container_name=None, + container_role=None, app_name=None, automation_id=None): + self.calls.append({"item_name": item_name, "by": by, + "container_name": container_name}) + return self.items.get(item_name) + + +def _inject(monkeypatch, backend): + import je_auto_control.utils.accessibility.backends as backends + monkeypatch.setattr(backends, "_cached_backend", backend, raising=False) + + +def test_realize_item_returns_realized_element(monkeypatch): + row = AccessibilityElement(name="Order 5000", role="ControlType_50007", + bounds=(10, 900, 200, 24), app_name="grid.exe") + fake = _FakeBackend({"Order 5000": row}) + _inject(monkeypatch, fake) + element = realize_item("Order 5000", container_name="Orders") + assert element is row + assert fake.calls[0]["item_name"] == "Order 5000" + assert fake.calls[0]["by"] == "name" + assert fake.calls[0]["container_name"] == "Orders" + + +def test_realize_item_not_found_returns_none(monkeypatch): + _inject(monkeypatch, _FakeBackend()) + assert realize_item("missing-row", container_name="Orders") is None + + +def test_realize_item_by_automation_id(monkeypatch): + fake = _FakeBackend() + _inject(monkeypatch, fake) + realize_item("row-42", by="automation_id", container_name="Grid") + assert fake.calls[0]["by"] == "automation_id" + + +def test_unsupported_backend_raises(monkeypatch): + from je_auto_control.utils.accessibility.element import ( + AccessibilityNotAvailableError) + _inject(monkeypatch, backend_base.AccessibilityBackend()) # all _unsupported + try: + realize_item("x") + raised = False + except AccessibilityNotAvailableError: + raised = True + assert raised is True + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_wraps_element(monkeypatch): + row = AccessibilityElement(name="Row 9", role="ControlType_50007", + bounds=(1, 2, 3, 4), app_name="x") + _inject(monkeypatch, _FakeBackend({"Row 9": row})) + from je_auto_control.utils.executor.action_executor import _realize_item + out = _realize_item("Row 9", container_name="List") + assert out["found"] is True + assert out["element"]["name"] == "Row 9" + assert _realize_item("nope")["found"] is False + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert "AC_realize_item" in known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_realize_item" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_realize_item" in specs + + +def test_facade_export(): + assert hasattr(ac, "realize_item") and "realize_item" in ac.__all__ From 0c76ec5f8b2c0d507b50a47e85083849ded92ab5 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 25 Jun 2026 02:44:13 +0800 Subject: [PATCH 2/3] Add ax_props: read rich UIA element properties before acting The flat element list carries only name/role/bounds/app/id, but automation needs more before it acts: is the control enabled (don't click a disabled button), is it off-screen, its item_status, help_text (tooltip), accelerator_key. get_element_properties reads those UIA properties; is_element_enabled is the common pre-action guard. Extends the backend ABC + Windows UIA backend via the same fake-backend seam. --- WHATS_NEW.md | 6 ++ .../doc/new_features/v196_features_doc.rst | 44 +++++++++ .../Zh/doc/new_features/v196_features_doc.rst | 40 ++++++++ je_auto_control/__init__.py | 5 + .../gui/script_builder/command_schema.py | 5 + .../utils/accessibility/backends/base.py | 14 +++ .../accessibility/backends/windows_backend.py | 34 +++++++ je_auto_control/utils/ax_props/__init__.py | 6 ++ je_auto_control/utils/ax_props/ax_props.py | 38 ++++++++ .../utils/executor/action_executor.py | 11 +++ .../utils/mcp_server/tools/_factories.py | 10 ++ .../utils/mcp_server/tools/_handlers.py | 7 ++ .../unit_test/headless/test_ax_props_batch.py | 93 +++++++++++++++++++ 13 files changed, 313 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v196_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v196_features_doc.rst create mode 100644 je_auto_control/utils/ax_props/__init__.py create mode 100644 je_auto_control/utils/ax_props/ax_props.py create mode 100644 test/unit_test/headless/test_ax_props_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 364d9264..c8bf18ea 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-25) — Rich UIA Element Properties + +Know if a control is enabled / off-screen / has a tooltip before you act. Full reference: [`docs/source/Eng/doc/new_features/v196_features_doc.rst`](docs/source/Eng/doc/new_features/v196_features_doc.rst). + +- **`get_element_properties` / `is_element_enabled`** (`AC_get_element_properties`): the flat element list carries only name/role/bounds/app/id, but automation needs more before it acts — **is the control enabled** (don't click a disabled button), **is it off-screen**, its **item_status** (field validation/error), **help_text** (tooltip), and **accelerator_key** (drive via hotkey). This reads those high-value UIA properties (`enabled`/`offscreen`/`help_text`/`item_status`/`accelerator_key`/`access_key`/`orientation`); `is_element_enabled` is the common pre-action guard. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA reads in the Windows backend). No `PySide6`. + ## What's new (2026-06-25) — Realize Off-Screen Items in Virtualized Lists / Grids Reach a row that isn't scrolled into view yet — the "element not found in a long list" fix. Full reference: [`docs/source/Eng/doc/new_features/v195_features_doc.rst`](docs/source/Eng/doc/new_features/v195_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v196_features_doc.rst b/docs/source/Eng/doc/new_features/v196_features_doc.rst new file mode 100644 index 00000000..3a097a4d --- /dev/null +++ b/docs/source/Eng/doc/new_features/v196_features_doc.rst @@ -0,0 +1,44 @@ +Rich UIA Element Properties +=========================== + +``list_accessibility_elements`` / ``AccessibilityElement`` carry only name / role / +bounds / app / pid / automation_id. Automation routinely needs more *before it +acts*: **is the control enabled** (don't click a disabled button), **is it +off-screen** (is it really visible?), its **item_status** (validation / error text +on a field), **help_text** (tooltip), and **accelerator_key** (drive it via a +hotkey instead of a click). ``ax_props`` exposes those high-value UIA properties. + +* :func:`get_element_properties` — the full property dict, +* :func:`is_element_enabled` — the common pre-action guard. + +Each function is a thin dispatch onto the injectable +``accessibility.backends.get_backend()`` seam — headless-testable on any platform +by injecting a fake backend; the real UIA property reads live in the Windows +backend. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import get_element_properties, is_element_enabled + + get_element_properties(name="Save", role="button") + # {"enabled": False, "offscreen": False, "help_text": "Save the file", + # "item_status": "", "accelerator_key": "Ctrl+S", "access_key": "S", + # "orientation": 0} + + if is_element_enabled(name="Submit"): + click_text("Submit") # don't click a disabled button + +The control is located by ``name`` / ``role`` / ``app_name`` / ``automation_id`` +(same as the other native-control reads). ``get_element_properties`` returns the +property dict or ``None`` when the control isn't found; ``is_element_enabled`` +returns the ``enabled`` flag (or ``None`` if not found). + +Executor commands +----------------- + +``AC_get_element_properties`` returns ``{found, properties}``. It is exposed as the +read-only ``ac_get_element_properties`` MCP tool and as a Script Builder command +under **Native UI**. diff --git a/docs/source/Zh/doc/new_features/v196_features_doc.rst b/docs/source/Zh/doc/new_features/v196_features_doc.rst new file mode 100644 index 00000000..a13380c9 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v196_features_doc.rst @@ -0,0 +1,40 @@ +豐富的 UIA 元素屬性 +=================== + +``list_accessibility_elements`` / ``AccessibilityElement``只帶有 name / role / bounds / +app / pid / automation_id。自動化在*動作之前*常需要更多資訊:**控制項是否啟用**(別點停用的 +按鈕)、**是否在畫面外**(是否真的可見)、其 **item_status**(欄位的驗證 / 錯誤文字)、 +**help_text**(工具提示),以及 **accelerator_key**(以快捷鍵而非點擊來驅動它)。``ax_props`` +就提供這些高價值的 UIA 屬性。 + +* :func:`get_element_properties` ——完整的屬性字典, +* :func:`is_element_enabled` ——常見的動作前守衛。 + +每個函式都是對可注入的 ``accessibility.backends.get_backend()`` 接縫的薄分派——可在任何平台透過 +注入 fake backend 進行無頭測試;真正的 UIA 屬性讀取位於 Windows 後端。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import get_element_properties, is_element_enabled + + get_element_properties(name="Save", role="button") + # {"enabled": False, "offscreen": False, "help_text": "Save the file", + # "item_status": "", "accelerator_key": "Ctrl+S", "access_key": "S", + # "orientation": 0} + + if is_element_enabled(name="Submit"): + click_text("Submit") # 別點停用的按鈕 + +控制項以 ``name`` / ``role`` / ``app_name`` / ``automation_id`` 定位(與其他原生控制讀取相同)。 +``get_element_properties`` 回傳屬性字典,找不到控制項時回傳 ``None``;``is_element_enabled`` +回傳 ``enabled`` 旗標(找不到則為 ``None``)。 + +執行器指令 +---------- + +``AC_get_element_properties`` 回傳 ``{found, properties}``。以唯讀 +``ac_get_element_properties`` MCP 工具及 Script Builder 指令(位於 **Native UI** 分類下) +形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 4113e55c..6e0a0d26 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -68,6 +68,10 @@ ) # Realize off-screen items in virtualized lists / grids (UIA VirtualizedItem) from je_auto_control.utils.virtualized import realize_item +# Rich UIA element properties (enabled / offscreen / help / status / keys) +from je_auto_control.utils.ax_props import ( + get_element_properties, is_element_enabled, +) # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1669,6 +1673,7 @@ def start_autocontrol_gui(*args, **kwargs): "assign_node_paths", "find_by_path", "is_interactive_role", "tab_order", "audit_focus_order", "focus_control", "realize_item", + "get_element_properties", "is_element_enabled", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index c407d839..18746e54 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1621,6 +1621,11 @@ def _add_native_control_specs(specs: List[CommandSpec]) -> None: ), description="Realize an off-screen item in a virtualized list/grid.", )) + specs.append(CommandSpec( + "AC_get_element_properties", "Native UI", "Get Element Properties", + fields=fields, + description="Read rich UIA props (enabled/offscreen/help/status/keys).", + )) specs.append(CommandSpec( "AC_get_control_text", "Native UI", "Get Control Text", fields=fields, diff --git a/je_auto_control/utils/accessibility/backends/base.py b/je_auto_control/utils/accessibility/backends/base.py index 2f40c8af..9379c68b 100644 --- a/je_auto_control/utils/accessibility/backends/base.py +++ b/je_auto_control/utils/accessibility/backends/base.py @@ -148,6 +148,20 @@ def find_virtual_item(self, item_name: Optional[str] = None, by: str = "name", """ self._unsupported("find_virtual_item") + # --- rich element properties ------------------------------------------- + + def get_properties(self, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Return rich UIA properties of the matched control, or None. + + Surfaces the high-value properties the flat element list omits — + ``enabled`` / ``offscreen`` / ``help_text`` / ``item_status`` / + ``accelerator_key`` / ``access_key`` / ``orientation``. + """ + self._unsupported("get_properties") + def _unsupported(self, operation: str): """Raise a clear error for an action this backend can't perform.""" raise AccessibilityNotAvailableError( diff --git a/je_auto_control/utils/accessibility/backends/windows_backend.py b/je_auto_control/utils/accessibility/backends/windows_backend.py index 0d0594fc..a11394ae 100644 --- a/je_auto_control/utils/accessibility/backends/windows_backend.py +++ b/je_auto_control/utils/accessibility/backends/windows_backend.py @@ -298,6 +298,13 @@ def find_virtual_item(self, item_name=None, by="name", container_name=None, self._realize(found) return _convert_uia(found) + def get_properties(self, name=None, role=None, app_name=None, + automation_id=None) -> Optional[Dict[str, Any]]: + raw = self._find_raw(name, role, app_name, automation_id) + if not raw: + return None + return _read_properties(raw) + def _text_pattern(self, name, role, app_name, automation_id): """Find a control and return its IUIAutomationTextPattern, or None.""" raw = self._find_raw(name, role, app_name, automation_id) @@ -366,6 +373,33 @@ def _read_row(pattern, row: int, cols: int): return cells +def _as_text(value) -> str: + return str(value or "") + + +# (key, UIA element attribute, cast) for the rich properties the flat list omits. +_PROPERTY_READS = ( + ("enabled", "CurrentIsEnabled", bool), + ("offscreen", "CurrentIsOffscreen", bool), + ("help_text", "CurrentHelpText", _as_text), + ("item_status", "CurrentItemStatus", _as_text), + ("accelerator_key", "CurrentAcceleratorKey", _as_text), + ("access_key", "CurrentAccessKey", _as_text), + ("orientation", "CurrentOrientation", int), +) + + +def _read_properties(raw) -> Dict[str, Any]: + """Read the rich UIA properties of a raw element into a plain dict.""" + properties: Dict[str, Any] = {} + for key, attribute, cast in _PROPERTY_READS: + try: + properties[key] = cast(getattr(raw, attribute)) + except (OSError, AttributeError, ValueError, TypeError): + properties[key] = None + return properties + + def _convert_uia(raw) -> Optional[AccessibilityElement]: try: name = str(raw.CurrentName or "") diff --git a/je_auto_control/utils/ax_props/__init__.py b/je_auto_control/utils/ax_props/__init__.py new file mode 100644 index 00000000..9d3bcebb --- /dev/null +++ b/je_auto_control/utils/ax_props/__init__.py @@ -0,0 +1,6 @@ +"""Read rich UIA element properties (enabled / offscreen / help / status / keys).""" +from je_auto_control.utils.ax_props.ax_props import ( + get_element_properties, is_element_enabled, +) + +__all__ = ["get_element_properties", "is_element_enabled"] diff --git a/je_auto_control/utils/ax_props/ax_props.py b/je_auto_control/utils/ax_props/ax_props.py new file mode 100644 index 00000000..8e185325 --- /dev/null +++ b/je_auto_control/utils/ax_props/ax_props.py @@ -0,0 +1,38 @@ +"""Read rich UI Automation properties the flat element list omits. + +``list_accessibility_elements`` / ``AccessibilityElement`` carry only name / role / +bounds / app / pid / automation_id. Automation routinely needs more before it +acts: **is the control enabled** (don't click a disabled button), **is it +off-screen** (is it really visible?), its **item_status** (validation / error text +on a field), **help_text** (tooltip), and **accelerator_key** (drive it via a +hotkey instead of a click). ``ax_props`` exposes those high-value UIA properties. + +Each function is a thin dispatch onto the injectable +``accessibility.backends.get_backend()`` seam, so the headless core is +unit-testable on any platform by injecting a fake backend; the real UIA property +reads live in the Windows backend. Imports no ``PySide6``. +""" +from typing import Any, Dict, Optional + + +def get_element_properties(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Return ``{enabled, offscreen, help_text, item_status, accelerator_key, + access_key, orientation}`` for the matched control, or None if not found.""" + from je_auto_control.utils.accessibility.backends import get_backend + return get_backend().get_properties(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def is_element_enabled(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Optional[bool]: + """Return whether the matched control is enabled (None if not found). + + The common guard before acting — don't click a disabled button. + """ + properties = get_element_properties(name=name, role=role, app_name=app_name, + automation_id=automation_id) + return properties.get("enabled") if properties else None diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index be846d81..487bf76d 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2476,6 +2476,16 @@ def _realize_item(item_name: str, by: str = "name", "element": element.to_dict() if element else None} +def _get_element_properties(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Dict[str, Any]: + """Adapter: read rich UIA properties (enabled/offscreen/help/status/keys).""" + from je_auto_control.utils.ax_props import get_element_properties + props = get_element_properties(name=name, role=role, app_name=app_name, + automation_id=automation_id) + return {"found": props is not None, "properties": props} + + def _get_control_text(name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, automation_id: Optional[str] = None) -> Dict[str, Any]: @@ -6420,6 +6430,7 @@ def __init__(self): "AC_set_control_range": _set_control_range, "AC_scroll_control_into_view": _scroll_control_into_view, "AC_realize_item": _realize_item, + "AC_get_element_properties": _get_element_properties, "AC_get_control_text": _get_control_text, "AC_get_selected_text": _get_selected_text, "AC_get_visible_text": _get_visible_text, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index fb805544..6369d8d2 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1178,6 +1178,16 @@ def a11y_control_tools() -> List[MCPTool]: handler=h.realize_item, annotations=READ_ONLY, ), + MCPTool( + name="ac_get_element_properties", + description=("Read rich UIA properties the flat list omits: " + "{found, properties:{enabled, offscreen, help_text, " + "item_status, accelerator_key, access_key, " + "orientation}}. Check 'enabled' before clicking."), + input_schema=schema(dict(_M)), + handler=h.get_element_properties, + annotations=READ_ONLY, + ), MCPTool( name="ac_get_control_text", description=("Read a control's full text via TextPattern: " diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index e9caff10..7942f130 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -802,6 +802,13 @@ def realize_item(item_name, by="name", container_name=None, container_role=None, app_name, automation_id) +def get_element_properties(name=None, role=None, app_name=None, + automation_id=None): + from je_auto_control.utils.executor.action_executor import ( + _get_element_properties) + return _get_element_properties(name, role, app_name, automation_id) + + def get_selected_text(name=None, role=None, app_name=None, automation_id=None): from je_auto_control.utils.executor.action_executor import _get_selected_text return _get_selected_text(name, role, app_name, automation_id) diff --git a/test/unit_test/headless/test_ax_props_batch.py b/test/unit_test/headless/test_ax_props_batch.py new file mode 100644 index 00000000..cf253988 --- /dev/null +++ b/test/unit_test/headless/test_ax_props_batch.py @@ -0,0 +1,93 @@ +"""Headless tests for rich UIA element property reads (fake backend seam).""" +import je_auto_control as ac +from je_auto_control.utils.accessibility.backends import base as backend_base +from je_auto_control.utils.ax_props import ( + get_element_properties, is_element_enabled, +) + +_PROPS = {"enabled": False, "offscreen": True, "help_text": "Save the file", + "item_status": "invalid", "accelerator_key": "Ctrl+S", + "access_key": "S", "orientation": 0} + + +class _FakeBackend(backend_base.AccessibilityBackend): + name = "fake" + available = True + + def __init__(self, props=None): + self.props = props + self.calls = [] + + def get_properties(self, name=None, role=None, app_name=None, + automation_id=None): + self.calls.append({"name": name, "role": role}) + return self.props + + +def _inject(monkeypatch, backend): + import je_auto_control.utils.accessibility.backends as backends + monkeypatch.setattr(backends, "_cached_backend", backend, raising=False) + + +def test_get_element_properties_dispatch(monkeypatch): + fake = _FakeBackend(dict(_PROPS)) + _inject(monkeypatch, fake) + props = get_element_properties(name="Save", role="button") + assert props == _PROPS + assert fake.calls[0] == {"name": "Save", "role": "button"} + + +def test_get_element_properties_not_found(monkeypatch): + _inject(monkeypatch, _FakeBackend(None)) + assert get_element_properties(name="missing") is None + + +def test_is_element_enabled_reads_flag(monkeypatch): + _inject(monkeypatch, _FakeBackend(dict(_PROPS))) + assert is_element_enabled(name="Save") is False + _inject(monkeypatch, _FakeBackend({"enabled": True})) + assert is_element_enabled(name="OK") is True + + +def test_is_element_enabled_none_when_not_found(monkeypatch): + _inject(monkeypatch, _FakeBackend(None)) + assert is_element_enabled(name="x") is None + + +def test_unsupported_backend_raises(monkeypatch): + from je_auto_control.utils.accessibility.element import ( + AccessibilityNotAvailableError) + _inject(monkeypatch, backend_base.AccessibilityBackend()) # all _unsupported + try: + get_element_properties(name="x") + raised = False + except AccessibilityNotAvailableError: + raised = True + assert raised is True + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapter_wraps_props(monkeypatch): + _inject(monkeypatch, _FakeBackend(dict(_PROPS))) + from je_auto_control.utils.executor.action_executor import ( + _get_element_properties) + out = _get_element_properties(name="Save") + assert out["found"] is True + assert out["properties"]["accelerator_key"] == "Ctrl+S" + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert "AC_get_element_properties" in known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert "ac_get_element_properties" in names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert "AC_get_element_properties" in specs + + +def test_facade_exports(): + for name in ("get_element_properties", "is_element_enabled"): + assert hasattr(ac, name) and name in ac.__all__ From fa11c4b19da0872d99e51d88b69032075850e536 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 25 Jun 2026 02:54:44 +0800 Subject: [PATCH 3/3] Add table_pattern: table headers + cell addressing for native grids read_control_table dumps a flat 2D list of cell names with no header labels and no way to address one cell by (header, row). Add the missing half: table_headers reads row/column header labels (TablePattern), table_cell reads the cell at (row, column) with its span (GridItemPattern), and cell_by_header resolves the column index from the headers so you can read the cell at (row, named column). Extends the backend ABC + Windows UIA backend via the same fake-backend seam. --- WHATS_NEW.md | 6 + .../doc/new_features/v197_features_doc.rst | 46 ++++++++ .../Zh/doc/new_features/v197_features_doc.rst | 42 +++++++ je_auto_control/__init__.py | 5 + .../gui/script_builder/command_schema.py | 17 +++ .../utils/accessibility/backends/base.py | 19 ++++ .../accessibility/backends/windows_backend.py | 72 ++++++++++++ .../utils/executor/action_executor.py | 33 ++++++ .../utils/mcp_server/tools/_factories.py | 31 +++++ .../utils/mcp_server/tools/_handlers.py | 18 +++ .../utils/table_pattern/__init__.py | 6 + .../utils/table_pattern/table_pattern.py | 59 ++++++++++ .../headless/test_table_pattern_batch.py | 107 ++++++++++++++++++ 13 files changed, 461 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v197_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v197_features_doc.rst create mode 100644 je_auto_control/utils/table_pattern/__init__.py create mode 100644 je_auto_control/utils/table_pattern/table_pattern.py create mode 100644 test/unit_test/headless/test_table_pattern_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index c8bf18ea..bb61e333 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,11 @@ # What's New — AutoControl +## What's new (2026-06-25) — Table Headers + Cell Addressing (UIA TablePattern) + +Assert "the Status column of row 5 says Shipped" — by header, not by guessing indices. Full reference: [`docs/source/Eng/doc/new_features/v197_features_doc.rst`](docs/source/Eng/doc/new_features/v197_features_doc.rst). + +- **`table_headers` / `table_cell` / `cell_by_header`** (`AC_table_headers`, `AC_table_cell`, `AC_cell_by_header`): `read_control_table` (GridPattern) dumps a flat 2-D list of cell names with no header labels and no way to address one cell by (header, row) — you can dump a grid but not test one. This adds the missing half: `table_headers` reads the row/column header labels (TablePattern), `table_cell` reads the cell at `(row, column)` with its span (GridItemPattern), and `cell_by_header` resolves the column index from the headers so you can read the cell at `(row, "Status")` directly. Dispatched through the injectable accessibility backend seam (headless-testable via a fake backend; real UIA in the Windows backend). No `PySide6`. + ## What's new (2026-06-25) — Rich UIA Element Properties Know if a control is enabled / off-screen / has a tooltip before you act. Full reference: [`docs/source/Eng/doc/new_features/v196_features_doc.rst`](docs/source/Eng/doc/new_features/v196_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v197_features_doc.rst b/docs/source/Eng/doc/new_features/v197_features_doc.rst new file mode 100644 index 00000000..eb70ad37 --- /dev/null +++ b/docs/source/Eng/doc/new_features/v197_features_doc.rst @@ -0,0 +1,46 @@ +Table Headers + Cell Addressing (UIA TablePattern) +================================================== + +``read_control_table`` (GridPattern) dumps a flat 2-D list of cell names with **no +header labels and no way to address a single cell by (header, row)** — so you can +*dump* a grid but not actually *test* one. ``table_pattern`` adds the missing +half: + +* :func:`table_headers` — the row / column header labels (TablePattern), +* :func:`table_cell` — the cell at ``(row, column)`` with its span + (GridPattern.GetItem + GridItemPattern), +* :func:`cell_by_header` — read the cell at ``(row, "Column Header")``, so you can + assert "the Status column of row 5 says Shipped" without guessing indices. + +Each is a thin dispatch onto the injectable ``accessibility.backends.get_backend()`` +seam — headless-testable by injecting a fake backend; the real UIA calls live in +the Windows backend. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import table_headers, table_cell, cell_by_header + + table_headers(name="Orders") + # {"columns": ["Order", "Status", "Total"], "rows": [...]} + + table_cell(0, 1, name="Orders") + # {"value": "Shipped", "row": 0, "column": 1, "row_span": 1, "column_span": 1} + + cell_by_header(0, "Status", name="Orders") # "Shipped" + +The table is located by ``name`` / ``role`` / ``app_name`` / ``automation_id`` +(same as ``read_control_table``). ``table_headers`` returns +``{columns, rows}`` (or ``None``); ``table_cell`` returns the cell record (or +``None``); ``cell_by_header`` resolves the column index from the headers and +returns the cell value (``None`` if the header or cell isn't found). + +Executor commands +----------------- + +``AC_table_headers`` (``{found, headers}``), ``AC_table_cell`` (``row`` / +``column`` → ``{found, cell}``) and ``AC_cell_by_header`` (``row`` / +``column_header`` → ``{found, value}``). They are exposed as read-only ``ac_*`` +MCP tools and as Script Builder commands under **Native UI**. diff --git a/docs/source/Zh/doc/new_features/v197_features_doc.rst b/docs/source/Zh/doc/new_features/v197_features_doc.rst new file mode 100644 index 00000000..a2eb05b1 --- /dev/null +++ b/docs/source/Zh/doc/new_features/v197_features_doc.rst @@ -0,0 +1,42 @@ +表頭 + 儲存格定址(UIA TablePattern) +==================================== + +``read_control_table``(GridPattern)輸出的是一份扁平的 2D 儲存格名稱清單,**沒有表頭標籤,也無法 +以(表頭, 列)定址單一儲存格**——所以你能*傾印*一個格線,卻無法真正*測試*它。``table_pattern`` +補上所缺的另一半: + +* :func:`table_headers` ——列 / 欄的表頭標籤(TablePattern), +* :func:`table_cell` ——位於 ``(row, column)`` 的儲存格及其跨距 + (GridPattern.GetItem + GridItemPattern), +* :func:`cell_by_header` ——讀取位於 ``(row, "欄表頭")`` 的儲存格,於是你可以斷言「第 5 列的 + Status 欄是 Shipped」,而不必猜索引。 + +每個都是對可注入的 ``accessibility.backends.get_backend()`` 接縫的薄分派——可透過注入 fake backend +進行無頭測試;真正的 UIA 呼叫位於 Windows 後端。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import table_headers, table_cell, cell_by_header + + table_headers(name="Orders") + # {"columns": ["Order", "Status", "Total"], "rows": [...]} + + table_cell(0, 1, name="Orders") + # {"value": "Shipped", "row": 0, "column": 1, "row_span": 1, "column_span": 1} + + cell_by_header(0, "Status", name="Orders") # "Shipped" + +表格以 ``name`` / ``role`` / ``app_name`` / ``automation_id`` 定位(與 ``read_control_table`` +相同)。``table_headers`` 回傳 ``{columns, rows}``(或 ``None``);``table_cell`` 回傳儲存格記錄 +(或 ``None``);``cell_by_header`` 由表頭解析欄索引並回傳儲存格值(找不到表頭或儲存格則為 +``None``)。 + +執行器指令 +---------- + +``AC_table_headers``(``{found, headers}``)、``AC_table_cell``(``row`` / ``column`` → +``{found, cell}``)與 ``AC_cell_by_header``(``row`` / ``column_header`` → ``{found, value}``)。 +皆以唯讀 ``ac_*`` MCP 工具及 Script Builder 指令(位於 **Native UI** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 6e0a0d26..6ae62724 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -72,6 +72,10 @@ from je_auto_control.utils.ax_props import ( get_element_properties, is_element_enabled, ) +# Table headers + cell addressing for native grids (UIA TablePattern) +from je_auto_control.utils.table_pattern import ( + cell_by_header, table_cell, table_headers, +) # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1674,6 +1678,7 @@ def start_autocontrol_gui(*args, **kwargs): "is_interactive_role", "tab_order", "audit_focus_order", "focus_control", "realize_item", "get_element_properties", "is_element_enabled", + "table_headers", "table_cell", "cell_by_header", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 18746e54..ebbaf161 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1626,6 +1626,23 @@ def _add_native_control_specs(specs: List[CommandSpec]) -> None: fields=fields, description="Read rich UIA props (enabled/offscreen/help/status/keys).", )) + specs.append(CommandSpec( + "AC_table_headers", "Native UI", "Get Table Headers", + fields=fields, + description="Read a table's row/column header labels (TablePattern).", + )) + specs.append(CommandSpec( + "AC_table_cell", "Native UI", "Get Table Cell (by index)", + fields=(FieldSpec("row", FieldType.INT), + FieldSpec("column", FieldType.INT)) + fields, + description="Read the cell at (row, column) with its span.", + )) + specs.append(CommandSpec( + "AC_cell_by_header", "Native UI", "Get Table Cell (by header)", + fields=(FieldSpec("row", FieldType.INT), + FieldSpec("column_header", FieldType.STRING)) + fields, + description="Read the cell at (row, named column) — assert by header.", + )) specs.append(CommandSpec( "AC_get_control_text", "Native UI", "Get Control Text", fields=fields, diff --git a/je_auto_control/utils/accessibility/backends/base.py b/je_auto_control/utils/accessibility/backends/base.py index 9379c68b..2624d5d3 100644 --- a/je_auto_control/utils/accessibility/backends/base.py +++ b/je_auto_control/utils/accessibility/backends/base.py @@ -162,6 +162,25 @@ def get_properties(self, name: Optional[str] = None, """ self._unsupported("get_properties") + # --- table headers + cell addressing (TablePattern / GridItemPattern) --- + + def get_table_headers(self, name: Optional[str] = None, + role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Return a table's header labels as ``{columns: [...], rows: [...]}``.""" + self._unsupported("get_table_headers") + + def get_grid_cell(self, row: int = 0, column: int = 0, + name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Return the cell at ``(row, column)`` as ``{value, row, column, + row_span, column_span}`` (GridPattern.GetItem + GridItemPattern).""" + self._unsupported("get_grid_cell") + def _unsupported(self, operation: str): """Raise a clear error for an action this backend can't perform.""" raise AccessibilityNotAvailableError( diff --git a/je_auto_control/utils/accessibility/backends/windows_backend.py b/je_auto_control/utils/accessibility/backends/windows_backend.py index a11394ae..e46c69e9 100644 --- a/je_auto_control/utils/accessibility/backends/windows_backend.py +++ b/je_auto_control/utils/accessibility/backends/windows_backend.py @@ -32,6 +32,8 @@ _UIA_TEXT_PATTERN_ID = 10014 _UIA_ITEMCONTAINER_PATTERN_ID = 10019 _UIA_VIRTUALIZEDITEM_PATTERN_ID = 10020 +_UIA_TABLE_PATTERN_ID = 10012 +_UIA_GRIDITEM_PATTERN_ID = 10007 _UIA_AUTOMATIONID_PROPERTY = 30011 _EXPAND_STATES = {0: "collapsed", 1: "expanded", 2: "partial", 3: "leaf"} @@ -305,6 +307,37 @@ def get_properties(self, name=None, role=None, app_name=None, return None return _read_properties(raw) + def get_table_headers(self, name=None, role=None, app_name=None, + automation_id=None) -> Optional[Dict[str, Any]]: + raw = self._find_raw(name, role, app_name, automation_id) + pattern = self._pattern(raw, _UIA_TABLE_PATTERN_ID, + "IUIAutomationTablePattern") if raw else None + if pattern is None: + return None + try: + columns = pattern.GetCurrentColumnHeaders() + rows = pattern.GetCurrentRowHeaders() + except (OSError, AttributeError): + return None + return {"columns": _header_names(columns), "rows": _header_names(rows)} + + def get_grid_cell(self, row=0, column=0, name=None, role=None, + app_name=None, automation_id=None) -> Optional[Dict[str, Any]]: + raw = self._find_raw(name, role, app_name, automation_id) + grid = self._pattern(raw, _UIA_GRID_PATTERN_ID, + "IUIAutomationGridPattern") if raw else None + if grid is None: + return None + try: + cell = grid.GetItem(int(row), int(column)) + except (OSError, AttributeError): + return None + if not cell: + return None + return _read_cell(self._pattern(cell, _UIA_GRIDITEM_PATTERN_ID, + "IUIAutomationGridItemPattern"), + cell, int(row), int(column)) + def _text_pattern(self, name, role, app_name, automation_id): """Find a control and return its IUIAutomationTextPattern, or None.""" raw = self._find_raw(name, role, app_name, automation_id) @@ -373,6 +406,45 @@ def _read_row(pattern, row: int, cols: int): return cells +def _header_names(array) -> List[str]: + """Read an IUIAutomationElementArray of header elements into name strings.""" + names: List[str] = [] + try: + count = int(array.Length or 0) + except (OSError, AttributeError): + return names + for index in range(count): + try: + names.append(str(array.GetElement(index).CurrentName or "")) + except (OSError, AttributeError): + names.append("") + return names + + +def _read_cell(item_pattern, cell, row: int, column: int) -> Dict[str, Any]: + """Build a cell record, enriching with GridItemPattern row/col/span if present.""" + info: Dict[str, Any] = { + "value": _safe_name(cell), "row": row, "column": column, + "row_span": 1, "column_span": 1, + } + if item_pattern is not None: + for key, attr in (("row", "CurrentRow"), ("column", "CurrentColumn"), + ("row_span", "CurrentRowSpan"), + ("column_span", "CurrentColumnSpan")): + try: + info[key] = int(getattr(item_pattern, attr)) + except (OSError, AttributeError, ValueError, TypeError): + pass + return info + + +def _safe_name(raw) -> str: + try: + return str(raw.CurrentName or "") + except (OSError, AttributeError): + return "" + + def _as_text(value) -> str: return str(value or "") diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 487bf76d..14e5265b 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2486,6 +2486,36 @@ def _get_element_properties(name: Optional[str] = None, role: Optional[str] = No return {"found": props is not None, "properties": props} +def _table_headers(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Dict[str, Any]: + """Adapter: a table's row/column header labels (TablePattern).""" + from je_auto_control.utils.table_pattern import table_headers + headers = table_headers(name=name, role=role, app_name=app_name, + automation_id=automation_id) + return {"found": headers is not None, "headers": headers} + + +def _table_cell(row: Any, column: Any, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Dict[str, Any]: + """Adapter: the cell at (row, column) with its span (GridItemPattern).""" + from je_auto_control.utils.table_pattern import table_cell + cell = table_cell(int(row), int(column), name=name, role=role, + app_name=app_name, automation_id=automation_id) + return {"found": cell is not None, "cell": cell} + + +def _cell_by_header(row: Any, column_header: str, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Dict[str, Any]: + """Adapter: read the cell at (row, named column) — assert by header.""" + from je_auto_control.utils.table_pattern import cell_by_header + value = cell_by_header(int(row), str(column_header), name=name, role=role, + app_name=app_name, automation_id=automation_id) + return {"found": value is not None, "value": value} + + def _get_control_text(name: Optional[str] = None, role: Optional[str] = None, app_name: Optional[str] = None, automation_id: Optional[str] = None) -> Dict[str, Any]: @@ -6431,6 +6461,9 @@ def __init__(self): "AC_scroll_control_into_view": _scroll_control_into_view, "AC_realize_item": _realize_item, "AC_get_element_properties": _get_element_properties, + "AC_table_headers": _table_headers, + "AC_table_cell": _table_cell, + "AC_cell_by_header": _cell_by_header, "AC_get_control_text": _get_control_text, "AC_get_selected_text": _get_selected_text, "AC_get_visible_text": _get_visible_text, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 6369d8d2..9ddc7998 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -1188,6 +1188,37 @@ def a11y_control_tools() -> List[MCPTool]: handler=h.get_element_properties, annotations=READ_ONLY, ), + MCPTool( + name="ac_table_headers", + description=("Read a native table's header labels via TablePattern: " + "{found, headers:{columns:[...], rows:[...]}}."), + input_schema=schema(dict(_M)), + handler=h.table_headers, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_table_cell", + description=("Read the cell at ('row','column') of a native grid: " + "{found, cell:{value,row,column,row_span,column_span}} " + "(GridPattern.GetItem + GridItemPattern)."), + input_schema=schema({"row": {"type": "integer"}, + "column": {"type": "integer"}, **_M}, + required=["row", "column"]), + handler=h.table_cell, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_cell_by_header", + description=("Read the cell at row 'row' in the column named " + "'column_header' — assert 'the Status column of row 5 " + "says Shipped' without guessing indices. Returns " + "{found, value}."), + input_schema=schema({"row": {"type": "integer"}, + "column_header": {"type": "string"}, **_M}, + required=["row", "column_header"]), + handler=h.cell_by_header, + annotations=READ_ONLY, + ), MCPTool( name="ac_get_control_text", description=("Read a control's full text via TextPattern: " diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 7942f130..069f90eb 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -809,6 +809,24 @@ def get_element_properties(name=None, role=None, app_name=None, return _get_element_properties(name, role, app_name, automation_id) +def table_headers(name=None, role=None, app_name=None, automation_id=None): + from je_auto_control.utils.executor.action_executor import _table_headers + return _table_headers(name, role, app_name, automation_id) + + +def table_cell(row, column, name=None, role=None, app_name=None, + automation_id=None): + from je_auto_control.utils.executor.action_executor import _table_cell + return _table_cell(row, column, name, role, app_name, automation_id) + + +def cell_by_header(row, column_header, name=None, role=None, app_name=None, + automation_id=None): + from je_auto_control.utils.executor.action_executor import _cell_by_header + return _cell_by_header(row, column_header, name, role, app_name, + automation_id) + + def get_selected_text(name=None, role=None, app_name=None, automation_id=None): from je_auto_control.utils.executor.action_executor import _get_selected_text return _get_selected_text(name, role, app_name, automation_id) diff --git a/je_auto_control/utils/table_pattern/__init__.py b/je_auto_control/utils/table_pattern/__init__.py new file mode 100644 index 00000000..35962beb --- /dev/null +++ b/je_auto_control/utils/table_pattern/__init__.py @@ -0,0 +1,6 @@ +"""Table headers + cell addressing for native grids (UIA TablePattern / GridItem).""" +from je_auto_control.utils.table_pattern.table_pattern import ( + cell_by_header, table_cell, table_headers, +) + +__all__ = ["table_headers", "table_cell", "cell_by_header"] diff --git a/je_auto_control/utils/table_pattern/table_pattern.py b/je_auto_control/utils/table_pattern/table_pattern.py new file mode 100644 index 00000000..315b7a49 --- /dev/null +++ b/je_auto_control/utils/table_pattern/table_pattern.py @@ -0,0 +1,59 @@ +"""Table headers and cell addressing for native grids (UIA TablePattern). + +``read_control_table`` (GridPattern) dumps a flat 2-D list of cell names with **no +header labels and no way to address a single cell by (header, row)** — so you can +dump a grid but not actually *test* one. ``table_pattern`` adds the missing half: + +* :func:`table_headers` — the row / column header labels (TablePattern), +* :func:`table_cell` — the cell at ``(row, column)`` with its span (GridItemPattern), +* :func:`cell_by_header` — read the cell at ``(row, "Column Header")`` — so you can + assert "the Status column of row 5 says Shipped" without guessing indices. + +Each is a thin dispatch onto the injectable ``accessibility.backends.get_backend()`` +seam, so the headless core is unit-testable by injecting a fake backend; the real +UIA calls live in the Windows backend. Imports no ``PySide6``. +""" +from typing import Any, Dict, Optional + + +def _backend(): + from je_auto_control.utils.accessibility.backends import get_backend + return get_backend() + + +def table_headers(name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + """Return a table's header labels as ``{columns: [...], rows: [...]}``, or None.""" + return _backend().get_table_headers(name=name, role=role, app_name=app_name, + automation_id=automation_id) + + +def table_cell(row: int, column: int, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Return the cell at ``(row, column)`` as ``{value, row, column, row_span, + column_span}``, or None if the grid / cell isn't found.""" + return _backend().get_grid_cell(int(row), int(column), name=name, role=role, + app_name=app_name, automation_id=automation_id) + + +def cell_by_header(row: int, column_header: str, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> Optional[str]: + """Return the value of the cell at ``row`` in the column named ``column_header``. + + Resolves the column index from the table's headers, then reads the cell — + ``None`` if the header or cell isn't found. + """ + headers = table_headers(name=name, role=role, app_name=app_name, + automation_id=automation_id) + if not headers: + return None + columns = headers.get("columns", []) + if column_header not in columns: + return None + cell = table_cell(int(row), columns.index(column_header), name=name, + role=role, app_name=app_name, automation_id=automation_id) + return cell.get("value") if cell else None diff --git a/test/unit_test/headless/test_table_pattern_batch.py b/test/unit_test/headless/test_table_pattern_batch.py new file mode 100644 index 00000000..8be70aff --- /dev/null +++ b/test/unit_test/headless/test_table_pattern_batch.py @@ -0,0 +1,107 @@ +"""Headless tests for table headers + cell addressing (fake backend seam).""" +import je_auto_control as ac +from je_auto_control.utils.accessibility.backends import base as backend_base +from je_auto_control.utils.table_pattern import ( + cell_by_header, table_cell, table_headers, +) + +_HEADERS = {"columns": ["Order", "Status", "Total"], "rows": ["r0", "r1"]} +_GRID = { + (0, 0): "Order 5", (0, 1): "Shipped", (0, 2): "$10", + (1, 0): "Order 6", (1, 1): "Pending", (1, 2): "$20", +} + + +class _FakeBackend(backend_base.AccessibilityBackend): + name = "fake" + available = True + + def __init__(self, headers=None, grid=None): + self._headers = headers + self._grid = grid or {} + + def get_table_headers(self, name=None, role=None, app_name=None, + automation_id=None): + return self._headers + + def get_grid_cell(self, row=0, column=0, name=None, role=None, + app_name=None, automation_id=None): + if (row, column) not in self._grid: + return None + return {"value": self._grid[(row, column)], "row": row, "column": column, + "row_span": 1, "column_span": 1} + + +def _inject(monkeypatch, backend): + import je_auto_control.utils.accessibility.backends as backends + monkeypatch.setattr(backends, "_cached_backend", backend, raising=False) + + +def test_table_headers(monkeypatch): + _inject(monkeypatch, _FakeBackend(dict(_HEADERS))) + assert table_headers(name="Orders") == _HEADERS + + +def test_table_cell_by_index(monkeypatch): + _inject(monkeypatch, _FakeBackend(grid=dict(_GRID))) + cell = table_cell(0, 1, name="Orders") + assert cell["value"] == "Shipped" + assert cell["row"] == 0 and cell["column"] == 1 + assert table_cell(9, 9, name="Orders") is None + + +def test_cell_by_header_resolves_column(monkeypatch): + _inject(monkeypatch, _FakeBackend(dict(_HEADERS), dict(_GRID))) + # row 0, "Status" column → resolves to column index 1 → "Shipped" + assert cell_by_header(0, "Status", name="Orders") == "Shipped" + assert cell_by_header(1, "Order", name="Orders") == "Order 6" + + +def test_cell_by_header_unknown_column(monkeypatch): + _inject(monkeypatch, _FakeBackend(dict(_HEADERS), dict(_GRID))) + assert cell_by_header(0, "Nonexistent", name="Orders") is None + + +def test_cell_by_header_no_headers(monkeypatch): + _inject(monkeypatch, _FakeBackend(None, dict(_GRID))) + assert cell_by_header(0, "Status", name="Orders") is None + + +def test_unsupported_backend_raises(monkeypatch): + from je_auto_control.utils.accessibility.element import ( + AccessibilityNotAvailableError) + _inject(monkeypatch, backend_base.AccessibilityBackend()) + try: + table_headers(name="x") + raised = False + except AccessibilityNotAvailableError: + raised = True + assert raised is True + + +# --- wiring --------------------------------------------------------------- + +def test_executor_adapters(monkeypatch): + _inject(monkeypatch, _FakeBackend(dict(_HEADERS), dict(_GRID))) + from je_auto_control.utils.executor.action_executor import ( + _cell_by_header, _table_cell, _table_headers) + assert _table_headers(name="Orders")["headers"]["columns"][1] == "Status" + assert _table_cell(0, 2, name="Orders")["cell"]["value"] == "$10" + out = _cell_by_header(0, "Status", name="Orders") + assert out["found"] is True and out["value"] == "Shipped" + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_table_headers", "AC_table_cell", "AC_cell_by_header"} <= known + from je_auto_control.utils.mcp_server.tools import build_default_tool_registry + names = {t.name for t in build_default_tool_registry()} + assert {"ac_table_headers", "ac_table_cell", "ac_cell_by_header"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_table_headers", "AC_table_cell", "AC_cell_by_header"} <= specs + + +def test_facade_exports(): + for name in ("table_headers", "table_cell", "cell_by_header"): + assert hasattr(ac, name) and name in ac.__all__