diff --git a/easybot_windows_agent/main.py b/easybot_windows_agent/main.py index a1ca11d..36cef5a 100644 --- a/easybot_windows_agent/main.py +++ b/easybot_windows_agent/main.py @@ -160,6 +160,46 @@ async def tool_list_windows(args: dict[str, Any]) -> dict[str, Any]: return {"error": str(e)} +def _get_ui_feedback() -> dict[str, Any]: + """Collect UI state feedback after an action: cursor position, focused element, active window.""" + feedback: dict[str, Any] = {} + try: + import pyautogui + mx, my = pyautogui.position() + feedback["cursor"] = {"x": mx, "y": my} + except Exception: + pass + try: + import uiautomation as auto + fw = auto.GetForegroundControl() + if fw: + feedback["active_window"] = fw.Name or "" + focused = auto.GetFocusedControl() + if focused: + fe: dict[str, Any] = { + "type": focused.ControlTypeName or "", + "name": focused.Name or "", + } + if focused.AutomationId: + fe["automation_id"] = focused.AutomationId + try: + vp = focused.GetValuePattern() + if vp: + fe["value"] = vp.Value + except Exception: + pass + try: + rect = focused.BoundingRectangle + if rect and rect.width() > 0: + fe["rect"] = {"x": rect.left, "y": rect.top, "w": rect.width(), "h": rect.height()} + except Exception: + pass + feedback["focused_element"] = fe + except Exception: + pass + return feedback + + async def tool_click_element(args: dict[str, Any]) -> dict[str, Any]: """Click a UI element by name, automation_id, or coordinates.""" name = args.get("name") @@ -178,7 +218,8 @@ async def tool_click_element(args: dict[str, Any]) -> dict[str, Any]: pyautogui.rightClick(x, y) else: pyautogui.click(x, y) - return {"success": True, "method": "coordinates", "x": x, "y": y} + await asyncio.sleep(0.3) + return {"success": True, "method": "coordinates", "x": x, "y": y, **_get_ui_feedback()} import uiautomation as auto ctrl = None @@ -192,7 +233,8 @@ async def tool_click_element(args: dict[str, Any]) -> dict[str, Any]: ctrl.DoubleClick() else: ctrl.Click() - return {"success": True, "method": "ui_automation", "element": ctrl.Name} + await asyncio.sleep(0.3) + return {"success": True, "method": "ui_automation", "element": ctrl.Name, **_get_ui_feedback()} else: return {"error": f"Element not found: name={name}, automation_id={auto_id}"} except Exception as e: @@ -203,6 +245,7 @@ async def tool_type_text(args: dict[str, Any]) -> dict[str, Any]: """Type text, optionally into a specific element.""" text = args.get("text", "") element_name = args.get("element_name") + clear_first = args.get("clear_first", False) if not text: return {"error": "No text provided"} @@ -213,15 +256,21 @@ async def tool_type_text(args: dict[str, Any]) -> dict[str, Any]: ctrl = auto.Control(searchDepth=8, Name=element_name) if ctrl.Exists(maxSearchSeconds=3): try: + if clear_first: + ctrl.GetValuePattern().SetValue("") ctrl.GetValuePattern().SetValue(text) - return {"success": True, "method": "set_value", "element": element_name} + return {"success": True, "method": "set_value", "element": element_name, **_get_ui_feedback()} except Exception: ctrl.Click() await asyncio.sleep(0.2) import pyautogui + if clear_first: + pyautogui.hotkey("ctrl", "a") + await asyncio.sleep(0.05) pyautogui.typewrite(text, interval=0.02) if text.isascii() else pyautogui.write(text) - return {"success": True, "method": "keyboard"} + await asyncio.sleep(0.1) + return {"success": True, "method": "keyboard", **_get_ui_feedback()} except Exception as e: return {"error": str(e)} @@ -239,14 +288,15 @@ async def tool_press_keys(args: dict[str, Any]) -> dict[str, Any]: pyautogui.hotkey(*parts) else: pyautogui.press(parts[0]) - return {"success": True, "keys": keys} + await asyncio.sleep(0.3) + return {"success": True, "keys": keys, **_get_ui_feedback()} except Exception as e: return {"error": str(e)} async def tool_screenshot(args: dict[str, Any]) -> dict[str, Any]: - """Take a screenshot and return as base64 PNG.""" - from PIL import ImageGrab + """Take a screenshot with UI annotations: cursor marker + focused element highlight.""" + from PIL import ImageGrab, ImageDraw try: region = args.get("region") @@ -256,6 +306,43 @@ async def tool_screenshot(args: dict[str, Any]) -> dict[str, Any]: else: img = ImageGrab.grab() + # Draw annotations on the screenshot + draw = ImageDraw.Draw(img) + + # 1. Draw cursor position (red crosshair) + try: + import pyautogui + mx, my = pyautogui.position() + # Adjust if region capture + if region: + mx -= region["x"] + my -= region["y"] + r = 12 + draw.ellipse([mx - r, my - r, mx + r, my + r], outline="red", width=3) + draw.line([mx - r - 4, my, mx + r + 4, my], fill="red", width=2) + draw.line([mx, my - r - 4, mx, my + r + 4], fill="red", width=2) + except Exception: + pass + + # 2. Highlight focused element (green rectangle + label) + try: + import uiautomation as auto + focused = auto.GetFocusedControl() + if focused: + rect = focused.BoundingRectangle + if rect and rect.width() > 0: + fx, fy = rect.left, rect.top + fw, fh = rect.width(), rect.height() + if region: + fx -= region["x"] + fy -= region["y"] + draw.rectangle([fx, fy, fx + fw, fy + fh], outline="lime", width=2) + label = focused.Name or focused.ControlTypeName or "" + if label: + draw.text((fx + 2, fy - 14), label[:40], fill="lime") + except Exception: + pass + # Resize for efficiency (max 1280px wide) if img.width > 1280: ratio = 1280 / img.width