feat: UI feedback on actions + annotated screenshots

easybot_windows_agent/main.py

@@ -160,6 +160,46 @@ async def tool_list_windows(args: dict[str, Any]) -> dict[str, Any]:
         return {"error": str(e)}
 
 
+def _get_ui_feedback() -> dict[str, Any]:
+    """Collect UI state feedback after an action: cursor position, focused element, active window."""
+    feedback: dict[str, Any] = {}
+    try:
+        import pyautogui
+        mx, my = pyautogui.position()
+        feedback["cursor"] = {"x": mx, "y": my}
+    except Exception:
+        pass
+    try:
+        import uiautomation as auto
+        fw = auto.GetForegroundControl()
+        if fw:
+            feedback["active_window"] = fw.Name or ""
+        focused = auto.GetFocusedControl()
+        if focused:
+            fe: dict[str, Any] = {
+                "type": focused.ControlTypeName or "",
+                "name": focused.Name or "",
+            }
+            if focused.AutomationId:
+                fe["automation_id"] = focused.AutomationId
+            try:
+                vp = focused.GetValuePattern()
+                if vp:
+                    fe["value"] = vp.Value
+            except Exception:
+                pass
+            try:
+                rect = focused.BoundingRectangle
+                if rect and rect.width() > 0:
+                    fe["rect"] = {"x": rect.left, "y": rect.top, "w": rect.width(), "h": rect.height()}
+            except Exception:
+                pass
+            feedback["focused_element"] = fe
+    except Exception:
+        pass
+    return feedback
+
+
 async def tool_click_element(args: dict[str, Any]) -> dict[str, Any]:
     """Click a UI element by name, automation_id, or coordinates."""
     name = args.get("name")
@@ -178,7 +218,8 @@ async def tool_click_element(args: dict[str, Any]) -> dict[str, Any]:
                 pyautogui.rightClick(x, y)
             else:
                 pyautogui.click(x, y)
-            return {"success": True, "method": "coordinates", "x": x, "y": y}
+            await asyncio.sleep(0.3)
+            return {"success": True, "method": "coordinates", "x": x, "y": y, **_get_ui_feedback()}
 
         import uiautomation as auto
         ctrl = None
@@ -192,7 +233,8 @@ async def tool_click_element(args: dict[str, Any]) -> dict[str, Any]:
                 ctrl.DoubleClick()
             else:
                 ctrl.Click()
-            return {"success": True, "method": "ui_automation", "element": ctrl.Name}
+            await asyncio.sleep(0.3)
+            return {"success": True, "method": "ui_automation", "element": ctrl.Name, **_get_ui_feedback()}
         else:
             return {"error": f"Element not found: name={name}, automation_id={auto_id}"}
     except Exception as e:
@@ -203,6 +245,7 @@ async def tool_type_text(args: dict[str, Any]) -> dict[str, Any]:
     """Type text, optionally into a specific element."""
     text = args.get("text", "")
     element_name = args.get("element_name")
+    clear_first = args.get("clear_first", False)
 
     if not text:
         return {"error": "No text provided"}
@@ -213,15 +256,21 @@ async def tool_type_text(args: dict[str, Any]) -> dict[str, Any]:
             ctrl = auto.Control(searchDepth=8, Name=element_name)
             if ctrl.Exists(maxSearchSeconds=3):
                 try:
+                    if clear_first:
+                        ctrl.GetValuePattern().SetValue("")
                     ctrl.GetValuePattern().SetValue(text)
-                    return {"success": True, "method": "set_value", "element": element_name}
+                    return {"success": True, "method": "set_value", "element": element_name, **_get_ui_feedback()}
                 except Exception:
                     ctrl.Click()
                     await asyncio.sleep(0.2)
 
         import pyautogui
+        if clear_first:
+            pyautogui.hotkey("ctrl", "a")
+            await asyncio.sleep(0.05)
         pyautogui.typewrite(text, interval=0.02) if text.isascii() else pyautogui.write(text)
-        return {"success": True, "method": "keyboard"}
+        await asyncio.sleep(0.1)
+        return {"success": True, "method": "keyboard", **_get_ui_feedback()}
     except Exception as e:
         return {"error": str(e)}
 
@@ -239,14 +288,15 @@ async def tool_press_keys(args: dict[str, Any]) -> dict[str, Any]:
             pyautogui.hotkey(*parts)
         else:
             pyautogui.press(parts[0])
-        return {"success": True, "keys": keys}
+        await asyncio.sleep(0.3)
+        return {"success": True, "keys": keys, **_get_ui_feedback()}
     except Exception as e:
         return {"error": str(e)}
 
 
 async def tool_screenshot(args: dict[str, Any]) -> dict[str, Any]:
-    """Take a screenshot and return as base64 PNG."""
+    """Take a screenshot with UI annotations: cursor marker + focused element highlight."""
-    from PIL import ImageGrab
+    from PIL import ImageGrab, ImageDraw
 
     try:
         region = args.get("region")
@@ -256,6 +306,43 @@ async def tool_screenshot(args: dict[str, Any]) -> dict[str, Any]:
         else:
             img = ImageGrab.grab()
 
+        # Draw annotations on the screenshot
+        draw = ImageDraw.Draw(img)
+
+        # 1. Draw cursor position (red crosshair)
+        try:
+            import pyautogui
+            mx, my = pyautogui.position()
+            # Adjust if region capture
+            if region:
+                mx -= region["x"]
+                my -= region["y"]
+            r = 12
+            draw.ellipse([mx - r, my - r, mx + r, my + r], outline="red", width=3)
+            draw.line([mx - r - 4, my, mx + r + 4, my], fill="red", width=2)
+            draw.line([mx, my - r - 4, mx, my + r + 4], fill="red", width=2)
+        except Exception:
+            pass
+
+        # 2. Highlight focused element (green rectangle + label)
+        try:
+            import uiautomation as auto
+            focused = auto.GetFocusedControl()
+            if focused:
+                rect = focused.BoundingRectangle
+                if rect and rect.width() > 0:
+                    fx, fy = rect.left, rect.top
+                    fw, fh = rect.width(), rect.height()
+                    if region:
+                        fx -= region["x"]
+                        fy -= region["y"]
+                    draw.rectangle([fx, fy, fx + fw, fy + fh], outline="lime", width=2)
+                    label = focused.Name or focused.ControlTypeName or ""
+                    if label:
+                        draw.text((fx + 2, fy - 14), label[:40], fill="lime")
+        except Exception:
+            pass
+
         # Resize for efficiency (max 1280px wide)
         if img.width > 1280:
             ratio = 1280 / img.width