feat: CDP tunnel + open_browser/close_browser + UI feedback on actions

easybot_windows_agent/main.py

@@ -439,6 +439,78 @@ async def tool_file_write(args: dict[str, Any]) -> dict[str, Any]:
         return {"error": str(e)}
 
 
+# ── Browser CDP Tool ─────────────────────────────────────────────────────────
+
+CDP_PORT = 9222
+_browser_process: subprocess.Popen | None = None
+
+
+async def tool_open_browser(args: dict[str, Any]) -> dict[str, Any]:
+    """Launch Edge/Chrome with CDP enabled. Returns the CDP port for tunnel."""
+    global _browser_process
+    url = args.get("url", "about:blank")
+    port = args.get("port", CDP_PORT)
+
+    # Find browser executable
+    browser_paths = [
+        r"C:\Program Files (x86)\Microsoft\Edge\Application\msedge.exe",
+        r"C:\Program Files\Microsoft\Edge\Application\msedge.exe",
+        r"C:\Program Files\Google\Chrome\Application\chrome.exe",
+        r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe",
+    ]
+    browser_exe = None
+    for p in browser_paths:
+        if os.path.exists(p):
+            browser_exe = p
+            break
+
+    if not browser_exe:
+        return {"error": "No Chromium browser found (Edge or Chrome)"}
+
+    try:
+        # Kill existing CDP browser if any
+        if _browser_process and _browser_process.poll() is None:
+            _browser_process.terminate()
+            await asyncio.sleep(0.5)
+
+        _browser_process = subprocess.Popen(
+            [browser_exe, f"--remote-debugging-port={port}",
+             "--no-first-run", "--no-default-browser-check", url],
+            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+            creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == "win32" else 0,
+        )
+        # Wait a bit for CDP to be ready
+        await asyncio.sleep(2)
+
+        # Verify CDP is listening
+        try:
+            reader, writer = await asyncio.open_connection("127.0.0.1", port)
+            writer.close()
+            await writer.wait_closed()
+        except OSError:
+            return {"error": f"Browser started but CDP port {port} not responding"}
+
+        return {
+            "success": True,
+            "browser": os.path.basename(browser_exe),
+            "cdp_port": port,
+            "pid": _browser_process.pid,
+            "message": f"Browser ready with CDP on port {port}. Tunnel available.",
+        }
+    except Exception as e:
+        return {"error": str(e)}
+
+
+async def tool_close_browser(args: dict[str, Any]) -> dict[str, Any]:
+    """Close the CDP browser."""
+    global _browser_process
+    if _browser_process and _browser_process.poll() is None:
+        _browser_process.terminate()
+        _browser_process = None
+        return {"success": True}
+    return {"success": True, "message": "No browser was running"}
+
+
 # ── Tool Registry ────────────────────────────────────────────────────────────
 
 TOOLS: dict[str, Any] = {
@@ -454,6 +526,8 @@ TOOLS: dict[str, Any] = {
     "focus_window": tool_focus_window,
     "file_read": tool_file_read,
     "file_write": tool_file_write,
+    "open_browser": tool_open_browser,
+    "close_browser": tool_close_browser,
 }
 
 
@@ -468,6 +542,8 @@ class AgentClient:
         self.current_call_id: str | None = None
         self.cancel_event = asyncio.Event()
         self._reconnect_delay = RECONNECT_DELAY_BASE
+        # TCP tunnel state: conn_id → (reader, writer)
+        self._tunnel_connections: dict[int, tuple[asyncio.StreamReader, asyncio.StreamWriter]] = {}
 
     async def run(self) -> None:
         """Main loop: connect → handle messages → reconnect on failure."""
@@ -511,11 +587,16 @@ class AgentClient:
 
             try:
                 async for raw in ws:
-                    msg = json.loads(raw)
+                    if isinstance(raw, bytes):
-                    await self._handle_message(msg)
+                        # Binary frame: tunnel data [4-byte connId BE][payload]
+                        await self._handle_tunnel_data(raw)
+                    else:
+                        msg = json.loads(raw)
+                        await self._handle_message(msg)
             finally:
                 self.ws = None
                 self.current_call_id = None
+                await self._close_all_tunnels()
 
     async def _handle_message(self, msg: dict[str, Any]) -> None:
         """Dispatch incoming messages from the server."""
@@ -537,6 +618,15 @@ class AgentClient:
                 logger.info("Cancel requested for %s", call_id)
                 self.cancel_event.set()
 
+        elif msg_type == "tunnel_open":
+            conn_id = msg["connection_id"]
+            port = msg.get("port", CDP_PORT)
+            asyncio.create_task(self._tunnel_open(conn_id, port))
+
+        elif msg_type == "tunnel_close":
+            conn_id = msg["connection_id"]
+            await self._tunnel_close(conn_id)
+
         else:
             logger.debug("Unknown message type: %s", msg_type)
 
@@ -595,6 +685,70 @@ class AgentClient:
         finally:
             self.current_call_id = None
 
+    # ── TCP Tunnel Methods ──────────────────────────────────────────────────
+
+    async def _tunnel_open(self, conn_id: int, port: int) -> None:
+        """Open a TCP connection to localhost:port and start forwarding."""
+        try:
+            reader, writer = await asyncio.open_connection("127.0.0.1", port)
+            self._tunnel_connections[conn_id] = (reader, writer)
+            await self._send({"type": "tunnel_open_ok", "connection_id": conn_id})
+            # Start reading from TCP and forwarding to WS
+            asyncio.create_task(self._tunnel_read_loop(conn_id, reader))
+            logger.debug("Tunnel connection %d opened to port %d", conn_id, port)
+        except Exception as e:
+            await self._send({"type": "tunnel_open_err", "connection_id": conn_id, "error": str(e)})
+
+    async def _tunnel_read_loop(self, conn_id: int, reader: asyncio.StreamReader) -> None:
+        """Read from TCP socket and send as binary frames to server."""
+        try:
+            while True:
+                data = await reader.read(65536)
+                if not data:
+                    break
+                # Binary frame: [4-byte connId BE][payload]
+                frame = conn_id.to_bytes(4, "big") + data
+                if self.ws:
+                    await self.ws.send(frame)
+        except (ConnectionError, asyncio.IncompleteReadError):
+            pass
+        finally:
+            await self._tunnel_close(conn_id)
+            await self._send({"type": "tunnel_close", "connection_id": conn_id})
+
+    async def _handle_tunnel_data(self, raw: bytes) -> None:
+        """Handle incoming binary frame: forward to TCP socket."""
+        if len(raw) < 5:
+            return
+        conn_id = int.from_bytes(raw[:4], "big")
+        payload = raw[4:]
+        conn = self._tunnel_connections.get(conn_id)
+        if conn:
+            _, writer = conn
+            try:
+                writer.write(payload)
+                await writer.drain()
+            except (ConnectionError, OSError):
+                await self._tunnel_close(conn_id)
+
+    async def _tunnel_close(self, conn_id: int) -> None:
+        """Close a tunnel connection."""
+        conn = self._tunnel_connections.pop(conn_id, None)
+        if conn:
+            _, writer = conn
+            try:
+                writer.close()
+                await writer.wait_closed()
+            except Exception:
+                pass
+
+    async def _close_all_tunnels(self) -> None:
+        """Close all tunnel connections."""
+        for conn_id in list(self._tunnel_connections.keys()):
+            await self._tunnel_close(conn_id)
+
+    # ── Communication ────────────────────────────────────────────────────────
+
     async def _send(self, msg: dict[str, Any]) -> None:
         """Send a JSON message to the server."""
         if self.ws: