f07d17e927
* feat(web): Oo computer browser panel with cowork and local preview Wire the in-app browser panel to the active Oo's live Chromium via the computer screen stream, page-browser navigation, and interactive cowork input with AgentCursor. User browser actions become system meta lines in thread context; localhost preview remains as Lokale Vorschau. Co-authored-by: cynarAI <cynarAI@users.noreply.github.com> * fix(i18n): sync Lingui catalogs for Oo browser panel Co-authored-by: cynarAI <cynarAI@users.noreply.github.com> * Fix the Oo browser's lint and E2E failures The shared-browser overlay is a focusable input surface, so it gets the application role and a translated label. The local preview focuses the address field again when no page is open, which the split out of InAppBrowser had dropped. The Oo browser spec opens the panel by its exact name, because the signed-up user's name also contains "Browser", and reads the user action from the RPC body's json envelope. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Wy1md9QE1ayJbDCwJ4vAQs --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: cynarAI <cynarAI@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com>
748 lines
29 KiB
Python
Executable File
748 lines
29 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Drive the live Chromium on this computer over CDP (stdlib only).
|
|
|
|
Attaches to the Chrome already showing on DISPLAY. Used by the host
|
|
BrowserProvider. Not a vendor SDK. On failure prints JSON with
|
|
fallback computer_act.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import socket
|
|
import struct
|
|
import sys
|
|
import time
|
|
import threading
|
|
import urllib.request
|
|
from urllib.parse import urlsplit
|
|
from typing import Any
|
|
|
|
|
|
def display_num() -> int:
|
|
raw = os.environ.get("DISPLAY", ":1")
|
|
try:
|
|
return int(str(raw).lstrip(":").split(".")[0] or "1")
|
|
except ValueError:
|
|
return 1
|
|
|
|
|
|
def cdp_port() -> int:
|
|
override = os.environ.get("RAKAZO_CDP_PORT")
|
|
if override:
|
|
return int(override)
|
|
# rakazo-browser uses 9221 + display number (:1 -> 9222).
|
|
return 9221 + display_num()
|
|
|
|
|
|
def http_get_json(url: str, timeout: float = 5.0) -> Any:
|
|
with urllib.request.urlopen(url, timeout=timeout) as resp:
|
|
data = resp.read(1024 * 1024 + 1)
|
|
if len(data) > 1024 * 1024:
|
|
raise RuntimeError("CDP discovery response too large")
|
|
return json.loads(data.decode("utf-8"))
|
|
|
|
|
|
def wait_for_cdp(port: int, timeout_s: float = 3.0) -> None:
|
|
deadline = time.time() + timeout_s
|
|
last_error = "unreachable"
|
|
while time.time() < deadline:
|
|
try:
|
|
http_get_json(f"http://127.0.0.1:{port}/json/version", timeout=1.0)
|
|
return
|
|
except Exception as exc: # noqa: BLE001
|
|
last_error = str(exc)
|
|
time.sleep(0.25)
|
|
raise RuntimeError(f"CDP not ready on port {port}: {last_error}")
|
|
|
|
|
|
def discover_ws_url(port: int) -> str:
|
|
version = http_get_json(f"http://127.0.0.1:{port}/json/version")
|
|
ws = version.get("webSocketDebuggerUrl")
|
|
if not isinstance(ws, str) or not ws:
|
|
raise RuntimeError("CDP version endpoint missing webSocketDebuggerUrl")
|
|
parsed = urlsplit(ws)
|
|
if parsed.scheme != "ws" or parsed.hostname != "127.0.0.1" or parsed.port != port:
|
|
raise RuntimeError("Unexpected CDP endpoint")
|
|
return ws
|
|
|
|
|
|
WORLD_NAME = "rakazo-page-browser"
|
|
|
|
|
|
class CdpClient:
|
|
def __init__(self, ws_url: str) -> None:
|
|
self._ws_url = ws_url
|
|
self._sock: socket.socket | None = None
|
|
self._buf = bytearray()
|
|
self._next_id = 1
|
|
self.context_id: int | None = None
|
|
self._pending: dict[int, dict[str, Any] | None] = {}
|
|
# Worlds live in Chromium until navigation; rediscover them per CLI process.
|
|
self._contexts: dict[int, dict[str, Any]] = {}
|
|
|
|
def connect(self) -> None:
|
|
assert self._ws_url.startswith("ws://"), self._ws_url
|
|
rest = self._ws_url[len("ws://") :]
|
|
host_port, _, path = rest.partition("/")
|
|
if ":" in host_port:
|
|
host, port_s = host_port.split(":", 1)
|
|
port = int(port_s)
|
|
else:
|
|
host, port = host_port, 80
|
|
path = "/" + path
|
|
sock = socket.create_connection((host, port), timeout=10)
|
|
key = base64.b64encode(os.urandom(16)).decode("ascii")
|
|
req = (
|
|
f"GET {path} HTTP/1.1\r\n"
|
|
f"Host: {host_port}\r\n"
|
|
"Upgrade: websocket\r\n"
|
|
"Connection: Upgrade\r\n"
|
|
f"Sec-WebSocket-Key: {key}\r\n"
|
|
"Sec-WebSocket-Version: 13\r\n"
|
|
"\r\n"
|
|
)
|
|
sock.sendall(req.encode("ascii"))
|
|
data = b""
|
|
while b"\r\n\r\n" not in data:
|
|
chunk = sock.recv(4096)
|
|
if not chunk:
|
|
raise RuntimeError("CDP websocket handshake closed")
|
|
data += chunk
|
|
if len(data) > 16_384:
|
|
raise RuntimeError("CDP handshake too large")
|
|
header, _, leftover = data.partition(b"\r\n\r\n")
|
|
if b"101" not in header.split(b"\r\n", 1)[0]:
|
|
raise RuntimeError(f"CDP websocket upgrade failed: {header[:200]!r}")
|
|
expected = base64.b64encode(
|
|
hashlib.sha1((key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").encode("ascii")).digest()
|
|
)
|
|
if expected not in header:
|
|
raise RuntimeError("CDP websocket accept mismatch")
|
|
self._sock = sock
|
|
self._buf.extend(leftover)
|
|
|
|
def close(self) -> None:
|
|
if self._sock is not None:
|
|
try:
|
|
self._sock.close()
|
|
except OSError:
|
|
pass
|
|
self._sock = None
|
|
|
|
def _send_frame(self, payload: bytes) -> None:
|
|
assert self._sock is not None
|
|
mask = os.urandom(4)
|
|
masked = bytes(b ^ mask[i % 4] for i, b in enumerate(payload))
|
|
header = bytearray([0x81])
|
|
n = len(payload)
|
|
if n < 126:
|
|
header.append(0x80 | n)
|
|
elif n < 65536:
|
|
header.append(0x80 | 126)
|
|
header.extend(struct.pack("!H", n))
|
|
else:
|
|
header.append(0x80 | 127)
|
|
header.extend(struct.pack("!Q", n))
|
|
header.extend(mask)
|
|
self._sock.sendall(header + masked)
|
|
|
|
def _send_control(self, opcode: int, payload: bytes) -> None:
|
|
assert self._sock is not None
|
|
mask = os.urandom(4)
|
|
masked = bytes(b ^ mask[i % 4] for i, b in enumerate(payload))
|
|
header = bytearray([0x80 | opcode, 0x80 | len(payload)])
|
|
header.extend(mask)
|
|
self._sock.sendall(header + masked)
|
|
|
|
def _read_more(self) -> None:
|
|
assert self._sock is not None
|
|
chunk = self._sock.recv(65536)
|
|
if not chunk:
|
|
raise RuntimeError("CDP websocket closed")
|
|
self._buf.extend(chunk)
|
|
|
|
def _read_frame(self) -> bytes:
|
|
assert self._sock is not None
|
|
while True:
|
|
while len(self._buf) < 2:
|
|
chunk = self._sock.recv(65536)
|
|
if not chunk:
|
|
raise RuntimeError("CDP websocket closed")
|
|
self._buf.extend(chunk)
|
|
b1, b2 = self._buf[0], self._buf[1]
|
|
opcode = b1 & 0x0F
|
|
masked = bool(b2 & 0x80)
|
|
length = b2 & 0x7F
|
|
offset = 2
|
|
if length == 126:
|
|
while len(self._buf) < 4:
|
|
self._read_more()
|
|
length = struct.unpack("!H", self._buf[2:4])[0]
|
|
offset = 4
|
|
elif length == 127:
|
|
while len(self._buf) < 10:
|
|
self._read_more()
|
|
length = struct.unpack("!Q", self._buf[2:10])[0]
|
|
offset = 10
|
|
if length > 2 * 1024 * 1024:
|
|
raise RuntimeError("CDP frame too large")
|
|
if not (b1 & 0x80):
|
|
raise RuntimeError("Fragmented CDP message unsupported")
|
|
mask = b""
|
|
if masked:
|
|
while len(self._buf) < offset + 4:
|
|
self._read_more()
|
|
mask = bytes(self._buf[offset : offset + 4])
|
|
offset += 4
|
|
while len(self._buf) < offset + length:
|
|
chunk = self._sock.recv(65536)
|
|
if not chunk:
|
|
raise RuntimeError("CDP websocket closed while reading frame")
|
|
self._buf.extend(chunk)
|
|
payload = bytes(self._buf[offset : offset + length])
|
|
del self._buf[: offset + length]
|
|
if masked:
|
|
payload = bytes(b ^ mask[i % 4] for i, b in enumerate(payload))
|
|
if opcode == 0x8:
|
|
raise RuntimeError("CDP websocket closed by peer")
|
|
if opcode == 0x9:
|
|
self._send_control(0xA, payload)
|
|
continue
|
|
if opcode in (0x1, 0x2, 0x0):
|
|
return payload
|
|
|
|
def _handle_event(self, message: dict[str, Any]) -> None:
|
|
method = message.get("method")
|
|
params = message.get("params") or {}
|
|
session_id = message.get("sessionId")
|
|
if method == "Runtime.executionContextCreated":
|
|
context = params.get("context")
|
|
if not isinstance(context, dict):
|
|
return
|
|
context_id = context.get("id")
|
|
if not isinstance(context_id, int):
|
|
return
|
|
recorded = dict(context)
|
|
recorded["_sessionId"] = session_id
|
|
self._contexts[context_id] = recorded
|
|
elif method == "Runtime.executionContextDestroyed":
|
|
context_id = params.get("executionContextId")
|
|
if isinstance(context_id, int):
|
|
self._contexts.pop(context_id, None)
|
|
elif method == "Runtime.executionContextsCleared":
|
|
if session_id is None:
|
|
self._contexts.clear()
|
|
else:
|
|
self._contexts = {
|
|
cid: ctx
|
|
for cid, ctx in self._contexts.items()
|
|
if ctx.get("_sessionId") != session_id
|
|
}
|
|
|
|
def find_world_context(self, world_name: str, frame_id: str, session_id: str) -> int | None:
|
|
for context_id, context in self._contexts.items():
|
|
if context.get("name") != world_name:
|
|
continue
|
|
if context.get("_sessionId") != session_id:
|
|
continue
|
|
aux = context.get("auxData") or {}
|
|
if not isinstance(aux, dict) or aux.get("frameId") != frame_id or aux.get("isDefault") is not False:
|
|
continue
|
|
return context_id
|
|
return None
|
|
|
|
def call(
|
|
self,
|
|
method: str,
|
|
params: dict[str, Any] | None = None,
|
|
*,
|
|
session_id: str | None = None,
|
|
timeout_s: float = 30.0,
|
|
) -> dict[str, Any]:
|
|
assert self._sock is not None
|
|
msg_id = self._next_id
|
|
self._next_id += 1
|
|
payload: dict[str, Any] = {"id": msg_id, "method": method, "params": params or {}}
|
|
if session_id:
|
|
payload["sessionId"] = session_id
|
|
self._pending[msg_id] = None
|
|
self._send_frame(json.dumps(payload).encode("utf-8"))
|
|
deadline = time.time() + timeout_s
|
|
while time.time() < deadline:
|
|
pending = self._pending.get(msg_id)
|
|
if pending is not None:
|
|
self._pending.pop(msg_id, None)
|
|
if "error" in pending:
|
|
raise RuntimeError(f"{method} failed: {pending['error']}")
|
|
result = pending.get("result")
|
|
return result if isinstance(result, dict) else {}
|
|
self._sock.settimeout(max(0.1, deadline - time.time()))
|
|
try:
|
|
frame = self._read_frame()
|
|
except socket.timeout:
|
|
continue
|
|
message = json.loads(frame.decode("utf-8"))
|
|
if not isinstance(message, dict):
|
|
continue
|
|
if message.get("id") in self._pending:
|
|
self._pending[int(message["id"])] = message
|
|
continue
|
|
if message.get("method"):
|
|
self._handle_event(message)
|
|
raise TimeoutError(f"CDP call timed out: {method}")
|
|
|
|
|
|
def attach_page(client: CdpClient) -> str:
|
|
targets = client.call("Target.getTargets").get("targetInfos") or []
|
|
pages = [t for t in targets if isinstance(t, dict) and t.get("type") == "page"]
|
|
candidates = []
|
|
for page in pages[:32]:
|
|
attached = client.call("Target.attachToTarget", {
|
|
"targetId": page["targetId"], "flatten": True,
|
|
})
|
|
session_id = attached["sessionId"]
|
|
try:
|
|
client.call("Page.enable", session_id=session_id)
|
|
ensure_world(client, session_id)
|
|
if eval_json(client, session_id, 'document.visibilityState === "visible"'):
|
|
focused = eval_json(client, session_id, "document.hasFocus()")
|
|
candidates.append((session_id, focused))
|
|
except Exception:
|
|
continue
|
|
focused = [session for session, focus in candidates if focus]
|
|
if len(focused) == 1:
|
|
selected = focused[0]
|
|
elif len(candidates) == 1:
|
|
selected = candidates[0][0]
|
|
else:
|
|
raise RuntimeError("No unambiguous visible browser tab. Use desktop tools to select a tab.")
|
|
ensure_world(client, selected)
|
|
return selected
|
|
|
|
|
|
def ensure_world(client: CdpClient, session_id: str) -> None:
|
|
# Rediscover only an isolated context bound to this frame and CDP session.
|
|
# Never run helpers in a page-owned default context with the same name.
|
|
frame = client.call("Page.getFrameTree", session_id=session_id)["frameTree"]["frame"]
|
|
frame_id = frame["id"]
|
|
client.call("Runtime.enable", session_id=session_id)
|
|
existing = client.find_world_context(WORLD_NAME, frame_id, session_id)
|
|
if existing is not None:
|
|
client.context_id = existing
|
|
return
|
|
world = client.call("Page.createIsolatedWorld", {
|
|
"frameId": frame_id, "worldName": WORLD_NAME,
|
|
}, session_id=session_id)
|
|
client.context_id = world["executionContextId"]
|
|
|
|
|
|
EVAL_HELPERS = r"""
|
|
(() => {
|
|
if (window.__rakazoPageBrowser) return true;
|
|
const refMap = new Map();
|
|
// New snapshots/documents never recycle references. Retain exact nodes, not selectors.
|
|
function newRef() {
|
|
return `e${Array.from(crypto.getRandomValues(new Uint32Array(4)), n => n.toString(16).padStart(8, "0")).join("")}`;
|
|
}
|
|
function visible(el) {
|
|
if (!(el instanceof Element)) return false;
|
|
const style = window.getComputedStyle(el);
|
|
if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") {
|
|
return false;
|
|
}
|
|
const rect = el.getBoundingClientRect();
|
|
return rect.width > 0 && rect.height > 0;
|
|
}
|
|
function roleOf(el) {
|
|
const explicit = el.getAttribute("role");
|
|
if (explicit) return explicit;
|
|
const tag = el.tagName.toLowerCase();
|
|
if (tag === "a") return "link";
|
|
if (tag === "button") return "button";
|
|
if (tag === "input") {
|
|
const type = (el.getAttribute("type") || "text").toLowerCase();
|
|
if (type === "submit" || type === "button") return "button";
|
|
if (type === "checkbox") return "checkbox";
|
|
if (type === "radio") return "radio";
|
|
return "textbox";
|
|
}
|
|
if (tag === "textarea" || el.isContentEditable) return "textbox";
|
|
if (tag === "select") return "combobox";
|
|
if (/^h[1-6]$/.test(tag)) return "heading";
|
|
return tag;
|
|
}
|
|
function nameOf(el) {
|
|
return (
|
|
el.getAttribute("aria-label") ||
|
|
Array.from(el.labels || []).map(label => label.textContent).join(" ").trim() ||
|
|
el.getAttribute("placeholder") ||
|
|
el.getAttribute("name") ||
|
|
el.getAttribute("title") ||
|
|
(el.innerText || el.textContent || "").trim().slice(0, 80) ||
|
|
""
|
|
);
|
|
}
|
|
function isInteractive(el) {
|
|
if (!(el instanceof Element)) return false;
|
|
const tag = el.tagName.toLowerCase();
|
|
if (["a", "button", "input", "textarea", "select", "summary"].includes(tag)) return true;
|
|
if (el.isContentEditable) return true;
|
|
const role = el.getAttribute("role");
|
|
if (
|
|
role &&
|
|
["button", "link", "textbox", "checkbox", "radio", "menuitem", "tab", "option", "switch"].includes(
|
|
role
|
|
)
|
|
) {
|
|
return true;
|
|
}
|
|
return typeof el.onclick === "function" || el.hasAttribute("tabindex");
|
|
}
|
|
function formatTree(elements) {
|
|
if (!elements.length) return "(no interactive elements)";
|
|
return elements
|
|
.map((el) => {
|
|
const value =
|
|
el.value !== undefined && el.value !== "" ? ` value=${JSON.stringify(el.value)}` : "";
|
|
return `- ${el.role} "${el.name}" [${el.ref}]${value}`;
|
|
})
|
|
.join("\n");
|
|
}
|
|
// Screen pixels of the element center's box, only while that center is in the viewport.
|
|
// Drives the visible bot cursor; the model never sees it.
|
|
function screenBox(el, viewport) {
|
|
const rect = el.getBoundingClientRect();
|
|
const cx = rect.left + rect.width / 2;
|
|
const cy = rect.top + rect.height / 2;
|
|
if (cx < 0 || cy < 0 || cx > window.innerWidth || cy > window.innerHeight) return undefined;
|
|
const box = [
|
|
viewport.x + rect.left * viewport.scale,
|
|
viewport.y + rect.top * viewport.scale,
|
|
rect.width * viewport.scale,
|
|
rect.height * viewport.scale,
|
|
].map(Math.round);
|
|
return box[0] >= 0 && box[1] >= 0 && box[2] > 0 && box[3] > 0 ? box : undefined;
|
|
}
|
|
function viewportOnScreen() {
|
|
const scale = window.devicePixelRatio || 1;
|
|
// Browser chrome sits above the page; side borders are split evenly.
|
|
return {
|
|
scale,
|
|
x: (window.screenX + Math.max(0, (window.outerWidth - window.innerWidth) / 2)) * scale,
|
|
y: (window.screenY + Math.max(0, window.outerHeight - window.innerHeight)) * scale,
|
|
screen: {
|
|
width: Math.round(window.screen.width * scale),
|
|
height: Math.round(window.screen.height * scale),
|
|
},
|
|
};
|
|
}
|
|
window.__rakazoPageBrowser = {
|
|
snapshot() {
|
|
refMap.clear();
|
|
const elements = [];
|
|
const viewport = viewportOnScreen();
|
|
for (const el of document.querySelectorAll("body *")) {
|
|
if (!isInteractive(el) || !visible(el)) continue;
|
|
const ref = newRef();
|
|
refMap.set(ref, el);
|
|
elements.push({
|
|
ref,
|
|
role: roleOf(el),
|
|
name: String(nameOf(el)).slice(0, 200),
|
|
value: el.type === "password" ? undefined
|
|
: ["checkbox", "radio"].includes(el.type) ? (el.checked ? "checked" : "unchecked")
|
|
: "value" in el ? String(el.value ?? "").slice(0, 2000) : undefined,
|
|
tag: el.tagName.toLowerCase(),
|
|
box: screenBox(el, viewport),
|
|
});
|
|
if (elements.length >= 80) break;
|
|
}
|
|
return {
|
|
url: location.href,
|
|
title: (document.title || "").slice(0, 200),
|
|
screen: viewport.screen,
|
|
elements,
|
|
tree: (document.body?.innerText || "").slice(0, 12000) + "\n" + formatTree(elements)
|
|
+ (document.querySelector("iframe,frame") ? "\nEmbedded frames require desktop tools." : ""),
|
|
};
|
|
},
|
|
resolve(ref) {
|
|
const el = refMap.get(String(ref));
|
|
if (!el || !el.isConnected || !visible(el)) return null;
|
|
if (el.matches(":disabled") || el.getAttribute("aria-disabled") === "true") {
|
|
throw new Error("Element is disabled");
|
|
}
|
|
return el;
|
|
},
|
|
click(ref) {
|
|
const el = this.resolve(ref);
|
|
if (!el) throw new Error(`Unknown element ref: ${ref}`);
|
|
el.focus();
|
|
el.click();
|
|
return { url: location.href, title: (document.title || "").slice(0, 200) };
|
|
},
|
|
fill(ref, value) {
|
|
const el = this.resolve(ref);
|
|
if (!el) throw new Error(`Unknown element ref: ${ref}`);
|
|
if (el.readOnly) throw new Error("Element is read-only");
|
|
el.focus();
|
|
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement) {
|
|
if (el instanceof HTMLInputElement && !["text", "search", "email", "url", "tel", "password", "number"].includes(el.type)) {
|
|
throw new Error("Input type requires desktop tools");
|
|
}
|
|
const prototype = el instanceof HTMLInputElement ? HTMLInputElement.prototype
|
|
: el instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLSelectElement.prototype;
|
|
Object.getOwnPropertyDescriptor(prototype, "value").set.call(el, String(value));
|
|
if (el.value !== String(value)) throw new Error("Input did not accept the requested value");
|
|
el.dispatchEvent(new Event("input", { bubbles: true }));
|
|
el.dispatchEvent(new Event("change", { bubbles: true }));
|
|
} else if (el.isContentEditable) {
|
|
el.textContent = String(value);
|
|
el.dispatchEvent(new Event("input", { bubbles: true }));
|
|
} else {
|
|
throw new Error(`Element is not fillable: ${ref}`);
|
|
}
|
|
return { url: location.href, title: (document.title || "").slice(0, 200) };
|
|
},
|
|
type(ref, text) {
|
|
const el = this.resolve(ref);
|
|
if (!el) throw new Error(`Unknown element ref: ${ref}`);
|
|
return this.fill(ref, `${"value" in el ? el.value : el.textContent || ""}${text}`);
|
|
},
|
|
};
|
|
return true;
|
|
})()
|
|
"""
|
|
|
|
|
|
def eval_json(client: CdpClient, session_id: str, expression: str) -> Any:
|
|
result = client.call(
|
|
"Runtime.evaluate",
|
|
{
|
|
"expression": expression,
|
|
"contextId": client.context_id,
|
|
"returnByValue": True,
|
|
"awaitPromise": True,
|
|
},
|
|
session_id=session_id,
|
|
)
|
|
if result.get("exceptionDetails"):
|
|
details = result["exceptionDetails"]
|
|
text = (
|
|
details.get("text")
|
|
or (details.get("exception") or {}).get("description")
|
|
or "evaluate failed"
|
|
)
|
|
raise RuntimeError(str(text))
|
|
return (result.get("result") or {}).get("value")
|
|
|
|
|
|
def ensure_helpers(client: CdpClient, session_id: str) -> None:
|
|
ensure_world(client, session_id)
|
|
eval_json(client, session_id, EVAL_HELPERS)
|
|
|
|
|
|
def navigate(client: CdpClient, session_id: str, url: str) -> dict[str, Any]:
|
|
parsed = urlsplit(url)
|
|
if parsed.scheme not in ("http", "https") or parsed.username or parsed.password:
|
|
raise RuntimeError("Only HTTP(S) URLs without embedded credentials are supported")
|
|
result = client.call("Page.navigate", {"url": url}, session_id=session_id, timeout_s=15.0)
|
|
if result.get("errorText") or result.get("isDownload"):
|
|
raise RuntimeError(result.get("errorText") or "Navigation started a download")
|
|
deadline = time.monotonic() + 10.0
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
frame = client.call("Page.getFrameTree", session_id=session_id)["frameTree"]["frame"]
|
|
if result.get("loaderId") and frame.get("loaderId") != result["loaderId"]:
|
|
time.sleep(0.1)
|
|
continue
|
|
ensure_world(client, session_id)
|
|
ready = eval_json(client, session_id, "document.readyState")
|
|
if ready in ("interactive", "complete"):
|
|
return {"ok": True, "url": eval_json(client, session_id, "location.href"),
|
|
"title": eval_json(client, session_id, "(document.title || '').slice(0, 200)")}
|
|
except Exception:
|
|
pass
|
|
time.sleep(0.1)
|
|
raise RuntimeError("Navigation did not settle; inspect the screen before continuing")
|
|
|
|
|
|
def snapshot(client: CdpClient, session_id: str) -> dict[str, Any]:
|
|
ensure_helpers(client, session_id)
|
|
data = eval_json(client, session_id, "window.__rakazoPageBrowser.snapshot()")
|
|
if not isinstance(data, dict):
|
|
raise RuntimeError("snapshot returned non-object")
|
|
return {"ok": True, **data}
|
|
|
|
|
|
def browser_state(client: CdpClient, session_id: str) -> dict[str, Any]:
|
|
ensure_helpers(client, session_id)
|
|
history = client.call("Page.getNavigationHistory", session_id=session_id)
|
|
entries = history.get("entries") if isinstance(history, dict) else None
|
|
current = history.get("currentIndex") if isinstance(history, dict) else None
|
|
can_back = isinstance(current, int) and current > 0
|
|
can_forward = (
|
|
isinstance(current, int)
|
|
and isinstance(entries, list)
|
|
and current >= 0
|
|
and current < len(entries) - 1
|
|
)
|
|
url = eval_json(client, session_id, "location.href")
|
|
title = eval_json(client, session_id, "(document.title || '').slice(0, 200)")
|
|
ready = eval_json(client, session_id, "document.readyState")
|
|
return {
|
|
"ok": True,
|
|
"url": url if isinstance(url, str) else "",
|
|
"title": title if isinstance(title, str) else "",
|
|
"loading": ready not in ("interactive", "complete"),
|
|
"canGoBack": can_back,
|
|
"canGoForward": can_forward,
|
|
}
|
|
|
|
|
|
def history_step(client: CdpClient, session_id: str, delta: int) -> dict[str, Any]:
|
|
ensure_helpers(client, session_id)
|
|
history = client.call("Page.getNavigationHistory", session_id=session_id)
|
|
current = history.get("currentIndex") if isinstance(history, dict) else None
|
|
entries = history.get("entries") if isinstance(history, dict) else None
|
|
if not isinstance(current, int) or not isinstance(entries, list):
|
|
raise RuntimeError("Navigation history unavailable")
|
|
target = current + delta
|
|
if target < 0 or target >= len(entries):
|
|
raise RuntimeError("Cannot navigate in that direction")
|
|
entry_id = entries[target].get("id")
|
|
if not isinstance(entry_id, int):
|
|
raise RuntimeError("Navigation entry missing")
|
|
client.call("Page.navigateToHistoryEntry", {"entryId": entry_id}, session_id=session_id)
|
|
deadline = time.monotonic() + 10.0
|
|
while time.monotonic() < deadline:
|
|
ready = eval_json(client, session_id, "document.readyState")
|
|
if ready in ("interactive", "complete"):
|
|
return browser_state(client, session_id)
|
|
time.sleep(0.1)
|
|
raise RuntimeError("History navigation did not settle")
|
|
|
|
|
|
def reload_page(client: CdpClient, session_id: str) -> dict[str, Any]:
|
|
client.call("Page.reload", session_id=session_id)
|
|
deadline = time.monotonic() + 10.0
|
|
while time.monotonic() < deadline:
|
|
ready = eval_json(client, session_id, "document.readyState")
|
|
if ready in ("interactive", "complete"):
|
|
return browser_state(client, session_id)
|
|
time.sleep(0.1)
|
|
raise RuntimeError("Reload did not settle")
|
|
|
|
|
|
def act(client: CdpClient, session_id: str, actions: list[dict[str, Any]]) -> dict[str, Any]:
|
|
if not actions or len(actions) > 24:
|
|
raise RuntimeError("Expected 1 to 24 actions")
|
|
for step in actions:
|
|
if not isinstance(step, dict) or step.get("kind") not in ("click", "fill", "type"):
|
|
raise RuntimeError("Invalid browser action")
|
|
if not isinstance(step.get("ref"), str) or not step["ref"]:
|
|
raise RuntimeError("Element ref is required")
|
|
if step["kind"] != "click" and not isinstance(step.get("text"), str):
|
|
raise RuntimeError("Text is required")
|
|
ensure_helpers(client, session_id)
|
|
completed = 0
|
|
uncertain = False
|
|
try:
|
|
for step in actions:
|
|
kind, ref = step["kind"], step["ref"]
|
|
# Preflight catches stale refs before a mutation is dispatched.
|
|
if not eval_json(client, session_id, f"Boolean(window.__rakazoPageBrowser.resolve({json.dumps(ref)}))"):
|
|
raise RuntimeError("Stale element ref. Take a fresh snapshot.")
|
|
arguments = json.dumps(ref)
|
|
if kind != "click":
|
|
arguments += ", " + json.dumps(step["text"])
|
|
uncertain = True
|
|
data = eval_json(client, session_id, f"window.__rakazoPageBrowser.{kind}({arguments})")
|
|
if not isinstance(data, dict):
|
|
raise RuntimeError("Action response unavailable")
|
|
completed += 1
|
|
uncertain = False
|
|
snap = snapshot(client, session_id)
|
|
return {**snap, "completed": completed}
|
|
except Exception as exc:
|
|
return {"ok": False, "completed": completed, "uncertain": uncertain,
|
|
"error": str(exc), "fallback": "computer_act"}
|
|
|
|
|
|
def exit_when_stdin_closes() -> None:
|
|
"""The supervisor owns this pipe; losing it cancels this helper, not Chromium."""
|
|
try:
|
|
while os.read(0, 1024):
|
|
pass
|
|
finally:
|
|
os._exit(130)
|
|
|
|
|
|
def run(argv: list[str]) -> int:
|
|
if os.environ.get("RAKAZO_BROWSER_WATCH_STDIN") == "1":
|
|
threading.Thread(target=exit_when_stdin_closes, daemon=True).start()
|
|
if len(argv) < 2:
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"ok": False,
|
|
"error": "usage: rakazo-page-browser <command> [json-args]",
|
|
"fallback": "computer_act",
|
|
}
|
|
)
|
|
)
|
|
return 2
|
|
command = argv[1]
|
|
args: dict[str, Any] = {}
|
|
if len(argv) >= 3 and argv[2].strip():
|
|
args = json.loads(argv[2])
|
|
port = cdp_port()
|
|
try:
|
|
wait_for_cdp(port)
|
|
ws_url = discover_ws_url(port)
|
|
client = CdpClient(ws_url)
|
|
client.connect()
|
|
try:
|
|
session_id = attach_page(client)
|
|
if command == "navigate":
|
|
result = navigate(client, session_id, str(args.get("url") or ""))
|
|
elif command == "state":
|
|
result = browser_state(client, session_id)
|
|
elif command == "back":
|
|
result = history_step(client, session_id, -1)
|
|
elif command == "forward":
|
|
result = history_step(client, session_id, 1)
|
|
elif command == "reload":
|
|
result = reload_page(client, session_id)
|
|
elif command == "snapshot":
|
|
result = snapshot(client, session_id)
|
|
elif command == "act":
|
|
actions = args.get("actions")
|
|
if not isinstance(actions, list):
|
|
raise RuntimeError("actions must be an array")
|
|
result = act(client, session_id, actions)
|
|
else:
|
|
result = {
|
|
"ok": False,
|
|
"error": f"unknown command: {command}",
|
|
"fallback": "computer_act",
|
|
}
|
|
print(json.dumps(result))
|
|
return 2
|
|
print(json.dumps(result))
|
|
return 0 # Structured failures preserve partial action progress.
|
|
finally:
|
|
client.close()
|
|
except Exception as exc: # noqa: BLE001
|
|
print(json.dumps({"ok": False, "error": str(exc), "fallback": "computer_act",
|
|
"uncertain": command == "act"}))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(run(sys.argv))
|