import base64
import json
import os
from openai import OpenAI
from playwright.sync_api import sync_playwright
client = OpenAI(base_url="https://api.hcompany.ai/v1/", api_key=os.environ["HAI_API_KEY"])
MODEL = "holo4-35b-a3b"
WIDTH, HEIGHT = 1280, 800
def fn(name: str, description: str, **properties) -> dict:
return {
"type": "function",
"function": {
"name": name,
"description": description,
"parameters": {"type": "object", "properties": properties, "required": list(properties)},
},
}
INT = {"type": "integer", "description": "Coordinate as integer in [0, 1000]"}
tools = [
fn("click", "Click at (x, y) coordinates", element={"type": "string", "description": "Detailed description of the target UI element"}, x=INT, y=INT),
fn("type", "Type text into the focused element, optionally pressing Enter", text={"type": "string"}, press_enter={"type": "boolean"}),
fn("scroll", "Scroll the page", direction={"type": "string", "enum": ["up", "down"]}),
fn("goto", "Navigate to a URL", url={"type": "string"}),
fn("answer", "Provide a final answer", content={"type": "string", "description": "The answer content"}),
]
def execute(page, name: str, args: dict) -> str:
if name == "click":
page.mouse.click(args["x"] / 1000 * WIDTH, args["y"] / 1000 * HEIGHT)
elif name == "type":
page.keyboard.type(args["text"])
if args.get("press_enter"):
page.keyboard.press("Enter")
elif name == "scroll":
page.mouse.wheel(0, HEIGHT * 0.8 * (1 if args["direction"] == "down" else -1))
elif name == "goto":
page.goto(args["url"])
page.wait_for_timeout(1000)
page.wait_for_load_state()
return f"Done. Current URL: {page.url}"
def observation(page) -> dict:
b64 = base64.b64encode(page.screenshot()).decode()
return {"role": "user", "content": [
{"type": "text", "text": "<observation>\n"},
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}},
{"type": "text", "text": "\n</observation>"},
]}
def trim_to_last_n_images(messages, n=3):
seen = 0
for msg in reversed(messages):
if msg["role"] != "user" or not isinstance(msg["content"], list):
continue
for chunk in msg["content"]:
if chunk.get("type") != "image_url":
continue
seen += 1
if seen > n:
chunk["type"] = "text"
chunk["text"] = "[screenshot evicted]"
chunk.pop("image_url", None)
def run(task: str, start_url: str, max_steps: int = 20) -> str:
with sync_playwright() as p:
page = p.chromium.launch().new_page(viewport={"width": WIDTH, "height": HEIGHT})
page.goto(start_url)
messages = [
{"role": "system", "content": "You are a web agent. You see the browser through screenshots and act with your tools."},
{"role": "user", "content": task},
]
for _ in range(max_steps):
messages.append(observation(page))
trim_to_last_n_images(messages)
resp = client.chat.completions.create(
model=MODEL,
messages=messages,
tools=tools,
tool_choice="required",
temperature=0.8,
extra_body={"chat_template_kwargs": {"enable_thinking": True}},
)
msg = resp.choices[0].message
messages.append({"role": "assistant", "content": msg.content, "tool_calls": msg.tool_calls})
if not msg.tool_calls:
continue
call = msg.tool_calls[0]
args = json.loads(call.function.arguments)
if call.function.name == "answer":
return args["content"]
print(f"--- {call.function.name}({args})")
messages.append({"role": "tool", "tool_call_id": call.id, "content": execute(page, call.function.name, args)})
return "Step budget exhausted"
if __name__ == "__main__":
print(run("Search Wikipedia for Ada Lovelace and tell me her date of birth.", "https://en.wikipedia.org"))
