Function calling

const tools = [
  {
    type: "function",
    function: {
      name: "click",
      description: "Click at (x, y) coordinates",
      parameters: {
        type: "object",
        properties: {
          element: { type: "string", description: "Detailed description of the target UI element" },
          x: { type: "integer", description: "X coordinate as integer in [0, 1000]" },
          y: { type: "integer", description: "Y coordinate as integer in [0, 1000]" },
        },
        required: ["element", "x", "y"],
      },
    },
  },
  {
    type: "function",
    function: {
      name: "answer",
      description: "Provide a final answer",
      parameters: {
        type: "object",
        properties: { content: { type: "string", description: "The answer content" } },
        required: ["content"],
      },
    },
  },
] as const;

import json

messages = [{"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": task}]

for _ in range(MAX_STEPS):
    messages.append(observation())
    trim_to_last_n_images(messages, n=3)

    resp = client.chat.completions.create(
        model="holo4-35b-a3b",
        messages=messages,
        tools=tools,
        tool_choice="required",
        temperature=0.8,
        extra_body={"chat_template_kwargs": {"enable_thinking": True}},
    )
    msg = resp.choices[0].message
    messages.append({"role": "assistant", "content": msg.content, "tool_calls": msg.tool_calls})
    if not msg.tool_calls:
        continue

    call = msg.tool_calls[0]
    args = json.loads(call.function.arguments)
    if call.function.name == "answer":
        return args["content"]
    messages.append({"role": "tool", "tool_call_id": call.id, "content": str(execute(call.function.name, args))})

const messages: any[] = [{ role: "system", content: SYSTEM_PROMPT }, { role: "user", content: task }];

for (let i = 0; i < MAX_STEPS; i++) {
  messages.push(await observation());
  trimToLastNImages(messages, 3);

  const resp = await client.chat.completions.create({
    model: "holo4-35b-a3b",
    messages,
    tools,
    tool_choice: "required",
    temperature: 0.8,
    ...({ chat_template_kwargs: { enable_thinking: true } } as any),
  });
  const msg = resp.choices[0].message;
  messages.push({ role: "assistant", content: msg.content, tool_calls: msg.tool_calls });
  if (!msg.tool_calls?.length) continue;

  const call = msg.tool_calls[0];
  const args = JSON.parse(call.function.arguments);
  if (call.function.name === "answer") return args.content;
  messages.push({ role: "tool", tool_call_id: call.id, content: String(await execute(call.function.name, args)) });
}