Computer Use

import os
import base64
from openai import OpenAI

# 初始化客户端
client = OpenAI(api_key=os.environ["API_KEY"])

# 模拟屏幕截图（实际应使用 pyautogui.screenshot）
with open("screenshot.png", "rb") as f:
    img_b64 = base64.b64encode(f.read()).decode()

# 调用多模态模型
resp = client.chat.completions.create(
    model="gpt-4o",
    messages=[
        {"role": "system", "content": "你是一个桌面操作助手。输出 JSON：{\"action\": \"click\", \"x\": 100, \"y\": 200}"},
        {"role": "user", "content": [
            {"type": "text", "text": "点击登录按钮"},
            {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}}
        ]}
    ],
    response_format={"type": "json_object"}
)

import json
action = json.loads(resp.choices[0].message.content)
print(f"执行操作: {action['action']} 坐标: ({action['x']}, {action['y']})")

最短闭环：读取截图 → 调用多模态模型 → 解析动作指令。真实环境需替换截图源与坐标执行逻辑。

{
  "id": "chatcmpl-xxx",
  "choices": [
    {
      "index": 0,
      "message": {
        "role": "assistant",
        "content": "{\"action\": \"click\", \"x\": 450, \"y\": 320}"
      },
      "finish_reason": "stop"
    }
  ],
  "usage": {
    "prompt_tokens": 1500,
    "completion_tokens": 25,
    "total_tokens": 1525
  }
}