"""Small budget-tier test via Ofox Chat Completions (2026-11-19).

Usage: OFOX_API_KEY=... python3 run_test.py [model ...]
Each model: triage batch (20 items) twice + 3 reasoning prompts once = 5 calls.
Default model settings: no temperature, no reasoning/thinking parameter.
Only max_tokens=32000 is set, as a spending cap.
Raw responses are saved to raw/<model>__<call>.json (no key stored).
"""
import json, os, sys, time, urllib.request, pathlib

HERE = pathlib.Path(__file__).parent
BASE = "https://api.ofox.ai/v1/chat/completions"
MODELS = sys.argv[1:] or [
    "google/gemini-4-hydrogen",
    "anthropic/claude-haiku-5.5",
    "openai/gpt-6-luna",
    "google/gemini-3.5-flash-lite",
]
MAX_TOKENS = 32000
# Optional follow-up: EXTRA='{"reasoning_effort":"low"}' SUFFIX=low ONLY=triage python3 run_test.py google/gemini-4-hydrogen
EXTRA = json.loads(os.environ.get("EXTRA", "{}"))
SUFFIX = os.environ.get("SUFFIX", "")
ONLY = os.environ.get("ONLY", "")
ts = json.loads((HERE / "testset.json").read_text())
raw_dir = HERE / "raw"
raw_dir.mkdir(exist_ok=True)


def call(model, messages):
    body = {"model": model, "messages": messages, "max_tokens": MAX_TOKENS, **EXTRA}
    req = urllib.request.Request(
        BASE,
        data=json.dumps(body).encode(),
        headers={
            "Authorization": "Bearer " + os.environ["OFOX_API_KEY"],
            "Content-Type": "application/json",
        },
    )
    t0 = time.time()
    try:
        with urllib.request.urlopen(req, timeout=600) as r:
            resp = json.loads(r.read())
            status = r.status
    except urllib.error.HTTPError as e:
        resp = {"error_body": e.read().decode(errors="replace")}
        status = e.code
    return body, resp, status, round(time.time() - t0, 2)


triage_user = "Messages:\n" + "\n".join(
    json.dumps({"id": it["id"], "text": it["text"]}, ensure_ascii=False) for it in ts["items"]
)

for model in MODELS:
    safe = model.replace("/", "_")
    jobs = [("triage_a", [{"role": "system", "content": ts["system_triage"]}, {"role": "user", "content": triage_user}]),
            ("triage_b", [{"role": "system", "content": ts["system_triage"]}, {"role": "user", "content": triage_user}])]
    for r in ts["reasoning"]:
        jobs.append((r["id"], [{"role": "user", "content": r["prompt"]}]))
    if ONLY == "triage":
        jobs = jobs[:2]
    for name, msgs in jobs:
        name = name + ("_" + SUFFIX if SUFFIX else "")
        body, resp, status, secs = call(model, msgs)
        rec = {"date": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "endpoint": BASE, "model": model,
               "request": body, "http_status": status, "wall_seconds": secs, "response": resp}
        (raw_dir / f"{safe}__{name}.json").write_text(json.dumps(rec, ensure_ascii=False, indent=1))
        u = resp.get("usage", {}) if isinstance(resp, dict) else {}
        print(model, name, status, secs, u.get("prompt_tokens"), u.get("completion_tokens"), flush=True)
