"""Score raw/ responses against testset.json and price them at official list rates.

Cost = official first-party Standard list price arithmetic (USD per 1M tokens,
checked 2026-11-19), applied to the usage the Ofox response reported.
It is NOT Ofox billing.
"""
import json, re, pathlib

HERE = pathlib.Path(__file__).parent
ts = json.loads((HERE / "testset.json").read_text())
PRICES = {  # input, output (output includes reasoning/thinking)
    "google/gemini-4-hydrogen": (1.50, 7.50),
    "anthropic/claude-haiku-5.5": (0.10, 0.50),   # prompt <= 100,000 tokens
    "openai/gpt-6-luna": (0.10, 0.50),            # input <= 272K tokens
    "google/gemini-3.5-flash-lite": (0.30, 2.50),
}
FIELDS = ("category", "order_id", "refund")


def parse_json_array(text):
    text = text.strip()
    m = re.search(r"\[.*\]", text, re.S)
    return json.loads(m.group(0)) if m else None


def last_answer(text):
    lines = [l for l in text.strip().splitlines() if "ANSWER:" in l.upper()]
    return lines[-1].split(":", 1)[1].strip() if lines else ""


def check_reasoning(r, ans):
    a = ans.lower().replace("*", "").strip()
    if r["check"] == "number":
        nums = re.findall(r"\d+(?:\.\d+)?", a.replace(",", ""))
        return bool(nums) and abs(float(nums[0]) - 3.8064) < 1e-6
    if r["check"] == "sla":
        return ("monday" in a and "13:00" in a and "tuesday" in a and "17:00" in a
                and a.index("monday") < a.index("tuesday"))
    if r["check"] == "yesno":
        return re.findall(r"\b(yes|no)\b", a) == ["yes", "no", "yes"]


out = {}
for model, (pin, pout) in PRICES.items():
    safe = model.replace("/", "_")
    res = {"triage": [], "reasoning": {}, "calls": []}
    for name in ["triage_a", "triage_b"] + [r["id"] for r in ts["reasoning"]]:
        rec = json.loads((HERE / "raw" / f"{safe}__{name}.json").read_text())
        resp = rec["response"]
        u = resp["usage"]
        text = resp["choices"][0]["message"]["content"] or ""
        cost = (u["prompt_tokens"] * pin + u["completion_tokens"] * pout) / 1e6
        rdet = (u.get("completion_tokens_details") or {}).get("reasoning_tokens")
        res["calls"].append({"call": name, "seconds": rec["wall_seconds"], "prompt_tokens": u["prompt_tokens"],
                             "completion_tokens": u["completion_tokens"], "reasoning_tokens": rdet,
                             "list_cost_usd": round(cost, 6), "finish": resp["choices"][0].get("finish_reason")})
        if name.startswith("triage"):
            arr = parse_json_array(text) or []
            byid = {int(o.get("id")): o for o in arr if isinstance(o, dict) and "id" in o}
            field_ok = 0
            item_ok = 0
            misses = []
            for it in ts["items"]:
                o = byid.get(it["id"], {})
                ok = [o.get(f) == it["gold"][f] for f in FIELDS]
                field_ok += sum(ok)
                item_ok += all(ok)
                if not all(ok):
                    misses.append({"id": it["id"], "got": {f: o.get(f) for f in FIELDS}, "gold": it["gold"]})
            res["triage"].append({"call": name, "items_all_correct": item_ok, "fields_correct": field_ok,
                                  "fields_total": 60, "returned": len(arr), "misses": misses})
        else:
            r = next(x for x in ts["reasoning"] if x["id"] == name)
            ans = last_answer(text)
            res["reasoning"][name] = {"answer": ans, "correct": check_reasoning(r, ans)}
    res["total_list_cost_usd"] = round(sum(c["list_cost_usd"] for c in res["calls"]), 6)
    out[model] = res

(HERE / "results.json").write_text(json.dumps(out, indent=1, ensure_ascii=False))
grand = 0
for m, r in out.items():
    grand += r["total_list_cost_usd"]
    print("\n==", m, "total list cost $", r["total_list_cost_usd"])
    for t in r["triage"]:
        print(" ", t["call"], "items", t["items_all_correct"], "/20 fields", t["fields_correct"], "/60 returned", t["returned"])
        for x in t["misses"]:
            print("    miss", x)
    for k, v in r["reasoning"].items():
        print(" ", k, v)
    for c in r["calls"]:
        print("  ", c)
print("\nGRAND list-price total $", round(grand, 6))
