42 lines
3.1 KiB
Python
Executable File
42 lines
3.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Natural-language checks against the real hosted model. Usage: nl_check.py BASE_URL
|
|
Each prompt should be answered by the LLM (not the fallback) with the expected panel."""
|
|
import json, sys, time, urllib.request, urllib.error
|
|
BASE = (sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:8795").rstrip("/")
|
|
fee_bps = json.load(urllib.request.urlopen(BASE + "/api/status"))["fee_bps"]
|
|
CASES = [
|
|
("I need something to get my kid to school, under $100", "show_cards", lambda p: any("kid" in l["title"].lower() for l in p["listings"]) and all(l["price"] <= 10000 for l in p["listings"])),
|
|
("what can I get for my garden?", "show_cards", lambda p: any("lawn" in l["title"].lower() for l in p["listings"])),
|
|
("list my old guitar for $120, cash only", "show_listing_form", lambda p: p["defaults"].get("price") == 12000 and p["defaults"].get("payment_options") == ["cash"] and "guitar" in p["defaults"].get("title", "").lower()),
|
|
("compare the two tutoring options", "show_compare", lambda p: len(p["listings"]) == 2 and all("tutoring" in l["title"].lower() for l in p["listings"])),
|
|
("I'd like to buy the ceramic lamp and pay online", "show_checkout", lambda p: p["intent"]["fee_cents"] == round(p["intent"]["subtotal_cents"] * fee_bps / 10000) and p["intent"]["fee_bps"] == fee_bps),
|
|
("can I just pay cash for the lawn mowing when they come round?", "show_cash_meetup", lambda p: p["deal"]["fee_cents"] == 0),
|
|
]
|
|
ok_n, times, history = 0, [], []
|
|
for prompt, want, pred in CASES:
|
|
t = time.time()
|
|
req = urllib.request.Request(BASE + "/api/chat", data=json.dumps({"message": prompt, "history": [], "actor": "guest-nl-test"}).encode(),
|
|
headers={"Content-Type": "application/json"})
|
|
try:
|
|
out = json.load(urllib.request.urlopen(req, timeout=90))
|
|
except urllib.error.HTTPError as e:
|
|
out = json.loads(e.read() or b"{}")
|
|
dt = time.time() - t
|
|
acts = [a for a in out.get("actions", []) if a["type"] == want]
|
|
good = out.get("agent") == "llm" and bool(acts) and pred(acts[0]["payload"])
|
|
ok_n += good; times.append(dt)
|
|
extra = ""
|
|
if acts and want == "show_checkout":
|
|
it = acts[0]["payload"]["intent"]; extra = f" price ${it['subtotal_cents']/100:.2f} fee ${it['fee_cents']/100:.2f} total ${it['total_cents']/100:.2f}"
|
|
if acts and want == "show_cash_meetup":
|
|
extra = f" cash fee ${acts[0]['payload']['deal']['fee_cents']/100:.2f}"
|
|
if acts and want in ("show_cards", "show_compare"):
|
|
extra = " -> " + " | ".join(l["title"] for l in acts[0]["payload"]["listings"])
|
|
if acts and want == "show_listing_form":
|
|
extra = " -> " + json.dumps(acts[0]["payload"]["defaults"])
|
|
print(f"{'PASS' if good else 'FAIL'} [{out.get('agent')}/{(out.get('model') or '').split('/')[-1]} {dt:.1f}s"
|
|
f"{' fb=' + ','.join(f['error'] for f in out.get('fallbacks', [])) if out.get('fallbacks') else ''}] {prompt!r}\n {[a['type'] for a in out.get('actions', [])]}{extra}\n reply: {out.get('reply', '')[:200]!r}")
|
|
times.sort()
|
|
print(f"\n{ok_n}/{len(CASES)} passed; median {times[len(times)//2]:.1f}s, max {times[-1]:.1f}s")
|
|
sys.exit(0 if ok_n == len(CASES) else 1)
|