leora
d785091d64
- Remove the keyword-rules agent from typed chat: fallback_agent, parse_look_command (+ tables), the copy-edit parser and _rules_check are gone. If every model fails: 'The assistant is busy, please try again in a moment.' with a Retry button (agent 'unavailable', no actions). Clicks, forms and payments stay deterministic (fallback_event); the landing view is a deterministic featured-listings panel; the Sell button opens the empty form. - Local model via llama.cpp llama-server directly (ops/local-llm-watchdog.sh, loopback only, restart on exit/hang, crash backoff, log rotation; deploy.sh ensures it runs). Lean: no vision projector, no weight repack (mmapped), ctx 6144, 1 slot, q8 KV, 6 threads at nice 10, poll 0, 1 GiB prompt cache. Idle 0% CPU, 2.9 GB RSS (0.6 GB private). - Local requests use a JSON GBNF grammar, worked examples in the cached system prompt, and a labelled visitor message. 10/10 correct on the real prompt, warm 4-13 s. - Primer re-primes after a model-server restart (llama-server /metrics counter) and otherwise stays idle. - Fix: hosted calls kept the Authorization header off when a local model was configured (cfg['local'] is a dict on hosted configs); now only the local server itself skips the key. Test added. - Badge/chat header: 'AI: busy', 'AI: quota used up', 'AI: unavailable'; no 'built-in'/'checked by rules' texts. - Tests: proxy checks expect the busy reply (never rules), clicks still work while busy, no-model server, local grammar/ prompt/no key; browser checks for busy + Retry; e2e_http accepts busy for typed turns unless REQUIRE_LLM=1; test_copyedit.py replaced by test_copy_defaults.py; README updated.
42 lines
3.2 KiB
Python
Executable File
42 lines
3.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Natural-language checks against a real model (hosted, or the local model). Usage: nl_check.py BASE_URL
|
|
Each prompt should be answered by a model (not the busy reply) with the expected panel."""
|
|
import json, sys, time, urllib.request, urllib.error
|
|
BASE = (sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:8795").rstrip("/")
|
|
fee_bps = json.load(urllib.request.urlopen(BASE + "/api/status"))["fee_bps"]
|
|
CASES = [
|
|
("I need something to get my kid to school, under $100", "show_cards", lambda p: any("kid" in l["title"].lower() for l in p["listings"]) and all(l["price"] <= 10000 for l in p["listings"])),
|
|
("what can I get for my garden?", "show_cards", lambda p: any("lawn" in l["title"].lower() for l in p["listings"])),
|
|
("list my old guitar for $120, cash only", "show_listing_form", lambda p: p["defaults"].get("price") == 12000 and p["defaults"].get("payment_options") == ["cash"] and "guitar" in p["defaults"].get("title", "").lower()),
|
|
("compare the two tutoring options", "show_compare", lambda p: len(p["listings"]) == 2 and all("tutoring" in l["title"].lower() for l in p["listings"])),
|
|
("I'd like to buy the ceramic lamp and pay online", "show_checkout", lambda p: p["intent"]["fee_cents"] == round(p["intent"]["subtotal_cents"] * fee_bps / 10000) and p["intent"]["fee_bps"] == fee_bps),
|
|
("can I just pay cash for the lawn mowing when they come round?", "show_cash_meetup", lambda p: p["deal"]["fee_cents"] == 0),
|
|
]
|
|
ok_n, times, history = 0, [], []
|
|
for prompt, want, pred in CASES:
|
|
t = time.time()
|
|
req = urllib.request.Request(BASE + "/api/chat", data=json.dumps({"message": prompt, "history": [], "actor": "guest-nl-test"}).encode(),
|
|
headers={"Content-Type": "application/json"})
|
|
try:
|
|
out = json.load(urllib.request.urlopen(req, timeout=90))
|
|
except urllib.error.HTTPError as e:
|
|
out = json.loads(e.read() or b"{}")
|
|
dt = time.time() - t
|
|
acts = [a for a in out.get("actions", []) if a["type"] == want]
|
|
good = out.get("agent") == "llm" and bool(acts) and pred(acts[0]["payload"])
|
|
ok_n += good; times.append(dt)
|
|
extra = ""
|
|
if acts and want == "show_checkout":
|
|
it = acts[0]["payload"]["intent"]; extra = f" price ${it['subtotal_cents']/100:.2f} fee ${it['fee_cents']/100:.2f} total ${it['total_cents']/100:.2f}"
|
|
if acts and want == "show_cash_meetup":
|
|
extra = f" cash fee ${acts[0]['payload']['deal']['fee_cents']/100:.2f}"
|
|
if acts and want in ("show_cards", "show_compare"):
|
|
extra = " -> " + " | ".join(l["title"] for l in acts[0]["payload"]["listings"])
|
|
if acts and want == "show_listing_form":
|
|
extra = " -> " + json.dumps(acts[0]["payload"]["defaults"])
|
|
print(f"{'PASS' if good else 'FAIL'} [{out.get('agent')}/{(out.get('model') or '').split('/')[-1]} {dt:.1f}s"
|
|
f"{' fb=' + ','.join(f['error'] for f in out.get('fallbacks', [])) if out.get('fallbacks') else ''}] {prompt!r}\n {[a['type'] for a in out.get('actions', [])]}{extra}\n reply: {out.get('reply', '')[:200]!r}")
|
|
times.sort()
|
|
print(f"\n{ok_n}/{len(CASES)} passed; median {times[len(times)//2]:.1f}s, max {times[-1]:.1f}s")
|
|
sys.exit(0 if ok_n == len(CASES) else 1)
|