leora
6c2a1d6a40
A 4B model miscopied 8-hex listing ids (picked a tutoring id for a kids' bike). The local prompt now shows L-refs, the server maps them back before tools/hydration (unknown refs are dropped as before), tool results are aliased the same way, and refs are scrubbed from replies. Hosted models still see real ids.
66 lines
4.5 KiB
Python
66 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Unit checks for the model-output normalisers used with the local backup model (no server, no model)."""
|
|
import os, re, sys, tempfile
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
os.environ["AGENT_MARKET_DATA"] = tempfile.mkdtemp()
|
|
for k in ("AGENT_MARKET_LLM_API_KEY_FILE", "AGENT_MARKET_LOCAL_LLM_URL"):
|
|
os.environ.pop(k, None)
|
|
import server # noqa: E402 (seeds a temp store)
|
|
from market import agent, theme # noqa: E402
|
|
|
|
results = []
|
|
def check(name, ok, d=""):
|
|
results.append(bool(ok)); print(("PASS " if ok else "FAIL ") + name + (f" [{d}]" if d else ""))
|
|
|
|
st = server.store
|
|
by = {L["title"]: L["id"] for L in st.all("listings")}
|
|
tut = next(i for t, i in by.items() if "tutoring" in t.lower()); bike = next(i for t, i in by.items() if "kids" in t.lower())
|
|
lawn = next(i for t, i in by.items() if "lawn" in t.lower())
|
|
f = lambda q, ids: agent._focus_results(st, q, [{"type": "show_cards", "payload": {"listing_ids": list(ids)}}])[0]["payload"]["listing_ids"]
|
|
check("focus: unrelated padding dropped when some picks match", f("show me bikes", [tut, bike]) == [bike])
|
|
check("focus: semantic picks kept when nothing matches lexically", f("anything for the garden?", [lawn]) == [lawn])
|
|
check("focus: no keywords -> unchanged", f("show me everything", [tut, bike]) == [tut, bike])
|
|
|
|
a = theme.build_theme_action({"preset": "sunset", "reset": True})
|
|
check("theme: preset + reset = start from the preset (not a bare reset)", a and a.get("preset") == "sunset" and a.get("base") and a.get("reset"), a)
|
|
a = theme.build_theme_action({"reset": True})
|
|
check("theme: bare reset still resets", a == {"reset": True, "patch": {}}, a)
|
|
|
|
h = server.hydrate([{"type": "show_listing_form", "payload": {"title": "Old guitar", "price_cents": 12000, "payment_options": ["cash"]}}], "t")
|
|
d = h[0]["payload"]["defaults"]
|
|
check("listing form: flat payload + price_cents normalised", d == {"title": "Old guitar", "price": 12000, "payment_options": ["cash"]}, d)
|
|
h = server.hydrate([{"type": "show_listing_form", "payload": {"defaults": {"title": "Lamp", "price": 2500}}}], "t")
|
|
check("listing form: normal shape untouched", h[0]["payload"]["defaults"] == {"title": "Lamp", "price": 2500})
|
|
|
|
cfg = agent._llm_config()
|
|
check("no hosted key and no local model -> no chain (typed messages get the busy reply)", cfg is None)
|
|
os.environ.update(AGENT_MARKET_LOCAL_LLM_URL="http://127.0.0.1:9/v1", AGENT_MARKET_LOCAL_LLM_MODEL="m")
|
|
cfg = agent._llm_config()
|
|
check("local model alone is a valid chain", cfg and cfg["models"] == [] and cfg["local"]["model"] == "m" and cfg["local"]["url"].endswith("/v1/chat/completions"))
|
|
# local model: short catalog refs in the prompt, mapped back to real ids; unknown refs left for hydrate() to drop
|
|
al = agent._aliases(st)
|
|
sp = agent._system_prompt(st, "", local=True)
|
|
check("local prompt uses short refs, not hex ids", '"id": "L1"' in sp and not re.search(r"lst_[0-9a-f]{6}", sp.split("CATALOG (all active listings)")[1]) and '"L7"' in sp, len(al))
|
|
plan = agent._unalias({"actions": [{"type": "show_cards", "payload": {"listing_ids": ["L1", "L999"]}},
|
|
{"type": "show_checkout", "payload": {"listing_id": "L2"}}]}, al)
|
|
check("refs mapped back (unknown kept for hydrate to drop)", plan["actions"][0]["payload"]["listing_ids"] == [al["L1"], "L999"] and
|
|
plan["actions"][1]["payload"]["listing_id"] == al["L2"])
|
|
check("hosted prompt keeps real ids", al["L1"] in agent._system_prompt(st, "", local=False))
|
|
|
|
# primer: re-prime after the local server restarts (llama-server /metrics counter goes back down) or is down
|
|
import http.server, threading
|
|
M = {"n": 500}
|
|
class _H(http.server.BaseHTTPRequestHandler):
|
|
def log_message(self, *a): pass
|
|
def do_GET(self):
|
|
b = f"# HELP x\nllamacpp:n_decode_total {M['n']}\n".encode(); self.send_response(200); self.end_headers(); self.wfile.write(b)
|
|
srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _H); threading.Thread(target=srv.serve_forever, daemon=True).start()
|
|
loc = {"url": f"http://127.0.0.1:{srv.server_port}/v1/chat/completions"}
|
|
agent._PRIMED["decodes"] = None
|
|
first = agent._local_restarted(loc); M["n"] = 520; grew = agent._local_restarted(loc); M["n"] = 7; dropped = agent._local_restarted(loc)
|
|
srv.shutdown(); down = agent._local_restarted(loc)
|
|
check("primer: restart detected from /metrics, normal use is not a restart, down = re-prime",
|
|
(first, grew, dropped, down) == (False, False, True, True), (first, grew, dropped, down))
|
|
print(f"\n{sum(results)}/{len(results)} passed")
|
|
sys.exit(0 if all(results) else 1)
|