leora
70f5b0525e
Root cause of the 45 s timeouts: the local model got the full hosted prompt (~3.1k tokens with examples). Any prompt-cache miss meant ~40 s of CPU prefill on the single slot, and every request queued behind one paid it too. - LOCAL_PROMPT/_local_system_prompt: terse action list, one-line examples, default page text, one line per listing (L-refs, dollar prices). No per-visitor data in it, so the cache always hits; it only changes with the catalog. - _local_user: look overrides (if any) + keyword/budget MATCHING LISTINGS hint + 'VISITOR: <message>'. - max_tokens 200 for the local model; over-budget card picks dropped when some fit (_respect_budget). - Buy vs sell examples separated (sell my own thing -> listing form, never checkout). - 18/18 correct on typical prompts, warm median 5.9 s, max 7.7 s; cold prefill ~13 s.
74 lines
5.4 KiB
Python
74 lines
5.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Unit checks for the model-output normalisers used with the local backup model (no server, no model)."""
|
|
import os, re, sys, tempfile
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
os.environ["AGENT_MARKET_DATA"] = tempfile.mkdtemp()
|
|
for k in ("AGENT_MARKET_LLM_API_KEY_FILE", "AGENT_MARKET_LOCAL_LLM_URL"):
|
|
os.environ.pop(k, None)
|
|
import server # noqa: E402 (seeds a temp store)
|
|
from market import agent, theme # noqa: E402
|
|
|
|
results = []
|
|
def check(name, ok, d=""):
|
|
results.append(bool(ok)); print(("PASS " if ok else "FAIL ") + name + (f" [{d}]" if d else ""))
|
|
|
|
st = server.store
|
|
by = {L["title"]: L["id"] for L in st.all("listings")}
|
|
tut = next(i for t, i in by.items() if "tutoring" in t.lower()); bike = next(i for t, i in by.items() if "kids" in t.lower())
|
|
lawn = next(i for t, i in by.items() if "lawn" in t.lower())
|
|
f = lambda q, ids: agent._focus_results(st, q, [{"type": "show_cards", "payload": {"listing_ids": list(ids)}}])[0]["payload"]["listing_ids"]
|
|
check("focus: unrelated padding dropped when some picks match", f("show me bikes", [tut, bike]) == [bike])
|
|
check("focus: semantic picks kept when nothing matches lexically", f("anything for the garden?", [lawn]) == [lawn])
|
|
check("focus: no keywords -> unchanged", f("show me everything", [tut, bike]) == [tut, bike])
|
|
|
|
a = theme.build_theme_action({"preset": "sunset", "reset": True})
|
|
check("theme: preset + reset = start from the preset (not a bare reset)", a and a.get("preset") == "sunset" and a.get("base") and a.get("reset"), a)
|
|
a = theme.build_theme_action({"reset": True})
|
|
check("theme: bare reset still resets", a == {"reset": True, "patch": {}}, a)
|
|
|
|
h = server.hydrate([{"type": "show_listing_form", "payload": {"title": "Old guitar", "price_cents": 12000, "payment_options": ["cash"]}}], "t")
|
|
d = h[0]["payload"]["defaults"]
|
|
check("listing form: flat payload + price_cents normalised", d == {"title": "Old guitar", "price": 12000, "payment_options": ["cash"]}, d)
|
|
h = server.hydrate([{"type": "show_listing_form", "payload": {"defaults": {"title": "Lamp", "price": 2500}}}], "t")
|
|
check("listing form: normal shape untouched", h[0]["payload"]["defaults"] == {"title": "Lamp", "price": 2500})
|
|
|
|
cfg = agent._llm_config()
|
|
check("no hosted key and no local model -> no chain (typed messages get the busy reply)", cfg is None)
|
|
os.environ.update(AGENT_MARKET_LOCAL_LLM_URL="http://127.0.0.1:9/v1", AGENT_MARKET_LOCAL_LLM_MODEL="m")
|
|
cfg = agent._llm_config()
|
|
check("local model alone is a valid chain", cfg and cfg["models"] == [] and cfg["local"]["model"] == "m" and cfg["local"]["url"].endswith("/v1/chat/completions"))
|
|
# local model: short catalog refs in the prompt, mapped back to real ids; unknown refs left for hydrate() to drop
|
|
al = agent._aliases(st)
|
|
sp = agent._local_system_prompt(st)
|
|
check("local prompt: compact, short refs + dollar prices, no hex ids, no per-visitor data",
|
|
"\nL1 | " in sp and not re.search(r"lst_[0-9a-f]{6}", sp) and "$" in sp.split("LISTINGS")[1] and "CURRENT LOOK" not in sp and len(sp) < 6000, len(sp))
|
|
check("local prompt is the same for every visitor (prompt cache hits)", sp == agent._local_system_prompt(st))
|
|
u = agent._local_user("show me bikes under $100", '{"mode":"light"}', st)
|
|
check("per-visitor part: look, budget-aware hints, message", u.startswith("CURRENT LOOK") and "MATCHING LISTINGS" in u and "within budget" in u and u.endswith("VISITOR: show me bikes under $100"), u)
|
|
check("no hint for a text edit", "MATCHING" not in agent._local_user("replace agent market to new internet on the top left corner", "", st))
|
|
plan = agent._unalias({"actions": [{"type": "show_cards", "payload": {"listing_ids": ["L1", "L999"]}},
|
|
{"type": "show_checkout", "payload": {"listing_id": "L2"}}]}, al)
|
|
check("refs mapped back (unknown kept for hydrate to drop)", plan["actions"][0]["payload"]["listing_ids"] == [al["L1"], "L999"] and
|
|
plan["actions"][1]["payload"]["listing_id"] == al["L2"])
|
|
check("hosted prompt keeps real ids", al["L1"] in agent._system_prompt(st, ""))
|
|
cheap = min(st.all("listings"), key=lambda L: L["price"]); dear = max(st.all("listings"), key=lambda L: L["price"])
|
|
b = agent._respect_budget(st, f"anything under ${cheap['price'] // 100 + 1}", [{"type": "show_cards", "payload": {"listing_ids": [dear["id"], cheap["id"]]}}])
|
|
check("budget: over-budget picks dropped when some fit", b[0]["payload"]["listing_ids"] == [cheap["id"]], b)
|
|
|
|
# primer: re-prime after the local server restarts (llama-server /metrics counter goes back down) or is down
|
|
import http.server, threading
|
|
M = {"n": 500}
|
|
class _H(http.server.BaseHTTPRequestHandler):
|
|
def log_message(self, *a): pass
|
|
def do_GET(self):
|
|
b = f"# HELP x\nllamacpp:n_decode_total {M['n']}\n".encode(); self.send_response(200); self.end_headers(); self.wfile.write(b)
|
|
srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _H); threading.Thread(target=srv.serve_forever, daemon=True).start()
|
|
loc = {"url": f"http://127.0.0.1:{srv.server_port}/v1/chat/completions"}
|
|
agent._PRIMED["decodes"] = None
|
|
first = agent._local_restarted(loc); M["n"] = 520; grew = agent._local_restarted(loc); M["n"] = 7; dropped = agent._local_restarted(loc)
|
|
srv.shutdown(); down = agent._local_restarted(loc)
|
|
check("primer: restart detected from /metrics, normal use is not a restart, down = re-prime",
|
|
(first, grew, dropped, down) == (False, False, True, True), (first, grew, dropped, down))
|
|
print(f"\n{sum(results)}/{len(results)} passed")
|
|
sys.exit(0 if all(results) else 1)
|