diff --git a/README.md b/README.md index 0e677b4..31c14d1 100644 --- a/README.md +++ b/README.md @@ -248,6 +248,7 @@ bash tests/llm_proxy_check.sh # fake LLM: 429 chain, python3 tests/nl_check.py http://127.0.0.1:8795 # real model: 6 natural-language prompts python3 tests/test_theme.py # theme spec + customCss sanitiser python3 tests/test_copyedit.py # built-in copy edits (logo/headline/... phrasings) +python3 tests/test_agent_units.py # local-model output normalisers (result focus, preset+reset, listing-form shape) NODE_PATH=... node tests/theme_check.js BASE ./screenshots # chat restyles, guardrail contrast, persistence, hostile stored theme, reset (REQUIRE_LLM=1 to insist on the model) NODE_PATH=/path/to/node_modules node tests/browser_e2e.js http://127.0.0.1:8795 ./screenshots # headless Chrome (14 checks) NODE_PATH=/path/to/node_modules bash tests/fake_model_browser.sh # React UI vs fake model: badges, fallback text, quota, rate-limit toast, sanitised set_theme diff --git a/market/agent.py b/market/agent.py index 926bf5f..2bb6bd7 100644 --- a/market/agent.py +++ b/market/agent.py @@ -677,6 +677,32 @@ def _llm_once(store, cfg, model, message, history, look="", timeout=None): return {"reply": (reply or "Done.")[:600], "actions": actions, "tools": tool_results} +_STOP = set("show me find search look looking for want need some any the a an and or with under over below above cheap " + "please can could you i my our get buy see list all everything something anything thing things stuff near " + "options option there here have has what which that this from about like".split()) + + +def _focus_results(store, message, actions): + """Small local models sometimes pad show_cards with unrelated listings. If some chosen listings share a keyword with + the request and others share none, keep only the matching ones (semantic picks with no keyword overlap stay).""" + words = {w.rstrip("s") for w in re.findall(r"[a-z]{3,}", (message or "").lower()) if w not in _STOP} + if not words: + return actions + for a in actions: + if a.get("type") != "show_cards" or not isinstance(a.get("payload"), dict): + continue + ids = [i for i in (a["payload"].get("listing_ids") or []) if isinstance(i, str)] + def hit(i): + L = store.get("listings", i) or {} + text = " ".join([L.get("title", ""), L.get("category") or "", " ".join(L.get("tags") or []), L.get("description", "")]).lower() + toks = {t.rstrip("s") for t in re.findall(r"[a-z]{3,}", text)} + return bool(words & toks) + keep = [i for i in ids if hit(i)] + if keep and len(keep) < len(ids): + a["payload"]["listing_ids"] = keep + return actions + + def llm_agent(store, message, history, cfg, look=""): """Hosted models in order (skipping ones cooling down after a 429, and all of them while the daily free quota is used up), then the local backup model with its own timeout. Raises if everything fails (keyword rules answer then).""" @@ -740,6 +766,7 @@ def llm_agent(store, message, history, cfg, look=""): out = fut.result(timeout=cap + 1) except concurrent.futures.TimeoutError: raise LLMError("timeout", f">{cap:.0f}s") + out["actions"] = _focus_results(store, message, out["actions"]) out.update(agent="llm", provider="local", local=True, model=local["model"], latency_ms=int((time.time() - t0) * 1000), fallbacks=errors) LAST.update(model=local["model"], at=time.time(), error=None, local=True) diff --git a/tests/test_agent_units.py b/tests/test_agent_units.py new file mode 100644 index 0000000..fe71d10 --- /dev/null +++ b/tests/test_agent_units.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Unit checks for the model-output normalisers used with the local backup model (no server, no model).""" +import os, sys, tempfile +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +os.environ["AGENT_MARKET_DATA"] = tempfile.mkdtemp() +for k in ("AGENT_MARKET_LLM_API_KEY_FILE", "AGENT_MARKET_LOCAL_LLM_URL"): + os.environ.pop(k, None) +import server # noqa: E402 (seeds a temp store) +from market import agent, theme # noqa: E402 + +results = [] +def check(name, ok, d=""): + results.append(bool(ok)); print(("PASS " if ok else "FAIL ") + name + (f" [{d}]" if d else "")) + +st = server.store +by = {L["title"]: L["id"] for L in st.all("listings")} +tut = next(i for t, i in by.items() if "tutoring" in t.lower()); bike = next(i for t, i in by.items() if "kids" in t.lower()) +lawn = next(i for t, i in by.items() if "lawn" in t.lower()) +f = lambda q, ids: agent._focus_results(st, q, [{"type": "show_cards", "payload": {"listing_ids": list(ids)}}])[0]["payload"]["listing_ids"] +check("focus: unrelated padding dropped when some picks match", f("show me bikes", [tut, bike]) == [bike]) +check("focus: semantic picks kept when nothing matches lexically", f("anything for the garden?", [lawn]) == [lawn]) +check("focus: no keywords -> unchanged", f("show me everything", [tut, bike]) == [tut, bike]) + +a = theme.build_theme_action({"preset": "sunset", "reset": True}) +check("theme: preset + reset = start from the preset (not a bare reset)", a and a.get("preset") == "sunset" and a.get("base") and a.get("reset"), a) +a = theme.build_theme_action({"reset": True}) +check("theme: bare reset still resets", a == {"reset": True, "patch": {}}, a) + +h = server.hydrate([{"type": "show_listing_form", "payload": {"title": "Old guitar", "price_cents": 12000, "payment_options": ["cash"]}}], "t") +d = h[0]["payload"]["defaults"] +check("listing form: flat payload + price_cents normalised", d == {"title": "Old guitar", "price": 12000, "payment_options": ["cash"]}, d) +h = server.hydrate([{"type": "show_listing_form", "payload": {"defaults": {"title": "Lamp", "price": 2500}}}], "t") +check("listing form: normal shape untouched", h[0]["payload"]["defaults"] == {"title": "Lamp", "price": 2500}) + +cfg = agent._llm_config() +check("no hosted key and no local model -> built-in agent only", cfg is None) +os.environ.update(AGENT_MARKET_LOCAL_LLM_URL="http://127.0.0.1:9/v1", AGENT_MARKET_LOCAL_LLM_MODEL="m") +cfg = agent._llm_config() +check("local model alone is a valid chain", cfg and cfg["models"] == [] and cfg["local"]["model"] == "m" and cfg["local"]["url"].endswith("/v1/chat/completions")) +print(f"\n{sum(results)}/{len(results)} passed") +sys.exit(0 if all(results) else 1)