From b396a01bc48df91aa779b2310aff395090339e6b Mon Sep 17 00:00:00 2001 From: leora Date: Thu, 8 Oct 2026 19:39:02 +0800 Subject: [PATCH] Rule-based look commands for when the model is unavailable; daily-quota-aware cooldown; honest badge when the built-in agent answers; theme tests --- market/agent.py | 70 +++++++++----------- market/theme.py | 139 +++++++++++++++++++++++++++++++++++++++ static/app.js | 8 ++- tests/e2e_http.py | 13 +++- tests/llm_proxy_check.sh | 17 ++++- tests/mock_llm.py | 13 ++++ tests/theme_check.js | 4 +- 7 files changed, 218 insertions(+), 46 deletions(-) diff --git a/market/agent.py b/market/agent.py index b1c9085..8a6e2b6 100644 --- a/market/agent.py +++ b/market/agent.py @@ -274,44 +274,16 @@ def fallback_event(store, ev: dict, actor: str): return {"reply": "I didn't recognise that action.", "actions": []} -def fallback_theme(low): - """Basic look commands for when the LLM is unavailable.""" - from .theme import PRESETS, PRESET_ALIASES - if re.search(r"\breset\b.*\b(look|theme|style|design|page)\b|\bdefault (look|theme)\b", low): - return {"reply": "Done: the page is back to its default look.", "actions": [{"type": "set_theme", "payload": {"reset": True}}]} - for name in list(PRESETS) + list(PRESET_ALIASES): - if re.search(r"\b" + re.escape(name) + r"\b", low) and re.search(r"look|theme|style|make it|switch|mode|preset|vibe", low): - real = PRESET_ALIASES.get(name, name) - return {"reply": f"Applied the {real} look.", "actions": [{"type": "set_theme", "payload": {"preset": real}}]} - patch, said = {}, [] - if re.search(r"\blight (mode|theme)\b", low): - patch["mode"] = "light"; said.append("light mode") - elif re.search(r"\bdark (mode|theme)\b", low): - patch["mode"] = "dark"; said.append("dark mode") - if re.search(r"as a list|list (view|layout)|not cards", low): - patch["layout"] = "list"; said.append("list layout") - elif re.search(r"grid (view|layout)|as cards", low): - patch["layout"] = "grid"; said.append("grid layout") - if re.search(r"chat.*\bleft\b", low): - patch["chat"] = {"position": "left"}; said.append("chat on the left") - elif re.search(r"chat.*\bright\b", low): - patch["chat"] = {"position": "right"}; said.append("chat on the right") - if re.search(r"(no|stop|turn off|disable) animation", low): - patch["animations"] = False; said.append("animations off") - if not patch: - return None - return {"reply": "Done: " + ", ".join(said) + ".", "actions": [{"type": "set_theme", "payload": patch}]} - - -def fallback_agent(store, message: str, history, actor: str = "demo-buyer"): +def fallback_agent(store, message: str, history, actor: str = "demo-buyer", look=None): m = (message or "").strip() low = m.lower() actions = [] reply = "" - th = fallback_theme(low) + from .theme import parse_look_command + th = parse_look_command(m, look if isinstance(look, dict) else {}) if th: - return th + return {"reply": th[0], "actions": [{"type": "set_theme", "payload": th[1]}]} # welcome / help if not m or low in ("hi", "hello", "hey", "help", "start"): @@ -458,6 +430,7 @@ _pool = concurrent.futures.ThreadPoolExecutor(max_workers=8, thread_name_prefix= COOLDOWN_429 = float(os.environ.get("AGENT_MARKET_LLM_COOLDOWN", "60")) _cooldown = {} # model -> unix time until which we skip it after a 429 LAST = {"model": None, "at": None, "error": None} +QUOTA = {"until": 0} class LLMError(Exception): @@ -510,12 +483,15 @@ def _call_llm(cfg, model, messages, json_mode=True): with urllib.request.urlopen(req, timeout=LLM_TIMEOUT) as r: data = json.loads(r.read().decode()) except urllib.error.HTTPError as e: - body = e.read().decode(errors="replace")[:300] + body = e.read().decode(errors="replace")[:2000] if e.code == 429: - raise LLMError("rate_limited", body) + m = re.search(r'"X-RateLimit-Reset":\s*"?(\d{12,})', body) + if "per-day" in body or "daily" in body: + raise LLMError("daily_quota", m.group(1) if m else "") + raise LLMError("rate_limited", body[:300]) if e.code == 400 and json_mode and "response_format" in body: return _call_llm(cfg, model, messages, json_mode=False) - raise LLMError(f"http_{e.code}", body) + raise LLMError(f"http_{e.code}", body[:300]) except (urllib.error.URLError, TimeoutError, OSError) as e: raise LLMError("timeout_or_network", type(e).__name__) if isinstance(data, dict) and data.get("error"): @@ -583,6 +559,8 @@ def llm_agent(store, message, history, cfg, look=""): """Try each model in the chain; skip models cooling down after a 429. Raises if all fail.""" errors = [] now_t = time.time() + if QUOTA["until"] > now_t: + raise LLMError("all_models_failed", json.dumps([{"model": "*", "error": "daily_quota", "ms": 0}])) chain = [m for m in cfg["models"] if _cooldown.get(m, 0) <= now_t] or cfg["models"][-1:] start = time.time() for model in chain: @@ -604,6 +582,15 @@ def llm_agent(store, message, history, cfg, look=""): except LLMError as e: if e.kind == "rate_limited": _cooldown[model] = time.time() + COOLDOWN_429 + elif e.kind == "daily_quota": + # the free-tier daily cap is per key: park every model until the reset (or 15 min if unknown) + d = str(e).split(": ", 1)[-1] + until = int(d) / 1000 if d.isdigit() else time.time() + 900 + for mm in cfg["models"]: + _cooldown[mm] = max(_cooldown.get(mm, 0), until) + QUOTA["until"] = until + errors.append({"model": model, "error": e.kind, "ms": int((time.time() - t0) * 1000)}) + break errors.append({"model": model, "error": e.kind, "ms": int((time.time() - t0) * 1000)}) except Exception as e: # never let the page break errors.append({"model": model, "error": type(e).__name__, "ms": int((time.time() - t0) * 1000)}) @@ -620,6 +607,10 @@ LLM_EVENTS = os.environ.get("AGENT_MARKET_LLM_EVENTS") == "1" def handle_chat(store, message: str, history=None, event=None, actor="demo-buyer", look=""): actor = (actor or "demo-buyer")[:60] t0 = time.time() + try: + look_d = json.loads(look) if look else {} + except ValueError: + look_d = {} if event and (event.get("type") == "landing" or not LLM_EVENTS): if event.get("type") == "landing": out = fallback_agent(store, "show me everything", [], actor) @@ -633,13 +624,13 @@ def handle_chat(store, message: str, history=None, event=None, actor="demo-buyer try: return llm_agent(store, text, history or [], cfg, look) except Exception as e: - out = fallback_event(store, event, actor) if event else fallback_agent(store, message, history or [], actor) + out = fallback_event(store, event, actor) if event else fallback_agent(store, message, history or [], actor, look_d) out.update(agent="fallback", llm_error=getattr(e, "kind", type(e).__name__), fallbacks=json.loads(str(e).split(": ", 1)[1]) if getattr(e, "kind", "") == "all_models_failed" else [], latency_ms=int((time.time() - t0) * 1000)) - out["reply"] = "(The AI model is busy right now, so the simple built-in agent answered.) " + out["reply"] + out["reply"] = "(AI model unavailable right now; the built-in agent answered.) " + out["reply"] return out - out = fallback_event(store, event, actor) if event else fallback_agent(store, message, history or [], actor) + out = fallback_event(store, event, actor) if event else fallback_agent(store, message, history or [], actor, look_d) out.update(agent="fallback", latency_ms=int((time.time() - t0) * 1000)) return out @@ -650,6 +641,7 @@ def agent_status(): now_t = time.time() return {"mode": "llm", "provider": cfg["provider"], "model": cfg["models"][0], "chain": cfg["models"], "cooling_down": [m for m in cfg["models"] if _cooldown.get(m, 0) > now_t], - "last_model": LAST["model"], "last_error": LAST["error"], "ui_events": "llm" if LLM_EVENTS else "deterministic"} + "last_model": LAST["model"], "last_error": LAST["error"], "ui_events": "llm" if LLM_EVENTS else "deterministic", + "daily_quota_exhausted_until_utc": time.strftime("%Y-%m-%dT%H:%MZ", time.gmtime(QUOTA["until"])) if QUOTA["until"] > now_t else None} return {"mode": "fallback", "provider": "deterministic", "model": None, "note": "No hosted LLM configured (set AGENT_MARKET_LLM_API_KEY_FILE or AGENT_MARKET_LLM_API_KEY, plus AGENT_MARKET_LLM_BASE_URL and AGENT_MARKET_LLM_MODELS). Using the labelled rule-based fallback agent."} diff --git a/market/theme.py b/market/theme.py index df3c07d..dff1502 100644 --- a/market/theme.py +++ b/market/theme.py @@ -278,3 +278,142 @@ def compact_current(theme): t.pop("customCss", None) s = json.dumps(t, separators=(",", ":")) return s if len(s) <= 900 else s[:900] + + +# ---------- rule-based look commands (used when the hosted model is unavailable) ---------- +NAMED = {"red": "#e5484d", "green": "#2fb36d", "blue": "#3d7bfd", "purple": "#8e4ec6", "violet": "#8e4ec6", "pink": "#e93d82", + "orange": "#f76b15", "yellow": "#ffd23f", "teal": "#12a594", "cyan": "#00b8d9", "black": "#0b0b0f", "white": "#ffffff", + "grey": "#8b8d98", "gray": "#8b8d98", "navy": "#1b2a4a", "gold": "#d4a72c", "brown": "#8d5a3b", "beige": "#efe3cf", + "cream": "#fff8e7", "mint": "#9de0c4", "lavender": "#c9b8f0", "coral": "#ff7f6e", "lime": "#9ae600", "magenta": "#d62fd6", + "maroon": "#6d1a2b", "olive": "#7a8c3c", "sky": "#8ecdf7", "peach": "#ffcfa8", "turquoise": "#2ad4c4", "indigo": "#4b4bd6"} +GRADIENTS = {"sunset": ["#ff7e5f", "#feb47b", "#6a3093"], "ocean": ["#0f2027", "#203a43", "#2c5364"], "forest": ["#134e5e", "#71b280"], + "candy": ["#f9a8d4", "#a5b4fc"], "fire": ["#f12711", "#f5af19"], "night": ["#0f0c29", "#302b63", "#24243e"], + "peach": ["#ffecd2", "#fcb69f"], "aurora": ["#00c9a7", "#845ec2", "#0081cf"], "rainbow": ["#ff5f6d", "#ffc371", "#47e891", "#3d7bfd", "#8e4ec6"]} +SECTION_WORDS = {"hero": "hero", "headline": "hero", "title": "hero", "chips": "chips", "quick actions": "chips", "quick buttons": "chips", + "buttons": "chips", "bullets": "bullets", "bullet points": "bullets", "facts": "bullets", "results": "results", + "listings": "results", "footer": "footer"} +_CW = "|".join(sorted(NAMED, key=len, reverse=True)) + + +def parse_look_command(text, current=None): + """Map a plain-English look request to a set_theme payload. Returns (reply, payload) or None.""" + low = (text or "").lower().strip() + cur = current if isinstance(current, dict) else {} + if re.search(r"\breset\b.*\b(look|theme|style|design|page|colou?rs?)\b|\b(default|original) (look|theme)\b|^reset$", low): + return "Done: the page is back to its default look.", {"reset": True} + for name in list(PRESETS) + list(PRESET_ALIASES): + if re.search(r"\b" + re.escape(name) + r"\b", low) and re.search(r"look|theme|style|make it|switch|preset|vibe|feel", low) \ + and not re.search(r"background|gradient", low): + real = PRESET_ALIASES.get(name, name) + return f"Applied the {real} look. Say “reset the look” to go back.", {"preset": real} + patch, said = {}, [] + m = re.search(r"(headline|title|heading)\s+(?:to|say|says|reads?|:)\s+[\"“']?(.+?)[\"”']?$", text or "", re.I) + if m: + patch.setdefault("copy", {})["headline"] = m.group(2).strip()[:120]; said.append("new headline") + m = re.search(r"(subtitle|subhead|tagline)\s+(?:to|:)\s+[\"“']?(.+?)[\"”']?$", text or "", re.I) + if m: + patch.setdefault("copy", {})["subhead"] = m.group(2).strip()[:240]; said.append("new subtitle") + m = re.search(r"(logo|site name|brand(?: name)?)\s+(?:to|:)\s+[\"“']?(.+?)[\"”']?$", text or "", re.I) + if m: + patch.setdefault("copy", {})["logo"] = m.group(2).strip()[:40]; said.append("new logo text") + if patch.get("copy"): + return "Done: " + ", ".join(said) + ".", patch + if re.search(r"\blight (mode|theme)\b|\bmake it light\b", low): + patch["mode"] = "light"; said.append("light mode") + elif re.search(r"\bdark (mode|theme)\b|\bmake it dark\b", low): + patch["mode"] = "dark"; said.append("dark mode") + if "background" in low or "gradient" in low or re.search(r"\b(dots|polka|stripes|checks|checkered|grid pattern|waves)\b", low): + g = next((k for k in GRADIENTS if k in low), None) + cols = re.findall(r"\b(" + _CW + r")\b", low) + pat = next((p for k, p in (("polka", "dots"), ("dot", "dots"), ("stripe", "stripes"), ("check", "checks"), + ("diagonal", "diagonal"), ("wave", "waves"), ("grid", "grid")) if k in low), None) + if g: + patch["background"] = {"kind": "gradient", "colors": GRADIENTS[g], "angle": 180}; said.append(f"{g} gradient background") + elif pat and "background" in low or pat and "pattern" in low: + c = [NAMED[x] for x in cols[:2]] or ["#1d212a", "#2a2f3a"] + patch["background"] = {"kind": "pattern", "pattern": pat, "colors": c if len(c) > 1 else [c[0], "#ffffff22"]}; said.append(f"{pat} pattern background") + elif "gradient" in low and cols: + c = [NAMED[x] for x in cols[:4]] + patch["background"] = {"kind": "gradient", "colors": c if len(c) > 1 else [c[0], "#000000"], "angle": 135}; said.append("gradient background") + elif cols: + patch["background"] = {"kind": "solid", "colors": [NAMED[cols[0]]]}; said.append(f"{cols[0]} background") + m = re.search(r"\b(" + _CW + r")\s+accent|accent(?: colou?r)?\s+(?:to\s+)?(" + _CW + r")\b", low) + if m: + c = m.group(1) or m.group(2); patch.setdefault("palette", {})["accent"] = NAMED[c]; said.append(f"{c} accent") + m = re.search(r"\b(" + _CW + r")\s+text\b|\btext\s+(?:colou?r\s+)?(?:to\s+)?(" + _CW + r")\b", low) + if m: + c = m.group(1) or m.group(2); patch.setdefault("palette", {})["text"] = NAMED[c]; said.append(f"{c} text") + for name in sorted(FONTS, key=len, reverse=True): + if name.lower() in low and name != "system": + patch["font"] = name; said.append(f"{name} font"); break + else: + for alias, name in (("monospace", "mono"), ("serif", "serif"), ("handwrit", "Pacifico"), ("script", "Pacifico"), + ("pixel", "Press Start 2P"), ("comic", "Comic Neue"), ("typewriter", "Space Mono"), ("sans", "Inter")): + if alias in low and "font" in low and not ("sans" in low and alias == "serif"): + patch["font"] = name; said.append(f"{name} font"); break + scale = float(cur.get("fontScale") or 1) + if re.search(r"(bigger|larger|increase|bump up).{0,12}(text|font)|(text|font).{0,12}(bigger|larger)", low): + patch["fontScale"] = round(min(1.75, scale + 0.15), 2); said.append(f"text size {patch['fontScale']}×") + elif re.search(r"(smaller|decrease|reduce).{0,12}(text|font)|(text|font).{0,12}smaller", low): + patch["fontScale"] = round(max(0.75, scale - 0.15), 2); said.append(f"text size {patch['fontScale']}×") + if re.search(r"sharp|square corners|no rounding", low): + patch["radius"] = 0; said.append("sharp corners") + elif re.search(r"pill|very rounded|super rounded", low): + patch["radius"] = 30; said.append("pill corners") + elif re.search(r"rounded|round corners|softer corners", low): + patch["radius"] = 22; said.append("rounded corners") + if re.search(r"compact|denser|tighter|less spac", low): + patch["density"] = "compact"; said.append("compact spacing") + elif re.search(r"spacious|airy|more space|roomier|comfy", low): + patch["density"] = "comfy"; said.append("roomier spacing") + for k, v in (("flat", "flat"), ("outlined", "outlined"), ("outline", "outlined"), ("shadow", "elevated"), ("elevated", "elevated"), ("glass", "glass")): + if re.search(r"\b" + k + r"\b", low) and ("card" in low or "style" in low): + patch["cardStyle"] = v; said.append(f"{v} cards"); break + if re.search(r"as a list|list (view|layout)|not cards|as rows", low): + patch["layout"] = "list"; said.append("list layout") + elif re.search(r"grid (view|layout)|as cards|as a grid", low): + patch["layout"] = "grid"; said.append("grid layout") + m = re.search(r"\b([1-6]|one|two|three|four|five|six)\s+columns?\b", low) + if m: + n = {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6}.get(m.group(1)) or int(m.group(1)) + patch["columns"] = n; patch.setdefault("layout", "grid"); said.append(f"{n} columns") + if "chat" in low: + ch = {} + if re.search(r"\bleft\b", low): + ch["position"] = "left" + elif re.search(r"\bright\b", low): + ch["position"] = "right" + if re.search(r"(bigger|larger|wider|taller)", low): + ch.update(width=480, height=680) + elif re.search(r"(smaller|narrower)", low): + ch.update(width=320, height=420) + if ch: + patch["chat"] = ch; said.append("chat " + ", ".join(f"{k} {v}" for k, v in ch.items())) + if re.search(r"(no|stop|turn off|disable|without) animations?|animations? off", low): + patch["animations"] = False; said.append("animations off") + elif re.search(r"(turn on|enable) animations?|animations? on", low): + patch["animations"] = True; said.append("animations on") + hide = re.search(r"\b(hide|remove|get rid of)\b(.+)", low) + show = re.search(r"\b(show|unhide|bring back)\b(.+)", low) + hidden = list((cur.get("sections") or {}).get("hidden") or []) + for verb, mm in (("hide", hide), ("show", show)): + if not mm: + continue + for w, sec in SECTION_WORDS.items(): + if re.search(r"\b" + w + r"\b", mm.group(2)): + if verb == "hide" and sec not in hidden: + hidden.append(sec); said.append(f"{sec} hidden"); patch["sections"] = {"hidden": hidden} + elif verb == "show" and sec in hidden: + hidden.remove(sec); said.append(f"{sec} shown"); patch["sections"] = {"hidden": hidden} + break + m = re.search(r"move (?:the )?(\w+(?: \w+)?) to the (top|bottom)", low) + if m and "chat" not in m.group(1): + sec = next((s for w, s in SECTION_WORDS.items() if w in m.group(1)), None) + if sec: + order = [x for x in ((cur.get("sections") or {}).get("order") or SECTIONS) if x != sec] + order = [sec] + order if m.group(2) == "top" else order + [sec] + patch.setdefault("sections", {})["order"] = order; said.append(f"{sec} moved to the {m.group(2)}") + patch = sanitize_patch(patch) + if not patch: + return None + return "Done: " + ", ".join(said) + ".", patch diff --git a/static/app.js b/static/app.js index 5c6e718..2ee48c8 100644 --- a/static/app.js +++ b/static/app.js @@ -51,7 +51,10 @@ history.push({ role: "assistant", content: out.reply || "" }); say("bot", out.reply || "(no reply)"); if (out.agent === "llm") setModel(out.model, out.latency_ms); - else if (out.agent === "fallback") $("#chatMode").textContent = "built-in fallback agent answered (AI model busy)"; + else if (out.agent === "fallback") { + $("#chatMode").textContent = "built-in agent answered (AI model unavailable)"; + $("#agentBadge").textContent = "agent: built-in (AI unavailable)"; $("#agentBadge").title = "The hosted model is rate-limited or out of free quota"; + } else if (out.agent === "rate-limit") toast(out.reply, "warn"); render(out.actions || []); } catch (e) { thinking.remove(); say("sys", "Agent error: " + e.message); } @@ -222,7 +225,8 @@ $("#resetLook").addEventListener("click", () => { AMTheme.reset(); toast("Look reset to default", "ok"); }); api("/api/status").then((s) => { feeBps = s.fee_bps; AMTheme.init(copyDefaults(feeBps)); - if (s.agent.mode === "llm") setModel(s.agent.last_model || s.agent.model); + if (s.agent.mode === "llm" && s.agent.daily_quota_exhausted_until_utc) { $("#agentBadge").textContent = "agent: built-in (AI quota used up)"; $("#agentBadge").title = "Hosted model free quota resets " + s.agent.daily_quota_exhausted_until_utc; } + else if (s.agent.mode === "llm") setModel(s.agent.last_model || s.agent.model); else { $("#agentBadge").textContent = "agent: rule-based fallback"; $("#agentBadge").title = s.agent.note || ""; } }); // Landing: show featured listings via the agent. diff --git a/tests/e2e_http.py b/tests/e2e_http.py index 2140aa2..869ff79 100755 --- a/tests/e2e_http.py +++ b/tests/e2e_http.py @@ -40,7 +40,11 @@ s, out = call("POST", "/api/chat", {"message": "show me bikes"}) cards = [a for a in out["actions"] if a["type"] == "show_cards"] titles = [l["title"] for l in cards[0]["payload"]["listings"]] if cards else [] check("agent search renders cards (hydrated listings)", cards and any("bike" in t.lower() for t in titles), ", ".join(titles)) -bike = next(l for l in cards[0]["payload"]["listings"] if l["kind"] == "product" and "online" in l["payment_options"]) +bike = next((l for l in cards[0]["payload"]["listings"] if l["kind"] == "product" and "online" in l["payment_options"]), None) if cards else None +if not bike: # earlier runs may have sold the seed bike; list a fresh test-only one + s, r = call("POST", "/api/actions", {"action": "create_listing", "args": {"title": "Test-only commuter bike", "kind": "product", "price": 18000, + "description": "Fake bike for automated tests", "payment_options": ["online", "cash"]}}, {"X-Agent-Id": "test-seller-e2e"}) + bike = r["listing"] s, out = call("POST", "/api/chat", {"message": "I want to sell something"}) check("agent opens listing form", any(a["type"] == "show_listing_form" for a in out["actions"])) @@ -73,7 +77,12 @@ check("forged webhook rejected", s == 401) # Cash in person: zero fee. s, out = call("POST", "/api/chat", {"message": "show lamp"}) -lamp = [l for a in out["actions"] if a["type"] == "show_cards" for l in a["payload"]["listings"] if "cash" in l["payment_options"]][0] +lamps = [l for a in out["actions"] if a["type"] == "show_cards" for l in a["payload"]["listings"] if "cash" in l["payment_options"] and l["kind"] == "product"] +if not lamps: + s, r = call("POST", "/api/actions", {"action": "create_listing", "args": {"title": "Test-only lamp", "kind": "product", "price": 3000, + "description": "Fake lamp for automated tests", "payment_options": ["online", "cash"]}}, {"X-Agent-Id": "test-seller-e2e"}) + lamps = [r["listing"]] +lamp = lamps[0] s, out = call("POST", "/api/chat", {"event": {"type": "buy_cash", "listing_id": lamp["id"]}, "actor": "guest-e2e"}) cm = [a for a in out["actions"] if a["type"] == "show_cash_meetup"] deal = cm[0]["payload"]["deal"] if cm else {} diff --git a/tests/llm_proxy_check.sh b/tests/llm_proxy_check.sh index c4a6d34..2eda74b 100755 --- a/tests/llm_proxy_check.sh +++ b/tests/llm_proxy_check.sh @@ -8,8 +8,8 @@ python3 tests/mock_llm.py 18798 & M=$! start() { # port models rpm AGENT_MARKET_PORT=$1 AGENT_MARKET_DATA=$D/d$1 AGENT_MARKET_LLM_API_KEY_FILE=$KEYF AGENT_MARKET_CHAT_RPM=$3 AGENT_MARKET_CHAT_BURST=3 \ AGENT_MARKET_LLM_BASE_URL=http://127.0.0.1:18798/v1 AGENT_MARKET_LLM_MODELS=$2 python3 server.py >/dev/null 2>&1 & echo $!; } -S1=$(start 18799 busy-model,mock-model 0); S2=$(start 18800 busy-model,busy-model-2 0); S3=$(start 18801 mock-model 2) -trap 'kill $M $S1 $S2 $S3 2>/dev/null; rm -rf $D' EXIT +S1=$(start 18799 busy-model,mock-model 0); S2=$(start 18800 busy-model,busy-model-2 0); S3=$(start 18801 mock-model 2); S4=$(start 18802 daily-model,mock-model 0) +trap 'kill $M $S1 $S2 $S3 $S4 2>/dev/null; rm -rf $D' EXIT sleep 1.5 chat() { curl -s localhost:$1/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}'; } OUT=$(chat 18799) @@ -34,3 +34,16 @@ OTHER=$(curl -s -o /dev/null -w '%{http_code}' localhost:18801/api/chat -H 'Cont EV=$(curl -s -o /dev/null -w '%{http_code}' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.9' -d '{"event":{"type":"landing"}}') [[ "$EV" == "200" ]] && echo "PASS UI events (no LLM call) not rate-limited" || { echo "FAIL event limited: $EV"; exit 1; } [[ "$CODES" == "200 200 200 429 429 " && "$OTHER" == "200" ]] && echo "PASS per-IP rate limit (burst 3: $CODES; other IP $OTHER)" || { echo "FAIL rate limit: $CODES / $OTHER"; exit 1; } +curl -s localhost:18799/api/chat -H 'Content-Type: application/json' -d '{"message":"make the background a sunset gradient"}' | python3 -c ' +import json,sys; d=json.load(sys.stdin); a=d["actions"][0]; p=a["payload"]["patch"] +assert a["type"]=="set_theme" and p["background"]["colors"]==["#ff7e5f","#feb47b"], a +assert "@import" not in p["customCss"] and "position" not in p["customCss"] and "border-width:3px" in p["customCss"], p +assert "<" not in p["copy"]["headline"] and "lst_" not in d["reply"], (p, d["reply"]) +print("PASS model set_theme is validated (bad colour, @import, position, HTML dropped; ids scrubbed from reply)")' +curl -s localhost:18802/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}' >/dev/null +curl -s localhost:18802/api/chat -H 'Content-Type: application/json' -d '{"message":"switch to light mode with a green accent"}' | python3 -c ' +import json,sys; d=json.load(sys.stdin) +assert d["agent"]=="fallback" and d["actions"][0]["type"]=="set_theme" and d["actions"][0]["payload"]["patch"]["mode"]=="light", d +print("PASS daily quota -> whole chain parked, built-in agent still restyles:", d["reply"][:80])' +curl -s localhost:18802/api/status | python3 -c ' +import json,sys; a=json.load(sys.stdin)["agent"]; assert a["daily_quota_exhausted_until_utc"], a; print("PASS status reports quota reset", a["daily_quota_exhausted_until_utc"])' diff --git a/tests/mock_llm.py b/tests/mock_llm.py index 4e3a48d..5828730 100755 --- a/tests/mock_llm.py +++ b/tests/mock_llm.py @@ -18,7 +18,20 @@ class H(BaseHTTPRequestHandler): return if self.headers.get("Authorization") == "Bearer test-dummy-key": SEEN["auth_ok"] += 1 + if body.get("model") == "daily-model": + b = json.dumps({"error": {"message": "Rate limit exceeded: free-models-per-day.", "code": 429, + "metadata": {"headers": {"X-RateLimit-Limit": "50", "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "4102444800000"}}}}).encode() + self.send_response(429); self.send_header("Content-Type", "application/json"); self.end_headers(); self.wfile.write(b) + return last = body["messages"][-1]["content"] + if "background" in last: + out = {"reply": "Done! Sunset gradient applied (lst_deadbeef).", "actions": [{"type": "set_theme", "payload": { + "background": {"kind": "gradient", "colors": ["#ff7e5f", "#feb47b", "javascript:alert(1)"], "angle": 180}, + "palette": {"accent": "orange"}, "customCss": "@import url(https://evil.example/x.css); .card{border-width:3px;position:fixed}", + "copy": {"headline": "Sunset"}}}]} + b = json.dumps({"choices": [{"message": {"role": "assistant", "content": json.dumps(out)}}]}).encode() + self.send_response(200); self.send_header("Content-Type", "application/json"); self.end_headers(); self.wfile.write(b) + return if last.startswith("TOOL_RESULTS"): ids = re.findall(r'"id": "(lst_[a-f0-9]+)"', last)[:3] out = {"reply": "Here are bikes I found (mock LLM). Payments are TEST MODE ONLY.", diff --git a/tests/theme_check.js b/tests/theme_check.js index 8a35283..eb972fb 100644 --- a/tests/theme_check.js +++ b/tests/theme_check.js @@ -115,7 +115,9 @@ const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); check("reset the look", Object.keys(t5).length === 0 && /just talk to/.test(h2), JSON.stringify(t5)); check("no internal ids in replies", replies.every((x) => !/\b(lst|pi|cash|ord)_[0-9a-f]{6,}/.test(x.reply))); - check("all restyles answered by the real model", replies.every((x) => /AI agent/.test(x.mode)), [...new Set(replies.map((x) => x.mode.split(" · ")[1]))].join(",")); + const who = [...new Set(replies.map((x) => (/AI agent/.test(x.mode) ? x.mode.split(" · ")[1] : "built-in agent")))].join(", "); + if (process.env.REQUIRE_LLM) check("all restyles answered by the real model", replies.every((x) => /AI agent/.test(x.mode)), who); + else console.log("answered by: " + who); check("no page JS errors", errors.length === 0, errors.join("; ")); const ms = replies.map((x) => x.ms).sort((a, b) => a - b); console.log(`\nmodel replies: median ${(ms[ms.length >> 1] / 1000).toFixed(1)}s, max ${(ms[ms.length - 1] / 1000).toFixed(1)}s`);