From b1a43c897cfa9b08ae3a4dd8aebf076406adbfbc Mon Sep 17 00:00:00 2001 From: leora Date: Thu, 8 Oct 2026 19:28:14 +0800 Subject: [PATCH] OpenRouter free-model LLM: key from file, model fallback chain with 429 cooldown and hard deadlines, per-IP chat rate limit, model badge, deterministic UI events --- deploy.sh | 7 + market/agent.py | 301 ++++++++++++++++++++++++--------------- server.py | 42 +++++- static/app.js | 21 ++- tests/llm_proxy_check.sh | 36 +++-- tests/mock_llm.py | 4 + 6 files changed, 278 insertions(+), 133 deletions(-) diff --git a/deploy.sh b/deploy.sh index ee65d15..93b5b09 100755 --- a/deploy.sh +++ b/deploy.sh @@ -15,6 +15,13 @@ mkdir -p "$ROOT" if [ ! -d "$ROOT/app/.git" ]; then git clone -q "$REPO" "$ROOT/app"; fi cd "$ROOT/app" && git fetch -q origin && git checkout -q --detach "origin/$REF" 2>/dev/null || git checkout -q --detach "$REF" echo "deploying $ENVN @ $(git rev-parse --short HEAD) on :$PORT" +# Optional config outside the repo (never committed): shared LLM settings, then per-env overrides. +# llm.env holds e.g. AGENT_MARKET_LLM_API_KEY_FILE / _BASE_URL / _MODELS (a path to the key, never the key itself). +CONF=$HOME/.config/agent-market +set -a +[ -f "$CONF/llm.env" ] && . "$CONF/llm.env" +[ -f "$CONF/$ENVN.env" ] && . "$CONF/$ENVN.env" +set +a PID=$(ss -ltnp 2>/dev/null | grep ":$PORT " | grep -o 'pid=[0-9]*' | cut -d= -f2 || true) [ -n "$PID" ] && kill "$PID" && sleep 0.5 AGENT_MARKET_PORT=$PORT AGENT_MARKET_ENV=$ENVN AGENT_MARKET_DATA=$ROOT/data \ diff --git a/market/agent.py b/market/agent.py index db737e4..465da6d 100644 --- a/market/agent.py +++ b/market/agent.py @@ -4,56 +4,40 @@ Returns {reply, actions} where actions is a list of UI action objects from a fix component set the frontend knows how to render. """ from __future__ import annotations -import json, os, re, urllib.request, urllib.error +import concurrent.futures, json, os, re, time, urllib.request, urllib.error from .store import new_id, now FEE_BPS = int(os.environ.get("AGENT_MARKET_FEE_BPS", "500")) # 5.00% FEE_RATE = FEE_BPS / 10000.0 UI_SCHEMA = """ -You are Market Guide, the concierge for Agent Market (a TEST DEMO marketplace). -You control the page by returning JSON UI actions the frontend renders. -Never invent IDs. Only use listing ids from SEARCH_RESULTS or CREATE_LISTING results. -Currency amounts are integer cents. Always mention that payments are TEST MODE ONLY. -Online purchases add a facilitation fee of FEE_PERCENT. Cash-in-person deals have $0 fee. +You are Market Guide, the AI agent that drives the page of Agent Market, a TEST DEMO marketplace +where people and AI agents buy and sell products and services. All listings are fake test data and +all payments are TEST MODE ONLY (no real money). -Reply ONLY with a single JSON object of this shape (no markdown fences): -{ - "reply": "short helpful message to the user", - "actions": [ /* 0 or more of the actions below */ ] -} +You control what the page shows by returning UI actions. Reply with ONE JSON object and nothing else +(no markdown, no prose outside JSON): +{"reply": "", "actions": [ ... ]} -Allowed actions (pick only from this set): -1. {"type":"show_hero","payload":{"title":"...","subtitle":"...","bullets":["..."]}} -2. {"type":"show_cards","payload":{"heading":"...","listing_ids":["lst_..."]}} -3. {"type":"show_listing_form","payload":{"defaults":{"title":"","kind":"product","price":0,"description":"","category":"","area":"","payment_options":["online","cash"]}}} -4. {"type":"show_compare","payload":{"listing_ids":["lst_...","lst_..."],"heading":"..."}} -5. {"type":"show_checkout","payload":{"listing_id":"lst_...","quantity":1}} -6. {"type":"show_cash_meetup","payload":{"listing_id":"lst_...","suggested_place":"...","suggested_time":"..."}} -7. {"type":"show_receipt","payload":{"order_id":"ord_..."}} -9. {"type":"clear_panel","payload":{}} -8. {"type":"toast","payload":{"level":"info|ok|warn","text":"..."}} +Allowed actions (use only these types; listing ids must come from CATALOG below, never invent them): +- {"type":"show_cards","payload":{"heading":"...","listing_ids":["lst_..."]}} list matching items +- {"type":"show_compare","payload":{"heading":"...","listing_ids":["lst_...","lst_..."]}} side-by-side, 2-4 ids +- {"type":"show_checkout","payload":{"listing_id":"lst_..."}} online purchase; the panel shows price + fee +- {"type":"show_cash_meetup","payload":{"listing_id":"lst_...","suggested_place":"...","suggested_time":"..."}} pay cash in person, $0 fee +- {"type":"show_listing_form","payload":{"defaults":{"title":"...","kind":"product|service","price":,"description":"...","category":"...","area":"Demo Town","payment_options":["online","cash"]}}} + use when the user wants to sell/list something; prefill everything you can infer, the user presses Publish + ("cash only" => payment_options ["cash"]; "$120" => price 12000) +- {"type":"show_hero","payload":{"title":"...","subtitle":"...","bullets":["..."]}} welcome / help +- {"type":"toast","payload":{"level":"info|ok|warn","text":"..."}} -You may also ask the server to run tools by including a "tools" array. The server -executes them and you get the results before the final reply. Tools: -- {"name":"search","args":{"q":"...","kind":"product|service|any","limit":6}} -- {"name":"get_listing","args":{"id":"lst_..."}} -- {"name":"create_listing","args":{"title":"...","kind":"product|service","price":cents,"description":"...","category":"...","area":"...","payment_options":["online","cash"],"seller":"demo-seller","seller_type":"human","emoji":"📦"}} -- {"name":"start_checkout","args":{"listing_id":"...","buyer":"demo-buyer","quantity":1}} -- {"name":"start_cash_deal","args":{"listing_id":"...","buyer":"demo-buyer","place":"...","time":"..."}} -- {"name":"complete_cash_deal","args":{"deal_id":"cash_..."}} - -Card details never reach you: the checkout panel posts them straight to the test payment -endpoint and then sends you a UI_EVENT {"type":"payment_result",...}. React with show_receipt. -UI_EVENT messages are clicks/forms from the page (JSON). Handle them: - view_listing -> show_cards with that id; buy_online -> start_checkout then show_checkout; - buy_cash -> start_cash_deal then show_cash_meetup; compare -> show_compare; - sell -> show_listing_form; submit_listing -> create_listing then show_cards with the new id; - payment_result -> show_receipt; cash_completed -> complete_cash_deal then show_receipt. - -Prefer one clear panel at a time. For "show me bikes" run search then show_cards. -For "I want to buy X" show_checkout. For "meet in person" show_cash_meetup. -For "I want to sell" show_listing_form. Keep reply under 80 words. +Rules: +- Interpret needs, not just keywords: e.g. "get my kid to school" => a kids' bike; "garden" => lawn/yard services. +- Respect budgets (prices in CATALOG are integer cents: 5500 = $55.00). +- If nothing fits, say so and show the closest options with show_cards. +- Online payment adds a FEE_PERCENT facilitation fee paid by the buyer; cash in person has no fee. Mention this when relevant. +- Usually return exactly one main panel action. Keep "reply" short and do not repeat the whole catalog in text. +- Optional: if you truly need a server action first, add "tools":[{"name":"search","args":{"q":"..."}}] (also + create_listing, start_checkout, start_cash_deal); you will get TOOL_RESULTS and then answer. Prefer not to. """.replace("FEE_PERCENT", f"{FEE_RATE*100:.1f}%") @@ -407,113 +391,202 @@ def fallback_agent(store, message: str, history, actor: str = "demo-buyer"): # ---------- LLM path ---------- -def _llm_config(): - """Hosted LLM config from env only (OpenAI-compatible chat completions). Never logs the key.""" - key_env = None +LLM_TIMEOUT = float(os.environ.get("AGENT_MARKET_LLM_TIMEOUT", "12")) # hard cap per model attempt +LLM_BUDGET = float(os.environ.get("AGENT_MARKET_LLM_BUDGET", "28")) # hard cap across the whole chain +_pool = concurrent.futures.ThreadPoolExecutor(max_workers=8, thread_name_prefix="llm") +COOLDOWN_429 = float(os.environ.get("AGENT_MARKET_LLM_COOLDOWN", "60")) +_cooldown = {} # model -> unix time until which we skip it after a 429 +LAST = {"model": None, "at": None, "error": None} + + +class LLMError(Exception): + def __init__(self, kind, detail=""): + super().__init__(f"{kind}: {detail}") + self.kind = kind + + +def _read_key(): + """Key from a file path (preferred) or env. Never logged, never returned to clients.""" + path = os.environ.get("AGENT_MARKET_LLM_API_KEY_FILE") + if path: + try: + with open(os.path.expanduser(path)) as f: + k = f.read().strip() + if k: + return k + except OSError: + pass for env in ("AGENT_MARKET_LLM_API_KEY", "XAI_API_KEY", "OPENAI_API_KEY"): if os.environ.get(env): - key_env = env - break - if not key_env: + return os.environ[env] + return None + + +def _llm_config(): + """Hosted LLM config (OpenAI-compatible chat completions) with an ordered model fallback chain.""" + if not _read_key(): return None - defaults = { - "XAI_API_KEY": ("https://api.x.ai/v1", "grok-3-mini"), - "OPENAI_API_KEY": ("https://api.openai.com/v1", "gpt-4o-mini"), - "AGENT_MARKET_LLM_API_KEY": ("", ""), - }[key_env] - base = os.environ.get("AGENT_MARKET_LLM_BASE_URL") or defaults[0] - model = os.environ.get("AGENT_MARKET_LLM_MODEL") or defaults[1] - if not base or not model: + base = os.environ.get("AGENT_MARKET_LLM_BASE_URL") or "" + if not base: + base = "https://api.x.ai/v1" if os.environ.get("XAI_API_KEY") else ("https://api.openai.com/v1" if os.environ.get("OPENAI_API_KEY") else "") + models = [m.strip() for m in (os.environ.get("AGENT_MARKET_LLM_MODELS") or os.environ.get("AGENT_MARKET_LLM_MODEL") or "").split(",") if m.strip()] + if not base or not models: return None return {"provider": base.split("//")[-1].split("/")[0], "url": base.rstrip("/") + "/chat/completions", - "model": model, "env": key_env} + "models": models, "model": models[0]} -def _call_llm(cfg, messages): - key = os.environ[cfg["env"]] - body = json.dumps({ - "model": cfg["model"], - "messages": messages, - "temperature": 0.2, - "response_format": {"type": "json_object"}, - }).encode() - req = urllib.request.Request(cfg["url"], data=body, method="POST", headers={ - "Authorization": f"Bearer {key}", +def _call_llm(cfg, model, messages, json_mode=True): + payload = {"model": model, "messages": messages, "temperature": 0.2, "max_tokens": 900} + if json_mode: + payload["response_format"] = {"type": "json_object"} + req = urllib.request.Request(cfg["url"], data=json.dumps(payload).encode(), method="POST", headers={ + "Authorization": "Bearer " + _read_key(), "Content-Type": "application/json", + "X-Title": "Agent Market demo", }) - with urllib.request.urlopen(req, timeout=45) as r: - return json.loads(r.read().decode()) + try: + with urllib.request.urlopen(req, timeout=LLM_TIMEOUT) as r: + data = json.loads(r.read().decode()) + except urllib.error.HTTPError as e: + body = e.read().decode(errors="replace")[:300] + if e.code == 429: + raise LLMError("rate_limited", body) + if e.code == 400 and json_mode and "response_format" in body: + return _call_llm(cfg, model, messages, json_mode=False) + raise LLMError(f"http_{e.code}", body) + except (urllib.error.URLError, TimeoutError, OSError) as e: + raise LLMError("timeout_or_network", type(e).__name__) + if isinstance(data, dict) and data.get("error"): + err = data["error"] + code = err.get("code") if isinstance(err, dict) else None + raise LLMError("rate_limited" if code == 429 else "provider_error", str(err)[:200]) + try: + return data["choices"][0]["message"].get("content") or "" + except (KeyError, IndexError, TypeError): + raise LLMError("bad_response", str(data)[:200]) def _parse_agent_json(text: str): - text = (text or "").strip() - if text.startswith("```"): - text = re.sub(r"^```(?:json)?\s*", "", text) - text = re.sub(r"\s*```$", "", text) - return json.loads(text) + text = re.sub(r".*?", "", text or "", flags=re.S).strip() + text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text).strip() + try: + obj = json.loads(text) + except ValueError: + m = re.search(r"\{.*\}", text, re.S) + if not m: + raise LLMError("not_json", text[:120]) + try: + obj = json.loads(m.group(0)) + except ValueError: + raise LLMError("not_json", text[:120]) + if not isinstance(obj, dict): + raise LLMError("not_json", "top-level not an object") + return obj -def llm_agent(store, message: str, history, cfg): - # Round 1: may request tools - catalog = [{"id": L["id"], "title": L["title"], "kind": L["kind"], "price": L["price"], - "tags": L.get("tags"), "payment_options": L.get("payment_options")} - for L in store.all("listings") if L.get("status") == "active"] - sys = UI_SCHEMA + "\n\nCURRENT_LISTINGS_SUMMARY:\n" + json.dumps(catalog)[:4000] - msgs = [{"role": "system", "content": sys}] - for h in (history or [])[-8:]: +def _catalog(store): + return [{"id": L["id"], "title": L["title"], "kind": L["kind"], "price_cents": L["price"], + "category": L.get("category"), "about": (L.get("description") or "")[:110], + "seller_type": L.get("seller_type"), "pay": L.get("payment_options")} + for L in store.all("listings") if L.get("status") == "active"][:60] + + +def _llm_once(store, cfg, model, message, history): + sys_prompt = UI_SCHEMA + "\nCATALOG (all active listings):\n" + json.dumps(_catalog(store))[:9000] + msgs = [{"role": "system", "content": sys_prompt}] + for h in (history or [])[-6:]: if h.get("role") in ("user", "assistant") and h.get("content"): - msgs.append({"role": h["role"], "content": str(h["content"])[:1000]}) + msgs.append({"role": h["role"], "content": str(h["content"])[:600]}) msgs.append({"role": "user", "content": message}) - - raw1 = _call_llm(cfg, msgs) - content1 = raw1["choices"][0]["message"]["content"] - plan = _parse_agent_json(content1) - tools = plan.get("tools") or [] - tool_results = run_tools(store, tools) if tools else [] - + content = _call_llm(cfg, model, msgs) + plan = _parse_agent_json(content) + tool_results = [] + tools = [t for t in (plan.get("tools") or []) if isinstance(t, dict)][:3] if tools: - msgs.append({"role": "assistant", "content": content1}) - msgs.append({"role": "user", "content": "TOOL_RESULTS:\n" + json.dumps(tool_results)[:6000] + - "\nNow return the final {reply, actions} JSON. Do not include tools."}) - raw2 = _call_llm(cfg, msgs) - content2 = raw2["choices"][0]["message"]["content"] - plan = _parse_agent_json(content2) + tool_results = run_tools(store, tools) + msgs += [{"role": "assistant", "content": content}, + {"role": "user", "content": "TOOL_RESULTS:\n" + json.dumps(tool_results)[:6000] + + "\nNow return the final {\"reply\",\"actions\"} JSON only."}] + plan = _parse_agent_json(_call_llm(cfg, model, msgs)) + actions = [a for a in (plan.get("actions") or []) if isinstance(a, dict) and a.get("type") in ALLOWED_ACTIONS] + reply = str(plan.get("reply") or "").strip() + if not reply and not actions: + raise LLMError("empty_plan", "") + return {"reply": (reply or "Here you go.")[:600], "actions": actions, "tools": tool_results} - actions = plan.get("actions") or [] - # sanitize action types - allowed = ALLOWED_ACTIONS - actions = [a for a in actions if isinstance(a, dict) and a.get("type") in allowed] - return { - "reply": str(plan.get("reply") or "Done.")[:500], - "actions": actions, - "agent": cfg["provider"], - "model": cfg["model"], - "tools": tool_results, - } + +def llm_agent(store, message, history, cfg): + """Try each model in the chain; skip models cooling down after a 429. Raises if all fail.""" + errors = [] + now_t = time.time() + chain = [m for m in cfg["models"] if _cooldown.get(m, 0) <= now_t] or cfg["models"][-1:] + start = time.time() + for model in chain: + t0 = time.time() + remaining = LLM_BUDGET - (t0 - start) + if remaining < 3: + errors.append({"model": model, "error": "skipped_budget", "ms": 0}) + continue + try: + fut = _pool.submit(_llm_once, store, cfg, model, message, history) + try: + out = fut.result(timeout=min(LLM_TIMEOUT, remaining)) + except concurrent.futures.TimeoutError: + raise LLMError("timeout", f">{min(LLM_TIMEOUT, remaining):.0f}s") + out.update(agent="llm", provider=cfg["provider"], model=model, + latency_ms=int((time.time() - t0) * 1000), fallbacks=errors) + LAST.update(model=model, at=time.time(), error=None) + return out + except LLMError as e: + if e.kind == "rate_limited": + _cooldown[model] = time.time() + COOLDOWN_429 + errors.append({"model": model, "error": e.kind, "ms": int((time.time() - t0) * 1000)}) + except Exception as e: # never let the page break + errors.append({"model": model, "error": type(e).__name__, "ms": int((time.time() - t0) * 1000)}) + LAST.update(error=errors[-1]["error"] if errors else "no_model", at=time.time()) + raise LLMError("all_models_failed", json.dumps(errors)) + + +# Structured UI events (button clicks, form submits, payment results) have one obvious outcome, +# so they are handled deterministically: fast, reliable, and they don't spend free-tier LLM quota. +# Free-text messages go to the LLM. Set AGENT_MARKET_LLM_EVENTS=1 to route events to the LLM too. +LLM_EVENTS = os.environ.get("AGENT_MARKET_LLM_EVENTS") == "1" def handle_chat(store, message: str, history=None, event=None, actor="demo-buyer"): actor = (actor or "demo-buyer")[:60] + t0 = time.time() + if event and (event.get("type") == "landing" or not LLM_EVENTS): + if event.get("type") == "landing": + out = fallback_agent(store, "show me everything", [], actor) + else: + out = fallback_event(store, event, actor) + out.update(agent="ui-event", latency_ms=int((time.time() - t0) * 1000)) + return out cfg = _llm_config() if cfg: - text = message - if event: - text = "UI_EVENT " + json.dumps(event)[:2000] + f"\n(actor/buyer id: {actor})" + text = message if not event else "UI_EVENT " + json.dumps(event)[:2000] + f"\n(actor/buyer id: {actor})" try: return llm_agent(store, text, history or [], cfg) except Exception as e: out = fallback_event(store, event, actor) if event else fallback_agent(store, message, history or [], actor) - out["agent"] = "fallback" - out["llm_error"] = type(e).__name__ + out.update(agent="fallback", llm_error=getattr(e, "kind", type(e).__name__), + fallbacks=json.loads(str(e).split(": ", 1)[1]) if getattr(e, "kind", "") == "all_models_failed" else [], + latency_ms=int((time.time() - t0) * 1000)) + out["reply"] = "(The AI model is busy right now, so the simple built-in agent answered.) " + out["reply"] return out out = fallback_event(store, event, actor) if event else fallback_agent(store, message, history or [], actor) - out["agent"] = "fallback" + out.update(agent="fallback", latency_ms=int((time.time() - t0) * 1000)) return out def agent_status(): cfg = _llm_config() if cfg: - return {"mode": "llm", "provider": cfg["provider"], "model": cfg["model"]} + now_t = time.time() + return {"mode": "llm", "provider": cfg["provider"], "model": cfg["models"][0], "chain": cfg["models"], + "cooling_down": [m for m in cfg["models"] if _cooldown.get(m, 0) > now_t], + "last_model": LAST["model"], "last_error": LAST["error"], "ui_events": "llm" if LLM_EVENTS else "deterministic"} return {"mode": "fallback", "provider": "deterministic", "model": None, - "note": "No hosted LLM key configured (set AGENT_MARKET_LLM_API_KEY + AGENT_MARKET_LLM_BASE_URL + AGENT_MARKET_LLM_MODEL, or XAI_API_KEY / OPENAI_API_KEY). Using the labelled rule-based fallback agent."} + "note": "No hosted LLM configured (set AGENT_MARKET_LLM_API_KEY_FILE or AGENT_MARKET_LLM_API_KEY, plus AGENT_MARKET_LLM_BASE_URL and AGENT_MARKET_LLM_MODELS). Using the labelled rule-based fallback agent."} diff --git a/server.py b/server.py index 10231c5..236af40 100755 --- a/server.py +++ b/server.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Agent Market demo server. Tiny on purpose: static files, an LLM proxy (key stays server-side), test-mode payment endpoints, and a small JSON action API shared by the chat agent and outside agents.""" -import json, os, re, secrets, sys, urllib.parse +import json, os, re, secrets, sys, threading, time, urllib.parse from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -16,6 +16,28 @@ ENV_LABEL = os.environ.get("AGENT_MARKET_ENV", "demo") STATIC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static") WEBHOOK_SECRET = os.environ.get("AGENT_MARKET_WEBHOOK_SECRET") or secrets.token_hex(16) # per-process if unset MAX_BODY = 64 * 1024 +CHAT_RPM = int(os.environ.get("AGENT_MARKET_CHAT_RPM", "12")) # per-IP chat requests per minute +CHAT_BURST = int(os.environ.get("AGENT_MARKET_CHAT_BURST", "6")) +_buckets = {} # ip -> [tokens, last_ts] +_bucket_lock = threading.Lock() + + +def rate_ok(ip): + """Token bucket per client IP. Returns (allowed, retry_after_seconds).""" + if CHAT_RPM <= 0: + return True, 0 + rate = CHAT_RPM / 60.0 + with _bucket_lock: + now_t = time.time() + tokens, last = _buckets.get(ip, (float(CHAT_BURST), now_t)) + tokens = min(float(CHAT_BURST), tokens + (now_t - last) * rate) + if tokens >= 1: + _buckets[ip] = (tokens - 1, now_t) + return True, 0 + _buckets[ip] = (tokens, now_t) + if len(_buckets) > 5000: + _buckets.clear() + return False, int((1 - tokens) / rate) + 1 store = Store(DATA_DIR) @@ -196,13 +218,24 @@ class H(BaseHTTPRequestHandler): a = self.headers.get("X-Agent-Id") or (body or {}).get("actor") or "guest" return re.sub(r"[^A-Za-z0-9:_\-. ()]", "", str(a))[:60] or "guest" - def _json(self, code, obj): + def _client_ip(self): + # Only trust proxy headers when the peer is the local tunnel (cloudflared) on loopback. + peer = self.client_address[0] + if peer in ("127.0.0.1", "::1"): + fwd = self.headers.get("CF-Connecting-IP") or (self.headers.get("X-Forwarded-For") or "").split(",")[0].strip() + if fwd: + return fwd + return peer + + def _json(self, code, obj, extra_headers=None): b = json.dumps(obj).encode() self.send_response(code) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(b))) self.send_header("Cache-Control", "no-store") self.send_header("Access-Control-Allow-Origin", "*") + for k, v in (extra_headers or {}).items(): + self.send_header(k, v) self.end_headers() self.wfile.write(b) @@ -279,6 +312,11 @@ class H(BaseHTTPRequestHandler): return self._json(200, self._webhook(body)) actor = self._actor(body) if path == "/api/chat": + ok, retry = rate_ok(self._client_ip()) + if not ok: + return self._json(429, {"error": "rate_limited", "retry_after": retry, "agent": "rate-limit", + "reply": f"You're sending messages quickly. Please wait about {retry}s and try again.", + "actions": []}, {"Retry-After": str(retry)}) msg = str(body.get("message") or "")[:2000] ev = body.get("event") if isinstance(body.get("event"), dict) else None hist = body.get("history") if isinstance(body.get("history"), list) else [] diff --git a/static/app.js b/static/app.js index f8fa44a..9c28a7b 100644 --- a/static/app.js +++ b/static/app.js @@ -33,6 +33,12 @@ return r.json(); } + function setModel(model, ms) { + const short = (model || "").replace(/:free$/, ""); + $("#agentBadge").textContent = "AI: " + short; + $("#agentBadge").title = (model || "") + " via OpenRouter (free tier)"; + $("#chatMode").textContent = `AI agent · ${short}${ms ? " · " + (ms / 1000).toFixed(1) + "s" : ""}`; + } async function talk({ message, event, echo }) { openChat(); if (echo) say("user", echo); @@ -44,7 +50,9 @@ if (event) history.push({ role: "user", content: "UI_EVENT " + JSON.stringify(event) }); history.push({ role: "assistant", content: out.reply || "" }); say("bot", out.reply || "(no reply)"); - $("#chatMode").textContent = out.agent === "fallback" ? "rule-based fallback agent (no LLM key)" : `LLM agent · ${out.model || out.agent}`; + if (out.agent === "llm") setModel(out.model, out.latency_ms); + else if (out.agent === "fallback") $("#chatMode").textContent = "built-in fallback agent answered (AI model busy)"; + else if (out.agent === "rate-limit") toast(out.reply, "warn"); render(out.actions || []); } catch (e) { thinking.remove(); say("sys", "Agent error: " + e.message); } } @@ -74,6 +82,7 @@ }, show_listing_form(p) { const d = p.defaults || {}; + const payOpts = Array.isArray(d.payment_options) && d.payment_options.length ? d.payment_options : ["online", "cash"]; const f = el("form", { class: "form", "data-component": "listing-form" }, el("label", { class: "full" }, "Title", el("input", { name: "title", required: "", value: d.title || "", placeholder: "e.g. Vintage desk chair" })), el("label", {}, "Type", (() => { const s = el("select", { name: "kind" }, el("option", { value: "product" }, "Product"), el("option", { value: "service" }, "Service")); s.value = d.kind || "product"; return s; })()), @@ -81,8 +90,8 @@ el("label", {}, "Category", el("input", { name: "category", value: d.category || "" })), el("label", {}, "Area (approximate)", el("input", { name: "area", value: d.area || "Demo Town" })), el("label", { class: "full" }, "Description", el("textarea", { name: "description", rows: "3", required: "" }, d.description || "")), - el("label", {}, el("span", {}, el("input", { type: "checkbox", name: "online", checked: "" }), ` Accept online payment (${feeBps / 100}% fee, paid by buyer)`)), - el("label", {}, el("span", {}, el("input", { type: "checkbox", name: "cash", checked: "" }), " Accept cash in person (free)")), + el("label", {}, el("span", {}, el("input", { type: "checkbox", name: "online", checked: payOpts.includes("online") ? "" : null }), ` Accept online payment (${feeBps / 100}% fee, paid by buyer)`)), + el("label", {}, el("span", {}, el("input", { type: "checkbox", name: "cash", checked: payOpts.includes("cash") ? "" : null }), " Accept cash in person (free)")), el("div", { class: "full" }, el("button", { class: "btn", type: "submit" }, "Publish test listing"))); f.addEventListener("submit", (e) => { e.preventDefault(); @@ -201,12 +210,12 @@ api("/api/status").then((s) => { feeBps = s.fee_bps; $("#feePct").textContent = (feeBps / 100).toFixed(feeBps % 100 ? 1 : 0) + "%"; - $("#agentBadge").textContent = s.agent.mode === "llm" ? `LLM: ${s.agent.model}` : "agent: rule-based fallback"; - $("#agentBadge").title = s.agent.note || ""; + if (s.agent.mode === "llm") setModel(s.agent.last_model || s.agent.model); + else { $("#agentBadge").textContent = "agent: rule-based fallback"; $("#agentBadge").title = s.agent.note || ""; } }); // Landing: show featured listings via the agent. (async () => { - const out = await api("/api/chat", { message: "show me everything", actor }); + const out = await api("/api/chat", { event: { type: "landing" }, actor }); render(out.actions || []); say("bot", "Hi! I run this page. Ask me to find, compare, sell, or buy. Payments are test mode only."); })(); diff --git a/tests/llm_proxy_check.sh b/tests/llm_proxy_check.sh index dbcbb66..c7e1ef5 100755 --- a/tests/llm_proxy_check.sh +++ b/tests/llm_proxy_check.sh @@ -1,20 +1,34 @@ #!/usr/bin/env bash -# Exercises the hosted-LLM proxy path with a local fake OpenAI-compatible server (no real key needed). +# LLM proxy path against a local fake OpenAI-compatible server (no real key needed): +# tool call -> hydrated cards, key stays server-side, 429 fallback chain, rule-based last resort, per-IP rate limit. set -euo pipefail cd "$(dirname "$0")/.." -D=$(mktemp -d) +D=$(mktemp -d); KEYF=$D/key; printf 'test-dummy-key' > $KEYF; chmod 600 $KEYF python3 tests/mock_llm.py 18798 & M=$! -AGENT_MARKET_PORT=18799 AGENT_MARKET_DATA=$D AGENT_MARKET_LLM_API_KEY=test-dummy-key \ - AGENT_MARKET_LLM_BASE_URL=http://127.0.0.1:18798/v1 AGENT_MARKET_LLM_MODEL=mock-model python3 server.py >/dev/null 2>&1 & S=$! -trap 'kill $M $S 2>/dev/null; rm -rf $D' EXIT -sleep 1.2 -OUT=$(curl -s localhost:18799/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}') +start() { # port models rpm + AGENT_MARKET_PORT=$1 AGENT_MARKET_DATA=$D/d$1 AGENT_MARKET_LLM_API_KEY_FILE=$KEYF AGENT_MARKET_CHAT_RPM=$3 AGENT_MARKET_CHAT_BURST=3 \ + AGENT_MARKET_LLM_BASE_URL=http://127.0.0.1:18798/v1 AGENT_MARKET_LLM_MODELS=$2 python3 server.py >/dev/null 2>&1 & echo $!; } +S1=$(start 18799 busy-model,mock-model 0); S2=$(start 18800 busy-model,busy-model-2 0); S3=$(start 18801 mock-model 2) +trap 'kill $M $S1 $S2 $S3 2>/dev/null; rm -rf $D' EXIT +sleep 1.5 +chat() { curl -s localhost:$1/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}'; } +OUT=$(chat 18799) echo "$OUT" | python3 -c ' import json,sys; d=json.load(sys.stdin) -assert d["agent"]!="fallback", d +assert d["agent"]=="llm" and d["model"]=="mock-model", d +assert d["fallbacks"][0]=={"model":"busy-model","error":"rate_limited","ms":d["fallbacks"][0]["ms"]}, d["fallbacks"] types=[a["type"] for a in d["actions"]] -assert types==["show_cards"], types # unknown component from the LLM is dropped +assert types==["show_cards"], types assert all("bike" in l["title"].lower() for l in d["actions"][0]["payload"]["listings"]) -print("PASS LLM proxy path: tool call -> hydrated cards; disallowed component dropped")' +print("PASS 429 on first model -> second model answered; cards hydrated; disallowed component dropped")' if echo "$OUT" | grep -q test-dummy-key; then echo "FAIL key leaked to browser"; exit 1; else echo "PASS key never sent to browser"; fi -test "$(curl -s localhost:18798/ | python3 -c 'import json,sys;print(json.load(sys.stdin)["auth_ok"])')" -ge 2 && echo "PASS server sent key to LLM upstream only" +test "$(curl -s localhost:18798/ | python3 -c 'import json,sys;print(json.load(sys.stdin)["auth_ok"])')" -ge 2 && echo "PASS key read from file and sent upstream only" +chat 18800 | python3 -c ' +import json,sys; d=json.load(sys.stdin) +assert d["agent"]=="fallback" and d["llm_error"]=="all_models_failed" and d["actions"], d +print("PASS all models rate-limited -> rule-based agent still renders", [a["type"] for a in d["actions"]])' +curl -s localhost:18800/api/status | python3 -c ' +import json,sys; a=json.load(sys.stdin)["agent"]; assert set(a["cooling_down"])=={"busy-model","busy-model-2"}, a; print("PASS 429ed models put on cooldown")' +CODES=$(for i in 1 2 3 4 5; do curl -s -o /dev/null -w '%{http_code} ' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.9' -d '{"event":{"type":"landing"}}'; done) +OTHER=$(curl -s -o /dev/null -w '%{http_code}' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.10' -d '{"event":{"type":"landing"}}') +[[ "$CODES" == "200 200 200 429 429 " && "$OTHER" == "200" ]] && echo "PASS per-IP rate limit (burst 3: $CODES; other IP $OTHER)" || { echo "FAIL rate limit: $CODES / $OTHER"; exit 1; } diff --git a/tests/mock_llm.py b/tests/mock_llm.py index 0b67d50..4e3a48d 100755 --- a/tests/mock_llm.py +++ b/tests/mock_llm.py @@ -12,6 +12,10 @@ class H(BaseHTTPRequestHandler): b = json.dumps(SEEN).encode(); self.send_response(200); self.end_headers(); self.wfile.write(b) def do_POST(self): body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + if body.get("model") in ("busy-model", "busy-model-2"): + b = b'{"error":{"message":"rate-limited upstream","code":429}}' + self.send_response(429); self.send_header("Content-Type", "application/json"); self.end_headers(); self.wfile.write(b) + return if self.headers.get("Authorization") == "Bearer test-dummy-key": SEEN["auth_ok"] += 1 last = body["messages"][-1]["content"]