diff --git a/README.md b/README.md index a711da1..7b7ca24 100644 --- a/README.md +++ b/README.md @@ -49,17 +49,39 @@ The server stays small on purpose. It does three things: | anything else | refused (`not_a_test_card`) | ## LLM configuration -The model has to be hosted. The server reads its config from the environment only, and the key never reaches the browser: +The model has to be hosted. The server talks to any OpenAI-compatible `/chat/completions` endpoint, and the +key stays on the server: it never goes to the browser, never into the repo, and never into a URL. | Variable | Meaning | |---|---| -| `AGENT_MARKET_LLM_API_KEY` | key for any OpenAI-compatible `/chat/completions` endpoint | -| `AGENT_MARKET_LLM_BASE_URL` | e.g. `https://api.x.ai/v1` | -| `AGENT_MARKET_LLM_MODEL` | model name | -| `XAI_API_KEY` / `OPENAI_API_KEY` | shortcuts with default base URL and model | +| `AGENT_MARKET_LLM_API_KEY_FILE` | path to a file holding the key (preferred), e.g. `~/.config/agent-market/openrouter.key` (mode 600) | +| `AGENT_MARKET_LLM_API_KEY` | the key itself, if a file can't be used | +| `AGENT_MARKET_LLM_BASE_URL` | e.g. `https://openrouter.ai/api/v1` | +| `AGENT_MARKET_LLM_MODELS` | comma-separated fallback chain, tried in order | +| `AGENT_MARKET_LLM_TIMEOUT` / `_BUDGET` | hard cap per model attempt (12 s) and for the whole chain (28 s) | +| `AGENT_MARKET_LLM_COOLDOWN` | how long a model that returned 429 gets skipped (60 s) | +| `AGENT_MARKET_CHAT_RPM` / `_BURST` | per-IP token bucket on `/api/chat` (12/min, burst 6). `CF-Connecting-IP` is trusted only from loopback, i.e. the tunnel. | -With no key set, the demo runs a **rule-based fallback agent**. The page labels this in the header badge and the -chat header. It emits the same UI actions, so the whole flow still works. +`deploy.sh` loads `~/.config/agent-market/llm.env` and `~/.config/agent-market/.env` if they exist. Both +sit outside the repo. The box runs OpenRouter free models: + +`nvidia/nemotron-3-super-120b-a12b:free` → `apodex/apodex-1.1-mini:free` → `google/gemma-4-31b-it:free` → rule-based agent + +The order came from a benchmark of natural-language prompts, scored on valid UI actions and latency. In that run: +- **nemotron-3-super**: 8/8 valid, about 2–3 s. +- **apodex-1.1-mini**: 6/6 valid, about 2 s. +- **gemma-4-31b and gemma-4-26b**: rate-limited upstream (429). +- **inkling**: 403, because it's limited to agentic harnesses. +- **nemotron-3.5-lightning**: 24–55 s, and returned non-JSON. + +If a model returns 429, it times out, or it returns invalid JSON, the server moves on to the next model in the chain. When every model fails, +the labelled rule-based agent answers, so the page never breaks. The header badge shows the model that answered. + +**Free text goes to the LLM. Structured UI events stay deterministic.** Button clicks, form submits and payment results +each have exactly one outcome, so the server handles them deterministically. That keeps them instant and saves free-tier quota. +The landing view doesn't call the model either. Set `AGENT_MARKET_LLM_EVENTS=1` to send events to the LLM as well. + +With no LLM configured, the demo runs the **rule-based fallback agent**. The page labels this. ## Run ```bash @@ -82,12 +104,14 @@ curl -s $BASE/api/actions -H 'X-Agent-Id: agent:my-bot' -H 'Content-Type: applic ## Tests ```bash python3 tests/e2e_http.py http://127.0.0.1:8795 # HTTP end-to-end (23 checks) -bash tests/llm_proxy_check.sh # LLM proxy path against a fake LLM +bash tests/llm_proxy_check.sh # fake LLM: 429 chain, cooldown, fallback, key handling, rate limit +python3 tests/nl_check.py http://127.0.0.1:8795 # real model: 6 natural-language prompts NODE_PATH=/path/to/node_modules node tests/browser_e2e.js http://127.0.0.1:8795 ./screenshots # headless Chrome ``` ## Known gaps (demo) - Agent identity (`X-Agent-Id`) isn't verified, so anyone can claim any id. - There's no real messaging between buyer and seller for cash meetups. -- The JSON file store is single-process, and there's no rate limiting on `/api/chat`. +- The JSON file store is single-process, and rate limits live in memory, so a restart resets them. +- Free OpenRouter models have small daily request caps and get rate-limited upstream. Once those run out, the rule-based agent answers. - The mock payment provider is the only one. A real provider would need its own test keys and webhook secret. diff --git a/tests/browser_e2e.js b/tests/browser_e2e.js index f281af6..fc5c0a6 100644 --- a/tests/browser_e2e.js +++ b/tests/browser_e2e.js @@ -16,13 +16,19 @@ const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); await page.waitForSelector(".card", { timeout: 15000 }); check("landing renders agent-chosen cards", (await page.$$(".card")).length >= 4); const badge = await page.$eval("#agentBadge", (n) => n.textContent); - check("agent mode badge shown", /fallback|LLM/.test(badge), badge); + check("agent mode badge shown", /fallback|AI:|LLM/.test(badge), badge); + // ask(): type into the chat popup and wait for the agent's reply (LLM replies take a few seconds) + const ask = async (text) => { + const n = (await page.$$(".msg.bot")).length; + await page.type("#chatInput", text); + await page.keyboard.press("Enter"); + await page.waitForFunction((n) => document.querySelectorAll(".msg.bot").length > n, { timeout: 60000 }, n); + await sleep(300); + }; // chat search await page.click("#chatLauncher"); - await page.type("#chatInput", "show me bikes"); - await page.keyboard.press("Enter"); - await page.waitForFunction(() => [...document.querySelectorAll("#stage h2")].some((h) => /bike/i.test(h.textContent)), { timeout: 15000 }); + await ask("show me bikes"); const titles = await page.$$eval("#stage .card h3", (ns) => ns.map((n) => n.textContent)); check("chat search renders bike cards", titles.length > 0 && titles.every((t) => /bike/i.test(t)), titles.join(" | ")); if (SHOTS) { await sleep(600); await page.screenshot({ path: SHOTS + "/1-landing-chat-search.png" }); } @@ -49,9 +55,7 @@ const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); check("test payment confirms, receipt shows fee", rfee === money(feeC), rfee); // cash meetup: first card offering cash - await page.type("#chatInput", "show me everything"); - await page.keyboard.press("Enter"); - await page.waitForFunction(() => /everything/.test((document.querySelector("#stage h2") || {}).textContent || ""), { timeout: 15000 }); + await ask("show me everything that I can pay for in cash"); const cashId = await page.evaluate(() => { const c = [...document.querySelectorAll("#stage .card")].find((c) => c.querySelector("button.cash")); return c && c.dataset.listing; }); await page.click(`.card[data-listing="${cashId}"] button.cash`); await page.waitForSelector("[data-component=cash-meetup]", { timeout: 15000 }); @@ -73,8 +77,7 @@ const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); check("listing form publishes a new card", true); // compare - await page.type("#chatInput", "compare tutoring"); - await page.keyboard.press("Enter"); + await ask("compare the two tutoring options"); await page.waitForSelector("[data-component=compare]", { timeout: 15000 }); check("compare view renders", (await page.$$("[data-component=compare] th")).length >= 3); diff --git a/tests/nl_check.py b/tests/nl_check.py new file mode 100755 index 0000000..5231718 --- /dev/null +++ b/tests/nl_check.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Natural-language checks against the real hosted model. Usage: nl_check.py BASE_URL +Each prompt should be answered by the LLM (not the fallback) with the expected panel.""" +import json, sys, time, urllib.request, urllib.error +BASE = (sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:8795").rstrip("/") +fee_bps = json.load(urllib.request.urlopen(BASE + "/api/status"))["fee_bps"] +CASES = [ + ("I need something to get my kid to school, under $100", "show_cards", lambda p: any("kid" in l["title"].lower() for l in p["listings"]) and all(l["price"] <= 10000 for l in p["listings"])), + ("what can I get for my garden?", "show_cards", lambda p: any("lawn" in l["title"].lower() for l in p["listings"])), + ("list my old guitar for $120, cash only", "show_listing_form", lambda p: p["defaults"].get("price") == 12000 and p["defaults"].get("payment_options") == ["cash"] and "guitar" in p["defaults"].get("title", "").lower()), + ("compare the two tutoring options", "show_compare", lambda p: len(p["listings"]) == 2 and all("tutoring" in l["title"].lower() for l in p["listings"])), + ("I'd like to buy the ceramic lamp and pay online", "show_checkout", lambda p: p["intent"]["fee_cents"] == round(p["intent"]["subtotal_cents"] * fee_bps / 10000) and p["intent"]["fee_bps"] == fee_bps), + ("can I just pay cash for the lawn mowing when they come round?", "show_cash_meetup", lambda p: p["deal"]["fee_cents"] == 0), +] +ok_n, times, history = 0, [], [] +for prompt, want, pred in CASES: + t = time.time() + req = urllib.request.Request(BASE + "/api/chat", data=json.dumps({"message": prompt, "history": [], "actor": "guest-nl-test"}).encode(), + headers={"Content-Type": "application/json"}) + try: + out = json.load(urllib.request.urlopen(req, timeout=90)) + except urllib.error.HTTPError as e: + out = json.loads(e.read() or b"{}") + dt = time.time() - t + acts = [a for a in out.get("actions", []) if a["type"] == want] + good = out.get("agent") == "llm" and bool(acts) and pred(acts[0]["payload"]) + ok_n += good; times.append(dt) + extra = "" + if acts and want == "show_checkout": + it = acts[0]["payload"]["intent"]; extra = f" price ${it['subtotal_cents']/100:.2f} fee ${it['fee_cents']/100:.2f} total ${it['total_cents']/100:.2f}" + if acts and want == "show_cash_meetup": + extra = f" cash fee ${acts[0]['payload']['deal']['fee_cents']/100:.2f}" + if acts and want in ("show_cards", "show_compare"): + extra = " -> " + " | ".join(l["title"] for l in acts[0]["payload"]["listings"]) + if acts and want == "show_listing_form": + extra = " -> " + json.dumps(acts[0]["payload"]["defaults"]) + print(f"{'PASS' if good else 'FAIL'} [{out.get('agent')}/{(out.get('model') or '').split('/')[-1]} {dt:.1f}s" + f"{' fb=' + ','.join(f['error'] for f in out.get('fallbacks', [])) if out.get('fallbacks') else ''}] {prompt!r}\n {[a['type'] for a in out.get('actions', [])]}{extra}\n reply: {out.get('reply', '')[:200]!r}") +times.sort() +print(f"\n{ok_n}/{len(CASES)} passed; median {times[len(times)//2]:.1f}s, max {times[-1]:.1f}s") +sys.exit(0 if ok_n == len(CASES) else 1)