leora
d785091d64
- Remove the keyword-rules agent from typed chat: fallback_agent, parse_look_command (+ tables), the copy-edit parser and _rules_check are gone. If every model fails: 'The assistant is busy, please try again in a moment.' with a Retry button (agent 'unavailable', no actions). Clicks, forms and payments stay deterministic (fallback_event); the landing view is a deterministic featured-listings panel; the Sell button opens the empty form. - Local model via llama.cpp llama-server directly (ops/local-llm-watchdog.sh, loopback only, restart on exit/hang, crash backoff, log rotation; deploy.sh ensures it runs). Lean: no vision projector, no weight repack (mmapped), ctx 6144, 1 slot, q8 KV, 6 threads at nice 10, poll 0, 1 GiB prompt cache. Idle 0% CPU, 2.9 GB RSS (0.6 GB private). - Local requests use a JSON GBNF grammar, worked examples in the cached system prompt, and a labelled visitor message. 10/10 correct on the real prompt, warm 4-13 s. - Primer re-primes after a model-server restart (llama-server /metrics counter) and otherwise stays idle. - Fix: hosted calls kept the Authorization header off when a local model was configured (cfg['local'] is a dict on hosted configs); now only the local server itself skips the key. Test added. - Badge/chat header: 'AI: busy', 'AI: quota used up', 'AI: unavailable'; no 'built-in'/'checked by rules' texts. - Tests: proxy checks expect the busy reply (never rules), clicks still work while busy, no-model server, local grammar/ prompt/no key; browser checks for busy + Retry; e2e_http accepts busy for typed turns unless REQUIRE_LLM=1; test_copyedit.py replaced by test_copy_defaults.py; README updated.
117 lines
11 KiB
Bash
Executable File
117 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# LLM proxy path against a local fake OpenAI-compatible server (no real key needed):
|
|
# tool call -> hydrated cards, key stays server-side, 429 fallback chain, local model last, busy reply (never rules), per-IP rate limit.
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
D=$(mktemp -d); KEYF=$D/key; printf 'test-dummy-key' > $KEYF; chmod 600 $KEYF
|
|
python3 tests/mock_llm.py 18798 & M=$!
|
|
start() { # port models rpm
|
|
AGENT_MARKET_PORT=$1 AGENT_MARKET_DATA=$D/d$1 AGENT_MARKET_LLM_API_KEY_FILE=$KEYF AGENT_MARKET_CHAT_RPM=$3 AGENT_MARKET_CHAT_BURST=3 \
|
|
AGENT_MARKET_LLM_BASE_URL=http://127.0.0.1:18798/v1 AGENT_MARKET_LLM_MODELS=$2 python3 server.py >/dev/null 2>&1 & echo $!; }
|
|
S1=$(start 18799 busy-model,mock-model 0); S2=$(start 18800 busy-model,busy-model-2 0); S3=$(start 18801 mock-model 2); S4=$(start 18802 daily-model,mock-model 0)
|
|
# local model (same fake server standing in for llama-server's OpenAI-compatible endpoint)
|
|
local_start() { # port hosted-models local-model timeout [nokey]
|
|
local K=$KEYF; [ -n "${5:-}" ] && K=$D/no-such-key
|
|
AGENT_MARKET_PORT=$1 AGENT_MARKET_DATA=$D/d$1 AGENT_MARKET_LLM_API_KEY_FILE=$K AGENT_MARKET_CHAT_RPM=0 \
|
|
AGENT_MARKET_LLM_BASE_URL=http://127.0.0.1:18798/v1 AGENT_MARKET_LLM_MODELS=$2 AGENT_MARKET_LOCAL_LLM_URL=http://127.0.0.1:18798/v1 \
|
|
AGENT_MARKET_LOCAL_LLM_MODEL=$3 AGENT_MARKET_LOCAL_LLM_TIMEOUT=$4 AGENT_MARKET_LOCAL_LLM_MIN=$4 python3 server.py >/dev/null 2>&1 & echo $!; }
|
|
S5=$(local_start 18808 daily-model local-model 30); S6=$(local_start 18809 busy-model slow-local 1); S7=$(local_start 18810 x local-model 30 nokey)
|
|
# no model at all (no key, no local model)
|
|
AGENT_MARKET_PORT=18811 AGENT_MARKET_DATA=$D/d18811 AGENT_MARKET_LLM_API_KEY_FILE=$D/no-such-key AGENT_MARKET_CHAT_RPM=0 python3 server.py >/dev/null 2>&1 & S8=$!
|
|
trap 'kill $M $S1 $S2 $S3 $S4 $S5 $S6 $S7 $S8 2>/dev/null; rm -rf $D' EXIT
|
|
sleep 1.5
|
|
chat() { curl -s localhost:$1/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}'; }
|
|
OUT=$(chat 18799)
|
|
echo "$OUT" | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin)
|
|
assert d["agent"]=="llm" and d["model"]=="mock-model", d
|
|
assert d["fallbacks"][0]=={"model":"busy-model","error":"rate_limited","ms":d["fallbacks"][0]["ms"]}, d["fallbacks"]
|
|
types=[a["type"] for a in d["actions"]]
|
|
assert types==["show_cards"], types
|
|
assert all("bike" in l["title"].lower() for l in d["actions"][0]["payload"]["listings"])
|
|
print("PASS 429 on first model -> second model answered; cards hydrated; disallowed component dropped")'
|
|
if echo "$OUT" | grep -q test-dummy-key; then echo "FAIL key leaked to browser"; exit 1; else echo "PASS key never sent to browser"; fi
|
|
test "$(curl -s localhost:18798/ | python3 -c 'import json,sys;print(json.load(sys.stdin)["auth_ok"])')" -ge 2 && echo "PASS key read from file and sent upstream only"
|
|
BUSY='import json,sys; d=json.load(sys.stdin)
|
|
assert d["agent"]=="unavailable" and d["reply"]=="The assistant is busy, please try again in a moment." and d["actions"]==[] and d["retry"] is True, d'
|
|
chat 18800 | python3 -c "$BUSY"'
|
|
assert d["llm_error"]=="all_models_failed" and [f["error"] for f in d["fallbacks"]]==["rate_limited","rate_limited"], d
|
|
print("PASS all models rate-limited -> honest busy reply with retry, no rule-based answer")'
|
|
curl -s localhost:18800/api/chat -H 'Content-Type: application/json' -d '{"message":"replace agent market to new internet on the top left corner","theme":{}}' | python3 -c "$BUSY"'
|
|
print("PASS look/text commands are never answered by rules either (busy reply)")'
|
|
curl -s localhost:18800/api/chat -H 'Content-Type: application/json' -d '{"event":{"type":"sell"}}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); assert d["agent"]=="ui-event" and d["actions"][0]["type"]=="show_listing_form", d
|
|
print("PASS clicks stay deterministic while the models are busy (Sell -> listing form)")'
|
|
curl -s localhost:18800/api/chat -H 'Content-Type: application/json' -d '{"event":{"type":"landing"}}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); a=d["actions"]
|
|
assert d["agent"]=="ui-event" and a[0]["type"]=="show_cards" and len(a[0]["payload"]["listings"])>=3, d
|
|
print("PASS landing view is deterministic (featured listings)")'
|
|
curl -s localhost:18800/api/status | python3 -c '
|
|
import json,sys; a=json.load(sys.stdin)["agent"]; assert set(a["cooling_down"])=={"busy-model","busy-model-2"}, a; print("PASS 429ed models put on cooldown")'
|
|
CODES=$(for i in 1 2 3 4 5; do curl -s -o /dev/null -w '%{http_code} ' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.9' -d '{"message":"show me bikes"}'; done)
|
|
OTHER=$(curl -s -o /dev/null -w '%{http_code}' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.10' -d '{"message":"show me bikes"}')
|
|
EV=$(curl -s -o /dev/null -w '%{http_code}' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.9' -d '{"event":{"type":"landing"}}')
|
|
[[ "$EV" == "200" ]] && echo "PASS UI events (no LLM call) not rate-limited" || { echo "FAIL event limited: $EV"; exit 1; }
|
|
[[ "$CODES" == "200 200 200 429 429 " && "$OTHER" == "200" ]] && echo "PASS per-IP rate limit (burst 3: $CODES; other IP $OTHER)" || { echo "FAIL rate limit: $CODES / $OTHER"; exit 1; }
|
|
curl -s localhost:18799/api/chat -H 'Content-Type: application/json' -d '{"message":"make the background a sunset gradient"}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); a=d["actions"][0]; p=a["payload"]["patch"]
|
|
assert a["type"]=="set_theme" and p["background"]["colors"]==["#ff7e5f","#feb47b"], a
|
|
assert "@import" not in p["customCss"] and "position" not in p["customCss"] and "border-width:3px" in p["customCss"], p
|
|
assert "<" not in p["copy"]["headline"] and "lst_" not in d["reply"], (p, d["reply"])
|
|
print("PASS model set_theme is validated (bad colour, @import, position, HTML dropped; ids scrubbed from reply)")'
|
|
curl -s localhost:18799/api/chat -H 'Content-Type: application/json' -d '{"message":"replace agent market to new internet on the top left corner","theme":{}}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); a=d["actions"]
|
|
assert d["agent"]=="llm" and a and a[0]["type"]=="set_theme" and a[0]["payload"]["patch"]["copy"]=={"logo":"New Internet"}, d
|
|
print("PASS model path: prompt explains top-left = logo + current page text -> copy.logo set")'
|
|
curl -s localhost:18802/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}' >/dev/null
|
|
curl -s localhost:18802/api/chat -H 'Content-Type: application/json' -d '{"message":"switch to light mode with a green accent"}' | python3 -c "$BUSY"'
|
|
assert d["fallbacks"]==[{"model":"*","error":"daily_quota","ms":0}], d["fallbacks"]
|
|
print("PASS daily quota, no local model -> whole chain parked, busy reply (no rules)")'
|
|
curl -s localhost:18802/api/status | python3 -c '
|
|
import json,sys; a=json.load(sys.stdin)["agent"]; assert a["daily_quota_exhausted_until_utc"], a; print("PASS status reports quota reset", a["daily_quota_exhausted_until_utc"])'
|
|
calls() { curl -s localhost:18798/ | python3 -c "import json,sys;print(json.load(sys.stdin)['calls'].get('$1',0))"; }
|
|
auth() { curl -s localhost:18798/ | python3 -c "import json,sys;print(json.load(sys.stdin)['auth'].get('$1',0))"; }
|
|
C0=$(calls daily-model); A0=$(auth daily-model)
|
|
chat 18808 | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin)
|
|
assert d["agent"]=="llm" and d.get("local") and d["provider"]=="local" and d["model"]=="local-model", d
|
|
assert d["fallbacks"][0]["model"]=="daily-model" and d["fallbacks"][0]["error"]=="daily_quota", d["fallbacks"]
|
|
print("PASS hosted daily quota -> local model answered (hosted -> local)")'
|
|
[ "$(auth daily-model)" = "$((A0+1))" ] && echo "PASS hosted calls still carry the key when a local model is configured" || { echo "FAIL hosted call without key"; exit 1; }
|
|
curl -s localhost:18798/ | python3 -c '
|
|
import json,sys; s=json.load(sys.stdin)
|
|
assert s["grammar"].get("local-model",0)>=1 and s["auth"].get("local-model",0)==0 and s["local_prompt"].get("local-model",0)>=1, s
|
|
assert s["grammar"].get("mock-model",0)==0 and s["local_prompt"].get("mock-model",0)==0, s
|
|
print("PASS local requests: JSON grammar, examples + visitor-message prompt, never the hosted key")'
|
|
chat 18808 | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); assert d.get("local") and d["fallbacks"]==[{"model":"*","error":"daily_quota","ms":0}], d'
|
|
[ "$(calls daily-model)" = "$((C0+1))" ] && echo "PASS quota parked: next turn goes straight to the local model (no hosted call)" || { echo "FAIL hosted called again: $(calls daily-model)"; exit 1; }
|
|
curl -s localhost:18808/api/status | python3 -c '
|
|
import json,sys; a=json.load(sys.stdin)["agent"]
|
|
assert a["local_model"]=="local-model" and a["local_ok"] is True and a["chain"]==["daily-model","local:local-model"] and a["last_local"] is True and a["daily_quota_exhausted_until_utc"], a
|
|
print("PASS status: chain hosted -> local, local server up, last answer local")'
|
|
kill $S5; sleep 0.5; S5=$(local_start 18808 daily-model local-model 30); sleep 1.2
|
|
curl -s localhost:18808/api/status | python3 -c '
|
|
import json,sys; a=json.load(sys.stdin)["agent"]; assert a["daily_quota_exhausted_until_utc"], a'
|
|
chat 18808 >/dev/null
|
|
[ "$(calls daily-model)" = "$((C0+1))" ] && echo "PASS quota stop survives a restart (no call spent to rediscover it)" || { echo "FAIL hosted called after restart"; exit 1; }
|
|
chat 18809 | python3 -c "$BUSY"'
|
|
assert [(f["model"], f["error"]) for f in d["fallbacks"]]==[("busy-model","rate_limited"),("slow-local","timeout")], d["fallbacks"]
|
|
print("PASS hosted 429 + local model over its own timeout -> busy reply with retry (no rules)")'
|
|
chat 18810 | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); assert d["agent"]=="llm" and d.get("local") and d["fallbacks"]==[], d
|
|
print("PASS local model works with no hosted key configured")'
|
|
curl -s localhost:18810/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}' | grep -q test-dummy-key && { echo "FAIL key"; exit 1; } || true
|
|
curl -s localhost:18810/api/chat -H 'Content-Type: application/json' -d '{"message":"replace agent market to new internet on the top left corner","theme":{}}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); a=d["actions"]
|
|
assert d["agent"]=="llm" and d.get("local") and "rules_checked" not in d and a[0]["payload"]["patch"]["copy"]=={"logo":"New Internet"}, d
|
|
print("PASS local model edits page text itself (exact logo phrase; no rules involved)")'
|
|
chat 18811 | python3 -c "$BUSY"'
|
|
assert d["llm_error"]=="no_model_configured", d
|
|
print("PASS no model configured -> busy reply for typed messages")'
|
|
curl -s localhost:18811/api/chat -H 'Content-Type: application/json' -d '{"event":{"type":"landing"}}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); assert d["agent"]=="ui-event" and d["actions"][0]["type"]=="show_cards", d'
|
|
curl -s localhost:18811/api/status | python3 -c '
|
|
import json,sys; a=json.load(sys.stdin)["agent"]; assert a["mode"]=="unavailable" and a["chain"]==[], a
|
|
print("PASS no model configured: landing still works, status says unavailable")'
|