92 lines
8.4 KiB
Bash
Executable File
92 lines
8.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# LLM proxy path against a local fake OpenAI-compatible server (no real key needed):
|
|
# tool call -> hydrated cards, key stays server-side, 429 fallback chain, rule-based last resort, per-IP rate limit.
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
D=$(mktemp -d); KEYF=$D/key; printf 'test-dummy-key' > $KEYF; chmod 600 $KEYF
|
|
python3 tests/mock_llm.py 18798 & M=$!
|
|
start() { # port models rpm
|
|
AGENT_MARKET_PORT=$1 AGENT_MARKET_DATA=$D/d$1 AGENT_MARKET_LLM_API_KEY_FILE=$KEYF AGENT_MARKET_CHAT_RPM=$3 AGENT_MARKET_CHAT_BURST=3 \
|
|
AGENT_MARKET_LLM_BASE_URL=http://127.0.0.1:18798/v1 AGENT_MARKET_LLM_MODELS=$2 python3 server.py >/dev/null 2>&1 & echo $!; }
|
|
S1=$(start 18799 busy-model,mock-model 0); S2=$(start 18800 busy-model,busy-model-2 0); S3=$(start 18801 mock-model 2); S4=$(start 18802 daily-model,mock-model 0)
|
|
# local backup model (same fake server standing in for Ollama's OpenAI-compatible endpoint)
|
|
local_start() { # port hosted-models local-model timeout [nokey]
|
|
local K=$KEYF; [ -n "${5:-}" ] && K=$D/no-such-key
|
|
AGENT_MARKET_PORT=$1 AGENT_MARKET_DATA=$D/d$1 AGENT_MARKET_LLM_API_KEY_FILE=$K AGENT_MARKET_CHAT_RPM=0 \
|
|
AGENT_MARKET_LLM_BASE_URL=http://127.0.0.1:18798/v1 AGENT_MARKET_LLM_MODELS=$2 AGENT_MARKET_LOCAL_LLM_URL=http://127.0.0.1:18798/v1 \
|
|
AGENT_MARKET_LOCAL_LLM_MODEL=$3 AGENT_MARKET_LOCAL_LLM_TIMEOUT=$4 AGENT_MARKET_LOCAL_LLM_MIN=$4 python3 server.py >/dev/null 2>&1 & echo $!; }
|
|
S5=$(local_start 18808 daily-model local-model 30); S6=$(local_start 18809 busy-model slow-local 1); S7=$(local_start 18810 x local-model 30 nokey)
|
|
trap 'kill $M $S1 $S2 $S3 $S4 $S5 $S6 $S7 2>/dev/null; rm -rf $D' EXIT
|
|
sleep 1.5
|
|
chat() { curl -s localhost:$1/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}'; }
|
|
OUT=$(chat 18799)
|
|
echo "$OUT" | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin)
|
|
assert d["agent"]=="llm" and d["model"]=="mock-model", d
|
|
assert d["fallbacks"][0]=={"model":"busy-model","error":"rate_limited","ms":d["fallbacks"][0]["ms"]}, d["fallbacks"]
|
|
types=[a["type"] for a in d["actions"]]
|
|
assert types==["show_cards"], types
|
|
assert all("bike" in l["title"].lower() for l in d["actions"][0]["payload"]["listings"])
|
|
print("PASS 429 on first model -> second model answered; cards hydrated; disallowed component dropped")'
|
|
if echo "$OUT" | grep -q test-dummy-key; then echo "FAIL key leaked to browser"; exit 1; else echo "PASS key never sent to browser"; fi
|
|
test "$(curl -s localhost:18798/ | python3 -c 'import json,sys;print(json.load(sys.stdin)["auth_ok"])')" -ge 2 && echo "PASS key read from file and sent upstream only"
|
|
chat 18800 | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin)
|
|
assert d["agent"]=="fallback" and d["llm_error"]=="all_models_failed" and d["actions"], d
|
|
print("PASS all models rate-limited -> rule-based agent still renders", [a["type"] for a in d["actions"]])'
|
|
curl -s localhost:18800/api/status | python3 -c '
|
|
import json,sys; a=json.load(sys.stdin)["agent"]; assert set(a["cooling_down"])=={"busy-model","busy-model-2"}, a; print("PASS 429ed models put on cooldown")'
|
|
CODES=$(for i in 1 2 3 4 5; do curl -s -o /dev/null -w '%{http_code} ' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.9' -d '{"message":"show me bikes"}'; done)
|
|
OTHER=$(curl -s -o /dev/null -w '%{http_code}' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.10' -d '{"message":"show me bikes"}')
|
|
EV=$(curl -s -o /dev/null -w '%{http_code}' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.9' -d '{"event":{"type":"landing"}}')
|
|
[[ "$EV" == "200" ]] && echo "PASS UI events (no LLM call) not rate-limited" || { echo "FAIL event limited: $EV"; exit 1; }
|
|
[[ "$CODES" == "200 200 200 429 429 " && "$OTHER" == "200" ]] && echo "PASS per-IP rate limit (burst 3: $CODES; other IP $OTHER)" || { echo "FAIL rate limit: $CODES / $OTHER"; exit 1; }
|
|
curl -s localhost:18799/api/chat -H 'Content-Type: application/json' -d '{"message":"make the background a sunset gradient"}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); a=d["actions"][0]; p=a["payload"]["patch"]
|
|
assert a["type"]=="set_theme" and p["background"]["colors"]==["#ff7e5f","#feb47b"], a
|
|
assert "@import" not in p["customCss"] and "position" not in p["customCss"] and "border-width:3px" in p["customCss"], p
|
|
assert "<" not in p["copy"]["headline"] and "lst_" not in d["reply"], (p, d["reply"])
|
|
print("PASS model set_theme is validated (bad colour, @import, position, HTML dropped; ids scrubbed from reply)")'
|
|
curl -s localhost:18799/api/chat -H 'Content-Type: application/json' -d '{"message":"replace agent market to new internet on the top left corner","theme":{}}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); a=d["actions"]
|
|
assert d["agent"]=="llm" and a and a[0]["type"]=="set_theme" and a[0]["payload"]["patch"]["copy"]=={"logo":"New Internet"}, d
|
|
print("PASS model path: prompt explains top-left = logo + current page text -> copy.logo set")'
|
|
curl -s localhost:18802/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}' >/dev/null
|
|
curl -s localhost:18802/api/chat -H 'Content-Type: application/json' -d '{"message":"switch to light mode with a green accent"}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin)
|
|
assert d["agent"]=="fallback" and d["actions"][0]["type"]=="set_theme" and d["actions"][0]["payload"]["patch"]["mode"]=="light", d
|
|
print("PASS daily quota -> whole chain parked, built-in agent still restyles:", d["reply"][:80])'
|
|
curl -s localhost:18802/api/status | python3 -c '
|
|
import json,sys; a=json.load(sys.stdin)["agent"]; assert a["daily_quota_exhausted_until_utc"], a; print("PASS status reports quota reset", a["daily_quota_exhausted_until_utc"])'
|
|
calls() { curl -s localhost:18798/ | python3 -c "import json,sys;print(json.load(sys.stdin)['calls'].get('$1',0))"; }
|
|
C0=$(calls daily-model)
|
|
chat 18808 | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin)
|
|
assert d["agent"]=="llm" and d.get("local") and d["provider"]=="local" and d["model"]=="local-model", d
|
|
assert d["fallbacks"][0]["model"]=="daily-model" and d["fallbacks"][0]["error"]=="daily_quota", d["fallbacks"]
|
|
print("PASS hosted daily quota -> local backup model answered (hosted -> local -> rules)")'
|
|
chat 18808 | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); assert d.get("local") and d["fallbacks"]==[{"model":"*","error":"daily_quota","ms":0}], d'
|
|
[ "$(calls daily-model)" = "$((C0+1))" ] && echo "PASS quota parked: next turn goes straight to the local model (no hosted call)" || { echo "FAIL hosted called again: $(calls daily-model)"; exit 1; }
|
|
curl -s localhost:18808/api/status | python3 -c '
|
|
import json,sys; a=json.load(sys.stdin)["agent"]
|
|
assert a["local_model"]=="local-model" and a["local_ok"] is True and a["chain"]==["daily-model","local:local-model"] and a["last_local"] is True and a["daily_quota_exhausted_until_utc"], a
|
|
print("PASS status: chain hosted -> local, local server up, last answer local")'
|
|
kill $S5; sleep 0.5; S5=$(local_start 18808 daily-model local-model 30); sleep 1.2
|
|
curl -s localhost:18808/api/status | python3 -c '
|
|
import json,sys; a=json.load(sys.stdin)["agent"]; assert a["daily_quota_exhausted_until_utc"], a'
|
|
chat 18808 >/dev/null
|
|
[ "$(calls daily-model)" = "$((C0+1))" ] && echo "PASS quota stop survives a restart (no call spent to rediscover it)" || { echo "FAIL hosted called after restart"; exit 1; }
|
|
chat 18809 | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin)
|
|
assert d["agent"]=="fallback" and d["actions"] and any(f["model"]=="slow-local" and f["error"]=="timeout" for f in d["fallbacks"]), d
|
|
print("PASS local model over its own timeout -> keyword rules answer (last resort)")'
|
|
chat 18810 | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); assert d["agent"]=="llm" and d.get("local") and d["fallbacks"]==[], d
|
|
print("PASS local model works with no hosted key configured")'
|
|
curl -s localhost:18810/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}' | grep -q test-dummy-key && { echo "FAIL key"; exit 1; } || true
|
|
curl -s localhost:18810/api/chat -H 'Content-Type: application/json' -d '{"message":"hide the bullet points","theme":{}}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); a=d["actions"]
|
|
assert d["agent"]=="llm" and d.get("local") and d.get("rules_checked") and [x["type"] for x in a]==["set_theme"] and a[0]["payload"]["patch"]["sections"]["hidden"]==["bullets"], d
|
|
print("PASS local model look command checked by the rules (validated patch wins, marked rules_checked)")'
|