50 lines
4.5 KiB
Bash
Executable File
50 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# LLM proxy path against a local fake OpenAI-compatible server (no real key needed):
|
|
# tool call -> hydrated cards, key stays server-side, 429 fallback chain, rule-based last resort, per-IP rate limit.
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
D=$(mktemp -d); KEYF=$D/key; printf 'test-dummy-key' > $KEYF; chmod 600 $KEYF
|
|
python3 tests/mock_llm.py 18798 & M=$!
|
|
start() { # port models rpm
|
|
AGENT_MARKET_PORT=$1 AGENT_MARKET_DATA=$D/d$1 AGENT_MARKET_LLM_API_KEY_FILE=$KEYF AGENT_MARKET_CHAT_RPM=$3 AGENT_MARKET_CHAT_BURST=3 \
|
|
AGENT_MARKET_LLM_BASE_URL=http://127.0.0.1:18798/v1 AGENT_MARKET_LLM_MODELS=$2 python3 server.py >/dev/null 2>&1 & echo $!; }
|
|
S1=$(start 18799 busy-model,mock-model 0); S2=$(start 18800 busy-model,busy-model-2 0); S3=$(start 18801 mock-model 2); S4=$(start 18802 daily-model,mock-model 0)
|
|
trap 'kill $M $S1 $S2 $S3 $S4 2>/dev/null; rm -rf $D' EXIT
|
|
sleep 1.5
|
|
chat() { curl -s localhost:$1/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}'; }
|
|
OUT=$(chat 18799)
|
|
echo "$OUT" | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin)
|
|
assert d["agent"]=="llm" and d["model"]=="mock-model", d
|
|
assert d["fallbacks"][0]=={"model":"busy-model","error":"rate_limited","ms":d["fallbacks"][0]["ms"]}, d["fallbacks"]
|
|
types=[a["type"] for a in d["actions"]]
|
|
assert types==["show_cards"], types
|
|
assert all("bike" in l["title"].lower() for l in d["actions"][0]["payload"]["listings"])
|
|
print("PASS 429 on first model -> second model answered; cards hydrated; disallowed component dropped")'
|
|
if echo "$OUT" | grep -q test-dummy-key; then echo "FAIL key leaked to browser"; exit 1; else echo "PASS key never sent to browser"; fi
|
|
test "$(curl -s localhost:18798/ | python3 -c 'import json,sys;print(json.load(sys.stdin)["auth_ok"])')" -ge 2 && echo "PASS key read from file and sent upstream only"
|
|
chat 18800 | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin)
|
|
assert d["agent"]=="fallback" and d["llm_error"]=="all_models_failed" and d["actions"], d
|
|
print("PASS all models rate-limited -> rule-based agent still renders", [a["type"] for a in d["actions"]])'
|
|
curl -s localhost:18800/api/status | python3 -c '
|
|
import json,sys; a=json.load(sys.stdin)["agent"]; assert set(a["cooling_down"])=={"busy-model","busy-model-2"}, a; print("PASS 429ed models put on cooldown")'
|
|
CODES=$(for i in 1 2 3 4 5; do curl -s -o /dev/null -w '%{http_code} ' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.9' -d '{"message":"show me bikes"}'; done)
|
|
OTHER=$(curl -s -o /dev/null -w '%{http_code}' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.10' -d '{"message":"show me bikes"}')
|
|
EV=$(curl -s -o /dev/null -w '%{http_code}' localhost:18801/api/chat -H 'Content-Type: application/json' -H 'CF-Connecting-IP: 203.0.113.9' -d '{"event":{"type":"landing"}}')
|
|
[[ "$EV" == "200" ]] && echo "PASS UI events (no LLM call) not rate-limited" || { echo "FAIL event limited: $EV"; exit 1; }
|
|
[[ "$CODES" == "200 200 200 429 429 " && "$OTHER" == "200" ]] && echo "PASS per-IP rate limit (burst 3: $CODES; other IP $OTHER)" || { echo "FAIL rate limit: $CODES / $OTHER"; exit 1; }
|
|
curl -s localhost:18799/api/chat -H 'Content-Type: application/json' -d '{"message":"make the background a sunset gradient"}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin); a=d["actions"][0]; p=a["payload"]["patch"]
|
|
assert a["type"]=="set_theme" and p["background"]["colors"]==["#ff7e5f","#feb47b"], a
|
|
assert "@import" not in p["customCss"] and "position" not in p["customCss"] and "border-width:3px" in p["customCss"], p
|
|
assert "<" not in p["copy"]["headline"] and "lst_" not in d["reply"], (p, d["reply"])
|
|
print("PASS model set_theme is validated (bad colour, @import, position, HTML dropped; ids scrubbed from reply)")'
|
|
curl -s localhost:18802/api/chat -H 'Content-Type: application/json' -d '{"message":"show me bikes"}' >/dev/null
|
|
curl -s localhost:18802/api/chat -H 'Content-Type: application/json' -d '{"message":"switch to light mode with a green accent"}' | python3 -c '
|
|
import json,sys; d=json.load(sys.stdin)
|
|
assert d["agent"]=="fallback" and d["actions"][0]["type"]=="set_theme" and d["actions"][0]["payload"]["patch"]["mode"]=="light", d
|
|
print("PASS daily quota -> whole chain parked, built-in agent still restyles:", d["reply"][:80])'
|
|
curl -s localhost:18802/api/status | python3 -c '
|
|
import json,sys; a=json.load(sys.stdin)["agent"]; assert a["daily_quota_exhausted_until_utc"], a; print("PASS status reports quota reset", a["daily_quota_exhausted_until_utc"])'
|