leora
d785091d64
- Remove the keyword-rules agent from typed chat: fallback_agent, parse_look_command (+ tables), the copy-edit parser and _rules_check are gone. If every model fails: 'The assistant is busy, please try again in a moment.' with a Retry button (agent 'unavailable', no actions). Clicks, forms and payments stay deterministic (fallback_event); the landing view is a deterministic featured-listings panel; the Sell button opens the empty form. - Local model via llama.cpp llama-server directly (ops/local-llm-watchdog.sh, loopback only, restart on exit/hang, crash backoff, log rotation; deploy.sh ensures it runs). Lean: no vision projector, no weight repack (mmapped), ctx 6144, 1 slot, q8 KV, 6 threads at nice 10, poll 0, 1 GiB prompt cache. Idle 0% CPU, 2.9 GB RSS (0.6 GB private). - Local requests use a JSON GBNF grammar, worked examples in the cached system prompt, and a labelled visitor message. 10/10 correct on the real prompt, warm 4-13 s. - Primer re-primes after a model-server restart (llama-server /metrics counter) and otherwise stays idle. - Fix: hosted calls kept the Authorization header off when a local model was configured (cfg['local'] is a dict on hosted configs); now only the local server itself skips the key. Test added. - Badge/chat header: 'AI: busy', 'AI: quota used up', 'AI: unavailable'; no 'built-in'/'checked by rules' texts. - Tests: proxy checks expect the busy reply (never rules), clicks still work while busy, no-model server, local grammar/ prompt/no key; browser checks for busy + Retry; e2e_http accepts busy for typed turns unless REQUIRE_LLM=1; test_copyedit.py replaced by test_copy_defaults.py; README updated.
65 lines
3.8 KiB
Bash
Executable File
65 lines
3.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Local LLM for Agent Market (the last link of the model chain): keeps llama.cpp's llama-server running on loopback only
|
|
# and restarts it if it exits or stops answering. Lean on purpose so the box keeps headroom for everything else:
|
|
# text-only (no vision projector), weights mmapped and not repacked (no second copy in RAM), 6k context, 1 slot,
|
|
# q8 KV cache, 6 threads at nice 10, 1 GiB prompt-cache RAM.
|
|
# start / ensure running (idempotent): ops/local-llm-watchdog.sh --ensure
|
|
# foreground: ops/local-llm-watchdog.sh
|
|
# stop: kill $(cat ~/llm/watchdog.pid) (the trap stops llama-server too)
|
|
# Settings can be overridden in ~/.config/agent-market/local-llm-server.env (outside the repo).
|
|
set -u
|
|
DIR=${LLM_DIR:-$HOME/llm}
|
|
mkdir -p "$DIR"
|
|
[ -f "$HOME/.config/agent-market/local-llm-server.env" ] && . "$HOME/.config/agent-market/local-llm-server.env"
|
|
LLAMA_SERVER=${LLAMA_SERVER:-$HOME/ollama/lib/ollama/llama-server} # any llama.cpp llama-server build works
|
|
LLAMA_LIBS=${LLAMA_LIBS:-$(dirname "$LLAMA_SERVER")}
|
|
MODEL_GGUF=${MODEL_GGUF:-$DIR/gemma3-4b-it-Q4_K_M.gguf} # gemma3 4B instruct, Q4_K_M, text weights only
|
|
MODEL_ALIAS=${MODEL_ALIAS:-gemma3:4b}
|
|
PORT=${LLM_PORT:-11434}
|
|
CTX=${LLM_CTX:-6144}
|
|
THREADS=${LLM_THREADS:-6}
|
|
NICE=${LLM_NICE:-10}
|
|
CACHE_RAM_MIB=${LLM_CACHE_RAM_MIB:-1024}
|
|
LOG=$DIR/llama-server.log
|
|
|
|
[ -x "$LLAMA_SERVER" ] && [ -f "$MODEL_GGUF" ] || { echo "missing $LLAMA_SERVER or $MODEL_GGUF (see README: Local model)"; exit 1; }
|
|
|
|
if [ "${1:-}" = "--ensure" ]; then
|
|
if [ -f "$DIR/watchdog.pid" ] && kill -0 "$(cat "$DIR/watchdog.pid")" 2>/dev/null; then echo "local LLM watchdog running (pid $(cat "$DIR/watchdog.pid"))"; exit 0; fi
|
|
# run a private copy: a later deploy rewriting the repo file must not change a script bash is still reading
|
|
install -m 755 "$0" "$DIR/watchdog.sh"
|
|
nohup setsid "$DIR/watchdog.sh" >> "$DIR/watchdog.log" 2>&1 < /dev/null &
|
|
echo "local LLM watchdog started (pid $!)"; exit 0
|
|
fi
|
|
|
|
exec 9> "$DIR/watchdog.lock"
|
|
flock -n 9 || { echo "another watchdog holds $DIR/watchdog.lock"; exit 0; }
|
|
echo $$ > "$DIR/watchdog.pid"
|
|
child=0
|
|
trap '[ $child -gt 0 ] && kill $child 2>/dev/null; exit 0' TERM INT
|
|
while true; do
|
|
# wait for a previous instance to release the port
|
|
for i in $(seq 1 15); do ss -ltn 2>/dev/null | grep -q "127.0.0.1:$PORT " || break; sleep 1; done
|
|
[ -f "$LOG" ] && [ "$(stat -c %s "$LOG")" -gt 20000000 ] && mv -f "$LOG" "$LOG.1"
|
|
LD_LIBRARY_PATH="$LLAMA_LIBS${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" nice -n "$NICE" "$LLAMA_SERVER" \
|
|
-m "$MODEL_GGUF" --alias "$MODEL_ALIAS" --host 127.0.0.1 --port "$PORT" \
|
|
-c "$CTX" -np 1 -t "$THREADS" -tb "$THREADS" -b 512 -ub 256 \
|
|
-ctk q8_0 -ctv q8_0 -fa on --swa-full --no-mmproj --no-repack --cache-ram "$CACHE_RAM_MIB" \
|
|
--poll 0 --threads-http 2 --no-webui --jinja --metrics >> "$LOG" 2>&1 &
|
|
child=$!; started=$(date +%s)
|
|
echo "$(date '+%F %T') started llama-server pid $child ($MODEL_ALIAS, ctx $CTX, $THREADS threads, nice $NICE)"
|
|
for i in $(seq 1 120); do curl -sf -m 2 "http://127.0.0.1:$PORT/health" >/dev/null && break; kill -0 $child 2>/dev/null || break; sleep 1; done
|
|
fails=0
|
|
while kill -0 $child 2>/dev/null; do
|
|
sleep 15
|
|
# /health is served by its own HTTP threads, so it answers while a reply is being generated
|
|
if curl -sf -m 10 "http://127.0.0.1:$PORT/health" >/dev/null; then fails=0; else fails=$((fails+1)); fi
|
|
if [ $fails -ge 3 ]; then echo "$(date '+%F %T') llama-server unresponsive, restarting"; kill $child; sleep 5; kill -9 $child 2>/dev/null; fi
|
|
[ -f "$LOG" ] && [ "$(stat -c %s "$LOG")" -gt 50000000 ] && : > "$LOG"
|
|
done
|
|
wait $child 2>/dev/null
|
|
st=$?; pause=3; [ $(( $(date +%s) - started )) -lt 60 ] && pause=30 # crashing right away: back off
|
|
echo "$(date '+%F %T') llama-server exited (status $st); restarting in ${pause}s"
|
|
sleep $pause
|
|
done
|