#!/usr/bin/env bash
# Local LLM for Agent Market (the last link of the model chain): keeps llama.cpp's llama-server running on loopback only
# and restarts it if it exits or stops answering. Lean on purpose so the box keeps headroom for everything else:
# text-only (no vision projector), weights mmapped and not repacked (no second copy in RAM), 6k context, 1 slot,
# q8 KV cache, 6 threads at nice 10, 1 GiB prompt-cache RAM.
#   start / ensure running (idempotent):  ops/local-llm-watchdog.sh --ensure
#   foreground:                           ops/local-llm-watchdog.sh
#   stop:                                 kill $(cat ~/llm/watchdog.pid)   (the trap stops llama-server too)
# Settings can be overridden in ~/.config/agent-market/local-llm-server.env (outside the repo).
set -u
DIR=${LLM_DIR:-$HOME/llm}
mkdir -p "$DIR"
[ -f "$HOME/.config/agent-market/local-llm-server.env" ] && . "$HOME/.config/agent-market/local-llm-server.env"
LLAMA_SERVER=${LLAMA_SERVER:-$HOME/ollama/lib/ollama/llama-server}   # any llama.cpp llama-server build works
LLAMA_LIBS=${LLAMA_LIBS:-$(dirname "$LLAMA_SERVER")}
MODEL_GGUF=${MODEL_GGUF:-$DIR/gemma3-4b-it-Q4_K_M.gguf}               # gemma3 4B instruct, Q4_K_M, text weights only
MODEL_ALIAS=${MODEL_ALIAS:-gemma3:4b}
PORT=${LLM_PORT:-11434}
CTX=${LLM_CTX:-6144}
THREADS=${LLM_THREADS:-6}
NICE=${LLM_NICE:-10}
CACHE_RAM_MIB=${LLM_CACHE_RAM_MIB:-1024}
LOG=$DIR/llama-server.log

[ -x "$LLAMA_SERVER" ] && [ -f "$MODEL_GGUF" ] || { echo "missing $LLAMA_SERVER or $MODEL_GGUF (see README: Local model)"; exit 1; }

if [ "${1:-}" = "--ensure" ]; then
  if [ -f "$DIR/watchdog.pid" ] && kill -0 "$(cat "$DIR/watchdog.pid")" 2>/dev/null; then echo "local LLM watchdog running (pid $(cat "$DIR/watchdog.pid"))"; exit 0; fi
  # run a private copy: a later deploy rewriting the repo file must not change a script bash is still reading
  install -m 755 "$0" "$DIR/watchdog.sh"
  nohup setsid "$DIR/watchdog.sh" >> "$DIR/watchdog.log" 2>&1 < /dev/null &
  echo "local LLM watchdog started (pid $!)"; exit 0
fi

exec 9> "$DIR/watchdog.lock"
flock -n 9 || { echo "another watchdog holds $DIR/watchdog.lock"; exit 0; }
echo $$ > "$DIR/watchdog.pid"
child=0
trap '[ $child -gt 0 ] && kill $child 2>/dev/null; exit 0' TERM INT
while true; do
  # wait for a previous instance to release the port
  for i in $(seq 1 15); do ss -ltn 2>/dev/null | grep -q "127.0.0.1:$PORT " || break; sleep 1; done
  [ -f "$LOG" ] && [ "$(stat -c %s "$LOG")" -gt 20000000 ] && mv -f "$LOG" "$LOG.1"
  LD_LIBRARY_PATH="$LLAMA_LIBS${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" nice -n "$NICE" "$LLAMA_SERVER" \
    -m "$MODEL_GGUF" --alias "$MODEL_ALIAS" --host 127.0.0.1 --port "$PORT" \
    -c "$CTX" -np 1 -t "$THREADS" -tb "$THREADS" -b 512 -ub 256 \
    -ctk q8_0 -ctv q8_0 -fa on --swa-full --no-mmproj --no-repack --cache-ram "$CACHE_RAM_MIB" \
    --poll 0 --threads-http 2 --no-webui --jinja --metrics >> "$LOG" 2>&1 &
  child=$!; started=$(date +%s)
  echo "$(date '+%F %T') started llama-server pid $child ($MODEL_ALIAS, ctx $CTX, $THREADS threads, nice $NICE)"
  for i in $(seq 1 120); do curl -sf -m 2 "http://127.0.0.1:$PORT/health" >/dev/null && break; kill -0 $child 2>/dev/null || break; sleep 1; done
  fails=0
  while kill -0 $child 2>/dev/null; do
    sleep 15
    # /health is served by its own HTTP threads, so it answers while a reply is being generated
    if curl -sf -m 10 "http://127.0.0.1:$PORT/health" >/dev/null; then fails=0; else fails=$((fails+1)); fi
    if [ $fails -ge 3 ]; then echo "$(date '+%F %T') llama-server unresponsive, restarting"; kill $child; sleep 5; kill -9 $child 2>/dev/null; fi
    [ -f "$LOG" ] && [ "$(stat -c %s "$LOG")" -gt 50000000 ] && : > "$LOG"
  done
  wait $child 2>/dev/null
  st=$?; pause=3; [ $(( $(date +%s) - started )) -lt 60 ] && pause=30   # crashing right away: back off
  echo "$(date '+%F %T') llama-server exited (status $st); restarting in ${pause}s"
  sleep $pause
done