#!/usr/bin/env bash
# Local LLM for Agent Market: keeps `ollama serve` running on localhost only and restarts it if it dies or hangs.
# Start: nohup setsid ~/ollama/watchdog.sh >> ~/ollama/watchdog.log 2>&1 &
# Stop:  kill $(cat ~/ollama/watchdog.pid)   (the trap stops ollama too)
set -u
export OLLAMA_HOST=127.0.0.1:11434          # loopback only, never exposed
export OLLAMA_MODELS=$HOME/ollama/models
export OLLAMA_KEEP_ALIVE=-1                  # keep the model in RAM
export OLLAMA_NUM_PARALLEL=1 OLLAMA_MAX_LOADED_MODELS=1 OLLAMA_MAX_QUEUE=8
export OLLAMA_CONTEXT_LENGTH=${OLLAMA_CONTEXT_LENGTH:-8192}
export OLLAMA_FLASH_ATTENTION=1 OLLAMA_KV_CACHE_TYPE=q8_0
export OLLAMA_NOPRUNE=1
# model to pre-load after each (re)start, from the Agent Market config (outside the repo)
WARM=$(sed -n 's/^AGENT_MARKET_LOCAL_LLM_MODEL=//p' "$HOME/.config/agent-market/local-llm.env" 2>/dev/null | tr -d '"' | head -1)
echo $$ > "$HOME/ollama/watchdog.pid"
child=0
trap '[ $child -gt 0 ] && kill $child 2>/dev/null; exit 0' TERM INT
while true; do
  # wait for a previous instance to release the port
  for i in $(seq 1 15); do ss -ltn 2>/dev/null | grep -q '127.0.0.1:11434 ' || break; sleep 1; done
  "$HOME/ollama/bin/ollama" serve >> "$HOME/ollama/ollama.log" 2>&1 &
  child=$!
  echo "$(date '+%F %T') started ollama pid $child"
  for i in $(seq 1 30); do curl -sf -m 2 http://127.0.0.1:11434/api/version >/dev/null && break; sleep 1; done
  [ -n "$WARM" ] && curl -s -m 300 http://127.0.0.1:11434/api/generate -d "{\"model\":\"$WARM\",\"keep_alive\":-1}" >/dev/null &
  fails=0
  while kill -0 $child 2>/dev/null; do
    sleep 20
    if curl -sf -m 10 http://127.0.0.1:11434/api/version >/dev/null; then fails=0; else fails=$((fails+1)); fi
    if [ $fails -ge 3 ]; then echo "$(date '+%F %T') ollama unresponsive, restarting"; kill $child; sleep 3; kill -9 $child 2>/dev/null; fi
  done
  echo "$(date '+%F %T') ollama exited; restarting in 3s"
  sleep 3
done