Local model: prime the prompt cache at start-up and when the catalog changes; accept price_cents in listing-form defaults; 2 Ollama slots

market/agent.py

@@ -599,10 +599,57 @@ def _page_text(look):
                        "chips": [x.get("label") for x in c["chips"]], "bullets": c["bullets"]}, ensure_ascii=False)
 
 
-def _llm_once(store, cfg, model, message, history, look="", timeout=None):
-    sys_prompt = (UI_SCHEMA + "\nCATALOG (all active listings):\n" + json.dumps(_catalog(store))[:9000] +
+def _system_prompt(store, look=""):
+    # stable parts first (schema, catalog) so a local model's prompt cache can reuse them across turns
+    return (UI_SCHEMA + "\nCATALOG (all active listings):\n" + json.dumps(_catalog(store))[:9000] +
             "\nCURRENT_LOOK (this visitor's theme overrides; {} = default dark look):\n" + (look or "{}") +
             "\nPAGE_TEXT (what the editable text slots show right now):\n" + _page_text(look))
+
+
+_PRIMED = {"key": None}
+
+
+def _prime_local_once(store):
+    """Process the current system prompt on the local model in the background (1 output token), so the next visitor
+    turn reuses Ollama's prompt cache instead of paying ~30 s of CPU prompt processing."""
+    local = _local_config()
+    if not local:
+        return False
+    sp = _system_prompt(store, "")
+    key = hash(sp)
+    if key == _PRIMED["key"] or not _local_slots.acquire(blocking=False):
+        return False
+    try:
+        body = {"model": local["model"], "messages": [{"role": "system", "content": sp}, {"role": "user", "content": "hi"}],
+                "max_tokens": 1, "temperature": 0, "keep_alive": -1}
+        req = urllib.request.Request(local["url"], data=json.dumps(body).encode(), headers={"Content-Type": "application/json"})
+        with urllib.request.urlopen(req, timeout=180) as r:
+            r.read()
+        _PRIMED["key"] = key
+        return True
+    except Exception:
+        return False
+    finally:
+        _local_slots.release()
+
+
+def start_local_primer(store, every=30):
+    """Background thread: prime at start-up and whenever the catalog (and so the prompt) changes."""
+    if not _local_config() or os.environ.get("AGENT_MARKET_LOCAL_LLM_PRIME", "1") != "1":
+        return
+
+    def loop():
+        while True:
+            try:
+                _prime_local_once(store)
+            except Exception:
+                pass
+            time.sleep(every)
+    threading.Thread(target=loop, name="local-llm-primer", daemon=True).start()
+
+
+def _llm_once(store, cfg, model, message, history, look="", timeout=None):
+    sys_prompt = _system_prompt(store, look)
     msgs = [{"role": "system", "content": sys_prompt}]
     for h in (history or [])[-6:]:
         if h.get("role") in ("user", "assistant") and h.get("content"):

ops/ollama-watchdog.sh

@@ -6,7 +6,7 @@ set -u
 export OLLAMA_HOST=127.0.0.1:11434          # loopback only, never exposed
 export OLLAMA_MODELS=$HOME/ollama/models
 export OLLAMA_KEEP_ALIVE=-1                  # keep the model in RAM
-export OLLAMA_NUM_PARALLEL=1 OLLAMA_MAX_LOADED_MODELS=1 OLLAMA_MAX_QUEUE=8
+export OLLAMA_NUM_PARALLEL=2 OLLAMA_MAX_LOADED_MODELS=1 OLLAMA_MAX_QUEUE=8
 export OLLAMA_CONTEXT_LENGTH=${OLLAMA_CONTEXT_LENGTH:-8192}
 export OLLAMA_FLASH_ATTENTION=1 OLLAMA_KV_CACHE_TYPE=q8_0
 export OLLAMA_NOPRUNE=1

server.py

@@ -150,7 +150,11 @@ def hydrate(actions, actor):
         elif t == "show_listing_form":
             # small models sometimes send the defaults flat ({"title":..., "price":...}); accept both shapes
             if not isinstance(p.get("defaults"), dict):
-                p = {"defaults": {k: v for k, v in p.items() if k in ("title", "kind", "price", "description", "category", "area", "payment_options")}}
+                p = {"defaults": {k: v for k, v in p.items() if k in ("title", "kind", "price", "price_cents", "description", "category", "area", "payment_options")}}
+            d = p["defaults"]
+            if d.get("price") is None and isinstance(d.get("price_cents"), (int, float)):
+                d["price"] = int(d["price_cents"])
+            d.pop("price_cents", None)
         elif t == "set_theme":
             act = TH.build_theme_action(p)
             if not act:
@@ -388,5 +392,6 @@ class H(BaseHTTPRequestHandler):
 if __name__ == "__main__":
     srv = ThreadingHTTPServer((HOST, PORT), H)
     st = A.agent_status()
+    A.start_local_primer(store)
     print(f"Agent Market [{ENV_LABEL}] on http://{HOST}:{PORT}  agent={st['mode']}  fee={A.FEE_BPS/100:.2f}%  payments=TEST(mock)", flush=True)
     srv.serve_forever()