diff --git a/market/agent.py b/market/agent.py index 27aeb8b..926bf5f 100644 --- a/market/agent.py +++ b/market/agent.py @@ -599,10 +599,57 @@ def _page_text(look): "chips": [x.get("label") for x in c["chips"]], "bullets": c["bullets"]}, ensure_ascii=False) +def _system_prompt(store, look=""): + # stable parts first (schema, catalog) so a local model's prompt cache can reuse them across turns + return (UI_SCHEMA + "\nCATALOG (all active listings):\n" + json.dumps(_catalog(store))[:9000] + + "\nCURRENT_LOOK (this visitor's theme overrides; {} = default dark look):\n" + (look or "{}") + + "\nPAGE_TEXT (what the editable text slots show right now):\n" + _page_text(look)) + + +_PRIMED = {"key": None} + + +def _prime_local_once(store): + """Process the current system prompt on the local model in the background (1 output token), so the next visitor + turn reuses Ollama's prompt cache instead of paying ~30 s of CPU prompt processing.""" + local = _local_config() + if not local: + return False + sp = _system_prompt(store, "") + key = hash(sp) + if key == _PRIMED["key"] or not _local_slots.acquire(blocking=False): + return False + try: + body = {"model": local["model"], "messages": [{"role": "system", "content": sp}, {"role": "user", "content": "hi"}], + "max_tokens": 1, "temperature": 0, "keep_alive": -1} + req = urllib.request.Request(local["url"], data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=180) as r: + r.read() + _PRIMED["key"] = key + return True + except Exception: + return False + finally: + _local_slots.release() + + +def start_local_primer(store, every=30): + """Background thread: prime at start-up and whenever the catalog (and so the prompt) changes.""" + if not _local_config() or os.environ.get("AGENT_MARKET_LOCAL_LLM_PRIME", "1") != "1": + return + + def loop(): + while True: + try: + _prime_local_once(store) + except Exception: + pass + time.sleep(every) + threading.Thread(target=loop, name="local-llm-primer", daemon=True).start() + + def _llm_once(store, cfg, model, message, history, look="", timeout=None): - sys_prompt = (UI_SCHEMA + "\nCATALOG (all active listings):\n" + json.dumps(_catalog(store))[:9000] + - "\nCURRENT_LOOK (this visitor's theme overrides; {} = default dark look):\n" + (look or "{}") + - "\nPAGE_TEXT (what the editable text slots show right now):\n" + _page_text(look)) + sys_prompt = _system_prompt(store, look) msgs = [{"role": "system", "content": sys_prompt}] for h in (history or [])[-6:]: if h.get("role") in ("user", "assistant") and h.get("content"): diff --git a/ops/ollama-watchdog.sh b/ops/ollama-watchdog.sh index 441a63c..1175f92 100755 --- a/ops/ollama-watchdog.sh +++ b/ops/ollama-watchdog.sh @@ -6,7 +6,7 @@ set -u export OLLAMA_HOST=127.0.0.1:11434 # loopback only, never exposed export OLLAMA_MODELS=$HOME/ollama/models export OLLAMA_KEEP_ALIVE=-1 # keep the model in RAM -export OLLAMA_NUM_PARALLEL=1 OLLAMA_MAX_LOADED_MODELS=1 OLLAMA_MAX_QUEUE=8 +export OLLAMA_NUM_PARALLEL=2 OLLAMA_MAX_LOADED_MODELS=1 OLLAMA_MAX_QUEUE=8 export OLLAMA_CONTEXT_LENGTH=${OLLAMA_CONTEXT_LENGTH:-8192} export OLLAMA_FLASH_ATTENTION=1 OLLAMA_KV_CACHE_TYPE=q8_0 export OLLAMA_NOPRUNE=1 diff --git a/server.py b/server.py index 3b42d25..9585aa8 100755 --- a/server.py +++ b/server.py @@ -150,7 +150,11 @@ def hydrate(actions, actor): elif t == "show_listing_form": # small models sometimes send the defaults flat ({"title":..., "price":...}); accept both shapes if not isinstance(p.get("defaults"), dict): - p = {"defaults": {k: v for k, v in p.items() if k in ("title", "kind", "price", "description", "category", "area", "payment_options")}} + p = {"defaults": {k: v for k, v in p.items() if k in ("title", "kind", "price", "price_cents", "description", "category", "area", "payment_options")}} + d = p["defaults"] + if d.get("price") is None and isinstance(d.get("price_cents"), (int, float)): + d["price"] = int(d["price_cents"]) + d.pop("price_cents", None) elif t == "set_theme": act = TH.build_theme_action(p) if not act: @@ -388,5 +392,6 @@ class H(BaseHTTPRequestHandler): if __name__ == "__main__": srv = ThreadingHTTPServer((HOST, PORT), H) st = A.agent_status() + A.start_local_primer(store) print(f"Agent Market [{ENV_LABEL}] on http://{HOST}:{PORT} agent={st['mode']} fee={A.FEE_BPS/100:.2f}% payments=TEST(mock)", flush=True) srv.serve_forever()