31 lines
1.5 KiB
Python
31 lines
1.5 KiB
Python
"""Local Gemma (llama-server, OpenAI-compatible) with optional Groq fallback. Plain text only."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import urllib.request
|
|
|
|
LOCAL = os.environ.get("NEWS_LLM_URL", "http://127.0.0.1:11434/v1/chat/completions")
|
|
MODEL = os.environ.get("NEWS_LLM_MODEL", "gemma3:4b")
|
|
GROQ_KEY_FILE = os.environ.get("NEWS_GROQ_KEY_FILE", "")
|
|
GROQ_MODEL = os.environ.get("NEWS_GROQ_MODEL", "llama-3.1-8b-instant")
|
|
|
|
|
|
def _post(url, payload, headers, timeout):
|
|
req = urllib.request.Request(url, json.dumps(payload).encode(), {"Content-Type": "application/json", **headers})
|
|
with urllib.request.urlopen(req, timeout=timeout) as r: # fixed, configured endpoints only
|
|
return json.loads(r.read())["choices"][0]["message"]["content"]
|
|
|
|
|
|
def chat(system: str, user: str, *, max_tokens: int = 300, timeout: float = 180) -> str:
|
|
msgs = [{"role": "system", "content": system}, {"role": "user", "content": user}]
|
|
try:
|
|
return _post(LOCAL, {"model": MODEL, "messages": msgs, "max_tokens": max_tokens, "temperature": 0.3}, {}, timeout).strip()
|
|
except Exception as exc: # noqa: BLE001
|
|
if GROQ_KEY_FILE and os.path.exists(GROQ_KEY_FILE):
|
|
key = open(GROQ_KEY_FILE).read().strip()
|
|
return _post("https://api.groq.com/openai/v1/chat/completions",
|
|
{"model": GROQ_MODEL, "messages": msgs, "max_tokens": max_tokens, "temperature": 0.3},
|
|
{"Authorization": "Bearer " + key}, 60).strip()
|
|
raise RuntimeError("llm_unavailable") from exc
|