news/digest/newsdigest/digest.py

"""Daily digest: 10 current articles on AI agents from reputable outlets, each with a short
original summary (written by the local model from the article's own text) and the source link.
Never stores or shows the article text itself."""
from __future__ import annotations

import datetime as dt
import email.utils
import re
import xml.etree.ElementTree as ET

from . import llm, safefetch, text
from .store import LOCK, Store

TZ = dt.timezone(dt.timedelta(hours=8))
FEEDS = [
    ("TechCrunch", "https://techcrunch.com/category/artificial-intelligence/feed/"),
    ("The Verge", "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml"),
    ("VentureBeat", "https://venturebeat.com/category/ai/feed/"),
    ("Ars Technica", "https://feeds.arstechnica.com/arstechnica/technology-lab"),
    ("MIT Technology Review", "https://www.technologyreview.com/topic/artificial-intelligence/feed"),
    ("The Register", "https://www.theregister.com/software/ai_ml/headlines.atom"),
    ("Wired", "https://www.wired.com/feed/tag/ai/latest/rss"),
    ("Hugging Face", "https://huggingface.co/blog/feed.xml"),
    ("Google", "https://blog.google/technology/ai/rss/"),
    ("OpenAI", "https://openai.com/news/rss.xml"),
    ("Microsoft", "https://blogs.microsoft.com/ai/feed/"),
    ("AWS Machine Learning", "https://aws.amazon.com/blogs/machine-learning/feed/"),
    ("ZDNET", "https://www.zdnet.com/topic/artificial-intelligence/rss.xml"),
    ("SiliconANGLE", "https://siliconangle.com/category/ai/feed/"),
]
STRONG = re.compile(r"\b(ai agents?|agentic|multi-agent|autonomous agents?|agent-to-agent|a2a|model context protocol|mcp|"
                    r"computer[- ]use|coding agents?|agent (?:framework|platform|economy|ecosystem|marketplace|protocol)|agents?)\b", re.I)
AI = re.compile(r"\b(ai|llm|model|openai|anthropic|claude|gemini|gpt|copilot|automation)\b", re.I)
MAX_PER_SOURCE = 3
MAX_AGE_DAYS = 3

SUMMARY_SYSTEM = (
    "You write short, original news summaries for a digest about AI agents. Use only facts in the provided "
    "article text. Write 2 or 3 plain sentences (max 70 words) in your own words. Do not quote sentences "
    "from the article, do not use markdown, do not add opinions or facts that are not in the text. "
    "End with one short sentence starting 'Why it matters:' about the relevance to AI agents.")


def _parse_date(s: str | None):
    if not s:
        return None
    s = s.strip()
    try:
        d = email.utils.parsedate_to_datetime(s)
    except (TypeError, ValueError):
        try:
            d = dt.datetime.fromisoformat(s.replace("Z", "+00:00"))
        except ValueError:
            return None
    if d.tzinfo is None:
        d = d.replace(tzinfo=dt.timezone.utc)
    return d


def parse_feed(body: bytes, source: str) -> list[dict]:
    if b"<!DOCTYPE" in body[:2000] or b"<!ENTITY" in body:
        return []          # no DTDs/entities (billion-laughs / XXE-style payloads)
    try:
        root = ET.fromstring(body)
    except ET.ParseError:
        return []
    out = []
    for el in root.iter():
        tag = el.tag.split("}")[-1]
        if tag not in ("item", "entry"):
            continue
        get = {c.tag.split("}")[-1]: c for c in el}
        title = text.plain(get["title"].text if "title" in get else "", 220)
        link = ""
        if "link" in get:
            link = get["link"].text or get["link"].get("href") or ""
        for c in el:
            if c.tag.split("}")[-1] == "link" and c.get("rel", "alternate") == "alternate" and c.get("href"):
                link = c.get("href")
        link = text.safe_url(link.strip())
        desc = ""
        for k in ("description", "summary", "content", "encoded"):
            if k in get and (get[k].text or "").strip():
                desc = text.plain(get[k].text, 1500)
                break
        date = None
        for k in ("pubDate", "published", "updated", "date"):
            if k in get:
                date = _parse_date(get[k].text)
                if date:
                    break
        if title and link:
            out.append({"source": source, "title": title, "url": link, "teaser": desc, "published": date})
    return out


def relevance(item: dict) -> int:
    t, d = item["title"], item.get("teaser", "")
    score = 3 * len(STRONG.findall(t)) + len(STRONG.findall(d))
    if score and AI.search(t + " " + d):
        score += 1
    return score


def candidates(now: dt.datetime, seen: set[str], fetch=safefetch.fetch, log=print) -> list[dict]:
    items = []
    for source, url in FEEDS:
        try:
            _, _, body = fetch(url)
        except safefetch.FetchError as exc:
            log(f"feed {source}: {exc}")
            continue
        got = parse_feed(body, source)
        log(f"feed {source}: {len(got)} items")
        items += got
    fresh = []
    for it in items:
        if it["url"] in seen or not it["published"]:
            continue
        age = now - it["published"]
        if age.days >= MAX_AGE_DAYS or age.total_seconds() < -3600:
            continue
        it["score"] = relevance(it)
        if it["score"] >= 2:
            fresh.append(it)
    fresh.sort(key=lambda i: (i["score"], i["published"]), reverse=True)
    picked, per, titles = [], {}, set()
    for it in fresh:
        key = re.sub(r"\W+", " ", it["title"].lower())[:60]
        if per.get(it["source"], 0) >= MAX_PER_SOURCE or key in titles:
            continue
        per[it["source"]] = per.get(it["source"], 0) + 1
        titles.add(key)
        picked.append(it)
    return picked


def summarise(it: dict, fetch=safefetch.fetch, chat=llm.chat) -> str | None:
    article = it.get("teaser", "")
    try:
        _, ctype, body = fetch(it["url"])
        info = text.page_info(body, ctype)
        if len(info["text"]) > len(article):
            article = info["text"]
    except safefetch.FetchError:
        pass
    if len(article) < 120:
        return None
    out = chat(SUMMARY_SYSTEM, f"Title: {it['title']}\nSource: {it['source']}\n\nArticle text:\n{article[:3500]}", max_tokens=220)
    summary = text.plain(out.replace("*", ""), 600)
    if not summary or copied(summary, article):
        return None
    return summary


def copied(summary: str, article: str, n: int = 12) -> bool:
    """True if the summary reuses any 12-word run from the article (we never reproduce text)."""
    w = re.findall(r"\w+", article.lower())
    grams = {" ".join(w[i:i + n]) for i in range(max(0, len(w) - n + 1))}
    s = re.findall(r"\w+", summary.lower())
    return any(" ".join(s[i:i + n]) in grams for i in range(max(0, len(s) - n + 1)))


def build(store: Store, *, now: dt.datetime | None = None, count: int = 10, fetch=safefetch.fetch,
          chat=llm.chat, log=print) -> dict:
    now = now or dt.datetime.now(TZ)
    day = now.astimezone(TZ).date().isoformat()
    # never repeat an article from an earlier day (a re-run of today may reuse today's picks)
    seen = {e["url"] for k, d in store.data["digests"].items() if k != day for e in d.get("entries", []) + d.get("community", [])}
    entries = []
    for it in candidates(now, seen, fetch, log):
        if len(entries) >= count:
            break
        try:
            s = summarise(it, fetch, chat)
        except RuntimeError as exc:
            log(f"summary failed: {exc}")
            s = None
        if not s:
            continue
        entries.append({"title": it["title"], "url": it["url"], "source": it["source"], "summary": s,
                        "published": it["published"].isoformat(), "kind": "digest"})
        log(f"picked {len(entries)}: {it['source']} | {it['title']}")
    with store.tx() as data:
        d = data["digests"].setdefault(day, {"date": day, "entries": [], "community": []})
        d["entries"] = entries
        d["generated_at"] = now.isoformat(timespec="seconds")
    return d