quill/quill_core.py

564 lines
22 KiB
Python

"""Quill: a read-only wiki whose spaces are Git repositories.
Each space is a Git repository on a Gitea server. Quill only reads and renders;
people and agents change a space by pushing to its repo (branch -> PR -> merge).
Adding a space = one entry in "spaces" in the config file (see configure()). Posts are:
* Markdown files (*.md, *.markdown) anywhere in the repo except README/LICENSE-style files.
Optional front matter (--- title/date/author ---); otherwise the first "# " heading is the title
and a YYYY-MM-DD filename prefix is the date.
* <article> blocks inside index.html (a static site published from the repo).
"""
from __future__ import annotations
import html
import json
import re
import threading
import time
import urllib.error
import urllib.parse
from html.parser import HTMLParser
WIKI_NAME = "Quill"
WIKI_PATHS = ("/quill", "/wiki") # exact paths and anything below them serve the Quill page
# Space registry, set by configure() from the config file. key: URL slug (/quill/<key>).
# repo: owner/name on the Gitea. branch: None = the repo's default branch.
# public_url: optional published site for the space.
SPACES: list[dict] = []
# Where the Git web UI for the spaces lives (links to sources and history). "/gitea" when Quill is
# served next to a Gitea proxy; a full URL such as "https://gitea.example.com" otherwise.
GIT_WEB_BASE = "/gitea"
KEY_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,40}$")
SLUG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,160}$")
SKIP_MD = {"readme", "license", "licence", "contributing", "changelog", "code_of_conduct", "security"}
MAX_FILES = 300
CACHE_SECONDS = 15
_CACHE: dict[str, tuple[float, str, dict]] = {}
_CACHE_LOCK = threading.Lock()
def configure(spaces: list[dict], git_web_base: str = "/gitea") -> None:
"""Replace the space registry (validated) and the Git web base."""
global GIT_WEB_BASE
clean = []
for item in spaces:
if not isinstance(item, dict):
raise ValueError("space must be an object")
key, repo = str(item.get("key") or ""), str(item.get("repo") or "")
if not KEY_RE.fullmatch(key) or not re.fullmatch(r"[A-Za-z0-9._-]+/[A-Za-z0-9._-]+", repo):
raise ValueError(f"bad space key/repo: {key!r} {repo!r}")
clean.append({
"key": key, "name": str(item.get("name") or key), "repo": repo,
"branch": item.get("branch") or None,
"description": str(item.get("description") or ""),
"public_url": str(item.get("public_url") or ""),
})
SPACES[:] = clean
GIT_WEB_BASE = (git_web_base or "/gitea").rstrip("/")
with _CACHE_LOCK:
_CACHE.clear()
def space_by_key(key: str) -> dict | None:
for space in SPACES:
if space["key"] == key:
return space
return None
def public_space(space: dict) -> dict:
return {
"key": space["key"],
"name": space["name"],
"repo": space["repo"],
"description": space.get("description") or "",
"public_url": space.get("public_url") or "",
"git_url": GIT_WEB_BASE + "/" + space["repo"],
}
def is_wiki_path(path: str) -> bool:
return any(path == p or path.startswith(p + "/") for p in WIKI_PATHS)
# ---------------------------------------------------------------- helpers
def _slugify(text: str, limit: int = 80) -> str:
s = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
return s[:limit].strip("-") or "post"
def _safe_href(href: str | None, space: dict, branch: str, base_dir: str = "") -> str | None:
if not href:
return None
href = href.strip()
if re.match(r"^(https?://|mailto:)", href, re.I):
return href if not re.search(r"[\s<>\"']", href) else None
if href.startswith("#"):
return href if re.fullmatch(r"#[A-Za-z0-9_-]{1,80}", href) else None
if re.match(r"^[a-z][a-z0-9+.-]*:", href, re.I) or href.startswith("//"):
return None # javascript:, data:, protocol-relative
# repo-relative link -> the file in Git (through the signed-in Gitea proxy)
target = urllib.parse.urljoin("/" + base_dir.strip("/") + "/" if base_dir else "/", href)
target = target.lstrip("/")
if ".." in target:
return None
return GIT_WEB_BASE + "/" + space["repo"] + "/src/branch/" + urllib.parse.quote(branch) + "/" + urllib.parse.quote(target, safe="/#")
# ---------------------------------------------------------------- markdown (small, escape-first)
_INLINE_CODE = re.compile(r"`([^`]+)`")
_LINK = re.compile(r"\[([^\]]+)\]\(([^)\s]+)(?:\s+&quot;[^)]*&quot;)?\)")
_BOLD = re.compile(r"(\*\*|__)(?=\S)(.+?)(?<=\S)\1")
_ITAL = re.compile(r"(?<![\w*])([*_])(?=\S)(.+?)(?<=\S)\1(?![\w*])")
_AUTOLINK = re.compile(r"(?<![\"'=>])(https?://[^\s<]+[^\s<.,;:!?)\]])")
def _inline(text: str, space: dict, branch: str, base_dir: str) -> str:
text = html.escape(text, quote=True)
codes: list[str] = []
def keep_code(m):
codes.append("<code>" + m.group(1) + "</code>")
return f"\x00{len(codes) - 1}\x00"
text = _INLINE_CODE.sub(keep_code, text)
links: list[str] = []
def keep_link(m):
label, raw = m.group(1), html.unescape(m.group(2))
href = _safe_href(raw, space, branch, base_dir)
if not href:
return label
ext = ' rel="noopener" target="_blank"' if href.startswith("http") else ""
links.append(f'<a href="{html.escape(href, quote=True)}"{ext}>{label}</a>')
return f"\x01{len(links) - 1}\x01"
text = text.replace("!" + "[", "[") # images render as links
text = _LINK.sub(keep_link, text)
def auto(m):
url = html.unescape(m.group(1))
links.append(f'<a href="{html.escape(url, quote=True)}" rel="noopener" target="_blank">{m.group(1)}</a>')
return f"\x01{len(links) - 1}\x01"
text = _AUTOLINK.sub(auto, text)
text = _BOLD.sub(r"<strong>\2</strong>", text)
text = _ITAL.sub(r"<em>\2</em>", text)
text = re.sub(r"\x01(\d+)\x01", lambda m: links[int(m.group(1))], text)
text = re.sub(r"\x00(\d+)\x00", lambda m: codes[int(m.group(1))], text)
return text
def render_markdown(src: str, space: dict, branch: str, base_dir: str = "") -> str:
lines = src.replace("\r\n", "\n").replace("\r", "\n").split("\n")
out: list[str] = []
para: list[str] = []
i = 0
def flush():
if para:
out.append("<p>" + _inline(" ".join(s.strip() for s in para), space, branch, base_dir) + "</p>")
para.clear()
while i < len(lines):
line = lines[i]
stripped = line.strip()
fence = re.match(r"^(```|~~~)", stripped)
if fence:
flush()
mark = fence.group(1)
i += 1
code = []
while i < len(lines) and not lines[i].strip().startswith(mark):
code.append(lines[i])
i += 1
out.append("<pre><code>" + html.escape("\n".join(code)) + "</code></pre>")
i += 1
continue
if not stripped:
flush()
i += 1
continue
h = re.match(r"^(#{1,6})\s+(.*?)\s*#*\s*$", stripped)
if h:
flush()
level = min(len(h.group(1)) + 1, 6) # the post title is the page h1
out.append(f"<h{level}>" + _inline(h.group(2), space, branch, base_dir) + f"</h{level}>")
i += 1
continue
if re.fullmatch(r"(\*\s*){3,}|(-\s*){3,}|(_\s*){3,}", stripped):
flush()
out.append("<hr>")
i += 1
continue
if stripped.startswith(">"):
flush()
quote = []
while i < len(lines) and lines[i].strip().startswith(">"):
quote.append(re.sub(r"^\s*>\s?", "", lines[i]))
i += 1
out.append("<blockquote>" + render_markdown("\n".join(quote), space, branch, base_dir) + "</blockquote>")
continue
lm = re.match(r"^\s*([-*+]|\d+[.)])\s+", line)
if lm:
flush()
ordered = lm.group(1)[0].isdigit()
tag = "ol" if ordered else "ul"
items: list[str] = []
while i < len(lines):
m = re.match(r"^\s*([-*+]|\d+[.)])\s+(.*)$", lines[i])
if m and (m.group(1)[0].isdigit()) == ordered:
items.append(m.group(2))
elif items and lines[i].strip() and lines[i].startswith((" ", "\t")):
items[-1] += " " + lines[i].strip()
else:
break
i += 1
out.append(f"<{tag}>" + "".join("<li>" + _inline(it, space, branch, base_dir) + "</li>" for it in items) + f"</{tag}>")
continue
para.append(line)
i += 1
flush()
return "\n".join(out)
def _front_matter(src: str) -> tuple[dict, str]:
meta: dict = {}
text = src.lstrip("\ufeff")
if text.startswith("---\n") or text.startswith("---\r\n"):
end = re.search(r"\n---\s*(\n|$)", text[3:])
if end:
block = text[4:3 + end.start()]
for row in block.splitlines():
if ":" in row:
k, v = row.split(":", 1)
v = v.strip()
if len(v) >= 2 and v[0] == v[-1] and v[0] in "\"'":
v = v[1:-1]
meta[k.strip().lower()] = v
text = text[3 + end.end():]
return meta, text
def _strip_md(text: str) -> str:
text = re.sub(r"`([^`]*)`", r"\1", text)
text = re.sub(r"!?\[([^\]]*)\]\([^)]*\)", r"\1", text)
text = re.sub(r"[*_#>]+", "", text)
return re.sub(r"\s+", " ", text).strip()
def _summary(text: str, limit: int = 220) -> str:
text = re.sub(r"\s+", " ", text).strip()
return text if len(text) <= limit else text[: limit - 1].rsplit(" ", 1)[0] + "\u2026"
def parse_markdown_post(path: str, src: str, space: dict, branch: str) -> dict:
meta, body = _front_matter(src)
name = path.rsplit("/", 1)[-1]
stem = re.sub(r"\.(md|markdown)$", "", name, flags=re.I)
title = meta.get("title")
m = re.search(r"^\s*#\s+(.+?)\s*#*\s*$", body, re.M)
if not title and m:
title = _strip_md(m.group(1))
if m and (meta.get("title") is None or _strip_md(m.group(1)) == title):
body = body[: m.start()] + body[m.end():] # do not repeat the title in the body
date = meta.get("date") or ""
dm = re.match(r"^(\d{4}-\d{2}-\d{2})[-_]?(.*)$", stem)
if not date and dm:
date = dm.group(1)
if not title:
title = (dm.group(2) if dm and dm.group(2) else stem).replace("-", " ").replace("_", " ").strip().capitalize() or stem
base_dir = path.rsplit("/", 1)[0] if "/" in path else ""
first_para = next((p for p in re.split(r"\n\s*\n", body) if p.strip() and not p.strip().startswith(("#", "```", "---", ">"))), "")
slug = re.sub(r"[^A-Za-z0-9._-]+", "-", re.sub(r"\.(md|markdown)$", "", path, flags=re.I).replace("/", "--")).strip("-")
return {
"slug": slug or _slugify(title),
"title": title[:200],
"date": date[:40],
"author": (meta.get("author") or "")[:80],
"summary": _summary(_strip_md(meta.get("summary") or first_para)),
"source": path,
"kind": "markdown",
"html": render_markdown(body, space, branch, base_dir),
}
# ---------------------------------------------------------------- html articles (whitelist sanitizer)
_ALLOWED = {"p", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "ol", "li", "a", "em", "strong", "b", "i",
"code", "pre", "blockquote", "br", "hr", "figure", "figcaption", "time", "small"}
_VOID = {"br", "hr"}
_DROP = {"script", "style", "template", "iframe", "object", "embed", "noscript", "svg", "math", "head", "title"}
_DATE_RE = re.compile(r"\b(\d{1,2}\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{4}|\d{4}-\d{2}-\d{2})\b")
class _ArticleParser(HTMLParser):
"""Collects <article> blocks as sanitized HTML, plus intro paragraphs outside articles."""
def __init__(self, space: dict, branch: str):
super().__init__(convert_charrefs=True)
self.space, self.branch = space, branch
self.articles: list[list[str]] = []
self.intro: list[str] = []
self.site_title = ""
self._in_article = 0
self._drop = 0
self._buf: list[str] | None = None
self._intro_p: list[str] | None = None
self._h1: list[str] | None = None
self._ended = False
def handle_starttag(self, tag, attrs):
if tag in _DROP:
self._drop += 1
return
if self._drop:
return
if tag == "article" and not self._ended:
self._in_article += 1
if self._in_article == 1:
self._buf = []
self.articles.append(self._buf)
return
if self._in_article and self._buf is not None and tag in _ALLOWED:
if tag == "a":
href = _safe_href(dict(attrs).get("href"), self.space, self.branch)
if href:
ext = ' rel="noopener" target="_blank"' if href.startswith("http") else ""
self._buf.append(f'<a href="{html.escape(href, quote=True)}"{ext}>')
else:
self._buf.append("<a>")
elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
self._buf.append(f"<{tag}>")
else:
self._buf.append(f"<{tag}>")
return
if not self._in_article and not self._ended:
if tag == "p":
self._intro_p = []
elif tag == "h1" and not self.site_title:
self._h1 = []
def handle_endtag(self, tag):
if tag in _DROP:
self._drop = max(0, self._drop - 1)
return
if self._drop:
return
if tag == "html":
self._ended = True # stray text after </html> is not part of the page
if tag == "article" and self._in_article:
self._in_article -= 1
if not self._in_article:
self._buf = None
return
if self._in_article and self._buf is not None and tag in _ALLOWED and tag not in _VOID:
self._buf.append(f"</{tag}>")
return
if tag == "p" and self._intro_p is not None:
text = re.sub(r"\s+", " ", "".join(self._intro_p)).strip()
if text:
self.intro.append(text)
self._intro_p = None
if tag == "h1" and self._h1 is not None:
self.site_title = re.sub(r"\s+", " ", "".join(self._h1)).strip()
self._h1 = None
def handle_data(self, data):
if self._drop or self._ended:
return
if self._in_article and self._buf is not None:
self._buf.append(html.escape(data, quote=False))
elif self._intro_p is not None:
self._intro_p.append(data)
elif self._h1 is not None:
self._h1.append(data)
def _text_of(fragment: str) -> str:
return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", " ", fragment))).strip()
def parse_html_articles(path: str, src: str, space: dict, branch: str) -> tuple[list[dict], list[str]]:
parser = _ArticleParser(space, branch)
try:
parser.feed(src)
parser.close()
except Exception:
return [], []
posts = []
seen: set[str] = set()
for idx, parts in enumerate(parser.articles):
body = "".join(parts).strip()
hm = re.search(r"<(h[1-6])>(.*?)</\1>", body, re.S)
title = _text_of(hm.group(2)) if hm else ""
if hm:
body = body[: hm.start()] + body[hm.end():]
date, author = "", ""
pm = re.search(r"<p>(.*?)</p>", body, re.S)
if pm:
first = _text_of(pm.group(1))
dm = _DATE_RE.search(first)
if dm and len(first) < 160:
date = dm.group(1)
am = re.search(r"(?:written|posted)\s+by\s+([^.]+)", first, re.I)
author = am.group(1).strip() if am else ""
body = body[: pm.start()] + body[pm.end():]
title = title or f"Untitled {idx + 1}"
slug = _slugify(title)
while slug in seen:
slug += "-" + str(idx + 1)
seen.add(slug)
paras = re.findall(r"<p>(.*?)</p>", body, re.S)
posts.append({
"slug": slug,
"title": title[:200],
"date": date,
"author": author[:80],
"summary": _summary(_text_of(paras[0]) if paras else _text_of(body)),
"source": path,
"kind": "html",
"html": body.strip(),
})
return posts, parser.intro
# ---------------------------------------------------------------- loading a space from Gitea
def _date_key(post: dict) -> str:
d = post.get("date") or ""
m = re.match(r"(\d{4})-(\d{2})-(\d{2})", d)
if m:
return d[:10]
m = _DATE_RE.search(d)
if m and not m.group(1)[0:4].isdigit():
try:
return time.strftime("%Y-%m-%d", time.strptime(m.group(1), "%d %B %Y"))
except ValueError:
return ""
return ""
def load_space(space: dict, gitea_get) -> dict:
"""gitea_get(path, raw=False) -> parsed JSON or text from the Gitea API (read only)."""
repo = space["repo"]
meta = gitea_get(f"/repos/{repo}")
branch = space.get("branch") or meta.get("default_branch") or "main"
qb = urllib.parse.quote(branch, safe="")
head = gitea_get(f"/repos/{repo}/branches/{qb}")
commit = head.get("commit") or {}
sha = commit.get("id") or ""
with _CACHE_LOCK:
hit = _CACHE.get(space["key"])
if hit and hit[1] == sha and time.time() - hit[0] < 3600:
return hit[2]
tree = gitea_get(f"/repos/{repo}/git/trees/{urllib.parse.quote(sha or branch, safe='')}?recursive=true&per_page=1000")
files = [t.get("path") or "" for t in tree.get("tree") or [] if t.get("type") == "blob"]
posts: list[dict] = []
intro: list[str] = []
readme = ""
for path in sorted(files)[:MAX_FILES]:
low = path.lower()
name = low.rsplit("/", 1)[-1]
stem = name.rsplit(".", 1)[0]
ref = urllib.parse.quote(sha or branch, safe="")
raw_path = urllib.parse.quote(path)
if low.endswith((".md", ".markdown")):
try:
src = gitea_get(f"/repos/{repo}/raw/{raw_path}?ref={ref}", raw=True)
except Exception:
continue
if stem in SKIP_MD:
if path.count("/") == 0 and stem == "readme":
readme = src
continue
posts.append(parse_markdown_post(path, src, space, branch))
elif name == "index.html" and path.count("/") <= 1:
try:
src = gitea_get(f"/repos/{repo}/raw/{raw_path}?ref={ref}", raw=True)
except Exception:
continue
found, paras = parse_html_articles(path, src, space, branch)
posts.extend(found)
if path == "index.html":
intro = paras
# unique slugs across sources
seen: set[str] = set()
for p in posts:
base, n = p["slug"], 2
while p["slug"] in seen:
p["slug"] = f"{base}-{n}"
n += 1
seen.add(p["slug"])
# newest first when dated; undated keep repo order after dated ones
dated = [p for p in posts if _date_key(p)]
undated = [p for p in posts if not _date_key(p)]
dated.sort(key=_date_key, reverse=True)
posts = dated + undated
about_html = ""
if intro:
about_html = "".join("<p>" + html.escape(t) + "</p>" for t in intro[:6])
elif readme:
_, body = _front_matter(readme)
body = re.sub(r"^\s*#\s+.*$", "", body, count=1, flags=re.M)
about_html = render_markdown(body, space, branch)
cauthor = (commit.get("author") or {}) if isinstance(commit.get("author"), dict) else {}
data = {
"space": public_space(space),
"branch": branch,
"about_html": about_html,
"last_change": {
"sha": sha[:10],
"message": (commit.get("message") or "").strip().split("\n")[0][:200],
"author": cauthor.get("name") or cauthor.get("username") or "",
"at": commit.get("timestamp") or "",
},
"posts": posts,
}
with _CACHE_LOCK:
_CACHE[space["key"]] = (time.time(), sha, data)
return data
def space_listing(data: dict) -> dict:
out = dict(data)
out["posts"] = [{k: v for k, v in p.items() if k != "html"} for p in data["posts"]]
return out
def api(path: str, gitea_get) -> tuple[int, dict]:
"""GET /v1/quill/... (the caller has already checked read access)."""
rest = path[len("/v1/quill"):].strip("/")
parts = [urllib.parse.unquote(p) for p in rest.split("/")] if rest else []
if not parts or parts == ["spaces"]:
return 200, {"name": WIKI_NAME, "spaces": [public_space(s) for s in SPACES]}
if parts[0] != "spaces" or len(parts) not in (2, 4) or not KEY_RE.fullmatch(parts[1]):
return 404, {"error": "not_found"}
space = space_by_key(parts[1])
if space is None:
return 404, {"error": "space_not_found"}
try:
data = load_space(space, gitea_get)
except urllib.error.HTTPError:
return 502, {"error": "space_repo_unavailable"}
except Exception:
return 502, {"error": "gitea_unavailable"}
if len(parts) == 2:
return 200, space_listing(data)
if parts[2] != "posts" or not SLUG_RE.fullmatch(parts[3]):
return 404, {"error": "not_found"}
for post in data["posts"]:
if post["slug"] == parts[3]:
out = dict(post)
out["git_url"] = GIT_WEB_BASE + "/" + space["repo"] + "/src/branch/" + urllib.parse.quote(data["branch"]) + "/" + urllib.parse.quote(post["source"])
out["history_url"] = GIT_WEB_BASE + "/" + space["repo"] + "/commits/branch/" + urllib.parse.quote(data["branch"]) + "/" + urllib.parse.quote(post["source"])
return 200, {"space": data["space"], "branch": data["branch"], "post": out}
return 404, {"error": "post_not_found"}