news/digest/newsdigest/safefetch.py

"""SSRF-safe HTTP(S) fetching.

Every hop: scheme http/https only, port 80/443 only, no userinfo, hostname resolved once and every
address must be public (no loopback, private, link-local, CGNAT, multicast, reserved, unspecified,
IPv4-mapped private). The connection is pinned to the vetted IP (no DNS-rebinding window), TLS still
verifies the real hostname. Redirects are followed by hand (max 3), each re-vetted. Body capped.
"""
from __future__ import annotations

import http.client
import ipaddress
import socket
import ssl
import json
import os
import urllib.parse

MAX_BYTES = 1_500_000
TIMEOUT = 10.0
MAX_REDIRECTS = 3
UA = "NewInternetNewsDigest/1.0 (+https://news.seamtro.ai)"
_CGNAT = ipaddress.ip_network("100.64.0.0/10")
_BENCH = ipaddress.ip_network("198.18.0.0/15")


class FetchError(Exception):
    pass


def ip_is_public(ip: str) -> bool:
    try:
        a = ipaddress.ip_address(ip)
    except ValueError:
        return False
    if isinstance(a, ipaddress.IPv6Address) and a.ipv4_mapped:
        a = a.ipv4_mapped
    if isinstance(a, ipaddress.IPv6Address) and (a.sixtofour or a.teredo):
        return False
    if (a.is_private or a.is_loopback or a.is_link_local or a.is_multicast or a.is_reserved
            or a.is_unspecified or not a.is_global):
        return False
    if isinstance(a, ipaddress.IPv4Address) and (a in _CGNAT or a in _BENCH):
        return False
    return True


def check_url(url: str) -> urllib.parse.SplitResult:
    if not isinstance(url, str) or len(url) > 2000 or any(c in url for c in "\r\n\t "):
        raise FetchError("bad_url")
    p = urllib.parse.urlsplit(url)
    if p.scheme not in ("http", "https") or not p.hostname:
        raise FetchError("bad_scheme")
    if p.username or p.password:
        raise FetchError("userinfo_not_allowed")
    try:
        port = p.port
    except ValueError:
        raise FetchError("bad_port") from None
    if port not in (None, 80, 443):
        raise FetchError("bad_port")
    host = p.hostname.rstrip(".").lower()
    if host == "localhost" or host.endswith((".localhost", ".local", ".internal", ".lan", ".home.arpa")):
        raise FetchError("private_host")
    try:
        ipaddress.ip_address(host)
        raise FetchError("ip_literal_not_allowed")
    except ValueError:
        pass
    if "." not in host:
        raise FetchError("private_host")
    return p


# The box sits behind a fake-IP egress gateway: the local resolver answers 198.18.0.1 for every
# name and the gateway resolves the real target. A local answer equal to one of these IPs is never
# trusted on its own: the real records are looked up over DNS-over-HTTPS and must all be public.
EGRESS_PROXY_IPS = {x.strip() for x in os.environ.get("NEWS_EGRESS_PROXY_IPS", "198.18.0.1").split(",") if x.strip()}
DOH_URL = "https://cloudflare-dns.com/dns-query"


def doh_lookup(host: str) -> list[str]:
    ips: list[str] = []
    for qtype in ("A", "AAAA"):
        q = urllib.parse.urlencode({"name": host, "type": qtype})
        conn = http.client.HTTPSConnection("cloudflare-dns.com", timeout=TIMEOUT, context=ssl.create_default_context())
        try:
            conn.request("GET", "/dns-query?" + q, headers={"Accept": "application/dns-json"})
            r = conn.getresponse()
            if r.status != 200:
                raise FetchError("dns_failed")
            data = json.loads(r.read(200_000))
        except (OSError, ValueError, http.client.HTTPException):
            raise FetchError("dns_failed") from None
        finally:
            conn.close()
        for ans in data.get("Answer") or []:
            if ans.get("type") in (1, 28):
                ips.append(str(ans.get("data")))
            elif ans.get("type") == 5:
                continue  # CNAME chain: the final A/AAAA records are listed too
    return ips


def resolve_public(host: str, port: int, resolver=socket.getaddrinfo, doh=None) -> str:
    """Returns the IP to connect to. Raises unless every real address of host is public."""
    try:
        infos = resolver(host, port, 0, socket.SOCK_STREAM)
    except (socket.gaierror, UnicodeError):
        raise FetchError("dns_failed") from None
    ips = [i[4][0] for i in infos]
    if not ips:
        raise FetchError("dns_failed")
    if all(ip in EGRESS_PROXY_IPS for ip in ips):
        real = (doh or doh_lookup)(host)
        if not real:
            raise FetchError("dns_failed")
        if not all(ip_is_public(ip) for ip in real):
            raise FetchError("private_address")
        return ips[0]            # connect through the gateway; it reaches the vetted public host
    # all records must be public: a mixed answer is a rebinding/SSRF attempt
    if not all(ip_is_public(ip) for ip in ips):
        raise FetchError("private_address")
    return ips[0]


class _PinnedHTTPS(http.client.HTTPSConnection):
    def __init__(self, host, ip, **kw):
        super().__init__(host, **kw)
        self._ip = ip

    def connect(self):
        sock = socket.create_connection((self._ip, self.port), self.timeout)
        self.sock = self._context.wrap_socket(sock, server_hostname=self.host)


class _PinnedHTTP(http.client.HTTPConnection):
    def __init__(self, host, ip, **kw):
        super().__init__(host, **kw)
        self._ip = ip

    def connect(self):
        self.sock = socket.create_connection((self._ip, self.port), self.timeout)


def fetch(url: str, *, max_bytes: int = MAX_BYTES, resolver=socket.getaddrinfo, doh=None,
          accept: str = "text/html,application/xhtml+xml,application/rss+xml,application/xml;q=0.9,*/*;q=0.5") -> tuple[str, str, bytes]:
    """Returns (final_url, content_type, body)."""
    for _ in range(MAX_REDIRECTS + 1):
        p = check_url(url)
        https = p.scheme == "https"
        port = p.port or (443 if https else 80)
        ip = resolve_public(p.hostname, port, resolver, doh)
        path = (p.path or "/") + (("?" + p.query) if p.query else "")
        if https:
            conn = _PinnedHTTPS(p.hostname, ip, port=port, timeout=TIMEOUT, context=ssl.create_default_context())
        else:
            conn = _PinnedHTTP(p.hostname, ip, port=port, timeout=TIMEOUT)
        try:
            conn.request("GET", path, headers={"User-Agent": UA, "Accept": accept, "Accept-Encoding": "identity"})
            r = conn.getresponse()
            if r.status in (301, 302, 303, 307, 308):
                loc = r.getheader("Location") or ""
                url = urllib.parse.urljoin(url, loc)
                continue
            if r.status != 200:
                raise FetchError(f"http_{r.status}")
            body = r.read(max_bytes + 1)
            if len(body) > max_bytes:
                body = body[:max_bytes]
            return url, (r.getheader("Content-Type") or "").lower(), body
        except (OSError, http.client.HTTPException, ssl.SSLError) as exc:
            raise FetchError("connect_failed") from exc
        finally:
            conn.close()
    raise FetchError("too_many_redirects")