"""SSRF-safe HTTP(S) fetching. Every hop: scheme http/https only, port 80/443 only, no userinfo, hostname resolved once and every address must be public (no loopback, private, link-local, CGNAT, multicast, reserved, unspecified, IPv4-mapped private). The connection is pinned to the vetted IP (no DNS-rebinding window), TLS still verifies the real hostname. Redirects are followed by hand (max 3), each re-vetted. Body capped. """ from __future__ import annotations import http.client import ipaddress import socket import ssl import json import os import urllib.parse MAX_BYTES = 1_500_000 TIMEOUT = 10.0 MAX_REDIRECTS = 3 UA = "NewInternetNewsDigest/1.0 (+https://news.seamtro.ai)" _CGNAT = ipaddress.ip_network("100.64.0.0/10") _BENCH = ipaddress.ip_network("198.18.0.0/15") class FetchError(Exception): pass def ip_is_public(ip: str) -> bool: try: a = ipaddress.ip_address(ip) except ValueError: return False if isinstance(a, ipaddress.IPv6Address) and a.ipv4_mapped: a = a.ipv4_mapped if isinstance(a, ipaddress.IPv6Address) and (a.sixtofour or a.teredo): return False if (a.is_private or a.is_loopback or a.is_link_local or a.is_multicast or a.is_reserved or a.is_unspecified or not a.is_global): return False if isinstance(a, ipaddress.IPv4Address) and (a in _CGNAT or a in _BENCH): return False return True def check_url(url: str) -> urllib.parse.SplitResult: if not isinstance(url, str) or len(url) > 2000 or any(c in url for c in "\r\n\t "): raise FetchError("bad_url") p = urllib.parse.urlsplit(url) if p.scheme not in ("http", "https") or not p.hostname: raise FetchError("bad_scheme") if p.username or p.password: raise FetchError("userinfo_not_allowed") try: port = p.port except ValueError: raise FetchError("bad_port") from None if port not in (None, 80, 443): raise FetchError("bad_port") host = p.hostname.rstrip(".").lower() if host == "localhost" or host.endswith((".localhost", ".local", ".internal", ".lan", ".home.arpa")): raise FetchError("private_host") try: ipaddress.ip_address(host) raise FetchError("ip_literal_not_allowed") except ValueError: pass if "." not in host: raise FetchError("private_host") return p # The box sits behind a fake-IP egress gateway: the local resolver answers 198.18.0.1 for every # name and the gateway resolves the real target. A local answer equal to one of these IPs is never # trusted on its own: the real records are looked up over DNS-over-HTTPS and must all be public. EGRESS_PROXY_IPS = {x.strip() for x in os.environ.get("NEWS_EGRESS_PROXY_IPS", "198.18.0.1").split(",") if x.strip()} DOH_URL = "https://cloudflare-dns.com/dns-query" def doh_lookup(host: str) -> list[str]: ips: list[str] = [] for qtype in ("A", "AAAA"): q = urllib.parse.urlencode({"name": host, "type": qtype}) conn = http.client.HTTPSConnection("cloudflare-dns.com", timeout=TIMEOUT, context=ssl.create_default_context()) try: conn.request("GET", "/dns-query?" + q, headers={"Accept": "application/dns-json"}) r = conn.getresponse() if r.status != 200: raise FetchError("dns_failed") data = json.loads(r.read(200_000)) except (OSError, ValueError, http.client.HTTPException): raise FetchError("dns_failed") from None finally: conn.close() for ans in data.get("Answer") or []: if ans.get("type") in (1, 28): ips.append(str(ans.get("data"))) elif ans.get("type") == 5: continue # CNAME chain: the final A/AAAA records are listed too return ips def resolve_public(host: str, port: int, resolver=socket.getaddrinfo, doh=None) -> str: """Returns the IP to connect to. Raises unless every real address of host is public.""" try: infos = resolver(host, port, 0, socket.SOCK_STREAM) except (socket.gaierror, UnicodeError): raise FetchError("dns_failed") from None ips = [i[4][0] for i in infos] if not ips: raise FetchError("dns_failed") if all(ip in EGRESS_PROXY_IPS for ip in ips): real = (doh or doh_lookup)(host) if not real: raise FetchError("dns_failed") if not all(ip_is_public(ip) for ip in real): raise FetchError("private_address") return ips[0] # connect through the gateway; it reaches the vetted public host # all records must be public: a mixed answer is a rebinding/SSRF attempt if not all(ip_is_public(ip) for ip in ips): raise FetchError("private_address") return ips[0] class _PinnedHTTPS(http.client.HTTPSConnection): def __init__(self, host, ip, **kw): super().__init__(host, **kw) self._ip = ip def connect(self): sock = socket.create_connection((self._ip, self.port), self.timeout) self.sock = self._context.wrap_socket(sock, server_hostname=self.host) class _PinnedHTTP(http.client.HTTPConnection): def __init__(self, host, ip, **kw): super().__init__(host, **kw) self._ip = ip def connect(self): self.sock = socket.create_connection((self._ip, self.port), self.timeout) def fetch(url: str, *, max_bytes: int = MAX_BYTES, resolver=socket.getaddrinfo, doh=None, accept: str = "text/html,application/xhtml+xml,application/rss+xml,application/xml;q=0.9,*/*;q=0.5") -> tuple[str, str, bytes]: """Returns (final_url, content_type, body).""" for _ in range(MAX_REDIRECTS + 1): p = check_url(url) https = p.scheme == "https" port = p.port or (443 if https else 80) ip = resolve_public(p.hostname, port, resolver, doh) path = (p.path or "/") + (("?" + p.query) if p.query else "") if https: conn = _PinnedHTTPS(p.hostname, ip, port=port, timeout=TIMEOUT, context=ssl.create_default_context()) else: conn = _PinnedHTTP(p.hostname, ip, port=port, timeout=TIMEOUT) try: conn.request("GET", path, headers={"User-Agent": UA, "Accept": accept, "Accept-Encoding": "identity"}) r = conn.getresponse() if r.status in (301, 302, 303, 307, 308): loc = r.getheader("Location") or "" url = urllib.parse.urljoin(url, loc) continue if r.status != 200: raise FetchError(f"http_{r.status}") body = r.read(max_bytes + 1) if len(body) > max_bytes: body = body[:max_bytes] return url, (r.getheader("Content-Type") or "").lower(), body except (OSError, http.client.HTTPException, ssl.SSLError) as exc: raise FetchError("connect_failed") from exc finally: conn.close() raise FetchError("too_many_redirects")