askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_reader.py · 13.3 KB · 346 lines
| 1 | """The reader: fetch a URL server-side, strip HTML to readable text. |
| 2 | |
| 3 | The lynx/w3m move for an AI + voice consumer (see |
| 4 | plan/vocabulary/57-reader-spec.md): a page becomes {title, text, links} |
| 5 | where links are numbered in document order so they double as speakable |
| 6 | navigation targets ("open link three"). stdlib only -- urllib for the |
| 7 | fetch, html.parser for the strip. No regex-only HTML scraping: regex |
| 8 | reliably misses attribute quoting, comments, and malformed markup in ways |
| 9 | that would either corrupt the text or -- worse -- leak script/style |
| 10 | content into what's supposed to be the readable body. |
| 11 | |
| 12 | THE SSRF GATE is the point of this module, not an afterthought. A server |
| 13 | that will fetch any URL a caller names is a standard pivot into internal |
| 14 | infrastructure: loopback, RFC1918 ranges, and especially link-local |
| 15 | 169.254.169.254 (cloud metadata, credentials for the asking). Every |
| 16 | hostname -- the original and every redirect hop -- is resolved and |
| 17 | checked BEFORE the socket opens. Redirects are followed manually (never |
| 18 | by urllib's own follower) precisely so each hop gets its own resolve+check |
| 19 | pass instead of trusting the first check to cover wherever the chain |
| 20 | ends up (DNS rebinding: a name can resolve to something public at check |
| 21 | time and something private moments later). Any resolution error or |
| 22 | ambiguity refuses the fetch -- fail closed, never fail open. |
| 23 | """ |
| 24 | |
| 25 | from __future__ import annotations |
| 26 | |
| 27 | import ipaddress |
| 28 | import socket |
| 29 | import urllib.error |
| 30 | import urllib.parse |
| 31 | import urllib.request |
| 32 | from html.parser import HTMLParser |
| 33 | from typing import Any |
| 34 | |
| 35 | #: Redirect hops allowed after the initial fetch. Kept small on purpose -- |
| 36 | #: a legitimate page does not need a long chain, and every hop is another |
| 37 | #: SSRF re-check surface. |
| 38 | MAX_REDIRECTS = 5 |
| 39 | |
| 40 | _REDIRECT_CODES = frozenset({301, 302, 303, 307, 308}) |
| 41 | _ALLOWED_CONTENT_TYPES = frozenset({"text/html", "application/xhtml+xml"}) |
| 42 | _ALLOWED_SCHEMES = frozenset({"http", "https"}) |
| 43 | |
| 44 | # Tags whose content is never readable body text: markup, chrome, and |
| 45 | # machine-only payloads, not prose. `head` is deliberately NOT here -- |
| 46 | # `<title>` lives inside it and is captured separately; head's other |
| 47 | # children (meta, link) carry no text data, so nothing else leaks through. |
| 48 | _SKIP_TEXT_TAGS = frozenset({"script", "style", "nav", "noscript", "template"}) |
| 49 | # Tags that mark a paragraph/line boundary -- their start inserts a break so |
| 50 | # the collapsed-whitespace pass below still keeps paragraphs apart. |
| 51 | _BLOCK_TAGS = frozenset({ |
| 52 | "p", "div", "br", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6", |
| 53 | "blockquote", "pre", "section", "article", "table", "header", "footer", |
| 54 | }) |
| 55 | _LINK_LABEL_MAX = 80 |
| 56 | _READ_CHUNK = 65536 |
| 57 | _USER_AGENT = "dbbasic-reader/1.0 (+text mode)" |
| 58 | |
| 59 | |
| 60 | class ReaderError(RuntimeError): |
| 61 | """Any refusal or fetch failure. Callers render the message, not a traceback.""" |
| 62 | |
| 63 | |
| 64 | class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): |
| 65 | """Disable urllib's own redirect-following. |
| 66 | |
| 67 | Returning None from redirect_request tells urllib "don't build a |
| 68 | follow-up request" -- urlopen then raises HTTPError for the 3xx status |
| 69 | instead of silently chasing it. That HTTPError's Location header is |
| 70 | read manually by read_page, which re-resolves and re-checks the next |
| 71 | hop before ever opening a connection to it. |
| 72 | """ |
| 73 | |
| 74 | def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: D102 |
| 75 | return None |
| 76 | |
| 77 | |
| 78 | def read_page(url: str, *, timeout: float = 10, max_bytes: int = 2_000_000) -> dict[str, Any]: |
| 79 | """Fetch one URL and return {title, text, links, final_url, truncated}. |
| 80 | |
| 81 | ``links`` is a list of {n, label, href} in document order, deduped by |
| 82 | href, resolved to absolute URLs against the page that carried them. |
| 83 | |
| 84 | Raises ReaderError for a disallowed scheme, an SSRF-refused host, too |
| 85 | many redirects, a non-HTML response, or any network failure/timeout. |
| 86 | Never raises anything else -- callers (the MCP tool, the view block, |
| 87 | the HTTP endpoint) map ReaderError to a visible error, not a 500. |
| 88 | """ |
| 89 | opener = urllib.request.build_opener(_NoRedirectHandler) |
| 90 | current = url |
| 91 | |
| 92 | for _hop in range(MAX_REDIRECTS + 1): |
| 93 | parsed = _check_url(current) |
| 94 | request = urllib.request.Request( |
| 95 | urllib.parse.urlunsplit(parsed), headers={"User-Agent": _USER_AGENT} |
| 96 | ) |
| 97 | |
| 98 | try: |
| 99 | response = opener.open(request, timeout=timeout) |
| 100 | except urllib.error.HTTPError as exc: |
| 101 | if exc.code in _REDIRECT_CODES: |
| 102 | current = _next_redirect_url(current, exc) |
| 103 | continue |
| 104 | raise ReaderError(f"Fetch failed: HTTP {exc.code}") from exc |
| 105 | except (socket.timeout, TimeoutError) as exc: |
| 106 | raise ReaderError(f"Fetch timed out after {timeout}s") from exc |
| 107 | except urllib.error.URLError as exc: |
| 108 | if isinstance(exc.reason, (socket.timeout, TimeoutError)): |
| 109 | raise ReaderError(f"Fetch timed out after {timeout}s") from exc |
| 110 | raise ReaderError(f"Fetch failed: {exc.reason}") from exc |
| 111 | |
| 112 | try: |
| 113 | with response: |
| 114 | content_type_header = response.headers.get("Content-Type") or "" |
| 115 | content_type = content_type_header.split(";", 1)[0].strip().lower() |
| 116 | if content_type not in _ALLOWED_CONTENT_TYPES: |
| 117 | raise ReaderError( |
| 118 | f"Refusing non-HTML content type: {content_type or 'unknown'}" |
| 119 | ) |
| 120 | final_url = response.geturl() |
| 121 | body, truncated = _read_capped(response, max_bytes=max_bytes) |
| 122 | except (socket.timeout, TimeoutError) as exc: |
| 123 | raise ReaderError(f"Fetch timed out after {timeout}s") from exc |
| 124 | except OSError as exc: |
| 125 | raise ReaderError(f"Fetch failed while reading response: {exc}") from exc |
| 126 | |
| 127 | charset = _charset_from_content_type(content_type_header) |
| 128 | try: |
| 129 | html_text = body.decode(charset, errors="replace") |
| 130 | except LookupError: |
| 131 | html_text = body.decode("utf-8", errors="replace") |
| 132 | |
| 133 | page = _strip_html(html_text, base_url=final_url) |
| 134 | page["final_url"] = final_url |
| 135 | page["truncated"] = truncated |
| 136 | return page |
| 137 | |
| 138 | raise ReaderError(f"Too many redirects (> {MAX_REDIRECTS})") |
| 139 | |
| 140 | |
| 141 | def _next_redirect_url(current: str, exc: urllib.error.HTTPError) -> str: |
| 142 | location = exc.headers.get("Location") if exc.headers is not None else None |
| 143 | if not location: |
| 144 | raise ReaderError(f"Redirect ({exc.code}) from {current} carried no Location") from exc |
| 145 | next_url = urllib.parse.urljoin(current, location) |
| 146 | next_scheme = urllib.parse.urlsplit(next_url).scheme |
| 147 | if next_scheme not in _ALLOWED_SCHEMES: |
| 148 | raise ReaderError( |
| 149 | f"Refusing redirect to non-http(s) scheme: {next_scheme or '(none)'}" |
| 150 | ) from exc |
| 151 | return next_url |
| 152 | |
| 153 | |
| 154 | def _check_url(url: str) -> urllib.parse.SplitResult: |
| 155 | """Parse `url`, enforce http/https, and run the SSRF host check. |
| 156 | |
| 157 | Called on the original URL and again on every redirect hop -- the |
| 158 | re-resolution on each hop is the DNS-rebinding defense the spec asks |
| 159 | for; a single check up front would not cover a hop that lands |
| 160 | somewhere different than the first hostname resolved to. |
| 161 | """ |
| 162 | parsed = urllib.parse.urlsplit(url) |
| 163 | if parsed.scheme not in _ALLOWED_SCHEMES: |
| 164 | raise ReaderError(f"Refusing non-http(s) scheme: {parsed.scheme or '(none)'}") |
| 165 | hostname = parsed.hostname |
| 166 | if not hostname: |
| 167 | raise ReaderError("URL has no host") |
| 168 | _refuse_internal_host(hostname) |
| 169 | return parsed |
| 170 | |
| 171 | |
| 172 | def _refuse_internal_host(hostname: str) -> None: |
| 173 | """Resolve `hostname` and refuse if ANY resolved address is internal. |
| 174 | |
| 175 | Fails closed: a DNS error, an empty resolution, or an address |
| 176 | ipaddress can't parse all refuse the fetch rather than let it through |
| 177 | on ambiguity -- the same posture as 02-public-write-hardening. |
| 178 | """ |
| 179 | try: |
| 180 | infos = socket.getaddrinfo(hostname, None) |
| 181 | except (socket.gaierror, UnicodeError, OSError) as exc: |
| 182 | raise ReaderError(f"Could not resolve host {hostname!r}: {exc}") from exc |
| 183 | |
| 184 | if not infos: |
| 185 | raise ReaderError(f"Host {hostname!r} resolved to no addresses") |
| 186 | |
| 187 | for info in infos: |
| 188 | raw_addr = info[4][0] |
| 189 | try: |
| 190 | addr = ipaddress.ip_address(raw_addr) |
| 191 | except ValueError as exc: |
| 192 | raise ReaderError( |
| 193 | f"Unparseable resolved address for {hostname!r}: {raw_addr!r}" |
| 194 | ) from exc |
| 195 | if ( |
| 196 | addr.is_private |
| 197 | or addr.is_loopback |
| 198 | or addr.is_link_local |
| 199 | or addr.is_reserved |
| 200 | or addr.is_multicast |
| 201 | or addr.is_unspecified |
| 202 | ): |
| 203 | raise ReaderError( |
| 204 | f"Refusing to fetch {hostname!r}: resolves to internal address {addr}" |
| 205 | ) |
| 206 | |
| 207 | |
| 208 | def _read_capped(response, *, max_bytes: int) -> tuple[bytes, bool]: |
| 209 | """Stream the response body, aborting past `max_bytes`.""" |
| 210 | chunks: list[bytes] = [] |
| 211 | total = 0 |
| 212 | truncated = False |
| 213 | while True: |
| 214 | chunk = response.read(_READ_CHUNK) |
| 215 | if not chunk: |
| 216 | break |
| 217 | total += len(chunk) |
| 218 | if total > max_bytes: |
| 219 | keep = max_bytes - (total - len(chunk)) |
| 220 | if keep > 0: |
| 221 | chunks.append(chunk[:keep]) |
| 222 | truncated = True |
| 223 | break |
| 224 | chunks.append(chunk) |
| 225 | return b"".join(chunks), truncated |
| 226 | |
| 227 | |
| 228 | def _charset_from_content_type(header: str) -> str: |
| 229 | for part in header.split(";")[1:]: |
| 230 | part = part.strip() |
| 231 | if part.lower().startswith("charset="): |
| 232 | charset = part.split("=", 1)[1].strip().strip('"').strip("'") |
| 233 | if charset: |
| 234 | return charset |
| 235 | return "utf-8" |
| 236 | |
| 237 | |
| 238 | class _Stripper(HTMLParser): |
| 239 | """Reduce one HTML document to (title, body text, numbered links). |
| 240 | |
| 241 | script/style/nav/head content is dropped entirely. Block-level tags |
| 242 | insert a paragraph break; everything else collapses to plain text. |
| 243 | Anchor text becomes both the link's label and part of the flowing |
| 244 | body text, same as how the text would read in a browser. |
| 245 | """ |
| 246 | |
| 247 | def __init__(self, base_url: str) -> None: |
| 248 | super().__init__(convert_charrefs=True) |
| 249 | self._base_url = base_url |
| 250 | self._skip_depth = 0 |
| 251 | self._in_title = False |
| 252 | self._title_parts: list[str] = [] |
| 253 | self._text_parts: list[str] = [] |
| 254 | self._links: list[dict[str, Any]] = [] |
| 255 | self._seen_hrefs: set[str] = set() |
| 256 | self._current_href: str | None = None |
| 257 | self._current_label_parts: list[str] = [] |
| 258 | |
| 259 | @property |
| 260 | def title(self) -> str: |
| 261 | return " ".join("".join(self._title_parts).split()) |
| 262 | |
| 263 | @property |
| 264 | def links(self) -> list[dict[str, Any]]: |
| 265 | return self._links |
| 266 | |
| 267 | def text(self) -> str: |
| 268 | paragraphs = [ |
| 269 | collapsed |
| 270 | for chunk in "".join(self._text_parts).split("\n") |
| 271 | if (collapsed := " ".join(chunk.split())) |
| 272 | ] |
| 273 | return "\n\n".join(paragraphs) |
| 274 | |
| 275 | def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: |
| 276 | if tag in _SKIP_TEXT_TAGS: |
| 277 | self._skip_depth += 1 |
| 278 | return |
| 279 | if self._skip_depth: |
| 280 | return |
| 281 | if tag == "title": |
| 282 | self._in_title = True |
| 283 | return |
| 284 | if tag == "a": |
| 285 | self._open_anchor(dict(attrs)) |
| 286 | return |
| 287 | if tag in _BLOCK_TAGS: |
| 288 | self._text_parts.append("\n") |
| 289 | |
| 290 | def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: |
| 291 | self.handle_starttag(tag, attrs) |
| 292 | if tag in _SKIP_TEXT_TAGS and self._skip_depth: |
| 293 | self._skip_depth -= 1 |
| 294 | |
| 295 | def handle_endtag(self, tag: str) -> None: |
| 296 | if tag in _SKIP_TEXT_TAGS: |
| 297 | if self._skip_depth: |
| 298 | self._skip_depth -= 1 |
| 299 | return |
| 300 | if self._skip_depth: |
| 301 | return |
| 302 | if tag == "title": |
| 303 | self._in_title = False |
| 304 | return |
| 305 | if tag == "a" and self._current_href is not None: |
| 306 | self._close_anchor() |
| 307 | elif tag in _BLOCK_TAGS: |
| 308 | self._text_parts.append("\n") |
| 309 | |
| 310 | def handle_data(self, data: str) -> None: |
| 311 | if self._skip_depth: |
| 312 | return |
| 313 | if self._in_title: |
| 314 | self._title_parts.append(data) |
| 315 | return |
| 316 | if self._current_href is not None: |
| 317 | self._current_label_parts.append(data) |
| 318 | self._text_parts.append(data) |
| 319 | |
| 320 | def _open_anchor(self, attrs: dict[str, str | None]) -> None: |
| 321 | href = (attrs.get("href") or "").strip() |
| 322 | if not href or href.startswith(("javascript:", "#")): |
| 323 | return |
| 324 | self._current_href = urllib.parse.urljoin(self._base_url, href) |
| 325 | self._current_label_parts = [] |
| 326 | |
| 327 | def _close_anchor(self) -> None: |
| 328 | href = self._current_href |
| 329 | self._current_href = None |
| 330 | label = " ".join("".join(self._current_label_parts).split()) |
| 331 | self._current_label_parts = [] |
| 332 | if href is None or href in self._seen_hrefs: |
| 333 | return |
| 334 | self._seen_hrefs.add(href) |
| 335 | if not label: |
| 336 | label = href |
| 337 | if len(label) > _LINK_LABEL_MAX: |
| 338 | label = label[: _LINK_LABEL_MAX - 1].rstrip() + "…" |
| 339 | self._links.append({"n": len(self._links) + 1, "label": label, "href": href}) |
| 340 | |
| 341 | |
| 342 | def _strip_html(html_text: str, *, base_url: str) -> dict[str, Any]: |
| 343 | stripper = _Stripper(base_url) |
| 344 | stripper.feed(html_text) |
| 345 | stripper.close() |
| 346 | return {"title": stripper.title, "text": stripper.text(), "links": stripper.links} |