Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_reader.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_reader.py · 13.3 KB · 346 lines
1"""The reader: fetch a URL server-side, strip HTML to readable text.
2
3The lynx/w3m move for an AI + voice consumer (see
4plan/vocabulary/57-reader-spec.md): a page becomes {title, text, links}
5where links are numbered in document order so they double as speakable
6navigation targets ("open link three"). stdlib only -- urllib for the
7fetch, html.parser for the strip. No regex-only HTML scraping: regex
8reliably misses attribute quoting, comments, and malformed markup in ways
9that would either corrupt the text or -- worse -- leak script/style
10content into what's supposed to be the readable body.
11
12THE SSRF GATE is the point of this module, not an afterthought. A server
13that will fetch any URL a caller names is a standard pivot into internal
14infrastructure: loopback, RFC1918 ranges, and especially link-local
15169.254.169.254 (cloud metadata, credentials for the asking). Every
16hostname -- the original and every redirect hop -- is resolved and
17checked BEFORE the socket opens. Redirects are followed manually (never
18by urllib's own follower) precisely so each hop gets its own resolve+check
19pass instead of trusting the first check to cover wherever the chain
20ends up (DNS rebinding: a name can resolve to something public at check
21time and something private moments later). Any resolution error or
22ambiguity refuses the fetch -- fail closed, never fail open.
23"""
24
25from __future__ import annotations
26
27import ipaddress
28import socket
29import urllib.error
30import urllib.parse
31import urllib.request
32from html.parser import HTMLParser
33from typing import Any
34
35#: Redirect hops allowed after the initial fetch. Kept small on purpose --
36#: a legitimate page does not need a long chain, and every hop is another
37#: SSRF re-check surface.
38MAX_REDIRECTS = 5
39
40_REDIRECT_CODES = frozenset({301, 302, 303, 307, 308})
41_ALLOWED_CONTENT_TYPES = frozenset({"text/html", "application/xhtml+xml"})
42_ALLOWED_SCHEMES = frozenset({"http", "https"})
43
44# Tags whose content is never readable body text: markup, chrome, and
45# machine-only payloads, not prose. `head` is deliberately NOT here --
46# `<title>` lives inside it and is captured separately; head's other
47# children (meta, link) carry no text data, so nothing else leaks through.
48_SKIP_TEXT_TAGS = frozenset({"script", "style", "nav", "noscript", "template"})
49# Tags that mark a paragraph/line boundary -- their start inserts a break so
50# the collapsed-whitespace pass below still keeps paragraphs apart.
51_BLOCK_TAGS = frozenset({
52 "p", "div", "br", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6",
53 "blockquote", "pre", "section", "article", "table", "header", "footer",
54})
55_LINK_LABEL_MAX = 80
56_READ_CHUNK = 65536
57_USER_AGENT = "dbbasic-reader/1.0 (+text mode)"
58
59
60class ReaderError(RuntimeError):
61 """Any refusal or fetch failure. Callers render the message, not a traceback."""
62
63
64class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
65 """Disable urllib's own redirect-following.
66
67 Returning None from redirect_request tells urllib "don't build a
68 follow-up request" -- urlopen then raises HTTPError for the 3xx status
69 instead of silently chasing it. That HTTPError's Location header is
70 read manually by read_page, which re-resolves and re-checks the next
71 hop before ever opening a connection to it.
72 """
73
74 def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: D102
75 return None
76
77
78def read_page(url: str, *, timeout: float = 10, max_bytes: int = 2_000_000) -> dict[str, Any]:
79 """Fetch one URL and return {title, text, links, final_url, truncated}.
80
81 ``links`` is a list of {n, label, href} in document order, deduped by
82 href, resolved to absolute URLs against the page that carried them.
83
84 Raises ReaderError for a disallowed scheme, an SSRF-refused host, too
85 many redirects, a non-HTML response, or any network failure/timeout.
86 Never raises anything else -- callers (the MCP tool, the view block,
87 the HTTP endpoint) map ReaderError to a visible error, not a 500.
88 """
89 opener = urllib.request.build_opener(_NoRedirectHandler)
90 current = url
91
92 for _hop in range(MAX_REDIRECTS + 1):
93 parsed = _check_url(current)
94 request = urllib.request.Request(
95 urllib.parse.urlunsplit(parsed), headers={"User-Agent": _USER_AGENT}
96 )
97
98 try:
99 response = opener.open(request, timeout=timeout)
100 except urllib.error.HTTPError as exc:
101 if exc.code in _REDIRECT_CODES:
102 current = _next_redirect_url(current, exc)
103 continue
104 raise ReaderError(f"Fetch failed: HTTP {exc.code}") from exc
105 except (socket.timeout, TimeoutError) as exc:
106 raise ReaderError(f"Fetch timed out after {timeout}s") from exc
107 except urllib.error.URLError as exc:
108 if isinstance(exc.reason, (socket.timeout, TimeoutError)):
109 raise ReaderError(f"Fetch timed out after {timeout}s") from exc
110 raise ReaderError(f"Fetch failed: {exc.reason}") from exc
111
112 try:
113 with response:
114 content_type_header = response.headers.get("Content-Type") or ""
115 content_type = content_type_header.split(";", 1)[0].strip().lower()
116 if content_type not in _ALLOWED_CONTENT_TYPES:
117 raise ReaderError(
118 f"Refusing non-HTML content type: {content_type or 'unknown'}"
119 )
120 final_url = response.geturl()
121 body, truncated = _read_capped(response, max_bytes=max_bytes)
122 except (socket.timeout, TimeoutError) as exc:
123 raise ReaderError(f"Fetch timed out after {timeout}s") from exc
124 except OSError as exc:
125 raise ReaderError(f"Fetch failed while reading response: {exc}") from exc
126
127 charset = _charset_from_content_type(content_type_header)
128 try:
129 html_text = body.decode(charset, errors="replace")
130 except LookupError:
131 html_text = body.decode("utf-8", errors="replace")
132
133 page = _strip_html(html_text, base_url=final_url)
134 page["final_url"] = final_url
135 page["truncated"] = truncated
136 return page
137
138 raise ReaderError(f"Too many redirects (> {MAX_REDIRECTS})")
139
140
141def _next_redirect_url(current: str, exc: urllib.error.HTTPError) -> str:
142 location = exc.headers.get("Location") if exc.headers is not None else None
143 if not location:
144 raise ReaderError(f"Redirect ({exc.code}) from {current} carried no Location") from exc
145 next_url = urllib.parse.urljoin(current, location)
146 next_scheme = urllib.parse.urlsplit(next_url).scheme
147 if next_scheme not in _ALLOWED_SCHEMES:
148 raise ReaderError(
149 f"Refusing redirect to non-http(s) scheme: {next_scheme or '(none)'}"
150 ) from exc
151 return next_url
152
153
154def _check_url(url: str) -> urllib.parse.SplitResult:
155 """Parse `url`, enforce http/https, and run the SSRF host check.
156
157 Called on the original URL and again on every redirect hop -- the
158 re-resolution on each hop is the DNS-rebinding defense the spec asks
159 for; a single check up front would not cover a hop that lands
160 somewhere different than the first hostname resolved to.
161 """
162 parsed = urllib.parse.urlsplit(url)
163 if parsed.scheme not in _ALLOWED_SCHEMES:
164 raise ReaderError(f"Refusing non-http(s) scheme: {parsed.scheme or '(none)'}")
165 hostname = parsed.hostname
166 if not hostname:
167 raise ReaderError("URL has no host")
168 _refuse_internal_host(hostname)
169 return parsed
170
171
172def _refuse_internal_host(hostname: str) -> None:
173 """Resolve `hostname` and refuse if ANY resolved address is internal.
174
175 Fails closed: a DNS error, an empty resolution, or an address
176 ipaddress can't parse all refuse the fetch rather than let it through
177 on ambiguity -- the same posture as 02-public-write-hardening.
178 """
179 try:
180 infos = socket.getaddrinfo(hostname, None)
181 except (socket.gaierror, UnicodeError, OSError) as exc:
182 raise ReaderError(f"Could not resolve host {hostname!r}: {exc}") from exc
183
184 if not infos:
185 raise ReaderError(f"Host {hostname!r} resolved to no addresses")
186
187 for info in infos:
188 raw_addr = info[4][0]
189 try:
190 addr = ipaddress.ip_address(raw_addr)
191 except ValueError as exc:
192 raise ReaderError(
193 f"Unparseable resolved address for {hostname!r}: {raw_addr!r}"
194 ) from exc
195 if (
196 addr.is_private
197 or addr.is_loopback
198 or addr.is_link_local
199 or addr.is_reserved
200 or addr.is_multicast
201 or addr.is_unspecified
202 ):
203 raise ReaderError(
204 f"Refusing to fetch {hostname!r}: resolves to internal address {addr}"
205 )
206
207
208def _read_capped(response, *, max_bytes: int) -> tuple[bytes, bool]:
209 """Stream the response body, aborting past `max_bytes`."""
210 chunks: list[bytes] = []
211 total = 0
212 truncated = False
213 while True:
214 chunk = response.read(_READ_CHUNK)
215 if not chunk:
216 break
217 total += len(chunk)
218 if total > max_bytes:
219 keep = max_bytes - (total - len(chunk))
220 if keep > 0:
221 chunks.append(chunk[:keep])
222 truncated = True
223 break
224 chunks.append(chunk)
225 return b"".join(chunks), truncated
226
227
228def _charset_from_content_type(header: str) -> str:
229 for part in header.split(";")[1:]:
230 part = part.strip()
231 if part.lower().startswith("charset="):
232 charset = part.split("=", 1)[1].strip().strip('"').strip("'")
233 if charset:
234 return charset
235 return "utf-8"
236
237
238class _Stripper(HTMLParser):
239 """Reduce one HTML document to (title, body text, numbered links).
240
241 script/style/nav/head content is dropped entirely. Block-level tags
242 insert a paragraph break; everything else collapses to plain text.
243 Anchor text becomes both the link's label and part of the flowing
244 body text, same as how the text would read in a browser.
245 """
246
247 def __init__(self, base_url: str) -> None:
248 super().__init__(convert_charrefs=True)
249 self._base_url = base_url
250 self._skip_depth = 0
251 self._in_title = False
252 self._title_parts: list[str] = []
253 self._text_parts: list[str] = []
254 self._links: list[dict[str, Any]] = []
255 self._seen_hrefs: set[str] = set()
256 self._current_href: str | None = None
257 self._current_label_parts: list[str] = []
258
259 @property
260 def title(self) -> str:
261 return " ".join("".join(self._title_parts).split())
262
263 @property
264 def links(self) -> list[dict[str, Any]]:
265 return self._links
266
267 def text(self) -> str:
268 paragraphs = [
269 collapsed
270 for chunk in "".join(self._text_parts).split("\n")
271 if (collapsed := " ".join(chunk.split()))
272 ]
273 return "\n\n".join(paragraphs)
274
275 def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
276 if tag in _SKIP_TEXT_TAGS:
277 self._skip_depth += 1
278 return
279 if self._skip_depth:
280 return
281 if tag == "title":
282 self._in_title = True
283 return
284 if tag == "a":
285 self._open_anchor(dict(attrs))
286 return
287 if tag in _BLOCK_TAGS:
288 self._text_parts.append("\n")
289
290 def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
291 self.handle_starttag(tag, attrs)
292 if tag in _SKIP_TEXT_TAGS and self._skip_depth:
293 self._skip_depth -= 1
294
295 def handle_endtag(self, tag: str) -> None:
296 if tag in _SKIP_TEXT_TAGS:
297 if self._skip_depth:
298 self._skip_depth -= 1
299 return
300 if self._skip_depth:
301 return
302 if tag == "title":
303 self._in_title = False
304 return
305 if tag == "a" and self._current_href is not None:
306 self._close_anchor()
307 elif tag in _BLOCK_TAGS:
308 self._text_parts.append("\n")
309
310 def handle_data(self, data: str) -> None:
311 if self._skip_depth:
312 return
313 if self._in_title:
314 self._title_parts.append(data)
315 return
316 if self._current_href is not None:
317 self._current_label_parts.append(data)
318 self._text_parts.append(data)
319
320 def _open_anchor(self, attrs: dict[str, str | None]) -> None:
321 href = (attrs.get("href") or "").strip()
322 if not href or href.startswith(("javascript:", "#")):
323 return
324 self._current_href = urllib.parse.urljoin(self._base_url, href)
325 self._current_label_parts = []
326
327 def _close_anchor(self) -> None:
328 href = self._current_href
329 self._current_href = None
330 label = " ".join("".join(self._current_label_parts).split())
331 self._current_label_parts = []
332 if href is None or href in self._seen_hrefs:
333 return
334 self._seen_hrefs.add(href)
335 if not label:
336 label = href
337 if len(label) > _LINK_LABEL_MAX:
338 label = label[: _LINK_LABEL_MAX - 1].rstrip() + "…"
339 self._links.append({"n": len(self._links) + 1, "label": label, "href": href})
340
341
342def _strip_html(html_text: str, *, base_url: str) -> dict[str, Any]:
343 stripper = _Stripper(base_url)
344 stripper.feed(html_text)
345 stripper.close()
346 return {"title": stripper.title, "text": stripper.text(), "links": stripper.links}