Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_object_reader.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_object_reader.py · 8.3 KB · 235 lines
1"""Tests for object_reader: the SSRF gate and the HTML-to-text strip.
2
3No real network calls -- socket.getaddrinfo and the urllib opener are
4monkeypatched, same convention as test_object_tts.py swapping out
5subprocess.run. The SSRF gate is the important half: see the spec at
6plan/vocabulary/57-reader-spec.md, "THE SECURITY GATE".
7"""
8
9from __future__ import annotations
10
11import io
12import socket
13
14import pytest
15
16import object_reader as reader
17
18
19class FakeHeaders(dict):
20 """Case-insensitive enough for the .get("Location") / .get("Content-Type") calls."""
21
22 def get(self, key, default=None):
23 for existing in self:
24 if existing.lower() == key.lower():
25 return self[existing]
26 return default
27
28
29class FakeResponse(io.BytesIO):
30 def __init__(self, data: bytes, url: str, headers: dict):
31 super().__init__(data)
32 self._url = url
33 self.headers = FakeHeaders(headers)
34
35 def geturl(self):
36 return self._url
37
38 def __enter__(self):
39 return self
40
41 def __exit__(self, *exc_info):
42 self.close()
43
44
45def addrinfo_for(mapping: dict[str, str]):
46 """Build a fake socket.getaddrinfo that answers from a host->ip map."""
47
48 def fake_getaddrinfo(host, *_args, **_kwargs):
49 if host not in mapping:
50 raise socket.gaierror(f"no mapping for {host!r} in this test")
51 ip = mapping[host]
52 family = socket.AF_INET6 if ":" in ip else socket.AF_INET
53 sockaddr = (ip, 0, 0, 0) if family == socket.AF_INET6 else (ip, 0)
54 return [(family, socket.SOCK_STREAM, 6, "", sockaddr)]
55
56 return fake_getaddrinfo
57
58
59SIMPLE_HTML = (
60 b"<html><head><title>Hello World</title><style>.x{color:red}</style></head>"
61 b"<body>\n"
62 b"<nav>site nav, not content</nav>\n"
63 b"<h1>Big Heading</h1>\n"
64 b"<p>First paragraph with a <a href=\"/about\">link about us</a> and more text.</p>\n"
65 b"<p>Second paragraph. <a href=\"https://example.com/x\">External</a></p>\n"
66 b"<script>var hostile = 1;</script>\n"
67 b"<a href=\"/about\">duplicate href, different label</a>\n"
68 b"</body></html>"
69)
70
71
72@pytest.mark.parametrize(
73 "address",
74 [
75 "127.0.0.1", # loopback
76 "10.0.0.5", # private (RFC1918)
77 "169.254.169.254", # link-local -- cloud metadata
78 "::1", # loopback, IPv6
79 ],
80)
81def test_refuses_when_resolved_address_is_internal(monkeypatch, address):
82 monkeypatch.setattr(reader.socket, "getaddrinfo", addrinfo_for({"internal.example": address}))
83
84 with pytest.raises(reader.ReaderError, match="internal address"):
85 reader.read_page("http://internal.example/secret")
86
87
88def test_refuses_when_dns_resolution_fails(monkeypatch):
89 def raising_getaddrinfo(*_args, **_kwargs):
90 raise socket.gaierror("name or service not known")
91
92 monkeypatch.setattr(reader.socket, "getaddrinfo", raising_getaddrinfo)
93
94 with pytest.raises(reader.ReaderError, match="Could not resolve"):
95 reader.read_page("http://nowhere.invalid/")
96
97
98@pytest.mark.parametrize("url", ["file:///etc/passwd", "ftp://example.com/x", "data:text/plain,hi"])
99def test_refuses_non_http_schemes(url):
100 with pytest.raises(reader.ReaderError, match="non-http"):
101 reader.read_page(url)
102
103
104def test_redirect_to_a_private_address_is_refused_on_the_hop(monkeypatch):
105 """DNS-rebinding defense: the SSRF check reruns on every redirect hop,
106 not just the first URL -- a page can be public but redirect somewhere
107 internal."""
108 monkeypatch.setattr(
109 reader.socket,
110 "getaddrinfo",
111 addrinfo_for({"public.example": "93.184.216.34", "internal.example": "10.0.0.9"}),
112 )
113
114 def fake_open(_self, request, timeout=None):
115 if "public.example" in request.full_url:
116 import urllib.error
117
118 raise urllib.error.HTTPError(
119 request.full_url,
120 302,
121 "Found",
122 FakeHeaders({"Location": "http://internal.example/secret"}),
123 None,
124 )
125 raise AssertionError(f"should never reach {request.full_url}")
126
127 monkeypatch.setattr("urllib.request.OpenerDirector.open", fake_open)
128
129 with pytest.raises(reader.ReaderError, match="internal address"):
130 reader.read_page("http://public.example/start")
131
132
133def test_redirect_follows_to_a_public_host_and_re_resolves(monkeypatch):
134 monkeypatch.setattr(
135 reader.socket,
136 "getaddrinfo",
137 addrinfo_for({"a.example": "93.184.216.34", "b.example": "93.184.216.35"}),
138 )
139
140 def fake_open(_self, request, timeout=None):
141 if "a.example" in request.full_url:
142 import urllib.error
143
144 raise urllib.error.HTTPError(
145 request.full_url, 302, "Found", FakeHeaders({"Location": "https://b.example/next"}), None
146 )
147 return FakeResponse(SIMPLE_HTML, request.full_url, {"Content-Type": "text/html; charset=utf-8"})
148
149 monkeypatch.setattr("urllib.request.OpenerDirector.open", fake_open)
150
151 result = reader.read_page("https://a.example/start")
152
153 assert result["final_url"] == "https://b.example/next"
154 assert result["title"] == "Hello World"
155
156
157def test_too_many_redirects_are_refused(monkeypatch):
158 monkeypatch.setattr(reader.socket, "getaddrinfo", addrinfo_for({"loop.example": "93.184.216.34"}))
159
160 def fake_open(_self, request, timeout=None):
161 import urllib.error
162
163 raise urllib.error.HTTPError(
164 request.full_url, 302, "Found", FakeHeaders({"Location": "https://loop.example/again"}), None
165 )
166
167 monkeypatch.setattr("urllib.request.OpenerDirector.open", fake_open)
168
169 with pytest.raises(reader.ReaderError, match="Too many redirects"):
170 reader.read_page("https://loop.example/start")
171
172
173def test_refuses_non_html_content_type(monkeypatch):
174 monkeypatch.setattr(reader.socket, "getaddrinfo", addrinfo_for({"api.example": "93.184.216.34"}))
175
176 def fake_open(_self, request, timeout=None):
177 return FakeResponse(b'{"ok": true}', request.full_url, {"Content-Type": "application/json"})
178
179 monkeypatch.setattr("urllib.request.OpenerDirector.open", fake_open)
180
181 with pytest.raises(reader.ReaderError, match="non-HTML"):
182 reader.read_page("https://api.example/data.json")
183
184
185def test_enforces_timeout(monkeypatch):
186 monkeypatch.setattr(reader.socket, "getaddrinfo", addrinfo_for({"slow.example": "93.184.216.34"}))
187
188 def fake_open(_self, request, timeout=None):
189 raise TimeoutError("timed out")
190
191 monkeypatch.setattr("urllib.request.OpenerDirector.open", fake_open)
192
193 with pytest.raises(reader.ReaderError, match="timed out"):
194 reader.read_page("https://slow.example/", timeout=1)
195
196
197def test_enforces_max_bytes_and_sets_truncated(monkeypatch):
198 monkeypatch.setattr(reader.socket, "getaddrinfo", addrinfo_for({"big.example": "93.184.216.34"}))
199 big_html = b"<html><title>Big</title><body>" + b"word " * 100_000 + b"</body></html>"
200
201 def fake_open(_self, request, timeout=None):
202 return FakeResponse(big_html, request.full_url, {"Content-Type": "text/html"})
203
204 monkeypatch.setattr("urllib.request.OpenerDirector.open", fake_open)
205
206 result = reader.read_page("https://big.example/", max_bytes=1000)
207
208 assert result["truncated"] is True
209
210
211def test_happy_path_returns_stripped_text_and_numbered_deduped_links(monkeypatch):
212 monkeypatch.setattr(reader.socket, "getaddrinfo", addrinfo_for({"example.com": "93.184.216.34"}))
213
214 def fake_open(_self, request, timeout=None):
215 return FakeResponse(
216 SIMPLE_HTML, "https://example.com/page", {"Content-Type": "text/html; charset=utf-8"}
217 )
218
219 monkeypatch.setattr("urllib.request.OpenerDirector.open", fake_open)
220
221 result = reader.read_page("https://example.com/page")
222
223 assert result["title"] == "Hello World"
224 # script/style/nav content never appears in the body text.
225 assert "hostile" not in result["text"]
226 assert "site nav" not in result["text"]
227 assert "Big Heading" in result["text"]
228 assert "First paragraph with a link about us and more text." in result["text"]
229 # Links are numbered in document order, deduped by href, absolute.
230 assert result["links"] == [
231 {"n": 1, "label": "link about us", "href": "https://example.com/about"},
232 {"n": 2, "label": "External", "href": "https://example.com/x"},
233 ]
234 assert result["final_url"] == "https://example.com/page"
235 assert result["truncated"] is False