A desktop browser with x402 built in: when a page asks for payment, it pays in stablecoins and shows the content.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-browser.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-browser.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-browser.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 site icons (favicons) in tabs, address bar suggestions and the bookmarks panel 7249579 15 hours, 50 minutes ago History
provider.ts · 22.3 KB · 483 lines
1/**
2 * A network layer the CALLER supplies.
3 *
4 * Normally the browser fetches every request itself (transport.ts). In this mode it makes none: each request a page
5 * would make is parked here and handed to the caller (through the automation API), which answers it with a response
6 * or refuses it with a reason. The browser then does with that answer exactly what it would do with a network one:
7 * a 3xx goes back to the page, which asks again (so every hop is the caller's decision), and a refusal ends the load
8 * with the reason the caller gave.
9 *
10 * - Nothing is fetched here, retried, or remembered. State (cookies, caches, pacing) is the caller's.
11 * - Waiting on the caller is not "a slow server": it has its own state, and ONE upper bound that the caller sets.
12 * Past the bound the request fails with its own reason ("caller-silent"), so a caller that never answers cannot
13 * hang a load for ever.
14 * - A main document the caller already fetched can be loaded as it is (`registerDocument`): it comes with its final
15 * address, status, headers and the redirects that led to it, so relative links resolve against the right address
16 * and the facts list the redirects even though this browser never saw them.
17 *
18 * No electron import: the pieces it needs from the browser are handed in, so it can be tested on its own.
19 */
20
21import type { FailureKind } from "./settle.js";
22import { FAILURE_KINDS } from "./settle.js";
23import type { NetEvent } from "./transport.js";
24
25export const DEFAULT_ANSWER_TIMEOUT_MS = 20_000;
26const MIN_ANSWER_TIMEOUT_MS = 1_000;
27const MAX_ANSWER_TIMEOUT_MS = 120_000;
28const MAX_BODY_BYTES = 32 * 1024 * 1024;
29const DOCUMENT_TTL_MS = 60_000;
30/** At most this many requests wait on the caller at once, and at most this many request-body bytes in total. */
31const MAX_PARKED = 64;
32const MAX_PARKED_BYTES = 64 * 1024 * 1024;
33/** `pending` returns at most this many at a time. */
34const PENDING_BATCH = 50;
35
36/** A request waiting for the caller. */
37export interface ParkedRequest {
38 id: number;
39 url: string;
40 method: string;
41 headers: Record<string, string>;
42 /** Base64 request body, for methods that have one. */
43 bodyBase64?: string;
44 /** The address of the page this request follows (its referrer), "" for a first visit. Lets the caller treat a move the page made itself as the same visit. */
45 follows: string;
46 /** What it is for ("document", "image", "script"…), when the browser knows. */
47 destination: string;
48 /** True for a top-level navigation. */
49 navigate: boolean;
50 waitingMs: number;
51}
52
53export interface Answer {
54 status: number;
55 statusText?: string;
56 headers?: Record<string, string | string[]>;
57 bodyBase64?: string;
58 bodyText?: string;
59}
60
61/** A main document the caller already has. */
62export interface Document extends Answer {
63 /** The address the document finally came from: relative links resolve against it. */
64 url: string;
65 /** The redirects that led to it, in order. */
66 redirects?: Array<{ status?: number; to: string }>;
67}
68
69export class ProviderError extends Error {
70 code: string;
71 constructor(code: string, message: string) {
72 super(message);
73 this.code = code;
74 }
75}
76
77/** Headers that describe how a body was TRANSPORTED. The body handed over is the body to show, so they never apply. */
78const DROPPED_HEADERS = new Set(["content-length", "content-encoding", "transfer-encoding", "connection", "keep-alive", "set-cookie", "set-cookie2"]);
79const NULL_BODY_STATUS = new Set([101, 204, 205, 304]);
80
81export const normUrl = (u: string): string => {
82 try {
83 return new URL(u).href;
84 } catch {
85 return u;
86 }
87};
88
89/** Turn a caller's answer into a Response, validating everything the caller sent. */
90export function buildResponse(requestUrl: string, a: Answer, method = "GET"): { response: Response; bytes: number; setCookies: string[] } {
91 if (!a || typeof a !== "object") throw new ProviderError("bad-params", "an answer is an object with a status");
92 const status = a.status;
93 if (!Number.isInteger(status) || status < 200 || status > 599) throw new ProviderError("bad-params", "status must be a whole number from 200 to 599");
94 if (a.bodyBase64 !== undefined && a.bodyText !== undefined) throw new ProviderError("bad-params", "give bodyBase64 or bodyText, not both");
95 let body: Buffer | null = null;
96 if (a.bodyBase64 !== undefined) {
97 if (typeof a.bodyBase64 !== "string" || !/^[A-Za-z0-9+/_-]*={0,2}$/.test(a.bodyBase64)) throw new ProviderError("bad-params", "bodyBase64 is not valid base64");
98 body = Buffer.from(a.bodyBase64, "base64");
99 } else if (a.bodyText !== undefined) {
100 if (typeof a.bodyText !== "string") throw new ProviderError("bad-params", "bodyText must be a string");
101 body = Buffer.from(a.bodyText, "utf8");
102 }
103 if (body && body.byteLength > MAX_BODY_BYTES) throw new ProviderError("too-large", `a body can be at most ${MAX_BODY_BYTES} bytes`);
104
105 const headers = new Headers();
106 const setCookies: string[] = [];
107 if (a.headers !== undefined) {
108 if (!a.headers || typeof a.headers !== "object" || Array.isArray(a.headers)) throw new ProviderError("bad-params", "headers must be an object of name → value");
109 for (const [k, v] of Object.entries(a.headers)) {
110 if (k.toLowerCase() === "set-cookie") {
111 // Never passed to the page; kept apart so a caller-chosen per-visit cookie jar can use them (see CookieJar).
112 for (const one of Array.isArray(v) ? v : [v]) if (typeof one === "string") setCookies.push(one);
113 continue;
114 }
115 if (DROPPED_HEADERS.has(k.toLowerCase())) continue;
116 for (const one of Array.isArray(v) ? v : [v]) {
117 if (typeof one !== "string") throw new ProviderError("bad-params", `header ${k} must be text`);
118 try {
119 headers.append(k, one);
120 } catch {
121 throw new ProviderError("bad-params", `header ${k} is not a valid header`);
122 }
123 }
124 }
125 }
126 if (status >= 300 && status < 400 && headers.has("location")) {
127 // The page resolves a relative Location itself, but the facts and the events want it absolute.
128 let target: URL;
129 try {
130 target = new URL(headers.get("location")!, requestUrl);
131 } catch {
132 throw new ProviderError("bad-params", "location is not a valid address");
133 }
134 if (!/^https?:$/.test(target.protocol)) throw new ProviderError("bad-params", "a redirect may only go to an http(s) address");
135 headers.set("location", target.href);
136 }
137 const nullBody = NULL_BODY_STATUS.has(status) || method === "HEAD" || (status >= 300 && status < 400 && !body);
138 return {
139 response: new Response(nullBody ? null : (body as unknown as BodyInit), { status, statusText: a.statusText ?? "", headers }),
140 bytes: nullBody ? 0 : (body?.byteLength ?? 0),
141 setCookies,
142 };
143}
144
145/**
146 * What a request is for. The request object a protocol handler gets carries no mode or destination, so this reads the
147 * two headers Chromium itself sets: only a navigation sends Upgrade-Insecure-Requests, and Accept says what a
148 * subresource is. Best effort; "" means the browser did not say.
149 */
150export function kindOf(h: Record<string, string>): { navigate: boolean; destination: string } {
151 if (h["upgrade-insecure-requests"]) return { navigate: true, destination: "document" };
152 const a = h["accept"] ?? "";
153 if (/^image\//.test(a)) return { navigate: false, destination: "image" };
154 if (/^text\/css/.test(a)) return { navigate: false, destination: "style" };
155 return { navigate: false, destination: "" };
156}
157
158/** A parsed Set-Cookie line, in the shape Electron's cookie store takes. */
159export interface ParsedCookie {
160 name: string;
161 value: string;
162 domain?: string;
163 path: string;
164 secure: boolean;
165 httpOnly: boolean;
166 sameSite: "unspecified" | "no_restriction" | "lax" | "strict";
167 /** Seconds since the epoch; absent = a session cookie. */
168 expirationDate?: number;
169}
170
171/** Cookie limits for one visit: how many, and how large. */
172export const MAX_COOKIES_PER_VISIT = 50;
173export const MAX_COOKIE_BYTES = 4096;
174
175/**
176 * Parse one Set-Cookie line (RFC 6265 §5.2, the parts that matter). Returns null for anything unusable. This only reads the line:
177 * whether the cookie may be set (prefixes such as __Host-, Secure on plain http, public-suffix domains) is decided by Chromium's own
178 * cookie store when it is stored, not here.
179 */
180export function parseSetCookie(line: string, requestUrl: string, nowSec = Date.now() / 1000): ParsedCookie | null {
181 if (typeof line !== "string" || line.length > MAX_COOKIE_BYTES * 2) return null;
182 const parts = line.split(";");
183 const first = parts.shift() ?? "";
184 const eq = first.indexOf("=");
185 if (eq <= 0) return null;
186 const name = first.slice(0, eq).trim();
187 const value = first.slice(eq + 1).trim();
188 if (!name || /[\x00-\x1f\x7f\s=;,]/.test(name) || /[\x00-\x1f\x7f;]/.test(value) || name.length + value.length > MAX_COOKIE_BYTES) return null;
189 let path = "";
190 const c: ParsedCookie = { name, value, path: "", secure: false, httpOnly: false, sameSite: "unspecified" };
191 let maxAge: number | null = null;
192 let expires: number | null = null;
193 for (const p of parts) {
194 const i = p.indexOf("=");
195 const k = (i < 0 ? p : p.slice(0, i)).trim().toLowerCase();
196 const v = i < 0 ? "" : p.slice(i + 1).trim();
197 if (k === "domain" && v) c.domain = v.replace(/^\./, "").toLowerCase().length ? (v.startsWith(".") ? v : "." + v).toLowerCase() : undefined;
198 else if (k === "path" && v.startsWith("/")) path = v;
199 else if (k === "secure") c.secure = true;
200 else if (k === "httponly") c.httpOnly = true;
201 else if (k === "samesite") c.sameSite = v.toLowerCase() === "strict" ? "strict" : v.toLowerCase() === "lax" ? "lax" : v.toLowerCase() === "none" ? "no_restriction" : "unspecified";
202 else if (k === "max-age" && /^-?\d+$/.test(v)) maxAge = Number(v);
203 else if (k === "expires") {
204 const t = Date.parse(v);
205 if (Number.isFinite(t)) expires = t / 1000;
206 }
207 }
208 if (!path) {
209 // RFC 6265 §5.1.4: the directory of the request path.
210 let p = "/";
211 try {
212 p = new URL(requestUrl).pathname;
213 } catch {
214 /* keep "/" */
215 }
216 path = p.length > 1 && p.lastIndexOf("/") > 0 ? p.slice(0, p.lastIndexOf("/")) : "/";
217 }
218 c.path = path;
219 if (maxAge !== null) c.expirationDate = nowSec + maxAge; // Max-Age wins over Expires
220 else if (expires !== null) c.expirationDate = expires;
221 return c;
222}
223
224/**
225 * Is a request first-party for cookie purposes? A navigation is. Anything else is only when the page it follows is on the same site
226 * (the same host, or the same last two name labels). Third-party requests get no cookies, and none are stored from them.
227 */
228export function firstParty(url: string, follows: string, navigate: boolean): boolean {
229 if (navigate) return true;
230 try {
231 const a = new URL(url).hostname;
232 const b = new URL(follows).hostname;
233 const site = (h: string) => h.split(".").slice(-2).join(".");
234 return a === b || site(a) === site(b);
235 } catch {
236 return false;
237 }
238}
239
240/**
241 * The per-visit cookie jar a caller can switch on. The browser's own cookie store holds the cookies (so Domain, Path, Secure, SameSite
242 * and prefixes are Chromium's rules, and a page's own document.cookie sees the same jar), and the visit's clean slate empties it.
243 */
244export interface CookieJar {
245 on(): boolean;
246 /** The Cookie header for a request, or "". */
247 header(url: string): Promise<string>;
248 /** Store a cookie from a response. Failures are the store's refusals and are ignored. */
249 store(url: string, cookie: ParsedCookie): Promise<void>;
250}
251
252export function clampAnswerTimeout(ms: unknown): number {
253 const n = Number(ms);
254 if (!Number.isFinite(n)) return DEFAULT_ANSWER_TIMEOUT_MS;
255 return Math.min(MAX_ANSWER_TIMEOUT_MS, Math.max(MIN_ANSWER_TIMEOUT_MS, Math.round(n)));
256}
257
258interface Entry {
259 req: ParkedRequest;
260 since: number;
261 timer: NodeJS.Timeout;
262 bytes: number;
263 resolve: (r: Response) => void;
264 reject: (e: unknown) => void;
265}
266
267export class CallerNetwork {
268 enabled = false;
269 userAgent = "";
270 answerTimeoutMs = DEFAULT_ANSWER_TIMEOUT_MS;
271 private parked = new Map<number, Entry>();
272 private waiters = new Set<() => void>();
273 private documents = new Map<string, { doc: Document; expires: number }>();
274 /** Set by the browser when the per-visit cookie jar is wanted; null = cookies are neither stored nor sent (the default). */
275 jar: CookieJar | null = null;
276 /** Per-visit limits; 0 = no limit. Past one, a request fails at once with a reason (kind "other", source "observed"). */
277 limits = { requests: 0, bytes: 0 };
278 private visitCookies = 0;
279 private visitRequests = 0;
280 private visitBytes = 0;
281 private emit: (e: NetEvent) => void;
282 private nextId: () => number;
283
284 constructor(emit: (e: NetEvent) => void, nextId: () => number) {
285 this.emit = emit;
286 this.nextId = nextId;
287 }
288
289 /** A new visit begins: the cookie allowance starts over (the cookies themselves are cleared by the browser). */
290 newVisit(): void {
291 this.visitCookies = 0;
292 this.visitRequests = 0;
293 this.visitBytes = 0;
294 }
295
296 /** Is a document the caller already has waiting for this address? */
297 hasDocumentFor(url: string): boolean {
298 const hit = this.documents.get(normUrl(url));
299 return !!hit && hit.expires >= Date.now();
300 }
301
302 /** Is a request for this address waiting on the caller right now? */
303 isParked(url: string): boolean {
304 const u = normUrl(url);
305 for (const e of this.parked.values()) if (normUrl(e.req.url) === u) return true;
306 return false;
307 }
308
309 parkedCount(): number {
310 return this.parked.size;
311 }
312
313 /** What the browser calls instead of fetching. */
314 async fetch(input: Request): Promise<Response> {
315 // A document the caller already has is served as it is, once.
316 const doc = this.takeDocument(input.url, input.method);
317 if (doc) return this.serveDocument(input.url, doc);
318
319 // Per-visit totals. A response already under way is never cut off; the limit stops the NEXT request.
320 if (this.limits.requests > 0 && this.visitRequests >= this.limits.requests) throw this.refuse(input.url, "other", `visit limit reached: ${this.limits.requests} requests`, "observed");
321 if (this.limits.bytes > 0 && this.visitBytes >= this.limits.bytes) throw this.refuse(input.url, "other", `visit limit reached: ${this.limits.bytes} bytes`, "observed");
322 this.visitRequests++;
323 // A page can ask for far more than a caller will ever answer. Past the limits the request fails at once, with a reason.
324 if (this.parked.size >= MAX_PARKED) throw this.refuse(input.url, "other", `too many requests are waiting on the caller (limit ${MAX_PARKED})`, "observed");
325 const hasBody = !!input.body && input.method !== "GET" && input.method !== "HEAD";
326 const bodyBuf = hasBody ? Buffer.from(await input.arrayBuffer()) : null;
327 const heldBytes = [...this.parked.values()].reduce((n, e) => n + e.bytes, 0);
328 if (bodyBuf && heldBytes + bodyBuf.byteLength > MAX_PARKED_BYTES) throw this.refuse(input.url, "other", "too much request data is waiting on the caller", "observed");
329 if (bodyBuf && bodyBuf.byteLength > MAX_BODY_BYTES) throw this.refuse(input.url, "other", "request body too large to hand over", "observed");
330
331 const headers: Record<string, string> = {};
332 input.headers.forEach((v, k) => (headers[k] = v));
333 headers["user-agent"] = this.userAgent; // the caller's identity, never the desktop browser's
334 const referrer = input.referrer && input.referrer !== "about:client" ? input.referrer : (input.headers.get("referer") ?? "");
335 const kind = kindOf(headers);
336 // The browser's Cookie header is the only one: whatever the page or Chromium put there is replaced, and without the jar there is none.
337 delete headers["cookie"];
338 if (this.jar?.on() && firstParty(input.url, referrer, kind.navigate)) {
339 const cookie = await this.jar.header(input.url);
340 if (cookie) headers["cookie"] = cookie;
341 }
342
343 const id = this.nextId();
344 this.emit({ id, url: input.url, type: "start" });
345 return new Promise<Response>((resolve, reject) => {
346 const req: ParkedRequest = {
347 id,
348 url: input.url,
349 method: input.method,
350 headers,
351 ...(bodyBuf ? { bodyBase64: bodyBuf.toString("base64") } : {}),
352 follows: referrer,
353 ...kind,
354 waitingMs: 0,
355 };
356 const timer = setTimeout(() => this.failEntry(id, "caller-silent", `the caller did not answer within ${this.answerTimeoutMs} ms`, "observed"), this.answerTimeoutMs);
357 timer.unref?.();
358 this.parked.set(id, { req, since: Date.now(), timer, bytes: bodyBuf?.byteLength ?? 0, resolve, reject });
359 this.emit({ id, url: input.url, type: "caller" });
360 input.signal.addEventListener("abort", () => {
361 const e = this.parked.get(id);
362 if (!e) return;
363 clearTimeout(e.timer);
364 this.parked.delete(id);
365 this.emit({ id, url: input.url, type: "error", error: "aborted" });
366 e.reject(new DOMException("The operation was aborted.", "AbortError"));
367 });
368 for (const w of [...this.waiters]) w();
369 });
370 }
371
372 /** Requests waiting for the caller, those after `afterId`. Waits up to `waitMs` for one to arrive. */
373 async pending(afterId: number, waitMs: number): Promise<ParkedRequest[]> {
374 const snapshot = () => [...this.parked.values()].filter((e) => e.req.id > afterId).sort((a, b) => a.req.id - b.req.id).slice(0, PENDING_BATCH).map((e) => ({ ...e.req, waitingMs: Date.now() - e.since }));
375 let out = snapshot();
376 if (out.length || waitMs <= 0) return out;
377 await new Promise<void>((resolve) => {
378 const done = () => {
379 clearTimeout(t);
380 this.waiters.delete(done);
381 resolve();
382 };
383 const t = setTimeout(done, Math.min(waitMs, 30_000));
384 this.waiters.add(done);
385 });
386 out = snapshot();
387 return out;
388 }
389
390 answer(id: number, a: Answer): void {
391 const e = this.parked.get(id);
392 if (!e) throw new ProviderError("no-request", `no request ${id} is waiting (answered already, timed out, or aborted)`);
393 const { response, bytes, setCookies } = buildResponse(e.req.url, a, e.req.method); // throws before anything changes if the answer is bad
394 clearTimeout(e.timer);
395 this.parked.delete(id);
396 this.emitAnswer(id, e.req.url, response, bytes);
397 this.visitBytes += bytes;
398 // The cookies are stored BEFORE the page sees the response, so the request that follows it (a redirect back) carries them.
399 const jar = this.jar;
400 if (jar?.on() && setCookies.length && firstParty(e.req.url, e.req.follows, e.req.navigate)) {
401 const stores: Promise<void>[] = [];
402 for (const line of setCookies) {
403 if (this.visitCookies >= MAX_COOKIES_PER_VISIT) break;
404 const parsed = parseSetCookie(line, e.req.url);
405 if (!parsed) continue;
406 this.visitCookies++;
407 stores.push(jar.store(e.req.url, parsed).catch(() => {}));
408 }
409 void Promise.all(stores).then(() => e.resolve(response));
410 } else e.resolve(response);
411 }
412
413 /** Refuse a request with a reason from the shared list. */
414 fail(id: number, kind: unknown, detail?: unknown): void {
415 if (typeof kind !== "string" || !(FAILURE_KINDS as readonly string[]).includes(kind) || kind === "caller-silent") {
416 throw new ProviderError("bad-params", `kind must be one of: ${FAILURE_KINDS.filter((k) => k !== "caller-silent").join(", ")}`);
417 }
418 if (!this.parked.has(id)) throw new ProviderError("no-request", `no request ${id} is waiting (answered already, timed out, or aborted)`);
419 this.failEntry(id, kind as FailureKind, typeof detail === "string" ? detail.slice(0, 300) : "refused by the caller", "caller");
420 }
421
422 /** Refuse every waiting request (used when the mode is turned off). */
423 failAll(kind: FailureKind, detail: string): void {
424 for (const id of [...this.parked.keys()]) this.failEntry(id, kind, detail, "observed");
425 }
426
427 /** Load a main document the caller already has. It is served once, when the browser asks for that address. */
428 registerDocument(doc: Document): void {
429 const answer = buildResponse(doc.url, doc); // validates it now, so a bad document is an error to the caller, not a broken page
430 void answer;
431 if (!/^https?:\/\//i.test(doc.url)) throw new ProviderError("bad-params", "url must be an http(s) address");
432 const redirects = doc.redirects ?? [];
433 if (!Array.isArray(redirects) || redirects.some((r) => !r || typeof r.to !== "string" || (r.status !== undefined && !Number.isInteger(r.status)))) throw new ProviderError("bad-params", "redirects must be a list of {status, to}");
434 this.documents.set(normUrl(doc.url), { doc: { ...doc, redirects }, expires: Date.now() + DOCUMENT_TTL_MS });
435 }
436
437 private takeDocument(url: string, method: string): Document | null {
438 const now = Date.now();
439 for (const [k, v] of this.documents) if (v.expires < now) this.documents.delete(k);
440 if (method !== "GET") return null;
441 const key = normUrl(url);
442 const hit = this.documents.get(key);
443 if (!hit) return null;
444 this.documents.delete(key);
445 return hit.doc;
446 }
447
448 private serveDocument(url: string, doc: Document): Response {
449 const id = this.nextId();
450 this.emit({ id, url, type: "start" });
451 const { response, bytes } = buildResponse(url, doc);
452 this.emitAnswer(id, url, response, bytes);
453 return response;
454 }
455
456 private emitAnswer(id: number, url: string, response: Response, bytes: number): void {
457 const ct = (response.headers.get("content-type") ?? "").split(";")[0]!.trim().toLowerCase();
458 const loc = response.headers.get("location");
459 if (response.status >= 300 && response.status < 400 && loc) this.emit({ id, url, type: "redirect", status: response.status, location: loc });
460 else {
461 this.emit({ id, url, type: "headers", status: response.status, total: bytes, ...(ct ? { contentType: ct } : {}) });
462 this.emit({ id, url, type: "end", bytes });
463 }
464 }
465
466 private failEntry(id: number, kind: FailureKind, detail: string, source: "observed" | "caller"): void {
467 const e = this.parked.get(id);
468 if (!e) return;
469 clearTimeout(e.timer);
470 this.parked.delete(id);
471 this.emit({ id, url: e.req.url, type: "error", error: `${detail}`, failureKind: kind, failureSource: source });
472 e.reject(new Error(`caller:${kind}: ${detail}`));
473 }
474
475 /** A failure raised before anything was parked. */
476 private refuse(url: string, kind: FailureKind, detail: string, source: "observed" | "caller"): Error {
477 // A request refused before it was parked still happened: it starts, then fails (so it is counted as requested and failed).
478 const id = this.nextId();
479 this.emit({ id, url, type: "start" });
480 this.emit({ id, url, type: "error", error: detail, failureKind: kind, failureSource: source });
481 return new Error(`caller:${kind}: ${detail}`);
482 }
483}