askrobots / dbbasic-browser
Public
A desktop browser with x402 built in: when a page asks for payment, it pays in stablecoins and shows the content.
danq9
site icons (favicons) in tabs, address bar suggestions and the bookmarks panel
7249579
15 hours, 50 minutes ago
History
transport.ts · 11.1 KB · 239 lines
| 1 | /** |
| 2 | * The browser's network transport: a fetch-shaped function built on Electron's |
| 3 | * `net.request`, whose ONE job beyond fetching is to NOT follow redirects itself. |
| 4 | * |
| 5 | * Why not `net.fetch`: inside `protocol.handle` every response we return is the final |
| 6 | * answer for the renderer's request. If we follow a 3xx internally, the renderer never |
| 7 | * learns it was redirected — the page shows the destination's content under the |
| 8 | * ORIGINAL url and origin. That breaks the address bar, relative links, same-origin |
| 9 | * checks and password-manager origin matching, and it silently dodges the per-request |
| 10 | * payment decision for the redirect target. `net.fetch` can't help: with |
| 11 | * `redirect: "manual"` it throws "Redirect was cancelled" and its Response carries no |
| 12 | * redirect information. |
| 13 | * |
| 14 | * `net.request` can: with `redirect: "manual"` it emits a `redirect` event (status, |
| 15 | * target, headers) and cancels the request unless `followRedirect()` is called. We never |
| 16 | * call it. Instead we return the 3xx as a normal Response (`Location` set to the absolute |
| 17 | * target), the renderer re-requests that URL, and the re-request re-enters our |
| 18 | * `protocol.handle` handler — so every hop gets its own payment decision, and the page's |
| 19 | * URL/origin are always the ones the user is actually looking at. |
| 20 | * |
| 21 | * Cookies: `useSessionCookies` + `credentials: "include"` make the request read and write |
| 22 | * the session's cookie jar exactly like a fetch would — including `Set-Cookie` on the 3xx |
| 23 | * itself, which bot checks such as Anubis depend on. |
| 24 | * |
| 25 | * Recursion: `net.request`, like `net.fetch`, IS routed through our own `protocol.handle` |
| 26 | * handler — without a bypass it would call us back forever. `net.fetch` avoids that with a |
| 27 | * `bypassCustomProtocolHandlers` option; `net.request` has no public equivalent, but |
| 28 | * Electron implements `net.fetch` ON TOP of `ClientRequest` and sets the same flag on the |
| 29 | * request's private `_urlLoaderOptions`. We do exactly what Electron does. That is a |
| 30 | * private field, so `assertBypassAvailable()` fails loudly at startup if a future Electron |
| 31 | * removes it, and the end-to-end transport tests cover it. |
| 32 | */ |
| 33 | |
| 34 | import { EventEmitter } from "node:events"; |
| 35 | import { Readable } from "node:stream"; |
| 36 | import { net, session, type Session } from "electron"; |
| 37 | import { chromiumRequestHeaders, type Hints, requestCredentials } from "./compat.js"; |
| 38 | import type { RawFetch } from "./intercept.js"; |
| 39 | |
| 40 | /** |
| 41 | * The engine's own client-hint values (from `navigator.userAgentData`), reported once by the |
| 42 | * main process after the chrome window loads. Requests wait briefly for it so the very first |
| 43 | * page load already carries hints that match what the page's JS will report; if it never |
| 44 | * arrives we proceed without hints rather than block browsing. |
| 45 | */ |
| 46 | let hintsValue: Hints | null = null; |
| 47 | let resolveHints: (h: Hints | null) => void = () => {}; |
| 48 | const hintsReady = new Promise<Hints | null>((resolve) => { |
| 49 | resolveHints = resolve; |
| 50 | setTimeout(() => resolve(null), 3000).unref?.(); |
| 51 | }); |
| 52 | export function setClientHints(h: Hints | null): void { |
| 53 | hintsValue = h; |
| 54 | resolveHints(h); |
| 55 | } |
| 56 | |
| 57 | /** |
| 58 | * Progress of every request through this transport, for the UI ("waiting for…", "receiving 1.2 MB…", |
| 59 | * "redirecting…"). Listen on the "event" channel. We can only report what is observable here — request |
| 60 | * out, response headers in, body bytes, redirect, error; DNS/TCP/TLS happen inside Chromium and are |
| 61 | * invisible, so they are folded into "waiting". |
| 62 | */ |
| 63 | export interface NetEvent { |
| 64 | id: number; |
| 65 | url: string; |
| 66 | type: "start" | "caller" | "headers" | "progress" | "end" | "redirect" | "error"; // "caller": parked, waiting for a caller-supplied network layer |
| 67 | status?: number; |
| 68 | /** Content-Length when the server sent one. */ |
| 69 | total?: number; |
| 70 | /** The response's Content-Type (lowercased, without parameters), e.g. "text/event-stream". */ |
| 71 | contentType?: string; |
| 72 | /** Body bytes received so far (progress/end). */ |
| 73 | bytes?: number; |
| 74 | location?: string; |
| 75 | error?: string; |
| 76 | /** For a failure the CALLER decided (or that waiting on the caller caused): the reason, and who said so. */ |
| 77 | failureKind?: string; |
| 78 | failureSource?: "observed" | "caller"; |
| 79 | } |
| 80 | export const netEvents = new EventEmitter(); |
| 81 | let netSeq = 0; |
| 82 | export const nextNetId = (): number => ++netSeq; |
| 83 | export const emitNet = (e: NetEvent): void => { netEvents.emit("event", e); }; |
| 84 | |
| 85 | /** Headers the network stack owns (or that would leak/duplicate state) — never copy these onto the request. */ |
| 86 | const REQUEST_HEADERS_WE_DONT_SET = new Set([ |
| 87 | "host", |
| 88 | "content-length", |
| 89 | "connection", |
| 90 | "keep-alive", |
| 91 | "transfer-encoding", |
| 92 | "upgrade", |
| 93 | "te", |
| 94 | "trailer", |
| 95 | "expect", |
| 96 | "proxy-connection", |
| 97 | "accept-encoding", // the network stack negotiates (and decodes) this itself |
| 98 | "cookie", // comes from the session's cookie jar via useSessionCookies |
| 99 | ]); |
| 100 | |
| 101 | /** Statuses that must not carry a body in a Response. */ |
| 102 | const NULL_BODY_STATUS = new Set([101, 204, 205, 304]); |
| 103 | |
| 104 | function toHeaders(src: Record<string, string | string[] | undefined>, drop: ReadonlySet<string> = new Set()): Headers { |
| 105 | const out = new Headers(); |
| 106 | for (const [k, v] of Object.entries(src)) { |
| 107 | if (v === undefined || drop.has(k.toLowerCase())) continue; |
| 108 | for (const one of Array.isArray(v) ? v : [v]) { |
| 109 | try { |
| 110 | out.append(k, one); |
| 111 | } catch { |
| 112 | /* a header name/value the Headers class rejects — drop it rather than fail the page */ |
| 113 | } |
| 114 | } |
| 115 | } |
| 116 | return out; |
| 117 | } |
| 118 | |
| 119 | /** The private option Electron's own net.fetch sets on a ClientRequest to skip protocol handlers. */ |
| 120 | type WithLoaderOptions = { _urlLoaderOptions?: { bypassCustomProtocolHandlers?: boolean } }; |
| 121 | |
| 122 | function bypassProtocolHandlers(req: unknown): void { |
| 123 | const opts = (req as WithLoaderOptions)._urlLoaderOptions; |
| 124 | if (!opts) { |
| 125 | // Fail loudly: silently continuing would recurse into our own handler and hang every page. |
| 126 | throw new Error("Electron changed ClientRequest internals (_urlLoaderOptions is gone); update transport.ts"); |
| 127 | } |
| 128 | opts.bypassCustomProtocolHandlers = true; |
| 129 | } |
| 130 | |
| 131 | export function netRequestFetch(ses: Session = session.defaultSession): RawFetch { |
| 132 | return async (input: Request): Promise<Response> => { |
| 133 | // Buffer the request body: simplest correct thing for ClientRequest, and bodies here are form posts/API calls. |
| 134 | const body = input.body ? Buffer.from(await input.arrayBuffer()) : null; |
| 135 | // What a real Chromium would send (client hints + fetch metadata) — net.request doesn't add them. |
| 136 | const headers = chromiumRequestHeaders(new URL(input.url), input.headers, hintsValue ?? (await hintsReady), { |
| 137 | referrer: input.referrer, |
| 138 | referrerPolicy: input.referrerPolicy, |
| 139 | }); |
| 140 | |
| 141 | const id = ++netSeq; |
| 142 | const url = input.url; |
| 143 | emitNet({ id, url, type: "start" }); |
| 144 | return new Promise<Response>((resolve, reject) => { |
| 145 | let settled = false; |
| 146 | const settle = (fn: () => void) => { |
| 147 | if (settled) return; // e.g. the "Redirect was cancelled" error that follows a handed-back 3xx |
| 148 | settled = true; |
| 149 | fn(); |
| 150 | }; |
| 151 | |
| 152 | // Cookies follow the page's own rule: a cross-site request that asked for none (<script crossorigin>, fetch with the default |
| 153 | // credentials) must not carry them, and a response with "Access-Control-Allow-Origin: *" is only acceptable to the network |
| 154 | // stack when the request carried none. Always asking for credentials made every such request fail (ERR_FAILED). |
| 155 | const credentials = requestCredentials(input, headers); |
| 156 | const req = net.request({ |
| 157 | method: input.method, |
| 158 | url: input.url, |
| 159 | session: ses, |
| 160 | redirect: "manual", |
| 161 | credentials, |
| 162 | useSessionCookies: credentials === "include", |
| 163 | }); |
| 164 | |
| 165 | bypassProtocolHandlers(req); |
| 166 | |
| 167 | headers.forEach((value, name) => { |
| 168 | if (REQUEST_HEADERS_WE_DONT_SET.has(name.toLowerCase())) return; |
| 169 | try { |
| 170 | req.setHeader(name, value); |
| 171 | } catch { |
| 172 | /* Chromium refuses a few restricted headers — the request still goes out without them */ |
| 173 | } |
| 174 | }); |
| 175 | |
| 176 | // A redirect: hand it to the renderer. We deliberately do NOT call req.followRedirect(); |
| 177 | // the request is cancelled, and the renderer re-requests `redirectUrl` through us. |
| 178 | req.on("redirect", (status, _method, redirectUrl, headers) => { |
| 179 | // The cookies are already stored by the network stack. The redirect's own body is never shown, so the headers that describe it go |
| 180 | // too: left in, a "text/plain" type with "nosniff" (unpkg sends exactly that on its redirects) made the browser refuse the script |
| 181 | // it was redirected to ("MIME type text/plain is not executable"). |
| 182 | const h = toHeaders(headers, new Set(["set-cookie", "content-type", "x-content-type-options", "content-length", "content-encoding", "content-disposition"])); |
| 183 | h.set("location", redirectUrl); |
| 184 | emitNet({ id, url, type: "redirect", status, location: redirectUrl }); |
| 185 | settle(() => resolve(new Response(null, { status, headers: h }))); |
| 186 | }); |
| 187 | |
| 188 | req.on("response", (incoming) => { |
| 189 | // Electron documents IncomingMessage as a Readable stream; its typings just don't say so. |
| 190 | const res = incoming as typeof incoming & Readable; |
| 191 | const status = res.statusCode; |
| 192 | const headers = toHeaders(res.headers); // passed through untouched, as Electron's own net.fetch does |
| 193 | const total = Number(res.headers["content-length"]); |
| 194 | const ct = String(res.headers["content-type"] ?? "").split(";")[0]!.trim().toLowerCase(); |
| 195 | emitNet({ id, url, type: "headers", status, ...(Number.isFinite(total) ? { total } : {}), ...(ct ? { contentType: ct } : {}) }); |
| 196 | let body: ReadableStream<Uint8Array> | null = null; |
| 197 | if (NULL_BODY_STATUS.has(status) || input.method === "HEAD") { |
| 198 | res.on("data", () => {}); |
| 199 | emitNet({ id, url, type: "end", bytes: 0 }); |
| 200 | } else { |
| 201 | // Count bytes in a pass-through stage (a 'data' listener on `res` would steal chunks from toWeb). |
| 202 | let bytes = 0; |
| 203 | let lastEmit = 0; |
| 204 | body = (Readable.toWeb(res) as ReadableStream<Uint8Array>).pipeThrough( |
| 205 | new TransformStream<Uint8Array, Uint8Array>({ |
| 206 | transform(chunk, controller) { |
| 207 | bytes += chunk.byteLength; |
| 208 | const now = Date.now(); |
| 209 | if (now - lastEmit > 120) { |
| 210 | lastEmit = now; |
| 211 | emitNet({ id, url, type: "progress", bytes }); |
| 212 | } |
| 213 | controller.enqueue(chunk); |
| 214 | }, |
| 215 | flush() { |
| 216 | emitNet({ id, url, type: "end", bytes }); |
| 217 | }, |
| 218 | }), |
| 219 | ); // handles backpressure and cancellation |
| 220 | } |
| 221 | settle(() => resolve(new Response(body, { status, statusText: res.statusMessage, headers }))); |
| 222 | }); |
| 223 | |
| 224 | req.on("error", (e) => { |
| 225 | if (!settled) emitNet({ id, url, type: "error", error: e.message }); |
| 226 | settle(() => reject(e)); |
| 227 | }); |
| 228 | |
| 229 | input.signal.addEventListener("abort", () => { |
| 230 | req.abort(); |
| 231 | if (!settled) emitNet({ id, url, type: "error", error: "aborted" }); |
| 232 | settle(() => reject(new DOMException("The operation was aborted.", "AbortError"))); |
| 233 | }); |
| 234 | |
| 235 | if (body) req.end(body); |
| 236 | else req.end(); |
| 237 | }); |
| 238 | }; |
| 239 | } |