askrobots / dbbasic-browser
Public
A desktop browser with x402 built in: when a page asks for payment, it pays in stablecoins and shows the content.
danq9
site icons (favicons) in tabs, address bar suggestions and the bookmarks panel
7249579
15 hours, 50 minutes ago
History
caller-network-client.mjs · 9.2 KB · 192 lines
| 1 | #!/usr/bin/env node |
| 2 | /** |
| 3 | * A small caller for the caller-supplied network layer (`net.provider`): it answers every request the browser would make |
| 4 | * by fetching it with a plain HTTP client, loads one address, and prints the verdict. One file, Node 18+, no dependencies. |
| 5 | * |
| 6 | * node caller-network-client.mjs --url https://example.com/ --user-agent "MyClient/1.0 (+https://me.example/about)" |
| 7 | * |
| 8 | * Options |
| 9 | * --url <address> what to load (required) |
| 10 | * --user-agent <identity> the identity this caller presents (required; the desktop browser's own is refused) |
| 11 | * --answer-timeout <ms> how long the browser waits for each answer before giving up on it (default 20000) |
| 12 | * --settle-timeout <ms> deadline for the page to finish loading (default 30000) |
| 13 | * --fetch-timeout <ms> how long this client waits on a server (default 15000) |
| 14 | * --max-bytes <n> largest response it will hand over (default 8388608); bigger ones are refused |
| 15 | * --min-gap-ms <ms> least time between two page loads from the same host (default 1000); page assets are not paced |
| 16 | * --allow-private also fetch loopback, local-network and link-local addresses (default: refused, since a page can ask for |
| 17 | * any address, including your own machine and your network's metadata service) |
| 18 | * --control-file <path> where the browser published its address and token (default $DBB_CONTROL_FILE or ~/.dbbasic/control.json; |
| 19 | * or set DBB_URL and DBB_TOKEN) |
| 20 | * --keep leave the caller network on when done (default: turn it off again) |
| 21 | * |
| 22 | * The browser must be running with its automation interface on (DBB_CONTROL=1). Start it with |
| 23 | * DBB_CALLER_NETWORK_USER_AGENT=<the same identity> and HOMEPAGE=about:blank to have it make no request of its own, not even |
| 24 | * for its first page. |
| 25 | * |
| 26 | * What it does not do: check what a site allows, or keep cookies. Those are the caller's decisions; the browser stores no state for |
| 27 | * you. A Set-Cookie in an answer is dropped unless the browser was told `net.provider {cookies: "visit"}`, in which case it keeps them |
| 28 | * for that one visit only. The private-address check is advisory: the name is resolved once to check it and again |
| 29 | * to connect, so a hostile name server could differ between the two; a firewall rule is the real guarantee. |
| 30 | */ |
| 31 | import { lookup } from "node:dns/promises"; |
| 32 | import { readFileSync } from "node:fs"; |
| 33 | import { isIP } from "node:net"; |
| 34 | import { homedir } from "node:os"; |
| 35 | import { join } from "node:path"; |
| 36 | import { parseArgs } from "node:util"; |
| 37 | |
| 38 | const { values: o } = parseArgs({ |
| 39 | options: { |
| 40 | url: { type: "string" }, |
| 41 | "user-agent": { type: "string" }, |
| 42 | "answer-timeout": { type: "string", default: "20000" }, |
| 43 | "settle-timeout": { type: "string", default: "30000" }, |
| 44 | "fetch-timeout": { type: "string", default: "15000" }, |
| 45 | "max-bytes": { type: "string", default: "8388608" }, |
| 46 | "min-gap-ms": { type: "string", default: "1000" }, |
| 47 | "allow-private": { type: "boolean", default: false }, |
| 48 | "control-file": { type: "string" }, |
| 49 | keep: { type: "boolean", default: false }, |
| 50 | }, |
| 51 | }); |
| 52 | if (!o.url || !o["user-agent"]) { |
| 53 | console.error('usage: caller-network-client.mjs --url <address> --user-agent "<identity>" [options] (see the top of this file)'); |
| 54 | process.exit(2); |
| 55 | } |
| 56 | |
| 57 | // ---- how to reach the browser ---- |
| 58 | let url = process.env.DBB_URL; |
| 59 | let token = process.env.DBB_TOKEN; |
| 60 | if (!url || !token) { |
| 61 | const file = o["control-file"] || process.env.DBB_CONTROL_FILE || join(homedir(), ".dbbasic", "control.json"); |
| 62 | try { |
| 63 | ({ url, token } = JSON.parse(readFileSync(file, "utf8"))); |
| 64 | } catch { |
| 65 | console.error(`cannot read ${file}: is the browser running with DBB_CONTROL=1?`); |
| 66 | process.exit(1); |
| 67 | } |
| 68 | } |
| 69 | let n = 0; |
| 70 | async function call(method, params = {}) { |
| 71 | // A call that gets no answer must not hang this script for ever: the browser answers every call, so 90 s means something is wrong. |
| 72 | const res = await fetch(url, { method: "POST", signal: AbortSignal.timeout(90_000), headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify({ id: ++n, method, params }) }); |
| 73 | const body = await res.json(); |
| 74 | if (!body.ok) throw Object.assign(new Error(`${body.error?.code}: ${body.error?.message}`), { code: body.error?.code }); |
| 75 | return body.result; |
| 76 | } |
| 77 | |
| 78 | // ---- answering requests ---- |
| 79 | const NOT_FORWARDED = new Set(["host", "content-length", "connection", "keep-alive", "transfer-encoding", "upgrade", "te", "trailer", "expect", "proxy-connection", "accept-encoding", "cookie"]); |
| 80 | |
| 81 | /** A reason from the shared list, from what went wrong while fetching. */ |
| 82 | function kindOf(err) { |
| 83 | const code = err?.cause?.code ?? err?.code ?? ""; |
| 84 | if (err?.code === "PRIVATE") return "blocked"; |
| 85 | if (err?.name === "TimeoutError" || /TIMEDOUT|TIMEOUT/.test(code)) return "timed-out"; |
| 86 | if (code === "ENOTFOUND" || code === "EAI_AGAIN") return "could-not-find-site"; |
| 87 | if (/^ECONN|EPIPE|UND_ERR_SOCKET|UND_ERR_CONNECT/.test(code)) return "could-not-connect"; |
| 88 | if (/CERT|SSL|TLS|DEPTH_ZERO|UNABLE_TO_VERIFY/.test(code)) return "certificate"; |
| 89 | return "other"; |
| 90 | } |
| 91 | |
| 92 | /** Loopback, private, link-local, CGNAT, unspecified and unique-local addresses. */ |
| 93 | function isPrivate(ip) { |
| 94 | const v4 = ip.startsWith("::ffff:") ? ip.slice(7) : ip; |
| 95 | if (isIP(v4) === 4) { |
| 96 | const [a, b] = v4.split(".").map(Number); |
| 97 | return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127); |
| 98 | } |
| 99 | const x = ip.toLowerCase(); |
| 100 | return x === "::1" || x === "::" || x.startsWith("fe8") || x.startsWith("fe9") || x.startsWith("fea") || x.startsWith("feb") || x.startsWith("fc") || x.startsWith("fd"); |
| 101 | } |
| 102 | async function assertPublic(host) { |
| 103 | if (o["allow-private"]) return; |
| 104 | const addrs = isIP(host) ? [{ address: host }] : await lookup(host, { all: true }); |
| 105 | if (addrs.some((a) => isPrivate(a.address))) throw Object.assign(new Error(`${host} is a private or local address (use --allow-private to fetch these)`), { code: "PRIVATE" }); |
| 106 | } |
| 107 | |
| 108 | // Page loads are spaced per host; a page's own files are not. |
| 109 | const lastLoad = new Map(); |
| 110 | async function pace(host) { |
| 111 | const gap = Number(o["min-gap-ms"]); |
| 112 | const wait = (lastLoad.get(host) ?? 0) + gap - Date.now(); |
| 113 | lastLoad.set(host, Math.max(Date.now(), (lastLoad.get(host) ?? 0) + gap)); |
| 114 | if (wait > 0) await new Promise((r) => setTimeout(r, wait)); |
| 115 | } |
| 116 | |
| 117 | /** Read a body, giving up past the size limit. */ |
| 118 | async function readCapped(res, limit) { |
| 119 | const chunks = []; |
| 120 | let n = 0; |
| 121 | for await (const c of res.body ?? []) { |
| 122 | n += c.byteLength; |
| 123 | if (n > limit) throw Object.assign(new Error(`response is larger than ${limit} bytes`), { code: "TOOBIG" }); |
| 124 | chunks.push(c); |
| 125 | } |
| 126 | return Buffer.concat(chunks); |
| 127 | } |
| 128 | |
| 129 | async function answer(r) { |
| 130 | try { |
| 131 | const target = new URL(r.url); |
| 132 | if (!/^https?:$/.test(target.protocol)) throw Object.assign(new Error("not an http(s) address"), { code: "UNSUPPORTED" }); |
| 133 | await assertPublic(target.hostname.replace(/^\[|\]$/g, "")); |
| 134 | if (r.navigate) await pace(target.host); |
| 135 | const headers = Object.fromEntries(Object.entries(r.headers).filter(([k]) => !NOT_FORWARDED.has(k.toLowerCase()))); |
| 136 | headers["user-agent"] = o["user-agent"]; |
| 137 | const res = await fetch(target, { |
| 138 | method: r.method, |
| 139 | headers, |
| 140 | redirect: "manual", // a redirect goes back to the page, which asks again, so every hop comes through here |
| 141 | signal: AbortSignal.timeout(Number(o["fetch-timeout"])), |
| 142 | ...(r.bodyBase64 ? { body: Buffer.from(r.bodyBase64, "base64") } : {}), |
| 143 | }); |
| 144 | const body = await readCapped(res, Number(o["max-bytes"])); |
| 145 | const outHeaders = {}; |
| 146 | res.headers.forEach((v, k) => (outHeaders[k] = v)); |
| 147 | await call("net.answer", { id: r.id, status: res.status, statusText: res.statusText, headers: outHeaders, bodyBase64: body.toString("base64") }); |
| 148 | log(`${res.status} ${r.method} ${r.url}`); |
| 149 | } catch (e) { |
| 150 | if (e.code === "no-request") return; // the browser gave up on it first |
| 151 | log(`refused ${r.url}: ${e.message}`); |
| 152 | await call("net.fail", { id: r.id, kind: kindOf(e), detail: String(e.message).slice(0, 200) }).catch(() => {}); |
| 153 | } |
| 154 | } |
| 155 | const log = (s) => console.error(`[caller] ${s}`); |
| 156 | |
| 157 | // ---- run ---- |
| 158 | let stop = false; |
| 159 | const state = await call("net.provider", { enabled: true, userAgent: o["user-agent"], answerTimeoutMs: Number(o["answer-timeout"]) }); |
| 160 | log(`caller network on (identity ${state.userAgent}, bound ${state.answerTimeoutMs} ms)`); |
| 161 | |
| 162 | const loop = (async () => { |
| 163 | let after = 0; |
| 164 | while (!stop) { |
| 165 | let requests = []; |
| 166 | try { |
| 167 | ({ requests } = await call("net.pending", { afterId: after, waitMs: 1000 })); |
| 168 | } catch (e) { |
| 169 | log(`waiting for the browser: ${e.message}`); |
| 170 | await new Promise((r) => setTimeout(r, 500)); |
| 171 | } |
| 172 | for (const r of requests) { |
| 173 | after = Math.max(after, r.id); |
| 174 | void answer(r); |
| 175 | } |
| 176 | } |
| 177 | })(); |
| 178 | |
| 179 | let code = 0; |
| 180 | try { |
| 181 | const loaded = await call("page.goto", { url: o.url, settle: { timeoutMs: Number(o["settle-timeout"]) } }); |
| 182 | console.log(JSON.stringify(loaded.settle, null, 2)); |
| 183 | if (!loaded.settle.settled || loaded.settle.outcome !== "ok") code = 3; |
| 184 | } catch (e) { |
| 185 | console.error(e.message); |
| 186 | code = 1; |
| 187 | } finally { |
| 188 | stop = true; |
| 189 | await loop; |
| 190 | if (!o.keep) await call("net.provider", { enabled: false }).catch(() => {}); |
| 191 | } |
| 192 | process.exit(code); |