A desktop browser with x402 built in: when a page asks for payment, it pays in stablecoins and shows the content.

0 downloads

dbbasic-browser/apps/browser/examples/caller-network-client.mjs
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-browser.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-browser.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-browser.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 site icons (favicons) in tabs, address bar suggestions and the bookmarks panel 7249579 15 hours, 50 minutes ago History
caller-network-client.mjs · 9.2 KB · 192 lines
1#!/usr/bin/env node
2/**
3 * A small caller for the caller-supplied network layer (`net.provider`): it answers every request the browser would make
4 * by fetching it with a plain HTTP client, loads one address, and prints the verdict. One file, Node 18+, no dependencies.
5 *
6 * node caller-network-client.mjs --url https://example.com/ --user-agent "MyClient/1.0 (+https://me.example/about)"
7 *
8 * Options
9 * --url <address> what to load (required)
10 * --user-agent <identity> the identity this caller presents (required; the desktop browser's own is refused)
11 * --answer-timeout <ms> how long the browser waits for each answer before giving up on it (default 20000)
12 * --settle-timeout <ms> deadline for the page to finish loading (default 30000)
13 * --fetch-timeout <ms> how long this client waits on a server (default 15000)
14 * --max-bytes <n> largest response it will hand over (default 8388608); bigger ones are refused
15 * --min-gap-ms <ms> least time between two page loads from the same host (default 1000); page assets are not paced
16 * --allow-private also fetch loopback, local-network and link-local addresses (default: refused, since a page can ask for
17 * any address, including your own machine and your network's metadata service)
18 * --control-file <path> where the browser published its address and token (default $DBB_CONTROL_FILE or ~/.dbbasic/control.json;
19 * or set DBB_URL and DBB_TOKEN)
20 * --keep leave the caller network on when done (default: turn it off again)
21 *
22 * The browser must be running with its automation interface on (DBB_CONTROL=1). Start it with
23 * DBB_CALLER_NETWORK_USER_AGENT=<the same identity> and HOMEPAGE=about:blank to have it make no request of its own, not even
24 * for its first page.
25 *
26 * What it does not do: check what a site allows, or keep cookies. Those are the caller's decisions; the browser stores no state for
27 * you. A Set-Cookie in an answer is dropped unless the browser was told `net.provider {cookies: "visit"}`, in which case it keeps them
28 * for that one visit only. The private-address check is advisory: the name is resolved once to check it and again
29 * to connect, so a hostile name server could differ between the two; a firewall rule is the real guarantee.
30 */
31import { lookup } from "node:dns/promises";
32import { readFileSync } from "node:fs";
33import { isIP } from "node:net";
34import { homedir } from "node:os";
35import { join } from "node:path";
36import { parseArgs } from "node:util";
37
38const { values: o } = parseArgs({
39 options: {
40 url: { type: "string" },
41 "user-agent": { type: "string" },
42 "answer-timeout": { type: "string", default: "20000" },
43 "settle-timeout": { type: "string", default: "30000" },
44 "fetch-timeout": { type: "string", default: "15000" },
45 "max-bytes": { type: "string", default: "8388608" },
46 "min-gap-ms": { type: "string", default: "1000" },
47 "allow-private": { type: "boolean", default: false },
48 "control-file": { type: "string" },
49 keep: { type: "boolean", default: false },
50 },
51});
52if (!o.url || !o["user-agent"]) {
53 console.error('usage: caller-network-client.mjs --url <address> --user-agent "<identity>" [options] (see the top of this file)');
54 process.exit(2);
55}
56
57// ---- how to reach the browser ----
58let url = process.env.DBB_URL;
59let token = process.env.DBB_TOKEN;
60if (!url || !token) {
61 const file = o["control-file"] || process.env.DBB_CONTROL_FILE || join(homedir(), ".dbbasic", "control.json");
62 try {
63 ({ url, token } = JSON.parse(readFileSync(file, "utf8")));
64 } catch {
65 console.error(`cannot read ${file}: is the browser running with DBB_CONTROL=1?`);
66 process.exit(1);
67 }
68}
69let n = 0;
70async function call(method, params = {}) {
71 // A call that gets no answer must not hang this script for ever: the browser answers every call, so 90 s means something is wrong.
72 const res = await fetch(url, { method: "POST", signal: AbortSignal.timeout(90_000), headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify({ id: ++n, method, params }) });
73 const body = await res.json();
74 if (!body.ok) throw Object.assign(new Error(`${body.error?.code}: ${body.error?.message}`), { code: body.error?.code });
75 return body.result;
76}
77
78// ---- answering requests ----
79const NOT_FORWARDED = new Set(["host", "content-length", "connection", "keep-alive", "transfer-encoding", "upgrade", "te", "trailer", "expect", "proxy-connection", "accept-encoding", "cookie"]);
80
81/** A reason from the shared list, from what went wrong while fetching. */
82function kindOf(err) {
83 const code = err?.cause?.code ?? err?.code ?? "";
84 if (err?.code === "PRIVATE") return "blocked";
85 if (err?.name === "TimeoutError" || /TIMEDOUT|TIMEOUT/.test(code)) return "timed-out";
86 if (code === "ENOTFOUND" || code === "EAI_AGAIN") return "could-not-find-site";
87 if (/^ECONN|EPIPE|UND_ERR_SOCKET|UND_ERR_CONNECT/.test(code)) return "could-not-connect";
88 if (/CERT|SSL|TLS|DEPTH_ZERO|UNABLE_TO_VERIFY/.test(code)) return "certificate";
89 return "other";
90}
91
92/** Loopback, private, link-local, CGNAT, unspecified and unique-local addresses. */
93function isPrivate(ip) {
94 const v4 = ip.startsWith("::ffff:") ? ip.slice(7) : ip;
95 if (isIP(v4) === 4) {
96 const [a, b] = v4.split(".").map(Number);
97 return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127);
98 }
99 const x = ip.toLowerCase();
100 return x === "::1" || x === "::" || x.startsWith("fe8") || x.startsWith("fe9") || x.startsWith("fea") || x.startsWith("feb") || x.startsWith("fc") || x.startsWith("fd");
101}
102async function assertPublic(host) {
103 if (o["allow-private"]) return;
104 const addrs = isIP(host) ? [{ address: host }] : await lookup(host, { all: true });
105 if (addrs.some((a) => isPrivate(a.address))) throw Object.assign(new Error(`${host} is a private or local address (use --allow-private to fetch these)`), { code: "PRIVATE" });
106}
107
108// Page loads are spaced per host; a page's own files are not.
109const lastLoad = new Map();
110async function pace(host) {
111 const gap = Number(o["min-gap-ms"]);
112 const wait = (lastLoad.get(host) ?? 0) + gap - Date.now();
113 lastLoad.set(host, Math.max(Date.now(), (lastLoad.get(host) ?? 0) + gap));
114 if (wait > 0) await new Promise((r) => setTimeout(r, wait));
115}
116
117/** Read a body, giving up past the size limit. */
118async function readCapped(res, limit) {
119 const chunks = [];
120 let n = 0;
121 for await (const c of res.body ?? []) {
122 n += c.byteLength;
123 if (n > limit) throw Object.assign(new Error(`response is larger than ${limit} bytes`), { code: "TOOBIG" });
124 chunks.push(c);
125 }
126 return Buffer.concat(chunks);
127}
128
129async function answer(r) {
130 try {
131 const target = new URL(r.url);
132 if (!/^https?:$/.test(target.protocol)) throw Object.assign(new Error("not an http(s) address"), { code: "UNSUPPORTED" });
133 await assertPublic(target.hostname.replace(/^\[|\]$/g, ""));
134 if (r.navigate) await pace(target.host);
135 const headers = Object.fromEntries(Object.entries(r.headers).filter(([k]) => !NOT_FORWARDED.has(k.toLowerCase())));
136 headers["user-agent"] = o["user-agent"];
137 const res = await fetch(target, {
138 method: r.method,
139 headers,
140 redirect: "manual", // a redirect goes back to the page, which asks again, so every hop comes through here
141 signal: AbortSignal.timeout(Number(o["fetch-timeout"])),
142 ...(r.bodyBase64 ? { body: Buffer.from(r.bodyBase64, "base64") } : {}),
143 });
144 const body = await readCapped(res, Number(o["max-bytes"]));
145 const outHeaders = {};
146 res.headers.forEach((v, k) => (outHeaders[k] = v));
147 await call("net.answer", { id: r.id, status: res.status, statusText: res.statusText, headers: outHeaders, bodyBase64: body.toString("base64") });
148 log(`${res.status} ${r.method} ${r.url}`);
149 } catch (e) {
150 if (e.code === "no-request") return; // the browser gave up on it first
151 log(`refused ${r.url}: ${e.message}`);
152 await call("net.fail", { id: r.id, kind: kindOf(e), detail: String(e.message).slice(0, 200) }).catch(() => {});
153 }
154}
155const log = (s) => console.error(`[caller] ${s}`);
156
157// ---- run ----
158let stop = false;
159const state = await call("net.provider", { enabled: true, userAgent: o["user-agent"], answerTimeoutMs: Number(o["answer-timeout"]) });
160log(`caller network on (identity ${state.userAgent}, bound ${state.answerTimeoutMs} ms)`);
161
162const loop = (async () => {
163 let after = 0;
164 while (!stop) {
165 let requests = [];
166 try {
167 ({ requests } = await call("net.pending", { afterId: after, waitMs: 1000 }));
168 } catch (e) {
169 log(`waiting for the browser: ${e.message}`);
170 await new Promise((r) => setTimeout(r, 500));
171 }
172 for (const r of requests) {
173 after = Math.max(after, r.id);
174 void answer(r);
175 }
176 }
177})();
178
179let code = 0;
180try {
181 const loaded = await call("page.goto", { url: o.url, settle: { timeoutMs: Number(o["settle-timeout"]) } });
182 console.log(JSON.stringify(loaded.settle, null, 2));
183 if (!loaded.settle.settled || loaded.settle.outcome !== "ok") code = 3;
184} catch (e) {
185 console.error(e.message);
186 code = 1;
187} finally {
188 stop = true;
189 await loop;
190 if (!o.keep) await call("net.provider", { enabled: false }).catch(() => {});
191}
192process.exit(code);