askrobots / dbbasic-browser
Public
A desktop browser with x402 built in: when a page asks for payment, it pays in stablecoins and shows the content.
danq9
site icons (favicons) in tabs, address bar suggestions and the bookmarks panel
7249579
15 hours, 50 minutes ago
History
settle.ts · 16.5 KB · 323 lines
| 1 | /** |
| 2 | * "Has this page finished loading?" — decided from what the browser actually observes, not from a guess. |
| 3 | * |
| 4 | * Outside tools wait by polling for "loaded", "network quiet for half a second" or a fixed sleep, and when |
| 5 | * they give up all they know is that time passed. This runs inside the browser, where the network layer |
| 6 | * can tell a finished page from one that is still receiving, and it answers two separate questions: |
| 7 | * |
| 8 | * rule — WHAT ENDED THE WAIT: the page settled (and which test proved it), or the deadline hit |
| 9 | * outcome — WHAT HAPPENED: fine, an error status, a blank page, a failed load (and why), a crash, |
| 10 | * a bot check, a payment request, a download instead of a page |
| 11 | * |
| 12 | * A page counts as settled when ALL of these hold: the load event fired; no request is in flight, other than |
| 13 | * long-lived streams that never finish by design (server-sent events, held-open responses); nothing has |
| 14 | * moved on the network for `quietMs`; web fonts are in; the visible images are in; and (unless turned off) |
| 15 | * two frames a moment apart are identical. A page that animates forever never matches, so after |
| 16 | * `frameTimeoutMs` it settles as "animating" instead of waiting for the deadline. |
| 17 | * |
| 18 | * The decision is a pure function of a snapshot so it can be tested without a browser; src/settle-service.ts |
| 19 | * gathers the snapshots. Known limit: WebSockets bypass the network layer, so a socket-driven page can look |
| 20 | * quiet while it is still receiving. |
| 21 | */ |
| 22 | |
| 23 | export interface SettleConfig { |
| 24 | /** Hard limit; at this point the call returns whatever happened, with what was still unmet. */ |
| 25 | timeoutMs: number; |
| 26 | /** No request activity (outside long-lived streams) for this long. */ |
| 27 | quietMs: number; |
| 28 | /** Compare two frames `frameGapMs` apart before calling the page settled. */ |
| 29 | frames: boolean; |
| 30 | frameGapMs: number; |
| 31 | /** After every other rule holds, how long to wait for two matching frames before calling it "animating". */ |
| 32 | frameTimeoutMs: number; |
| 33 | /** A response with headers but no Content-Length that is still open after this long counts as a stream. */ |
| 34 | streamAfterMs: number; |
| 35 | /** A settled page with less visible text than this (and no image, canvas, video or svg) is called "blank". 0 turns the test off. */ |
| 36 | minText: number; |
| 37 | /** When a page would settle as blank, keep checking for this long for text or media to appear, then settle (still blank if it stayed blank). */ |
| 38 | blankWaitMs: number; |
| 39 | pollMs: number; |
| 40 | } |
| 41 | |
| 42 | export const SETTLE_DEFAULTS: SettleConfig = { |
| 43 | timeoutMs: 30_000, |
| 44 | quietMs: 500, |
| 45 | frames: true, |
| 46 | frameGapMs: 250, |
| 47 | frameTimeoutMs: 3_000, |
| 48 | streamAfterMs: 1_000, |
| 49 | minText: 20, |
| 50 | blankWaitMs: 0, |
| 51 | pollMs: 100, |
| 52 | }; |
| 53 | |
| 54 | /** Clamp caller-supplied options to sane values; unknown or invalid ones fall back to the defaults. */ |
| 55 | export function resolveConfig(input: unknown): SettleConfig { |
| 56 | const o = (input && typeof input === "object" ? input : {}) as Record<string, unknown>; |
| 57 | const num = (k: keyof SettleConfig, min: number, max: number) => { |
| 58 | const v = Number(o[k]); |
| 59 | return Number.isFinite(v) && o[k] !== undefined ? Math.min(max, Math.max(min, v)) : (SETTLE_DEFAULTS[k] as number); |
| 60 | }; |
| 61 | return { |
| 62 | timeoutMs: num("timeoutMs", 500, 300_000), |
| 63 | quietMs: num("quietMs", 0, 10_000), |
| 64 | frames: o.frames === undefined ? SETTLE_DEFAULTS.frames : o.frames !== false, |
| 65 | frameGapMs: num("frameGapMs", 50, 2_000), |
| 66 | frameTimeoutMs: num("frameTimeoutMs", 0, 30_000), |
| 67 | streamAfterMs: num("streamAfterMs", 100, 60_000), |
| 68 | minText: num("minText", 0, 10_000), |
| 69 | blankWaitMs: num("blankWaitMs", 0, 120_000), |
| 70 | pollMs: num("pollMs", 20, 1_000), |
| 71 | }; |
| 72 | } |
| 73 | |
| 74 | export type Rule = |
| 75 | | "quiet" // every rule held (frames check off) |
| 76 | | "frames-match" // every rule held and two frames a moment apart were identical |
| 77 | | "animating" // every rule held but the frames never matched: the page keeps changing on its own |
| 78 | | "navigation-failed" |
| 79 | | "crashed" |
| 80 | | "download" |
| 81 | | "payment" |
| 82 | | "deadline"; |
| 83 | |
| 84 | export type Outcome = |
| 85 | | "ok" |
| 86 | | "error-status" // the page loaded, but the server answered 4xx/5xx |
| 87 | | "blank" // it loaded and settled, but there is nothing on it |
| 88 | | "failed" |
| 89 | | "crashed" |
| 90 | | "bot-check" |
| 91 | | "payment-required" |
| 92 | | "not-a-page" // the address was a download, not a page |
| 93 | | "deadline"; // gave up; `blockedBy` says what was still unmet |
| 94 | |
| 95 | export const FAILURE_KINDS = ["could-not-find-site", "could-not-connect", "certificate", "redirect-loop", "offline", "timed-out", "blocked", "other", "caller-silent"] as const; |
| 96 | /** Why a load failed. "caller-silent" = a caller-supplied network layer did not answer within its own bound (not "server too slow"). */ |
| 97 | export type FailureKind = (typeof FAILURE_KINDS)[number]; |
| 98 | /** Who decided it: the browser saw it happen, or the caller said so. */ |
| 99 | export type FailureSource = "observed" | "caller"; |
| 100 | |
| 101 | /** Map Chromium's error description (e.g. "ERR_CERT_DATE_INVALID") to a reason a person can act on. */ |
| 102 | export function classifyFailure(description: string): FailureKind { |
| 103 | const d = String(description || "").toUpperCase(); |
| 104 | if (/ERR_NAME_NOT_RESOLVED|ERR_NAME_RESOLUTION_FAILED|DNS/.test(d)) return "could-not-find-site"; |
| 105 | if (/ERR_CERT|ERR_SSL|CERTIFICATE|ERR_BAD_SSL|ERR_SECURITY/.test(d)) return "certificate"; |
| 106 | if (/TOO_MANY_REDIRECTS|REDIRECT_LOOP/.test(d)) return "redirect-loop"; |
| 107 | if (/INTERNET_DISCONNECTED|NETWORK_CHANGED|ERR_NETWORK_ACCESS|PROXY/.test(d)) return "offline"; |
| 108 | if (/TIMED_OUT|TIMEOUT/.test(d)) return "timed-out"; |
| 109 | if (/BLOCKED|ERR_ACCESS_DENIED|ERR_ABORTED_BY_BLOCKER|ERR_UNSAFE_PORT|ERR_UNSAFE_REDIRECT/.test(d)) return "blocked"; // the browser itself refused it |
| 110 | if (/ERR_CONNECTION|ERR_EMPTY_RESPONSE|ERR_ADDRESS|ERR_SOCKET|ERR_CONNECT|REFUSED|RESET/.test(d)) return "could-not-connect"; |
| 111 | return "other"; |
| 112 | } |
| 113 | |
| 114 | /** What the network layer counted for a visit, as the browser UI keeps it. */ |
| 115 | export interface ResourceStats { |
| 116 | requests: number; |
| 117 | answered: number; |
| 118 | errorStatus: number; |
| 119 | cancelled: number; |
| 120 | failures: Array<{ kind: string | null; error: string; /** Without its query or fragment. */ url?: string }>; |
| 121 | /** Up to ten responses with an error status, as addresses without queries. */ |
| 122 | statusExamples?: Array<{ url: string; status: number }>; |
| 123 | } |
| 124 | |
| 125 | /** How much of a page arrived. "ok" says the main document settled; this says what happened to everything else it asked for. */ |
| 126 | export interface Resources { |
| 127 | /** Requests started (the main document, its redirects, and everything the page asked for). */ |
| 128 | requested: number; |
| 129 | /** Got a response (a redirect counts). Of these, `errorStatus` answered with 4xx or 5xx. */ |
| 130 | answered: number; |
| 131 | errorStatus: number; |
| 132 | /** Ended in an error: refused, could not connect, timed out... by reason below. */ |
| 133 | failed: number; |
| 134 | failedByKind: Partial<Record<FailureKind, number>>; |
| 135 | /** The page gave up on them itself (it navigated away or cancelled). Not counted as failures. */ |
| 136 | cancelled: number; |
| 137 | /** Started, not answered or failed yet when the verdict was given. */ |
| 138 | waiting: number; |
| 139 | /** The page itself: "answered" once its response status is known, "failed" if the load failed, else "pending". */ |
| 140 | mainDocument: "answered" | "failed" | "pending"; |
| 141 | /** Which requests failed, and which got an error status (up to ten each, addresses without their query), when known: what to look at first. */ |
| 142 | failedExamples?: Array<{ url: string; error: string }>; |
| 143 | errorStatusExamples?: Array<{ url: string; status: number }>; |
| 144 | } |
| 145 | |
| 146 | export function summarizeResources(stats: ResourceStats, main: { failed: boolean; status: number | null }): Resources { |
| 147 | const failedByKind: Partial<Record<FailureKind, number>> = {}; |
| 148 | for (const f of stats.failures) { |
| 149 | const kind = (FAILURE_KINDS as readonly string[]).includes(f.kind ?? "") ? (f.kind as FailureKind) : classifyFailure(f.error); |
| 150 | failedByKind[kind] = (failedByKind[kind] ?? 0) + 1; |
| 151 | } |
| 152 | const failed = stats.failures.length; |
| 153 | const failedExamples = stats.failures.filter((f) => f.url).slice(0, 10).map((f) => ({ url: f.url as string, error: f.error })); |
| 154 | const errorStatusExamples = (stats.statusExamples ?? []).slice(0, 10); |
| 155 | return { |
| 156 | requested: stats.requests, |
| 157 | answered: stats.answered, |
| 158 | errorStatus: stats.errorStatus, |
| 159 | failed, |
| 160 | failedByKind, |
| 161 | cancelled: stats.cancelled, |
| 162 | waiting: Math.max(0, stats.requests - stats.answered - failed - stats.cancelled), |
| 163 | mainDocument: main.failed ? "failed" : main.status !== null ? "answered" : "pending", |
| 164 | ...(failedExamples.length ? { failedExamples } : {}), |
| 165 | ...(errorStatusExamples.length ? { errorStatusExamples } : {}), |
| 166 | }; |
| 167 | } |
| 168 | |
| 169 | export interface InFlight { |
| 170 | url: string; |
| 171 | /** "caller": a caller-supplied network layer has the request and has not answered yet. */ |
| 172 | phase: "waiting" | "receiving" | "caller"; |
| 173 | ageMs: number; |
| 174 | /** Headers arrived but the body is open-ended (event stream, or no Content-Length for a long time). */ |
| 175 | longLived: boolean; |
| 176 | } |
| 177 | |
| 178 | export interface Snapshot { |
| 179 | /** Milliseconds since the wait began. */ |
| 180 | nowMs: number; |
| 181 | lifecycle: { |
| 182 | crashed: boolean; |
| 183 | failed: { description: string; code?: number; kind?: FailureKind; source?: FailureSource } | null; |
| 184 | /** File name of a download that started instead of a page. */ |
| 185 | download: string | null; |
| 186 | /** HTTP status of the main document, once known. */ |
| 187 | status: number | null; |
| 188 | /** How many top-level navigations this visit has made so far (the page moving itself adds to it). */ |
| 189 | navigations?: number; |
| 190 | }; |
| 191 | net: { |
| 192 | inFlight: InFlight[]; |
| 193 | /** Time since the last request activity that is not a long-lived stream. */ |
| 194 | sinceActivityMs: number; |
| 195 | /** The tab's own state ("challenge" while a bot check runs, …). */ |
| 196 | state: string | null; |
| 197 | /** An x402 payment waiting for a decision, or refused. */ |
| 198 | pay: "pending" | "refused" | null; |
| 199 | }; |
| 200 | /** What the page itself says; null if it could not be asked. */ |
| 201 | page: { readyState: string; fonts: string; imagesPending: number; textLength: number; hasMedia: boolean } | null; |
| 202 | /** Result of the latest two-frame comparison, if one has been made. */ |
| 203 | frames?: { matched: boolean }; |
| 204 | /** Time spent waiting on a caller-supplied network layer. It does not count toward the deadline: the caller's own bound ends that wait. */ |
| 205 | callerWaitMs?: number; |
| 206 | } |
| 207 | |
| 208 | export interface Blocker { |
| 209 | rule: "load-event" | "caller" | "network" | "network-quiet" | "fonts" | "images" | "page-unreachable"; |
| 210 | detail?: string; |
| 211 | } |
| 212 | |
| 213 | export interface Verdict { |
| 214 | settled: boolean; |
| 215 | outcome: Outcome; |
| 216 | rule: Rule; |
| 217 | waitedMs: number; |
| 218 | status?: number; |
| 219 | /** For outcome "error-status": the server asked us to slow down (HTTP 429). */ |
| 220 | slowDown?: boolean; |
| 221 | failure?: { kind: FailureKind; description: string; source: FailureSource }; |
| 222 | download?: string; |
| 223 | /** For the deadline: what was still unmet. */ |
| 224 | blockedBy?: Blocker[]; |
| 225 | /** True when frames never matched: the page keeps changing by itself. */ |
| 226 | animating?: boolean; |
| 227 | /** Long-lived streams that were ignored (a page can be finished while holding one open). */ |
| 228 | streams?: number; |
| 229 | } |
| 230 | |
| 231 | export interface Memo { |
| 232 | /** When every rule except frames first held, continuously. */ |
| 233 | okSince: number | null; |
| 234 | /** The navigation count last seen: a page that navigates itself restarts the wait. */ |
| 235 | navs: number | null; |
| 236 | /** When the page first looked blank at the point of settling (for blankWaitMs). */ |
| 237 | blankSince: number | null; |
| 238 | } |
| 239 | |
| 240 | export type Step = { done: true; verdict: Verdict } | { done: false; action?: "check-frames" }; |
| 241 | |
| 242 | export const newMemo = (): Memo => ({ okSince: null, navs: null, blankSince: null }); |
| 243 | |
| 244 | /** The unmet rules in a snapshot (empty = everything holds, apart from the frame comparison). */ |
| 245 | export function blockers(s: Snapshot, cfg: SettleConfig): Blocker[] { |
| 246 | const out: Blocker[] = []; |
| 247 | const withCaller = s.net.inFlight.filter((r) => r.phase === "caller"); |
| 248 | const pending = s.net.inFlight.filter((r) => !r.longLived && r.phase !== "caller"); |
| 249 | if (withCaller.length) out.push({ rule: "caller", detail: `${withCaller.length} request(s) waiting on the caller, e.g. ${withCaller[0]!.url}` }); |
| 250 | if (pending.length) { |
| 251 | const oldest = pending.reduce((a, b) => (b.ageMs > a.ageMs ? b : a)); |
| 252 | out.push({ rule: "network", detail: `${pending.length} request(s) in flight, e.g. ${oldest.url} (${oldest.phase}, ${Math.round(oldest.ageMs)} ms)` }); |
| 253 | } else if (s.net.sinceActivityMs < cfg.quietMs) out.push({ rule: "network-quiet", detail: `activity ${Math.round(s.net.sinceActivityMs)} ms ago` }); |
| 254 | // While a navigation is stuck pending, the page can't be asked anything; the request that never answered is still |
| 255 | // the useful answer, so it is reported first and "page-unreachable" after it. |
| 256 | if (!s.page) { |
| 257 | out.push({ rule: "page-unreachable" }); |
| 258 | return out; |
| 259 | } |
| 260 | if (s.page.readyState !== "complete") out.push({ rule: "load-event", detail: s.page.readyState }); |
| 261 | if (s.page.fonts === "loading") out.push({ rule: "fonts" }); |
| 262 | if (s.page.imagesPending > 0) out.push({ rule: "images", detail: `${s.page.imagesPending} visible image(s) still loading` }); |
| 263 | return out; |
| 264 | } |
| 265 | |
| 266 | /** What actually happened, once we know the page settled. */ |
| 267 | function settledOutcome(s: Snapshot, cfg: SettleConfig): Pick<Verdict, "outcome" | "status" | "slowDown"> { |
| 268 | const status = s.lifecycle.status ?? undefined; |
| 269 | if (status !== undefined && status >= 400) return { outcome: "error-status", status, ...(status === 429 ? { slowDown: true } : {}) }; |
| 270 | if (s.page && s.page.textLength < cfg.minText && !s.page.hasMedia) return { outcome: "blank", ...(status !== undefined ? { status } : {}) }; |
| 271 | return { outcome: "ok", ...(status !== undefined ? { status } : {}) }; |
| 272 | } |
| 273 | |
| 274 | const streams = (s: Snapshot) => s.net.inFlight.filter((r) => r.longLived).length; |
| 275 | |
| 276 | /** How much waiting on the caller is forgiven: all of it, but never more than twice the deadline (so a call always ends within 3x). */ |
| 277 | export const callerAllowance = (callerWaitMs: number | undefined, cfg: SettleConfig): number => Math.min(callerWaitMs ?? 0, 2 * cfg.timeoutMs); |
| 278 | |
| 279 | export function step(s: Snapshot, cfg: SettleConfig, memo: Memo): Step { |
| 280 | const base = { waitedMs: Math.round(s.nowMs) }; |
| 281 | const sc = streams(s); |
| 282 | const streamField = sc ? { streams: sc } : {}; |
| 283 | |
| 284 | // Things that end the wait right away, whatever else is going on. |
| 285 | if (s.lifecycle.crashed) return { done: true, verdict: { settled: false, outcome: "crashed", rule: "crashed", ...base } }; |
| 286 | if (s.lifecycle.download) return { done: true, verdict: { settled: true, outcome: "not-a-page", rule: "download", download: s.lifecycle.download, ...base } }; |
| 287 | if (s.lifecycle.failed) { |
| 288 | const f = s.lifecycle.failed; |
| 289 | return { done: true, verdict: { settled: false, outcome: "failed", rule: "navigation-failed", failure: { kind: f.kind ?? classifyFailure(f.description), description: f.description, source: f.source ?? "observed" }, ...base } }; |
| 290 | } |
| 291 | if (s.net.pay) return { done: true, verdict: { settled: false, outcome: "payment-required", rule: "payment", ...base } }; |
| 292 | |
| 293 | // The page moved itself (a script, a meta refresh, a link): what was true of the page we were on no longer counts. |
| 294 | const navs = s.lifecycle.navigations ?? null; |
| 295 | if (navs !== null && memo.navs !== null && navs !== memo.navs) { |
| 296 | memo.okSince = null; |
| 297 | memo.blankSince = null; |
| 298 | } |
| 299 | memo.navs = navs; |
| 300 | |
| 301 | const unmet = blockers(s, cfg); |
| 302 | if (unmet.length === 0) { |
| 303 | memo.okSince ??= s.nowMs; |
| 304 | // Would it settle as blank? Give it blankWaitMs to draw something first (a page that fills itself in late). Still blank after |
| 305 | // that, it is reported blank; the time waited is part of waitedMs. |
| 306 | if (cfg.blankWaitMs > 0 && settledOutcome(s, cfg).outcome === "blank") { |
| 307 | memo.blankSince ??= s.nowMs; |
| 308 | if (s.nowMs - memo.blankSince < cfg.blankWaitMs) return { done: false }; |
| 309 | } else memo.blankSince = null; |
| 310 | if (!cfg.frames) return { done: true, verdict: { settled: true, rule: "quiet", ...settledOutcome(s, cfg), ...streamField, ...base } }; |
| 311 | if (s.frames?.matched) return { done: true, verdict: { settled: true, rule: "frames-match", ...settledOutcome(s, cfg), ...streamField, ...base } }; |
| 312 | if (s.nowMs - memo.okSince >= cfg.frameTimeoutMs) return { done: true, verdict: { settled: true, rule: "animating", animating: true, ...settledOutcome(s, cfg), ...streamField, ...base } }; |
| 313 | return { done: false, action: "check-frames" }; |
| 314 | } |
| 315 | memo.okSince = null; // something moved again: the "everything holds" clock restarts |
| 316 | |
| 317 | // Time on the caller does not count, up to twice the deadline: a page that always has one request outstanding cannot freeze the clock for ever. |
| 318 | if (s.nowMs - callerAllowance(s.callerWaitMs, cfg) >= cfg.timeoutMs) { |
| 319 | const botCheck = s.net.state === "challenge"; |
| 320 | return { done: true, verdict: { settled: false, outcome: botCheck ? "bot-check" : "deadline", rule: "deadline", blockedBy: unmet, ...streamField, ...base } }; |
| 321 | } |
| 322 | return { done: false }; |
| 323 | } |