askrobots / dbbasic-browser
Public
A desktop browser with x402 built in: when a page asks for payment, it pays in stablecoins and shows the content.
danq9
site icons (favicons) in tabs, address bar suggestions and the bookmarks panel
7249579
15 hours, 50 minutes ago
History
reader.ts · 15.0 KB · 318 lines
| 1 | /** |
| 2 | * The AI Reader's document model and page builder — Electron-free so it is unit-tested. |
| 3 | * |
| 4 | * Trust boundary: extraction runs INSIDE the (hostile) page, so whatever it returns is untrusted data — |
| 5 | * a page can tamper with the extractor, lie about its structure, or try to smuggle markup through. |
| 6 | * Nothing it returns is ever used as HTML. `validateExtraction` accepts only a small plain-data |
| 7 | * vocabulary (headings, paragraphs, lists, quotes, code, images, links), clamps every size, and drops |
| 8 | * anything else; `buildReaderHtml` then writes the page from scratch with every string escaped. The |
| 9 | * result is a script-free document under a strict CSP (no scripts, no forms, no frames, no connections). |
| 10 | */ |
| 11 | |
| 12 | import type { Summary } from "./ai.js"; |
| 13 | |
| 14 | export interface Run { |
| 15 | x: string; |
| 16 | b?: 1; |
| 17 | i?: 1; |
| 18 | c?: 1; |
| 19 | href?: string; |
| 20 | } |
| 21 | |
| 22 | export type Block = |
| 23 | | { t: "h"; level: number; runs: Run[] } |
| 24 | | { t: "p"; runs: Run[] } |
| 25 | | { t: "ul" | "ol"; items: Run[][] } |
| 26 | | { t: "quote"; runs: Run[] } |
| 27 | | { t: "pre"; text: string } |
| 28 | | { t: "img"; src: string; alt: string } |
| 29 | | { t: "hr" }; |
| 30 | |
| 31 | export interface Extracted { |
| 32 | /** "article" when a main article was found; "text" when we fell back to the page's visible text. */ |
| 33 | mode: "article" | "text"; |
| 34 | url: string; |
| 35 | title: string; |
| 36 | byline: string; |
| 37 | siteName: string; |
| 38 | lang: string; |
| 39 | blocks: Block[]; |
| 40 | } |
| 41 | |
| 42 | export const LIMITS = { blocks: 800, runs: 400, items: 200, runChars: 4000, totalChars: 250_000, title: 300 } as const; |
| 43 | |
| 44 | const str = (v: unknown, max: number): string => (typeof v === "string" ? v : v == null ? "" : String(v)).slice(0, max); |
| 45 | |
| 46 | /** http(s) only, no credentials — anything else (javascript:, data:, file:, blob:) is dropped. */ |
| 47 | export function safeHttpUrl(v: unknown): string | null { |
| 48 | if (typeof v !== "string" || v.length > 4000) return null; |
| 49 | try { |
| 50 | const u = new URL(v); |
| 51 | if ((u.protocol !== "http:" && u.protocol !== "https:") || u.username || u.password) return null; |
| 52 | return u.href; |
| 53 | } catch { |
| 54 | return null; |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | function cleanRuns(v: unknown, budget: { left: number }): Run[] { |
| 59 | if (!Array.isArray(v)) return []; |
| 60 | const out: Run[] = []; |
| 61 | for (const r of v.slice(0, LIMITS.runs)) { |
| 62 | if (!r || typeof r !== "object" || budget.left <= 0) continue; |
| 63 | const o = r as Record<string, unknown>; |
| 64 | const x = str(o.x, LIMITS.runChars); |
| 65 | if (!x) continue; |
| 66 | budget.left -= x.length; |
| 67 | const run: Run = { x }; |
| 68 | if (o.b) run.b = 1; |
| 69 | if (o.i) run.i = 1; |
| 70 | if (o.c) run.c = 1; |
| 71 | const href = safeHttpUrl(o.href); |
| 72 | if (href) run.href = href; |
| 73 | out.push(run); |
| 74 | } |
| 75 | return out; |
| 76 | } |
| 77 | |
| 78 | /** Accept only well-formed plain data; null if it isn't an extraction at all. */ |
| 79 | export function validateExtraction(raw: unknown): Extracted | null { |
| 80 | if (!raw || typeof raw !== "object") return null; |
| 81 | const o = raw as Record<string, unknown>; |
| 82 | const url = safeHttpUrl(o.url); |
| 83 | if (!url || !Array.isArray(o.blocks)) return null; |
| 84 | const budget = { left: LIMITS.totalChars }; |
| 85 | const blocks: Block[] = []; |
| 86 | for (const b of o.blocks.slice(0, LIMITS.blocks)) { |
| 87 | if (!b || typeof b !== "object" || budget.left <= 0) continue; |
| 88 | const r = b as Record<string, unknown>; |
| 89 | switch (r.t) { |
| 90 | case "h": { |
| 91 | const runs = cleanRuns(r.runs, budget); |
| 92 | if (runs.length) blocks.push({ t: "h", level: Math.min(6, Math.max(2, Math.trunc(Number(r.level)) || 2)), runs }); |
| 93 | break; |
| 94 | } |
| 95 | case "p": |
| 96 | case "quote": { |
| 97 | const runs = cleanRuns(r.runs, budget); |
| 98 | if (runs.length) blocks.push({ t: r.t, runs }); |
| 99 | break; |
| 100 | } |
| 101 | case "ul": |
| 102 | case "ol": { |
| 103 | const items = (Array.isArray(r.items) ? r.items.slice(0, LIMITS.items) : []).map((it) => cleanRuns(it, budget)).filter((it) => it.length); |
| 104 | if (items.length) blocks.push({ t: r.t, items }); |
| 105 | break; |
| 106 | } |
| 107 | case "pre": { |
| 108 | const text = str(r.text, LIMITS.runChars * 4); |
| 109 | if (text.trim()) { |
| 110 | budget.left -= text.length; |
| 111 | blocks.push({ t: "pre", text }); |
| 112 | } |
| 113 | break; |
| 114 | } |
| 115 | case "img": { |
| 116 | const src = safeHttpUrl(r.src); |
| 117 | if (src) blocks.push({ t: "img", src, alt: str(r.alt, 300) }); |
| 118 | break; |
| 119 | } |
| 120 | case "hr": |
| 121 | blocks.push({ t: "hr" }); |
| 122 | break; |
| 123 | default: |
| 124 | break; // unknown vocabulary: dropped |
| 125 | } |
| 126 | } |
| 127 | if (blocks.length === 0) return null; |
| 128 | return { |
| 129 | mode: o.mode === "article" ? "article" : "text", |
| 130 | url, |
| 131 | title: str(o.title, LIMITS.title).replace(/\s+/g, " ").trim() || new URL(url).host, |
| 132 | byline: str(o.byline, 200).replace(/\s+/g, " ").trim(), |
| 133 | siteName: str(o.siteName, 100).replace(/\s+/g, " ").trim(), |
| 134 | lang: /^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})*$/.test(str(o.lang, 20)) ? str(o.lang, 20) : "en", |
| 135 | blocks, |
| 136 | }; |
| 137 | } |
| 138 | |
| 139 | const runsText = (runs: Run[]) => runs.map((r) => r.x).join(""); |
| 140 | |
| 141 | /** The article as plain text (what the summarizer is given). */ |
| 142 | export function blocksToText(blocks: Block[]): string { |
| 143 | const parts: string[] = []; |
| 144 | for (const b of blocks) { |
| 145 | if (b.t === "h") parts.push(`${"#".repeat(Math.min(b.level, 4))} ${runsText(b.runs)}`); |
| 146 | else if (b.t === "p" || b.t === "quote") parts.push(runsText(b.runs)); |
| 147 | else if (b.t === "ul" || b.t === "ol") parts.push(b.items.map((it, i) => `${b.t === "ol" ? `${i + 1}.` : "-"} ${runsText(it)}`).join("\n")); |
| 148 | else if (b.t === "pre") parts.push(b.text); |
| 149 | else if (b.t === "img" && b.alt) parts.push(`[image: ${b.alt}]`); |
| 150 | } |
| 151 | return parts.join("\n\n"); |
| 152 | } |
| 153 | |
| 154 | export const wordCount = (text: string) => (text.match(/\S+/g) ?? []).length; |
| 155 | |
| 156 | export function escapeHtml(s: string): string { |
| 157 | return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'"); |
| 158 | } |
| 159 | |
| 160 | function runsHtml(runs: Run[]): string { |
| 161 | return runs |
| 162 | .map((r) => { |
| 163 | let h = escapeHtml(r.x); |
| 164 | if (r.c) h = `<code>${h}</code>`; |
| 165 | if (r.i) h = `<em>${h}</em>`; |
| 166 | if (r.b) h = `<strong>${h}</strong>`; |
| 167 | if (r.href) h = `<a href="${escapeHtml(r.href)}" rel="noopener noreferrer">${h}</a>`; |
| 168 | return h; |
| 169 | }) |
| 170 | .join(""); |
| 171 | } |
| 172 | |
| 173 | function blockHtml(b: Block): string { |
| 174 | switch (b.t) { |
| 175 | case "h": |
| 176 | return `<h${b.level}>${runsHtml(b.runs)}</h${b.level}>`; |
| 177 | case "p": |
| 178 | return `<p>${runsHtml(b.runs)}</p>`; |
| 179 | case "quote": |
| 180 | return `<blockquote>${runsHtml(b.runs)}</blockquote>`; |
| 181 | case "ul": |
| 182 | case "ol": |
| 183 | return `<${b.t}>${b.items.map((it) => `<li>${runsHtml(it)}</li>`).join("")}</${b.t}>`; |
| 184 | case "pre": |
| 185 | return `<pre>${escapeHtml(b.text)}</pre>`; |
| 186 | case "img": |
| 187 | return `<figure><img src="${escapeHtml(b.src)}" alt="${escapeHtml(b.alt)}" loading="lazy" referrerpolicy="no-referrer"></figure>`; |
| 188 | case "hr": |
| 189 | return "<hr>"; |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | export type AiState = "pending" | "none" | "declined" | "error"; |
| 194 | export interface ReaderView { |
| 195 | ai: |
| 196 | | { state: AiState; provider?: string; model?: string; message?: string } |
| 197 | /** The finished summary — what a Back/Forward/reload of the reader page must show. */ |
| 198 | | { state: "done"; tldr: string; points: string[]; foot: string }; |
| 199 | } |
| 200 | |
| 201 | export const READER_CSP = "default-src 'none'; img-src http: https:; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'"; |
| 202 | |
| 203 | const STYLE = ` |
| 204 | :root { color-scheme: light dark; --bg:#faf8f4; --fg:#1d1d1f; --muted:#6b6b72; --rule:#e4e0d8; --card:#fff; --accent:#2f5fd0; --tldr:#eef3ff; --tldr-rule:#c9d8ff; --sk:#dfe6f7; } |
| 205 | @media (prefers-color-scheme: dark) { :root { --bg:#16161a; --fg:#e8e8ea; --muted:#9a9aa2; --rule:#2c2c33; --card:#1e1e24; --accent:#7aa2ff; --tldr:#1a2033; --tldr-rule:#2c3d6b; --sk:#26304f; } } |
| 206 | * { box-sizing: border-box; } |
| 207 | body { margin: 0; background: var(--bg); color: var(--fg); font: 19px/1.72 Charter, "Iowan Old Style", Georgia, "Times New Roman", serif; } |
| 208 | main { max-width: 700px; margin: 0 auto; padding: 44px 22px 90px; } |
| 209 | .badge { display: inline-block; font: 600 11px/1 -apple-system, system-ui, sans-serif; letter-spacing: .08em; text-transform: uppercase; color: var(--accent); border: 1px solid var(--tldr-rule); border-radius: 999px; padding: 5px 10px; margin-bottom: 18px; } |
| 210 | h1 { font: 700 34px/1.2 -apple-system, "Helvetica Neue", system-ui, sans-serif; letter-spacing: -.02em; margin: 0 0 12px; } |
| 211 | .meta { font: 14px/1.5 -apple-system, system-ui, sans-serif; color: var(--muted); margin: 0 0 28px; } |
| 212 | .meta a { color: var(--muted); } |
| 213 | .tldr { background: var(--tldr); border: 1px solid var(--tldr-rule); border-radius: 14px; padding: 18px 22px 16px; margin: 0 0 38px; font: 16.5px/1.6 -apple-system, system-ui, sans-serif; } |
| 214 | .tldr .lbl { font-weight: 700; font-size: 12px; letter-spacing: .1em; color: var(--accent); margin-bottom: 8px; } |
| 215 | .tldr #tldr-body { font-size: 18px; line-height: 1.55; font-weight: 500; } |
| 216 | .tldr ul { margin: 12px 0 0; padding-left: 20px; } .tldr li { margin: 4px 0; } |
| 217 | .tldr .foot { margin-top: 12px; font-size: 12px; color: var(--muted); } |
| 218 | .tldr .sk { display: none; } |
| 219 | .tldr.pending #tldr-body { display: none; } |
| 220 | .tldr.pending .sk { display: block; height: 13px; border-radius: 7px; background: var(--sk); margin: 9px 0; animation: pulse 1.3s ease-in-out infinite; } |
| 221 | .tldr.pending .sk:nth-child(3) { width: 82%; } .tldr.pending .sk:nth-child(4) { width: 58%; } |
| 222 | .tldr.none, .tldr.declined, .tldr.error { background: var(--card); border-color: var(--rule); } |
| 223 | .tldr.none .lbl, .tldr.declined .lbl, .tldr.error .lbl { color: var(--muted); } |
| 224 | .tldr.none #tldr-body, .tldr.declined #tldr-body, .tldr.error #tldr-body { font-weight: 400; font-size: 15px; color: var(--muted); } |
| 225 | @keyframes pulse { 50% { opacity: .45; } } |
| 226 | @media (prefers-reduced-motion: reduce) { .tldr.pending .sk { animation: none; } } |
| 227 | article h2, article h3, article h4, article h5, article h6 { font-family: -apple-system, "Helvetica Neue", system-ui, sans-serif; line-height: 1.3; letter-spacing: -.01em; margin: 1.9em 0 .6em; } |
| 228 | article h2 { font-size: 26px; } article h3 { font-size: 21px; } article h4, article h5, article h6 { font-size: 18px; } |
| 229 | article p { margin: 0 0 1.1em; } article a { color: var(--accent); } |
| 230 | article ul, article ol { padding-left: 1.3em; margin: 0 0 1.1em; } article li { margin: .3em 0; } |
| 231 | article blockquote { margin: 1.2em 0; padding: .1em 0 .1em 1.1em; border-left: 3px solid var(--rule); color: var(--muted); } |
| 232 | article pre { overflow: auto; background: var(--card); border: 1px solid var(--rule); border-radius: 10px; padding: 14px 16px; font: 14px/1.5 ui-monospace, Menlo, monospace; } |
| 233 | article code { font: .86em ui-monospace, Menlo, monospace; background: var(--card); border: 1px solid var(--rule); border-radius: 5px; padding: .08em .35em; } |
| 234 | article figure { margin: 1.6em 0; } article img { max-width: 100%; height: auto; border-radius: 8px; display: block; } |
| 235 | hr { border: 0; border-top: 1px solid var(--rule); margin: 2em 0; } |
| 236 | footer { margin-top: 48px; padding-top: 18px; border-top: 1px solid var(--rule); font: 14px/1.5 -apple-system, system-ui, sans-serif; color: var(--muted); } |
| 237 | footer a { color: var(--accent); } |
| 238 | `; |
| 239 | |
| 240 | const AI_NOTE: Record<Exclude<AiState, "pending">, string> = { |
| 241 | none: "No AI summary: add an Anthropic or OpenAI API key (see Help ▸ AI Reader).", |
| 242 | declined: "No summary was written, so nothing from this page was sent to an AI provider.", |
| 243 | error: "The summary could not be written.", |
| 244 | }; |
| 245 | |
| 246 | export function buildReaderHtml(ex: Extracted, view: ReaderView): string { |
| 247 | const text = blocksToText(ex.blocks); |
| 248 | const words = wordCount(text); |
| 249 | const minutes = Math.max(1, Math.round(words / 220)); |
| 250 | let host = ex.url; |
| 251 | try { |
| 252 | host = new URL(ex.url).host; |
| 253 | } catch { |
| 254 | /* keep the url */ |
| 255 | } |
| 256 | const meta = [ex.byline, ex.siteName && ex.siteName !== host ? ex.siteName : host, `${minutes} min read`].filter(Boolean).map(escapeHtml).join(" · "); |
| 257 | const ai = view.ai; |
| 258 | const pending = ai.state === "pending"; |
| 259 | const body = pending |
| 260 | ? `<div class="sk"></div><div class="sk"></div><div class="sk"></div><div id="tldr-body"></div>` |
| 261 | : ai.state === "done" |
| 262 | ? `<div id="tldr-body">${escapeHtml(ai.tldr)}</div>` |
| 263 | : `<div id="tldr-body">${escapeHtml(ai.state === "error" && ai.message ? `${AI_NOTE.error} ${ai.message}` : AI_NOTE[ai.state as Exclude<AiState, "pending">])}</div>`; |
| 264 | const points = ai.state === "done" ? ai.points.map((p) => `<li>${escapeHtml(p)}</li>`).join("") : ""; |
| 265 | const foot = |
| 266 | ai.state === "done" |
| 267 | ? escapeHtml(ai.foot) |
| 268 | : pending && ai.provider |
| 269 | ? `Summarizing with ${escapeHtml(ai.provider)}${ai.model ? ` · ${escapeHtml(ai.model)}` : ""}…` |
| 270 | : ""; |
| 271 | const note = ex.mode === "text" ? `<p class="meta">This page isn't an article, so this is its visible text, not a cleaned-up article.</p>` : ""; |
| 272 | return `<!doctype html> |
| 273 | <html lang="${escapeHtml(ex.lang)}"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"> |
| 274 | <meta http-equiv="Content-Security-Policy" content="${READER_CSP}"><meta name="referrer" content="no-referrer"> |
| 275 | <title>${escapeHtml(ex.title)}</title><style>${STYLE}</style></head> |
| 276 | <body><main> |
| 277 | <header><div class="badge">AI Reader</div><h1>${escapeHtml(ex.title)}</h1><p class="meta">${meta} · <a href="${escapeHtml(ex.url)}" rel="noopener noreferrer">original</a></p></header> |
| 278 | <section id="tldr" class="tldr ${ai.state}"><div class="lbl">TL;DR</div>${body}<ul id="tldr-points">${points}</ul><div id="tldr-foot" class="foot">${foot}</div></section> |
| 279 | ${note}<article>${ex.blocks.map(blockHtml).join("\n")}</article> |
| 280 | <footer>Read on the original page: <a href="${escapeHtml(ex.url)}" rel="noopener noreferrer">${escapeHtml(host)}</a></footer> |
| 281 | </main></body></html>`; |
| 282 | } |
| 283 | |
| 284 | export function readerDataUrl(html: string): string { |
| 285 | return `data:text/html;charset=utf-8;base64,${Buffer.from(html, "utf8").toString("base64")}`; |
| 286 | } |
| 287 | |
| 288 | export interface TldrUpdate { |
| 289 | state: "done" | "error"; |
| 290 | tldr: string; |
| 291 | points: string[]; |
| 292 | foot: string; |
| 293 | } |
| 294 | |
| 295 | /** |
| 296 | * JavaScript that fills the TL;DR in an already-open reader page. Values are embedded as JSON string |
| 297 | * literals and written with textContent only — never as HTML. |
| 298 | */ |
| 299 | export function tldrUpdateScript(u: TldrUpdate): string { |
| 300 | return `(() => { |
| 301 | const d = document, sec = d.getElementById("tldr"); |
| 302 | if (!sec) return false; |
| 303 | sec.className = "tldr " + ${JSON.stringify(u.state === "done" ? "done" : "error")}; |
| 304 | d.getElementById("tldr-body").textContent = ${JSON.stringify(u.tldr)}; |
| 305 | const ul = d.getElementById("tldr-points"); ul.textContent = ""; |
| 306 | for (const p of ${JSON.stringify(u.points)}) { const li = d.createElement("li"); li.textContent = p; ul.appendChild(li); } |
| 307 | d.getElementById("tldr-foot").textContent = ${JSON.stringify(u.foot)}; |
| 308 | return true; |
| 309 | })()`; |
| 310 | } |
| 311 | |
| 312 | export function footFor(r: { provider: string; model: string; ms: number; usage: { inputTokens: number; outputTokens: number }; costUsd: number | null; truncated: boolean }): string { |
| 313 | const cost = r.costUsd == null ? "" : ` · ≈$${r.costUsd < 0.01 ? r.costUsd.toFixed(4) : r.costUsd.toFixed(3)}`; |
| 314 | const cut = r.truncated ? " · long page: only the first part was summarized" : ""; |
| 315 | return `Summarized by ${r.model} (${r.provider}) in ${(r.ms / 1000).toFixed(1)} s · ${r.usage.inputTokens.toLocaleString("en-US")} in / ${r.usage.outputTokens.toLocaleString("en-US")} out tokens${cost}${cut}`; |
| 316 | } |
| 317 | |
| 318 | export const summaryToUpdate = (s: Summary, foot: string): TldrUpdate => ({ state: "done", tldr: s.tldr, points: s.points, foot }); |