askrobots / dbbasic-browser
Public
A desktop browser with x402 built in: when a page asks for payment, it pays in stablecoins and shows the content.
danq9
site icons (favicons) in tabs, address bar suggestions and the bookmarks panel
7249579
15 hours, 50 minutes ago
History
reader-extractor.js · 6.8 KB · 137 lines
| 1 | // Runs INSIDE the page being read (injected by the main process together with Mozilla's Readability). |
| 2 | // It returns PLAIN DATA only — never HTML. The page is hostile and may tamper with this code, so the |
| 3 | // main process re-validates everything (src/reader.ts validateExtraction) and builds the reader page itself. |
| 4 | // |
| 5 | // Shape: { mode: "article"|"text", url, title, byline, siteName, lang, blocks: [...] } |
| 6 | // or { error: "login" | "extract-failed" } |
| 7 | (function (Readability) { |
| 8 | "use strict"; |
| 9 | var MAX_BLOCKS = 800, MAX_CHARS = 250000; |
| 10 | var total = 0; |
| 11 | var doc = document; |
| 12 | var SKIP = /^(SCRIPT|STYLE|NOSCRIPT|TEMPLATE|SVG|IFRAME|OBJECT|EMBED|SELECT|TEXTAREA|INPUT|CANVAS|VIDEO|AUDIO|LINK|META)$/; |
| 13 | var INLINE = /^(A|SPAN|STRONG|B|EM|I|CODE|KBD|SAMP|SMALL|MARK|ABBR|CITE|Q|SUB|SUP|U|S|DEL|INS|TIME|BDI|BDO|WBR)$/; |
| 14 | |
| 15 | function visible(el) { return !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length); } |
| 16 | // Never read a login page: it may contain (or be about to contain) credentials. |
| 17 | var pw = doc.querySelectorAll('input[type="password"]'); |
| 18 | for (var i = 0; i < pw.length; i++) if (visible(pw[i])) return { error: "login" }; |
| 19 | |
| 20 | function abs(u) { try { return new URL(u, doc.baseURI).href; } catch (e) { return ""; } } |
| 21 | function http(u) { return /^https?:/i.test(u) ? u : ""; } |
| 22 | |
| 23 | function inline(node, out, fmt) { |
| 24 | var kids = node.childNodes; |
| 25 | for (var i = 0; i < kids.length; i++) { |
| 26 | if (total > MAX_CHARS) return; |
| 27 | var n = kids[i]; |
| 28 | if (n.nodeType === 3) { |
| 29 | var x = n.nodeValue.replace(/\s+/g, " "); |
| 30 | if (x.trim() || (x === " " && out.length)) { out.push(Object.assign({ x: x }, fmt)); total += x.length; } |
| 31 | } else if (n.nodeType === 1) { |
| 32 | var tag = n.tagName; |
| 33 | if (SKIP.test(tag) || tag === "IMG") continue; |
| 34 | if (tag === "BR") { out.push({ x: "\n" }); continue; } |
| 35 | var f = Object.assign({}, fmt); |
| 36 | if (tag === "STRONG" || tag === "B") f.b = 1; |
| 37 | else if (tag === "EM" || tag === "I") f.i = 1; |
| 38 | else if (tag === "CODE" || tag === "KBD" || tag === "SAMP") f.c = 1; |
| 39 | else if (tag === "A") { var h = http(abs(n.getAttribute("href") || "")); if (h) f.href = h; } |
| 40 | inline(n, out, f); |
| 41 | } |
| 42 | } |
| 43 | } |
| 44 | function trimRuns(runs) { |
| 45 | while (runs.length && !runs[0].x.trim()) runs.shift(); |
| 46 | while (runs.length && !runs[runs.length - 1].x.trim()) runs.pop(); |
| 47 | if (runs.length) { runs[0].x = runs[0].x.replace(/^\s+/, ""); runs[runs.length - 1].x = runs[runs.length - 1].x.replace(/\s+$/, ""); } |
| 48 | return runs; |
| 49 | } |
| 50 | function paragraphFrom(nodes, out, type) { |
| 51 | var runs = []; |
| 52 | for (var i = 0; i < nodes.length; i++) inline({ childNodes: [nodes[i]] }, runs, {}); |
| 53 | runs = trimRuns(runs); |
| 54 | if (runs.length) out.push({ t: type || "p", runs: runs }); |
| 55 | } |
| 56 | function image(img, out) { |
| 57 | var src = http(abs(img.currentSrc || img.getAttribute("src") || img.getAttribute("data-src") || "")); |
| 58 | var w = parseInt(img.getAttribute("width") || "0", 10), h = parseInt(img.getAttribute("height") || "0", 10); |
| 59 | if (!src || (w && w < 60) || (h && h < 60)) return; // icons, spacers and tracking pixels |
| 60 | out.push({ t: "img", src: src, alt: img.getAttribute("alt") || "" }); |
| 61 | } |
| 62 | |
| 63 | function blocks(root, out) { |
| 64 | var kids = root.childNodes, run = []; |
| 65 | function flush() { if (run.length) { paragraphFrom(run, out); run = []; } } |
| 66 | for (var i = 0; i < kids.length; i++) { |
| 67 | if (out.length >= MAX_BLOCKS || total > MAX_CHARS) break; |
| 68 | var n = kids[i]; |
| 69 | if (n.nodeType === 3) { if (n.nodeValue.trim()) run.push(n); continue; } |
| 70 | if (n.nodeType !== 1) continue; |
| 71 | var tag = n.tagName; |
| 72 | if (SKIP.test(tag)) continue; |
| 73 | if (INLINE.test(tag)) { run.push(n); continue; } |
| 74 | flush(); |
| 75 | if (/^H[1-6]$/.test(tag)) { |
| 76 | var hr = []; inline(n, hr, {}); hr = trimRuns(hr); |
| 77 | if (hr.length) out.push({ t: "h", level: Number(tag.charAt(1)), runs: hr }); |
| 78 | } else if (tag === "P") { |
| 79 | paragraphFrom(n.childNodes, out); |
| 80 | var imgs = n.querySelectorAll("img"); for (var k = 0; k < imgs.length; k++) image(imgs[k], out); |
| 81 | } else if (tag === "UL" || tag === "OL") { |
| 82 | var items = [], lis = n.children; |
| 83 | for (var j = 0; j < lis.length; j++) if (lis[j].tagName === "LI") { var r = []; inline(lis[j], r, {}); r = trimRuns(r); if (r.length) items.push(r); } |
| 84 | if (items.length) out.push({ t: tag === "OL" ? "ol" : "ul", items: items }); |
| 85 | } else if (tag === "BLOCKQUOTE") { |
| 86 | paragraphFrom(n.childNodes, out, "quote"); |
| 87 | } else if (tag === "PRE") { |
| 88 | var text = n.textContent || ""; if (text.trim()) { out.push({ t: "pre", text: text }); total += text.length; } |
| 89 | } else if (tag === "IMG") { |
| 90 | image(n, out); |
| 91 | } else if (tag === "PICTURE") { |
| 92 | var pi = n.querySelector("img"); if (pi) image(pi, out); |
| 93 | } else if (tag === "HR") { |
| 94 | out.push({ t: "hr" }); |
| 95 | } else if (tag === "TABLE") { |
| 96 | var rows = n.querySelectorAll("tr"); |
| 97 | for (var q = 0; q < rows.length && out.length < MAX_BLOCKS; q++) { |
| 98 | var cells = rows[q].querySelectorAll("th,td"), parts = []; |
| 99 | for (var c = 0; c < cells.length; c++) { var t = (cells[c].textContent || "").replace(/\s+/g, " ").trim(); if (t) parts.push(t); } |
| 100 | if (parts.length) { out.push({ t: "p", runs: [{ x: parts.join(" | ") }] }); total += parts.join(" | ").length; } |
| 101 | } |
| 102 | } else { |
| 103 | blocks(n, out); // DIV, SECTION, ARTICLE, FIGURE, FIGCAPTION, MAIN, … |
| 104 | } |
| 105 | } |
| 106 | flush(); |
| 107 | } |
| 108 | |
| 109 | try { |
| 110 | var out = [], mode = "text", title = "", byline = "", siteName = "", lang = ""; |
| 111 | var art = null; |
| 112 | if (Readability) { |
| 113 | try { art = new Readability(doc.cloneNode(true), { charThreshold: 250 }).parse(); } catch (e) { art = null; } |
| 114 | } |
| 115 | if (art && art.textContent && art.textContent.trim().length > 400) { |
| 116 | var parsed = new DOMParser().parseFromString(art.content, "text/html"); |
| 117 | blocks(parsed.body, out); |
| 118 | mode = "article"; title = art.title || ""; byline = art.byline || ""; siteName = art.siteName || ""; lang = art.lang || ""; |
| 119 | } |
| 120 | if (out.length === 0) { |
| 121 | // Not an article (a home page, results, a dashboard…): the page's VISIBLE text — innerText skips what is hidden. |
| 122 | var root = doc.querySelector("main, article, [role=main]") || doc.body; |
| 123 | var parts = String(root.innerText || "").split(/\n\s*\n+/); |
| 124 | for (var p = 0; p < parts.length && out.length < MAX_BLOCKS && total < MAX_CHARS; p++) { |
| 125 | var s = parts[p].replace(/\s+/g, " ").trim(); |
| 126 | if (s) { out.push({ t: "p", runs: [{ x: s }] }); total += s.length; } |
| 127 | } |
| 128 | mode = "text"; |
| 129 | } |
| 130 | return { |
| 131 | mode: mode, url: location.href, title: title || doc.title || "", byline: byline, siteName: siteName || "", |
| 132 | lang: lang || doc.documentElement.lang || "", blocks: out |
| 133 | }; |
| 134 | } catch (e) { |
| 135 | return { error: "extract-failed" }; |
| 136 | } |
| 137 | }) |