A desktop browser with x402 built in: when a page asks for payment, it pays in stablecoins and shows the content.

0 downloads

dbbasic-browser/apps/browser/src/reader-extractor.js
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-browser.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-browser.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-browser.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 site icons (favicons) in tabs, address bar suggestions and the bookmarks panel 7249579 15 hours, 50 minutes ago History
reader-extractor.js · 6.8 KB · 137 lines
1// Runs INSIDE the page being read (injected by the main process together with Mozilla's Readability).
2// It returns PLAIN DATA only — never HTML. The page is hostile and may tamper with this code, so the
3// main process re-validates everything (src/reader.ts validateExtraction) and builds the reader page itself.
4//
5// Shape: { mode: "article"|"text", url, title, byline, siteName, lang, blocks: [...] }
6// or { error: "login" | "extract-failed" }
7(function (Readability) {
8 "use strict";
9 var MAX_BLOCKS = 800, MAX_CHARS = 250000;
10 var total = 0;
11 var doc = document;
12 var SKIP = /^(SCRIPT|STYLE|NOSCRIPT|TEMPLATE|SVG|IFRAME|OBJECT|EMBED|SELECT|TEXTAREA|INPUT|CANVAS|VIDEO|AUDIO|LINK|META)$/;
13 var INLINE = /^(A|SPAN|STRONG|B|EM|I|CODE|KBD|SAMP|SMALL|MARK|ABBR|CITE|Q|SUB|SUP|U|S|DEL|INS|TIME|BDI|BDO|WBR)$/;
14
15 function visible(el) { return !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length); }
16 // Never read a login page: it may contain (or be about to contain) credentials.
17 var pw = doc.querySelectorAll('input[type="password"]');
18 for (var i = 0; i < pw.length; i++) if (visible(pw[i])) return { error: "login" };
19
20 function abs(u) { try { return new URL(u, doc.baseURI).href; } catch (e) { return ""; } }
21 function http(u) { return /^https?:/i.test(u) ? u : ""; }
22
23 function inline(node, out, fmt) {
24 var kids = node.childNodes;
25 for (var i = 0; i < kids.length; i++) {
26 if (total > MAX_CHARS) return;
27 var n = kids[i];
28 if (n.nodeType === 3) {
29 var x = n.nodeValue.replace(/\s+/g, " ");
30 if (x.trim() || (x === " " && out.length)) { out.push(Object.assign({ x: x }, fmt)); total += x.length; }
31 } else if (n.nodeType === 1) {
32 var tag = n.tagName;
33 if (SKIP.test(tag) || tag === "IMG") continue;
34 if (tag === "BR") { out.push({ x: "\n" }); continue; }
35 var f = Object.assign({}, fmt);
36 if (tag === "STRONG" || tag === "B") f.b = 1;
37 else if (tag === "EM" || tag === "I") f.i = 1;
38 else if (tag === "CODE" || tag === "KBD" || tag === "SAMP") f.c = 1;
39 else if (tag === "A") { var h = http(abs(n.getAttribute("href") || "")); if (h) f.href = h; }
40 inline(n, out, f);
41 }
42 }
43 }
44 function trimRuns(runs) {
45 while (runs.length && !runs[0].x.trim()) runs.shift();
46 while (runs.length && !runs[runs.length - 1].x.trim()) runs.pop();
47 if (runs.length) { runs[0].x = runs[0].x.replace(/^\s+/, ""); runs[runs.length - 1].x = runs[runs.length - 1].x.replace(/\s+$/, ""); }
48 return runs;
49 }
50 function paragraphFrom(nodes, out, type) {
51 var runs = [];
52 for (var i = 0; i < nodes.length; i++) inline({ childNodes: [nodes[i]] }, runs, {});
53 runs = trimRuns(runs);
54 if (runs.length) out.push({ t: type || "p", runs: runs });
55 }
56 function image(img, out) {
57 var src = http(abs(img.currentSrc || img.getAttribute("src") || img.getAttribute("data-src") || ""));
58 var w = parseInt(img.getAttribute("width") || "0", 10), h = parseInt(img.getAttribute("height") || "0", 10);
59 if (!src || (w && w < 60) || (h && h < 60)) return; // icons, spacers and tracking pixels
60 out.push({ t: "img", src: src, alt: img.getAttribute("alt") || "" });
61 }
62
63 function blocks(root, out) {
64 var kids = root.childNodes, run = [];
65 function flush() { if (run.length) { paragraphFrom(run, out); run = []; } }
66 for (var i = 0; i < kids.length; i++) {
67 if (out.length >= MAX_BLOCKS || total > MAX_CHARS) break;
68 var n = kids[i];
69 if (n.nodeType === 3) { if (n.nodeValue.trim()) run.push(n); continue; }
70 if (n.nodeType !== 1) continue;
71 var tag = n.tagName;
72 if (SKIP.test(tag)) continue;
73 if (INLINE.test(tag)) { run.push(n); continue; }
74 flush();
75 if (/^H[1-6]$/.test(tag)) {
76 var hr = []; inline(n, hr, {}); hr = trimRuns(hr);
77 if (hr.length) out.push({ t: "h", level: Number(tag.charAt(1)), runs: hr });
78 } else if (tag === "P") {
79 paragraphFrom(n.childNodes, out);
80 var imgs = n.querySelectorAll("img"); for (var k = 0; k < imgs.length; k++) image(imgs[k], out);
81 } else if (tag === "UL" || tag === "OL") {
82 var items = [], lis = n.children;
83 for (var j = 0; j < lis.length; j++) if (lis[j].tagName === "LI") { var r = []; inline(lis[j], r, {}); r = trimRuns(r); if (r.length) items.push(r); }
84 if (items.length) out.push({ t: tag === "OL" ? "ol" : "ul", items: items });
85 } else if (tag === "BLOCKQUOTE") {
86 paragraphFrom(n.childNodes, out, "quote");
87 } else if (tag === "PRE") {
88 var text = n.textContent || ""; if (text.trim()) { out.push({ t: "pre", text: text }); total += text.length; }
89 } else if (tag === "IMG") {
90 image(n, out);
91 } else if (tag === "PICTURE") {
92 var pi = n.querySelector("img"); if (pi) image(pi, out);
93 } else if (tag === "HR") {
94 out.push({ t: "hr" });
95 } else if (tag === "TABLE") {
96 var rows = n.querySelectorAll("tr");
97 for (var q = 0; q < rows.length && out.length < MAX_BLOCKS; q++) {
98 var cells = rows[q].querySelectorAll("th,td"), parts = [];
99 for (var c = 0; c < cells.length; c++) { var t = (cells[c].textContent || "").replace(/\s+/g, " ").trim(); if (t) parts.push(t); }
100 if (parts.length) { out.push({ t: "p", runs: [{ x: parts.join(" | ") }] }); total += parts.join(" | ").length; }
101 }
102 } else {
103 blocks(n, out); // DIV, SECTION, ARTICLE, FIGURE, FIGCAPTION, MAIN, …
104 }
105 }
106 flush();
107 }
108
109 try {
110 var out = [], mode = "text", title = "", byline = "", siteName = "", lang = "";
111 var art = null;
112 if (Readability) {
113 try { art = new Readability(doc.cloneNode(true), { charThreshold: 250 }).parse(); } catch (e) { art = null; }
114 }
115 if (art && art.textContent && art.textContent.trim().length > 400) {
116 var parsed = new DOMParser().parseFromString(art.content, "text/html");
117 blocks(parsed.body, out);
118 mode = "article"; title = art.title || ""; byline = art.byline || ""; siteName = art.siteName || ""; lang = art.lang || "";
119 }
120 if (out.length === 0) {
121 // Not an article (a home page, results, a dashboard…): the page's VISIBLE text — innerText skips what is hidden.
122 var root = doc.querySelector("main, article, [role=main]") || doc.body;
123 var parts = String(root.innerText || "").split(/\n\s*\n+/);
124 for (var p = 0; p < parts.length && out.length < MAX_BLOCKS && total < MAX_CHARS; p++) {
125 var s = parts[p].replace(/\s+/g, " ").trim();
126 if (s) { out.push({ t: "p", runs: [{ x: s }] }); total += s.length; }
127 }
128 mode = "text";
129 }
130 return {
131 mode: mode, url: location.href, title: title || doc.title || "", byline: byline, siteName: siteName || "",
132 lang: lang || doc.documentElement.lang || "", blocks: out
133 };
134 } catch (e) {
135 return { error: "extract-failed" };
136 }
137})