A desktop browser with x402 built in: when a page asks for payment, it pays in stablecoins and shows the content.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-browser.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-browser.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-browser.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 site icons (favicons) in tabs, address bar suggestions and the bookmarks panel 7249579 15 hours, 50 minutes ago History
history.ts · 16.9 KB · 399 lines
1/**
2 * Browsing history: the pages you visit, kept on your askrobots.com account so you can find one later by the words you
3 * remember. Electron-free (main.ts tells it what the tabs do and gives it the connection), so it is unit-tested.
4 *
5 * Rules (the point of the design):
6 * - Only while signed in to askrobots with the "history" permission. Signed out: nothing is recorded, nothing kept.
7 * - Never recorded: anything but http(s); sign-in and payment pages (by address, by title, or a visible password field);
8 * this machine and the private network; sites on the never-record list; anything while paused or while the caller
9 * network is on. Glances shorter than MIN_SECONDS are not visits.
10 * - Addresses lose their fragment, user info and every query parameter except the few that say which page it is
11 * (YouTube's v and list, Hacker News' id). askrobots keeps the same list, as a backstop.
12 * - Text: the page's visible text, at most 50,000 bytes.
13 * - A visit is sent when the tab leaves the page, or once it has been open SEND_OPEN_AFTER seconds (a page you keep
14 * reading is still history; it is then sent with the seconds so far, once). Quitting sends every open page.
15 * - Visits wait in a small in-memory queue (sent in batches, kept while offline). Signing out or in again empties it,
16 * so one person's pages never go to another person's account. Forgetting also takes matching visits out of the queue.
17 * Pages already open when history switches on (signing in, resuming) count from that moment.
18 * - Nothing here goes to the search site: only the askrobots account's own history tools.
19 */
20
21export const MIN_SECONDS = 2;
22/** A page still open after this long is sent without waiting for the tab to leave it. */
23export const SEND_OPEN_AFTER = 30;
24export const MAX_TEXT_BYTES = 50_000;
25export const QUEUE_CAP = 1000;
26export const BATCH = 100;
27
28/** Query parameters that say which page it is, per site (www. and m. removed). Every other parameter is dropped. */
29const KEEP_PARAMS: Record<string, string[]> = {
30 "youtube.com": ["v", "list"],
31 "news.ycombinator.com": ["id"],
32};
33
34/** The host without "www." or "m.", lower case. */
35export function bareHost(host: string): string {
36 return host.toLowerCase().replace(/^(www|m)\./, "");
37}
38
39/** The address as history keeps it, or null when it is not a web page. */
40export function cleanHistoryUrl(raw: string): string | null {
41 let u: URL;
42 try {
43 u = new URL(raw);
44 } catch {
45 return null;
46 }
47 if (u.protocol !== "http:" && u.protocol !== "https:") return null;
48 u.hash = "";
49 u.username = "";
50 u.password = "";
51 const keep = KEEP_PARAMS[bareHost(u.hostname)] ?? [];
52 const kept = new URLSearchParams();
53 for (const [k, v] of u.searchParams) if (keep.includes(k)) kept.append(k, v);
54 u.search = kept.toString();
55 return u.href;
56}
57
58const SIGN_IN_HOSTS = /^(login|signin|sso|auth|accounts|account|checkout|pay|payments|secure|billing)\./i;
59const SIGN_IN_SITES = new Set(["accounts.google.com", "paypal.com", "stripe.com", "checkout.stripe.com", "appleid.apple.com", "login.microsoftonline.com", "login.live.com", "id.atlassian.com"]);
60const SIGN_IN_PATHS = /\/(log-?in|sign-?in|sign-?up|oauth2?|authorize|sso|checkout|payment|payments|pay|billing|account\/security|password|reset-password|2fa|mfa)(\/|$|\.)/i;
61const SIGN_IN_TITLES = /^\s*(sign in|log in|login|sign up|checkout|payment|pay|two-step verification)\b/i;
62
63/** This machine or the private network: never history (development servers, routers, printers). */
64function privateHost(host: string): boolean {
65 const h = host.replace(/^\[|\]$/g, "").toLowerCase();
66 return h === "localhost" || h.endsWith(".localhost") || h.endsWith(".local") || h === "::1" || /^127\./.test(h) || /^10\./.test(h) || /^192\.168\./.test(h) || /^172\.(1[6-9]|2\d|3[01])\./.test(h) || /^169\.254\./.test(h) || /^f[cd][0-9a-f]{2}:/.test(h);
67}
68
69/** Does this site (or a parent of it) appear on the never-record list? Entries are bare hosts and cover subdomains. */
70export function onNeverList(host: string, never: readonly string[]): boolean {
71 const h = bareHost(host);
72 return never.some((n) => {
73 const b = bareHost(n);
74 return b && (h === b || h.endsWith("." + b));
75 });
76}
77
78export type SkipReason = "not-web" | "private-network" | "sign-in-or-payment" | "never-this-site";
79
80/** Why a page is never recorded, or null when it may be. (Pausing, sign-in and time on page are decided by the service.) */
81export function skipReason(p: { url: string; title?: string; hasPassword?: boolean }, never: readonly string[] = [], allowPrivate = false): SkipReason | null {
82 let u: URL;
83 try {
84 u = new URL(p.url);
85 } catch {
86 return "not-web";
87 }
88 if (u.protocol !== "http:" && u.protocol !== "https:") return "not-web";
89 if (!allowPrivate && privateHost(u.hostname)) return "private-network";
90 const host = u.hostname.toLowerCase();
91 if (p.hasPassword || SIGN_IN_HOSTS.test(host) || SIGN_IN_SITES.has(bareHost(host)) || SIGN_IN_PATHS.test(u.pathname) || SIGN_IN_TITLES.test(p.title ?? "")) return "sign-in-or-payment";
92 if (onNeverList(host, never)) return "never-this-site";
93 return null;
94}
95
96/** At most `max` bytes of UTF-8, cut on a character boundary, without NUL characters. */
97export function clipText(text: string, max = MAX_TEXT_BYTES): string {
98 const t = text.replace(/\u0000/g, "");
99 if (Buffer.byteLength(t, "utf8") <= max) return t;
100 const b = Buffer.from(t, "utf8").subarray(0, max);
101 return b.toString("utf8").replace(/�$/, "");
102}
103
104export interface Visit {
105 url: string;
106 title?: string;
107 visited_at: string;
108 seconds_on_page: number;
109 text?: string;
110}
111
112export interface HistoryState {
113 /** Signed in with the history permission: pages are being recorded (unless paused). */
114 on: boolean;
115 /** Why it is off, in a few words, when it is. */
116 why?: string;
117 paused: boolean;
118 neverSites: string[];
119 queued: number;
120 sent: number;
121 /** The last time visits reached askrobots, and the last problem, if any. */
122 lastSentAt?: number;
123 error?: string;
124}
125
126/**
127 * A page without a title shows its address as the title, parameters and all. That is not a title (and would keep the
128 * parameters cleanHistoryUrl drops), so it becomes "".
129 */
130export function realTitle(title: string, rawUrl: string): string {
131 const t = title.trim();
132 if (!t) return "";
133 const bare = (u: string) => u.replace(/^[a-z][a-z0-9+.-]*:\/\//i, "").replace(/\/$/, "");
134 if (bare(t) === bare(rawUrl) || /^[a-z][a-z0-9+.-]*:\/\//i.test(t) || bare(rawUrl).startsWith(bare(t)) && /[/:]/.test(t)) return "";
135 return t;
136}
137
138/** Which visits a forget covers (for the copy kept here: see history-copy.ts). `at` is when the visit was, in ms. */
139export function forgetMatch(scope: "url" | "last_hour" | "site" | "all", target: string | undefined, now: number): (v: { url: string; at: number }) => boolean {
140 if (scope === "all") return () => true;
141 if (scope === "last_hour") return (v) => v.at >= now - 3600_000;
142 if (scope === "url") {
143 const url = target ? cleanHistoryUrl(target) : null;
144 return (v) => !!url && v.url === url;
145 }
146 let site = "";
147 try {
148 site = target ? bareHost(target.includes("://") ? new URL(target).hostname : target) : "";
149 } catch {
150 site = "";
151 }
152 return (v) => {
153 try {
154 return !!site && onNeverList(new URL(v.url).hostname, [site]);
155 } catch {
156 return false;
157 }
158 };
159}
160
161export interface HistoryDeps {
162 /** Call one of the account's history tools; resolves to the tool's text. */
163 call(tool: string, args: Record<string, unknown>): Promise<string>;
164 /** Signed in, with the history permission granted. */
165 connected(): boolean;
166 /** A short reason when not connected ("not signed in", "connect again to allow history"). */
167 whyOff(): string;
168 callerNetworkOn(): boolean;
169 now(): number;
170 changed?(state: HistoryState): void;
171 /** Tests only: record pages on this machine (the stand-in server runs on 127.0.0.1). */
172 allowPrivateHosts?: boolean;
173}
174
175interface Open {
176 url: string;
177 /** The address as the tab had it (only to tell a missing title, which shows as the address, from a real one). */
178 raw: string;
179 title: string;
180 startedAt: number;
181 text?: string;
182 hasPassword?: boolean;
183 /** Already queued while still open (see SEND_OPEN_AFTER); leaving it adds nothing more. */
184 queued?: boolean;
185}
186
187export class HistoryService {
188 private open = new Map<number, Open>();
189 private queue: Visit[] = [];
190 private paused = false;
191 private never: string[] = [];
192 private sent = 0;
193 private lastSentAt: number | undefined;
194 private error: string | undefined;
195 private flushing: Promise<void> | null = null;
196
197 constructor(private d: HistoryDeps) {}
198
199 state(): HistoryState {
200 const on = this.d.connected() && !this.d.callerNetworkOn();
201 return {
202 on,
203 ...(on ? {} : { why: this.d.callerNetworkOn() ? "off while the caller network is on" : this.d.whyOff() }),
204 paused: this.paused,
205 neverSites: [...this.never],
206 queued: this.queue.length,
207 sent: this.sent,
208 ...(this.lastSentAt ? { lastSentAt: this.lastSentAt } : {}),
209 ...(this.error ? { error: this.error } : {}),
210 };
211 }
212
213 private emit(): void {
214 this.d.changed?.(this.state());
215 }
216
217 /** A tab's main frame now shows this address (a new page, or the same page at a new address). */
218 shown(tab: number, url: string, title = ""): void {
219 const clean = cleanHistoryUrl(url);
220 const cur = this.open.get(tab);
221 if (cur && clean && cur.url === clean) return; // the same page (a fragment change, a dropped parameter)
222 this.left(tab);
223 if (clean) this.open.set(tab, { url: clean, title: realTitle(title, url), startedAt: this.d.now(), raw: url });
224 }
225
226 title(tab: number, title: string): void {
227 const cur = this.open.get(tab);
228 if (cur) cur.title = realTitle(title, cur.raw);
229 }
230
231 /** What the page says (its visible text), and whether a password field is showing. */
232 text(tab: number, url: string, text: string, hasPassword: boolean): void {
233 const cur = this.open.get(tab);
234 if (!cur || cur.url !== cleanHistoryUrl(url)) return; // the tab has moved on
235 cur.text = clipText(text);
236 cur.hasPassword = cur.hasPassword || hasPassword;
237 }
238
239 /** The tab moved away from its page or closed: the visit is complete. */
240 left(tab: number): void {
241 const cur = this.open.get(tab);
242 if (!cur) return;
243 this.open.delete(tab);
244 if (this.enqueue(cur)) this.emit();
245 }
246
247 /** Every tab leaves its page (the browser is quitting). */
248 leaveAll(): void {
249 for (const tab of [...this.open.keys()]) this.left(tab);
250 }
251
252 /** Queue pages that have been open SEND_OPEN_AFTER seconds or more, once each. */
253 private queueLongOpen(): void {
254 if (!this.d.connected() || this.d.callerNetworkOn() || this.paused) return;
255 let any = false;
256 for (const cur of this.open.values()) {
257 if (cur.queued || this.d.now() - cur.startedAt < SEND_OPEN_AFTER * 1000) continue;
258 if (this.enqueue(cur)) any = true;
259 cur.queued = true; // (also when skipped: the answer will not change while it stays open)
260 }
261 if (any) this.emit();
262 }
263
264 /** Pages open right now start counting again from this moment (history has just switched on). */
265 private restartOpen(): void {
266 const now = this.d.now();
267 for (const cur of this.open.values()) {
268 cur.startedAt = now;
269 cur.queued = false;
270 }
271 }
272
273 /** Add one visit to the queue if it is one we record. True when it was added. */
274 private enqueue(cur: Open): boolean {
275 if (cur.queued) return false;
276 const seconds = Math.round((this.d.now() - cur.startedAt) / 1000);
277 if (seconds < MIN_SECONDS) return false;
278 if (!this.d.connected() || this.d.callerNetworkOn() || this.paused) return false;
279 if (skipReason({ url: cur.url, title: cur.title, hasPassword: !!cur.hasPassword }, this.never, this.d.allowPrivateHosts)) return false;
280 this.queue.push({
281 url: cur.url,
282 ...(cur.title ? { title: cur.title.slice(0, 500) } : {}),
283 visited_at: new Date(cur.startedAt).toISOString(),
284 seconds_on_page: seconds,
285 ...(cur.text ? { text: cur.text } : {}),
286 });
287 if (this.queue.length > QUEUE_CAP) this.queue.splice(0, this.queue.length - QUEUE_CAP); // the oldest go first
288 return true;
289 }
290
291 /** Send what is waiting, in batches. Failures leave the visits queued for the next try. */
292 flush(): Promise<void> {
293 if (this.flushing) return this.flushing;
294 this.queueLongOpen();
295 // (The lock is cleared in .finally, which always runs after this assignment, even when there is nothing to send.)
296 this.flushing = this.sendQueued().finally(() => {
297 this.flushing = null;
298 this.emit();
299 });
300 return this.flushing;
301 }
302
303 private async sendQueued(): Promise<void> {
304 while (this.queue.length && this.d.connected() && !this.d.callerNetworkOn()) {
305 const batch = this.queue.slice(0, BATCH);
306 let res: { accepted?: number; duplicates?: number; paused?: boolean; error?: string };
307 try {
308 res = JSON.parse(await this.d.call("add_history", { visits: batch })) as typeof res;
309 } catch (e) {
310 this.error = e instanceof Error ? e.message.slice(0, 200) : String(e);
311 return;
312 }
313 if (res.error) {
314 // e.g. over the plan's storage: nothing in the batch was kept; keep it here and say so
315 this.error = String(res.error).slice(0, 200);
316 return;
317 }
318 // (A batch only leaves the queue once the account has it; anything added meanwhile stays behind it.)
319 this.queue.splice(0, batch.length);
320 this.sent += res.accepted ?? 0;
321 this.lastSentAt = this.d.now();
322 this.error = undefined;
323 if (res.paused) this.paused = true;
324 }
325 }
326
327 /** Signed out, or signed in again (perhaps as someone else): nothing waiting may go to the new account. */
328 resetAccount(): void {
329 this.queue = [];
330 this.restartOpen(); // what the tabs show now is not lost, but nothing from before counts
331 this.sent = 0;
332 this.lastSentAt = undefined;
333 this.error = undefined;
334 this.emit();
335 }
336
337 /** Read pause and the never-record list from the account. */
338 async loadSettings(): Promise<void> {
339 if (!this.d.connected()) return;
340 try {
341 this.applySettings(JSON.parse(await this.d.call("history_settings", {})));
342 } catch (e) {
343 this.error = e instanceof Error ? e.message.slice(0, 200) : String(e);
344 this.emit();
345 }
346 }
347
348 private applySettings(s: { paused?: unknown; never_sites?: unknown; error?: unknown }): void {
349 if (typeof s.error === "string") throw new Error(s.error);
350 if (typeof s.paused === "boolean") this.paused = s.paused;
351 if (Array.isArray(s.never_sites)) this.never = s.never_sites.filter((x): x is string => typeof x === "string");
352 this.emit();
353 }
354
355 /** Pause or resume, here and on the account. Pausing also drops the time spent so far on the pages open now. */
356 async setPaused(paused: boolean): Promise<HistoryState> {
357 this.paused = paused;
358 this.restartOpen(); // paused: what is open now is not recorded; resumed: it counts from now
359 this.emit();
360 if (this.d.connected()) this.applySettings(JSON.parse(await this.d.call("history_settings", { paused })));
361 return this.state();
362 }
363
364 /** Never record this site again (it and its subdomains). Past visits stay until forgotten. */
365 async neverRecord(host: string): Promise<HistoryState> {
366 const h = bareHost(host);
367 if (!h) return this.state();
368 const list = [...new Set([...this.never, h])];
369 this.never = list;
370 this.queue = this.queue.filter((v) => !onNeverList(new URL(v.url).hostname, [h]));
371 this.emit();
372 if (this.d.connected()) this.applySettings(JSON.parse(await this.d.call("history_settings", { never_sites: list })));
373 return this.state();
374 }
375
376 /** Forget: one page, the last hour, a site, or everything. Takes matching visits out of the queue too. */
377 async forget(scope: "url" | "last_hour" | "site" | "all", target?: string): Promise<{ forgotten: number }> {
378 const hourAgo = this.d.now() - 3600_000;
379 const url = scope === "url" && target ? cleanHistoryUrl(target) : null;
380 const site = scope === "site" && target ? bareHost(target.includes("://") ? new URL(target).hostname : target) : null;
381 const gone = (v: Visit) =>
382 scope === "all" ||
383 (scope === "last_hour" && Date.parse(v.visited_at) >= hourAgo) ||
384 (scope === "url" && v.url === url) ||
385 (scope === "site" && !!site && onNeverList(new URL(v.url).hostname, [site]));
386 const before = this.queue.length;
387 this.queue = this.queue.filter((v) => !gone(v));
388 if (scope === "all" || scope === "last_hour") this.open.clear();
389 else for (const [tab, o] of this.open) if (gone({ url: o.url, visited_at: new Date(o.startedAt).toISOString(), seconds_on_page: 0 })) this.open.delete(tab);
390 let forgotten = before - this.queue.length;
391 if (this.d.connected()) {
392 const res = JSON.parse(await this.d.call("forget_history", { scope, ...(url ? { url } : {}), ...(site ? { site } : {}) })) as { forgotten?: number; error?: string };
393 if (res.error) throw new Error(res.error);
394 forgotten += res.forgotten ?? 0;
395 }
396 this.emit();
397 return { forgotten };
398 }
399}