askrobots / dbbasic-browser
Public
A desktop browser with x402 built in: when a page asks for payment, it pays in stablecoins and shows the content.
danq9
site icons (favicons) in tabs, address bar suggestions and the bookmarks panel
7249579
15 hours, 50 minutes ago
History
history.ts · 16.9 KB · 399 lines
| 1 | /** |
| 2 | * Browsing history: the pages you visit, kept on your askrobots.com account so you can find one later by the words you |
| 3 | * remember. Electron-free (main.ts tells it what the tabs do and gives it the connection), so it is unit-tested. |
| 4 | * |
| 5 | * Rules (the point of the design): |
| 6 | * - Only while signed in to askrobots with the "history" permission. Signed out: nothing is recorded, nothing kept. |
| 7 | * - Never recorded: anything but http(s); sign-in and payment pages (by address, by title, or a visible password field); |
| 8 | * this machine and the private network; sites on the never-record list; anything while paused or while the caller |
| 9 | * network is on. Glances shorter than MIN_SECONDS are not visits. |
| 10 | * - Addresses lose their fragment, user info and every query parameter except the few that say which page it is |
| 11 | * (YouTube's v and list, Hacker News' id). askrobots keeps the same list, as a backstop. |
| 12 | * - Text: the page's visible text, at most 50,000 bytes. |
| 13 | * - A visit is sent when the tab leaves the page, or once it has been open SEND_OPEN_AFTER seconds (a page you keep |
| 14 | * reading is still history; it is then sent with the seconds so far, once). Quitting sends every open page. |
| 15 | * - Visits wait in a small in-memory queue (sent in batches, kept while offline). Signing out or in again empties it, |
| 16 | * so one person's pages never go to another person's account. Forgetting also takes matching visits out of the queue. |
| 17 | * Pages already open when history switches on (signing in, resuming) count from that moment. |
| 18 | * - Nothing here goes to the search site: only the askrobots account's own history tools. |
| 19 | */ |
| 20 | |
| 21 | export const MIN_SECONDS = 2; |
| 22 | /** A page still open after this long is sent without waiting for the tab to leave it. */ |
| 23 | export const SEND_OPEN_AFTER = 30; |
| 24 | export const MAX_TEXT_BYTES = 50_000; |
| 25 | export const QUEUE_CAP = 1000; |
| 26 | export const BATCH = 100; |
| 27 | |
| 28 | /** Query parameters that say which page it is, per site (www. and m. removed). Every other parameter is dropped. */ |
| 29 | const KEEP_PARAMS: Record<string, string[]> = { |
| 30 | "youtube.com": ["v", "list"], |
| 31 | "news.ycombinator.com": ["id"], |
| 32 | }; |
| 33 | |
| 34 | /** The host without "www." or "m.", lower case. */ |
| 35 | export function bareHost(host: string): string { |
| 36 | return host.toLowerCase().replace(/^(www|m)\./, ""); |
| 37 | } |
| 38 | |
| 39 | /** The address as history keeps it, or null when it is not a web page. */ |
| 40 | export function cleanHistoryUrl(raw: string): string | null { |
| 41 | let u: URL; |
| 42 | try { |
| 43 | u = new URL(raw); |
| 44 | } catch { |
| 45 | return null; |
| 46 | } |
| 47 | if (u.protocol !== "http:" && u.protocol !== "https:") return null; |
| 48 | u.hash = ""; |
| 49 | u.username = ""; |
| 50 | u.password = ""; |
| 51 | const keep = KEEP_PARAMS[bareHost(u.hostname)] ?? []; |
| 52 | const kept = new URLSearchParams(); |
| 53 | for (const [k, v] of u.searchParams) if (keep.includes(k)) kept.append(k, v); |
| 54 | u.search = kept.toString(); |
| 55 | return u.href; |
| 56 | } |
| 57 | |
| 58 | const SIGN_IN_HOSTS = /^(login|signin|sso|auth|accounts|account|checkout|pay|payments|secure|billing)\./i; |
| 59 | const SIGN_IN_SITES = new Set(["accounts.google.com", "paypal.com", "stripe.com", "checkout.stripe.com", "appleid.apple.com", "login.microsoftonline.com", "login.live.com", "id.atlassian.com"]); |
| 60 | const SIGN_IN_PATHS = /\/(log-?in|sign-?in|sign-?up|oauth2?|authorize|sso|checkout|payment|payments|pay|billing|account\/security|password|reset-password|2fa|mfa)(\/|$|\.)/i; |
| 61 | const SIGN_IN_TITLES = /^\s*(sign in|log in|login|sign up|checkout|payment|pay|two-step verification)\b/i; |
| 62 | |
| 63 | /** This machine or the private network: never history (development servers, routers, printers). */ |
| 64 | function privateHost(host: string): boolean { |
| 65 | const h = host.replace(/^\[|\]$/g, "").toLowerCase(); |
| 66 | return h === "localhost" || h.endsWith(".localhost") || h.endsWith(".local") || h === "::1" || /^127\./.test(h) || /^10\./.test(h) || /^192\.168\./.test(h) || /^172\.(1[6-9]|2\d|3[01])\./.test(h) || /^169\.254\./.test(h) || /^f[cd][0-9a-f]{2}:/.test(h); |
| 67 | } |
| 68 | |
| 69 | /** Does this site (or a parent of it) appear on the never-record list? Entries are bare hosts and cover subdomains. */ |
| 70 | export function onNeverList(host: string, never: readonly string[]): boolean { |
| 71 | const h = bareHost(host); |
| 72 | return never.some((n) => { |
| 73 | const b = bareHost(n); |
| 74 | return b && (h === b || h.endsWith("." + b)); |
| 75 | }); |
| 76 | } |
| 77 | |
| 78 | export type SkipReason = "not-web" | "private-network" | "sign-in-or-payment" | "never-this-site"; |
| 79 | |
| 80 | /** Why a page is never recorded, or null when it may be. (Pausing, sign-in and time on page are decided by the service.) */ |
| 81 | export function skipReason(p: { url: string; title?: string; hasPassword?: boolean }, never: readonly string[] = [], allowPrivate = false): SkipReason | null { |
| 82 | let u: URL; |
| 83 | try { |
| 84 | u = new URL(p.url); |
| 85 | } catch { |
| 86 | return "not-web"; |
| 87 | } |
| 88 | if (u.protocol !== "http:" && u.protocol !== "https:") return "not-web"; |
| 89 | if (!allowPrivate && privateHost(u.hostname)) return "private-network"; |
| 90 | const host = u.hostname.toLowerCase(); |
| 91 | if (p.hasPassword || SIGN_IN_HOSTS.test(host) || SIGN_IN_SITES.has(bareHost(host)) || SIGN_IN_PATHS.test(u.pathname) || SIGN_IN_TITLES.test(p.title ?? "")) return "sign-in-or-payment"; |
| 92 | if (onNeverList(host, never)) return "never-this-site"; |
| 93 | return null; |
| 94 | } |
| 95 | |
| 96 | /** At most `max` bytes of UTF-8, cut on a character boundary, without NUL characters. */ |
| 97 | export function clipText(text: string, max = MAX_TEXT_BYTES): string { |
| 98 | const t = text.replace(/\u0000/g, ""); |
| 99 | if (Buffer.byteLength(t, "utf8") <= max) return t; |
| 100 | const b = Buffer.from(t, "utf8").subarray(0, max); |
| 101 | return b.toString("utf8").replace(/�$/, ""); |
| 102 | } |
| 103 | |
| 104 | export interface Visit { |
| 105 | url: string; |
| 106 | title?: string; |
| 107 | visited_at: string; |
| 108 | seconds_on_page: number; |
| 109 | text?: string; |
| 110 | } |
| 111 | |
| 112 | export interface HistoryState { |
| 113 | /** Signed in with the history permission: pages are being recorded (unless paused). */ |
| 114 | on: boolean; |
| 115 | /** Why it is off, in a few words, when it is. */ |
| 116 | why?: string; |
| 117 | paused: boolean; |
| 118 | neverSites: string[]; |
| 119 | queued: number; |
| 120 | sent: number; |
| 121 | /** The last time visits reached askrobots, and the last problem, if any. */ |
| 122 | lastSentAt?: number; |
| 123 | error?: string; |
| 124 | } |
| 125 | |
| 126 | /** |
| 127 | * A page without a title shows its address as the title, parameters and all. That is not a title (and would keep the |
| 128 | * parameters cleanHistoryUrl drops), so it becomes "". |
| 129 | */ |
| 130 | export function realTitle(title: string, rawUrl: string): string { |
| 131 | const t = title.trim(); |
| 132 | if (!t) return ""; |
| 133 | const bare = (u: string) => u.replace(/^[a-z][a-z0-9+.-]*:\/\//i, "").replace(/\/$/, ""); |
| 134 | if (bare(t) === bare(rawUrl) || /^[a-z][a-z0-9+.-]*:\/\//i.test(t) || bare(rawUrl).startsWith(bare(t)) && /[/:]/.test(t)) return ""; |
| 135 | return t; |
| 136 | } |
| 137 | |
| 138 | /** Which visits a forget covers (for the copy kept here: see history-copy.ts). `at` is when the visit was, in ms. */ |
| 139 | export function forgetMatch(scope: "url" | "last_hour" | "site" | "all", target: string | undefined, now: number): (v: { url: string; at: number }) => boolean { |
| 140 | if (scope === "all") return () => true; |
| 141 | if (scope === "last_hour") return (v) => v.at >= now - 3600_000; |
| 142 | if (scope === "url") { |
| 143 | const url = target ? cleanHistoryUrl(target) : null; |
| 144 | return (v) => !!url && v.url === url; |
| 145 | } |
| 146 | let site = ""; |
| 147 | try { |
| 148 | site = target ? bareHost(target.includes("://") ? new URL(target).hostname : target) : ""; |
| 149 | } catch { |
| 150 | site = ""; |
| 151 | } |
| 152 | return (v) => { |
| 153 | try { |
| 154 | return !!site && onNeverList(new URL(v.url).hostname, [site]); |
| 155 | } catch { |
| 156 | return false; |
| 157 | } |
| 158 | }; |
| 159 | } |
| 160 | |
| 161 | export interface HistoryDeps { |
| 162 | /** Call one of the account's history tools; resolves to the tool's text. */ |
| 163 | call(tool: string, args: Record<string, unknown>): Promise<string>; |
| 164 | /** Signed in, with the history permission granted. */ |
| 165 | connected(): boolean; |
| 166 | /** A short reason when not connected ("not signed in", "connect again to allow history"). */ |
| 167 | whyOff(): string; |
| 168 | callerNetworkOn(): boolean; |
| 169 | now(): number; |
| 170 | changed?(state: HistoryState): void; |
| 171 | /** Tests only: record pages on this machine (the stand-in server runs on 127.0.0.1). */ |
| 172 | allowPrivateHosts?: boolean; |
| 173 | } |
| 174 | |
| 175 | interface Open { |
| 176 | url: string; |
| 177 | /** The address as the tab had it (only to tell a missing title, which shows as the address, from a real one). */ |
| 178 | raw: string; |
| 179 | title: string; |
| 180 | startedAt: number; |
| 181 | text?: string; |
| 182 | hasPassword?: boolean; |
| 183 | /** Already queued while still open (see SEND_OPEN_AFTER); leaving it adds nothing more. */ |
| 184 | queued?: boolean; |
| 185 | } |
| 186 | |
| 187 | export class HistoryService { |
| 188 | private open = new Map<number, Open>(); |
| 189 | private queue: Visit[] = []; |
| 190 | private paused = false; |
| 191 | private never: string[] = []; |
| 192 | private sent = 0; |
| 193 | private lastSentAt: number | undefined; |
| 194 | private error: string | undefined; |
| 195 | private flushing: Promise<void> | null = null; |
| 196 | |
| 197 | constructor(private d: HistoryDeps) {} |
| 198 | |
| 199 | state(): HistoryState { |
| 200 | const on = this.d.connected() && !this.d.callerNetworkOn(); |
| 201 | return { |
| 202 | on, |
| 203 | ...(on ? {} : { why: this.d.callerNetworkOn() ? "off while the caller network is on" : this.d.whyOff() }), |
| 204 | paused: this.paused, |
| 205 | neverSites: [...this.never], |
| 206 | queued: this.queue.length, |
| 207 | sent: this.sent, |
| 208 | ...(this.lastSentAt ? { lastSentAt: this.lastSentAt } : {}), |
| 209 | ...(this.error ? { error: this.error } : {}), |
| 210 | }; |
| 211 | } |
| 212 | |
| 213 | private emit(): void { |
| 214 | this.d.changed?.(this.state()); |
| 215 | } |
| 216 | |
| 217 | /** A tab's main frame now shows this address (a new page, or the same page at a new address). */ |
| 218 | shown(tab: number, url: string, title = ""): void { |
| 219 | const clean = cleanHistoryUrl(url); |
| 220 | const cur = this.open.get(tab); |
| 221 | if (cur && clean && cur.url === clean) return; // the same page (a fragment change, a dropped parameter) |
| 222 | this.left(tab); |
| 223 | if (clean) this.open.set(tab, { url: clean, title: realTitle(title, url), startedAt: this.d.now(), raw: url }); |
| 224 | } |
| 225 | |
| 226 | title(tab: number, title: string): void { |
| 227 | const cur = this.open.get(tab); |
| 228 | if (cur) cur.title = realTitle(title, cur.raw); |
| 229 | } |
| 230 | |
| 231 | /** What the page says (its visible text), and whether a password field is showing. */ |
| 232 | text(tab: number, url: string, text: string, hasPassword: boolean): void { |
| 233 | const cur = this.open.get(tab); |
| 234 | if (!cur || cur.url !== cleanHistoryUrl(url)) return; // the tab has moved on |
| 235 | cur.text = clipText(text); |
| 236 | cur.hasPassword = cur.hasPassword || hasPassword; |
| 237 | } |
| 238 | |
| 239 | /** The tab moved away from its page or closed: the visit is complete. */ |
| 240 | left(tab: number): void { |
| 241 | const cur = this.open.get(tab); |
| 242 | if (!cur) return; |
| 243 | this.open.delete(tab); |
| 244 | if (this.enqueue(cur)) this.emit(); |
| 245 | } |
| 246 | |
| 247 | /** Every tab leaves its page (the browser is quitting). */ |
| 248 | leaveAll(): void { |
| 249 | for (const tab of [...this.open.keys()]) this.left(tab); |
| 250 | } |
| 251 | |
| 252 | /** Queue pages that have been open SEND_OPEN_AFTER seconds or more, once each. */ |
| 253 | private queueLongOpen(): void { |
| 254 | if (!this.d.connected() || this.d.callerNetworkOn() || this.paused) return; |
| 255 | let any = false; |
| 256 | for (const cur of this.open.values()) { |
| 257 | if (cur.queued || this.d.now() - cur.startedAt < SEND_OPEN_AFTER * 1000) continue; |
| 258 | if (this.enqueue(cur)) any = true; |
| 259 | cur.queued = true; // (also when skipped: the answer will not change while it stays open) |
| 260 | } |
| 261 | if (any) this.emit(); |
| 262 | } |
| 263 | |
| 264 | /** Pages open right now start counting again from this moment (history has just switched on). */ |
| 265 | private restartOpen(): void { |
| 266 | const now = this.d.now(); |
| 267 | for (const cur of this.open.values()) { |
| 268 | cur.startedAt = now; |
| 269 | cur.queued = false; |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | /** Add one visit to the queue if it is one we record. True when it was added. */ |
| 274 | private enqueue(cur: Open): boolean { |
| 275 | if (cur.queued) return false; |
| 276 | const seconds = Math.round((this.d.now() - cur.startedAt) / 1000); |
| 277 | if (seconds < MIN_SECONDS) return false; |
| 278 | if (!this.d.connected() || this.d.callerNetworkOn() || this.paused) return false; |
| 279 | if (skipReason({ url: cur.url, title: cur.title, hasPassword: !!cur.hasPassword }, this.never, this.d.allowPrivateHosts)) return false; |
| 280 | this.queue.push({ |
| 281 | url: cur.url, |
| 282 | ...(cur.title ? { title: cur.title.slice(0, 500) } : {}), |
| 283 | visited_at: new Date(cur.startedAt).toISOString(), |
| 284 | seconds_on_page: seconds, |
| 285 | ...(cur.text ? { text: cur.text } : {}), |
| 286 | }); |
| 287 | if (this.queue.length > QUEUE_CAP) this.queue.splice(0, this.queue.length - QUEUE_CAP); // the oldest go first |
| 288 | return true; |
| 289 | } |
| 290 | |
| 291 | /** Send what is waiting, in batches. Failures leave the visits queued for the next try. */ |
| 292 | flush(): Promise<void> { |
| 293 | if (this.flushing) return this.flushing; |
| 294 | this.queueLongOpen(); |
| 295 | // (The lock is cleared in .finally, which always runs after this assignment, even when there is nothing to send.) |
| 296 | this.flushing = this.sendQueued().finally(() => { |
| 297 | this.flushing = null; |
| 298 | this.emit(); |
| 299 | }); |
| 300 | return this.flushing; |
| 301 | } |
| 302 | |
| 303 | private async sendQueued(): Promise<void> { |
| 304 | while (this.queue.length && this.d.connected() && !this.d.callerNetworkOn()) { |
| 305 | const batch = this.queue.slice(0, BATCH); |
| 306 | let res: { accepted?: number; duplicates?: number; paused?: boolean; error?: string }; |
| 307 | try { |
| 308 | res = JSON.parse(await this.d.call("add_history", { visits: batch })) as typeof res; |
| 309 | } catch (e) { |
| 310 | this.error = e instanceof Error ? e.message.slice(0, 200) : String(e); |
| 311 | return; |
| 312 | } |
| 313 | if (res.error) { |
| 314 | // e.g. over the plan's storage: nothing in the batch was kept; keep it here and say so |
| 315 | this.error = String(res.error).slice(0, 200); |
| 316 | return; |
| 317 | } |
| 318 | // (A batch only leaves the queue once the account has it; anything added meanwhile stays behind it.) |
| 319 | this.queue.splice(0, batch.length); |
| 320 | this.sent += res.accepted ?? 0; |
| 321 | this.lastSentAt = this.d.now(); |
| 322 | this.error = undefined; |
| 323 | if (res.paused) this.paused = true; |
| 324 | } |
| 325 | } |
| 326 | |
| 327 | /** Signed out, or signed in again (perhaps as someone else): nothing waiting may go to the new account. */ |
| 328 | resetAccount(): void { |
| 329 | this.queue = []; |
| 330 | this.restartOpen(); // what the tabs show now is not lost, but nothing from before counts |
| 331 | this.sent = 0; |
| 332 | this.lastSentAt = undefined; |
| 333 | this.error = undefined; |
| 334 | this.emit(); |
| 335 | } |
| 336 | |
| 337 | /** Read pause and the never-record list from the account. */ |
| 338 | async loadSettings(): Promise<void> { |
| 339 | if (!this.d.connected()) return; |
| 340 | try { |
| 341 | this.applySettings(JSON.parse(await this.d.call("history_settings", {}))); |
| 342 | } catch (e) { |
| 343 | this.error = e instanceof Error ? e.message.slice(0, 200) : String(e); |
| 344 | this.emit(); |
| 345 | } |
| 346 | } |
| 347 | |
| 348 | private applySettings(s: { paused?: unknown; never_sites?: unknown; error?: unknown }): void { |
| 349 | if (typeof s.error === "string") throw new Error(s.error); |
| 350 | if (typeof s.paused === "boolean") this.paused = s.paused; |
| 351 | if (Array.isArray(s.never_sites)) this.never = s.never_sites.filter((x): x is string => typeof x === "string"); |
| 352 | this.emit(); |
| 353 | } |
| 354 | |
| 355 | /** Pause or resume, here and on the account. Pausing also drops the time spent so far on the pages open now. */ |
| 356 | async setPaused(paused: boolean): Promise<HistoryState> { |
| 357 | this.paused = paused; |
| 358 | this.restartOpen(); // paused: what is open now is not recorded; resumed: it counts from now |
| 359 | this.emit(); |
| 360 | if (this.d.connected()) this.applySettings(JSON.parse(await this.d.call("history_settings", { paused }))); |
| 361 | return this.state(); |
| 362 | } |
| 363 | |
| 364 | /** Never record this site again (it and its subdomains). Past visits stay until forgotten. */ |
| 365 | async neverRecord(host: string): Promise<HistoryState> { |
| 366 | const h = bareHost(host); |
| 367 | if (!h) return this.state(); |
| 368 | const list = [...new Set([...this.never, h])]; |
| 369 | this.never = list; |
| 370 | this.queue = this.queue.filter((v) => !onNeverList(new URL(v.url).hostname, [h])); |
| 371 | this.emit(); |
| 372 | if (this.d.connected()) this.applySettings(JSON.parse(await this.d.call("history_settings", { never_sites: list }))); |
| 373 | return this.state(); |
| 374 | } |
| 375 | |
| 376 | /** Forget: one page, the last hour, a site, or everything. Takes matching visits out of the queue too. */ |
| 377 | async forget(scope: "url" | "last_hour" | "site" | "all", target?: string): Promise<{ forgotten: number }> { |
| 378 | const hourAgo = this.d.now() - 3600_000; |
| 379 | const url = scope === "url" && target ? cleanHistoryUrl(target) : null; |
| 380 | const site = scope === "site" && target ? bareHost(target.includes("://") ? new URL(target).hostname : target) : null; |
| 381 | const gone = (v: Visit) => |
| 382 | scope === "all" || |
| 383 | (scope === "last_hour" && Date.parse(v.visited_at) >= hourAgo) || |
| 384 | (scope === "url" && v.url === url) || |
| 385 | (scope === "site" && !!site && onNeverList(new URL(v.url).hostname, [site])); |
| 386 | const before = this.queue.length; |
| 387 | this.queue = this.queue.filter((v) => !gone(v)); |
| 388 | if (scope === "all" || scope === "last_hour") this.open.clear(); |
| 389 | else for (const [tab, o] of this.open) if (gone({ url: o.url, visited_at: new Date(o.startedAt).toISOString(), seconds_on_page: 0 })) this.open.delete(tab); |
| 390 | let forgotten = before - this.queue.length; |
| 391 | if (this.d.connected()) { |
| 392 | const res = JSON.parse(await this.d.call("forget_history", { scope, ...(url ? { url } : {}), ...(site ? { site } : {}) })) as { forgotten?: number; error?: string }; |
| 393 | if (res.error) throw new Error(res.error); |
| 394 | forgotten += res.forgotten ?? 0; |
| 395 | } |
| 396 | this.emit(); |
| 397 | return { forgotten }; |
| 398 | } |
| 399 | } |