askrobots / dbbasic-browser
Public
A desktop browser with x402 built in: when a page asks for payment, it pays in stablecoins and shows the content.
danq9
site icons (favicons) in tabs, address bar suggestions and the bookmarks panel
7249579
15 hours, 50 minutes ago
History
chat-tools.ts · 16.3 KB · 293 lines
| 1 | /** |
| 2 | * The tools the chat can use on the open tabs. Each one is a thin layer over the browser's own automation methods |
| 3 | * (the same ones scripts use), run in-process through `run`. Reading tools just run; tools that change something say |
| 4 | * what they are about to do first (see chat-core.ts). In Auto mode only a short allow-list runs unasked: |
| 5 | * - switching tab, going back, keys that only move (Tab, arrows, Page Up/Down…), closing a tab, saving a tab as a bookmark, |
| 6 | * keeping a copy of a tab; |
| 7 | * - typing into an ordinary field and choosing in a dropdown; |
| 8 | * - clicking a link to a site that is already open, a checkbox or radio button, or a disclosure triangle. |
| 9 | * Everything else asks: any other click (a "Post" or "Accept" button need not say anything scary), Enter, opening an address on |
| 10 | * a site that is not open. Clicks that submit a form or are about paying, buying, deleting or sending are always marked risky. |
| 11 | * After the assistant has read private data from a connected account, typing and opening addresses ask in every mode. |
| 12 | * Fields for passwords, card numbers and one-time codes are refused outright. |
| 13 | */ |
| 14 | |
| 15 | import type { ToolDef } from "./chat-core.js"; |
| 16 | import { CHOOSE_JS, REF_INFO_JS } from "./page-tools.js"; |
| 17 | |
| 18 | export type Runner = (method: string, params: Record<string, unknown>) => Promise<unknown>; |
| 19 | |
| 20 | const DANGER = /\b(buy|pay|purchase|order|place|checkout|check out|delete|remove|submit|confirm|send|sign ?up|subscribe|donate|transfer|withdraw|unsubscribe|cancel (my|the) )/i; |
| 21 | const KEYS = ["Enter", "Tab", "Escape", "Backspace", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "PageUp", "PageDown", "Home", "End"]; |
| 22 | |
| 23 | interface RefInfo { |
| 24 | found: boolean; |
| 25 | tag?: string; |
| 26 | type?: string | null; |
| 27 | text?: string; |
| 28 | label?: string | null; |
| 29 | name?: string | null; |
| 30 | sensitive?: string | null; |
| 31 | submits?: boolean; |
| 32 | link?: string | null; |
| 33 | } |
| 34 | |
| 35 | const str = (input: Record<string, unknown>, k: string): string => { |
| 36 | const v = input[k]; |
| 37 | if (typeof v !== "string" || !v.trim()) throw new Error(`"${k}" (text) is required`); |
| 38 | return v; |
| 39 | }; |
| 40 | const tabParam = (input: Record<string, unknown>): Record<string, unknown> => (typeof input.tab === "number" ? { tab: input.tab } : {}); |
| 41 | const short = (s: string, n = 80) => (s.length > n ? `${s.slice(0, n - 1)}…` : s); |
| 42 | |
| 43 | /** Only plain http(s) addresses, with no embedded credentials. */ |
| 44 | export function webAddress(v: string): URL { |
| 45 | let u: URL; |
| 46 | try { |
| 47 | u = new URL(v); |
| 48 | } catch { |
| 49 | throw new Error(`"${short(v, 60)}" is not a web address`); |
| 50 | } |
| 51 | if ((u.protocol !== "https:" && u.protocol !== "http:") || u.username || u.password) throw new Error("only plain http(s) addresses without a login in them can be opened"); |
| 52 | return u; |
| 53 | } |
| 54 | |
| 55 | export function pageTools(run: Runner): ToolDef[] { |
| 56 | const info = async (input: Record<string, unknown>): Promise<RefInfo> => { |
| 57 | const ref = str(input, "ref"); |
| 58 | const r = (await run("page.eval", { js: REF_INFO_JS(ref), ...tabParam(input) })) as { value: RefInfo }; |
| 59 | return r.value; |
| 60 | }; |
| 61 | const needFound = (i: RefInfo): string | undefined => (i.found ? undefined : "that ref is not on the page any more; call list_fields or find again to get a fresh one"); |
| 62 | const nameOf = (i: RefInfo) => short(i.label || i.text || i.name || i.tag || "element", 60); |
| 63 | const openHosts = async (): Promise<Set<string>> => { |
| 64 | const tabs = (await run("tabs.list", {})) as Array<{ url: string }>; |
| 65 | const hosts = new Set<string>(); |
| 66 | for (const t of tabs) { |
| 67 | try { |
| 68 | hosts.add(new URL(t.url).host); |
| 69 | } catch { |
| 70 | /* about:blank and friends */ |
| 71 | } |
| 72 | } |
| 73 | return hosts; |
| 74 | }; |
| 75 | const tabProp = { tab: { type: "integer", description: "A tab id from list_tabs. Leave out for the tab the person is looking at." } }; |
| 76 | const refProp = { ref: { type: "string", description: "A ref from list_fields or find, such as e12." } }; |
| 77 | |
| 78 | return [ |
| 79 | { |
| 80 | name: "list_tabs", |
| 81 | description: "List the open tabs: id, address, title, and which one is active.", |
| 82 | input_schema: { type: "object", properties: {}, additionalProperties: false }, |
| 83 | risk: "read", |
| 84 | run: async () => (await run("tabs.list", {})) as unknown, |
| 85 | }, |
| 86 | { |
| 87 | name: "read_page", |
| 88 | description: "Read the visible text of a page. Long pages are cut, so for something specific prefer find.", |
| 89 | input_schema: { type: "object", properties: tabProp, additionalProperties: false }, |
| 90 | risk: "read", |
| 91 | run: async (input) => { |
| 92 | const tabs = (await run("tabs.list", {})) as Array<{ id: number; url: string; title: string; active: boolean }>; |
| 93 | const t = typeof input.tab === "number" ? tabs.find((x) => x.id === input.tab) : tabs.find((x) => x.active); |
| 94 | const r = (await run("page.text", tabParam(input))) as { text: string | null }; |
| 95 | return `Address: ${t?.url ?? ""}\nTitle: ${t?.title ?? ""}\n\n${r.text ?? ""}`; |
| 96 | }, |
| 97 | }, |
| 98 | { |
| 99 | name: "list_fields", |
| 100 | description: |
| 101 | "List the form fields and buttons on a page: label, type, options, current value, and a ref for each. Passwords, card numbers and codes are listed without their values. Use the refs with type_text, choose and click.", |
| 102 | input_schema: { type: "object", properties: { ...tabProp, includeHidden: { type: "boolean" } }, additionalProperties: false }, |
| 103 | risk: "read", |
| 104 | run: async (input) => run("page.forms", { ...tabParam(input), includeHidden: input.includeHidden === true }), |
| 105 | }, |
| 106 | { |
| 107 | name: "find", |
| 108 | description: "Find links, buttons and fields on a page by their words, label or role (link, button, textbox, checkbox…). Returns refs.", |
| 109 | input_schema: { type: "object", properties: { ...tabProp, text: { type: "string" }, label: { type: "string" }, role: { type: "string" } }, additionalProperties: false }, |
| 110 | risk: "read", |
| 111 | run: async (input) => run("page.find", { ...tabParam(input), ...(typeof input.text === "string" ? { text: input.text } : {}), ...(typeof input.label === "string" ? { label: input.label } : {}), ...(typeof input.role === "string" ? { role: input.role } : {}), limit: 20 }), |
| 112 | }, |
| 113 | { |
| 114 | name: "open_tab", |
| 115 | origin: "browser", |
| 116 | description: "Open an address in a new tab and wait for it to load.", |
| 117 | input_schema: { type: "object", properties: { url: { type: "string" } }, required: ["url"], additionalProperties: false }, |
| 118 | risk: "act", |
| 119 | plan: async (input) => { |
| 120 | const u = webAddress(str(input, "url")); |
| 121 | const known = (await openHosts()).has(u.host); |
| 122 | return { summary: `Open ${short(u.href, 160)} in a new tab`, danger: !known, carriesData: true }; |
| 123 | }, |
| 124 | run: async (input) => run("tabs.new", { url: webAddress(str(input, "url")).href }), |
| 125 | }, |
| 126 | { |
| 127 | name: "go_to", |
| 128 | origin: "browser", |
| 129 | description: "Go to an address in a tab and wait for it to load.", |
| 130 | input_schema: { type: "object", properties: { ...tabProp, url: { type: "string" } }, required: ["url"], additionalProperties: false }, |
| 131 | risk: "act", |
| 132 | plan: async (input) => { |
| 133 | const u = webAddress(str(input, "url")); |
| 134 | const known = (await openHosts()).has(u.host); |
| 135 | return { summary: `Go to ${short(u.href, 160)}`, danger: !known, safe: known, carriesData: true }; |
| 136 | }, |
| 137 | run: async (input) => run("page.goto", { ...tabParam(input), url: webAddress(str(input, "url")).href }), |
| 138 | }, |
| 139 | { |
| 140 | name: "back", |
| 141 | origin: "browser", |
| 142 | description: "Go back one page in a tab.", |
| 143 | input_schema: { type: "object", properties: tabProp, additionalProperties: false }, |
| 144 | risk: "act", |
| 145 | plan: async () => ({ summary: "Go back one page", safe: true }), |
| 146 | run: async (input) => run("page.back", tabParam(input)), |
| 147 | }, |
| 148 | { |
| 149 | name: "switch_tab", |
| 150 | origin: "browser", |
| 151 | description: "Bring a tab to the front.", |
| 152 | input_schema: { type: "object", properties: { tab: tabProp.tab }, required: ["tab"], additionalProperties: false }, |
| 153 | risk: "act", |
| 154 | plan: async (input) => ({ summary: `Switch to tab ${String(input.tab)}`, safe: true }), |
| 155 | run: async (input) => run("tabs.activate", tabParam(input)), |
| 156 | }, |
| 157 | { |
| 158 | name: "close_tab", |
| 159 | origin: "browser", |
| 160 | description: "Close a tab by its id from list_tabs. Use it to tidy up when the person asks; save what they want to keep first (save_bookmark).", |
| 161 | input_schema: { type: "object", properties: { tab: tabProp.tab }, required: ["tab"], additionalProperties: false }, |
| 162 | risk: "act", |
| 163 | plan: async (input) => { |
| 164 | const tabs = (await run("tabs.list", {})) as Array<{ id: number; url: string; title: string }>; |
| 165 | const t = tabs.find((x) => x.id === input.tab); |
| 166 | if (!t) return { summary: "", refuse: `there is no tab ${String(input.tab)}; call list_tabs` }; |
| 167 | let host = t.url; |
| 168 | try { |
| 169 | host = new URL(t.url).host; |
| 170 | } catch { |
| 171 | /* keep the address */ |
| 172 | } |
| 173 | return { summary: `Close tab “${short(t.title || host, 60)}” (${short(host, 40)})`, safe: true }; |
| 174 | }, |
| 175 | run: async (input) => run("tabs.close", tabParam(input)), |
| 176 | }, |
| 177 | { |
| 178 | name: "save_bookmark", |
| 179 | origin: "browser", |
| 180 | description: "Save a tab's page to the person's bookmarks (their saved links on askrobots). Leave out tab for the one they are looking at.", |
| 181 | input_schema: { type: "object", properties: tabProp, additionalProperties: false }, |
| 182 | risk: "act", |
| 183 | plan: async (input) => { |
| 184 | const tabs = (await run("tabs.list", {})) as Array<{ id: number; url: string; title: string; active: boolean }>; |
| 185 | const t = typeof input.tab === "number" ? tabs.find((x) => x.id === input.tab) : tabs.find((x) => x.active); |
| 186 | if (!t) return { summary: "", refuse: "there is no such tab; call list_tabs" }; |
| 187 | if (!/^https?:\/\//i.test(t.url)) return { summary: "", refuse: "only web pages can be saved" }; |
| 188 | return { summary: `Save “${short(t.title || t.url, 60)}” to your bookmarks (${short(t.url, 80)})`, safe: true, carriesData: true }; |
| 189 | }, |
| 190 | run: async (input) => { |
| 191 | const tabs = (await run("tabs.list", {})) as Array<{ id: number; url: string; title: string; active: boolean }>; |
| 192 | const t = typeof input.tab === "number" ? tabs.find((x) => x.id === input.tab) : tabs.find((x) => x.active); |
| 193 | if (!t) throw new Error("there is no such tab"); |
| 194 | const r = (await run("bookmarks.add", { url: t.url, title: t.title })) as { ok: boolean; already?: boolean; message?: string }; |
| 195 | if (!r.ok) throw new Error(r.message || "could not save it"); |
| 196 | return r.already ? "already in the bookmarks" : "saved"; |
| 197 | }, |
| 198 | }, |
| 199 | { |
| 200 | name: "keep_tab", |
| 201 | origin: "browser", |
| 202 | description: "Keep a copy of a tab's page on this computer (what is on screen and its text), save it as a bookmark when connected, and close the tab. For tidying up without losing anything.", |
| 203 | input_schema: { type: "object", properties: { ...tabProp, reason: { type: "string", description: "One line on why it was open, if the person said." } }, additionalProperties: false }, |
| 204 | risk: "act", |
| 205 | plan: async (input) => { |
| 206 | const tabs = (await run("tabs.list", {})) as Array<{ id: number; url: string; title: string; active: boolean }>; |
| 207 | const t = typeof input.tab === "number" ? tabs.find((x) => x.id === input.tab) : tabs.find((x) => x.active); |
| 208 | if (!t) return { summary: "", refuse: "there is no such tab; call list_tabs" }; |
| 209 | if (!/^https?:\/\//i.test(t.url)) return { summary: "", refuse: "only web pages can be kept" }; |
| 210 | return { summary: `Keep a copy of “${short(t.title || t.url, 60)}” and close the tab`, safe: true }; |
| 211 | }, |
| 212 | run: async (input) => { |
| 213 | const r = (await run("keep.tab", { ...tabParam(input), ...(typeof input.reason === "string" ? { reason: input.reason } : {}) })) as { ok: boolean; message?: string; bookmarked?: boolean }; |
| 214 | if (!r.ok) throw new Error(r.message || "could not keep it"); |
| 215 | return r.bookmarked ? "kept, saved as a bookmark, and closed" : "kept and closed"; |
| 216 | }, |
| 217 | }, |
| 218 | { |
| 219 | name: "click", |
| 220 | origin: "browser", |
| 221 | description: "Click a link, button, checkbox or radio button by its ref. A click that submits a form or is about buying, paying, deleting or sending is always asked about.", |
| 222 | input_schema: { type: "object", properties: { ...tabProp, ...refProp }, required: ["ref"], additionalProperties: false }, |
| 223 | risk: "act", |
| 224 | plan: async (input) => { |
| 225 | const i = await info(input); |
| 226 | const gone = needFound(i); |
| 227 | if (gone) return { summary: "", refuse: gone }; |
| 228 | const words = `${i.text ?? ""} ${i.label ?? ""}`; |
| 229 | const danger = i.submits === true || DANGER.test(words); |
| 230 | const kind = i.tag === "a" ? "link" : i.tag === "button" || i.type === "submit" ? "button" : (i.type ?? i.tag ?? "element"); |
| 231 | // Safe to run unasked: a link to a site that is already open, a checkbox or radio button, a disclosure triangle. |
| 232 | let toOpenSite = false; |
| 233 | if (i.link) { |
| 234 | try { |
| 235 | toOpenSite = (await openHosts()).has(new URL(i.link).host); |
| 236 | } catch { |
| 237 | /* not an address */ |
| 238 | } |
| 239 | } |
| 240 | const safe = !danger && ((i.tag === "a" && toOpenSite) || i.type === "checkbox" || i.type === "radio" || i.tag === "summary"); |
| 241 | return { summary: `Click ${kind} “${nameOf(i)}”${i.submits ? " (submits the form)" : ""}${i.link ? ` → ${short(i.link, 100)}` : ""}`, danger, safe }; |
| 242 | }, |
| 243 | run: async (input) => run("page.click", { ...tabParam(input), ref: str(input, "ref") }), |
| 244 | }, |
| 245 | { |
| 246 | name: "type_text", |
| 247 | origin: "browser", |
| 248 | description: "Type into a text field by its ref. By default it replaces what is there. Never for password, card or code fields.", |
| 249 | input_schema: { type: "object", properties: { ...tabProp, ...refProp, text: { type: "string" }, clear: { type: "boolean", description: "Replace the current value (default true)." } }, required: ["ref", "text"], additionalProperties: false }, |
| 250 | risk: "act", |
| 251 | plan: async (input) => { |
| 252 | const i = await info(input); |
| 253 | const gone = needFound(i); |
| 254 | if (gone) return { summary: "", refuse: gone }; |
| 255 | if (i.sensitive) return { summary: "", refuse: `“${nameOf(i)}” is a ${i.sensitive} field. I never fill those; the person has to type it themselves.` }; |
| 256 | return { summary: `Type “${short(String(input.text ?? ""), 80)}” into “${nameOf(i)}”`, safe: true, carriesData: true }; |
| 257 | }, |
| 258 | run: async (input) => run("page.type", { ...tabParam(input), ref: str(input, "ref"), text: String(input.text ?? ""), clear: input.clear !== false }), |
| 259 | }, |
| 260 | { |
| 261 | name: "choose", |
| 262 | origin: "browser", |
| 263 | description: "Choose an option in a dropdown by its ref and the option's value or words.", |
| 264 | input_schema: { type: "object", properties: { ...tabProp, ...refProp, option: { type: "string" } }, required: ["ref", "option"], additionalProperties: false }, |
| 265 | risk: "act", |
| 266 | plan: async (input) => { |
| 267 | const i = await info(input); |
| 268 | const gone = needFound(i); |
| 269 | if (gone) return { summary: "", refuse: gone }; |
| 270 | if (i.sensitive) return { summary: "", refuse: `“${nameOf(i)}” is a ${i.sensitive} field.` }; |
| 271 | return { summary: `Choose “${short(str(input, "option"), 60)}” in “${nameOf(i)}”`, safe: true }; |
| 272 | }, |
| 273 | run: async (input) => { |
| 274 | const r = (await run("page.eval", { js: CHOOSE_JS(str(input, "ref"), str(input, "option")), ...tabParam(input) })) as { value: { ok: boolean; why?: string; options?: string[]; chosen?: string } }; |
| 275 | if (!r.value.ok) throw new Error(r.value.why === "no-such-option" ? `no such option; the choices are: ${(r.value.options ?? []).join(", ")}` : `could not choose (${r.value.why})`); |
| 276 | return { chosen: r.value.chosen }; |
| 277 | }, |
| 278 | }, |
| 279 | { |
| 280 | name: "press_key", |
| 281 | origin: "browser", |
| 282 | description: `Press one key in the page: ${KEYS.join(", ")}. Enter is always asked about because it can submit a form.`, |
| 283 | input_schema: { type: "object", properties: { ...tabProp, key: { type: "string", enum: KEYS } }, required: ["key"], additionalProperties: false }, |
| 284 | risk: "act", |
| 285 | plan: async (input) => { |
| 286 | const k = str(input, "key"); |
| 287 | if (!KEYS.includes(k)) return { summary: "", refuse: `only these keys can be pressed: ${KEYS.join(", ")}` }; |
| 288 | return { summary: k === "Enter" ? "Press Enter (this can submit a form)" : `Press ${k}`, danger: k === "Enter", safe: k !== "Enter" }; |
| 289 | }, |
| 290 | run: async (input) => run("page.press", { ...tabParam(input), key: str(input, "key") }), |
| 291 | }, |
| 292 | ]; |
| 293 | } |