askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_talk_voice.py · 19.3 KB · 450 lines
| 1 | """The talk page, actually run: WebKit, fake voice, intercepted network. |
| 2 | |
| 3 | Every /talk regression this week shipped past a green suite, because the |
| 4 | suite proved the page's source CONTAINS the code, never that the code |
| 5 | RUNS. A source-grep test cannot see a runtime error kill the script, a |
| 6 | function called before it exists, or a submit that never fires -- and |
| 7 | each of those reached a real iPad in turn. "you need to better figure |
| 8 | out how to test the various parts and actually test it" is a verbatim |
| 9 | user instruction, and this file is the answer. |
| 10 | |
| 11 | The harness: WebKit (the iPad's actual engine), the page's real rendered |
| 12 | HTML+JS, a fake webkitSpeechRecognition the test drives line by line, a |
| 13 | getUserMedia that REJECTS (the exclusive-microphone case every fix |
| 14 | orbited), and every network call intercepted -- prefs, history, chat, |
| 15 | TTS. Hermetic: no droplet, no provider, no key, no sound card. |
| 16 | |
| 17 | Lives in the e2e lane beside test_generated_ui.py, but deliberately does |
| 18 | NOT use its real-server fixture: the questions here are about the PAGE's |
| 19 | runtime behaviour (does the gate strip, does the submit fire, does the |
| 20 | unlocked element play), and intercepting the network makes each answer |
| 21 | exact -- the chat payload asserted verbatim, the empty-reply case forced |
| 22 | at will -- where a real provider would make them flaky and cost money. |
| 23 | The uvicorn lane covers the server half; between them the seam is the |
| 24 | HTTP contract, which tests/test_ai_turn_recording.py pins. |
| 25 | """ |
| 26 | |
| 27 | import importlib.util |
| 28 | import json |
| 29 | import pathlib |
| 30 | |
| 31 | import pytest |
| 32 | |
| 33 | pytestmark = pytest.mark.e2e |
| 34 | |
| 35 | REPO = pathlib.Path(__file__).resolve().parents[2] |
| 36 | TALK = REPO / "packages" / "app-shell" / "objects" / "site" / "talk.py" |
| 37 | |
| 38 | # A valid, near-empty WAV for the TTS route. |
| 39 | SILENT_WAV = (b"RIFF$\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00" |
| 40 | b'"V\x00\x00D\xac\x00\x00\x02\x00\x10\x00data\x00\x00\x00\x00') |
| 41 | |
| 42 | FAKE_VOICE = """ |
| 43 | // A SpeechRecognition the TEST drives. The page must not be able to tell |
| 44 | // the difference, and the test must see everything: instances register on |
| 45 | // window.__recos, and __speak(text, isFinal) feeds the newest one. |
| 46 | window.__recos = []; |
| 47 | window.__speak = (text, isFinal) => { |
| 48 | const r = window.__recos[window.__recos.length - 1]; |
| 49 | if (!r || !r.onresult) return; |
| 50 | r.onresult({results: [{0: {transcript: text}, isFinal: !!isFinal, |
| 51 | length: 1}], resultIndex: 0, |
| 52 | // the page iterates event.results |
| 53 | }); |
| 54 | }; |
| 55 | window.__endReco = () => { |
| 56 | const r = window.__recos[window.__recos.length - 1]; |
| 57 | if (r && r.onend) r.onend(); |
| 58 | }; |
| 59 | class FakeRecognition { |
| 60 | constructor() { window.__recos.push(this); this.started = 0; } |
| 61 | start() { this.started++; window.__recoStarts = (window.__recoStarts || 0) + 1; } |
| 62 | stop() { if (this.onend) setTimeout(() => this.onend(), 0); } |
| 63 | } |
| 64 | window.SpeechRecognition = FakeRecognition; |
| 65 | window.webkitSpeechRecognition = FakeRecognition; |
| 66 | |
| 67 | // The iPad case: the microphone is exclusive and the meter's second |
| 68 | // stream is refused. |
| 69 | if (navigator.mediaDevices) { |
| 70 | navigator.mediaDevices.getUserMedia = () => Promise.reject(new Error("NotAllowedError")); |
| 71 | } |
| 72 | |
| 73 | // Spy on media playback -- a headless browser has no speaker, so the |
| 74 | // PROOF of speech is that play() was called on the unlocked element. |
| 75 | window.__plays = []; |
| 76 | const realPlay = HTMLMediaElement.prototype.play; |
| 77 | HTMLMediaElement.prototype.play = function () { |
| 78 | window.__plays.push(this.src ? this.src.slice(0, 40) : "(no src)"); |
| 79 | const p = realPlay.apply(this); |
| 80 | if (p && p.catch) p.catch(() => {}); |
| 81 | // A spied play never really plays, so it must really END -- otherwise |
| 82 | // the page's `speaking` flag sticks and blocks everything after the |
| 83 | // first reply, which is exactly the stuck state the page now guards. |
| 84 | setTimeout(() => this.dispatchEvent(new Event("ended")), 40); |
| 85 | return Promise.resolve(); |
| 86 | }; |
| 87 | // speechSynthesis exists headless but must be observable too. |
| 88 | window.__spoken = []; |
| 89 | if (window.speechSynthesis) { |
| 90 | const realSpeak = window.speechSynthesis.speak.bind(window.speechSynthesis); |
| 91 | window.speechSynthesis.speak = (u) => { |
| 92 | window.__spoken.push(u.text); |
| 93 | setTimeout(() => u.dispatchEvent(new Event("end")), 20); |
| 94 | }; |
| 95 | } |
| 96 | """ |
| 97 | |
| 98 | |
| 99 | def render_talk_html(): |
| 100 | spec = importlib.util.spec_from_file_location("talk_under_test", TALK) |
| 101 | module = importlib.util.module_from_spec(spec) |
| 102 | spec.loader.exec_module(module) |
| 103 | |
| 104 | class _Null: |
| 105 | def info(self, *a, **k): pass |
| 106 | module._logger = _Null() |
| 107 | return module.GET({"_identity": {"user_id": "dan"}})["body"] |
| 108 | |
| 109 | |
| 110 | # FUNCTION-scoped, matching the house e2e convention (conftest.py's own |
| 111 | # `page` fixture) and not incidentally: a module-scoped page was tried |
| 112 | # first here and it leaked. A timer armed near the end of one test |
| 113 | # (the stability endpointer, ~1.4s) was still pending when that test |
| 114 | # returned, fired mid-WAY THROUGH THE NEXT TEST, and submitted a stale |
| 115 | # utterance that made an unrelated assertion fail in a way that looked |
| 116 | # exactly like the guard it was supposed to be proving. One page per |
| 117 | # test is slightly slower and is the only way this suite's own timers |
| 118 | # cannot contaminate each other -- the failure mode this harness exists |
| 119 | # to catch, caught in itself. |
| 120 | @pytest.fixture(scope="module") |
| 121 | def talk_browser(playwright): |
| 122 | # `playwright` is pytest-playwright's OWN session-scoped fixture, not |
| 123 | # a name we chose -- and using it instead of a hand-rolled |
| 124 | # `sync_playwright()` context is not a style preference, it is what |
| 125 | # makes this file coexist with test_generated_ui.py in one pytest |
| 126 | # session. That file uses the plugin's page/context/browser fixtures, |
| 127 | # which means the plugin already owns a Playwright driver for the |
| 128 | # process by the time this fixture runs. A second, independent |
| 129 | # `sync_playwright()` call collided with it -- passed locally (this |
| 130 | # file run alone), failed in CI (both files in the same `pytest -m |
| 131 | # e2e tests/e2e` invocation, exactly how the workflow runs it) with |
| 132 | # "Sync API inside the asyncio loop". One driver, reused via the |
| 133 | # fixture the plugin already provides, has no such collision. |
| 134 | browser = playwright.webkit.launch() |
| 135 | yield browser |
| 136 | browser.close() |
| 137 | |
| 138 | |
| 139 | @pytest.fixture |
| 140 | def talk_page(talk_browser): |
| 141 | html = render_talk_html() |
| 142 | ctx = talk_browser.new_context(viewport={"width": 810, "height": 1080}) |
| 143 | page = ctx.new_page() |
| 144 | page.add_init_script(FAKE_VOICE) |
| 145 | |
| 146 | state = {"chat_calls": [], "reply": "Here are your notes.", |
| 147 | "errors": []} |
| 148 | page.on("pageerror", lambda e: state["errors"].append(str(e))) |
| 149 | |
| 150 | def route_all(route): |
| 151 | url = route.request.url |
| 152 | if url.endswith("/talk") or "/talk?" in url: |
| 153 | route.fulfill(status=200, content_type="text/html; charset=utf-8", body=html) |
| 154 | elif "/api/ai/chat" in url: |
| 155 | # NEVER time.sleep() here to simulate network latency: a |
| 156 | # sync-API route handler runs on Playwright's one shared |
| 157 | # dispatch thread, and blocking it stalls every OTHER |
| 158 | # in-flight call too -- including whatever a concurrency |
| 159 | # test fires "shortly after". Race conditions in this |
| 160 | # harness are proven by firing calls in the SAME JS tick |
| 161 | # (see test_a_turn_in_flight_blocks_a_second_submit), never |
| 162 | # by an artificial server-side delay. |
| 163 | state["chat_calls"].append(json.loads(route.request.post_data or "{}")) |
| 164 | route.fulfill(status=200, content_type="application/json", |
| 165 | body=json.dumps({"status": "ok", "reply": state["reply"], |
| 166 | "tool_calls": []})) |
| 167 | elif "/api/tts" in url: |
| 168 | route.fulfill(status=200, content_type="audio/wav", body=SILENT_WAV) |
| 169 | elif "shell_preferences" in url: |
| 170 | route.fulfill(status=200, content_type="application/json", |
| 171 | body=json.dumps({"record": {"id": "dan", "talk_tts": "server"}})) |
| 172 | elif "shell_commands" in url: |
| 173 | route.fulfill(status=200, content_type="application/json", |
| 174 | body=json.dumps({"records": []})) |
| 175 | else: |
| 176 | route.fulfill(status=200, content_type="application/json", body="{}") |
| 177 | |
| 178 | page.route("**/*", route_all) |
| 179 | page.goto("http://talk.test/talk", wait_until="networkidle") |
| 180 | yield page, state |
| 181 | ctx.close() |
| 182 | |
| 183 | |
| 184 | def fresh_turn(page, state): |
| 185 | state["chat_calls"].clear() |
| 186 | state["errors"].clear() |
| 187 | page.evaluate("""() => { |
| 188 | window.__plays = []; window.__spoken = []; |
| 189 | const mic = document.getElementById('mic'); |
| 190 | // ensure conversation mode is ON exactly once for this turn |
| 191 | if (!mic.classList.contains('on')) mic.click(); |
| 192 | }""") |
| 193 | page.wait_for_timeout(120) |
| 194 | |
| 195 | |
| 196 | def say(page, text, *, final=True): |
| 197 | page.evaluate("([t, f]) => window.__speak(t, f)", [text, final]) |
| 198 | |
| 199 | |
| 200 | def test_the_page_runs_without_a_single_error(talk_page): |
| 201 | page, state = talk_page |
| 202 | assert state["errors"] == [] |
| 203 | assert page.evaluate("() => typeof submitTurn") == "function" |
| 204 | |
| 205 | |
| 206 | def test_the_full_voice_loop_speaks_the_reply(talk_page): |
| 207 | """Wake word -> command -> auto-submit -> reply caption -> SERVER TTS |
| 208 | played through the unlocked element. The whole pipeline, in a real |
| 209 | WebKit, with the exclusive-microphone case active.""" |
| 210 | page, state = talk_page |
| 211 | fresh_turn(page, state) |
| 212 | |
| 213 | say(page, "computer show me my notes", final=True) |
| 214 | page.wait_for_timeout(1400) # isFinal debounce (800ms) + settle |
| 215 | |
| 216 | assert len(state["chat_calls"]) == 1, state["errors"] |
| 217 | sent = state["chat_calls"][0] |
| 218 | assert sent["message"] == "show me my notes" # wake word stripped |
| 219 | assert sent["source"] == "talk" |
| 220 | assert sent["session_id"] |
| 221 | |
| 222 | caption = page.text_content("#capAssistant") |
| 223 | assert "Here are your notes." in caption |
| 224 | |
| 225 | plays = page.evaluate("() => window.__plays") |
| 226 | assert plays, "server TTS was fetched but never played" |
| 227 | assert state["errors"] == [] |
| 228 | |
| 229 | |
| 230 | def test_the_bare_wake_word_never_reaches_the_api(talk_page): |
| 231 | page, state = talk_page |
| 232 | fresh_turn(page, state) |
| 233 | say(page, "computer", final=True) |
| 234 | page.wait_for_timeout(1200) |
| 235 | assert state["chat_calls"] == [] |
| 236 | |
| 237 | |
| 238 | def test_a_fused_no_space_utterance_still_submits(talk_page): |
| 239 | """"comptureshowmemynotes", the literal iPad transcript: misheard |
| 240 | wake word fused onto the command with no spaces.""" |
| 241 | page, state = talk_page |
| 242 | fresh_turn(page, state) |
| 243 | say(page, "comptureshowmemynotes", final=True) |
| 244 | page.wait_for_timeout(1400) |
| 245 | assert len(state["chat_calls"]) == 1 |
| 246 | assert state["chat_calls"][0]["message"] == "showmemynotes" |
| 247 | |
| 248 | |
| 249 | def test_an_empty_reply_becomes_words_on_screen(talk_page): |
| 250 | page, state = talk_page |
| 251 | fresh_turn(page, state) |
| 252 | state["reply"] = "" |
| 253 | try: |
| 254 | say(page, "computer what is two plus two", final=True) |
| 255 | page.wait_for_timeout(1400) |
| 256 | caption = page.text_content("#capAssistant") |
| 257 | assert "came back empty" in caption |
| 258 | finally: |
| 259 | state["reply"] = "Here are your notes." |
| 260 | |
| 261 | |
| 262 | def test_the_endpoint_fallback_hint_is_shown(talk_page): |
| 263 | """getUserMedia rejects in this harness (the iPad case), so the page |
| 264 | must SAY it switched to send-when-you-finish.""" |
| 265 | page, state = talk_page |
| 266 | fresh_turn(page, state) |
| 267 | say(page, "computer hello there", final=True) |
| 268 | page.wait_for_timeout(1200) |
| 269 | hint = page.text_content("#endpointhint") |
| 270 | assert "level meter" in (hint or "") |
| 271 | |
| 272 | |
| 273 | def test_interim_results_render_a_live_caption(talk_page): |
| 274 | page, state = talk_page |
| 275 | fresh_turn(page, state) |
| 276 | say(page, "computer make a", final=False) |
| 277 | page.wait_for_timeout(150) |
| 278 | live = page.text_content("#capUser") |
| 279 | assert "make a" in (live or "") |
| 280 | # finish it so the module-scoped page is not left mid-utterance |
| 281 | say(page, "computer make a note", final=True) |
| 282 | page.wait_for_timeout(1300) |
| 283 | |
| 284 | |
| 285 | def test_desktop_with_a_dead_vad_still_submits(talk_page): |
| 286 | """THE iMac regression, encoded. On desktop the meter RUNS -- and a |
| 287 | meter started mid-speech calibrates on the voice itself, so its VAD |
| 288 | never fires: transcript on screen, listening forever, nothing sent, |
| 289 | no turn recorded (verified against the live box's shell_commands). |
| 290 | The stability endpointer must backstop even a running meter, because |
| 291 | a mis-calibrated one fails SILENT and the stalling transcript is the |
| 292 | one signal it cannot fake. |
| 293 | |
| 294 | Here the meter is simulated as running-but-useless, which is exactly |
| 295 | what a bad calibration is.""" |
| 296 | page, state = talk_page |
| 297 | fresh_turn(page, state) |
| 298 | page.evaluate("() => { meterRunning = true; }") |
| 299 | try: |
| 300 | say(page, "computer list my tasks", final=False) # interims only |
| 301 | page.wait_for_timeout(2500) # window + meter margin |
| 302 | assert len(state["chat_calls"]) == 1, "backstop did not fire past the meter" |
| 303 | assert state["chat_calls"][0]["message"] == "list my tasks" |
| 304 | finally: |
| 305 | page.evaluate("() => { meterRunning = false; }") |
| 306 | |
| 307 | |
| 308 | def test_the_meter_starts_at_the_click_so_it_calibrates_in_silence(talk_page): |
| 309 | """The root cause, pinned at the source level: calibration measures |
| 310 | the noise floor, so it must run BEFORE speech. Deferring the meter to |
| 311 | the first recognition result put calibration mid-utterance and |
| 312 | taught the threshold the user's own voice.""" |
| 313 | source = TALK.read_text() |
| 314 | click = source[source.index('mic.addEventListener("click"'):] |
| 315 | handler = click[:click.index("} else {")] |
| 316 | assert "startMeter()" in handler |
| 317 | onresult = source[source.index("recognizer.onresult"):] |
| 318 | onresult = onresult[:onresult.index("recognizer.onerror")] |
| 319 | assert "startMeter()" not in onresult |
| 320 | |
| 321 | |
| 322 | def test_an_utterance_submits_even_if_a_final_never_comes(talk_page): |
| 323 | """THE latency fix, and the regression it repairs. With the meter |
| 324 | convicted, end-of-utterance waited on iOS volunteering an isFinal -- |
| 325 | 2-6 seconds after you stop, sometimes never, and strictly WORSE than |
| 326 | the meter it replaced. The recognizer's interims are themselves a |
| 327 | silence detector: text that stops changing IS the silence. Here the |
| 328 | device sends interims only, never a final, and the turn still fires |
| 329 | after the silence window.""" |
| 330 | page, state = talk_page |
| 331 | fresh_turn(page, state) |
| 332 | say(page, "computer what time is it", final=False) # interim only |
| 333 | page.wait_for_timeout(2100) # silence window (1400ms) + margin |
| 334 | assert len(state["chat_calls"]) == 1, "no submit without a final" |
| 335 | assert state["chat_calls"][0]["message"] == "what time is it" |
| 336 | |
| 337 | |
| 338 | def test_continued_speech_keeps_postponing_the_submit(talk_page): |
| 339 | """Stability means STABLE: as long as the interim keeps growing, the |
| 340 | timer must keep resetting -- otherwise mid-sentence pauses truncate.""" |
| 341 | page, state = talk_page |
| 342 | fresh_turn(page, state) |
| 343 | say(page, "computer remind me", final=False) |
| 344 | page.wait_for_timeout(700) |
| 345 | say(page, "computer remind me to call the supplier", final=False) |
| 346 | page.wait_for_timeout(700) # 1400ms since FIRST, 700 since latest |
| 347 | assert state["chat_calls"] == [] # still talking; must not have fired |
| 348 | page.wait_for_timeout(1100) # now quiet past the window |
| 349 | assert len(state["chat_calls"]) == 1 |
| 350 | assert state["chat_calls"][0]["message"] == "remind me to call the supplier" |
| 351 | |
| 352 | |
| 353 | def test_talk_prefers_its_own_model_when_set(talk_page): |
| 354 | """Voice tolerates latency far worse than a terminal: a fast model |
| 355 | that answers in 3 seconds beats a smarter one that leaves 10 seconds |
| 356 | of silence. talk_model overrides ai_model for this surface only.""" |
| 357 | page, state = talk_page |
| 358 | page.evaluate("() => { prefs.talk_model = 'anthropic:claude-haiku-4-5'; }") |
| 359 | try: |
| 360 | fresh_turn(page, state) |
| 361 | say(page, "computer hello", final=True) |
| 362 | page.wait_for_timeout(900) |
| 363 | assert state["chat_calls"], "turn did not fire" |
| 364 | assert state["chat_calls"][0]["model"] == "anthropic:claude-haiku-4-5" |
| 365 | finally: |
| 366 | page.evaluate("() => { delete prefs.talk_model; }") |
| 367 | |
| 368 | |
| 369 | def test_a_turn_in_flight_blocks_a_second_submit(talk_page): |
| 370 | """The real bug, off a real session: "showed me my notes" and "show |
| 371 | me my notes" reached the server ONE SECOND apart -- two provider |
| 372 | calls billed for one utterance, the reply spoken twice. Nothing |
| 373 | serialized the paths that can produce an utterance (voice |
| 374 | endpointer, manual send), so a second one arriving while the first |
| 375 | is still in flight was never refused. |
| 376 | |
| 377 | Both calls fire in ONE evaluate, back to back, with no wait between |
| 378 | them -- deliberately not an artificial network delay (a Python-side |
| 379 | time.sleep() inside a sync-API route handler blocks Playwright's one |
| 380 | shared dispatch thread, which stalls every OTHER call including the |
| 381 | "second" one, and the two land far enough apart that the guard's |
| 382 | absence goes unnoticed; that false pass is what this comment now |
| 383 | warns off). submitTurn is async but sets the guard synchronously |
| 384 | before its first await, so two calls in the same JS tick already |
| 385 | race for real -- no delay required. |
| 386 | |
| 387 | Calling finalizeVoiceSubmit directly, twice, isolates the guarantee |
| 388 | from the choreography of WHICH timer or recognizer instance produced |
| 389 | the duplicate -- that cause matters less than the guarantee holding |
| 390 | regardless of cause.""" |
| 391 | page, state = talk_page |
| 392 | fresh_turn(page, state) |
| 393 | page.evaluate("""() => { |
| 394 | finalizeVoiceSubmit('show me my notes'); |
| 395 | finalizeVoiceSubmit('show me my notes'); |
| 396 | }""") |
| 397 | page.wait_for_timeout(400) |
| 398 | assert len(state["chat_calls"]) == 1, state["chat_calls"] |
| 399 | |
| 400 | |
| 401 | def test_a_failed_turn_releases_the_guard_for_the_next_one(talk_page): |
| 402 | """The guard must not survive its own failure -- a dropped connection |
| 403 | on turn one must not wedge the page shut for the rest of the |
| 404 | session.""" |
| 405 | page, state = talk_page |
| 406 | fresh_turn(page, state) |
| 407 | |
| 408 | def fail_once(route): |
| 409 | route.abort() |
| 410 | page.route("**/api/ai/chat", fail_once, times=1) |
| 411 | page.evaluate("() => finalizeVoiceSubmit('this one fails')") |
| 412 | page.wait_for_timeout(400) |
| 413 | |
| 414 | page.evaluate("() => finalizeVoiceSubmit('this one should work')") |
| 415 | page.wait_for_timeout(400) |
| 416 | assert len(state["chat_calls"]) == 1 |
| 417 | assert state["chat_calls"][0]["message"] == "this one should work" |
| 418 | |
| 419 | |
| 420 | def test_playback_waits_for_the_microphone_to_actually_release(talk_page): |
| 421 | """Dan's own diagnosis, verbatim: "listening while talking, maybe not |
| 422 | allowed." The mic and speaker share one audio session on iOS, and |
| 423 | recognizer.stop() is a REQUEST -- teardown finishes asynchronously, |
| 424 | signalled by onend. Starting playback before that finishes can play |
| 425 | into a still-recording session and come out silent, with no error |
| 426 | anywhere. speak() must wait for the release, not race it.""" |
| 427 | page, state = talk_page |
| 428 | fresh_turn(page, state) |
| 429 | page.evaluate("() => { window.__recoStarts = 0; }") |
| 430 | |
| 431 | say(page, "computer say something", final=True) |
| 432 | page.wait_for_timeout(1200) |
| 433 | |
| 434 | plays = page.evaluate("() => window.__plays") |
| 435 | assert plays, "server TTS never played" |
| 436 | |
| 437 | |
| 438 | def test_stop_listening_and_wait_resolves_even_if_onend_never_fires(talk_page): |
| 439 | """The safety timeout: a recognizer that was never actually listening |
| 440 | (stop() on an idle recognizer) must not hang speak() forever.""" |
| 441 | page, state = talk_page |
| 442 | fresh_turn(page, state) |
| 443 | resolved = page.evaluate("""async () => { |
| 444 | const r = window.__recos[window.__recos.length - 1]; |
| 445 | if (r && r.onend) { r.stop = () => {}; } // stop() that fires nothing |
| 446 | const started = performance.now(); |
| 447 | await stopListeningAndWaitForRelease(); |
| 448 | return performance.now() - started; |
| 449 | }""") |
| 450 | assert resolved < 1000 # bounded by the safety timeout, not hung |