Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/e2e/test_talk_voice.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_talk_voice.py · 19.3 KB · 450 lines
1"""The talk page, actually run: WebKit, fake voice, intercepted network.
2
3Every /talk regression this week shipped past a green suite, because the
4suite proved the page's source CONTAINS the code, never that the code
5RUNS. A source-grep test cannot see a runtime error kill the script, a
6function called before it exists, or a submit that never fires -- and
7each of those reached a real iPad in turn. "you need to better figure
8out how to test the various parts and actually test it" is a verbatim
9user instruction, and this file is the answer.
10
11The harness: WebKit (the iPad's actual engine), the page's real rendered
12HTML+JS, a fake webkitSpeechRecognition the test drives line by line, a
13getUserMedia that REJECTS (the exclusive-microphone case every fix
14orbited), and every network call intercepted -- prefs, history, chat,
15TTS. Hermetic: no droplet, no provider, no key, no sound card.
16
17Lives in the e2e lane beside test_generated_ui.py, but deliberately does
18NOT use its real-server fixture: the questions here are about the PAGE's
19runtime behaviour (does the gate strip, does the submit fire, does the
20unlocked element play), and intercepting the network makes each answer
21exact -- the chat payload asserted verbatim, the empty-reply case forced
22at will -- where a real provider would make them flaky and cost money.
23The uvicorn lane covers the server half; between them the seam is the
24HTTP contract, which tests/test_ai_turn_recording.py pins.
25"""
26
27import importlib.util
28import json
29import pathlib
30
31import pytest
32
33pytestmark = pytest.mark.e2e
34
35REPO = pathlib.Path(__file__).resolve().parents[2]
36TALK = REPO / "packages" / "app-shell" / "objects" / "site" / "talk.py"
37
38# A valid, near-empty WAV for the TTS route.
39SILENT_WAV = (b"RIFF$\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00"
40 b'"V\x00\x00D\xac\x00\x00\x02\x00\x10\x00data\x00\x00\x00\x00')
41
42FAKE_VOICE = """
43// A SpeechRecognition the TEST drives. The page must not be able to tell
44// the difference, and the test must see everything: instances register on
45// window.__recos, and __speak(text, isFinal) feeds the newest one.
46window.__recos = [];
47window.__speak = (text, isFinal) => {
48 const r = window.__recos[window.__recos.length - 1];
49 if (!r || !r.onresult) return;
50 r.onresult({results: [{0: {transcript: text}, isFinal: !!isFinal,
51 length: 1}], resultIndex: 0,
52 // the page iterates event.results
53 });
54};
55window.__endReco = () => {
56 const r = window.__recos[window.__recos.length - 1];
57 if (r && r.onend) r.onend();
58};
59class FakeRecognition {
60 constructor() { window.__recos.push(this); this.started = 0; }
61 start() { this.started++; window.__recoStarts = (window.__recoStarts || 0) + 1; }
62 stop() { if (this.onend) setTimeout(() => this.onend(), 0); }
63}
64window.SpeechRecognition = FakeRecognition;
65window.webkitSpeechRecognition = FakeRecognition;
66
67// The iPad case: the microphone is exclusive and the meter's second
68// stream is refused.
69if (navigator.mediaDevices) {
70 navigator.mediaDevices.getUserMedia = () => Promise.reject(new Error("NotAllowedError"));
71}
72
73// Spy on media playback -- a headless browser has no speaker, so the
74// PROOF of speech is that play() was called on the unlocked element.
75window.__plays = [];
76const realPlay = HTMLMediaElement.prototype.play;
77HTMLMediaElement.prototype.play = function () {
78 window.__plays.push(this.src ? this.src.slice(0, 40) : "(no src)");
79 const p = realPlay.apply(this);
80 if (p && p.catch) p.catch(() => {});
81 // A spied play never really plays, so it must really END -- otherwise
82 // the page's `speaking` flag sticks and blocks everything after the
83 // first reply, which is exactly the stuck state the page now guards.
84 setTimeout(() => this.dispatchEvent(new Event("ended")), 40);
85 return Promise.resolve();
86};
87// speechSynthesis exists headless but must be observable too.
88window.__spoken = [];
89if (window.speechSynthesis) {
90 const realSpeak = window.speechSynthesis.speak.bind(window.speechSynthesis);
91 window.speechSynthesis.speak = (u) => {
92 window.__spoken.push(u.text);
93 setTimeout(() => u.dispatchEvent(new Event("end")), 20);
94 };
95}
96"""
97
98
99def render_talk_html():
100 spec = importlib.util.spec_from_file_location("talk_under_test", TALK)
101 module = importlib.util.module_from_spec(spec)
102 spec.loader.exec_module(module)
103
104 class _Null:
105 def info(self, *a, **k): pass
106 module._logger = _Null()
107 return module.GET({"_identity": {"user_id": "dan"}})["body"]
108
109
110# FUNCTION-scoped, matching the house e2e convention (conftest.py's own
111# `page` fixture) and not incidentally: a module-scoped page was tried
112# first here and it leaked. A timer armed near the end of one test
113# (the stability endpointer, ~1.4s) was still pending when that test
114# returned, fired mid-WAY THROUGH THE NEXT TEST, and submitted a stale
115# utterance that made an unrelated assertion fail in a way that looked
116# exactly like the guard it was supposed to be proving. One page per
117# test is slightly slower and is the only way this suite's own timers
118# cannot contaminate each other -- the failure mode this harness exists
119# to catch, caught in itself.
120@pytest.fixture(scope="module")
121def talk_browser(playwright):
122 # `playwright` is pytest-playwright's OWN session-scoped fixture, not
123 # a name we chose -- and using it instead of a hand-rolled
124 # `sync_playwright()` context is not a style preference, it is what
125 # makes this file coexist with test_generated_ui.py in one pytest
126 # session. That file uses the plugin's page/context/browser fixtures,
127 # which means the plugin already owns a Playwright driver for the
128 # process by the time this fixture runs. A second, independent
129 # `sync_playwright()` call collided with it -- passed locally (this
130 # file run alone), failed in CI (both files in the same `pytest -m
131 # e2e tests/e2e` invocation, exactly how the workflow runs it) with
132 # "Sync API inside the asyncio loop". One driver, reused via the
133 # fixture the plugin already provides, has no such collision.
134 browser = playwright.webkit.launch()
135 yield browser
136 browser.close()
137
138
139@pytest.fixture
140def talk_page(talk_browser):
141 html = render_talk_html()
142 ctx = talk_browser.new_context(viewport={"width": 810, "height": 1080})
143 page = ctx.new_page()
144 page.add_init_script(FAKE_VOICE)
145
146 state = {"chat_calls": [], "reply": "Here are your notes.",
147 "errors": []}
148 page.on("pageerror", lambda e: state["errors"].append(str(e)))
149
150 def route_all(route):
151 url = route.request.url
152 if url.endswith("/talk") or "/talk?" in url:
153 route.fulfill(status=200, content_type="text/html; charset=utf-8", body=html)
154 elif "/api/ai/chat" in url:
155 # NEVER time.sleep() here to simulate network latency: a
156 # sync-API route handler runs on Playwright's one shared
157 # dispatch thread, and blocking it stalls every OTHER
158 # in-flight call too -- including whatever a concurrency
159 # test fires "shortly after". Race conditions in this
160 # harness are proven by firing calls in the SAME JS tick
161 # (see test_a_turn_in_flight_blocks_a_second_submit), never
162 # by an artificial server-side delay.
163 state["chat_calls"].append(json.loads(route.request.post_data or "{}"))
164 route.fulfill(status=200, content_type="application/json",
165 body=json.dumps({"status": "ok", "reply": state["reply"],
166 "tool_calls": []}))
167 elif "/api/tts" in url:
168 route.fulfill(status=200, content_type="audio/wav", body=SILENT_WAV)
169 elif "shell_preferences" in url:
170 route.fulfill(status=200, content_type="application/json",
171 body=json.dumps({"record": {"id": "dan", "talk_tts": "server"}}))
172 elif "shell_commands" in url:
173 route.fulfill(status=200, content_type="application/json",
174 body=json.dumps({"records": []}))
175 else:
176 route.fulfill(status=200, content_type="application/json", body="{}")
177
178 page.route("**/*", route_all)
179 page.goto("http://talk.test/talk", wait_until="networkidle")
180 yield page, state
181 ctx.close()
182
183
184def fresh_turn(page, state):
185 state["chat_calls"].clear()
186 state["errors"].clear()
187 page.evaluate("""() => {
188 window.__plays = []; window.__spoken = [];
189 const mic = document.getElementById('mic');
190 // ensure conversation mode is ON exactly once for this turn
191 if (!mic.classList.contains('on')) mic.click();
192 }""")
193 page.wait_for_timeout(120)
194
195
196def say(page, text, *, final=True):
197 page.evaluate("([t, f]) => window.__speak(t, f)", [text, final])
198
199
200def test_the_page_runs_without_a_single_error(talk_page):
201 page, state = talk_page
202 assert state["errors"] == []
203 assert page.evaluate("() => typeof submitTurn") == "function"
204
205
206def test_the_full_voice_loop_speaks_the_reply(talk_page):
207 """Wake word -> command -> auto-submit -> reply caption -> SERVER TTS
208 played through the unlocked element. The whole pipeline, in a real
209 WebKit, with the exclusive-microphone case active."""
210 page, state = talk_page
211 fresh_turn(page, state)
212
213 say(page, "computer show me my notes", final=True)
214 page.wait_for_timeout(1400) # isFinal debounce (800ms) + settle
215
216 assert len(state["chat_calls"]) == 1, state["errors"]
217 sent = state["chat_calls"][0]
218 assert sent["message"] == "show me my notes" # wake word stripped
219 assert sent["source"] == "talk"
220 assert sent["session_id"]
221
222 caption = page.text_content("#capAssistant")
223 assert "Here are your notes." in caption
224
225 plays = page.evaluate("() => window.__plays")
226 assert plays, "server TTS was fetched but never played"
227 assert state["errors"] == []
228
229
230def test_the_bare_wake_word_never_reaches_the_api(talk_page):
231 page, state = talk_page
232 fresh_turn(page, state)
233 say(page, "computer", final=True)
234 page.wait_for_timeout(1200)
235 assert state["chat_calls"] == []
236
237
238def test_a_fused_no_space_utterance_still_submits(talk_page):
239 """"comptureshowmemynotes", the literal iPad transcript: misheard
240 wake word fused onto the command with no spaces."""
241 page, state = talk_page
242 fresh_turn(page, state)
243 say(page, "comptureshowmemynotes", final=True)
244 page.wait_for_timeout(1400)
245 assert len(state["chat_calls"]) == 1
246 assert state["chat_calls"][0]["message"] == "showmemynotes"
247
248
249def test_an_empty_reply_becomes_words_on_screen(talk_page):
250 page, state = talk_page
251 fresh_turn(page, state)
252 state["reply"] = ""
253 try:
254 say(page, "computer what is two plus two", final=True)
255 page.wait_for_timeout(1400)
256 caption = page.text_content("#capAssistant")
257 assert "came back empty" in caption
258 finally:
259 state["reply"] = "Here are your notes."
260
261
262def test_the_endpoint_fallback_hint_is_shown(talk_page):
263 """getUserMedia rejects in this harness (the iPad case), so the page
264 must SAY it switched to send-when-you-finish."""
265 page, state = talk_page
266 fresh_turn(page, state)
267 say(page, "computer hello there", final=True)
268 page.wait_for_timeout(1200)
269 hint = page.text_content("#endpointhint")
270 assert "level meter" in (hint or "")
271
272
273def test_interim_results_render_a_live_caption(talk_page):
274 page, state = talk_page
275 fresh_turn(page, state)
276 say(page, "computer make a", final=False)
277 page.wait_for_timeout(150)
278 live = page.text_content("#capUser")
279 assert "make a" in (live or "")
280 # finish it so the module-scoped page is not left mid-utterance
281 say(page, "computer make a note", final=True)
282 page.wait_for_timeout(1300)
283
284
285def test_desktop_with_a_dead_vad_still_submits(talk_page):
286 """THE iMac regression, encoded. On desktop the meter RUNS -- and a
287 meter started mid-speech calibrates on the voice itself, so its VAD
288 never fires: transcript on screen, listening forever, nothing sent,
289 no turn recorded (verified against the live box's shell_commands).
290 The stability endpointer must backstop even a running meter, because
291 a mis-calibrated one fails SILENT and the stalling transcript is the
292 one signal it cannot fake.
293
294 Here the meter is simulated as running-but-useless, which is exactly
295 what a bad calibration is."""
296 page, state = talk_page
297 fresh_turn(page, state)
298 page.evaluate("() => { meterRunning = true; }")
299 try:
300 say(page, "computer list my tasks", final=False) # interims only
301 page.wait_for_timeout(2500) # window + meter margin
302 assert len(state["chat_calls"]) == 1, "backstop did not fire past the meter"
303 assert state["chat_calls"][0]["message"] == "list my tasks"
304 finally:
305 page.evaluate("() => { meterRunning = false; }")
306
307
308def test_the_meter_starts_at_the_click_so_it_calibrates_in_silence(talk_page):
309 """The root cause, pinned at the source level: calibration measures
310 the noise floor, so it must run BEFORE speech. Deferring the meter to
311 the first recognition result put calibration mid-utterance and
312 taught the threshold the user's own voice."""
313 source = TALK.read_text()
314 click = source[source.index('mic.addEventListener("click"'):]
315 handler = click[:click.index("} else {")]
316 assert "startMeter()" in handler
317 onresult = source[source.index("recognizer.onresult"):]
318 onresult = onresult[:onresult.index("recognizer.onerror")]
319 assert "startMeter()" not in onresult
320
321
322def test_an_utterance_submits_even_if_a_final_never_comes(talk_page):
323 """THE latency fix, and the regression it repairs. With the meter
324 convicted, end-of-utterance waited on iOS volunteering an isFinal --
325 2-6 seconds after you stop, sometimes never, and strictly WORSE than
326 the meter it replaced. The recognizer's interims are themselves a
327 silence detector: text that stops changing IS the silence. Here the
328 device sends interims only, never a final, and the turn still fires
329 after the silence window."""
330 page, state = talk_page
331 fresh_turn(page, state)
332 say(page, "computer what time is it", final=False) # interim only
333 page.wait_for_timeout(2100) # silence window (1400ms) + margin
334 assert len(state["chat_calls"]) == 1, "no submit without a final"
335 assert state["chat_calls"][0]["message"] == "what time is it"
336
337
338def test_continued_speech_keeps_postponing_the_submit(talk_page):
339 """Stability means STABLE: as long as the interim keeps growing, the
340 timer must keep resetting -- otherwise mid-sentence pauses truncate."""
341 page, state = talk_page
342 fresh_turn(page, state)
343 say(page, "computer remind me", final=False)
344 page.wait_for_timeout(700)
345 say(page, "computer remind me to call the supplier", final=False)
346 page.wait_for_timeout(700) # 1400ms since FIRST, 700 since latest
347 assert state["chat_calls"] == [] # still talking; must not have fired
348 page.wait_for_timeout(1100) # now quiet past the window
349 assert len(state["chat_calls"]) == 1
350 assert state["chat_calls"][0]["message"] == "remind me to call the supplier"
351
352
353def test_talk_prefers_its_own_model_when_set(talk_page):
354 """Voice tolerates latency far worse than a terminal: a fast model
355 that answers in 3 seconds beats a smarter one that leaves 10 seconds
356 of silence. talk_model overrides ai_model for this surface only."""
357 page, state = talk_page
358 page.evaluate("() => { prefs.talk_model = 'anthropic:claude-haiku-4-5'; }")
359 try:
360 fresh_turn(page, state)
361 say(page, "computer hello", final=True)
362 page.wait_for_timeout(900)
363 assert state["chat_calls"], "turn did not fire"
364 assert state["chat_calls"][0]["model"] == "anthropic:claude-haiku-4-5"
365 finally:
366 page.evaluate("() => { delete prefs.talk_model; }")
367
368
369def test_a_turn_in_flight_blocks_a_second_submit(talk_page):
370 """The real bug, off a real session: "showed me my notes" and "show
371 me my notes" reached the server ONE SECOND apart -- two provider
372 calls billed for one utterance, the reply spoken twice. Nothing
373 serialized the paths that can produce an utterance (voice
374 endpointer, manual send), so a second one arriving while the first
375 is still in flight was never refused.
376
377 Both calls fire in ONE evaluate, back to back, with no wait between
378 them -- deliberately not an artificial network delay (a Python-side
379 time.sleep() inside a sync-API route handler blocks Playwright's one
380 shared dispatch thread, which stalls every OTHER call including the
381 "second" one, and the two land far enough apart that the guard's
382 absence goes unnoticed; that false pass is what this comment now
383 warns off). submitTurn is async but sets the guard synchronously
384 before its first await, so two calls in the same JS tick already
385 race for real -- no delay required.
386
387 Calling finalizeVoiceSubmit directly, twice, isolates the guarantee
388 from the choreography of WHICH timer or recognizer instance produced
389 the duplicate -- that cause matters less than the guarantee holding
390 regardless of cause."""
391 page, state = talk_page
392 fresh_turn(page, state)
393 page.evaluate("""() => {
394 finalizeVoiceSubmit('show me my notes');
395 finalizeVoiceSubmit('show me my notes');
396 }""")
397 page.wait_for_timeout(400)
398 assert len(state["chat_calls"]) == 1, state["chat_calls"]
399
400
401def test_a_failed_turn_releases_the_guard_for_the_next_one(talk_page):
402 """The guard must not survive its own failure -- a dropped connection
403 on turn one must not wedge the page shut for the rest of the
404 session."""
405 page, state = talk_page
406 fresh_turn(page, state)
407
408 def fail_once(route):
409 route.abort()
410 page.route("**/api/ai/chat", fail_once, times=1)
411 page.evaluate("() => finalizeVoiceSubmit('this one fails')")
412 page.wait_for_timeout(400)
413
414 page.evaluate("() => finalizeVoiceSubmit('this one should work')")
415 page.wait_for_timeout(400)
416 assert len(state["chat_calls"]) == 1
417 assert state["chat_calls"][0]["message"] == "this one should work"
418
419
420def test_playback_waits_for_the_microphone_to_actually_release(talk_page):
421 """Dan's own diagnosis, verbatim: "listening while talking, maybe not
422 allowed." The mic and speaker share one audio session on iOS, and
423 recognizer.stop() is a REQUEST -- teardown finishes asynchronously,
424 signalled by onend. Starting playback before that finishes can play
425 into a still-recording session and come out silent, with no error
426 anywhere. speak() must wait for the release, not race it."""
427 page, state = talk_page
428 fresh_turn(page, state)
429 page.evaluate("() => { window.__recoStarts = 0; }")
430
431 say(page, "computer say something", final=True)
432 page.wait_for_timeout(1200)
433
434 plays = page.evaluate("() => window.__plays")
435 assert plays, "server TTS never played"
436
437
438def test_stop_listening_and_wait_resolves_even_if_onend_never_fires(talk_page):
439 """The safety timeout: a recognizer that was never actually listening
440 (stop() on an idle recognizer) must not hang speak() forever."""
441 page, state = talk_page
442 fresh_turn(page, state)
443 resolved = page.evaluate("""async () => {
444 const r = window.__recos[window.__recos.length - 1];
445 if (r && r.onend) { r.stop = () => {}; } // stop() that fires nothing
446 const started = performance.now();
447 await stopListeningAndWaitForRelease();
448 return performance.now() - started;
449 }""")
450 assert resolved < 1000 # bounded by the safety timeout, not hung