Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_talk_mobile.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_talk_mobile.py · 13.8 KB · 310 lines
1"""/talk on a tablet: three failures reported from an older iPad.
2
3All three are the same shape -- a capability the page silently assumed,
4on a device that does not have it -- and all three were invisible on a
5desktop browser where the assumption holds.
6
71. **It never sent.** Speech showed on screen, but nothing happened until
8 the mic was tapped a second time. Silence endpointing depends on a
9 SECOND microphone stream (getUserMedia feeding an AnalyserNode, since
10 SpeechRecognition exposes no raw audio). On iOS the microphone is
11 EXCLUSIVE: the second consumer to ask simply fails. With no meter,
12 nothing in `silence` mode ever calls finalizeVoiceSubmit.
13
142. **It never spoke.** iOS refuses speechSynthesis and audio playback
15 that was not started by a user gesture, and a reply arrives long after
16 the click that started listening.
17
183. **The controls did not fit.** The stage claimed 85vh of its own on top
19 of captions, controls and nav, so the page was taller than the screen
20 by construction -- you could see the nav or the text box, never both.
21"""
22
23import pathlib
24
25import pytest
26
27TALK = (pathlib.Path(__file__).resolve().parents[1] / "packages" / "app-shell"
28 / "objects" / "site" / "talk.py")
29
30
31@pytest.fixture(scope="module")
32def source():
33 return TALK.read_text()
34
35
36# --- 0. meter vs recognition: settled by evidence, not scheduling ---------------
37#
38# Three designs, each disproven by a real machine. (1) Start both at the
39# click: iOS's exclusive microphone let the meter starve recognition --
40# ring animating, no transcript ever. (2) Defer the meter until
41# recognition's first result: the meter then calibrated MID-SPEECH,
42# learned the user's own voice as the noise floor, and every desktop's
43# VAD went dead -- transcript on screen, listening forever, nothing
44# sent. (3) Current: both start at the click so calibration happens in
45# the pre-speech silence; the iOS theft is detected by the CONVICTION
46# (meter hears voice, recognition silent for 2s) and the convicted meter
47# stays off for the session; and the transcript-stability endpointer
48# backstops EVERYTHING, because a mis-calibrated meter fails silent and
49# a stalling transcript is the one signal it cannot fake.
50
51def test_the_meter_starts_at_the_click_not_on_first_result(source):
52 """Calibration measures the noise floor; it must run BEFORE speech.
53 Design (2) put it after the first result -- mid-utterance -- and
54 silently killed every desktop's endpointing."""
55 click = source[source.index('mic.addEventListener("click"'):]
56 handler = click[:click.index("} else {")]
57 assert "startMeter()" in handler
58 onresult = source[source.index("recognizer.onresult"):]
59 onresult = onresult[:onresult.index("recognizer.onerror")]
60 assert "startMeter()" not in onresult
61
62
63def test_a_convicted_meter_is_not_restarted_by_the_next_click(source):
64 """Conviction is per-session: re-opening the stream on the next mic
65 tap would re-steal the microphone one utterance later, forever."""
66 click = source[source.index('mic.addEventListener("click"'):]
67 handler = click[:click.index("} else {")]
68 assert "!meterContended" in handler
69
70
71def test_a_device_that_returns_no_recognition_says_so(source):
72 """A listening button that will never do anything is worse than one
73 that admits it cannot hear -- the user can then just type."""
74 assert "watchForRecognition" in source
75 assert "not returning speech recognition" in source
76
77
78def test_the_proof_resets_when_listening_stops(source):
79 """Otherwise the next session starts the meter immediately and
80 reintroduces the contention."""
81 assert "recognitionProven = false" in source
82
83
84# --- 1. endpointing without a level meter ---------------------------------------
85
86def test_silence_mode_falls_back_to_isFinal_when_the_meter_is_absent(source):
87 """THE bug. Without this, `silence` mode is a dead end on any device
88 where the microphone cannot be opened twice: the user speaks, sees
89 their words, and has to tap the mic to make anything happen."""
90 assert 'endpoint === "silence" && !meterRunning' in source
91
92
93def test_the_meter_reports_whether_it_actually_came_up(source):
94 """startMeter returns early on four separate failures. Each one has to
95 say so, or the fallback above can never fire."""
96 assert "meterRunning = false" in source
97 assert "meterRunning = true" in source
98 assert source.count("noteEndpointFallback()") >= 4
99
100
101def test_the_fallback_is_visible_to_the_user(source):
102 """Absent capability, STATED -- the house rule. The hint is the only
103 way to tell 'waiting for you to stop talking' from 'waiting for you to
104 tap', and both look identical on screen."""
105 assert 'id="endpointhint"' in source
106 assert "hint.hidden = false" in source
107
108
109# --- 2. iOS will not make sound without a gesture --------------------------------
110
111def test_audio_is_primed_inside_the_mic_click(source):
112 """Not at load, not when the reply arrives -- inside the gesture. iOS
113 gives no useful error when it refuses; the reply just never becomes
114 sound."""
115 click = source[source.index('mic.addEventListener("click"'):]
116 handler = click[:click.index("conversationMode = !conversationMode")]
117 assert "primeAudioForIOS();" in handler, \
118 "priming must run BEFORE the mode toggles, so it is inside the " \
119 "gesture even on the click that turns listening off"
120
121
122def test_priming_is_silent(source):
123 """A warm-up that made a noise would be worse than the bug."""
124 prime = source[source.index("function primeAudioForIOS"):]
125 prime = prime[:prime.index("async function api")]
126 assert "volume = 0" in prime
127 assert 'SpeechSynthesisUtterance("")' in prime
128
129
130def test_priming_also_warms_the_voice_list(source):
131 """getVoices() is empty on iOS until the engine loads, and
132 hasLocalVoice() routes on it -- asking inside the gesture is what
133 makes the answer true by the time a reply arrives."""
134 prime = source[source.index("function primeAudioForIOS"):]
135 assert "getVoices()" in prime[:900]
136
137
138def test_the_unlocked_context_is_not_closed(source):
139 """Closing it re-locks playback on iOS, which would undo the fix on
140 the second reply rather than the first."""
141 assert "__dbbasicTalkUnlockCtx" in source
142
143
144# --- 3. the page has to fit the screen ------------------------------------------
145
146def test_the_stage_no_longer_claims_a_fixed_share_of_the_viewport(source):
147 """85vh of stage plus captions plus controls plus nav is taller than
148 the screen by construction."""
149 assert "flex: 0 0 85vh" not in source
150 assert "flex: 1 1 auto; min-height: 0" in source
151
152
153def test_the_wrapper_is_bounded_by_the_viewport(source):
154 assert "overflow: hidden" in source
155 assert "max-height: calc(100dvh" in source
156
157
158def test_the_height_ladder_degrades_for_older_ipados(source):
159 """dvh where supported (iOS toolbars shrink the visible area and 100vh
160 lies about it), -webkit-fill-available for older iPadOS that has
161 neither dvh nor svh, plain vh last."""
162 wrap = source[source.index(".talkwrap {"):]
163 wrap = wrap[:wrap.index("}")]
164 assert "100vh" in wrap
165 assert "-webkit-fill-available" in wrap
166 assert "100dvh" in wrap
167 assert wrap.index("100vh") < wrap.index("-webkit-fill-available") < wrap.index("100dvh")
168
169
170def test_the_page_still_encodes_and_compiles(source):
171 """The lone-surrogate lesson from the shell: a page that dies on
172 encode passes every source-parsing test."""
173 import importlib.util
174 spec = importlib.util.spec_from_file_location("talkmod", TALK)
175 module = importlib.util.module_from_spec(spec)
176 spec.loader.exec_module(module)
177 for value in vars(module).values():
178 if isinstance(value, str):
179 value.encode("utf-8")
180
181
182# --- the meter must be able to LOSE ---------------------------------------------
183
184def test_the_meter_convicts_itself_of_taking_the_microphone(source):
185 """The deferred start was not enough, and the same iPad proved it:
186 recognition delivered ONE result ("computer"), that proof started the
187 meter, and the meter's stream took the microphone back -- first word
188 on screen, then nothing, the original bug moved one result later.
189
190 The conviction uses the meter's own evidence: RMS above the speech
191 threshold (it can hear voice) while recognition has produced nothing
192 for CONTENTION_MS. On a working platform results flow continuously
193 while you speak, so the signature cannot fire there."""
194 assert "meterContended = true" in source
195 assert "CONTENTION_MS" in source
196 frame = source[source.index("function frame()"):]
197 frame = frame[:frame.index("function stopMeter")]
198 assert "rms >= speechThreshold && now - lastResultTs > CONTENTION_MS" in frame
199
200
201def test_conviction_frees_the_mic_and_kicks_the_starved_recognizer(source):
202 frame = source[source.index("function frame()"):]
203 conviction = frame[frame.index("meterContended = true"):]
204 conviction = conviction[:conviction.index("evaluateVAD")]
205 assert "stopMeter();" in conviction
206 assert "stopListening();" in conviction # onend restarts it, mic now free
207
208
209def test_every_recognition_result_stamps_the_heartbeat(source):
210 """lastResultTs is the evidence the conviction reads; a result that
211 does not stamp it makes an innocent meter convictable."""
212 onresult = source[source.index("recognizer.onresult"):]
213 assert "lastResultTs = performance.now();" in onresult[:220]
214
215
216# --- a misheard wake word must not discard the utterance ------------------------
217
218def test_the_wake_word_tolerates_recognition_mishears(source):
219 """"compture" arrived for "computer" -- on screen, visibly heard --
220 and the exact match then threw away everything said after it. The
221 asymmetry decides the design: a false arm captures a sentence the
222 user SEES (recoverable); a strict miss discards it invisibly (not)."""
223 assert "function matchesWakeWord" in source
224 assert "matchesWakeWord(key, target)" in source
225
226
227def test_the_tolerance_is_damerau_because_mishears_are_transpositions(source):
228 """"compture" is two adjacent swaps: Damerau distance 2, plain
229 Levenshtein 3. Plain distance would reject the exact observed case
230 this tolerance exists for."""
231 assert "prevPrev[j - 2] + 1" in source
232 assert "a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]" in source
233
234
235def test_short_wake_words_stay_exact(source):
236 """"hey" must not match "they": the budget scales with length and is
237 zero below four letters."""
238 budget = source[source.index("function wakeBudget"):]
239 budget = budget[:budget.index("function matchesWakeWord")]
240 assert "return 0" in budget and "return 2" in budget
241
242
243# --- iOS joins segments without spaces ------------------------------------------
244
245def test_result_segments_are_joined_with_spaces(source):
246 """"comptureshowmemynotes", verbatim off the iPad. Chrome includes
247 leading spaces on continuation segments; iOS does not, and the raw
248 concatenation produced one giant token the whitespace tokenizer could
249 never split -- so the wake gate could never match, and every
250 utterance was invisibly discarded WITH the transcript on screen. The
251 round that taught the most: recognition was working the whole time;
252 the page was destroying its output."""
253 onresult = source[source.index("recognizer.onresult"):]
254 onresult = onresult[:onresult.index("recognizer.onerror")]
255 assert 'text += (text ? " " : "") + seg;' in onresult
256 assert "text += event.results[i][0].transcript;" not in onresult
257
258
259def test_the_shell_gets_the_same_join_fix(source):
260 """Same loop, same platform, same bug waiting."""
261 shell = (TALK.parent / "shell.py").read_text()
262 assert 'text += (text ? " " : "") + seg;' in shell
263 assert "text += event.results[i][0].transcript;" not in shell
264
265
266def test_a_fused_wake_word_still_arms_via_its_prefix(source):
267 """Belt beside the braces: if a no-space final still arrives fused,
268 a long token whose PREFIX matches the wake word arms, and the
269 remainder becomes the command -- unspaced, but a model reads
270 unspaced text far better than this page reads a discarded
271 utterance."""
272 split = source[source.index("function findWakeSplit"):]
273 split = split[:split.index("// If `text`")]
274 assert "key.slice(0, len)" in split
275 assert "key.length > target.length + wakeBudget(target)" in split
276
277
278# --- a knock is not a question --------------------------------------------------
279
280def test_the_bare_wake_word_is_never_submitted(source):
281 """"Computer", alone, reached the API on a live iPad: a 2.4-second
282 provider round-trip that answered "" -- money and silence for a turn
283 nobody asked. The check is fuzzy like the gate, so a misheard bare
284 "compture" is swallowed too."""
285 finalize = source[source.index("function finalizeVoiceSubmit"):]
286 finalize = finalize[:finalize.index("function processTranscript")]
287 assert "matchesWakeWord(wordKey(only[0])" in finalize
288 assert finalize.index("matchesWakeWord") < finalize.index("submitTurn(clean)")
289
290
291def test_an_empty_reply_becomes_words_rather_than_silence(source):
292 """A successful round-trip with empty text looks exactly like a hang
293 on a voice surface. The recorded turn (out: "") is real."""
294 assert 'body.reply || "I came back empty' in source
295
296
297def test_server_tts_reuses_one_gesture_unlocked_audio_element(source):
298 """"i didn't hear it talk", with the server voice configured and the
299 WAV verified good. iOS licenses programmatic play() only on an
300 element that has already played inside a user gesture -- `new
301 Audio(url)` per reply is born locked, its play() rejects, and the
302 fallback is speechSynthesis... which the mute switch silences. One
303 persistent element, played silent during the mic tap, reused for
304 every reply."""
305 assert "function sharedTtsAudio()" in source
306 assert "new Audio(url)" not in source
307 prime = source[source.index("function primeAudioForIOS"):]
308 prime = prime[:prime.index("async function api")]
309 assert "sharedTtsAudio()" in prime
310 assert "data:audio/wav;base64," in prime