askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_talk_mobile.py · 13.8 KB · 310 lines
| 1 | """/talk on a tablet: three failures reported from an older iPad. |
| 2 | |
| 3 | All three are the same shape -- a capability the page silently assumed, |
| 4 | on a device that does not have it -- and all three were invisible on a |
| 5 | desktop browser where the assumption holds. |
| 6 | |
| 7 | 1. **It never sent.** Speech showed on screen, but nothing happened until |
| 8 | the mic was tapped a second time. Silence endpointing depends on a |
| 9 | SECOND microphone stream (getUserMedia feeding an AnalyserNode, since |
| 10 | SpeechRecognition exposes no raw audio). On iOS the microphone is |
| 11 | EXCLUSIVE: the second consumer to ask simply fails. With no meter, |
| 12 | nothing in `silence` mode ever calls finalizeVoiceSubmit. |
| 13 | |
| 14 | 2. **It never spoke.** iOS refuses speechSynthesis and audio playback |
| 15 | that was not started by a user gesture, and a reply arrives long after |
| 16 | the click that started listening. |
| 17 | |
| 18 | 3. **The controls did not fit.** The stage claimed 85vh of its own on top |
| 19 | of captions, controls and nav, so the page was taller than the screen |
| 20 | by construction -- you could see the nav or the text box, never both. |
| 21 | """ |
| 22 | |
| 23 | import pathlib |
| 24 | |
| 25 | import pytest |
| 26 | |
| 27 | TALK = (pathlib.Path(__file__).resolve().parents[1] / "packages" / "app-shell" |
| 28 | / "objects" / "site" / "talk.py") |
| 29 | |
| 30 | |
| 31 | @pytest.fixture(scope="module") |
| 32 | def source(): |
| 33 | return TALK.read_text() |
| 34 | |
| 35 | |
| 36 | # --- 0. meter vs recognition: settled by evidence, not scheduling --------------- |
| 37 | # |
| 38 | # Three designs, each disproven by a real machine. (1) Start both at the |
| 39 | # click: iOS's exclusive microphone let the meter starve recognition -- |
| 40 | # ring animating, no transcript ever. (2) Defer the meter until |
| 41 | # recognition's first result: the meter then calibrated MID-SPEECH, |
| 42 | # learned the user's own voice as the noise floor, and every desktop's |
| 43 | # VAD went dead -- transcript on screen, listening forever, nothing |
| 44 | # sent. (3) Current: both start at the click so calibration happens in |
| 45 | # the pre-speech silence; the iOS theft is detected by the CONVICTION |
| 46 | # (meter hears voice, recognition silent for 2s) and the convicted meter |
| 47 | # stays off for the session; and the transcript-stability endpointer |
| 48 | # backstops EVERYTHING, because a mis-calibrated meter fails silent and |
| 49 | # a stalling transcript is the one signal it cannot fake. |
| 50 | |
| 51 | def test_the_meter_starts_at_the_click_not_on_first_result(source): |
| 52 | """Calibration measures the noise floor; it must run BEFORE speech. |
| 53 | Design (2) put it after the first result -- mid-utterance -- and |
| 54 | silently killed every desktop's endpointing.""" |
| 55 | click = source[source.index('mic.addEventListener("click"'):] |
| 56 | handler = click[:click.index("} else {")] |
| 57 | assert "startMeter()" in handler |
| 58 | onresult = source[source.index("recognizer.onresult"):] |
| 59 | onresult = onresult[:onresult.index("recognizer.onerror")] |
| 60 | assert "startMeter()" not in onresult |
| 61 | |
| 62 | |
| 63 | def test_a_convicted_meter_is_not_restarted_by_the_next_click(source): |
| 64 | """Conviction is per-session: re-opening the stream on the next mic |
| 65 | tap would re-steal the microphone one utterance later, forever.""" |
| 66 | click = source[source.index('mic.addEventListener("click"'):] |
| 67 | handler = click[:click.index("} else {")] |
| 68 | assert "!meterContended" in handler |
| 69 | |
| 70 | |
| 71 | def test_a_device_that_returns_no_recognition_says_so(source): |
| 72 | """A listening button that will never do anything is worse than one |
| 73 | that admits it cannot hear -- the user can then just type.""" |
| 74 | assert "watchForRecognition" in source |
| 75 | assert "not returning speech recognition" in source |
| 76 | |
| 77 | |
| 78 | def test_the_proof_resets_when_listening_stops(source): |
| 79 | """Otherwise the next session starts the meter immediately and |
| 80 | reintroduces the contention.""" |
| 81 | assert "recognitionProven = false" in source |
| 82 | |
| 83 | |
| 84 | # --- 1. endpointing without a level meter --------------------------------------- |
| 85 | |
| 86 | def test_silence_mode_falls_back_to_isFinal_when_the_meter_is_absent(source): |
| 87 | """THE bug. Without this, `silence` mode is a dead end on any device |
| 88 | where the microphone cannot be opened twice: the user speaks, sees |
| 89 | their words, and has to tap the mic to make anything happen.""" |
| 90 | assert 'endpoint === "silence" && !meterRunning' in source |
| 91 | |
| 92 | |
| 93 | def test_the_meter_reports_whether_it_actually_came_up(source): |
| 94 | """startMeter returns early on four separate failures. Each one has to |
| 95 | say so, or the fallback above can never fire.""" |
| 96 | assert "meterRunning = false" in source |
| 97 | assert "meterRunning = true" in source |
| 98 | assert source.count("noteEndpointFallback()") >= 4 |
| 99 | |
| 100 | |
| 101 | def test_the_fallback_is_visible_to_the_user(source): |
| 102 | """Absent capability, STATED -- the house rule. The hint is the only |
| 103 | way to tell 'waiting for you to stop talking' from 'waiting for you to |
| 104 | tap', and both look identical on screen.""" |
| 105 | assert 'id="endpointhint"' in source |
| 106 | assert "hint.hidden = false" in source |
| 107 | |
| 108 | |
| 109 | # --- 2. iOS will not make sound without a gesture -------------------------------- |
| 110 | |
| 111 | def test_audio_is_primed_inside_the_mic_click(source): |
| 112 | """Not at load, not when the reply arrives -- inside the gesture. iOS |
| 113 | gives no useful error when it refuses; the reply just never becomes |
| 114 | sound.""" |
| 115 | click = source[source.index('mic.addEventListener("click"'):] |
| 116 | handler = click[:click.index("conversationMode = !conversationMode")] |
| 117 | assert "primeAudioForIOS();" in handler, \ |
| 118 | "priming must run BEFORE the mode toggles, so it is inside the " \ |
| 119 | "gesture even on the click that turns listening off" |
| 120 | |
| 121 | |
| 122 | def test_priming_is_silent(source): |
| 123 | """A warm-up that made a noise would be worse than the bug.""" |
| 124 | prime = source[source.index("function primeAudioForIOS"):] |
| 125 | prime = prime[:prime.index("async function api")] |
| 126 | assert "volume = 0" in prime |
| 127 | assert 'SpeechSynthesisUtterance("")' in prime |
| 128 | |
| 129 | |
| 130 | def test_priming_also_warms_the_voice_list(source): |
| 131 | """getVoices() is empty on iOS until the engine loads, and |
| 132 | hasLocalVoice() routes on it -- asking inside the gesture is what |
| 133 | makes the answer true by the time a reply arrives.""" |
| 134 | prime = source[source.index("function primeAudioForIOS"):] |
| 135 | assert "getVoices()" in prime[:900] |
| 136 | |
| 137 | |
| 138 | def test_the_unlocked_context_is_not_closed(source): |
| 139 | """Closing it re-locks playback on iOS, which would undo the fix on |
| 140 | the second reply rather than the first.""" |
| 141 | assert "__dbbasicTalkUnlockCtx" in source |
| 142 | |
| 143 | |
| 144 | # --- 3. the page has to fit the screen ------------------------------------------ |
| 145 | |
| 146 | def test_the_stage_no_longer_claims_a_fixed_share_of_the_viewport(source): |
| 147 | """85vh of stage plus captions plus controls plus nav is taller than |
| 148 | the screen by construction.""" |
| 149 | assert "flex: 0 0 85vh" not in source |
| 150 | assert "flex: 1 1 auto; min-height: 0" in source |
| 151 | |
| 152 | |
| 153 | def test_the_wrapper_is_bounded_by_the_viewport(source): |
| 154 | assert "overflow: hidden" in source |
| 155 | assert "max-height: calc(100dvh" in source |
| 156 | |
| 157 | |
| 158 | def test_the_height_ladder_degrades_for_older_ipados(source): |
| 159 | """dvh where supported (iOS toolbars shrink the visible area and 100vh |
| 160 | lies about it), -webkit-fill-available for older iPadOS that has |
| 161 | neither dvh nor svh, plain vh last.""" |
| 162 | wrap = source[source.index(".talkwrap {"):] |
| 163 | wrap = wrap[:wrap.index("}")] |
| 164 | assert "100vh" in wrap |
| 165 | assert "-webkit-fill-available" in wrap |
| 166 | assert "100dvh" in wrap |
| 167 | assert wrap.index("100vh") < wrap.index("-webkit-fill-available") < wrap.index("100dvh") |
| 168 | |
| 169 | |
| 170 | def test_the_page_still_encodes_and_compiles(source): |
| 171 | """The lone-surrogate lesson from the shell: a page that dies on |
| 172 | encode passes every source-parsing test.""" |
| 173 | import importlib.util |
| 174 | spec = importlib.util.spec_from_file_location("talkmod", TALK) |
| 175 | module = importlib.util.module_from_spec(spec) |
| 176 | spec.loader.exec_module(module) |
| 177 | for value in vars(module).values(): |
| 178 | if isinstance(value, str): |
| 179 | value.encode("utf-8") |
| 180 | |
| 181 | |
| 182 | # --- the meter must be able to LOSE --------------------------------------------- |
| 183 | |
| 184 | def test_the_meter_convicts_itself_of_taking_the_microphone(source): |
| 185 | """The deferred start was not enough, and the same iPad proved it: |
| 186 | recognition delivered ONE result ("computer"), that proof started the |
| 187 | meter, and the meter's stream took the microphone back -- first word |
| 188 | on screen, then nothing, the original bug moved one result later. |
| 189 | |
| 190 | The conviction uses the meter's own evidence: RMS above the speech |
| 191 | threshold (it can hear voice) while recognition has produced nothing |
| 192 | for CONTENTION_MS. On a working platform results flow continuously |
| 193 | while you speak, so the signature cannot fire there.""" |
| 194 | assert "meterContended = true" in source |
| 195 | assert "CONTENTION_MS" in source |
| 196 | frame = source[source.index("function frame()"):] |
| 197 | frame = frame[:frame.index("function stopMeter")] |
| 198 | assert "rms >= speechThreshold && now - lastResultTs > CONTENTION_MS" in frame |
| 199 | |
| 200 | |
| 201 | def test_conviction_frees_the_mic_and_kicks_the_starved_recognizer(source): |
| 202 | frame = source[source.index("function frame()"):] |
| 203 | conviction = frame[frame.index("meterContended = true"):] |
| 204 | conviction = conviction[:conviction.index("evaluateVAD")] |
| 205 | assert "stopMeter();" in conviction |
| 206 | assert "stopListening();" in conviction # onend restarts it, mic now free |
| 207 | |
| 208 | |
| 209 | def test_every_recognition_result_stamps_the_heartbeat(source): |
| 210 | """lastResultTs is the evidence the conviction reads; a result that |
| 211 | does not stamp it makes an innocent meter convictable.""" |
| 212 | onresult = source[source.index("recognizer.onresult"):] |
| 213 | assert "lastResultTs = performance.now();" in onresult[:220] |
| 214 | |
| 215 | |
| 216 | # --- a misheard wake word must not discard the utterance ------------------------ |
| 217 | |
| 218 | def test_the_wake_word_tolerates_recognition_mishears(source): |
| 219 | """"compture" arrived for "computer" -- on screen, visibly heard -- |
| 220 | and the exact match then threw away everything said after it. The |
| 221 | asymmetry decides the design: a false arm captures a sentence the |
| 222 | user SEES (recoverable); a strict miss discards it invisibly (not).""" |
| 223 | assert "function matchesWakeWord" in source |
| 224 | assert "matchesWakeWord(key, target)" in source |
| 225 | |
| 226 | |
| 227 | def test_the_tolerance_is_damerau_because_mishears_are_transpositions(source): |
| 228 | """"compture" is two adjacent swaps: Damerau distance 2, plain |
| 229 | Levenshtein 3. Plain distance would reject the exact observed case |
| 230 | this tolerance exists for.""" |
| 231 | assert "prevPrev[j - 2] + 1" in source |
| 232 | assert "a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]" in source |
| 233 | |
| 234 | |
| 235 | def test_short_wake_words_stay_exact(source): |
| 236 | """"hey" must not match "they": the budget scales with length and is |
| 237 | zero below four letters.""" |
| 238 | budget = source[source.index("function wakeBudget"):] |
| 239 | budget = budget[:budget.index("function matchesWakeWord")] |
| 240 | assert "return 0" in budget and "return 2" in budget |
| 241 | |
| 242 | |
| 243 | # --- iOS joins segments without spaces ------------------------------------------ |
| 244 | |
| 245 | def test_result_segments_are_joined_with_spaces(source): |
| 246 | """"comptureshowmemynotes", verbatim off the iPad. Chrome includes |
| 247 | leading spaces on continuation segments; iOS does not, and the raw |
| 248 | concatenation produced one giant token the whitespace tokenizer could |
| 249 | never split -- so the wake gate could never match, and every |
| 250 | utterance was invisibly discarded WITH the transcript on screen. The |
| 251 | round that taught the most: recognition was working the whole time; |
| 252 | the page was destroying its output.""" |
| 253 | onresult = source[source.index("recognizer.onresult"):] |
| 254 | onresult = onresult[:onresult.index("recognizer.onerror")] |
| 255 | assert 'text += (text ? " " : "") + seg;' in onresult |
| 256 | assert "text += event.results[i][0].transcript;" not in onresult |
| 257 | |
| 258 | |
| 259 | def test_the_shell_gets_the_same_join_fix(source): |
| 260 | """Same loop, same platform, same bug waiting.""" |
| 261 | shell = (TALK.parent / "shell.py").read_text() |
| 262 | assert 'text += (text ? " " : "") + seg;' in shell |
| 263 | assert "text += event.results[i][0].transcript;" not in shell |
| 264 | |
| 265 | |
| 266 | def test_a_fused_wake_word_still_arms_via_its_prefix(source): |
| 267 | """Belt beside the braces: if a no-space final still arrives fused, |
| 268 | a long token whose PREFIX matches the wake word arms, and the |
| 269 | remainder becomes the command -- unspaced, but a model reads |
| 270 | unspaced text far better than this page reads a discarded |
| 271 | utterance.""" |
| 272 | split = source[source.index("function findWakeSplit"):] |
| 273 | split = split[:split.index("// If `text`")] |
| 274 | assert "key.slice(0, len)" in split |
| 275 | assert "key.length > target.length + wakeBudget(target)" in split |
| 276 | |
| 277 | |
| 278 | # --- a knock is not a question -------------------------------------------------- |
| 279 | |
| 280 | def test_the_bare_wake_word_is_never_submitted(source): |
| 281 | """"Computer", alone, reached the API on a live iPad: a 2.4-second |
| 282 | provider round-trip that answered "" -- money and silence for a turn |
| 283 | nobody asked. The check is fuzzy like the gate, so a misheard bare |
| 284 | "compture" is swallowed too.""" |
| 285 | finalize = source[source.index("function finalizeVoiceSubmit"):] |
| 286 | finalize = finalize[:finalize.index("function processTranscript")] |
| 287 | assert "matchesWakeWord(wordKey(only[0])" in finalize |
| 288 | assert finalize.index("matchesWakeWord") < finalize.index("submitTurn(clean)") |
| 289 | |
| 290 | |
| 291 | def test_an_empty_reply_becomes_words_rather_than_silence(source): |
| 292 | """A successful round-trip with empty text looks exactly like a hang |
| 293 | on a voice surface. The recorded turn (out: "") is real.""" |
| 294 | assert 'body.reply || "I came back empty' in source |
| 295 | |
| 296 | |
| 297 | def test_server_tts_reuses_one_gesture_unlocked_audio_element(source): |
| 298 | """"i didn't hear it talk", with the server voice configured and the |
| 299 | WAV verified good. iOS licenses programmatic play() only on an |
| 300 | element that has already played inside a user gesture -- `new |
| 301 | Audio(url)` per reply is born locked, its play() rejects, and the |
| 302 | fallback is speechSynthesis... which the mute switch silences. One |
| 303 | persistent element, played silent during the mic tap, reused for |
| 304 | every reply.""" |
| 305 | assert "function sharedTtsAudio()" in source |
| 306 | assert "new Audio(url)" not in source |
| 307 | prime = source[source.index("function primeAudioForIOS"):] |
| 308 | prime = prime[:prime.index("async function api")] |
| 309 | assert "sharedTtsAudio()" in prime |
| 310 | assert "data:audio/wav;base64," in prime |