askrobots / dbbasic-web-native
Public
100+ copy-paste UI components in plain HTML and CSS: no dependencies, no build step.
danq9
Add canonical.html v1.0 - The single source of truth for usability standards
fb6e650
1 year ago
History
voice-system.esm.js · 17.7 KB · 550 lines
| 1 | /** |
| 2 | * Semantic Components v1.0.0 |
| 3 | * voice-system.js |
| 4 | * |
| 5 | * Intent-based, context-aware, modality-agnostic web components |
| 6 | * https://semantic-components.dev |
| 7 | * |
| 8 | * Copyright (c) 2025 Semantic Components Team |
| 9 | * Released under the MIT License |
| 10 | */ |
| 11 | |
| 12 | /** |
| 13 | * Voice Command System |
| 14 | * Automatically registers voice commands based on semantic component intents |
| 15 | * Integrates with Web Speech API for recognition and synthesis |
| 16 | */ |
| 17 | |
| 18 | class VoiceCommandSystem { |
| 19 | constructor(contextManager) { |
| 20 | this.context = contextManager; |
| 21 | this.recognition = null; |
| 22 | this.synthesis = window.speechSynthesis; |
| 23 | this.commands = new Map(); |
| 24 | this.isListening = false; |
| 25 | this.confidenceThreshold = 0.7; |
| 26 | |
| 27 | this.init(); |
| 28 | } |
| 29 | |
| 30 | init() { |
| 31 | // Check for Web Speech API support |
| 32 | if (!('webkitSpeechRecognition' in window) && !('SpeechRecognition' in window)) { |
| 33 | console.warn('Web Speech API not supported in this browser'); |
| 34 | return; |
| 35 | } |
| 36 | |
| 37 | const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; |
| 38 | this.recognition = new SpeechRecognition(); |
| 39 | |
| 40 | this.recognition.continuous = true; |
| 41 | this.recognition.interimResults = true; |
| 42 | this.recognition.lang = 'en-US'; |
| 43 | |
| 44 | this.setupRecognitionHandlers(); |
| 45 | this.registerDefaultCommands(); |
| 46 | this.observeComponents(); |
| 47 | } |
| 48 | |
| 49 | setupRecognitionHandlers() { |
| 50 | this.recognition.onresult = (event) => { |
| 51 | const results = Array.from(event.results); |
| 52 | const latestResult = results[results.length - 1]; |
| 53 | |
| 54 | if (latestResult.isFinal) { |
| 55 | const transcript = latestResult[0].transcript.toLowerCase().trim(); |
| 56 | const confidence = latestResult[0].confidence; |
| 57 | |
| 58 | console.log(`Voice input: "${transcript}" (confidence: ${confidence})`); |
| 59 | |
| 60 | if (confidence >= this.confidenceThreshold) { |
| 61 | this.processCommand(transcript); |
| 62 | } |
| 63 | } |
| 64 | }; |
| 65 | |
| 66 | this.recognition.onerror = (event) => { |
| 67 | console.error('Speech recognition error:', event.error); |
| 68 | |
| 69 | if (event.error === 'no-speech') { |
| 70 | // Silence is okay, just keep listening |
| 71 | return; |
| 72 | } |
| 73 | |
| 74 | if (event.error === 'aborted' || event.error === 'audio-capture') { |
| 75 | this.stop(); |
| 76 | } |
| 77 | }; |
| 78 | |
| 79 | this.recognition.onend = () => { |
| 80 | // Restart if we're supposed to be listening |
| 81 | if (this.isListening) { |
| 82 | this.recognition.start(); |
| 83 | } |
| 84 | }; |
| 85 | } |
| 86 | |
| 87 | registerDefaultCommands() { |
| 88 | // System commands |
| 89 | this.registerCommand({ |
| 90 | patterns: ['stop listening', 'voice off', 'disable voice'], |
| 91 | handler: () => this.stop(), |
| 92 | description: 'Stop voice recognition' |
| 93 | }); |
| 94 | |
| 95 | this.registerCommand({ |
| 96 | patterns: ['what can i say', 'help', 'voice commands'], |
| 97 | handler: () => this.listCommands(), |
| 98 | description: 'List available commands' |
| 99 | }); |
| 100 | |
| 101 | this.registerCommand({ |
| 102 | patterns: ['scroll up', 'scroll down', 'scroll to top', 'scroll to bottom'], |
| 103 | handler: (transcript) => this.handleScroll(transcript), |
| 104 | description: 'Scroll the page' |
| 105 | }); |
| 106 | } |
| 107 | |
| 108 | registerCommand(command) { |
| 109 | command.patterns.forEach(pattern => { |
| 110 | this.commands.set(pattern, command); |
| 111 | }); |
| 112 | } |
| 113 | |
| 114 | observeComponents() { |
| 115 | // Watch for components being added/removed |
| 116 | const observer = new MutationObserver(() => { |
| 117 | this.autoRegisterComponents(); |
| 118 | }); |
| 119 | |
| 120 | observer.observe(document.body, { |
| 121 | childList: true, |
| 122 | subtree: true |
| 123 | }); |
| 124 | |
| 125 | // Initial registration |
| 126 | this.autoRegisterComponents(); |
| 127 | } |
| 128 | |
| 129 | autoRegisterComponents() { |
| 130 | // Register commands for semantic-action components |
| 131 | document.querySelectorAll('semantic-action').forEach(action => { |
| 132 | const text = action.textContent.trim(); |
| 133 | if (text) { |
| 134 | const patterns = this.generatePatterns(text); |
| 135 | |
| 136 | this.registerCommand({ |
| 137 | patterns, |
| 138 | handler: () => { |
| 139 | action.click(); |
| 140 | this.speak(`Activated ${text}`); |
| 141 | }, |
| 142 | description: `Activate: ${text}`, |
| 143 | component: action |
| 144 | }); |
| 145 | } |
| 146 | }); |
| 147 | |
| 148 | // Register commands for navigation |
| 149 | document.querySelectorAll('[data-nav-id]').forEach(nav => { |
| 150 | const id = nav.getAttribute('data-nav-id'); |
| 151 | const text = nav.textContent.trim(); |
| 152 | |
| 153 | this.registerCommand({ |
| 154 | patterns: [ |
| 155 | `go to ${text.toLowerCase()}`, |
| 156 | `navigate to ${text.toLowerCase()}`, |
| 157 | `open ${text.toLowerCase()}`, |
| 158 | `show ${text.toLowerCase()}` |
| 159 | ], |
| 160 | handler: () => { |
| 161 | nav.click(); |
| 162 | this.speak(`Navigating to ${text}`); |
| 163 | }, |
| 164 | description: `Navigate to: ${text}`, |
| 165 | component: nav |
| 166 | }); |
| 167 | }); |
| 168 | |
| 169 | // Register commands for cards |
| 170 | document.querySelectorAll('semantic-card card-title').forEach(card => { |
| 171 | const title = card.textContent.trim(); |
| 172 | const cardElement = card.closest('semantic-card'); |
| 173 | |
| 174 | this.registerCommand({ |
| 175 | patterns: [ |
| 176 | `open ${title.toLowerCase()}`, |
| 177 | `show ${title.toLowerCase()}`, |
| 178 | `inspect ${title.toLowerCase()}` |
| 179 | ], |
| 180 | handler: () => { |
| 181 | cardElement.click(); |
| 182 | this.speak(`Opening ${title}`); |
| 183 | }, |
| 184 | description: `Open card: ${title}`, |
| 185 | component: cardElement |
| 186 | }); |
| 187 | }); |
| 188 | |
| 189 | // Register commands for inputs |
| 190 | document.querySelectorAll('semantic-input[label]').forEach(input => { |
| 191 | const label = input.getAttribute('label').toLowerCase(); |
| 192 | |
| 193 | this.registerCommand({ |
| 194 | patterns: [ |
| 195 | `type ${label}`, |
| 196 | `enter ${label}`, |
| 197 | `input ${label}`, |
| 198 | `fill ${label}` |
| 199 | ], |
| 200 | handler: () => { |
| 201 | input.querySelector('input, textarea')?.focus(); |
| 202 | this.speak(`Ready to input ${label}. Please speak your text.`); |
| 203 | // Start dictation mode |
| 204 | this.startDictation(input); |
| 205 | }, |
| 206 | description: `Focus input: ${label}`, |
| 207 | component: input |
| 208 | }); |
| 209 | }); |
| 210 | |
| 211 | // Register commands for adjusters |
| 212 | document.querySelectorAll('semantic-adjuster[label]').forEach(adjuster => { |
| 213 | const label = adjuster.getAttribute('label').toLowerCase(); |
| 214 | |
| 215 | this.registerCommand({ |
| 216 | patterns: [ |
| 217 | `increase ${label}`, |
| 218 | `raise ${label}`, |
| 219 | `turn up ${label}` |
| 220 | ], |
| 221 | handler: () => { |
| 222 | const step = parseFloat(adjuster.getAttribute('step') || '5'); |
| 223 | adjuster.value = adjuster.value + step; |
| 224 | this.speak(`${label} increased to ${adjuster.value}`); |
| 225 | }, |
| 226 | description: `Increase: ${label}`, |
| 227 | component: adjuster |
| 228 | }); |
| 229 | |
| 230 | this.registerCommand({ |
| 231 | patterns: [ |
| 232 | `decrease ${label}`, |
| 233 | `lower ${label}`, |
| 234 | `turn down ${label}` |
| 235 | ], |
| 236 | handler: () => { |
| 237 | const step = parseFloat(adjuster.getAttribute('step') || '5'); |
| 238 | adjuster.value = adjuster.value - step; |
| 239 | this.speak(`${label} decreased to ${adjuster.value}`); |
| 240 | }, |
| 241 | description: `Decrease: ${label}`, |
| 242 | component: adjuster |
| 243 | }); |
| 244 | |
| 245 | this.registerCommand({ |
| 246 | patterns: [ |
| 247 | `set ${label} to *`, |
| 248 | `change ${label} to *` |
| 249 | ], |
| 250 | handler: (transcript) => { |
| 251 | const match = transcript.match(/to (\d+)/); |
| 252 | if (match) { |
| 253 | const value = parseInt(match[1]); |
| 254 | adjuster.value = value; |
| 255 | this.speak(`${label} set to ${value}`); |
| 256 | } |
| 257 | }, |
| 258 | description: `Set ${label} to a value`, |
| 259 | component: adjuster |
| 260 | }); |
| 261 | }); |
| 262 | |
| 263 | // Register commands for modals |
| 264 | document.querySelectorAll('semantic-modal[id]').forEach(modal => { |
| 265 | const id = modal.getAttribute('id'); |
| 266 | const header = modal.querySelector('[slot="header"]')?.textContent.trim() || id; |
| 267 | |
| 268 | this.registerCommand({ |
| 269 | patterns: [ |
| 270 | `open ${header.toLowerCase()}`, |
| 271 | `show ${header.toLowerCase()}` |
| 272 | ], |
| 273 | handler: () => { |
| 274 | modal.open(); |
| 275 | this.speak(`Opening ${header}`); |
| 276 | }, |
| 277 | description: `Open modal: ${header}`, |
| 278 | component: modal |
| 279 | }); |
| 280 | |
| 281 | this.registerCommand({ |
| 282 | patterns: [ |
| 283 | `close ${header.toLowerCase()}`, |
| 284 | `dismiss ${header.toLowerCase()}` |
| 285 | ], |
| 286 | handler: () => { |
| 287 | modal.close(); |
| 288 | this.speak(`Closing ${header}`); |
| 289 | }, |
| 290 | description: `Close modal: ${header}`, |
| 291 | component: modal |
| 292 | }); |
| 293 | }); |
| 294 | |
| 295 | // Generic close commands |
| 296 | this.registerCommand({ |
| 297 | patterns: ['close', 'dismiss', 'cancel', 'go back'], |
| 298 | handler: () => { |
| 299 | // Try to close any open modal |
| 300 | const openModal = document.querySelector('semantic-modal[open]'); |
| 301 | if (openModal) { |
| 302 | openModal.close(); |
| 303 | this.speak('Closed'); |
| 304 | return; |
| 305 | } |
| 306 | |
| 307 | // Try to close any open menu |
| 308 | const openMenu = document.querySelector('semantic-menu[open]'); |
| 309 | if (openMenu) { |
| 310 | openMenu.close(); |
| 311 | this.speak('Closed menu'); |
| 312 | return; |
| 313 | } |
| 314 | |
| 315 | this.speak('Nothing to close'); |
| 316 | }, |
| 317 | description: 'Close current dialog or menu' |
| 318 | }); |
| 319 | } |
| 320 | |
| 321 | generatePatterns(text) { |
| 322 | const lower = text.toLowerCase(); |
| 323 | return [ |
| 324 | lower, |
| 325 | `click ${lower}`, |
| 326 | `press ${lower}`, |
| 327 | `activate ${lower}`, |
| 328 | `select ${lower}` |
| 329 | ]; |
| 330 | } |
| 331 | |
| 332 | processCommand(transcript) { |
| 333 | // Try exact match first |
| 334 | if (this.commands.has(transcript)) { |
| 335 | const command = this.commands.get(transcript); |
| 336 | command.handler(transcript); |
| 337 | return; |
| 338 | } |
| 339 | |
| 340 | // Try wildcard patterns |
| 341 | for (const [pattern, command] of this.commands.entries()) { |
| 342 | if (pattern.includes('*')) { |
| 343 | const regex = new RegExp('^' + pattern.replace('*', '.*') + '$'); |
| 344 | if (regex.test(transcript)) { |
| 345 | command.handler(transcript); |
| 346 | return; |
| 347 | } |
| 348 | } |
| 349 | } |
| 350 | |
| 351 | // Try fuzzy matching for close matches |
| 352 | const matches = this.fuzzyMatch(transcript); |
| 353 | if (matches.length > 0) { |
| 354 | const bestMatch = matches[0]; |
| 355 | console.log(`Using fuzzy match: "${bestMatch.pattern}" for "${transcript}"`); |
| 356 | bestMatch.command.handler(transcript); |
| 357 | return; |
| 358 | } |
| 359 | |
| 360 | // No match found |
| 361 | console.log(`No command found for: "${transcript}"`); |
| 362 | this.speak("I didn't understand that command. Say 'help' for available commands."); |
| 363 | } |
| 364 | |
| 365 | fuzzyMatch(transcript) { |
| 366 | const matches = []; |
| 367 | |
| 368 | for (const [pattern, command] of this.commands.entries()) { |
| 369 | const similarity = this.similarity(transcript, pattern); |
| 370 | if (similarity > 0.7) { |
| 371 | matches.push({ pattern, command, similarity }); |
| 372 | } |
| 373 | } |
| 374 | |
| 375 | return matches.sort((a, b) => b.similarity - a.similarity); |
| 376 | } |
| 377 | |
| 378 | similarity(s1, s2) { |
| 379 | // Levenshtein distance for fuzzy matching |
| 380 | const longer = s1.length > s2.length ? s1 : s2; |
| 381 | const shorter = s1.length > s2.length ? s2 : s1; |
| 382 | |
| 383 | if (longer.length === 0) return 1.0; |
| 384 | |
| 385 | const distance = this.levenshteinDistance(longer, shorter); |
| 386 | return (longer.length - distance) / longer.length; |
| 387 | } |
| 388 | |
| 389 | levenshteinDistance(s1, s2) { |
| 390 | const matrix = []; |
| 391 | |
| 392 | for (let i = 0; i <= s2.length; i++) { |
| 393 | matrix[i] = [i]; |
| 394 | } |
| 395 | |
| 396 | for (let j = 0; j <= s1.length; j++) { |
| 397 | matrix[0][j] = j; |
| 398 | } |
| 399 | |
| 400 | for (let i = 1; i <= s2.length; i++) { |
| 401 | for (let j = 1; j <= s1.length; j++) { |
| 402 | if (s2.charAt(i - 1) === s1.charAt(j - 1)) { |
| 403 | matrix[i][j] = matrix[i - 1][j - 1]; |
| 404 | } else { |
| 405 | matrix[i][j] = Math.min( |
| 406 | matrix[i - 1][j - 1] + 1, |
| 407 | matrix[i][j - 1] + 1, |
| 408 | matrix[i - 1][j] + 1 |
| 409 | ); |
| 410 | } |
| 411 | } |
| 412 | } |
| 413 | |
| 414 | return matrix[s2.length][s1.length]; |
| 415 | } |
| 416 | |
| 417 | handleScroll(transcript) { |
| 418 | if (transcript.includes('up')) { |
| 419 | window.scrollBy({ top: -300, behavior: 'smooth' }); |
| 420 | this.speak('Scrolling up'); |
| 421 | } else if (transcript.includes('down')) { |
| 422 | window.scrollBy({ top: 300, behavior: 'smooth' }); |
| 423 | this.speak('Scrolling down'); |
| 424 | } else if (transcript.includes('top')) { |
| 425 | window.scrollTo({ top: 0, behavior: 'smooth' }); |
| 426 | this.speak('Scrolling to top'); |
| 427 | } else if (transcript.includes('bottom')) { |
| 428 | window.scrollTo({ top: document.body.scrollHeight, behavior: 'smooth' }); |
| 429 | this.speak('Scrolling to bottom'); |
| 430 | } |
| 431 | } |
| 432 | |
| 433 | startDictation(inputComponent) { |
| 434 | // Special dictation mode for filling inputs |
| 435 | const input = inputComponent.shadowRoot.querySelector('input, textarea'); |
| 436 | if (!input) return; |
| 437 | |
| 438 | const dictationHandler = (event) => { |
| 439 | const results = Array.from(event.results); |
| 440 | const latestResult = results[results.length - 1]; |
| 441 | |
| 442 | if (latestResult.isFinal) { |
| 443 | const transcript = latestResult[0].transcript; |
| 444 | input.value = transcript; |
| 445 | inputComponent.value = transcript; |
| 446 | |
| 447 | this.speak('Text entered'); |
| 448 | this.recognition.removeEventListener('result', dictationHandler); |
| 449 | } |
| 450 | }; |
| 451 | |
| 452 | this.recognition.addEventListener('result', dictationHandler); |
| 453 | } |
| 454 | |
| 455 | listCommands() { |
| 456 | const commandList = Array.from(this.commands.values()) |
| 457 | .filter((cmd, idx, self) => self.indexOf(cmd) === idx) // Unique commands |
| 458 | .slice(0, 10) // Top 10 |
| 459 | .map(cmd => cmd.description) |
| 460 | .join('. '); |
| 461 | |
| 462 | this.speak(`Available commands: ${commandList}`); |
| 463 | } |
| 464 | |
| 465 | speak(text, options = {}) { |
| 466 | if (!this.synthesis) return; |
| 467 | |
| 468 | // Cancel any ongoing speech |
| 469 | this.synthesis.cancel(); |
| 470 | |
| 471 | // Adjust speech based on context |
| 472 | const context = this.context.context; |
| 473 | |
| 474 | // Don't speak if user is busy or noise level is high |
| 475 | if (context.user.isBusy && options.urgency !== 'critical') return; |
| 476 | if (context.environment.noiseLevel > 70 && options.urgency !== 'critical') return; |
| 477 | |
| 478 | const utterance = new SpeechSynthesisUtterance(text); |
| 479 | |
| 480 | // Adjust rate and pitch based on urgency |
| 481 | utterance.rate = options.urgency === 'critical' ? 0.9 : 1.0; |
| 482 | utterance.pitch = options.urgency === 'critical' ? 1.2 : 1.0; |
| 483 | utterance.volume = Math.max(0.5, (100 - context.environment.noiseLevel) / 100); |
| 484 | |
| 485 | this.synthesis.speak(utterance); |
| 486 | } |
| 487 | |
| 488 | start() { |
| 489 | if (!this.recognition) { |
| 490 | console.warn('Speech recognition not available'); |
| 491 | return; |
| 492 | } |
| 493 | |
| 494 | if (this.isListening) return; |
| 495 | |
| 496 | this.isListening = true; |
| 497 | this.recognition.start(); |
| 498 | this.speak('Voice commands activated'); |
| 499 | |
| 500 | // Update context |
| 501 | this.context.setModality('voice-screen'); |
| 502 | |
| 503 | console.log('Voice recognition started'); |
| 504 | } |
| 505 | |
| 506 | stop() { |
| 507 | if (!this.recognition || !this.isListening) return; |
| 508 | |
| 509 | this.isListening = false; |
| 510 | this.recognition.stop(); |
| 511 | this.speak('Voice commands deactivated'); |
| 512 | |
| 513 | // Reset to screen modality |
| 514 | this.context.setModality('screen'); |
| 515 | |
| 516 | console.log('Voice recognition stopped'); |
| 517 | } |
| 518 | |
| 519 | toggle() { |
| 520 | if (this.isListening) { |
| 521 | this.stop(); |
| 522 | } else { |
| 523 | this.start(); |
| 524 | } |
| 525 | } |
| 526 | } |
| 527 | |
| 528 | // Initialize voice system when context is ready |
| 529 | if (typeof window !== 'undefined') { |
| 530 | window.addEventListener('DOMContentLoaded', () => { |
| 531 | if (window.semanticContext) { |
| 532 | window.voiceSystem = new VoiceCommandSystem(window.semanticContext); |
| 533 | |
| 534 | // Add keyboard shortcut to toggle voice (Ctrl+Shift+V) |
| 535 | document.addEventListener('keydown', (e) => { |
| 536 | if (e.ctrlKey && e.shiftKey && e.key === 'V') { |
| 537 | e.preventDefault(); |
| 538 | window.voiceSystem.toggle(); |
| 539 | } |
| 540 | }); |
| 541 | |
| 542 | console.log('Voice command system initialized. Press Ctrl+Shift+V to toggle voice commands.'); |
| 543 | } |
| 544 | }); |
| 545 | } |
| 546 | |
| 547 | // Export |
| 548 | |
| 549 | |
| 550 | export default window.semanticContext || {}; |