askrobots / dbbasic-web-native
Public
100+ copy-paste UI components in plain HTML and CSS: no dependencies, no build step.
danq9
Add canonical.html v1.0 - The single source of truth for usability standards
fb6e650
1 year ago
History
test_voice.py · 3.8 KB · 132 lines
| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | Text-based AI tests for voice command system. |
| 4 | Uses Claude Haiku (cheap) to validate voice hints and commands. |
| 5 | Cost: ~$0.01 per full run |
| 6 | Run: Every commit |
| 7 | """ |
| 8 | |
| 9 | from anthropic import Anthropic |
| 10 | from pathlib import Path |
| 11 | import json |
| 12 | import os |
| 13 | |
| 14 | COMPONENT_PAGES = [ |
| 15 | 'semantic-action.html', |
| 16 | 'semantic-card.html', |
| 17 | 'semantic-input.html', |
| 18 | 'semantic-modal.html', |
| 19 | 'semantic-navigator.html', |
| 20 | 'semantic-list.html', |
| 21 | 'semantic-menu.html', |
| 22 | 'semantic-feedback.html', |
| 23 | 'semantic-adjuster.html' |
| 24 | ] |
| 25 | |
| 26 | class VoiceTester: |
| 27 | def __init__(self, api_key): |
| 28 | self.client = Anthropic(api_key=api_key) |
| 29 | self.results = [] |
| 30 | self.docs_dir = Path(__file__).parent / 'docs' |
| 31 | |
| 32 | def test_voice_hints(self, component_name, html_content): |
| 33 | """Test voice command hints using text AI""" |
| 34 | |
| 35 | prompt = f"""Analyze this HTML for voice command accessibility. |
| 36 | |
| 37 | Component: {component_name} |
| 38 | |
| 39 | Requirements: |
| 40 | 1. Interactive elements should have clear voice commands |
| 41 | 2. aria-label attributes should be present for screen readers |
| 42 | 3. Voice hints should be actionable (e.g., "Say 'submit' to...") |
| 43 | 4. Elements should be uniquely identifiable by voice |
| 44 | |
| 45 | Check for: |
| 46 | - Missing aria-label on buttons/inputs |
| 47 | - Unclear or missing voice command hints |
| 48 | - Duplicate voice commands (ambiguous) |
| 49 | - Elements that can't be activated by voice |
| 50 | |
| 51 | Respond with JSON: |
| 52 | {{ |
| 53 | "passed": true/false, |
| 54 | "issues": [ |
| 55 | {{"type": "missing_aria_label", "element": "button text", "severity": "error"}}, |
| 56 | {{"type": "unclear_voice_hint", "element": "...", "severity": "warning"}} |
| 57 | ] |
| 58 | }} |
| 59 | |
| 60 | HTML: |
| 61 | {html_content[:5000]} |
| 62 | """ |
| 63 | |
| 64 | response = self.client.messages.create( |
| 65 | model="claude-haiku-3-5-20241022", # Cheapest model |
| 66 | max_tokens=1024, |
| 67 | messages=[{"role": "user", "content": prompt}] |
| 68 | ) |
| 69 | |
| 70 | try: |
| 71 | result = json.loads(response.content[0].text) |
| 72 | return result |
| 73 | except: |
| 74 | return { |
| 75 | "passed": False, |
| 76 | "issues": [{"type": "parse_error", "element": "response", "severity": "error"}] |
| 77 | } |
| 78 | |
| 79 | def run_all_tests(self): |
| 80 | """Test all components""" |
| 81 | print("\n🎤 Running voice command tests...\n") |
| 82 | |
| 83 | for component in COMPONENT_PAGES: |
| 84 | doc_path = self.docs_dir / component |
| 85 | html_content = doc_path.read_text() |
| 86 | |
| 87 | print(f"🔍 Testing {component}...") |
| 88 | result = self.test_voice_hints(component, html_content) |
| 89 | |
| 90 | test_result = { |
| 91 | 'component': component, |
| 92 | 'passed': result.get('passed', False), |
| 93 | 'issues': result.get('issues', []), |
| 94 | 'issue_count': len(result.get('issues', [])) |
| 95 | } |
| 96 | |
| 97 | self.results.append(test_result) |
| 98 | |
| 99 | if test_result['passed']: |
| 100 | print(f"✅ {component} - PASSED") |
| 101 | else: |
| 102 | print(f"❌ {component} - {test_result['issue_count']} issues") |
| 103 | |
| 104 | self.generate_report() |
| 105 | |
| 106 | def generate_report(self): |
| 107 | """Generate test report""" |
| 108 | passed = sum(1 for r in self.results if r['passed']) |
| 109 | total = len(self.results) |
| 110 | |
| 111 | # Save JSON |
| 112 | results_path = Path(__file__).parent / 'test-results' / 'results-voice.json' |
| 113 | results_path.parent.mkdir(exist_ok=True) |
| 114 | with open(results_path, 'w') as f: |
| 115 | json.dump(self.results, f, indent=2) |
| 116 | |
| 117 | print(f"\n📊 Voice tests complete!") |
| 118 | print(f" Passed: {passed}/{total}") |
| 119 | print(f" Results: {results_path}") |
| 120 | |
| 121 | # Exit with error if any tests failed |
| 122 | if passed < total: |
| 123 | exit(1) |
| 124 | |
| 125 | if __name__ == '__main__': |
| 126 | api_key = os.getenv('ANTHROPIC_API_KEY') |
| 127 | if not api_key: |
| 128 | print("❌ ANTHROPIC_API_KEY environment variable not set") |
| 129 | exit(1) |
| 130 | |
| 131 | tester = VoiceTester(api_key) |
| 132 | tester.run_all_tests() |