askrobots / dbbasic-web-native
Public
100+ copy-paste UI components in plain HTML and CSS: no dependencies, no build step.
danq9
Add canonical.html v1.0 - The single source of truth for usability standards
fb6e650
1 year ago
History
test_visual_ai.py · 17.9 KB · 494 lines
| 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | """ |
| 4 | Automated Visual Accessibility Testing with AI Vision |
| 5 | |
| 6 | 1. Takes screenshots of all component docs using Playwright |
| 7 | 2. Analyzes screenshots using Claude's vision API |
| 8 | 3. Checks for: contrast, visual feedback, affordances |
| 9 | 4. Generates HTML report with pass/fail results |
| 10 | |
| 11 | Usage: python3 test_visual_ai.py |
| 12 | """ |
| 13 | |
| 14 | import os |
| 15 | import base64 |
| 16 | import json |
| 17 | from pathlib import Path |
| 18 | from datetime import datetime |
| 19 | from anthropic import Anthropic |
| 20 | from playwright.sync_api import sync_playwright |
| 21 | |
| 22 | # Load .env file if it exists |
| 23 | def load_env(): |
| 24 | env_path = Path(__file__).parent / '.env' |
| 25 | if env_path.exists(): |
| 26 | with open(env_path) as f: |
| 27 | for line in f: |
| 28 | line = line.strip() |
| 29 | if line and not line.startswith('#') and '=' in line: |
| 30 | key, value = line.split('=', 1) |
| 31 | os.environ[key.strip()] = value.strip() |
| 32 | |
| 33 | load_env() |
| 34 | |
| 35 | # Component pages that should have screenshots |
| 36 | COMPONENT_PAGES = [ |
| 37 | 'semantic-action.html', |
| 38 | 'semantic-card.html', |
| 39 | 'semantic-input.html', |
| 40 | 'semantic-modal.html', |
| 41 | 'semantic-navigator.html', |
| 42 | 'semantic-list.html', |
| 43 | 'semantic-menu.html', |
| 44 | 'semantic-feedback.html', |
| 45 | 'semantic-adjuster.html' |
| 46 | ] |
| 47 | |
| 48 | # AI prompts for accessibility testing |
| 49 | ACCESSIBILITY_PROMPTS = { |
| 50 | 'contrast': """Analyze this component screenshot for WCAG AA contrast compliance: |
| 51 | |
| 52 | 1. Identify all text elements (labels, buttons, input text, etc.) |
| 53 | 2. Measure approximate contrast ratios between text and backgrounds |
| 54 | 3. Flag any text below 4.5:1 contrast (normal text) or 3:1 (large text 18pt+) |
| 55 | 4. List specific failures with colors if detected |
| 56 | |
| 57 | Return ONLY valid JSON (no markdown): |
| 58 | { |
| 59 | "passed": true/false, |
| 60 | "issues": [ |
| 61 | {"element": "Save button", "contrast": "3.2:1", "colors": "white on #007aff", "severity": "fail"} |
| 62 | ] |
| 63 | }""", |
| 64 | |
| 65 | 'visual_feedback': """Analyze this component's visual feedback states: |
| 66 | |
| 67 | 1. Check if disabled elements look OBVIOUSLY non-interactive (grey, not just faded colors) |
| 68 | 2. Check if hover states are clearly visible |
| 69 | 3. Check if selection/active states are unmistakable |
| 70 | 4. Check if focus indicators are present and visible |
| 71 | |
| 72 | Return ONLY valid JSON (no markdown): |
| 73 | { |
| 74 | "passed": true/false, |
| 75 | "issues": [ |
| 76 | {"state": "disabled", "problem": "still looks blue, not grey", "severity": "fail"}, |
| 77 | {"state": "hover", "problem": "no visible change detected", "severity": "warning"} |
| 78 | ] |
| 79 | }""", |
| 80 | |
| 81 | 'affordance': """Analyze if interactive elements are obviously clickable: |
| 82 | |
| 83 | 1. Do buttons look like buttons (not plain text)? |
| 84 | 2. Are touch targets at least 44px (visible size)? |
| 85 | 3. Do disabled states look obviously non-interactive? |
| 86 | 4. Are dangerous actions (delete, etc.) visually distinct (red)? |
| 87 | |
| 88 | Return ONLY valid JSON (no markdown): |
| 89 | { |
| 90 | "passed": true/false, |
| 91 | "issues": [ |
| 92 | {"element": "Delete button", "problem": "same blue as Save button", "severity": "fail"} |
| 93 | ] |
| 94 | }""", |
| 95 | |
| 96 | 'state_differentiation': """Analyze the Interactive States section (if present): |
| 97 | |
| 98 | CRITICAL: Check if buttons labeled with different states (Default, Hover, Active, Focus, Disabled) actually LOOK DIFFERENT from each other. |
| 99 | |
| 100 | 1. Do ALL state buttons look identical/same color? (CRITICAL FAIL) |
| 101 | 2. Does the Hover state button show visual elevation or shadow? |
| 102 | 3. Does the Active state button show scale-down effect? |
| 103 | 4. Does the Focus state button show an outline? |
| 104 | 5. Does the Disabled state button look grey/non-interactive? |
| 105 | |
| 106 | If there is NO Interactive States section, pass this test. |
| 107 | |
| 108 | Return ONLY valid JSON (no markdown): |
| 109 | { |
| 110 | "passed": true/false, |
| 111 | "issues": [ |
| 112 | {"state": "all_identical", "problem": "All 5 state buttons look identical (same grey color)", "severity": "fail"}, |
| 113 | {"state": "hover", "problem": "Hover state button shows no visual difference from default", "severity": "fail"} |
| 114 | ] |
| 115 | }""" |
| 116 | } |
| 117 | |
| 118 | |
| 119 | class VisualAIAnalyzer: |
| 120 | def __init__(self, api_key: str): |
| 121 | self.client = Anthropic(api_key=api_key) |
| 122 | self.screenshots_dir = Path(__file__).parent / 'screenshots' |
| 123 | self.results_dir = Path(__file__).parent / 'test-results' |
| 124 | self.docs_dir = Path(__file__).parent / 'docs' |
| 125 | self.results = [] |
| 126 | |
| 127 | # Create directories if needed |
| 128 | self.screenshots_dir.mkdir(exist_ok=True) |
| 129 | self.results_dir.mkdir(exist_ok=True) |
| 130 | |
| 131 | def capture_screenshots(self): |
| 132 | """Capture screenshots of all component docs using Playwright""" |
| 133 | print("\n📸 Capturing screenshots with Playwright...\n") |
| 134 | |
| 135 | with sync_playwright() as p: |
| 136 | browser = p.chromium.launch(headless=True) |
| 137 | page = browser.new_page(viewport={'width': 1400, 'height': 1200}) |
| 138 | |
| 139 | for component_page in COMPONENT_PAGES: |
| 140 | doc_path = self.docs_dir / component_page |
| 141 | |
| 142 | if not doc_path.exists(): |
| 143 | print(f"⚠️ Doc not found: {component_page}") |
| 144 | continue |
| 145 | |
| 146 | print(f"📸 Screenshotting {component_page}...") |
| 147 | |
| 148 | # Load page |
| 149 | page.goto(f"file://{doc_path}") |
| 150 | page.wait_for_load_state('networkidle') |
| 151 | |
| 152 | # Screenshot full page |
| 153 | screenshot_path = self.screenshots_dir / f"{component_page}.png" |
| 154 | page.screenshot(path=str(screenshot_path), full_page=True) |
| 155 | |
| 156 | # Screenshot individual sections |
| 157 | sections = page.query_selector_all('.section') |
| 158 | for i, section in enumerate(sections): |
| 159 | try: |
| 160 | title_elem = section.query_selector('.section-title') |
| 161 | title = title_elem.inner_text().strip() if title_elem else f"section-{i}" |
| 162 | section_filename = f"{component_page}_{title.lower().replace(' ', '-').replace('&', 'and')}.png" |
| 163 | section_path = self.screenshots_dir / section_filename |
| 164 | section.screenshot(path=str(section_path)) |
| 165 | except Exception as e: |
| 166 | print(f" ⚠️ Could not screenshot section {i}: {e}") |
| 167 | |
| 168 | browser.close() |
| 169 | |
| 170 | print(f"\n✅ Screenshots saved to {self.screenshots_dir}\n") |
| 171 | |
| 172 | def encode_image(self, image_path: Path) -> str: |
| 173 | """Read image and encode as base64""" |
| 174 | with open(image_path, 'rb') as f: |
| 175 | return base64.standard_b64encode(f.read()).decode('utf-8') |
| 176 | |
| 177 | def analyze_image(self, image_path: Path, prompt: str) -> dict: |
| 178 | """Analyze image with Claude's vision API""" |
| 179 | print(f"🤖 Analyzing {image_path.name}...") |
| 180 | |
| 181 | # Encode image |
| 182 | image_data = self.encode_image(image_path) |
| 183 | |
| 184 | # Call Claude API |
| 185 | try: |
| 186 | message = self.client.messages.create( |
| 187 | model="claude-sonnet-4-5", |
| 188 | max_tokens=1024, |
| 189 | messages=[{ |
| 190 | "role": "user", |
| 191 | "content": [ |
| 192 | { |
| 193 | "type": "image", |
| 194 | "source": { |
| 195 | "type": "base64", |
| 196 | "media_type": "image/png", |
| 197 | "data": image_data |
| 198 | } |
| 199 | }, |
| 200 | { |
| 201 | "type": "text", |
| 202 | "text": prompt |
| 203 | } |
| 204 | ] |
| 205 | }] |
| 206 | ) |
| 207 | |
| 208 | # Extract text from response |
| 209 | response_text = message.content[0].text |
| 210 | |
| 211 | # Parse JSON (handle markdown code blocks if present) |
| 212 | response_text = response_text.strip() |
| 213 | if response_text.startswith('```'): |
| 214 | # Remove markdown code blocks |
| 215 | lines = response_text.split('\n') |
| 216 | response_text = '\n'.join(lines[1:-1]) # Remove first and last line |
| 217 | |
| 218 | result = json.loads(response_text) |
| 219 | return result |
| 220 | |
| 221 | except Exception as e: |
| 222 | print(f"❌ Error analyzing {image_path.name}: {e}") |
| 223 | return {"passed": False, "issues": [{"error": str(e)}]} |
| 224 | |
| 225 | def test_component(self, component_page: str) -> dict: |
| 226 | """Test a single component page""" |
| 227 | print(f"\n🧪 Testing {component_page}...") |
| 228 | |
| 229 | # Find screenshot |
| 230 | screenshot_path = self.screenshots_dir / f"{component_page}.png" |
| 231 | |
| 232 | if not screenshot_path.exists(): |
| 233 | print(f"⚠️ Screenshot not found: {screenshot_path}") |
| 234 | return None |
| 235 | |
| 236 | # Run all accessibility tests |
| 237 | contrast_result = self.analyze_image(screenshot_path, ACCESSIBILITY_PROMPTS['contrast']) |
| 238 | feedback_result = self.analyze_image(screenshot_path, ACCESSIBILITY_PROMPTS['visual_feedback']) |
| 239 | affordance_result = self.analyze_image(screenshot_path, ACCESSIBILITY_PROMPTS['affordance']) |
| 240 | state_diff_result = self.analyze_image(screenshot_path, ACCESSIBILITY_PROMPTS['state_differentiation']) |
| 241 | |
| 242 | # Compile results |
| 243 | result = { |
| 244 | 'component': component_page.replace('.html', ''), |
| 245 | 'timestamp': datetime.now().isoformat(), |
| 246 | 'screenshot': str(screenshot_path.relative_to(Path.cwd())), |
| 247 | 'tests': { |
| 248 | 'contrast': contrast_result, |
| 249 | 'visual_feedback': feedback_result, |
| 250 | 'affordance': affordance_result, |
| 251 | 'state_differentiation': state_diff_result |
| 252 | }, |
| 253 | 'passed': all([ |
| 254 | contrast_result.get('passed', False), |
| 255 | feedback_result.get('passed', False), |
| 256 | affordance_result.get('passed', False), |
| 257 | state_diff_result.get('passed', False) |
| 258 | ]) |
| 259 | } |
| 260 | |
| 261 | self.results.append(result) |
| 262 | |
| 263 | # Log results |
| 264 | if result['passed']: |
| 265 | print(f"✅ {component_page} - ALL TESTS PASSED") |
| 266 | else: |
| 267 | print(f"❌ {component_page} - FAILURES DETECTED:") |
| 268 | if not contrast_result.get('passed', False): |
| 269 | print(f" - Contrast issues: {len(contrast_result.get('issues', []))}") |
| 270 | if not feedback_result.get('passed', False): |
| 271 | print(f" - Visual feedback issues: {len(feedback_result.get('issues', []))}") |
| 272 | if not affordance_result.get('passed', False): |
| 273 | print(f" - Affordance issues: {len(affordance_result.get('issues', []))}") |
| 274 | if not state_diff_result.get('passed', False): |
| 275 | print(f" - State differentiation issues: {len(state_diff_result.get('issues', []))}") |
| 276 | |
| 277 | return result |
| 278 | |
| 279 | def generate_html_report(self) -> Path: |
| 280 | """Generate HTML report from results""" |
| 281 | passed_count = sum(1 for r in self.results if r['passed']) |
| 282 | failed_count = len(self.results) - passed_count |
| 283 | pass_rate = round((passed_count / len(self.results)) * 100) if self.results else 0 |
| 284 | |
| 285 | # Build result rows |
| 286 | result_rows = [] |
| 287 | for result in self.results: |
| 288 | status_icon = '✅' if result['passed'] else '❌' |
| 289 | status_class = 'pass' if result['passed'] else 'fail' |
| 290 | |
| 291 | # Collect issues |
| 292 | issues_html = [] |
| 293 | for test_type, test_result in result['tests'].items(): |
| 294 | if not test_result.get('passed', False) and test_result.get('issues'): |
| 295 | issue_items = [] |
| 296 | for issue in test_result['issues']: |
| 297 | if 'error' in issue: |
| 298 | issue_items.append(f"<li>Error: {issue['error']}</li>") |
| 299 | else: |
| 300 | element = issue.get('element') or issue.get('state', 'Unknown') |
| 301 | problem = issue.get('problem') or issue.get('contrast', 'Issue detected') |
| 302 | severity = issue.get('severity', 'warning') |
| 303 | issue_items.append(f"<li>{element}: {problem} ({severity})</li>") |
| 304 | |
| 305 | issues_html.append(f""" |
| 306 | <div class="issue-group"> |
| 307 | <strong>{test_type.replace('_', ' ').title()}:</strong> |
| 308 | <ul>{''.join(issue_items)}</ul> |
| 309 | </div> |
| 310 | """) |
| 311 | |
| 312 | issues_content = ''.join(issues_html) if issues_html else '<span class="no-issues">No issues detected</span>' |
| 313 | |
| 314 | result_rows.append(f""" |
| 315 | <tr class="{status_class}"> |
| 316 | <td>{status_icon} {result['component']}</td> |
| 317 | <td>{'Pass' if result['passed'] else 'Fail'}</td> |
| 318 | <td> |
| 319 | <a href="../{result['screenshot']}" target="_blank">View Screenshot</a> |
| 320 | </td> |
| 321 | <td>{issues_content}</td> |
| 322 | </tr> |
| 323 | """) |
| 324 | |
| 325 | html = f"""<!DOCTYPE html> |
| 326 | <html lang="en"> |
| 327 | <head> |
| 328 | <meta charset="UTF-8"> |
| 329 | <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| 330 | <title>Visual Accessibility Test Report</title> |
| 331 | <style> |
| 332 | * {{ margin: 0; padding: 0; box-sizing: border-box; }} |
| 333 | body {{ |
| 334 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif; |
| 335 | background: #000; |
| 336 | color: #fff; |
| 337 | padding: 48px 24px; |
| 338 | }} |
| 339 | .container {{ max-width: 1400px; margin: 0 auto; }} |
| 340 | h1 {{ font-size: 48px; margin-bottom: 24px; color: #409cff; }} |
| 341 | .summary {{ |
| 342 | display: flex; |
| 343 | gap: 24px; |
| 344 | margin-bottom: 48px; |
| 345 | }} |
| 346 | .summary-card {{ |
| 347 | flex: 1; |
| 348 | background: #1c1c1e; |
| 349 | border: 2px solid #38383a; |
| 350 | border-radius: 12px; |
| 351 | padding: 24px; |
| 352 | text-align: center; |
| 353 | }} |
| 354 | .summary-card.pass {{ border-color: #34c759; }} |
| 355 | .summary-card.fail {{ border-color: #ff3b30; }} |
| 356 | .summary-number {{ font-size: 48px; font-weight: 900; margin-bottom: 8px; }} |
| 357 | .summary-card.pass .summary-number {{ color: #34c759; }} |
| 358 | .summary-card.fail .summary-number {{ color: #ff3b30; }} |
| 359 | table {{ |
| 360 | width: 100%; |
| 361 | border-collapse: collapse; |
| 362 | background: #1c1c1e; |
| 363 | border: 2px solid #38383a; |
| 364 | border-radius: 12px; |
| 365 | overflow: hidden; |
| 366 | }} |
| 367 | th {{ |
| 368 | background: rgba(64, 156, 255, 0.1); |
| 369 | padding: 16px; |
| 370 | text-align: left; |
| 371 | font-weight: 700; |
| 372 | color: #409cff; |
| 373 | border-bottom: 2px solid #38383a; |
| 374 | }} |
| 375 | td {{ |
| 376 | padding: 16px; |
| 377 | border-bottom: 1px solid #38383a; |
| 378 | }} |
| 379 | tr:last-child td {{ border-bottom: none; }} |
| 380 | tr.pass td:first-child {{ color: #34c759; }} |
| 381 | tr.fail td:first-child {{ color: #ff3b30; }} |
| 382 | .issue-group {{ |
| 383 | margin-bottom: 12px; |
| 384 | padding: 12px; |
| 385 | background: rgba(255, 59, 48, 0.1); |
| 386 | border-left: 3px solid #ff3b30; |
| 387 | border-radius: 4px; |
| 388 | }} |
| 389 | .issue-group ul {{ |
| 390 | margin-top: 8px; |
| 391 | padding-left: 24px; |
| 392 | }} |
| 393 | .issue-group li {{ |
| 394 | margin-bottom: 4px; |
| 395 | color: #ababab; |
| 396 | }} |
| 397 | .no-issues {{ color: #6e6e73; }} |
| 398 | a {{ color: #409cff; text-decoration: none; }} |
| 399 | a:hover {{ text-decoration: underline; }} |
| 400 | </style> |
| 401 | </head> |
| 402 | <body> |
| 403 | <div class="container"> |
| 404 | <h1>🤖 Visual Accessibility Test Report</h1> |
| 405 | <p style="color: #ababab; margin-bottom: 48px; font-size: 18px;"> |
| 406 | Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} |
| 407 | </p> |
| 408 | |
| 409 | <div class="summary"> |
| 410 | <div class="summary-card pass"> |
| 411 | <div class="summary-number">{passed_count}</div> |
| 412 | <div style="color: #ababab;">Components Passed</div> |
| 413 | </div> |
| 414 | <div class="summary-card fail"> |
| 415 | <div class="summary-number">{failed_count}</div> |
| 416 | <div style="color: #ababab;">Components Failed</div> |
| 417 | </div> |
| 418 | <div class="summary-card"> |
| 419 | <div class="summary-number">{pass_rate}%</div> |
| 420 | <div style="color: #ababab;">Pass Rate</div> |
| 421 | </div> |
| 422 | </div> |
| 423 | |
| 424 | <table> |
| 425 | <thead> |
| 426 | <tr> |
| 427 | <th>Component</th> |
| 428 | <th>Status</th> |
| 429 | <th>Screenshot</th> |
| 430 | <th>Issues</th> |
| 431 | </tr> |
| 432 | </thead> |
| 433 | <tbody> |
| 434 | {''.join(result_rows)} |
| 435 | </tbody> |
| 436 | </table> |
| 437 | </div> |
| 438 | </body> |
| 439 | </html>""" |
| 440 | |
| 441 | report_path = self.results_dir / 'accessibility-report-ai.html' |
| 442 | report_path.write_text(html) |
| 443 | print(f"\n📊 Report generated: {report_path}") |
| 444 | |
| 445 | return report_path |
| 446 | |
| 447 | def run_all(self, skip_screenshots=False): |
| 448 | """Run tests on all components""" |
| 449 | # Capture screenshots first |
| 450 | if not skip_screenshots: |
| 451 | self.capture_screenshots() |
| 452 | |
| 453 | print(f"\n🧪 Testing {len(COMPONENT_PAGES)} components with AI vision...\n") |
| 454 | |
| 455 | for component_page in COMPONENT_PAGES: |
| 456 | self.test_component(component_page) |
| 457 | |
| 458 | # Generate reports |
| 459 | report_path = self.generate_html_report() |
| 460 | |
| 461 | # Save JSON results |
| 462 | json_path = self.results_dir / 'results-ai.json' |
| 463 | with open(json_path, 'w') as f: |
| 464 | json.dump(self.results, f, indent=2) |
| 465 | |
| 466 | print(f"\n✅ Testing complete!") |
| 467 | print(f" Passed: {sum(1 for r in self.results if r['passed'])}/{len(self.results)}") |
| 468 | print(f" HTML Report: {report_path}") |
| 469 | print(f" JSON Results: {json_path}") |
| 470 | |
| 471 | return self.results |
| 472 | |
| 473 | |
| 474 | if __name__ == '__main__': |
| 475 | # Get API key from environment |
| 476 | api_key = os.environ.get('ANTHROPIC_API_KEY') |
| 477 | |
| 478 | if not api_key: |
| 479 | print("⚠️ Warning: ANTHROPIC_API_KEY not found in .env or environment") |
| 480 | print("Running in screenshot-only mode (no AI analysis)\n") |
| 481 | |
| 482 | # Create analyzer with dummy key |
| 483 | analyzer = VisualAIAnalyzer("dummy-key") |
| 484 | |
| 485 | # Just capture screenshots |
| 486 | analyzer.capture_screenshots() |
| 487 | |
| 488 | print("\n✅ Screenshots captured successfully!") |
| 489 | print("Add ANTHROPIC_API_KEY to .env file to enable AI analysis") |
| 490 | exit(0) |
| 491 | |
| 492 | # Run full tests with AI analysis |
| 493 | analyzer = VisualAIAnalyzer(api_key) |
| 494 | analyzer.run_all() |