100+ copy-paste UI components in plain HTML and CSS: no dependencies, no build step.

0 downloads

dbbasic-web-native/test_visual_ai.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-web-native.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-web-native.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-web-native.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Add canonical.html v1.0 - The single source of truth for usability standards fb6e650 1 year ago History
test_visual_ai.py · 17.9 KB · 494 lines
1#!/usr/bin/env python3
2
3"""
4Automated Visual Accessibility Testing with AI Vision
5
61. Takes screenshots of all component docs using Playwright
72. Analyzes screenshots using Claude's vision API
83. Checks for: contrast, visual feedback, affordances
94. Generates HTML report with pass/fail results
10
11Usage: python3 test_visual_ai.py
12"""
13
14import os
15import base64
16import json
17from pathlib import Path
18from datetime import datetime
19from anthropic import Anthropic
20from playwright.sync_api import sync_playwright
21
22# Load .env file if it exists
23def load_env():
24 env_path = Path(__file__).parent / '.env'
25 if env_path.exists():
26 with open(env_path) as f:
27 for line in f:
28 line = line.strip()
29 if line and not line.startswith('#') and '=' in line:
30 key, value = line.split('=', 1)
31 os.environ[key.strip()] = value.strip()
32
33load_env()
34
35# Component pages that should have screenshots
36COMPONENT_PAGES = [
37 'semantic-action.html',
38 'semantic-card.html',
39 'semantic-input.html',
40 'semantic-modal.html',
41 'semantic-navigator.html',
42 'semantic-list.html',
43 'semantic-menu.html',
44 'semantic-feedback.html',
45 'semantic-adjuster.html'
46]
47
48# AI prompts for accessibility testing
49ACCESSIBILITY_PROMPTS = {
50 'contrast': """Analyze this component screenshot for WCAG AA contrast compliance:
51
521. Identify all text elements (labels, buttons, input text, etc.)
532. Measure approximate contrast ratios between text and backgrounds
543. Flag any text below 4.5:1 contrast (normal text) or 3:1 (large text 18pt+)
554. List specific failures with colors if detected
56
57Return ONLY valid JSON (no markdown):
58{
59 "passed": true/false,
60 "issues": [
61 {"element": "Save button", "contrast": "3.2:1", "colors": "white on #007aff", "severity": "fail"}
62 ]
63}""",
64
65 'visual_feedback': """Analyze this component's visual feedback states:
66
671. Check if disabled elements look OBVIOUSLY non-interactive (grey, not just faded colors)
682. Check if hover states are clearly visible
693. Check if selection/active states are unmistakable
704. Check if focus indicators are present and visible
71
72Return ONLY valid JSON (no markdown):
73{
74 "passed": true/false,
75 "issues": [
76 {"state": "disabled", "problem": "still looks blue, not grey", "severity": "fail"},
77 {"state": "hover", "problem": "no visible change detected", "severity": "warning"}
78 ]
79}""",
80
81 'affordance': """Analyze if interactive elements are obviously clickable:
82
831. Do buttons look like buttons (not plain text)?
842. Are touch targets at least 44px (visible size)?
853. Do disabled states look obviously non-interactive?
864. Are dangerous actions (delete, etc.) visually distinct (red)?
87
88Return ONLY valid JSON (no markdown):
89{
90 "passed": true/false,
91 "issues": [
92 {"element": "Delete button", "problem": "same blue as Save button", "severity": "fail"}
93 ]
94}""",
95
96 'state_differentiation': """Analyze the Interactive States section (if present):
97
98CRITICAL: Check if buttons labeled with different states (Default, Hover, Active, Focus, Disabled) actually LOOK DIFFERENT from each other.
99
1001. Do ALL state buttons look identical/same color? (CRITICAL FAIL)
1012. Does the Hover state button show visual elevation or shadow?
1023. Does the Active state button show scale-down effect?
1034. Does the Focus state button show an outline?
1045. Does the Disabled state button look grey/non-interactive?
105
106If there is NO Interactive States section, pass this test.
107
108Return ONLY valid JSON (no markdown):
109{
110 "passed": true/false,
111 "issues": [
112 {"state": "all_identical", "problem": "All 5 state buttons look identical (same grey color)", "severity": "fail"},
113 {"state": "hover", "problem": "Hover state button shows no visual difference from default", "severity": "fail"}
114 ]
115}"""
116}
117
118
119class VisualAIAnalyzer:
120 def __init__(self, api_key: str):
121 self.client = Anthropic(api_key=api_key)
122 self.screenshots_dir = Path(__file__).parent / 'screenshots'
123 self.results_dir = Path(__file__).parent / 'test-results'
124 self.docs_dir = Path(__file__).parent / 'docs'
125 self.results = []
126
127 # Create directories if needed
128 self.screenshots_dir.mkdir(exist_ok=True)
129 self.results_dir.mkdir(exist_ok=True)
130
131 def capture_screenshots(self):
132 """Capture screenshots of all component docs using Playwright"""
133 print("\n📸 Capturing screenshots with Playwright...\n")
134
135 with sync_playwright() as p:
136 browser = p.chromium.launch(headless=True)
137 page = browser.new_page(viewport={'width': 1400, 'height': 1200})
138
139 for component_page in COMPONENT_PAGES:
140 doc_path = self.docs_dir / component_page
141
142 if not doc_path.exists():
143 print(f"⚠️ Doc not found: {component_page}")
144 continue
145
146 print(f"📸 Screenshotting {component_page}...")
147
148 # Load page
149 page.goto(f"file://{doc_path}")
150 page.wait_for_load_state('networkidle')
151
152 # Screenshot full page
153 screenshot_path = self.screenshots_dir / f"{component_page}.png"
154 page.screenshot(path=str(screenshot_path), full_page=True)
155
156 # Screenshot individual sections
157 sections = page.query_selector_all('.section')
158 for i, section in enumerate(sections):
159 try:
160 title_elem = section.query_selector('.section-title')
161 title = title_elem.inner_text().strip() if title_elem else f"section-{i}"
162 section_filename = f"{component_page}_{title.lower().replace(' ', '-').replace('&', 'and')}.png"
163 section_path = self.screenshots_dir / section_filename
164 section.screenshot(path=str(section_path))
165 except Exception as e:
166 print(f" ⚠️ Could not screenshot section {i}: {e}")
167
168 browser.close()
169
170 print(f"\n✅ Screenshots saved to {self.screenshots_dir}\n")
171
172 def encode_image(self, image_path: Path) -> str:
173 """Read image and encode as base64"""
174 with open(image_path, 'rb') as f:
175 return base64.standard_b64encode(f.read()).decode('utf-8')
176
177 def analyze_image(self, image_path: Path, prompt: str) -> dict:
178 """Analyze image with Claude's vision API"""
179 print(f"🤖 Analyzing {image_path.name}...")
180
181 # Encode image
182 image_data = self.encode_image(image_path)
183
184 # Call Claude API
185 try:
186 message = self.client.messages.create(
187 model="claude-sonnet-4-5",
188 max_tokens=1024,
189 messages=[{
190 "role": "user",
191 "content": [
192 {
193 "type": "image",
194 "source": {
195 "type": "base64",
196 "media_type": "image/png",
197 "data": image_data
198 }
199 },
200 {
201 "type": "text",
202 "text": prompt
203 }
204 ]
205 }]
206 )
207
208 # Extract text from response
209 response_text = message.content[0].text
210
211 # Parse JSON (handle markdown code blocks if present)
212 response_text = response_text.strip()
213 if response_text.startswith('```'):
214 # Remove markdown code blocks
215 lines = response_text.split('\n')
216 response_text = '\n'.join(lines[1:-1]) # Remove first and last line
217
218 result = json.loads(response_text)
219 return result
220
221 except Exception as e:
222 print(f"❌ Error analyzing {image_path.name}: {e}")
223 return {"passed": False, "issues": [{"error": str(e)}]}
224
225 def test_component(self, component_page: str) -> dict:
226 """Test a single component page"""
227 print(f"\n🧪 Testing {component_page}...")
228
229 # Find screenshot
230 screenshot_path = self.screenshots_dir / f"{component_page}.png"
231
232 if not screenshot_path.exists():
233 print(f"⚠️ Screenshot not found: {screenshot_path}")
234 return None
235
236 # Run all accessibility tests
237 contrast_result = self.analyze_image(screenshot_path, ACCESSIBILITY_PROMPTS['contrast'])
238 feedback_result = self.analyze_image(screenshot_path, ACCESSIBILITY_PROMPTS['visual_feedback'])
239 affordance_result = self.analyze_image(screenshot_path, ACCESSIBILITY_PROMPTS['affordance'])
240 state_diff_result = self.analyze_image(screenshot_path, ACCESSIBILITY_PROMPTS['state_differentiation'])
241
242 # Compile results
243 result = {
244 'component': component_page.replace('.html', ''),
245 'timestamp': datetime.now().isoformat(),
246 'screenshot': str(screenshot_path.relative_to(Path.cwd())),
247 'tests': {
248 'contrast': contrast_result,
249 'visual_feedback': feedback_result,
250 'affordance': affordance_result,
251 'state_differentiation': state_diff_result
252 },
253 'passed': all([
254 contrast_result.get('passed', False),
255 feedback_result.get('passed', False),
256 affordance_result.get('passed', False),
257 state_diff_result.get('passed', False)
258 ])
259 }
260
261 self.results.append(result)
262
263 # Log results
264 if result['passed']:
265 print(f"✅ {component_page} - ALL TESTS PASSED")
266 else:
267 print(f"❌ {component_page} - FAILURES DETECTED:")
268 if not contrast_result.get('passed', False):
269 print(f" - Contrast issues: {len(contrast_result.get('issues', []))}")
270 if not feedback_result.get('passed', False):
271 print(f" - Visual feedback issues: {len(feedback_result.get('issues', []))}")
272 if not affordance_result.get('passed', False):
273 print(f" - Affordance issues: {len(affordance_result.get('issues', []))}")
274 if not state_diff_result.get('passed', False):
275 print(f" - State differentiation issues: {len(state_diff_result.get('issues', []))}")
276
277 return result
278
279 def generate_html_report(self) -> Path:
280 """Generate HTML report from results"""
281 passed_count = sum(1 for r in self.results if r['passed'])
282 failed_count = len(self.results) - passed_count
283 pass_rate = round((passed_count / len(self.results)) * 100) if self.results else 0
284
285 # Build result rows
286 result_rows = []
287 for result in self.results:
288 status_icon = '✅' if result['passed'] else '❌'
289 status_class = 'pass' if result['passed'] else 'fail'
290
291 # Collect issues
292 issues_html = []
293 for test_type, test_result in result['tests'].items():
294 if not test_result.get('passed', False) and test_result.get('issues'):
295 issue_items = []
296 for issue in test_result['issues']:
297 if 'error' in issue:
298 issue_items.append(f"<li>Error: {issue['error']}</li>")
299 else:
300 element = issue.get('element') or issue.get('state', 'Unknown')
301 problem = issue.get('problem') or issue.get('contrast', 'Issue detected')
302 severity = issue.get('severity', 'warning')
303 issue_items.append(f"<li>{element}: {problem} ({severity})</li>")
304
305 issues_html.append(f"""
306 <div class="issue-group">
307 <strong>{test_type.replace('_', ' ').title()}:</strong>
308 <ul>{''.join(issue_items)}</ul>
309 </div>
310 """)
311
312 issues_content = ''.join(issues_html) if issues_html else '<span class="no-issues">No issues detected</span>'
313
314 result_rows.append(f"""
315 <tr class="{status_class}">
316 <td>{status_icon} {result['component']}</td>
317 <td>{'Pass' if result['passed'] else 'Fail'}</td>
318 <td>
319 <a href="../{result['screenshot']}" target="_blank">View Screenshot</a>
320 </td>
321 <td>{issues_content}</td>
322 </tr>
323 """)
324
325 html = f"""<!DOCTYPE html>
326<html lang="en">
327<head>
328 <meta charset="UTF-8">
329 <meta name="viewport" content="width=device-width, initial-scale=1.0">
330 <title>Visual Accessibility Test Report</title>
331 <style>
332 * {{ margin: 0; padding: 0; box-sizing: border-box; }}
333 body {{
334 font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', system-ui, sans-serif;
335 background: #000;
336 color: #fff;
337 padding: 48px 24px;
338 }}
339 .container {{ max-width: 1400px; margin: 0 auto; }}
340 h1 {{ font-size: 48px; margin-bottom: 24px; color: #409cff; }}
341 .summary {{
342 display: flex;
343 gap: 24px;
344 margin-bottom: 48px;
345 }}
346 .summary-card {{
347 flex: 1;
348 background: #1c1c1e;
349 border: 2px solid #38383a;
350 border-radius: 12px;
351 padding: 24px;
352 text-align: center;
353 }}
354 .summary-card.pass {{ border-color: #34c759; }}
355 .summary-card.fail {{ border-color: #ff3b30; }}
356 .summary-number {{ font-size: 48px; font-weight: 900; margin-bottom: 8px; }}
357 .summary-card.pass .summary-number {{ color: #34c759; }}
358 .summary-card.fail .summary-number {{ color: #ff3b30; }}
359 table {{
360 width: 100%;
361 border-collapse: collapse;
362 background: #1c1c1e;
363 border: 2px solid #38383a;
364 border-radius: 12px;
365 overflow: hidden;
366 }}
367 th {{
368 background: rgba(64, 156, 255, 0.1);
369 padding: 16px;
370 text-align: left;
371 font-weight: 700;
372 color: #409cff;
373 border-bottom: 2px solid #38383a;
374 }}
375 td {{
376 padding: 16px;
377 border-bottom: 1px solid #38383a;
378 }}
379 tr:last-child td {{ border-bottom: none; }}
380 tr.pass td:first-child {{ color: #34c759; }}
381 tr.fail td:first-child {{ color: #ff3b30; }}
382 .issue-group {{
383 margin-bottom: 12px;
384 padding: 12px;
385 background: rgba(255, 59, 48, 0.1);
386 border-left: 3px solid #ff3b30;
387 border-radius: 4px;
388 }}
389 .issue-group ul {{
390 margin-top: 8px;
391 padding-left: 24px;
392 }}
393 .issue-group li {{
394 margin-bottom: 4px;
395 color: #ababab;
396 }}
397 .no-issues {{ color: #6e6e73; }}
398 a {{ color: #409cff; text-decoration: none; }}
399 a:hover {{ text-decoration: underline; }}
400 </style>
401</head>
402<body>
403 <div class="container">
404 <h1>🤖 Visual Accessibility Test Report</h1>
405 <p style="color: #ababab; margin-bottom: 48px; font-size: 18px;">
406 Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
407 </p>
408
409 <div class="summary">
410 <div class="summary-card pass">
411 <div class="summary-number">{passed_count}</div>
412 <div style="color: #ababab;">Components Passed</div>
413 </div>
414 <div class="summary-card fail">
415 <div class="summary-number">{failed_count}</div>
416 <div style="color: #ababab;">Components Failed</div>
417 </div>
418 <div class="summary-card">
419 <div class="summary-number">{pass_rate}%</div>
420 <div style="color: #ababab;">Pass Rate</div>
421 </div>
422 </div>
423
424 <table>
425 <thead>
426 <tr>
427 <th>Component</th>
428 <th>Status</th>
429 <th>Screenshot</th>
430 <th>Issues</th>
431 </tr>
432 </thead>
433 <tbody>
434 {''.join(result_rows)}
435 </tbody>
436 </table>
437 </div>
438</body>
439</html>"""
440
441 report_path = self.results_dir / 'accessibility-report-ai.html'
442 report_path.write_text(html)
443 print(f"\n📊 Report generated: {report_path}")
444
445 return report_path
446
447 def run_all(self, skip_screenshots=False):
448 """Run tests on all components"""
449 # Capture screenshots first
450 if not skip_screenshots:
451 self.capture_screenshots()
452
453 print(f"\n🧪 Testing {len(COMPONENT_PAGES)} components with AI vision...\n")
454
455 for component_page in COMPONENT_PAGES:
456 self.test_component(component_page)
457
458 # Generate reports
459 report_path = self.generate_html_report()
460
461 # Save JSON results
462 json_path = self.results_dir / 'results-ai.json'
463 with open(json_path, 'w') as f:
464 json.dump(self.results, f, indent=2)
465
466 print(f"\n✅ Testing complete!")
467 print(f" Passed: {sum(1 for r in self.results if r['passed'])}/{len(self.results)}")
468 print(f" HTML Report: {report_path}")
469 print(f" JSON Results: {json_path}")
470
471 return self.results
472
473
474if __name__ == '__main__':
475 # Get API key from environment
476 api_key = os.environ.get('ANTHROPIC_API_KEY')
477
478 if not api_key:
479 print("⚠️ Warning: ANTHROPIC_API_KEY not found in .env or environment")
480 print("Running in screenshot-only mode (no AI analysis)\n")
481
482 # Create analyzer with dummy key
483 analyzer = VisualAIAnalyzer("dummy-key")
484
485 # Just capture screenshots
486 analyzer.capture_screenshots()
487
488 print("\n✅ Screenshots captured successfully!")
489 print("Add ANTHROPIC_API_KEY to .env file to enable AI analysis")
490 exit(0)
491
492 # Run full tests with AI analysis
493 analyzer = VisualAIAnalyzer(api_key)
494 analyzer.run_all()