askrobots / dbbasic-web-native
Public
100+ copy-paste UI components in plain HTML and CSS: no dependencies, no build step.
danq9
Add canonical.html v1.0 - The single source of truth for usability standards
fb6e650
1 year ago
History
test_single_visual.py · 12.9 KB · 390 lines
| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | Single Component AI Vision Test |
| 4 | Tests just one component at a time for quick verification |
| 5 | Usage: python3 test_single_visual.py <component-name> |
| 6 | Example: python3 test_single_visual.py semantic-card |
| 7 | """ |
| 8 | |
| 9 | import sys |
| 10 | import json |
| 11 | import base64 |
| 12 | import anthropic |
| 13 | import os |
| 14 | import time |
| 15 | from datetime import datetime |
| 16 | from pathlib import Path |
| 17 | from playwright.sync_api import sync_playwright |
| 18 | |
| 19 | # Configuration |
| 20 | SCREENSHOT_DIR = Path("screenshots") |
| 21 | RESULTS_DIR = Path("test-results") |
| 22 | API_LOG_FILE = Path("test-results/api-calls.log") |
| 23 | ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY") |
| 24 | |
| 25 | # API cost tracking |
| 26 | API_COSTS = { |
| 27 | "claude-sonnet-4-5": { |
| 28 | "input": 3.00 / 1_000_000, # $3 per million input tokens |
| 29 | "output": 15.00 / 1_000_000 # $15 per million output tokens |
| 30 | } |
| 31 | } |
| 32 | |
| 33 | MODEL = "claude-sonnet-4-5" |
| 34 | |
| 35 | api_call_stats = { |
| 36 | "total_calls": 0, |
| 37 | "total_input_tokens": 0, |
| 38 | "total_output_tokens": 0, |
| 39 | "total_cost": 0.0, |
| 40 | "calls": [] |
| 41 | } |
| 42 | |
| 43 | if not ANTHROPIC_API_KEY: |
| 44 | print("❌ ANTHROPIC_API_KEY environment variable not set") |
| 45 | sys.exit(1) |
| 46 | |
| 47 | client = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY) |
| 48 | |
| 49 | COMPONENTS = [ |
| 50 | "semantic-action", |
| 51 | "semantic-card", |
| 52 | "semantic-input", |
| 53 | "semantic-modal", |
| 54 | "semantic-navigator", |
| 55 | "semantic-list", |
| 56 | "semantic-menu", |
| 57 | "semantic-feedback", |
| 58 | "semantic-adjuster" |
| 59 | ] |
| 60 | |
| 61 | def take_screenshot(component_name): |
| 62 | """Take screenshot of a single component""" |
| 63 | print(f"📸 Taking screenshot of {component_name}.html...") |
| 64 | |
| 65 | with sync_playwright() as p: |
| 66 | browser = p.chromium.launch() |
| 67 | page = browser.new_page(viewport={"width": 1400, "height": 1200}) |
| 68 | |
| 69 | file_path = Path(f"docs/{component_name}.html").absolute() |
| 70 | page.goto(f"file://{file_path}") |
| 71 | page.wait_for_timeout(1000) |
| 72 | |
| 73 | screenshot_path = SCREENSHOT_DIR / f"{component_name}.html.png" |
| 74 | page.screenshot(path=str(screenshot_path), full_page=True) |
| 75 | |
| 76 | browser.close() |
| 77 | |
| 78 | print(f"✅ Screenshot saved: {screenshot_path}") |
| 79 | return screenshot_path |
| 80 | |
| 81 | def log_api_call(test_name, model, input_tokens, output_tokens, cost): |
| 82 | """Log API call for cost tracking""" |
| 83 | global api_call_stats |
| 84 | |
| 85 | call_data = { |
| 86 | "timestamp": datetime.now().isoformat(), |
| 87 | "test": test_name, |
| 88 | "model": model, |
| 89 | "input_tokens": input_tokens, |
| 90 | "output_tokens": output_tokens, |
| 91 | "cost": cost |
| 92 | } |
| 93 | |
| 94 | api_call_stats["total_calls"] += 1 |
| 95 | api_call_stats["total_input_tokens"] += input_tokens |
| 96 | api_call_stats["total_output_tokens"] += output_tokens |
| 97 | api_call_stats["total_cost"] += cost |
| 98 | api_call_stats["calls"].append(call_data) |
| 99 | |
| 100 | # Append to log file |
| 101 | with open(API_LOG_FILE, "a") as f: |
| 102 | f.write(f"{datetime.now().isoformat()} | {test_name:25s} | Input: {input_tokens:6d} | Output: {output_tokens:5d} | Cost: ${cost:.4f}\n") |
| 103 | |
| 104 | def analyze_with_vision(component_name, screenshot_path): |
| 105 | """Analyze component with Claude's vision API""" |
| 106 | print(f"🤖 Analyzing {component_name} with AI vision...") |
| 107 | |
| 108 | # Read and encode image |
| 109 | with open(screenshot_path, "rb") as f: |
| 110 | image_data = base64.standard_b64encode(f.read()).decode("utf-8") |
| 111 | |
| 112 | results = { |
| 113 | "component": component_name, |
| 114 | "screenshot": str(screenshot_path), |
| 115 | "tests": {} |
| 116 | } |
| 117 | |
| 118 | # Test 1: Contrast |
| 119 | print(f" 🔍 Testing contrast...") |
| 120 | contrast_response = client.messages.create( |
| 121 | model=MODEL, |
| 122 | max_tokens=1024, |
| 123 | messages=[{ |
| 124 | "role": "user", |
| 125 | "content": [ |
| 126 | { |
| 127 | "type": "image", |
| 128 | "source": { |
| 129 | "type": "base64", |
| 130 | "media_type": "image/png", |
| 131 | "data": image_data, |
| 132 | }, |
| 133 | }, |
| 134 | { |
| 135 | "type": "text", |
| 136 | "text": """Analyze this component documentation page for WCAG AA contrast compliance (4.5:1 minimum for normal text, 3:1 for large text). |
| 137 | |
| 138 | Check all text elements including: |
| 139 | - Headings and titles |
| 140 | - Body text and descriptions |
| 141 | - Code examples |
| 142 | - Table text |
| 143 | - State labels and descriptions |
| 144 | - Button text |
| 145 | |
| 146 | For each contrast failure, report: |
| 147 | - Element description |
| 148 | - Estimated contrast ratio |
| 149 | - Colors involved |
| 150 | - Severity (fail if below 4.5:1, warning if 4.5-7.0:1) |
| 151 | |
| 152 | Return JSON: {"passed": true/false, "issues": [{"element": "...", "contrast": "...", "colors": "...", "severity": "fail/warning"}]}""" |
| 153 | } |
| 154 | ] |
| 155 | }] |
| 156 | ) |
| 157 | |
| 158 | # Log API call |
| 159 | usage = contrast_response.usage |
| 160 | cost = (usage.input_tokens * API_COSTS[MODEL]["input"]) + (usage.output_tokens * API_COSTS[MODEL]["output"]) |
| 161 | log_api_call(f"{component_name}:contrast", MODEL, usage.input_tokens, usage.output_tokens, cost) |
| 162 | |
| 163 | contrast_text = contrast_response.content[0].text |
| 164 | # Extract JSON from response |
| 165 | try: |
| 166 | if "```json" in contrast_text: |
| 167 | contrast_text = contrast_text.split("```json")[1].split("```")[0].strip() |
| 168 | results["tests"]["contrast"] = json.loads(contrast_text) |
| 169 | except: |
| 170 | results["tests"]["contrast"] = {"passed": False, "issues": [{"element": "parse_error", "contrast": "N/A", "colors": contrast_text[:200], "severity": "fail"}]} |
| 171 | |
| 172 | # Test 2: State Differentiation |
| 173 | print(f" 🔍 Testing state differentiation...") |
| 174 | states_response = client.messages.create( |
| 175 | model=MODEL, |
| 176 | max_tokens=1024, |
| 177 | messages=[{ |
| 178 | "role": "user", |
| 179 | "content": [ |
| 180 | { |
| 181 | "type": "image", |
| 182 | "source": { |
| 183 | "type": "base64", |
| 184 | "media_type": "image/png", |
| 185 | "data": image_data, |
| 186 | }, |
| 187 | }, |
| 188 | { |
| 189 | "type": "text", |
| 190 | "text": """Look at the "Interactive States" section. Are the different states (Default, Hover, Active, Focus, Disabled) visually distinguishable? |
| 191 | |
| 192 | Each state should look CLEARLY different from the others. Check for: |
| 193 | - Color changes |
| 194 | - Size/scale changes |
| 195 | - Shadow/elevation changes |
| 196 | - Border changes |
| 197 | - Visual effects (brightness, blur, etc) |
| 198 | |
| 199 | Report any states that look identical or too similar. |
| 200 | |
| 201 | Return JSON: {"passed": true/false, "issues": [{"states": "which states look the same", "problem": "description", "severity": "fail/warning"}]}""" |
| 202 | } |
| 203 | ] |
| 204 | }] |
| 205 | ) |
| 206 | |
| 207 | # Log API call |
| 208 | usage = states_response.usage |
| 209 | cost = (usage.input_tokens * API_COSTS[MODEL]["input"]) + (usage.output_tokens * API_COSTS[MODEL]["output"]) |
| 210 | log_api_call(f"{component_name}:state_differentiation", MODEL, usage.input_tokens, usage.output_tokens, cost) |
| 211 | |
| 212 | states_text = states_response.content[0].text |
| 213 | try: |
| 214 | if "```json" in states_text: |
| 215 | states_text = states_text.split("```json")[1].split("```")[0].strip() |
| 216 | results["tests"]["state_differentiation"] = json.loads(states_text) |
| 217 | except: |
| 218 | results["tests"]["state_differentiation"] = {"passed": False, "issues": [{"states": "parse_error", "problem": states_text[:200], "severity": "fail"}]} |
| 219 | |
| 220 | # Test 3: Affordance |
| 221 | print(f" 🔍 Testing affordance...") |
| 222 | affordance_response = client.messages.create( |
| 223 | model=MODEL, |
| 224 | max_tokens=1024, |
| 225 | messages=[{ |
| 226 | "role": "user", |
| 227 | "content": [ |
| 228 | { |
| 229 | "type": "image", |
| 230 | "source": { |
| 231 | "type": "base64", |
| 232 | "media_type": "image/png", |
| 233 | "data": image_data, |
| 234 | }, |
| 235 | }, |
| 236 | { |
| 237 | "type": "text", |
| 238 | "text": """Check if interactive elements clearly look interactive (buttons, inputs, cards, etc). |
| 239 | |
| 240 | Good affordance signals: |
| 241 | - Buttons look pressable (3D depth, shadows, borders) |
| 242 | - Inputs have clear input fields |
| 243 | - Clickable cards have hover effects shown |
| 244 | - Links are underlined or colored distinctly |
| 245 | |
| 246 | Report any interactive elements that don't look interactive. |
| 247 | |
| 248 | Return JSON: {"passed": true/false, "issues": [{"element": "...", "problem": "...", "severity": "fail/warning"}]}""" |
| 249 | } |
| 250 | ] |
| 251 | }] |
| 252 | ) |
| 253 | |
| 254 | # Log API call |
| 255 | usage = affordance_response.usage |
| 256 | cost = (usage.input_tokens * API_COSTS[MODEL]["input"]) + (usage.output_tokens * API_COSTS[MODEL]["output"]) |
| 257 | log_api_call(f"{component_name}:affordance", MODEL, usage.input_tokens, usage.output_tokens, cost) |
| 258 | |
| 259 | affordance_text = affordance_response.content[0].text |
| 260 | try: |
| 261 | if "```json" in affordance_text: |
| 262 | affordance_text = affordance_text.split("```json")[1].split("```")[0].strip() |
| 263 | results["tests"]["affordance"] = json.loads(affordance_text) |
| 264 | except: |
| 265 | results["tests"]["affordance"] = {"passed": False, "issues": [{"element": "parse_error", "problem": affordance_text[:200], "severity": "fail"}]} |
| 266 | |
| 267 | # Test 4: Visual Feedback |
| 268 | print(f" 🔍 Testing visual feedback...") |
| 269 | feedback_response = client.messages.create( |
| 270 | model=MODEL, |
| 271 | max_tokens=1024, |
| 272 | messages=[{ |
| 273 | "role": "user", |
| 274 | "content": [ |
| 275 | { |
| 276 | "type": "image", |
| 277 | "source": { |
| 278 | "type": "base64", |
| 279 | "media_type": "image/png", |
| 280 | "data": image_data, |
| 281 | }, |
| 282 | }, |
| 283 | { |
| 284 | "type": "text", |
| 285 | "text": """Check if state changes provide clear visual feedback. |
| 286 | |
| 287 | Look at the Interactive States section - do hover/active/focus states show OBVIOUS changes? |
| 288 | - Hover should be clearly visible (lift, glow, color change) |
| 289 | - Active/pressed should be obvious (scale down, darker) |
| 290 | - Focus should have clear outline |
| 291 | - Disabled should look clearly disabled (greyed out) |
| 292 | |
| 293 | Report any states with insufficient visual feedback. |
| 294 | |
| 295 | Return JSON: {"passed": true/false, "issues": [{"state": "...", "problem": "...", "severity": "fail/warning"}]}""" |
| 296 | } |
| 297 | ] |
| 298 | }] |
| 299 | ) |
| 300 | |
| 301 | # Log API call |
| 302 | usage = feedback_response.usage |
| 303 | cost = (usage.input_tokens * API_COSTS[MODEL]["input"]) + (usage.output_tokens * API_COSTS[MODEL]["output"]) |
| 304 | log_api_call(f"{component_name}:visual_feedback", MODEL, usage.input_tokens, usage.output_tokens, cost) |
| 305 | |
| 306 | feedback_text = feedback_response.content[0].text |
| 307 | try: |
| 308 | if "```json" in feedback_text: |
| 309 | feedback_text = feedback_text.split("```json")[1].split("```")[0].strip() |
| 310 | results["tests"]["visual_feedback"] = json.loads(feedback_text) |
| 311 | except: |
| 312 | results["tests"]["visual_feedback"] = {"passed": False, "issues": [{"state": "parse_error", "problem": feedback_text[:200], "severity": "fail"}]} |
| 313 | |
| 314 | # Determine overall pass/fail |
| 315 | results["passed"] = all( |
| 316 | test.get("passed", False) |
| 317 | for test in results["tests"].values() |
| 318 | ) |
| 319 | |
| 320 | return results |
| 321 | |
| 322 | def print_results(results): |
| 323 | """Print results in a readable format""" |
| 324 | component = results["component"] |
| 325 | |
| 326 | print(f"\n{'='*60}") |
| 327 | print(f"📊 Results for {component}") |
| 328 | print(f"{'='*60}\n") |
| 329 | |
| 330 | for test_name, test_data in results["tests"].items(): |
| 331 | status = "✅ PASSED" if test_data.get("passed", False) else "❌ FAILED" |
| 332 | print(f"{test_name.upper()}: {status}") |
| 333 | |
| 334 | if test_data.get("issues"): |
| 335 | for issue in test_data["issues"]: |
| 336 | severity = issue.get("severity", "unknown") |
| 337 | emoji = "❌" if severity == "fail" else "⚠️" |
| 338 | print(f" {emoji} {issue}") |
| 339 | print() |
| 340 | |
| 341 | overall = "✅ PASSED" if results["passed"] else "❌ FAILED" |
| 342 | print(f"Overall: {overall}\n") |
| 343 | |
| 344 | def main(): |
| 345 | if len(sys.argv) != 2: |
| 346 | print("Usage: python3 test_single_visual.py <component-name>") |
| 347 | print(f"Available components: {', '.join(COMPONENTS)}") |
| 348 | sys.exit(1) |
| 349 | |
| 350 | component_name = sys.argv[1] |
| 351 | |
| 352 | if component_name not in COMPONENTS: |
| 353 | print(f"❌ Unknown component: {component_name}") |
| 354 | print(f"Available components: {', '.join(COMPONENTS)}") |
| 355 | sys.exit(1) |
| 356 | |
| 357 | # Ensure directories exist |
| 358 | SCREENSHOT_DIR.mkdir(exist_ok=True) |
| 359 | RESULTS_DIR.mkdir(exist_ok=True) |
| 360 | |
| 361 | # Take screenshot |
| 362 | screenshot_path = take_screenshot(component_name) |
| 363 | |
| 364 | # Analyze with AI |
| 365 | results = analyze_with_vision(component_name, screenshot_path) |
| 366 | |
| 367 | # Save results |
| 368 | result_file = RESULTS_DIR / f"result-{component_name}.json" |
| 369 | with open(result_file, "w") as f: |
| 370 | json.dump(results, f, indent=2) |
| 371 | print(f"💾 Results saved to {result_file}") |
| 372 | |
| 373 | # Print results |
| 374 | print_results(results) |
| 375 | |
| 376 | # Print API cost summary |
| 377 | print(f"\n{'='*60}") |
| 378 | print(f"💰 API Cost Summary") |
| 379 | print(f"{'='*60}") |
| 380 | print(f"Total API Calls: {api_call_stats['total_calls']}") |
| 381 | print(f"Input Tokens: {api_call_stats['total_input_tokens']:,}") |
| 382 | print(f"Output Tokens: {api_call_stats['total_output_tokens']:,}") |
| 383 | print(f"Total Cost: ${api_call_stats['total_cost']:.4f}") |
| 384 | print() |
| 385 | |
| 386 | # Exit with appropriate code |
| 387 | sys.exit(0 if results["passed"] else 1) |
| 388 | |
| 389 | if __name__ == "__main__": |
| 390 | main() |