askrobots / dbbasic-content
Public
WordPress-style content management with block-based pages, stored as JSON files on disk instead of MySQL.
wordpress_importer.py · 11.3 KB · 371 lines
| 1 | """ |
| 2 | WordPress importer: Convert wp_posts to JSON blocks. |
| 3 | |
| 4 | Extracts content from WordPress MySQL database and converts to |
| 5 | block-based JSON format. |
| 6 | """ |
| 7 | |
| 8 | import json |
| 9 | import re |
| 10 | from pathlib import Path |
| 11 | from typing import Dict, List, Any, Optional |
| 12 | from html.parser import HTMLParser |
| 13 | |
| 14 | |
| 15 | class HTMLToBlocksParser(HTMLParser): |
| 16 | """ |
| 17 | Parse WordPress HTML content into blocks. |
| 18 | |
| 19 | WordPress stores content as HTML. We need to extract it into |
| 20 | structured blocks. |
| 21 | """ |
| 22 | |
| 23 | def __init__(self): |
| 24 | super().__init__() |
| 25 | self.blocks = [] |
| 26 | self.current_tag = None |
| 27 | self.current_data = [] |
| 28 | self.list_items = [] |
| 29 | self.in_list = False |
| 30 | |
| 31 | def handle_starttag(self, tag, attrs): |
| 32 | """Handle opening HTML tags.""" |
| 33 | self.current_tag = tag |
| 34 | |
| 35 | if tag in ['ul', 'ol']: |
| 36 | self.in_list = True |
| 37 | self.list_type = 'unordered' if tag == 'ul' else 'ordered' |
| 38 | self.list_items = [] |
| 39 | |
| 40 | def handle_endtag(self, tag): |
| 41 | """Handle closing HTML tags.""" |
| 42 | if tag == 'p' and self.current_data: |
| 43 | content = ''.join(self.current_data).strip() |
| 44 | if content: |
| 45 | self.blocks.append({ |
| 46 | 'type': 'paragraph', |
| 47 | 'data': {'content': content} |
| 48 | }) |
| 49 | self.current_data = [] |
| 50 | |
| 51 | elif tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'] and self.current_data: |
| 52 | level = int(tag[1]) |
| 53 | content = ''.join(self.current_data).strip() |
| 54 | if content: |
| 55 | self.blocks.append({ |
| 56 | 'type': 'heading', |
| 57 | 'data': {'level': level, 'content': content} |
| 58 | }) |
| 59 | self.current_data = [] |
| 60 | |
| 61 | elif tag == 'li' and self.current_data: |
| 62 | content = ''.join(self.current_data).strip() |
| 63 | if content: |
| 64 | self.list_items.append(content) |
| 65 | self.current_data = [] |
| 66 | |
| 67 | elif tag in ['ul', 'ol']: |
| 68 | if self.list_items: |
| 69 | self.blocks.append({ |
| 70 | 'type': 'list', |
| 71 | 'data': { |
| 72 | 'listType': self.list_type, |
| 73 | 'items': self.list_items |
| 74 | } |
| 75 | }) |
| 76 | self.list_items = [] |
| 77 | self.in_list = False |
| 78 | |
| 79 | elif tag == 'blockquote' and self.current_data: |
| 80 | content = ''.join(self.current_data).strip() |
| 81 | if content: |
| 82 | self.blocks.append({ |
| 83 | 'type': 'quote', |
| 84 | 'data': {'content': content} |
| 85 | }) |
| 86 | self.current_data = [] |
| 87 | |
| 88 | elif tag == 'pre' and self.current_data: |
| 89 | code = ''.join(self.current_data).strip() |
| 90 | if code: |
| 91 | self.blocks.append({ |
| 92 | 'type': 'code', |
| 93 | 'data': {'code': code, 'language': 'text'} |
| 94 | }) |
| 95 | self.current_data = [] |
| 96 | |
| 97 | self.current_tag = None |
| 98 | |
| 99 | def handle_data(self, data): |
| 100 | """Handle text data between tags.""" |
| 101 | if self.current_tag: |
| 102 | self.current_data.append(data) |
| 103 | else: |
| 104 | # Text outside tags - treat as paragraph |
| 105 | data = data.strip() |
| 106 | if data: |
| 107 | self.current_data.append(data) |
| 108 | |
| 109 | def get_blocks(self) -> List[Dict[str, Any]]: |
| 110 | """Get parsed blocks.""" |
| 111 | # Handle any remaining data |
| 112 | if self.current_data and not self.in_list: |
| 113 | content = ''.join(self.current_data).strip() |
| 114 | if content: |
| 115 | self.blocks.append({ |
| 116 | 'type': 'paragraph', |
| 117 | 'data': {'content': content} |
| 118 | }) |
| 119 | |
| 120 | return self.blocks |
| 121 | |
| 122 | |
| 123 | class WordPressImporter: |
| 124 | """ |
| 125 | Import WordPress content to JSON block format. |
| 126 | |
| 127 | Connects to WordPress MySQL database and extracts posts, |
| 128 | converting them to our block-based JSON format. |
| 129 | """ |
| 130 | |
| 131 | def __init__( |
| 132 | self, |
| 133 | host: str = "localhost", |
| 134 | database: str = "wordpress", |
| 135 | user: str = "root", |
| 136 | password: str = "", |
| 137 | port: int = 3306, |
| 138 | table_prefix: str = "wp_" |
| 139 | ): |
| 140 | """ |
| 141 | Initialize WordPress importer. |
| 142 | |
| 143 | Args: |
| 144 | host: MySQL host |
| 145 | database: Database name |
| 146 | user: MySQL user |
| 147 | password: MySQL password |
| 148 | port: MySQL port |
| 149 | table_prefix: WordPress table prefix (default: wp_) |
| 150 | """ |
| 151 | self.host = host |
| 152 | self.database = database |
| 153 | self.user = user |
| 154 | self.password = password |
| 155 | self.port = port |
| 156 | self.table_prefix = table_prefix |
| 157 | self._connection = None |
| 158 | |
| 159 | def _connect(self): |
| 160 | """Connect to MySQL database (lazy).""" |
| 161 | if self._connection is None: |
| 162 | try: |
| 163 | import pymysql |
| 164 | except ImportError: |
| 165 | raise ImportError( |
| 166 | "pymysql required for WordPress import. " |
| 167 | "Install with: pip install dbbasic-content[wordpress]" |
| 168 | ) |
| 169 | |
| 170 | self._connection = pymysql.connect( |
| 171 | host=self.host, |
| 172 | port=self.port, |
| 173 | user=self.user, |
| 174 | password=self.password, |
| 175 | database=self.database, |
| 176 | charset='utf8mb4', |
| 177 | cursorclass=pymysql.cursors.DictCursor |
| 178 | ) |
| 179 | |
| 180 | return self._connection |
| 181 | |
| 182 | def _get_posts(self, post_type: str = 'post') -> List[Dict[str, Any]]: |
| 183 | """ |
| 184 | Get posts from WordPress database. |
| 185 | |
| 186 | Args: |
| 187 | post_type: WordPress post type (post, page, etc.) |
| 188 | |
| 189 | Returns: |
| 190 | List of post dicts |
| 191 | """ |
| 192 | conn = self._connect() |
| 193 | with conn.cursor() as cursor: |
| 194 | query = f""" |
| 195 | SELECT |
| 196 | ID, |
| 197 | post_title, |
| 198 | post_name, |
| 199 | post_content, |
| 200 | post_excerpt, |
| 201 | post_status, |
| 202 | post_date, |
| 203 | post_author |
| 204 | FROM {self.table_prefix}posts |
| 205 | WHERE post_type = %s |
| 206 | AND post_status IN ('publish', 'draft') |
| 207 | ORDER BY post_date DESC |
| 208 | """ |
| 209 | cursor.execute(query, (post_type,)) |
| 210 | return cursor.fetchall() |
| 211 | |
| 212 | def _get_post_meta(self, post_id: int) -> Dict[str, Any]: |
| 213 | """Get post metadata.""" |
| 214 | conn = self._connect() |
| 215 | meta = {} |
| 216 | with conn.cursor() as cursor: |
| 217 | query = f""" |
| 218 | SELECT meta_key, meta_value |
| 219 | FROM {self.table_prefix}postmeta |
| 220 | WHERE post_id = %s |
| 221 | """ |
| 222 | cursor.execute(query, (post_id,)) |
| 223 | for row in cursor.fetchall(): |
| 224 | meta[row['meta_key']] = row['meta_value'] |
| 225 | return meta |
| 226 | |
| 227 | def _get_terms(self, post_id: int, taxonomy: str) -> List[str]: |
| 228 | """Get post terms (categories, tags).""" |
| 229 | conn = self._connect() |
| 230 | terms = [] |
| 231 | with conn.cursor() as cursor: |
| 232 | query = f""" |
| 233 | SELECT t.name |
| 234 | FROM {self.table_prefix}terms t |
| 235 | JOIN {self.table_prefix}term_taxonomy tt ON t.term_id = tt.term_id |
| 236 | JOIN {self.table_prefix}term_relationships tr ON tt.term_taxonomy_id = tr.term_taxonomy_id |
| 237 | WHERE tr.object_id = %s |
| 238 | AND tt.taxonomy = %s |
| 239 | """ |
| 240 | cursor.execute(query, (post_id, taxonomy)) |
| 241 | terms = [row['name'] for row in cursor.fetchall()] |
| 242 | return terms |
| 243 | |
| 244 | def _convert_content_to_blocks(self, content: str) -> List[Dict[str, Any]]: |
| 245 | """ |
| 246 | Convert WordPress HTML content to blocks. |
| 247 | |
| 248 | Args: |
| 249 | content: WordPress post_content HTML |
| 250 | |
| 251 | Returns: |
| 252 | List of blocks |
| 253 | """ |
| 254 | if not content or not content.strip(): |
| 255 | return [] |
| 256 | |
| 257 | # Parse HTML into blocks |
| 258 | parser = HTMLToBlocksParser() |
| 259 | parser.feed(content) |
| 260 | blocks = parser.get_blocks() |
| 261 | |
| 262 | # If no blocks parsed, treat entire content as single paragraph |
| 263 | if not blocks: |
| 264 | blocks = [{ |
| 265 | 'type': 'paragraph', |
| 266 | 'data': {'content': content.strip()} |
| 267 | }] |
| 268 | |
| 269 | return blocks |
| 270 | |
| 271 | def import_posts( |
| 272 | self, |
| 273 | output_dir: str, |
| 274 | post_type: str = 'post', |
| 275 | status_filter: Optional[List[str]] = None |
| 276 | ) -> int: |
| 277 | """ |
| 278 | Import WordPress posts to JSON files. |
| 279 | |
| 280 | Args: |
| 281 | output_dir: Directory to write JSON files |
| 282 | post_type: WordPress post type to import |
| 283 | status_filter: Filter by post status (None = all) |
| 284 | |
| 285 | Returns: |
| 286 | Number of posts imported |
| 287 | """ |
| 288 | output_path = Path(output_dir) |
| 289 | articles_dir = output_path / "articles" |
| 290 | articles_dir.mkdir(parents=True, exist_ok=True) |
| 291 | |
| 292 | posts = self._get_posts(post_type) |
| 293 | imported_count = 0 |
| 294 | |
| 295 | for wp_post in posts: |
| 296 | # Filter by status if specified |
| 297 | if status_filter and wp_post['post_status'] not in status_filter: |
| 298 | continue |
| 299 | |
| 300 | # Get additional data |
| 301 | categories = self._get_terms(wp_post['ID'], 'category') |
| 302 | tags = self._get_terms(wp_post['ID'], 'post_tag') |
| 303 | meta = self._get_post_meta(wp_post['ID']) |
| 304 | |
| 305 | # Convert content to blocks |
| 306 | blocks = self._convert_content_to_blocks(wp_post['post_content']) |
| 307 | |
| 308 | # Build article JSON |
| 309 | article = { |
| 310 | 'slug': wp_post['post_name'] or f"post-{wp_post['ID']}", |
| 311 | 'title': wp_post['post_title'], |
| 312 | 'date': wp_post['post_date'].strftime('%Y-%m-%d'), |
| 313 | 'author': str(wp_post['post_author']), |
| 314 | 'status': wp_post['post_status'], |
| 315 | 'blocks': blocks, |
| 316 | } |
| 317 | |
| 318 | if wp_post['post_excerpt']: |
| 319 | article['description'] = wp_post['post_excerpt'] |
| 320 | |
| 321 | if categories: |
| 322 | article['categories'] = categories |
| 323 | |
| 324 | if tags: |
| 325 | article['tags'] = tags |
| 326 | |
| 327 | # Add WordPress ID for reference |
| 328 | article['wordpress_id'] = wp_post['ID'] |
| 329 | |
| 330 | # Write to file |
| 331 | filename = f"{article['slug']}.json" |
| 332 | filepath = articles_dir / filename |
| 333 | |
| 334 | with open(filepath, 'w', encoding='utf-8') as f: |
| 335 | json.dump(article, f, indent=2, ensure_ascii=False) |
| 336 | |
| 337 | imported_count += 1 |
| 338 | |
| 339 | return imported_count |
| 340 | |
| 341 | def import_to(self, output_dir: str) -> Dict[str, int]: |
| 342 | """ |
| 343 | Import all WordPress content to directory. |
| 344 | |
| 345 | Args: |
| 346 | output_dir: Directory to write content |
| 347 | |
| 348 | Returns: |
| 349 | Dict with counts of imported items |
| 350 | """ |
| 351 | stats = { |
| 352 | 'posts': self.import_posts(output_dir, post_type='post', status_filter=['publish']), |
| 353 | 'pages': self.import_posts(output_dir, post_type='page', status_filter=['publish']), |
| 354 | 'drafts': self.import_posts(output_dir, post_type='post', status_filter=['draft']), |
| 355 | } |
| 356 | |
| 357 | return stats |
| 358 | |
| 359 | def close(self): |
| 360 | """Close database connection.""" |
| 361 | if self._connection: |
| 362 | self._connection.close() |
| 363 | self._connection = None |
| 364 | |
| 365 | def __enter__(self): |
| 366 | """Context manager support.""" |
| 367 | return self |
| 368 | |
| 369 | def __exit__(self, exc_type, exc_val, exc_tb): |
| 370 | """Context manager cleanup.""" |
| 371 | self.close() |