WordPress-style content management with block-based pages, stored as JSON files on disk instead of MySQL.

0 downloads

dbbasic-content/dbbasic_content/wordpress_importer.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-content.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-content.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-content.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 Release v0.2.0 - Pages support 25b7e5e 1 year, 1 month ago History
wordpress_importer.py · 11.3 KB · 371 lines
1"""
2WordPress importer: Convert wp_posts to JSON blocks.
3
4Extracts content from WordPress MySQL database and converts to
5block-based JSON format.
6"""
7
8import json
9import re
10from pathlib import Path
11from typing import Dict, List, Any, Optional
12from html.parser import HTMLParser
13
14
15class HTMLToBlocksParser(HTMLParser):
16 """
17 Parse WordPress HTML content into blocks.
18
19 WordPress stores content as HTML. We need to extract it into
20 structured blocks.
21 """
22
23 def __init__(self):
24 super().__init__()
25 self.blocks = []
26 self.current_tag = None
27 self.current_data = []
28 self.list_items = []
29 self.in_list = False
30
31 def handle_starttag(self, tag, attrs):
32 """Handle opening HTML tags."""
33 self.current_tag = tag
34
35 if tag in ['ul', 'ol']:
36 self.in_list = True
37 self.list_type = 'unordered' if tag == 'ul' else 'ordered'
38 self.list_items = []
39
40 def handle_endtag(self, tag):
41 """Handle closing HTML tags."""
42 if tag == 'p' and self.current_data:
43 content = ''.join(self.current_data).strip()
44 if content:
45 self.blocks.append({
46 'type': 'paragraph',
47 'data': {'content': content}
48 })
49 self.current_data = []
50
51 elif tag in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'] and self.current_data:
52 level = int(tag[1])
53 content = ''.join(self.current_data).strip()
54 if content:
55 self.blocks.append({
56 'type': 'heading',
57 'data': {'level': level, 'content': content}
58 })
59 self.current_data = []
60
61 elif tag == 'li' and self.current_data:
62 content = ''.join(self.current_data).strip()
63 if content:
64 self.list_items.append(content)
65 self.current_data = []
66
67 elif tag in ['ul', 'ol']:
68 if self.list_items:
69 self.blocks.append({
70 'type': 'list',
71 'data': {
72 'listType': self.list_type,
73 'items': self.list_items
74 }
75 })
76 self.list_items = []
77 self.in_list = False
78
79 elif tag == 'blockquote' and self.current_data:
80 content = ''.join(self.current_data).strip()
81 if content:
82 self.blocks.append({
83 'type': 'quote',
84 'data': {'content': content}
85 })
86 self.current_data = []
87
88 elif tag == 'pre' and self.current_data:
89 code = ''.join(self.current_data).strip()
90 if code:
91 self.blocks.append({
92 'type': 'code',
93 'data': {'code': code, 'language': 'text'}
94 })
95 self.current_data = []
96
97 self.current_tag = None
98
99 def handle_data(self, data):
100 """Handle text data between tags."""
101 if self.current_tag:
102 self.current_data.append(data)
103 else:
104 # Text outside tags - treat as paragraph
105 data = data.strip()
106 if data:
107 self.current_data.append(data)
108
109 def get_blocks(self) -> List[Dict[str, Any]]:
110 """Get parsed blocks."""
111 # Handle any remaining data
112 if self.current_data and not self.in_list:
113 content = ''.join(self.current_data).strip()
114 if content:
115 self.blocks.append({
116 'type': 'paragraph',
117 'data': {'content': content}
118 })
119
120 return self.blocks
121
122
123class WordPressImporter:
124 """
125 Import WordPress content to JSON block format.
126
127 Connects to WordPress MySQL database and extracts posts,
128 converting them to our block-based JSON format.
129 """
130
131 def __init__(
132 self,
133 host: str = "localhost",
134 database: str = "wordpress",
135 user: str = "root",
136 password: str = "",
137 port: int = 3306,
138 table_prefix: str = "wp_"
139 ):
140 """
141 Initialize WordPress importer.
142
143 Args:
144 host: MySQL host
145 database: Database name
146 user: MySQL user
147 password: MySQL password
148 port: MySQL port
149 table_prefix: WordPress table prefix (default: wp_)
150 """
151 self.host = host
152 self.database = database
153 self.user = user
154 self.password = password
155 self.port = port
156 self.table_prefix = table_prefix
157 self._connection = None
158
159 def _connect(self):
160 """Connect to MySQL database (lazy)."""
161 if self._connection is None:
162 try:
163 import pymysql
164 except ImportError:
165 raise ImportError(
166 "pymysql required for WordPress import. "
167 "Install with: pip install dbbasic-content[wordpress]"
168 )
169
170 self._connection = pymysql.connect(
171 host=self.host,
172 port=self.port,
173 user=self.user,
174 password=self.password,
175 database=self.database,
176 charset='utf8mb4',
177 cursorclass=pymysql.cursors.DictCursor
178 )
179
180 return self._connection
181
182 def _get_posts(self, post_type: str = 'post') -> List[Dict[str, Any]]:
183 """
184 Get posts from WordPress database.
185
186 Args:
187 post_type: WordPress post type (post, page, etc.)
188
189 Returns:
190 List of post dicts
191 """
192 conn = self._connect()
193 with conn.cursor() as cursor:
194 query = f"""
195 SELECT
196 ID,
197 post_title,
198 post_name,
199 post_content,
200 post_excerpt,
201 post_status,
202 post_date,
203 post_author
204 FROM {self.table_prefix}posts
205 WHERE post_type = %s
206 AND post_status IN ('publish', 'draft')
207 ORDER BY post_date DESC
208 """
209 cursor.execute(query, (post_type,))
210 return cursor.fetchall()
211
212 def _get_post_meta(self, post_id: int) -> Dict[str, Any]:
213 """Get post metadata."""
214 conn = self._connect()
215 meta = {}
216 with conn.cursor() as cursor:
217 query = f"""
218 SELECT meta_key, meta_value
219 FROM {self.table_prefix}postmeta
220 WHERE post_id = %s
221 """
222 cursor.execute(query, (post_id,))
223 for row in cursor.fetchall():
224 meta[row['meta_key']] = row['meta_value']
225 return meta
226
227 def _get_terms(self, post_id: int, taxonomy: str) -> List[str]:
228 """Get post terms (categories, tags)."""
229 conn = self._connect()
230 terms = []
231 with conn.cursor() as cursor:
232 query = f"""
233 SELECT t.name
234 FROM {self.table_prefix}terms t
235 JOIN {self.table_prefix}term_taxonomy tt ON t.term_id = tt.term_id
236 JOIN {self.table_prefix}term_relationships tr ON tt.term_taxonomy_id = tr.term_taxonomy_id
237 WHERE tr.object_id = %s
238 AND tt.taxonomy = %s
239 """
240 cursor.execute(query, (post_id, taxonomy))
241 terms = [row['name'] for row in cursor.fetchall()]
242 return terms
243
244 def _convert_content_to_blocks(self, content: str) -> List[Dict[str, Any]]:
245 """
246 Convert WordPress HTML content to blocks.
247
248 Args:
249 content: WordPress post_content HTML
250
251 Returns:
252 List of blocks
253 """
254 if not content or not content.strip():
255 return []
256
257 # Parse HTML into blocks
258 parser = HTMLToBlocksParser()
259 parser.feed(content)
260 blocks = parser.get_blocks()
261
262 # If no blocks parsed, treat entire content as single paragraph
263 if not blocks:
264 blocks = [{
265 'type': 'paragraph',
266 'data': {'content': content.strip()}
267 }]
268
269 return blocks
270
271 def import_posts(
272 self,
273 output_dir: str,
274 post_type: str = 'post',
275 status_filter: Optional[List[str]] = None
276 ) -> int:
277 """
278 Import WordPress posts to JSON files.
279
280 Args:
281 output_dir: Directory to write JSON files
282 post_type: WordPress post type to import
283 status_filter: Filter by post status (None = all)
284
285 Returns:
286 Number of posts imported
287 """
288 output_path = Path(output_dir)
289 articles_dir = output_path / "articles"
290 articles_dir.mkdir(parents=True, exist_ok=True)
291
292 posts = self._get_posts(post_type)
293 imported_count = 0
294
295 for wp_post in posts:
296 # Filter by status if specified
297 if status_filter and wp_post['post_status'] not in status_filter:
298 continue
299
300 # Get additional data
301 categories = self._get_terms(wp_post['ID'], 'category')
302 tags = self._get_terms(wp_post['ID'], 'post_tag')
303 meta = self._get_post_meta(wp_post['ID'])
304
305 # Convert content to blocks
306 blocks = self._convert_content_to_blocks(wp_post['post_content'])
307
308 # Build article JSON
309 article = {
310 'slug': wp_post['post_name'] or f"post-{wp_post['ID']}",
311 'title': wp_post['post_title'],
312 'date': wp_post['post_date'].strftime('%Y-%m-%d'),
313 'author': str(wp_post['post_author']),
314 'status': wp_post['post_status'],
315 'blocks': blocks,
316 }
317
318 if wp_post['post_excerpt']:
319 article['description'] = wp_post['post_excerpt']
320
321 if categories:
322 article['categories'] = categories
323
324 if tags:
325 article['tags'] = tags
326
327 # Add WordPress ID for reference
328 article['wordpress_id'] = wp_post['ID']
329
330 # Write to file
331 filename = f"{article['slug']}.json"
332 filepath = articles_dir / filename
333
334 with open(filepath, 'w', encoding='utf-8') as f:
335 json.dump(article, f, indent=2, ensure_ascii=False)
336
337 imported_count += 1
338
339 return imported_count
340
341 def import_to(self, output_dir: str) -> Dict[str, int]:
342 """
343 Import all WordPress content to directory.
344
345 Args:
346 output_dir: Directory to write content
347
348 Returns:
349 Dict with counts of imported items
350 """
351 stats = {
352 'posts': self.import_posts(output_dir, post_type='post', status_filter=['publish']),
353 'pages': self.import_posts(output_dir, post_type='page', status_filter=['publish']),
354 'drafts': self.import_posts(output_dir, post_type='post', status_filter=['draft']),
355 }
356
357 return stats
358
359 def close(self):
360 """Close database connection."""
361 if self._connection:
362 self._connection.close()
363 self._connection = None
364
365 def __enter__(self):
366 """Context manager support."""
367 return self
368
369 def __exit__(self, exc_type, exc_val, exc_tb):
370 """Context manager cleanup."""
371 self.close()