wpsync 1.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of wpsync might be problematic. Click here for more details.

@@ -0,0 +1,565 @@
1
+ """Post content: WordPress HTML to the post Markdown dialect and back.
2
+
3
+ Conversion is checked against a model of what WordPress displays. Content
4
+ with block markup is displayed block by block; content without it goes
5
+ through WordPress's `wpautop` filter. The stored post and the post a push
6
+ would send are both put through that model, and any part of the post whose
7
+ Markdown would not display the same is kept as its exact original source.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import re
14
+ from collections.abc import Iterator
15
+ from dataclasses import dataclass
16
+
17
+ from markdown_it.token import Token
18
+
19
+ from wpsync.markup.dom import (
20
+ HTML_SPACE,
21
+ Comment,
22
+ Element,
23
+ Node,
24
+ canonical,
25
+ parse_html,
26
+ to_html,
27
+ trim_block_whitespace,
28
+ )
29
+ from wpsync.markup.markdown import (
30
+ Converter,
31
+ Unconvertible,
32
+ join_blocks,
33
+ kept_markdown,
34
+ pieces,
35
+ render_inline,
36
+ render_tokens,
37
+ )
38
+ from wpsync.media import Media
39
+ from wpsync.posts import Conversion, UnrepresentableContent
40
+ from wpsync.wordpress.autop import BLOCK_NAMES, autop
41
+
42
+ EDITOR_CLASSES = frozenset(
43
+ {
44
+ "wp-block-heading",
45
+ "wp-block-list",
46
+ "wp-block-quote",
47
+ "wp-block-code",
48
+ "wp-block-separator",
49
+ "has-alpha-channel-opacity",
50
+ "wp-block-paragraph",
51
+ }
52
+ )
53
+ """Classes the block editor adds to unstyled elements."""
54
+
55
+ STATIC_BLOCKS = frozenset(
56
+ "core/" + name
57
+ for name in """paragraph heading list list-item quote code preformatted separator
58
+ image gallery table html columns column group spacer pullquote verse buttons
59
+ button details freeform more nextpage media-text cover audio video file""".split()
60
+ )
61
+ """Blocks whose stored HTML is what WordPress displays."""
62
+
63
+ DELIMITER = re.compile(
64
+ r"<!--\s+(?P<closer>/)?wp:(?P<namespace>[a-z][a-z0-9_-]*/)?"
65
+ r"(?P<name>[a-z][a-z0-9_-]*)\s+(?P<attrs>\{.*?\}\s+)?(?P<void>/)?-->",
66
+ re.DOTALL,
67
+ )
68
+ WP_IMAGE_CLASS = re.compile(r"wp-image-(\d+)")
69
+ SIZE_CLASS = re.compile(r"size-([\w-]+)")
70
+ PLACEMENT_CLASS = re.compile(r"align(?:left|right|center|wide|full|none)")
71
+ CAPTION_CLASS = "wp-element-caption"
72
+ EMBED_ATTRS = frozenset({"url", "type", "providerNameSlug", "responsive"})
73
+ EMBED_CLASS = re.compile(
74
+ r"wp-block-embed(?:-[\w-]+)?|is-type-[\w-]+|is-provider-[\w-]+"
75
+ r"|wp-embed-aspect-[\w-]+|wp-has-aspect-ratio"
76
+ )
77
+ EMBED_URL = re.compile(r"https?://[^\s<>\"]+")
78
+ CLASSIC_MARKUP = re.compile(
79
+ r"<!--.*?-->|<(script|style|pre|textarea)\b.*?</\1\s*>"
80
+ r"|<(/?)([a-zA-Z][\w:-]*)\b[^>]*?(/?)>|(\n[ \t]*\n)",
81
+ re.DOTALL | re.IGNORECASE,
82
+ )
83
+ VOID_NAMES = frozenset(
84
+ "area base br col embed hr img input link meta param source track wbr".split()
85
+ )
86
+
87
+
88
+ def is_block_content(raw: str) -> bool:
89
+ return "<!-- wp:" in raw
90
+
91
+
92
+ # Push ----------------------------------------------------------------------
93
+
94
+
95
+ def render_html(markdown: str, media: Media) -> str:
96
+ """The HTML a push sends; raw HTML from the Markdown is sent exactly."""
97
+ html = []
98
+ for piece in pieces(markdown):
99
+ if piece.raw is not None:
100
+ html.append(piece.raw)
101
+ else:
102
+ html.append(to_html(render_tokens(_present(piece.tokens, media))))
103
+ return "\n\n".join(html)
104
+
105
+
106
+ def _present(tokens: list[Token], media: Media) -> list[Token]:
107
+ """Use WordPress's presentation for Markdown images and tables."""
108
+ presented: list[Token] = []
109
+ index = 0
110
+ while index < len(tokens):
111
+ token = tokens[index]
112
+ window = tokens[index : index + 3]
113
+ if [t.type for t in window] == ["paragraph_open", "inline", "paragraph_close"]:
114
+ children = [
115
+ c
116
+ for c in window[1].children or []
117
+ if not (c.type == "text" and not c.content.strip(HTML_SPACE))
118
+ ]
119
+ if len(children) == 1 and children[0].type == "image":
120
+ image = render_inline(children[0])[0]
121
+ assert isinstance(image, Element)
122
+ presented.append(_html_token(to_html([_image_figure(image, media)])))
123
+ index += 3
124
+ continue
125
+ if token.type == "table_open":
126
+ presented.append(_html_token('<figure class="wp-block-table">'))
127
+ presented.append(token)
128
+ if token.type == "table_close":
129
+ presented.append(_html_token("</figure>"))
130
+ index += 1
131
+ return presented
132
+
133
+
134
+ def _html_token(html: str) -> Token:
135
+ token = Token("html_block", "", 0)
136
+ token.content = html + "\n"
137
+ return token
138
+
139
+
140
+ def _image_figure(image: Element, media: Media) -> Element:
141
+ """WordPress's image figure: placement on the figure, the rest on the image."""
142
+ attrs = dict(image.attrs)
143
+ title = attrs.pop("title", None)
144
+ classes = attrs.pop("class", "").split()
145
+ figure_classes = ["wp-block-image"]
146
+ image_classes = [c for c in classes if not PLACEMENT_CLASS.fullmatch(c)]
147
+ library = media.lookup(attrs["src"])
148
+ if library is not None:
149
+ attrs["src"] = library.url
150
+ figure_classes.append(f"size-{library.size}")
151
+ image_classes.append(f"wp-image-{library.id}")
152
+ figure_classes.extend(c for c in classes if PLACEMENT_CLASS.fullmatch(c))
153
+ if image_classes:
154
+ attrs["class"] = " ".join(image_classes)
155
+ children: list[Node] = [Element("img", attrs)]
156
+ if title:
157
+ children.append(Element("figcaption", {"class": CAPTION_CLASS}, [title]))
158
+ return Element("figure", {"class": " ".join(figure_classes)}, children)
159
+
160
+
161
+ # What WordPress displays ---------------------------------------------------
162
+
163
+
164
+ @dataclass(frozen=True)
165
+ class _Unit:
166
+ """A part of the stored content that converts, or is kept, as a whole."""
167
+
168
+ source: str
169
+ """The exact original text."""
170
+ nodes: list[Node]
171
+ """What WordPress displays for it."""
172
+ label: str
173
+ convertible: bool
174
+ separator: str = ""
175
+ """The whitespace before it in the original."""
176
+
177
+
178
+ def display(content: str) -> list[Node]:
179
+ """What WordPress displays for stored or pushed post content."""
180
+ if not is_block_content(content):
181
+ return parse_html(autop(content))
182
+ nodes: list[Node] = []
183
+ for unit in _block_units(content):
184
+ if unit.separator and nodes:
185
+ nodes.append(" ")
186
+ nodes.extend(unit.nodes)
187
+ return nodes
188
+
189
+
190
+ def same_display(stored: str, pushed: str) -> bool:
191
+ """Whether WordPress displays two post contents the same."""
192
+ return _comparable(display(stored)) == _comparable(display(pushed))
193
+
194
+
195
+ def _comparable(nodes: list[Node]) -> tuple:
196
+ return canonical(_canonical_media(nodes), EDITOR_CLASSES)
197
+
198
+
199
+ def _displays_same(first: list[Node], second: list[Node]) -> bool:
200
+ return _comparable(first) == _comparable(second)
201
+
202
+
203
+ def _blank(node: Node) -> bool:
204
+ return isinstance(node, str) and not node.strip(HTML_SPACE)
205
+
206
+
207
+ def _block_units(raw: str) -> list[_Unit]:
208
+ try:
209
+ return list(_scan_blocks(raw))
210
+ except ValueError:
211
+ return [_Unit(raw, parse_html(raw), "malformed block markup", False)]
212
+
213
+
214
+ def _scan_blocks(raw: str) -> Iterator[_Unit]:
215
+ depth = 0
216
+ start = cursor = 0
217
+ name = ""
218
+ attrs: dict = {}
219
+ names: set[str] = set()
220
+ inner_start = 0
221
+ for match in DELIMITER.finditer(raw):
222
+ block = (match["namespace"] or "core/") + match["name"]
223
+ if match["closer"]:
224
+ depth -= 1
225
+ if depth < 0:
226
+ raise ValueError(f"unexpected closing delimiter for {block}")
227
+ if depth == 0:
228
+ if block != name:
229
+ raise ValueError(f"{block} closes {name}")
230
+ yield from _freeform(raw[cursor:start])
231
+ yield _block_unit(
232
+ raw[start : match.end()],
233
+ name,
234
+ attrs,
235
+ names,
236
+ raw[inner_start : match.start()],
237
+ raw[cursor:start],
238
+ )
239
+ cursor = match.end()
240
+ continue
241
+ if depth == 0:
242
+ name, start, inner_start, names = block, match.start(), match.end(), set()
243
+ try:
244
+ attrs = json.loads(match["attrs"]) if match["attrs"] else {}
245
+ except json.JSONDecodeError as error:
246
+ raise ValueError("invalid block attributes") from error
247
+ names.add(block)
248
+ if match["void"]:
249
+ if depth == 0:
250
+ yield from _freeform(raw[cursor:start])
251
+ yield _Unit(match.group(0), parse_html(match.group(0)), block, False)
252
+ cursor = match.end()
253
+ continue
254
+ depth += 1
255
+ if depth:
256
+ raise ValueError(f"unclosed block {name}")
257
+ yield from _freeform(raw[cursor:])
258
+
259
+
260
+ def _freeform(text: str) -> Iterator[_Unit]:
261
+ source = text.strip(HTML_SPACE)
262
+ if source:
263
+ separator = text[: len(text) - len(text.lstrip(HTML_SPACE))]
264
+ yield _Unit(source, parse_html(source), "classic content", True, separator)
265
+
266
+
267
+ def _block_unit(
268
+ markup: str, name: str, attrs: dict, names: set[str], inner: str, before: str
269
+ ) -> _Unit:
270
+ separator = before[len(before.rstrip(HTML_SPACE)) :]
271
+ if name == "core/embed" or name.startswith("core-embed/"):
272
+ url = _plain_embed_url(attrs, inner)
273
+ if url is not None:
274
+ embed = Element("p", {"data-wpsync-embed": url})
275
+ return _Unit(markup, [embed], name, True, separator)
276
+ if name == "core/shortcode":
277
+ return _Unit(markup, parse_html(autop(inner)), name, True, separator)
278
+ if names <= STATIC_BLOCKS:
279
+ nodes = parse_html(DELIMITER.sub("", inner))
280
+ return _Unit(markup, nodes, name, True, separator)
281
+ return _Unit(markup, parse_html(markup), name, False, separator)
282
+
283
+
284
+ def _plain_embed_url(attrs: dict, inner: str) -> str | None:
285
+ """The URL of an embed WordPress displays from its address alone."""
286
+ url = attrs.get("url")
287
+ if not isinstance(url, str) or set(attrs) - EMBED_ATTRS:
288
+ return None
289
+ figures = [n for n in parse_html(inner) if not _blank(n)]
290
+ if len(figures) != 1 or not isinstance(figures[0], Element):
291
+ return None
292
+ figure = figures[0]
293
+ if figure.tag != "figure" or set(figure.attrs) != {"class"}:
294
+ return None
295
+ if not all(EMBED_CLASS.fullmatch(c) for c in figure.attrs["class"].split()):
296
+ return None
297
+ wrappers = [n for n in figure.children if not _blank(n)]
298
+ if len(wrappers) != 1 or not isinstance(wrappers[0], Element):
299
+ return None
300
+ text = wrappers[0].children
301
+ if len(text) != 1 or not isinstance(text[0], str) or text[0].strip() != url:
302
+ return None
303
+ return url
304
+
305
+
306
+ def _classic_units(raw: str) -> list[_Unit]:
307
+ """Split classic content where `wpautop` starts a new paragraph or block.
308
+
309
+ Boundaries are blank lines and the edges of block elements, counted only
310
+ outside every other element.
311
+ """
312
+ cuts = [0]
313
+ depth = 0
314
+ for match in CLASSIC_MARKUP.finditer(raw):
315
+ if match[5] is not None:
316
+ if depth == 0:
317
+ cuts.append(match.start())
318
+ continue
319
+ name = (match[1] or match[3] or "").lower()
320
+ block = name in BLOCK_NAMES
321
+ if match[1] or (match[3] and not match[2] and not match[4]):
322
+ if depth == 0 and block:
323
+ cuts.append(match.start())
324
+ if match[3] and name not in VOID_NAMES:
325
+ depth += 1
326
+ if match[1] or (match[3] and match[2]):
327
+ if match[3]:
328
+ depth = max(0, depth - 1)
329
+ if depth == 0 and block:
330
+ cuts.append(match.end())
331
+ cuts.append(len(raw))
332
+ chunks = (raw[start:end] for start, end in zip(cuts, cuts[1:]))
333
+ units = []
334
+ for chunk in chunks:
335
+ source = chunk.strip(HTML_SPACE)
336
+ if source:
337
+ nodes = parse_html(autop(source))
338
+ units.append(_Unit(source, nodes, _classic_label(nodes), True))
339
+ return units
340
+
341
+
342
+ def _classic_label(nodes: list[Node]) -> str:
343
+ elements = [n for n in nodes if isinstance(n, Element)]
344
+ if len(elements) == 1 and elements[0].tag == "p":
345
+ elements = [
346
+ c for c in elements[0].children if isinstance(c, Element)
347
+ ] or elements
348
+ return f"<{elements[0].tag}>" if elements else "classic text"
349
+
350
+
351
+ # Comparable forms of WordPress presentation --------------------------------
352
+
353
+
354
+ @dataclass(frozen=True)
355
+ class _Image:
356
+ """An image as either editor presents it."""
357
+
358
+ src: str
359
+ alt: str | None
360
+ caption: str | None
361
+ media_id: str | None
362
+ size: str | None
363
+ placement: tuple[str, ...]
364
+ """Alignment classes, which may sit on the image or its figure."""
365
+ figure_classes: tuple[str, ...]
366
+ image_classes: tuple[str, ...]
367
+ attrs: tuple[tuple[str, str], ...]
368
+
369
+
370
+ def _parse_image(node: Node) -> _Image | None:
371
+ """Read a block-editor image figure or a classic paragraph holding one image."""
372
+ if not isinstance(node, Element):
373
+ return None
374
+ children = [c for c in node.children if not _blank(c)]
375
+ caption = None
376
+ figure_classes: list[str] = []
377
+ if node.tag == "figure" and "wp-block-image" in node.attrs.get("class", "").split():
378
+ if set(node.attrs) != {"class"} or not children:
379
+ return None
380
+ if len(children) == 2:
381
+ caption = _caption_text(children[1])
382
+ if caption is None:
383
+ return None
384
+ elif len(children) != 1:
385
+ return None
386
+ figure_classes = [
387
+ c for c in node.attrs["class"].split() if c != "wp-block-image"
388
+ ]
389
+ elif not (node.tag == "p" and not node.attrs and len(children) == 1):
390
+ return None
391
+ image = children[0]
392
+ if not isinstance(image, Element) or image.tag != "img" or "src" not in image.attrs:
393
+ return None
394
+ attrs = dict(image.attrs)
395
+ src, alt = attrs.pop("src"), attrs.pop("alt", None)
396
+ image_classes = attrs.pop("class", "").split()
397
+ pooled = figure_classes + image_classes
398
+ ids = [m[1] for c in image_classes if (m := WP_IMAGE_CLASS.fullmatch(c))]
399
+ sizes = [m[1] for c in pooled if (m := SIZE_CLASS.fullmatch(c))]
400
+ if len(ids) > 1 or len(sizes) > 1 or len(ids) != len(sizes):
401
+ return None
402
+
403
+ def own(classes: list[str]) -> tuple[str, ...]:
404
+ return tuple(
405
+ c
406
+ for c in classes
407
+ if not (
408
+ WP_IMAGE_CLASS.fullmatch(c)
409
+ or SIZE_CLASS.fullmatch(c)
410
+ or PLACEMENT_CLASS.fullmatch(c)
411
+ )
412
+ )
413
+
414
+ return _Image(
415
+ src,
416
+ alt,
417
+ caption,
418
+ ids[0] if ids else None,
419
+ sizes[0] if sizes else None,
420
+ tuple(sorted({c for c in pooled if PLACEMENT_CLASS.fullmatch(c)})),
421
+ own(figure_classes),
422
+ own(image_classes),
423
+ tuple(attrs.items()),
424
+ )
425
+
426
+
427
+ def _caption_text(node: Node) -> str | None:
428
+ """A plain figure caption's text; styled or attributed captions return None."""
429
+ if not isinstance(node, Element) or node.tag != "figcaption":
430
+ return None
431
+ if node.attrs and node.attrs != {"class": CAPTION_CLASS}:
432
+ return None
433
+ if not all(isinstance(child, str) for child in node.children):
434
+ return None
435
+ return "".join(node.children).strip(HTML_SPACE) # type: ignore[arg-type]
436
+
437
+
438
+ def _canonical_media(nodes: list[Node]) -> list[Node]:
439
+ """Give each displayed image, embed, and comment paragraph one comparable form.
440
+
441
+ The forms are block elements, like the figures and paragraphs they stand for.
442
+ """
443
+ result: list[Node] = []
444
+ for node in nodes:
445
+ image = _parse_image(node)
446
+ if image is not None:
447
+ node = Element(
448
+ "figure",
449
+ {
450
+ "data-wpsync-image": "",
451
+ "src": image.src,
452
+ "alt": "" if image.alt is None else image.alt,
453
+ "has-alt": str(image.alt is not None),
454
+ "caption": image.caption or "",
455
+ "media": f"{image.media_id}:{image.size}",
456
+ "placement": " ".join(image.placement),
457
+ "figure-class": " ".join(sorted(image.figure_classes)),
458
+ "image-class": " ".join(sorted(image.image_classes)),
459
+ **dict(image.attrs),
460
+ },
461
+ )
462
+ elif isinstance(node, Element) and node.tag == "p" and not node.attrs:
463
+ content = [c for c in node.children if not _blank(c)]
464
+ if len(content) == 1 and isinstance(content[0], str):
465
+ url = content[0].strip(HTML_SPACE)
466
+ if EMBED_URL.fullmatch(url):
467
+ node = Element("p", {"data-wpsync-embed": url})
468
+ elif content and all(isinstance(c, Comment) for c in content):
469
+ # WordPress wraps a lone <!--more--> in an empty paragraph.
470
+ result.extend(content)
471
+ continue
472
+ if isinstance(node, Element) and not _is_media_form(node):
473
+ node = Element(node.tag, node.attrs, _canonical_media(node.children))
474
+ result.append(node)
475
+ return result
476
+
477
+
478
+ def _is_media_form(node: Element) -> bool:
479
+ return "data-wpsync-image" in node.attrs or "data-wpsync-embed" in node.attrs
480
+
481
+
482
+ # Pull ----------------------------------------------------------------------
483
+
484
+
485
+ def to_markdown(raw: str, media: Media) -> Conversion:
486
+ """Convert stored content, keeping as exact source what would not display alike."""
487
+ units = _block_units(raw) if is_block_content(raw) else _classic_units(raw)
488
+ converter = Converter(EDITOR_CLASSES)
489
+ fragments = [_convert(unit, converter, media) for unit in units]
490
+ converted = [index for index, fragment in enumerate(fragments) if fragment]
491
+ expected = _comparable(display(raw))
492
+
493
+ def document(chosen: list[int]) -> str:
494
+ parts = [
495
+ fragments[index] if index in chosen else kept_markdown(unit.source)
496
+ for index, unit in enumerate(units)
497
+ ]
498
+ return join_blocks(parts) # type: ignore[arg-type]
499
+
500
+ def displays_alike(chosen: list[int]) -> bool:
501
+ pushed = render_html(document(chosen), media)
502
+ return _comparable(display(pushed)) == expected
503
+
504
+ if not displays_alike(converted):
505
+ if not displays_alike([]):
506
+ raise UnrepresentableContent(
507
+ "the post displays differently even when kept entirely as HTML"
508
+ )
509
+ while not displays_alike(converted):
510
+ # Find the first conversion that breaks the document and keep it.
511
+ good, bad = 0, len(converted)
512
+ while bad - good > 1:
513
+ middle = (good + bad) // 2
514
+ if displays_alike(converted[:middle]):
515
+ good = middle
516
+ else:
517
+ bad = middle
518
+ converted.pop(bad - 1)
519
+ kept = tuple(
520
+ unit.label for index, unit in enumerate(units) if index not in converted
521
+ )
522
+ return Conversion(document(converted), kept)
523
+
524
+
525
+ def _convert(unit: _Unit, converter: Converter, media: Media) -> str | None:
526
+ if not unit.convertible:
527
+ return None
528
+ try:
529
+ nodes = trim_block_whitespace(unit.nodes)
530
+ parts = [converter.block(_for_conversion(node, media)) for node in nodes]
531
+ except Unconvertible:
532
+ return None
533
+ fragment = join_blocks(parts)
534
+ if not _displays_same(display(render_html(fragment, media)), unit.nodes):
535
+ return None
536
+ return fragment
537
+
538
+
539
+ def _for_conversion(node: Node, media: Media) -> Node:
540
+ """Rewrite WordPress presentation markup into the form the converter reads."""
541
+ if not isinstance(node, Element):
542
+ return node
543
+ if "data-wpsync-embed" in node.attrs:
544
+ return Element("p", {}, [node.attrs["data-wpsync-embed"]])
545
+ classes = node.attrs.get("class", "").split()
546
+ if node.tag == "figure" and "wp-block-table" in classes:
547
+ tables = [c for c in node.children if isinstance(c, Element)]
548
+ if len(classes) == 1 and len(tables) == 1 and tables[0].tag == "table":
549
+ return tables[0]
550
+ image = _parse_image(node)
551
+ if image is not None and not image.figure_classes:
552
+ attrs = {"src": image.src, **dict(image.attrs)}
553
+ if image.alt is not None:
554
+ attrs["alt"] = image.alt
555
+ if image.media_id is not None:
556
+ attrs["src"] = media.remember(image.src, image.media_id, image.size) # type: ignore[arg-type]
557
+ if image.caption:
558
+ attrs["title"] = image.caption
559
+ classes = list(image.placement) + list(image.image_classes)
560
+ if classes:
561
+ attrs["class"] = " ".join(classes)
562
+ return Element("p", {}, [Element("img", attrs)])
563
+ return Element(
564
+ node.tag, node.attrs, [_for_conversion(c, media) for c in node.children]
565
+ )
@@ -0,0 +1,100 @@
1
+ """WordPress posts in WPSync's platform-neutral form."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from datetime import tzinfo
7
+ from typing import Any
8
+ from urllib.parse import unquote
9
+
10
+ import httpx
11
+
12
+ from wpsync.media import Media
13
+ from wpsync.posts import Conversion, FetchedPost, Post
14
+ from wpsync.wordpress.client import (
15
+ FINGERPRINT_FIELDS,
16
+ HEADERS,
17
+ STATUSES,
18
+ TIMEOUT,
19
+ WordPressClient,
20
+ WordPressError,
21
+ parse_gmt,
22
+ remote_state,
23
+ request,
24
+ )
25
+ from wpsync.wordpress.content import to_markdown
26
+
27
+ PULL_FIELDS = FINGERPRINT_FIELDS + ",generated_slug"
28
+
29
+
30
+ class WordPressRemote:
31
+ """Reads posts and media from one WordPress site."""
32
+
33
+ def __init__(
34
+ self, client: WordPressClient, transport: httpx.BaseTransport | None = None
35
+ ) -> None:
36
+ self.client = client
37
+ # Images are often served from another host, so this client never
38
+ # carries the site's credentials.
39
+ self.downloads = httpx.Client(
40
+ headers=HEADERS, timeout=TIMEOUT, follow_redirects=True, transport=transport
41
+ )
42
+ self._timezone: tzinfo | None = None
43
+ self._terms: dict[str, dict[int, str]] = {}
44
+
45
+ def close(self) -> None:
46
+ self.downloads.close()
47
+
48
+ def fetch_posts(
49
+ self, ids: list[str] | None = None, slug: str | None = None
50
+ ) -> list[FetchedPost]:
51
+ return [
52
+ self._fetched(post)
53
+ for post in self.client.posts(PULL_FIELDS, ids=ids, slug=slug)
54
+ ]
55
+
56
+ def convert(self, post: FetchedPost, media: Media) -> Conversion:
57
+ return to_markdown(post.content, media)
58
+
59
+ def download(self, url: str) -> bytes:
60
+ response = request(self.downloads, "GET", url)
61
+ if response.is_error:
62
+ raise WordPressError(
63
+ f"could not download {url} (HTTP {response.status_code})"
64
+ )
65
+ return response.content
66
+
67
+ def _fetched(self, post: dict[str, Any]) -> FetchedPost:
68
+ if self._timezone is None:
69
+ self._timezone = self.client.site_timezone()
70
+ names = {
71
+ taxonomy: self._term_names(taxonomy) for taxonomy in ("categories", "tags")
72
+ }
73
+ fields = Post(
74
+ title=post["title"]["raw"],
75
+ slug=unquote(post["slug"]),
76
+ date=parse_gmt(post["date_gmt"]).astimezone(self._timezone),
77
+ status=STATUSES[post["status"]],
78
+ categories=tuple(
79
+ sorted({names["categories"][i] for i in post["categories"]})
80
+ ),
81
+ tags=tuple(sorted({names["tags"][i] for i in post["tags"]})),
82
+ )
83
+ return FetchedPost(
84
+ str(post["id"]),
85
+ fields,
86
+ post["content"]["raw"],
87
+ remote_state(post),
88
+ _file_stem(post),
89
+ )
90
+
91
+ def _term_names(self, taxonomy: str) -> dict[int, str]:
92
+ if taxonomy not in self._terms:
93
+ self._terms[taxonomy] = self.client.term_names(taxonomy)
94
+ return self._terms[taxonomy]
95
+
96
+
97
+ def _file_stem(post: dict[str, Any]) -> str:
98
+ slug = unquote(post["slug"] or post.get("generated_slug") or "")
99
+ slug = re.sub(r"[/\\:\x00]", "-", slug).strip(".-")
100
+ return slug or f"post-{post['id']}"