wpsync 1.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of wpsync might be problematic. Click here for more details.
- wpsync/__init__.py +3 -0
- wpsync/__main__.py +3 -0
- wpsync/cli.py +740 -0
- wpsync/config.py +173 -0
- wpsync/documents.py +258 -0
- wpsync/markup/__init__.py +0 -0
- wpsync/markup/dom.py +313 -0
- wpsync/markup/markdown.py +526 -0
- wpsync/media.py +92 -0
- wpsync/posts.py +76 -0
- wpsync/pull.py +324 -0
- wpsync/state.py +177 -0
- wpsync/storage.py +28 -0
- wpsync/sync.py +260 -0
- wpsync/wordpress/__init__.py +0 -0
- wpsync/wordpress/autop.py +114 -0
- wpsync/wordpress/client.py +208 -0
- wpsync/wordpress/content.py +565 -0
- wpsync/wordpress/remote.py +100 -0
- wpsync-1.2.1.dist-info/METADATA +312 -0
- wpsync-1.2.1.dist-info/RECORD +24 -0
- wpsync-1.2.1.dist-info/WHEEL +4 -0
- wpsync-1.2.1.dist-info/entry_points.txt +2 -0
- wpsync-1.2.1.dist-info/licenses/LICENSE +61 -0
|
@@ -0,0 +1,565 @@
|
|
|
1
|
+
"""Post content: WordPress HTML to the post Markdown dialect and back.
|
|
2
|
+
|
|
3
|
+
Conversion is checked against a model of what WordPress displays. Content
|
|
4
|
+
with block markup is displayed block by block; content without it goes
|
|
5
|
+
through WordPress's `wpautop` filter. The stored post and the post a push
|
|
6
|
+
would send are both put through that model, and any part of the post whose
|
|
7
|
+
Markdown would not display the same is kept as its exact original source.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
from collections.abc import Iterator
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
|
|
17
|
+
from markdown_it.token import Token
|
|
18
|
+
|
|
19
|
+
from wpsync.markup.dom import (
|
|
20
|
+
HTML_SPACE,
|
|
21
|
+
Comment,
|
|
22
|
+
Element,
|
|
23
|
+
Node,
|
|
24
|
+
canonical,
|
|
25
|
+
parse_html,
|
|
26
|
+
to_html,
|
|
27
|
+
trim_block_whitespace,
|
|
28
|
+
)
|
|
29
|
+
from wpsync.markup.markdown import (
|
|
30
|
+
Converter,
|
|
31
|
+
Unconvertible,
|
|
32
|
+
join_blocks,
|
|
33
|
+
kept_markdown,
|
|
34
|
+
pieces,
|
|
35
|
+
render_inline,
|
|
36
|
+
render_tokens,
|
|
37
|
+
)
|
|
38
|
+
from wpsync.media import Media
|
|
39
|
+
from wpsync.posts import Conversion, UnrepresentableContent
|
|
40
|
+
from wpsync.wordpress.autop import BLOCK_NAMES, autop
|
|
41
|
+
|
|
42
|
+
EDITOR_CLASSES = frozenset(
|
|
43
|
+
{
|
|
44
|
+
"wp-block-heading",
|
|
45
|
+
"wp-block-list",
|
|
46
|
+
"wp-block-quote",
|
|
47
|
+
"wp-block-code",
|
|
48
|
+
"wp-block-separator",
|
|
49
|
+
"has-alpha-channel-opacity",
|
|
50
|
+
"wp-block-paragraph",
|
|
51
|
+
}
|
|
52
|
+
)
|
|
53
|
+
"""Classes the block editor adds to unstyled elements."""
|
|
54
|
+
|
|
55
|
+
STATIC_BLOCKS = frozenset(
|
|
56
|
+
"core/" + name
|
|
57
|
+
for name in """paragraph heading list list-item quote code preformatted separator
|
|
58
|
+
image gallery table html columns column group spacer pullquote verse buttons
|
|
59
|
+
button details freeform more nextpage media-text cover audio video file""".split()
|
|
60
|
+
)
|
|
61
|
+
"""Blocks whose stored HTML is what WordPress displays."""
|
|
62
|
+
|
|
63
|
+
DELIMITER = re.compile(
|
|
64
|
+
r"<!--\s+(?P<closer>/)?wp:(?P<namespace>[a-z][a-z0-9_-]*/)?"
|
|
65
|
+
r"(?P<name>[a-z][a-z0-9_-]*)\s+(?P<attrs>\{.*?\}\s+)?(?P<void>/)?-->",
|
|
66
|
+
re.DOTALL,
|
|
67
|
+
)
|
|
68
|
+
WP_IMAGE_CLASS = re.compile(r"wp-image-(\d+)")
|
|
69
|
+
SIZE_CLASS = re.compile(r"size-([\w-]+)")
|
|
70
|
+
PLACEMENT_CLASS = re.compile(r"align(?:left|right|center|wide|full|none)")
|
|
71
|
+
CAPTION_CLASS = "wp-element-caption"
|
|
72
|
+
EMBED_ATTRS = frozenset({"url", "type", "providerNameSlug", "responsive"})
|
|
73
|
+
EMBED_CLASS = re.compile(
|
|
74
|
+
r"wp-block-embed(?:-[\w-]+)?|is-type-[\w-]+|is-provider-[\w-]+"
|
|
75
|
+
r"|wp-embed-aspect-[\w-]+|wp-has-aspect-ratio"
|
|
76
|
+
)
|
|
77
|
+
EMBED_URL = re.compile(r"https?://[^\s<>\"]+")
|
|
78
|
+
CLASSIC_MARKUP = re.compile(
|
|
79
|
+
r"<!--.*?-->|<(script|style|pre|textarea)\b.*?</\1\s*>"
|
|
80
|
+
r"|<(/?)([a-zA-Z][\w:-]*)\b[^>]*?(/?)>|(\n[ \t]*\n)",
|
|
81
|
+
re.DOTALL | re.IGNORECASE,
|
|
82
|
+
)
|
|
83
|
+
VOID_NAMES = frozenset(
|
|
84
|
+
"area base br col embed hr img input link meta param source track wbr".split()
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def is_block_content(raw: str) -> bool:
|
|
89
|
+
return "<!-- wp:" in raw
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# Push ----------------------------------------------------------------------
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def render_html(markdown: str, media: Media) -> str:
|
|
96
|
+
"""The HTML a push sends; raw HTML from the Markdown is sent exactly."""
|
|
97
|
+
html = []
|
|
98
|
+
for piece in pieces(markdown):
|
|
99
|
+
if piece.raw is not None:
|
|
100
|
+
html.append(piece.raw)
|
|
101
|
+
else:
|
|
102
|
+
html.append(to_html(render_tokens(_present(piece.tokens, media))))
|
|
103
|
+
return "\n\n".join(html)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _present(tokens: list[Token], media: Media) -> list[Token]:
|
|
107
|
+
"""Use WordPress's presentation for Markdown images and tables."""
|
|
108
|
+
presented: list[Token] = []
|
|
109
|
+
index = 0
|
|
110
|
+
while index < len(tokens):
|
|
111
|
+
token = tokens[index]
|
|
112
|
+
window = tokens[index : index + 3]
|
|
113
|
+
if [t.type for t in window] == ["paragraph_open", "inline", "paragraph_close"]:
|
|
114
|
+
children = [
|
|
115
|
+
c
|
|
116
|
+
for c in window[1].children or []
|
|
117
|
+
if not (c.type == "text" and not c.content.strip(HTML_SPACE))
|
|
118
|
+
]
|
|
119
|
+
if len(children) == 1 and children[0].type == "image":
|
|
120
|
+
image = render_inline(children[0])[0]
|
|
121
|
+
assert isinstance(image, Element)
|
|
122
|
+
presented.append(_html_token(to_html([_image_figure(image, media)])))
|
|
123
|
+
index += 3
|
|
124
|
+
continue
|
|
125
|
+
if token.type == "table_open":
|
|
126
|
+
presented.append(_html_token('<figure class="wp-block-table">'))
|
|
127
|
+
presented.append(token)
|
|
128
|
+
if token.type == "table_close":
|
|
129
|
+
presented.append(_html_token("</figure>"))
|
|
130
|
+
index += 1
|
|
131
|
+
return presented
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _html_token(html: str) -> Token:
|
|
135
|
+
token = Token("html_block", "", 0)
|
|
136
|
+
token.content = html + "\n"
|
|
137
|
+
return token
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _image_figure(image: Element, media: Media) -> Element:
|
|
141
|
+
"""WordPress's image figure: placement on the figure, the rest on the image."""
|
|
142
|
+
attrs = dict(image.attrs)
|
|
143
|
+
title = attrs.pop("title", None)
|
|
144
|
+
classes = attrs.pop("class", "").split()
|
|
145
|
+
figure_classes = ["wp-block-image"]
|
|
146
|
+
image_classes = [c for c in classes if not PLACEMENT_CLASS.fullmatch(c)]
|
|
147
|
+
library = media.lookup(attrs["src"])
|
|
148
|
+
if library is not None:
|
|
149
|
+
attrs["src"] = library.url
|
|
150
|
+
figure_classes.append(f"size-{library.size}")
|
|
151
|
+
image_classes.append(f"wp-image-{library.id}")
|
|
152
|
+
figure_classes.extend(c for c in classes if PLACEMENT_CLASS.fullmatch(c))
|
|
153
|
+
if image_classes:
|
|
154
|
+
attrs["class"] = " ".join(image_classes)
|
|
155
|
+
children: list[Node] = [Element("img", attrs)]
|
|
156
|
+
if title:
|
|
157
|
+
children.append(Element("figcaption", {"class": CAPTION_CLASS}, [title]))
|
|
158
|
+
return Element("figure", {"class": " ".join(figure_classes)}, children)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# What WordPress displays ---------------------------------------------------
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclass(frozen=True)
|
|
165
|
+
class _Unit:
|
|
166
|
+
"""A part of the stored content that converts, or is kept, as a whole."""
|
|
167
|
+
|
|
168
|
+
source: str
|
|
169
|
+
"""The exact original text."""
|
|
170
|
+
nodes: list[Node]
|
|
171
|
+
"""What WordPress displays for it."""
|
|
172
|
+
label: str
|
|
173
|
+
convertible: bool
|
|
174
|
+
separator: str = ""
|
|
175
|
+
"""The whitespace before it in the original."""
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def display(content: str) -> list[Node]:
|
|
179
|
+
"""What WordPress displays for stored or pushed post content."""
|
|
180
|
+
if not is_block_content(content):
|
|
181
|
+
return parse_html(autop(content))
|
|
182
|
+
nodes: list[Node] = []
|
|
183
|
+
for unit in _block_units(content):
|
|
184
|
+
if unit.separator and nodes:
|
|
185
|
+
nodes.append(" ")
|
|
186
|
+
nodes.extend(unit.nodes)
|
|
187
|
+
return nodes
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def same_display(stored: str, pushed: str) -> bool:
|
|
191
|
+
"""Whether WordPress displays two post contents the same."""
|
|
192
|
+
return _comparable(display(stored)) == _comparable(display(pushed))
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _comparable(nodes: list[Node]) -> tuple:
|
|
196
|
+
return canonical(_canonical_media(nodes), EDITOR_CLASSES)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _displays_same(first: list[Node], second: list[Node]) -> bool:
|
|
200
|
+
return _comparable(first) == _comparable(second)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _blank(node: Node) -> bool:
|
|
204
|
+
return isinstance(node, str) and not node.strip(HTML_SPACE)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _block_units(raw: str) -> list[_Unit]:
|
|
208
|
+
try:
|
|
209
|
+
return list(_scan_blocks(raw))
|
|
210
|
+
except ValueError:
|
|
211
|
+
return [_Unit(raw, parse_html(raw), "malformed block markup", False)]
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _scan_blocks(raw: str) -> Iterator[_Unit]:
|
|
215
|
+
depth = 0
|
|
216
|
+
start = cursor = 0
|
|
217
|
+
name = ""
|
|
218
|
+
attrs: dict = {}
|
|
219
|
+
names: set[str] = set()
|
|
220
|
+
inner_start = 0
|
|
221
|
+
for match in DELIMITER.finditer(raw):
|
|
222
|
+
block = (match["namespace"] or "core/") + match["name"]
|
|
223
|
+
if match["closer"]:
|
|
224
|
+
depth -= 1
|
|
225
|
+
if depth < 0:
|
|
226
|
+
raise ValueError(f"unexpected closing delimiter for {block}")
|
|
227
|
+
if depth == 0:
|
|
228
|
+
if block != name:
|
|
229
|
+
raise ValueError(f"{block} closes {name}")
|
|
230
|
+
yield from _freeform(raw[cursor:start])
|
|
231
|
+
yield _block_unit(
|
|
232
|
+
raw[start : match.end()],
|
|
233
|
+
name,
|
|
234
|
+
attrs,
|
|
235
|
+
names,
|
|
236
|
+
raw[inner_start : match.start()],
|
|
237
|
+
raw[cursor:start],
|
|
238
|
+
)
|
|
239
|
+
cursor = match.end()
|
|
240
|
+
continue
|
|
241
|
+
if depth == 0:
|
|
242
|
+
name, start, inner_start, names = block, match.start(), match.end(), set()
|
|
243
|
+
try:
|
|
244
|
+
attrs = json.loads(match["attrs"]) if match["attrs"] else {}
|
|
245
|
+
except json.JSONDecodeError as error:
|
|
246
|
+
raise ValueError("invalid block attributes") from error
|
|
247
|
+
names.add(block)
|
|
248
|
+
if match["void"]:
|
|
249
|
+
if depth == 0:
|
|
250
|
+
yield from _freeform(raw[cursor:start])
|
|
251
|
+
yield _Unit(match.group(0), parse_html(match.group(0)), block, False)
|
|
252
|
+
cursor = match.end()
|
|
253
|
+
continue
|
|
254
|
+
depth += 1
|
|
255
|
+
if depth:
|
|
256
|
+
raise ValueError(f"unclosed block {name}")
|
|
257
|
+
yield from _freeform(raw[cursor:])
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _freeform(text: str) -> Iterator[_Unit]:
|
|
261
|
+
source = text.strip(HTML_SPACE)
|
|
262
|
+
if source:
|
|
263
|
+
separator = text[: len(text) - len(text.lstrip(HTML_SPACE))]
|
|
264
|
+
yield _Unit(source, parse_html(source), "classic content", True, separator)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _block_unit(
|
|
268
|
+
markup: str, name: str, attrs: dict, names: set[str], inner: str, before: str
|
|
269
|
+
) -> _Unit:
|
|
270
|
+
separator = before[len(before.rstrip(HTML_SPACE)) :]
|
|
271
|
+
if name == "core/embed" or name.startswith("core-embed/"):
|
|
272
|
+
url = _plain_embed_url(attrs, inner)
|
|
273
|
+
if url is not None:
|
|
274
|
+
embed = Element("p", {"data-wpsync-embed": url})
|
|
275
|
+
return _Unit(markup, [embed], name, True, separator)
|
|
276
|
+
if name == "core/shortcode":
|
|
277
|
+
return _Unit(markup, parse_html(autop(inner)), name, True, separator)
|
|
278
|
+
if names <= STATIC_BLOCKS:
|
|
279
|
+
nodes = parse_html(DELIMITER.sub("", inner))
|
|
280
|
+
return _Unit(markup, nodes, name, True, separator)
|
|
281
|
+
return _Unit(markup, parse_html(markup), name, False, separator)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _plain_embed_url(attrs: dict, inner: str) -> str | None:
|
|
285
|
+
"""The URL of an embed WordPress displays from its address alone."""
|
|
286
|
+
url = attrs.get("url")
|
|
287
|
+
if not isinstance(url, str) or set(attrs) - EMBED_ATTRS:
|
|
288
|
+
return None
|
|
289
|
+
figures = [n for n in parse_html(inner) if not _blank(n)]
|
|
290
|
+
if len(figures) != 1 or not isinstance(figures[0], Element):
|
|
291
|
+
return None
|
|
292
|
+
figure = figures[0]
|
|
293
|
+
if figure.tag != "figure" or set(figure.attrs) != {"class"}:
|
|
294
|
+
return None
|
|
295
|
+
if not all(EMBED_CLASS.fullmatch(c) for c in figure.attrs["class"].split()):
|
|
296
|
+
return None
|
|
297
|
+
wrappers = [n for n in figure.children if not _blank(n)]
|
|
298
|
+
if len(wrappers) != 1 or not isinstance(wrappers[0], Element):
|
|
299
|
+
return None
|
|
300
|
+
text = wrappers[0].children
|
|
301
|
+
if len(text) != 1 or not isinstance(text[0], str) or text[0].strip() != url:
|
|
302
|
+
return None
|
|
303
|
+
return url
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _classic_units(raw: str) -> list[_Unit]:
|
|
307
|
+
"""Split classic content where `wpautop` starts a new paragraph or block.
|
|
308
|
+
|
|
309
|
+
Boundaries are blank lines and the edges of block elements, counted only
|
|
310
|
+
outside every other element.
|
|
311
|
+
"""
|
|
312
|
+
cuts = [0]
|
|
313
|
+
depth = 0
|
|
314
|
+
for match in CLASSIC_MARKUP.finditer(raw):
|
|
315
|
+
if match[5] is not None:
|
|
316
|
+
if depth == 0:
|
|
317
|
+
cuts.append(match.start())
|
|
318
|
+
continue
|
|
319
|
+
name = (match[1] or match[3] or "").lower()
|
|
320
|
+
block = name in BLOCK_NAMES
|
|
321
|
+
if match[1] or (match[3] and not match[2] and not match[4]):
|
|
322
|
+
if depth == 0 and block:
|
|
323
|
+
cuts.append(match.start())
|
|
324
|
+
if match[3] and name not in VOID_NAMES:
|
|
325
|
+
depth += 1
|
|
326
|
+
if match[1] or (match[3] and match[2]):
|
|
327
|
+
if match[3]:
|
|
328
|
+
depth = max(0, depth - 1)
|
|
329
|
+
if depth == 0 and block:
|
|
330
|
+
cuts.append(match.end())
|
|
331
|
+
cuts.append(len(raw))
|
|
332
|
+
chunks = (raw[start:end] for start, end in zip(cuts, cuts[1:]))
|
|
333
|
+
units = []
|
|
334
|
+
for chunk in chunks:
|
|
335
|
+
source = chunk.strip(HTML_SPACE)
|
|
336
|
+
if source:
|
|
337
|
+
nodes = parse_html(autop(source))
|
|
338
|
+
units.append(_Unit(source, nodes, _classic_label(nodes), True))
|
|
339
|
+
return units
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _classic_label(nodes: list[Node]) -> str:
|
|
343
|
+
elements = [n for n in nodes if isinstance(n, Element)]
|
|
344
|
+
if len(elements) == 1 and elements[0].tag == "p":
|
|
345
|
+
elements = [
|
|
346
|
+
c for c in elements[0].children if isinstance(c, Element)
|
|
347
|
+
] or elements
|
|
348
|
+
return f"<{elements[0].tag}>" if elements else "classic text"
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
# Comparable forms of WordPress presentation --------------------------------
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
@dataclass(frozen=True)
|
|
355
|
+
class _Image:
|
|
356
|
+
"""An image as either editor presents it."""
|
|
357
|
+
|
|
358
|
+
src: str
|
|
359
|
+
alt: str | None
|
|
360
|
+
caption: str | None
|
|
361
|
+
media_id: str | None
|
|
362
|
+
size: str | None
|
|
363
|
+
placement: tuple[str, ...]
|
|
364
|
+
"""Alignment classes, which may sit on the image or its figure."""
|
|
365
|
+
figure_classes: tuple[str, ...]
|
|
366
|
+
image_classes: tuple[str, ...]
|
|
367
|
+
attrs: tuple[tuple[str, str], ...]
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _parse_image(node: Node) -> _Image | None:
|
|
371
|
+
"""Read a block-editor image figure or a classic paragraph holding one image."""
|
|
372
|
+
if not isinstance(node, Element):
|
|
373
|
+
return None
|
|
374
|
+
children = [c for c in node.children if not _blank(c)]
|
|
375
|
+
caption = None
|
|
376
|
+
figure_classes: list[str] = []
|
|
377
|
+
if node.tag == "figure" and "wp-block-image" in node.attrs.get("class", "").split():
|
|
378
|
+
if set(node.attrs) != {"class"} or not children:
|
|
379
|
+
return None
|
|
380
|
+
if len(children) == 2:
|
|
381
|
+
caption = _caption_text(children[1])
|
|
382
|
+
if caption is None:
|
|
383
|
+
return None
|
|
384
|
+
elif len(children) != 1:
|
|
385
|
+
return None
|
|
386
|
+
figure_classes = [
|
|
387
|
+
c for c in node.attrs["class"].split() if c != "wp-block-image"
|
|
388
|
+
]
|
|
389
|
+
elif not (node.tag == "p" and not node.attrs and len(children) == 1):
|
|
390
|
+
return None
|
|
391
|
+
image = children[0]
|
|
392
|
+
if not isinstance(image, Element) or image.tag != "img" or "src" not in image.attrs:
|
|
393
|
+
return None
|
|
394
|
+
attrs = dict(image.attrs)
|
|
395
|
+
src, alt = attrs.pop("src"), attrs.pop("alt", None)
|
|
396
|
+
image_classes = attrs.pop("class", "").split()
|
|
397
|
+
pooled = figure_classes + image_classes
|
|
398
|
+
ids = [m[1] for c in image_classes if (m := WP_IMAGE_CLASS.fullmatch(c))]
|
|
399
|
+
sizes = [m[1] for c in pooled if (m := SIZE_CLASS.fullmatch(c))]
|
|
400
|
+
if len(ids) > 1 or len(sizes) > 1 or len(ids) != len(sizes):
|
|
401
|
+
return None
|
|
402
|
+
|
|
403
|
+
def own(classes: list[str]) -> tuple[str, ...]:
|
|
404
|
+
return tuple(
|
|
405
|
+
c
|
|
406
|
+
for c in classes
|
|
407
|
+
if not (
|
|
408
|
+
WP_IMAGE_CLASS.fullmatch(c)
|
|
409
|
+
or SIZE_CLASS.fullmatch(c)
|
|
410
|
+
or PLACEMENT_CLASS.fullmatch(c)
|
|
411
|
+
)
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
return _Image(
|
|
415
|
+
src,
|
|
416
|
+
alt,
|
|
417
|
+
caption,
|
|
418
|
+
ids[0] if ids else None,
|
|
419
|
+
sizes[0] if sizes else None,
|
|
420
|
+
tuple(sorted({c for c in pooled if PLACEMENT_CLASS.fullmatch(c)})),
|
|
421
|
+
own(figure_classes),
|
|
422
|
+
own(image_classes),
|
|
423
|
+
tuple(attrs.items()),
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _caption_text(node: Node) -> str | None:
|
|
428
|
+
"""A plain figure caption's text; styled or attributed captions return None."""
|
|
429
|
+
if not isinstance(node, Element) or node.tag != "figcaption":
|
|
430
|
+
return None
|
|
431
|
+
if node.attrs and node.attrs != {"class": CAPTION_CLASS}:
|
|
432
|
+
return None
|
|
433
|
+
if not all(isinstance(child, str) for child in node.children):
|
|
434
|
+
return None
|
|
435
|
+
return "".join(node.children).strip(HTML_SPACE) # type: ignore[arg-type]
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _canonical_media(nodes: list[Node]) -> list[Node]:
|
|
439
|
+
"""Give each displayed image, embed, and comment paragraph one comparable form.
|
|
440
|
+
|
|
441
|
+
The forms are block elements, like the figures and paragraphs they stand for.
|
|
442
|
+
"""
|
|
443
|
+
result: list[Node] = []
|
|
444
|
+
for node in nodes:
|
|
445
|
+
image = _parse_image(node)
|
|
446
|
+
if image is not None:
|
|
447
|
+
node = Element(
|
|
448
|
+
"figure",
|
|
449
|
+
{
|
|
450
|
+
"data-wpsync-image": "",
|
|
451
|
+
"src": image.src,
|
|
452
|
+
"alt": "" if image.alt is None else image.alt,
|
|
453
|
+
"has-alt": str(image.alt is not None),
|
|
454
|
+
"caption": image.caption or "",
|
|
455
|
+
"media": f"{image.media_id}:{image.size}",
|
|
456
|
+
"placement": " ".join(image.placement),
|
|
457
|
+
"figure-class": " ".join(sorted(image.figure_classes)),
|
|
458
|
+
"image-class": " ".join(sorted(image.image_classes)),
|
|
459
|
+
**dict(image.attrs),
|
|
460
|
+
},
|
|
461
|
+
)
|
|
462
|
+
elif isinstance(node, Element) and node.tag == "p" and not node.attrs:
|
|
463
|
+
content = [c for c in node.children if not _blank(c)]
|
|
464
|
+
if len(content) == 1 and isinstance(content[0], str):
|
|
465
|
+
url = content[0].strip(HTML_SPACE)
|
|
466
|
+
if EMBED_URL.fullmatch(url):
|
|
467
|
+
node = Element("p", {"data-wpsync-embed": url})
|
|
468
|
+
elif content and all(isinstance(c, Comment) for c in content):
|
|
469
|
+
# WordPress wraps a lone <!--more--> in an empty paragraph.
|
|
470
|
+
result.extend(content)
|
|
471
|
+
continue
|
|
472
|
+
if isinstance(node, Element) and not _is_media_form(node):
|
|
473
|
+
node = Element(node.tag, node.attrs, _canonical_media(node.children))
|
|
474
|
+
result.append(node)
|
|
475
|
+
return result
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _is_media_form(node: Element) -> bool:
|
|
479
|
+
return "data-wpsync-image" in node.attrs or "data-wpsync-embed" in node.attrs
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
# Pull ----------------------------------------------------------------------
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def to_markdown(raw: str, media: Media) -> Conversion:
|
|
486
|
+
"""Convert stored content, keeping as exact source what would not display alike."""
|
|
487
|
+
units = _block_units(raw) if is_block_content(raw) else _classic_units(raw)
|
|
488
|
+
converter = Converter(EDITOR_CLASSES)
|
|
489
|
+
fragments = [_convert(unit, converter, media) for unit in units]
|
|
490
|
+
converted = [index for index, fragment in enumerate(fragments) if fragment]
|
|
491
|
+
expected = _comparable(display(raw))
|
|
492
|
+
|
|
493
|
+
def document(chosen: list[int]) -> str:
|
|
494
|
+
parts = [
|
|
495
|
+
fragments[index] if index in chosen else kept_markdown(unit.source)
|
|
496
|
+
for index, unit in enumerate(units)
|
|
497
|
+
]
|
|
498
|
+
return join_blocks(parts) # type: ignore[arg-type]
|
|
499
|
+
|
|
500
|
+
def displays_alike(chosen: list[int]) -> bool:
|
|
501
|
+
pushed = render_html(document(chosen), media)
|
|
502
|
+
return _comparable(display(pushed)) == expected
|
|
503
|
+
|
|
504
|
+
if not displays_alike(converted):
|
|
505
|
+
if not displays_alike([]):
|
|
506
|
+
raise UnrepresentableContent(
|
|
507
|
+
"the post displays differently even when kept entirely as HTML"
|
|
508
|
+
)
|
|
509
|
+
while not displays_alike(converted):
|
|
510
|
+
# Find the first conversion that breaks the document and keep it.
|
|
511
|
+
good, bad = 0, len(converted)
|
|
512
|
+
while bad - good > 1:
|
|
513
|
+
middle = (good + bad) // 2
|
|
514
|
+
if displays_alike(converted[:middle]):
|
|
515
|
+
good = middle
|
|
516
|
+
else:
|
|
517
|
+
bad = middle
|
|
518
|
+
converted.pop(bad - 1)
|
|
519
|
+
kept = tuple(
|
|
520
|
+
unit.label for index, unit in enumerate(units) if index not in converted
|
|
521
|
+
)
|
|
522
|
+
return Conversion(document(converted), kept)
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def _convert(unit: _Unit, converter: Converter, media: Media) -> str | None:
|
|
526
|
+
if not unit.convertible:
|
|
527
|
+
return None
|
|
528
|
+
try:
|
|
529
|
+
nodes = trim_block_whitespace(unit.nodes)
|
|
530
|
+
parts = [converter.block(_for_conversion(node, media)) for node in nodes]
|
|
531
|
+
except Unconvertible:
|
|
532
|
+
return None
|
|
533
|
+
fragment = join_blocks(parts)
|
|
534
|
+
if not _displays_same(display(render_html(fragment, media)), unit.nodes):
|
|
535
|
+
return None
|
|
536
|
+
return fragment
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _for_conversion(node: Node, media: Media) -> Node:
|
|
540
|
+
"""Rewrite WordPress presentation markup into the form the converter reads."""
|
|
541
|
+
if not isinstance(node, Element):
|
|
542
|
+
return node
|
|
543
|
+
if "data-wpsync-embed" in node.attrs:
|
|
544
|
+
return Element("p", {}, [node.attrs["data-wpsync-embed"]])
|
|
545
|
+
classes = node.attrs.get("class", "").split()
|
|
546
|
+
if node.tag == "figure" and "wp-block-table" in classes:
|
|
547
|
+
tables = [c for c in node.children if isinstance(c, Element)]
|
|
548
|
+
if len(classes) == 1 and len(tables) == 1 and tables[0].tag == "table":
|
|
549
|
+
return tables[0]
|
|
550
|
+
image = _parse_image(node)
|
|
551
|
+
if image is not None and not image.figure_classes:
|
|
552
|
+
attrs = {"src": image.src, **dict(image.attrs)}
|
|
553
|
+
if image.alt is not None:
|
|
554
|
+
attrs["alt"] = image.alt
|
|
555
|
+
if image.media_id is not None:
|
|
556
|
+
attrs["src"] = media.remember(image.src, image.media_id, image.size) # type: ignore[arg-type]
|
|
557
|
+
if image.caption:
|
|
558
|
+
attrs["title"] = image.caption
|
|
559
|
+
classes = list(image.placement) + list(image.image_classes)
|
|
560
|
+
if classes:
|
|
561
|
+
attrs["class"] = " ".join(classes)
|
|
562
|
+
return Element("p", {}, [Element("img", attrs)])
|
|
563
|
+
return Element(
|
|
564
|
+
node.tag, node.attrs, [_for_conversion(c, media) for c in node.children]
|
|
565
|
+
)
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""WordPress posts in WPSync's platform-neutral form."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from datetime import tzinfo
|
|
7
|
+
from typing import Any
|
|
8
|
+
from urllib.parse import unquote
|
|
9
|
+
|
|
10
|
+
import httpx
|
|
11
|
+
|
|
12
|
+
from wpsync.media import Media
|
|
13
|
+
from wpsync.posts import Conversion, FetchedPost, Post
|
|
14
|
+
from wpsync.wordpress.client import (
|
|
15
|
+
FINGERPRINT_FIELDS,
|
|
16
|
+
HEADERS,
|
|
17
|
+
STATUSES,
|
|
18
|
+
TIMEOUT,
|
|
19
|
+
WordPressClient,
|
|
20
|
+
WordPressError,
|
|
21
|
+
parse_gmt,
|
|
22
|
+
remote_state,
|
|
23
|
+
request,
|
|
24
|
+
)
|
|
25
|
+
from wpsync.wordpress.content import to_markdown
|
|
26
|
+
|
|
27
|
+
PULL_FIELDS = FINGERPRINT_FIELDS + ",generated_slug"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class WordPressRemote:
|
|
31
|
+
"""Reads posts and media from one WordPress site."""
|
|
32
|
+
|
|
33
|
+
def __init__(
|
|
34
|
+
self, client: WordPressClient, transport: httpx.BaseTransport | None = None
|
|
35
|
+
) -> None:
|
|
36
|
+
self.client = client
|
|
37
|
+
# Images are often served from another host, so this client never
|
|
38
|
+
# carries the site's credentials.
|
|
39
|
+
self.downloads = httpx.Client(
|
|
40
|
+
headers=HEADERS, timeout=TIMEOUT, follow_redirects=True, transport=transport
|
|
41
|
+
)
|
|
42
|
+
self._timezone: tzinfo | None = None
|
|
43
|
+
self._terms: dict[str, dict[int, str]] = {}
|
|
44
|
+
|
|
45
|
+
def close(self) -> None:
|
|
46
|
+
self.downloads.close()
|
|
47
|
+
|
|
48
|
+
def fetch_posts(
|
|
49
|
+
self, ids: list[str] | None = None, slug: str | None = None
|
|
50
|
+
) -> list[FetchedPost]:
|
|
51
|
+
return [
|
|
52
|
+
self._fetched(post)
|
|
53
|
+
for post in self.client.posts(PULL_FIELDS, ids=ids, slug=slug)
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
def convert(self, post: FetchedPost, media: Media) -> Conversion:
|
|
57
|
+
return to_markdown(post.content, media)
|
|
58
|
+
|
|
59
|
+
def download(self, url: str) -> bytes:
|
|
60
|
+
response = request(self.downloads, "GET", url)
|
|
61
|
+
if response.is_error:
|
|
62
|
+
raise WordPressError(
|
|
63
|
+
f"could not download {url} (HTTP {response.status_code})"
|
|
64
|
+
)
|
|
65
|
+
return response.content
|
|
66
|
+
|
|
67
|
+
def _fetched(self, post: dict[str, Any]) -> FetchedPost:
|
|
68
|
+
if self._timezone is None:
|
|
69
|
+
self._timezone = self.client.site_timezone()
|
|
70
|
+
names = {
|
|
71
|
+
taxonomy: self._term_names(taxonomy) for taxonomy in ("categories", "tags")
|
|
72
|
+
}
|
|
73
|
+
fields = Post(
|
|
74
|
+
title=post["title"]["raw"],
|
|
75
|
+
slug=unquote(post["slug"]),
|
|
76
|
+
date=parse_gmt(post["date_gmt"]).astimezone(self._timezone),
|
|
77
|
+
status=STATUSES[post["status"]],
|
|
78
|
+
categories=tuple(
|
|
79
|
+
sorted({names["categories"][i] for i in post["categories"]})
|
|
80
|
+
),
|
|
81
|
+
tags=tuple(sorted({names["tags"][i] for i in post["tags"]})),
|
|
82
|
+
)
|
|
83
|
+
return FetchedPost(
|
|
84
|
+
str(post["id"]),
|
|
85
|
+
fields,
|
|
86
|
+
post["content"]["raw"],
|
|
87
|
+
remote_state(post),
|
|
88
|
+
_file_stem(post),
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
def _term_names(self, taxonomy: str) -> dict[int, str]:
|
|
92
|
+
if taxonomy not in self._terms:
|
|
93
|
+
self._terms[taxonomy] = self.client.term_names(taxonomy)
|
|
94
|
+
return self._terms[taxonomy]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _file_stem(post: dict[str, Any]) -> str:
|
|
98
|
+
slug = unquote(post["slug"] or post.get("generated_slug") or "")
|
|
99
|
+
slug = re.sub(r"[/\\:\x00]", "-", slug).strip(".-")
|
|
100
|
+
return slug or f"post-{post['id']}"
|