wpsync 1.17.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
wpsync/markup/dom.py ADDED
@@ -0,0 +1,371 @@
1
+ """A small HTML tree for converting and comparing post content.
2
+
3
+ Markup is parsed with html5lib, which follows the HTML parsing rules browsers
4
+ use, so the tree is what a reader's browser builds.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import re
10
+ import xml.etree.ElementTree as ET
11
+ from dataclasses import dataclass, field
12
+
13
+ import html5lib
14
+
15
+ VOID_TAGS = frozenset(
16
+ "area base br col embed hr img input link meta param source track wbr".split()
17
+ )
18
+ RAW_TEXT_TAGS = frozenset({"script", "style"})
19
+ PRESERVED_WHITESPACE_TAGS = frozenset({"pre", "textarea", "script", "style"})
20
+ BLOCK_TAGS = frozenset(
21
+ """address article aside blockquote caption col colgroup dd details dialog div
22
+ dl dt fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 header hgroup hr
23
+ li main menu nav ol p pre section summary table tbody td tfoot th thead tr
24
+ ul""".split()
25
+ )
26
+ """Elements browsers display as blocks, so whitespace around their edges is hidden."""
27
+ TAG_ALIASES = {"b": "strong", "i": "em"}
28
+ EDGE_SPACE_TAGS = frozenset({"strong", "em"})
29
+ EMPTY_FORMATTING_TAGS = frozenset({"strong", "em", "b", "i", "span"})
30
+ WHITESPACE = re.compile(r"[ \t\n\r\f]+")
31
+ HTML_SPACE = " \t\n\r\f"
32
+ NO_BREAK_SPACE = "\xa0"
33
+ LAYOUT_STYLE = re.compile(r"(?:^|;)\s*(?:white-space|display)\s*:", re.IGNORECASE)
34
+ NAMESPACE_PREFIXES = {
35
+ "http://www.w3.org/1999/xlink": "xlink",
36
+ "http://www.w3.org/XML/1998/namespace": "xml",
37
+ "http://www.w3.org/2000/xmlns/": "xmlns",
38
+ }
39
+
40
+
41
+ @dataclass
42
+ class Element:
43
+ tag: str
44
+ attrs: dict[str, str] = field(default_factory=dict)
45
+ children: list[Node] = field(default_factory=list)
46
+
47
+
48
+ @dataclass(frozen=True)
49
+ class Comment:
50
+ text: str
51
+
52
+
53
+ Node = Element | Comment | str
54
+
55
+ _PARSER = html5lib.HTMLParser(namespaceHTMLElements=False)
56
+
57
+
58
+ def _local_name(name: str) -> str:
59
+ if not name.startswith("{"):
60
+ return name
61
+ namespace, local = name[1:].split("}", 1)
62
+ prefix = NAMESPACE_PREFIXES.get(namespace)
63
+ return f"{prefix}:{local}" if prefix else local
64
+
65
+
66
+ def _append_node(nodes: list[Node], node: Node) -> None:
67
+ if isinstance(node, str) and nodes and isinstance(nodes[-1], str):
68
+ nodes[-1] += node
69
+ elif node != "":
70
+ nodes.append(node)
71
+
72
+
73
+ def _children(element: ET.Element) -> list[Node]:
74
+ nodes: list[Node] = []
75
+ _append_node(nodes, element.text or "")
76
+ for child in element:
77
+ if child.tag is ET.Comment:
78
+ nodes.append(Comment(child.text or ""))
79
+ else:
80
+ attrs = {_local_name(name): value for name, value in child.attrib.items()}
81
+ nodes.append(Element(_local_name(child.tag), attrs, _children(child)))
82
+ _append_node(nodes, child.tail or "")
83
+ return nodes
84
+
85
+
86
+ def parse_html(markup: str) -> list[Node]:
87
+ return _children(_PARSER.parseFragment(markup))
88
+
89
+
90
+ def _escape_text(text: str) -> str:
91
+ return text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
92
+
93
+
94
+ def _escape_attribute(value: str) -> str:
95
+ return (
96
+ value.replace("&", "&amp;")
97
+ .replace('"', "&quot;")
98
+ .replace("<", "&lt;")
99
+ .replace("\n", "&#10;")
100
+ )
101
+
102
+
103
+ def start_tag(element: Element) -> str:
104
+ attributes = "".join(
105
+ f' {name}="{_escape_attribute(value)}"' for name, value in element.attrs.items()
106
+ )
107
+ return f"<{element.tag}{attributes}>"
108
+
109
+
110
+ def preserves_whitespace(element: Element) -> bool:
111
+ """Whether whitespace in and around the element may be displayed as written."""
112
+ return element.tag in PRESERVED_WHITESPACE_TAGS or bool(
113
+ LAYOUT_STYLE.search(element.attrs.get("style", ""))
114
+ )
115
+
116
+
117
+ def serialize(nodes: list[Node], *, preserve_whitespace: bool = False) -> str:
118
+ """Serialize without line breaks outside elements that preserve whitespace."""
119
+ out = []
120
+ for node in nodes:
121
+ if isinstance(node, str):
122
+ text = node if preserve_whitespace else node.replace("\n", " ")
123
+ out.append(_escape_text(text))
124
+ elif isinstance(node, Comment):
125
+ if node.text.strip(HTML_SPACE):
126
+ out.append(f"<!--{node.text}-->")
127
+ elif node.tag in VOID_TAGS:
128
+ out.append(start_tag(node)[:-1] + " />")
129
+ else:
130
+ preserve = preserve_whitespace or preserves_whitespace(node)
131
+ if node.tag in RAW_TEXT_TAGS:
132
+ inner = "".join(
133
+ child for child in node.children if isinstance(child, str)
134
+ )
135
+ else:
136
+ inner = serialize(node.children, preserve_whitespace=preserve)
137
+ out.append(f"{start_tag(node)}{inner}</{node.tag}>")
138
+ return "".join(out)
139
+
140
+
141
+ def to_html(nodes: list[Node]) -> str:
142
+ return serialize(trim_block_whitespace(nodes))
143
+
144
+
145
+ def is_block(node: Node) -> bool:
146
+ return (
147
+ isinstance(node, Element)
148
+ and node.tag in BLOCK_TAGS
149
+ and not LAYOUT_STYLE.search(node.attrs.get("style", ""))
150
+ )
151
+
152
+
153
+ def _breaks_line(node: Node | None) -> bool:
154
+ """Whitespace next to a block or a line break is not displayed."""
155
+ return is_block(node) or (isinstance(node, Element) and node.tag == "br")
156
+
157
+
158
+ def trim_block_whitespace(
159
+ nodes: list[Node], *, block_context: bool = True
160
+ ) -> list[Node]:
161
+ """Drop whitespace that cannot affect display, and spacing made of markup.
162
+
163
+ Browsers do not display whitespace around block elements or at the start
164
+ and end of a line, including inside inline elements at those positions.
165
+ Blank paragraphs and repeated line breaks are how authors force vertical
166
+ space in WordPress, which is not content: Markdown has no use for them.
167
+ """
168
+ result: list[Node] = []
169
+ for node in nodes:
170
+ if isinstance(node, Element) and not preserves_whitespace(node):
171
+ children = trim_block_whitespace(
172
+ node.children, block_context=is_block(node)
173
+ )
174
+ if is_block(node):
175
+ parts = _split_at_blank_lines(_trim_breaks(children))
176
+ if node.tag == "p" and not node.attrs:
177
+ result.extend(Element("p", {}, part) for part in parts)
178
+ continue
179
+ children = parts[0]
180
+ for part in parts[1:]:
181
+ children = [*children, Element("br"), *part]
182
+ node = Element(node.tag, node.attrs, children)
183
+ result.append(node)
184
+ result = _strip_after_breaks(result, block_context, leading=True)
185
+ result.reverse()
186
+ result = _strip_after_breaks(result, block_context, leading=False)
187
+ result.reverse()
188
+ return [node for node in result if not _displays_nothing(node)]
189
+
190
+
191
+ def _displays_nothing(node: Node) -> bool:
192
+ """Plain formatting with nothing inside, or a plain paragraph of blank lines."""
193
+ if not isinstance(node, Element) or node.attrs:
194
+ return False
195
+ if node.tag in EMPTY_FORMATTING_TAGS:
196
+ return not node.children
197
+ return node.tag == "p" and all(_is_filler(child) for child in node.children)
198
+
199
+
200
+ def _is_break(node: Node) -> bool:
201
+ return node == Element("br")
202
+
203
+
204
+ def _is_filler(node: Node) -> bool:
205
+ """Spaces, line breaks, or plain formatting holding only those."""
206
+ if isinstance(node, str):
207
+ return not node.strip(HTML_SPACE + NO_BREAK_SPACE)
208
+ return (
209
+ isinstance(node, Element)
210
+ and not node.attrs
211
+ and (
212
+ node.tag == "br"
213
+ or (
214
+ node.tag in EMPTY_FORMATTING_TAGS
215
+ and all(_is_filler(child) for child in node.children)
216
+ )
217
+ )
218
+ )
219
+
220
+
221
+ def _trim_breaks(children: list[Node]) -> list[Node]:
222
+ start, end = 0, len(children)
223
+ while start < end and _is_break(children[start]):
224
+ start += 1
225
+ while end > start and _is_break(children[end - 1]):
226
+ end -= 1
227
+ return children[start:end]
228
+
229
+
230
+ def _split_at_blank_lines(children: list[Node]) -> list[list[Node]]:
231
+ """Split a block's content at blank lines made of repeated line breaks."""
232
+ parts: list[list[Node]] = [[]]
233
+ filler: list[Node] = []
234
+ for child in children:
235
+ if _is_filler(child):
236
+ filler.append(child)
237
+ continue
238
+ if sum(_is_break(node) for node in filler) > 1:
239
+ parts.append([])
240
+ else:
241
+ parts[-1].extend(filler)
242
+ filler = []
243
+ parts[-1].append(child)
244
+ parts[-1].extend(filler)
245
+ return parts
246
+
247
+
248
+ def _strip_after_breaks(
249
+ nodes: list[Node], block_context: bool, *, leading: bool
250
+ ) -> list[Node]:
251
+ kept: list[Node] = []
252
+ for node in nodes:
253
+ previous = kept[-1] if kept else None
254
+ if (previous is None and block_context) or _breaks_line(previous):
255
+ node = _strip_edge(node, leading=leading)
256
+ if node != "":
257
+ kept.append(node)
258
+ return kept
259
+
260
+
261
+ def _strip_edge(node: Node, *, leading: bool) -> Node:
262
+ if isinstance(node, str):
263
+ return node.lstrip(HTML_SPACE) if leading else node.rstrip(HTML_SPACE)
264
+ if (
265
+ not isinstance(node, Element)
266
+ or is_block(node)
267
+ or preserves_whitespace(node)
268
+ or not node.children
269
+ ):
270
+ return node
271
+ children = list(node.children)
272
+ index = 0 if leading else -1
273
+ children[index] = _strip_edge(children[index], leading=leading)
274
+ return Element(node.tag, node.attrs, [child for child in children if child != ""])
275
+
276
+
277
+ Canonical = tuple
278
+
279
+
280
+ def _canonical_attrs(attrs: dict[str, str], ignored_classes: frozenset[str]) -> tuple:
281
+ items = []
282
+ for name, value in attrs.items():
283
+ if name == "class":
284
+ classes = sorted(set(value.split()) - ignored_classes)
285
+ if not classes:
286
+ continue
287
+ value = " ".join(classes)
288
+ items.append((name, value))
289
+ return tuple(sorted(items))
290
+
291
+
292
+ def _canonical_children(
293
+ nodes: list[Node], ignored_classes: frozenset[str], preserve: bool
294
+ ) -> list[Canonical]:
295
+ out: list[Canonical] = []
296
+ for node in nodes:
297
+ if isinstance(node, str):
298
+ _append_text(out, node if preserve else WHITESPACE.sub(" ", node))
299
+ elif isinstance(node, Comment):
300
+ # Comments are never displayed; only their words matter.
301
+ text = WHITESPACE.sub(" ", node.text).strip()
302
+ if text:
303
+ out.append(("comment", text))
304
+ else:
305
+ tag = TAG_ALIASES.get(node.tag, node.tag)
306
+ attrs = _canonical_attrs(node.attrs, ignored_classes)
307
+ inner = _canonical_children(
308
+ node.children, ignored_classes, preserve or preserves_whitespace(node)
309
+ )
310
+ if tag == "pre":
311
+ inner = _trim_trailing_newline(inner)
312
+ if tag == "span" and not attrs:
313
+ for child in inner:
314
+ if child[0] == "text":
315
+ _append_text(out, child[1])
316
+ else:
317
+ out.append(child)
318
+ elif tag in EDGE_SPACE_TAGS and not attrs and not preserve:
319
+ lead, inner, trail = _edge_spaces(inner)
320
+ _append_text(out, lead)
321
+ out.append(("element", tag, attrs, tuple(inner)))
322
+ _append_text(out, trail)
323
+ else:
324
+ out.append(("element", tag, attrs, tuple(inner)))
325
+ return out
326
+
327
+
328
+ def _append_text(out: list[Canonical], text: str) -> None:
329
+ if not text:
330
+ return
331
+ if out and out[-1][0] == "text":
332
+ out[-1] = ("text", out[-1][1] + text)
333
+ else:
334
+ out.append(("text", text))
335
+
336
+
337
+ def _edge_spaces(inner: list[Canonical]) -> tuple[str, list[Canonical], str]:
338
+ """Spaces at the edges of plain bold or italic text display as if outside it."""
339
+ inner = list(inner)
340
+ lead = trail = ""
341
+ if inner and inner[0][0] == "text" and inner[0][1].startswith(" "):
342
+ lead = " "
343
+ inner[0] = ("text", inner[0][1].lstrip(" "))
344
+ if inner and inner[-1][0] == "text" and inner[-1][1].endswith(" "):
345
+ trail = " "
346
+ inner[-1] = ("text", inner[-1][1].rstrip(" "))
347
+ return lead, [node for node in inner if node != ("text", "")], trail
348
+
349
+
350
+ def _trim_trailing_newline(children: list[Canonical]) -> list[Canonical]:
351
+ """A single line break at the very end of `<pre>` is not displayed."""
352
+ if not children:
353
+ return children
354
+ children = list(children)
355
+ last = children[-1]
356
+ if last[0] == "text":
357
+ text = last[1].removesuffix("\n")
358
+ children[-1] = ("text", text)
359
+ return [child for child in children if child != ("text", "")]
360
+ if last[0] == "element":
361
+ children[-1] = (*last[:3], tuple(_trim_trailing_newline(list(last[3]))))
362
+ return children
363
+
364
+
365
+ def canonical(
366
+ nodes: list[Node], ignored_classes: frozenset[str] = frozenset()
367
+ ) -> Canonical:
368
+ """A comparable form that is equal when two trees display the same."""
369
+ return tuple(
370
+ _canonical_children(trim_block_whitespace(nodes), ignored_classes, False)
371
+ )