blockspec 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
blockspec/document.py ADDED
@@ -0,0 +1,1212 @@
1
+ # SPDX-License-Identifier: Apache-2.0 OR MIT
2
+ """Read a blockspec draft 0 document: refuse it, or return its structure.
3
+
4
+ Draft 0 is decided by `docs/adjudications/0001-syntax.md` and `0002-block.md`
5
+ (each section's *What draft 0 carries*, and 0002's frontmatter addendum). In
6
+ short: a pinned CommonMark 0.31.2 block-level profile with GFM pipe tables,
7
+ inline content opaque, region markers as the one convention, and a fixed list
8
+ of refusals.
9
+
10
+ The block parser is the line-by-line algorithm of CommonMark's reference
11
+ implementation (commonmark.js), cut down to the profile: every block the
12
+ profile refuses raises `Refusal` where it is recognised. GFM's table extension
13
+ is a block start tried after every core one, as cmark-gfm tries its
14
+ extensions. Where this reader and GFM part ways on tables, it takes the
15
+ narrower reading: a row with no unescaped pipe ends a table, where GFM would
16
+ make it a row, and a header row with no pipe over a one-cell delimiter row, a
17
+ table to GitHub alone, is refused. A row that is only `|` ends a table too, as
18
+ GitHub ends it. Standard library only, and one file on purpose:
19
+ `conformance/run_cases.py --impl FILE` loads this file by path, and so does
20
+ the mutation gate, from a mutated copy of it. The anchor resolver of section
21
+ 11 is `anchor.py` beside it, which `resolve` loads by path in turn and hands
22
+ this file's reader.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import importlib.util
28
+ import os
29
+ import re
30
+ import string
31
+ import sys
32
+ from types import SimpleNamespace
33
+
34
+ __all__ = ["Document", "Refusal", "conform", "parse", "resolve"]
35
+
36
+
37
+ class Refusal(Exception):
38
+ """The document is not in the profile. `code` is one of the codes in
39
+ `conformance/cases/case-body.schema.json`."""
40
+
41
+ def __init__(self, code: str, message: str) -> None:
42
+ super().__init__(f"{code}: {message}")
43
+ self.code = code
44
+ self.message = message
45
+
46
+
47
+ LEAVES = ("paragraph", "heading", "code_block", "html_block", "thematic_break", "table")
48
+ CONTAINERS = ("document", "block_quote", "list", "list_item")
49
+
50
+ _PUNCT = frozenset(string.punctuation)
51
+ # CommonMark 0.31.2 section 6.6's spacing in a tag: "spaces, tabs, and up to
52
+ # one line ending". _WS may be empty; _WS1, before an attribute, may not.
53
+ _EOL = "(?:\\r\\n|\\r|\\n)"
54
+ _WS = f"[ \\t]*(?:{_EOL}[ \\t]*)?"
55
+ _WS1 = f"(?:[ \\t]+(?:{_EOL}[ \\t]*)?|{_EOL}[ \\t]*)"
56
+
57
+ # A git conflict-marker line: a column-0 run of seven or more `<`, `>` or `|`
58
+ # followed by a space or the end of the line, or seven or more `=` alone
59
+ # (conformance/cases/README.md, reading 14).
60
+ _CONFLICT = re.compile(r"^(?:(?:<{7,}|>{7,}|\|{7,})(?: |$)|={7,}$)")
61
+
62
+ # D8 section 7.2's region marker, in its two forms.
63
+ _MARKER = re.compile(r"^[ \t]*<!--[ \t]*#([a-z0-9][a-z0-9-]*)(:begin|:end)?[ \t]*-->[ \t]*$")
64
+
65
+ _ATX = re.compile(r"^#{1,6}(?:[ \t]+|$)")
66
+ _FENCE = re.compile(r"^`{3,}(?!.*`)|^~{3,}")
67
+ _CLOSING_FENCE = re.compile(r"^(?:`{3,}|~{3,})(?=[ \t]*$)")
68
+ _THEMATIC = re.compile(r"^(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,})$")
69
+ _SETEXT = re.compile(r"^(?:=+|-+)[ \t]*$")
70
+ _BULLET = re.compile(r"^[*+-]")
71
+ _ORDERED = re.compile(r"^(\d{1,9})([.)])")
72
+ _DELIMITER_CELL = re.compile(r"^:?-+:?$")
73
+
74
+ _TAGNAME = "[A-Za-z][A-Za-z0-9-]*"
75
+ _ATTRIBUTE = (
76
+ f"(?:{_WS1}[a-zA-Z_:][a-zA-Z0-9_.:-]*"
77
+ f"(?:{_WS}={_WS}(?:[^ \\t\\n\\r\"'=<>`]+|'[^']*'|\"[^\"]*\"))?)"
78
+ )
79
+ _OPENTAG = f"<{_TAGNAME}{_ATTRIBUTE}*{_WS}/?>"
80
+ _CLOSETAG = f"</{_TAGNAME}{_WS}>"
81
+ _BLOCK_TAGS = (
82
+ "address|article|aside|base|basefont|blockquote|body|caption|center|col|colgroup|dd|"
83
+ "details|dialog|dir|div|dl|dt|fieldset|figcaption|figure|footer|form|frame|frameset|"
84
+ "h[1-6]|head|header|hr|html|iframe|legend|li|link|main|menu|menuitem|nav|noframes|ol|"
85
+ "optgroup|option|p|param|search|section|summary|table|tbody|td|tfoot|th|thead|title|"
86
+ "tr|track|ul"
87
+ )
88
+ # CommonMark 0.31.2 section 4.6: the start conditions of HTML block types 1-7.
89
+ _HTML_OPEN = (
90
+ re.compile(r"^<(?:pre|script|style|textarea)(?:[ \t>]|$)", re.I),
91
+ re.compile(r"^<!--"),
92
+ re.compile(r"^<\?"),
93
+ re.compile(r"^<![A-Za-z]"),
94
+ re.compile(r"^<!\[CDATA\["),
95
+ re.compile(f"^</?(?:{_BLOCK_TAGS})(?:[ \\t]|/?>|$)", re.I),
96
+ re.compile(f"^(?:{_OPENTAG}|{_CLOSETAG})[ \\t]*$", re.I),
97
+ )
98
+
99
+
100
+ class Node:
101
+ """One node of the CommonMark tree, with what the profile reports of it."""
102
+
103
+ def __init__(self, kind: str, line: int) -> None:
104
+ self.type = kind
105
+ self.parent: Node | None = None
106
+ self.children: list[Node] = []
107
+ self.open = True
108
+ self.start = line
109
+ self.end = line
110
+ #: The last line on which this node, or a node inside it, held a
111
+ #: non-blank line. A table's neighbours are measured with it.
112
+ self.last_content = line
113
+ self.lines: list[tuple[int, str]] = [] # paragraph lines, code lines
114
+ #: A paragraph's lazy continuation lines as they stand, leading
115
+ #: whitespace kept, keyed by line number. GitHub reads a table header
116
+ #: from a lazy line without stripping it.
117
+ self.lazy: dict[int, str] = {}
118
+ self.text = "" # heading content, or an html_block's line
119
+ self.rows: list[list[str]] = [] # table
120
+ self.list_data: dict = {}
121
+ self.fence = ("", 0, 0) # character, length, indentation
122
+ self.marker: tuple[str, str | None] | None = None
123
+ self.path: tuple[int, ...] = ()
124
+
125
+ def content(self) -> str | list[list[str]]:
126
+ if self.type == "paragraph":
127
+ return "\n".join(text for _, text in self.lines).rstrip(" \t")
128
+ if self.type == "heading":
129
+ return self.text
130
+ if self.type == "code_block":
131
+ return "".join(text + "\n" for _, text in self.lines)
132
+ if self.type == "html_block":
133
+ return self.text + "\n"
134
+ if self.type == "table":
135
+ width = len(self.rows[0])
136
+ return [self.rows[0]] + [(row + [""] * width)[:width] for row in self.rows[1:]]
137
+ return ""
138
+
139
+ def ref(self) -> dict:
140
+ return {"node": list(self.path), "type": self.type}
141
+
142
+
143
+ def _split_lines(text: str) -> list[str]:
144
+ lines = re.split(r"\r\n|\r|\n", text)
145
+ if lines and lines[-1] == "":
146
+ lines.pop()
147
+ return lines
148
+
149
+
150
+ def _blank(line: str) -> bool:
151
+ return not line.strip(" \t")
152
+
153
+
154
+ def _has_pipe(text: str) -> bool:
155
+ """Whether `text` holds a `|` that no backslash escapes."""
156
+ return re.search(r"(?:^|[^\\])\|", text) is not None
157
+
158
+
159
+ #: A delimiter-like line once form feeds and vertical tabs are allowed in it:
160
+ #: only `|`, `:`, `-` and whitespace, with at least one `-`.
161
+ _FFVT_DELIMITER = re.compile(r"^[ \t\f\v|:-]*-[ \t\f\v|:-]*$")
162
+
163
+
164
+ def _split_row(text: str, keep_lead: bool = False) -> list[str]:
165
+ """A table row's cells as GFM splits them: on unescaped pipes, outer pipes
166
+ dropped, each cell trimmed of spaces and tabs, `\\|` read as `|`. With
167
+ `keep_lead`, whitespace before a leading pipe is not stripped first, so
168
+ it makes an empty first cell, as GitHub reads a lazy line."""
169
+ row = text.rstrip(" \t") if keep_lead else text.strip(" \t")
170
+ cells: list[str] = []
171
+ cell: list[str] = []
172
+ ends_with_pipe = False
173
+ i = 0
174
+ while i < len(row):
175
+ c = row[i]
176
+ if c == "\\" and row[i + 1 : i + 2] == "|":
177
+ cell.append("|")
178
+ i += 2
179
+ ends_with_pipe = False
180
+ continue
181
+ if c == "|":
182
+ cells.append("".join(cell))
183
+ cell = []
184
+ ends_with_pipe = True
185
+ else:
186
+ cell.append(c)
187
+ ends_with_pipe = False
188
+ i += 1
189
+ cells.append("".join(cell))
190
+ if row.startswith("|"):
191
+ cells.pop(0)
192
+ if ends_with_pipe and cells:
193
+ cells.pop()
194
+ return [cell.strip(" \t") for cell in cells]
195
+
196
+
197
+ def _link_reference_definition(text: str) -> bool:
198
+ """Whether a paragraph's raw content opens with a link reference
199
+ definition (CommonMark 0.31.2 section 4.7)."""
200
+ n = len(text)
201
+ if not text.startswith("["):
202
+ return False
203
+ j = 1
204
+ while j < n:
205
+ c = text[j]
206
+ if c == "\\" and j + 1 < n and text[j + 1] in _PUNCT:
207
+ j += 2
208
+ continue
209
+ if c == "[":
210
+ return False
211
+ if c == "]":
212
+ break
213
+ j += 1
214
+ else:
215
+ return False
216
+ label = text[1:j]
217
+ if len(label) > 999 or not label.strip(" \t\n"):
218
+ return False
219
+ j += 1
220
+ if j >= n or text[j] != ":":
221
+ return False
222
+ j += 1
223
+ while j < n and text[j] in " \t":
224
+ j += 1
225
+ if j < n and text[j] == "\n":
226
+ j += 1
227
+ while j < n and text[j] in " \t":
228
+ j += 1
229
+ if j >= n:
230
+ return False
231
+ k = j
232
+ if text[k] == "<":
233
+ k += 1
234
+ while k < n:
235
+ c = text[k]
236
+ if c == "\\" and k + 1 < n and text[k + 1] in _PUNCT:
237
+ k += 2
238
+ continue
239
+ if c in "\n<":
240
+ return False
241
+ if c == ">":
242
+ break
243
+ k += 1
244
+ else:
245
+ return False
246
+ k += 1
247
+ else:
248
+ depth = 0
249
+ while k < n:
250
+ c = text[k]
251
+ if c == "\\" and k + 1 < n and text[k + 1] in _PUNCT:
252
+ k += 2
253
+ continue
254
+ if c == "(":
255
+ depth += 1
256
+ elif c == ")":
257
+ if depth == 0:
258
+ break
259
+ depth -= 1
260
+ elif ord(c) <= 0x20 or c == "\x7f":
261
+ break
262
+ k += 1
263
+ if k == j or depth:
264
+ return False
265
+ p = k
266
+ while p < n and text[p] in " \t":
267
+ p += 1
268
+ destination_ends_its_line = p >= n or text[p] == "\n"
269
+ q = p
270
+ if q < n and text[q] == "\n":
271
+ q += 1
272
+ while q < n and text[q] in " \t":
273
+ q += 1
274
+ if q > k and q < n and text[q] in "\"'(":
275
+ opener = text[q]
276
+ closer = ")" if opener == "(" else opener
277
+ r = q + 1
278
+ while r < n:
279
+ c = text[r]
280
+ if c == "\\" and r + 1 < n and text[r + 1] in _PUNCT:
281
+ r += 2
282
+ continue
283
+ if c == closer:
284
+ r += 1
285
+ while r < n and text[r] in " \t":
286
+ r += 1
287
+ if r >= n or text[r] == "\n":
288
+ return True
289
+ break
290
+ if opener == "(" and c == "(":
291
+ break
292
+ r += 1
293
+ return destination_ends_its_line
294
+
295
+
296
+ class _Parser:
297
+ """commonmark.js's block parser, cut down to the profile."""
298
+
299
+ def __init__(self) -> None:
300
+ self.doc = Node("document", 1)
301
+ self.tip = self.doc
302
+ self.oldtip = self.doc
303
+ self.last_matched = self.doc
304
+ self.all_closed = True
305
+ self.line = ""
306
+ self.line_no = 0
307
+ self.offset = 0
308
+ self.column = 0
309
+ self.next_nonspace = 0
310
+ self.next_nonspace_column = 0
311
+ self.indent = 0
312
+ self.indented = False
313
+ self.blank = False
314
+ self.partially_consumed_tab = False
315
+
316
+ # --- the cursor ---------------------------------------------------------
317
+
318
+ def find_next_nonspace(self) -> None:
319
+ i, cols = self.offset, self.column
320
+ while i < len(self.line):
321
+ c = self.line[i]
322
+ if c == " ":
323
+ i += 1
324
+ cols += 1
325
+ elif c == "\t":
326
+ i += 1
327
+ cols += 4 - cols % 4
328
+ else:
329
+ break
330
+ self.blank = i >= len(self.line)
331
+ self.next_nonspace = i
332
+ self.next_nonspace_column = cols
333
+ self.indent = cols - self.column
334
+ self.indented = self.indent >= 4
335
+
336
+ def advance_next_nonspace(self) -> None:
337
+ self.offset = self.next_nonspace
338
+ self.column = self.next_nonspace_column
339
+ self.partially_consumed_tab = False
340
+
341
+ def advance_offset(self, count: int, columns: bool) -> None:
342
+ while count > 0 and self.offset < len(self.line):
343
+ if self.line[self.offset] == "\t":
344
+ to_tab = 4 - self.column % 4
345
+ if columns:
346
+ self.partially_consumed_tab = to_tab > count
347
+ step = min(to_tab, count)
348
+ self.column += step
349
+ self.offset += 0 if self.partially_consumed_tab else 1
350
+ count -= step
351
+ else:
352
+ self.partially_consumed_tab = False
353
+ self.column += to_tab
354
+ self.offset += 1
355
+ count -= 1
356
+ else:
357
+ self.partially_consumed_tab = False
358
+ self.offset += 1
359
+ self.column += 1
360
+ count -= 1
361
+
362
+ def peek(self, at: int) -> str:
363
+ return self.line[at] if at < len(self.line) else ""
364
+
365
+ def rest(self) -> str:
366
+ """The rest of the line from the offset, a partly consumed tab
367
+ counting as the spaces it has left."""
368
+ prefix = ""
369
+ if self.partially_consumed_tab:
370
+ self.offset += 1
371
+ prefix = " " * (4 - self.column % 4)
372
+ return prefix + self.line[self.offset :]
373
+
374
+ # --- the tree -----------------------------------------------------------
375
+
376
+ def add_child(self, kind: str) -> Node:
377
+ allowed = {
378
+ "document": lambda t: t != "list_item",
379
+ "block_quote": lambda t: t != "list_item",
380
+ "list_item": lambda t: t != "list_item",
381
+ "list": lambda t: t == "list_item",
382
+ }
383
+ while not allowed.get(self.tip.type, lambda t: False)(kind):
384
+ self.finalize(self.tip, self.line_no - 1)
385
+ node = Node(kind, self.line_no)
386
+ node.parent = self.tip
387
+ self.tip.children.append(node)
388
+ self.tip = node
389
+ return node
390
+
391
+ def finalize(self, node: Node, line: int) -> None:
392
+ node.open = False
393
+ node.end = line
394
+ if node.type == "paragraph" and _link_reference_definition(node.content()):
395
+ raise Refusal("link-reference-definition", "a link reference definition")
396
+ self.tip = node.parent
397
+
398
+ def close_unmatched(self) -> None:
399
+ if not self.all_closed:
400
+ while self.oldtip is not self.last_matched:
401
+ parent = self.oldtip.parent
402
+ self.finalize(self.oldtip, self.line_no - 1)
403
+ self.oldtip = parent
404
+ self.all_closed = True
405
+
406
+ def touch(self, node: Node | None) -> None:
407
+ while node is not None:
408
+ node.last_content = self.line_no
409
+ node = node.parent
410
+
411
+ # --- continuation: 0 matched, 1 not matched, 2 line consumed ------------
412
+
413
+ def proceed(self, node: Node) -> int:
414
+ kind = node.type
415
+ if kind == "block_quote":
416
+ if not self.indented and self.peek(self.next_nonspace) == ">":
417
+ self.advance_next_nonspace()
418
+ self.advance_offset(1, False)
419
+ if self.peek(self.offset) in (" ", "\t"):
420
+ self.advance_offset(1, True)
421
+ return 0
422
+ return 1
423
+ if kind == "list_item":
424
+ data = node.list_data
425
+ if self.blank:
426
+ if not node.children:
427
+ return 1
428
+ self.advance_next_nonspace()
429
+ elif self.indent >= data["offset"] + data["padding"]:
430
+ self.advance_offset(data["offset"] + data["padding"], True)
431
+ else:
432
+ return 1
433
+ return 0
434
+ if kind == "list":
435
+ return 0
436
+ if kind == "code_block":
437
+ char, length, indent = node.fence
438
+ closing = self.indent <= 3 and self.peek(self.next_nonspace) == char
439
+ match = closing and _CLOSING_FENCE.match(self.line[self.next_nonspace :])
440
+ if match and len(match.group()) >= length:
441
+ self.finalize(node, self.line_no)
442
+ return 2
443
+ while indent > 0 and self.peek(self.offset) in (" ", "\t"):
444
+ self.advance_offset(1, True)
445
+ indent -= 1
446
+ return 0
447
+ if kind == "paragraph":
448
+ return 1 if self.blank else 0
449
+ if kind == "table":
450
+ row = self.line[self.next_nonspace :]
451
+ if self.blank or not _has_pipe(row) or not _split_row(row):
452
+ return 1
453
+ return 0
454
+ return 1
455
+
456
+ # --- block starts: 0 none, 1 container, 2 leaf, 3 line consumed --------
457
+
458
+ def block_quote(self, container: Node) -> int:
459
+ if self.indented or self.peek(self.next_nonspace) != ">":
460
+ return 0
461
+ self.advance_next_nonspace()
462
+ self.advance_offset(1, False)
463
+ if self.peek(self.offset) in (" ", "\t"):
464
+ self.advance_offset(1, True)
465
+ self.close_unmatched()
466
+ self.add_child("block_quote")
467
+ return 1
468
+
469
+ def atx_heading(self, container: Node) -> int:
470
+ match = None if self.indented else _ATX.match(self.line[self.next_nonspace :])
471
+ if not match:
472
+ return 0
473
+ self.advance_next_nonspace()
474
+ self.advance_offset(len(match.group()), False)
475
+ self.close_unmatched()
476
+ node = self.add_child("heading")
477
+ text = re.sub(r"^[ \t]*#+[ \t]*$", "", self.line[self.offset :])
478
+ node.text = re.sub(r"[ \t]+#+[ \t]*$", "", text).strip(" \t")
479
+ self.finalize(node, self.line_no)
480
+ return 3
481
+
482
+ def fenced_code(self, container: Node) -> int:
483
+ match = None if self.indented else _FENCE.match(self.line[self.next_nonspace :])
484
+ if not match:
485
+ return 0
486
+ self.close_unmatched()
487
+ node = self.add_child("code_block")
488
+ node.fence = (match.group()[0], len(match.group()), self.indent)
489
+ return 3
490
+
491
+ def html_block(self, container: Node) -> int:
492
+ if self.indented or self.peek(self.next_nonspace) != "<":
493
+ return 0
494
+ text = self.line[self.next_nonspace :]
495
+ for kind, pattern in enumerate(_HTML_OPEN, 1):
496
+ if not pattern.match(text):
497
+ continue
498
+ if kind == 7 and (
499
+ container.type == "paragraph"
500
+ or (not self.all_closed and not self.blank and self.tip.type == "paragraph")
501
+ ):
502
+ continue
503
+ marker = _MARKER.match(text) if kind == 2 else None
504
+ if marker is None:
505
+ raise Refusal("raw-html", "an HTML block that is not a region marker")
506
+ self.close_unmatched()
507
+ node = self.add_child("html_block")
508
+ node.text = self.rest()
509
+ node.marker = (marker.group(1), marker.group(2))
510
+ self.finalize(node, self.line_no)
511
+ return 3
512
+ return 0
513
+
514
+ def setext_heading(self, container: Node) -> int:
515
+ if self.indented or container.type != "paragraph":
516
+ return 0
517
+ if not _SETEXT.match(self.line[self.next_nonspace :]):
518
+ return 0
519
+ if _link_reference_definition(container.content()):
520
+ raise Refusal("link-reference-definition", "a link reference definition")
521
+ raise Refusal("setext-heading", "a setext heading")
522
+
523
+ def table(self, container: Node) -> int:
524
+ """GFM's table extension: a delimiter row under a paragraph line whose
525
+ cells it matches makes that line a table's header row.
526
+
527
+ Tried after every core block start, as cmark-gfm tries its extensions,
528
+ so `- | -` under a paragraph line is a list item, not a delimiter row.
529
+ A delimiter row and the paragraph line over it are judged as two
530
+ lines, without the line above them (0002, addendum on ambiguous
531
+ tables). Renderers disagree on form feeds and vertical tabs, so when
532
+ either line holds one and the delimiter line is only `|`, `:`, `-`
533
+ and whitespace with at least one `-`, the lines are refused as
534
+ ambiguous-table. Otherwise they are a table candidate when the
535
+ delimiter row's cells are `:?-+:?` and the header has as many. GitHub
536
+ reads every candidate as a table. pandoc and markdown-it read one
537
+ only when both lines hold an unescaped pipe, so a candidate without
538
+ one is refused as ambiguous-table. A lazy header line is read as it
539
+ stands, as GitHub reads it: whitespace before its leading pipe is an
540
+ empty first cell."""
541
+ if self.indented or container.type != "paragraph":
542
+ return 0
543
+ delimiter_text = self.line[self.next_nonspace :]
544
+ header_line, header_text = container.lines[-1]
545
+ lazy = header_line in container.lazy
546
+ header_raw = container.lazy.get(header_line, header_text)
547
+ if any(c in delimiter_text or c in header_raw for c in "\f\v"):
548
+ if _FFVT_DELIMITER.match(delimiter_text):
549
+ raise Refusal(
550
+ "ambiguous-table",
551
+ "a table-like pair of lines holding a form feed or a vertical tab",
552
+ )
553
+ return 0
554
+ delimiter = _split_row(delimiter_text)
555
+ if not delimiter or not all(_DELIMITER_CELL.match(cell) for cell in delimiter):
556
+ return 0
557
+ header = _split_row(header_raw, keep_lead=lazy)
558
+ if len(header) != len(delimiter):
559
+ return 0
560
+ if not (_has_pipe(delimiter_text) and _has_pipe(header_raw)):
561
+ raise Refusal(
562
+ "ambiguous-table",
563
+ "a table to GitHub and a paragraph to pandoc and markdown-it",
564
+ )
565
+ parent = container.parent
566
+ if len(container.lines) > 1:
567
+ raise Refusal("table-not-blank-above", "a table directly under a paragraph line")
568
+ at = parent.children.index(container)
569
+ if at and parent.children[at - 1].last_content == header_line - 1:
570
+ raise Refusal("table-not-blank-above", "a table directly under a non-blank line")
571
+ parent.children.pop(at)
572
+ self.tip = parent
573
+ node = self.add_child("table")
574
+ node.start = header_line
575
+ node.rows = [header]
576
+ self.advance_offset(len(self.line) - self.offset, False)
577
+ return 3
578
+
579
+ def thematic_break(self, container: Node) -> int:
580
+ if self.indented or not _THEMATIC.match(self.line[self.next_nonspace :]):
581
+ return 0
582
+ self.close_unmatched()
583
+ node = self.add_child("thematic_break")
584
+ self.finalize(node, self.line_no)
585
+ return 3
586
+
587
+ def list_item(self, container: Node) -> int:
588
+ if self.indented and container.type != "list":
589
+ return 0
590
+ data = self.list_marker(container)
591
+ if data is None:
592
+ return 0
593
+ self.close_unmatched()
594
+ if self.tip.type != "list" or not all(
595
+ self.tip.list_data[key] == data[key] for key in ("kind", "bullet", "delimiter")
596
+ ):
597
+ self.add_child("list").list_data = data
598
+ self.add_child("list_item").list_data = data
599
+ return 1
600
+
601
+ def list_marker(self, container: Node) -> dict | None:
602
+ if self.indent >= 4:
603
+ return None
604
+ rest = self.line[self.next_nonspace :]
605
+ data = {"kind": "", "bullet": "", "delimiter": "", "offset": self.indent, "padding": 0}
606
+ bullet = _BULLET.match(rest)
607
+ ordered = _ORDERED.match(rest)
608
+ if bullet:
609
+ match = bullet
610
+ data["kind"], data["bullet"] = "bullet", bullet.group()
611
+ elif ordered and (container.type != "paragraph" or int(ordered.group(1)) == 1):
612
+ match = ordered
613
+ data["kind"], data["delimiter"] = "ordered", ordered.group(2)
614
+ else:
615
+ return None
616
+ width = len(match.group())
617
+ if self.peek(self.next_nonspace + width) not in ("", " ", "\t"):
618
+ return None
619
+ if container.type == "paragraph" and _blank(self.line[self.next_nonspace + width :]):
620
+ return None
621
+ self.advance_next_nonspace()
622
+ self.advance_offset(width, True)
623
+ start_column, start_offset = self.column, self.offset
624
+ while True:
625
+ self.advance_offset(1, True)
626
+ nxt = self.peek(self.offset)
627
+ if not (self.column - start_column < 5 and nxt in (" ", "\t")):
628
+ break
629
+ blank_item = self.peek(self.offset) == ""
630
+ spaces = self.column - start_column
631
+ if spaces >= 5 or spaces < 1 or blank_item:
632
+ data["padding"] = width + 1
633
+ self.column, self.offset = start_column, start_offset
634
+ if self.peek(self.offset) in (" ", "\t"):
635
+ self.advance_offset(1, True)
636
+ else:
637
+ data["padding"] = width + spaces
638
+ return data
639
+
640
+ def indented_code(self, container: Node) -> int:
641
+ if self.indented and self.tip.type != "paragraph" and not self.blank:
642
+ raise Refusal("indented-code", "an indented code block")
643
+ return 0
644
+
645
+ # --- one line -----------------------------------------------------------
646
+
647
+ def incorporate(self, line: str, line_no: int) -> None:
648
+ self.line, self.line_no = line, line_no
649
+ self.offset = self.column = 0
650
+ self.blank = self.partially_consumed_tab = False
651
+ self.oldtip = self.tip
652
+ container = self.doc
653
+ while container.children and container.children[-1].open:
654
+ container = container.children[-1]
655
+ self.find_next_nonspace()
656
+ outcome = self.proceed(container)
657
+ if outcome == 2:
658
+ self.touch(container)
659
+ return
660
+ if outcome == 1:
661
+ container = container.parent
662
+ break
663
+ self.all_closed = container is self.oldtip
664
+ self.last_matched = container
665
+ matched_offset = self.offset
666
+ starts = (
667
+ self.block_quote,
668
+ self.atx_heading,
669
+ self.fenced_code,
670
+ self.html_block,
671
+ self.setext_heading,
672
+ self.thematic_break,
673
+ self.list_item,
674
+ self.indented_code,
675
+ self.table,
676
+ )
677
+ leaf = container.type == "code_block"
678
+ while not leaf:
679
+ self.find_next_nonspace()
680
+ for start in starts:
681
+ outcome = start(container)
682
+ if outcome:
683
+ break
684
+ else:
685
+ self.advance_next_nonspace()
686
+ break
687
+ container = self.tip
688
+ if outcome == 3:
689
+ self.touch(self.tip)
690
+ return
691
+ leaf = outcome == 2
692
+ if not self.all_closed and not self.blank and self.tip.type == "paragraph":
693
+ self.tip.lazy[line_no] = line[matched_offset:]
694
+ self.tip.lines.append((line_no, self.rest()))
695
+ else:
696
+ self.close_unmatched()
697
+ if container.type in ("paragraph", "code_block"):
698
+ container.lines.append((line_no, self.rest()))
699
+ elif container.type == "table":
700
+ container.rows.append(_split_row(self.line[self.next_nonspace :]))
701
+ elif self.offset < len(line) and not self.blank:
702
+ node = self.add_child("paragraph")
703
+ self.advance_next_nonspace()
704
+ node.lines.append((line_no, self.rest()))
705
+ if not _blank(line):
706
+ self.touch(self.tip)
707
+
708
+ def finish(self, last_line: int) -> Node:
709
+ self.line_no = last_line + 1
710
+ while self.tip is not None:
711
+ self.finalize(self.tip, last_line)
712
+ return self.doc
713
+
714
+
715
+ #: github-slugger 2.0.0's removed characters, as hex code point ranges. Generated
716
+ #: from its regex.js by evaluating the regex on every code point with node, so
717
+ #: it matches code point for code point rather than by reading the class:
718
+ #: tools/slugger/ranges.mjs makes it, and tools/slugger/check.py checks it.
719
+ _SLUG_REMOVED_RANGES = (
720
+ "0-1f 21-2c 2e-2f 3a-40 5b-5e 60 7b-a9 ab-b4 b6-b9 bb-bf d7 f7 2c2-2c5 2d2-2df 2e5-2eb "
721
+ "2ed 2ef-2ff 375 378-379 37e 380-385 387 38b 38d 3a2 3f6 482 530 557-558 55a-55f 589-590 "
722
+ "5be 5c0 5c3 5c6 5c8-5cf 5eb-5ee 5f3-60f 61b-61f 66a-66d 6d4 6dd-6de 6e9 6fd-6fe 700-70f "
723
+ "74b-74c 7b2-7bf 7f6-7f9 7fb-7fc 7fe-7ff 82e-83f 85c-85f 86b-89f 8b5 8c8-8d2 8e2 964-965 "
724
+ "970 984 98d-98e 991-992 9a9 9b1 9b3-9b5 9ba-9bb 9c5-9c6 9c9-9ca 9cf-9d6 9d8-9db 9de "
725
+ "9e4-9e5 9f2-9fb 9fd 9ff-a00 a04 a0b-a0e a11-a12 a29 a31 a34 a37 a3a-a3b a3d a43-a46 "
726
+ "a49-a4a a4e-a50 a52-a58 a5d a5f-a65 a76-a80 a84 a8e a92 aa9 ab1 ab4 aba-abb ac6 aca "
727
+ "ace-acf ad1-adf ae4-ae5 af0-af8 b00 b04 b0d-b0e b11-b12 b29 b31 b34 b3a-b3b b45-b46 "
728
+ "b49-b4a b4e-b54 b58-b5b b5e b64-b65 b70 b72-b81 b84 b8b-b8d b91 b96-b98 b9b b9d ba0-ba2 "
729
+ "ba5-ba7 bab-bad bba-bbd bc3-bc5 bc9 bce-bcf bd1-bd6 bd8-be5 bf0-bff c0d c11 c29 c3a-c3c "
730
+ "c45 c49 c4e-c54 c57 c5b-c5f c64-c65 c70-c7f c84 c8d c91 ca9 cb4 cba-cbb cc5 cc9 cce-cd4 "
731
+ "cd7-cdd cdf ce4-ce5 cf0 cf3-cff d0d d11 d45 d49 d4f-d53 d58-d5e d64-d65 d70-d79 d80 d84 "
732
+ "d97-d99 db2 dbc dbe-dbf dc7-dc9 dcb-dce dd5 dd7 de0-de5 df0-df1 df4-e00 e3b-e3f e4f "
733
+ "e5a-e80 e83 e85 e8b ea4 ea6 ebe-ebf ec5 ec7 ece-ecf eda-edb ee0-eff f01-f17 f1a-f1f "
734
+ "f2a-f34 f36 f38 f3a-f3d f48 f6d-f70 f85 f98 fbd-fc5 fc7-fff 104a-104f 109e-109f 10c6 "
735
+ "10c8-10cc 10ce-10cf 10fb 1249 124e-124f 1257 1259 125e-125f 1289 128e-128f 12b1 "
736
+ "12b6-12b7 12bf 12c1 12c6-12c7 12d7 1311 1316-1317 135b-135c 1360-137f 1390-139f "
737
+ "13f6-13f7 13fe-1400 166d-166e 1680 169b-169f 16eb-16ed 16f9-16ff 170d 1715-171f "
738
+ "1735-173f 1754-175f 176d 1771 1774-177f 17d4-17d6 17d8-17db 17de-17df 17ea-180a "
739
+ "180e-180f 181a-181f 1879-187f 18ab-18af 18f6-18ff 191f 192c-192f 193c-1945 196e-196f "
740
+ "1975-197f 19ac-19af 19ca-19cf 19da-19ff 1a1c-1a1f 1a5f 1a7d-1a7e 1a8a-1a8f 1a9a-1aa6 "
741
+ "1aa8-1aaf 1ac1-1aff 1b4c-1b4f 1b5a-1b6a 1b74-1b7f 1bf4-1bff 1c38-1c3f 1c4a-1c4c "
742
+ "1c7e-1c7f 1c89-1c8f 1cbb-1cbc 1cc0-1ccf 1cd3 1cfb-1cff 1dfa 1f16-1f17 1f1e-1f1f "
743
+ "1f46-1f47 1f4e-1f4f 1f58 1f5a 1f5c 1f5e 1f7e-1f7f 1fb5 1fbd 1fbf-1fc1 1fc5 1fcd-1fcf "
744
+ "1fd4-1fd5 1fdc-1fdf 1fed-1ff1 1ff5 1ffd-203e 2041-2053 2055-2070 2072-207e 2080-208f "
745
+ "209d-20cf 20f1-2101 2103-2106 2108-2109 2114 2116-2118 211e-2123 2125 2127 2129 212e "
746
+ "213a-213b 2140-2144 214a-214d 214f-215f 2189-24b5 24ea-2bff 2c2f 2c5f 2ce5-2cea "
747
+ "2cf4-2cff 2d26 2d28-2d2c 2d2e-2d2f 2d68-2d6e 2d70-2d7e 2d97-2d9f 2da7 2daf 2db7 2dbf "
748
+ "2dc7 2dcf 2dd7 2ddf 2e00-2e2e 2e30-3004 3008-3020 3030 3036-3037 303d-3040 3097-3098 "
749
+ "309b-309c 30a0 30fb 3100-3104 3130 318f-319f 31c0-31ef 3200-33ff 4dc0-4dff 9ffd-9fff "
750
+ "a48d-a4cf a4fe-a4ff a60d-a60f a62c-a63f a673 a67e a6f2-a716 a720-a721 a789-a78a "
751
+ "a7c0-a7c1 a7cb-a7f4 a828-a82b a82d-a83f a874-a87f a8c6-a8cf a8da-a8df a8f8-a8fa a8fc "
752
+ "a92e-a92f a954-a95f a97d-a97f a9c1-a9ce a9da-a9df a9ff aa37-aa3f aa4e-aa4f aa5a-aa5f "
753
+ "aa77-aa79 aac3-aada aade-aadf aaf0-aaf1 aaf7-ab00 ab07-ab08 ab0f-ab10 ab17-ab1f ab27 "
754
+ "ab2f ab5b ab6a-ab6f abeb abee-abef abfa-abff d7a4-d7af d7c7-d7ca d7fc-d7ff e000-f8ff "
755
+ "fa6e-fa6f fada-faff fb07-fb12 fb18-fb1c fb29 fb37 fb3d fb3f fb42 fb45 fbb2-fbd2 "
756
+ "fd3e-fd4f fd90-fd91 fdc8-fdef fdfc-fdff fe10-fe1f fe30-fe32 fe35-fe4c fe50-fe6f fe75 "
757
+ "fefd-ff0f ff1a-ff20 ff3b-ff3e ff40 ff5b-ff65 ffbf-ffc1 ffc8-ffc9 ffd0-ffd1 ffd8-ffd9 "
758
+ "ffdd-ffff 1000c 10027 1003b 1003e 1004e-1004f 1005e-1007f 100fb-1013f 10175-101fc "
759
+ "101fe-1027f 1029d-1029f 102d1-102df 102e1-102ff 10320-1032c 1034b-1034f 1037b-1037f "
760
+ "1039e-1039f 103c4-103c7 103d0 103d6-103ff 1049e-1049f 104aa-104af 104d4-104d7 "
761
+ "104fc-104ff 10528-1052f 10564-105ff 10737-1073f 10756-1075f 10768-107ff 10806-10807 "
762
+ "10809 10836 10839-1083b 1083d-1083e 10856-1085f 10877-1087f 1089f-108df 108f3 "
763
+ "108f6-108ff 10916-1091f 1093a-1097f 109b8-109bd 109c0-109ff 10a04 10a07-10a0b 10a14 "
764
+ "10a18 10a36-10a37 10a3b-10a3e 10a40-10a5f 10a7d-10a7f 10a9d-10abf 10ac8 10ae7-10aff "
765
+ "10b36-10b3f 10b56-10b5f 10b73-10b7f 10b92-10bff 10c49-10c7f 10cb3-10cbf 10cf3-10cff "
766
+ "10d28-10d2f 10d3a-10e7f 10eaa 10ead-10eaf 10eb2-10eff 10f1d-10f26 10f28-10f2f "
767
+ "10f51-10faf 10fc5-10fdf 10ff7-10fff 11047-11065 11070-1107e 110bb-110cf 110e9-110ef "
768
+ "110fa-110ff 11135 11140-11143 11148-1114f 11174-11175 11177-1117f 111c5-111c8 111cd "
769
+ "111db 111dd-111ff 11212 11238-1123d 1123f-1127f 11287 11289 1128e 1129e 112a9-112af "
770
+ "112eb-112ef 112fa-112ff 11304 1130d-1130e 11311-11312 11329 11331 11334 1133a "
771
+ "11345-11346 11349-1134a 1134e-1134f 11351-11356 11358-1135c 11364-11365 1136d-1136f "
772
+ "11375-113ff 1144b-1144f 1145a-1145d 11462-1147f 114c6 114c8-114cf 114da-1157f "
773
+ "115b6-115b7 115c1-115d7 115de-115ff 11641-11643 11645-1164f 1165a-1167f 116b9-116bf "
774
+ "116ca-116ff 1171b-1171c 1172c-1172f 1173a-117ff 1183b-1189f 118ea-118fe 11907-11908 "
775
+ "1190a-1190b 11914 11917 11936 11939-1193a 11944-1194f 1195a-1199f 119a8-119a9 "
776
+ "119d8-119d9 119e2 119e5-119ff 11a3f-11a46 11a48-11a4f 11a9a-11a9c 11a9e-11abf "
777
+ "11af9-11bff 11c09 11c37 11c41-11c4f 11c5a-11c71 11c90-11c91 11ca8 11cb7-11cff 11d07 "
778
+ "11d0a 11d37-11d39 11d3b 11d3e 11d48-11d4f 11d5a-11d5f 11d66 11d69 11d8f 11d92 "
779
+ "11d99-11d9f 11daa-11edf 11ef7-11faf 11fb1-11fff 1239a-123ff 1246f-1247f 12544-12fff "
780
+ "1342f-143ff 14647-167ff 16a39-16a3f 16a5f 16a6a-16acf 16aee-16aef 16af5-16aff "
781
+ "16b37-16b3f 16b44-16b4f 16b5a-16b62 16b78-16b7c 16b90-16e3f 16e80-16eff 16f4b-16f4e "
782
+ "16f88-16f8e 16fa0-16fdf 16fe2 16fe5-16fef 16ff2-16fff 187f8-187ff 18cd6-18cff "
783
+ "18d09-1afff 1b11f-1b14f 1b153-1b163 1b168-1b16f 1b2fc-1bbff 1bc6b-1bc6f 1bc7d-1bc7f "
784
+ "1bc89-1bc8f 1bc9a-1bc9c 1bc9f-1d164 1d16a-1d16c 1d173-1d17a 1d183-1d184 1d18c-1d1a9 "
785
+ "1d1ae-1d241 1d245-1d3ff 1d455 1d49d 1d4a0-1d4a1 1d4a3-1d4a4 1d4a7-1d4a8 1d4ad 1d4ba "
786
+ "1d4bc 1d4c4 1d506 1d50b-1d50c 1d515 1d51d 1d53a 1d53f 1d545 1d547-1d549 1d551 "
787
+ "1d6a6-1d6a7 1d6c1 1d6db 1d6fb 1d715 1d735 1d74f 1d76f 1d789 1d7a9 1d7c3 1d7cc-1d7cd "
788
+ "1d800-1d9ff 1da37-1da3a 1da6d-1da74 1da76-1da83 1da85-1da9a 1daa0 1dab0-1dfff 1e007 "
789
+ "1e019-1e01a 1e022 1e025 1e02b-1e0ff 1e12d-1e12f 1e13e-1e13f 1e14a-1e14d 1e14f-1e2bf "
790
+ "1e2fa-1e7ff 1e8c5-1e8cf 1e8d7-1e8ff 1e94c-1e94f 1e95a-1edff 1ee04 1ee20 1ee23 "
791
+ "1ee25-1ee26 1ee28 1ee33 1ee38 1ee3a 1ee3c-1ee41 1ee43-1ee46 1ee48 1ee4a 1ee4c 1ee50 "
792
+ "1ee53 1ee55-1ee56 1ee58 1ee5a 1ee5c 1ee5e 1ee60 1ee63 1ee65-1ee66 1ee6b 1ee73 1ee78 "
793
+ "1ee7d 1ee7f 1ee8a 1ee9c-1eea0 1eea4 1eeaa 1eebc-1f12f 1f14a-1f14f 1f16a-1f16f "
794
+ "1f18a-1fbef 1fbfa-1ffff 2a6de-2a6ff 2b735-2b73f 2b81e-2b81f 2cea2-2ceaf 2ebe1-2f7ff "
795
+ "2fa1e-2ffff 3134b-e00ff e01f0-10ffff"
796
+ )
797
+ _SLUG_REMOVED = re.compile(
798
+ "["
799
+ + "".join(
800
+ f"\\U{int(lo, 16):08x}-\\U{int(hi or lo, 16):08x}"
801
+ for lo, _, hi in (span.partition("-") for span in _SLUG_REMOVED_RANGES.split())
802
+ )
803
+ + "]"
804
+ )
805
+ #: A heading whose content holds one of these carries inline markup, and has
806
+ #: no slug (0001, addendum on the marker line and heading slugs).
807
+ _INLINE_MARKUP = frozenset("*_`[<&")
808
+
809
+
810
+ def _slug(text: str) -> str | None:
811
+ """GitHub's heading slug, as github-slugger 2.0.0 computes it from the
812
+ heading's raw content: lower-cased, its removed characters dropped, each
813
+ space turned into `-`. None for a heading whose content holds an
814
+ inline-markup character or `&`, whose rendered text the raw content is
815
+ not. Lower-casing is the runtime's: Python's Unicode tables, not node's,
816
+ so the two can differ where their Unicode versions do (U+A7CB and U+A7DC
817
+ lower-case under node 24's Unicode 17, not under Python 3.12's 15.0)."""
818
+ if _INLINE_MARKUP.intersection(text):
819
+ return None
820
+ return _SLUG_REMOVED.sub("", text.lower()).replace(" ", "-") or None
821
+
822
+
823
+ class Document:
824
+ """An accepted document: its frontmatter, blocks, leaves and names."""
825
+
826
+ def __init__(self, root: Node, lines: list[str], frontmatter: list[int] | None) -> None:
827
+ self.root = root
828
+ self.frontmatter = frontmatter
829
+ self.blocks = []
830
+ for node in root.children:
831
+ end = node.end
832
+ while end > node.start and _blank(lines[end - 1]):
833
+ end -= 1
834
+ self.blocks.append({"type": node.type, "lines": [node.start, end]})
835
+ self.nodes = list(self._walk(root, ()))
836
+ self.leaves = [
837
+ {"node": list(node.path), "type": node.type, "content": node.content()}
838
+ for node in self.nodes
839
+ if node.type in LEAVES
840
+ ]
841
+ self.names = self._names()
842
+
843
+ def _walk(self, node: Node, path: tuple[int, ...]):
844
+ for i, child in enumerate(node.children):
845
+ child.path = (*path, i)
846
+ yield child
847
+ yield from self._walk(child, child.path)
848
+
849
+ def _names(self) -> dict[str, dict]:
850
+ """What each marker names. Every problem is collected first, so that
851
+ when several refusals apply the one reported does not depend on
852
+ document order: duplicate-name, then unpaired-region, then
853
+ dangling-marker (0002, addendum on markers that name nothing). A name is declared by a
854
+ one-line marker or a `:begin`; an `:end` only closes one."""
855
+ declared: dict[str, list[dict]] = {}
856
+ unpaired: list[str] = []
857
+ dangling: list[str] = []
858
+ for container in [self.root, *self.nodes]:
859
+ if container.type not in ("document", "block_quote", "list_item"):
860
+ continue
861
+ siblings = container.children
862
+ begun: dict[str, int] = {}
863
+ for i, node in enumerate(siblings):
864
+ if node.marker is None:
865
+ continue
866
+ name, form = node.marker
867
+ if form is None:
868
+ target = siblings[i + 1] if i + 1 < len(siblings) else None
869
+ if target is None or target.marker is not None:
870
+ dangling.append(f"#{name} names no node")
871
+ declared.setdefault(name, []).append({"dangling": True})
872
+ else:
873
+ declared.setdefault(name, []).append(target.ref())
874
+ elif form == ":begin":
875
+ declared.setdefault(name, []).append({})
876
+ begun.setdefault(name, i)
877
+ elif name in begun:
878
+ first = begun.pop(name)
879
+ region = [n.ref() for n in siblings[first + 1 : i]]
880
+ if not region:
881
+ dangling.append(f"#{name}:begin and #{name}:end name no node")
882
+ declared[name][declared[name].index({})] = {"region": region}
883
+ else:
884
+ unpaired.append(f"#{name}:end has no :begin beside it")
885
+ unpaired += [f"#{name}:begin has no :end beside it" for name in begun]
886
+ for name, targets in declared.items():
887
+ if len(targets) > 1:
888
+ raise Refusal("duplicate-name", f"#{name} is declared twice")
889
+ if unpaired:
890
+ raise Refusal("unpaired-region", unpaired[0])
891
+ if dangling:
892
+ raise Refusal("dangling-marker", dangling[0])
893
+ return {name: targets[0] for name, targets in declared.items()}
894
+
895
+ def resolve(self, name: str) -> dict | str:
896
+ """Markers first, then heading slugs. Zero or several matches is
897
+ `#REF!(name)`."""
898
+ if name in self.names:
899
+ return self.names[name]
900
+ headings = [n for n in self.nodes if n.type == "heading" and _slug(n.text) == name]
901
+ if len(headings) == 1:
902
+ return headings[0].ref()
903
+ return f"#REF!({name})"
904
+
905
+
906
+ def parse(text: str, *, record: bool = False) -> Document:
907
+ """Read `text` as a blockspec document, or raise `Refusal`. A comment
908
+ record's body names things in the commented document, so `record` leaves
909
+ its links unread (SPEC.md section 9, `dangling-link`)."""
910
+ # SPEC.md section 3: U+0000 is replaced with U+FFFD (CommonMark 0.31.2
911
+ # section 2.3), before anything else reads the text.
912
+ text = text.replace("\x00", "\ufffd")
913
+ lines = _split_lines(text)
914
+ for line in lines:
915
+ if _CONFLICT.match(line):
916
+ raise Refusal("conflict-marker", "a git conflict-marker line")
917
+ frontmatter = None
918
+ first = 0
919
+ if len(lines) > 1 and lines[0] == "---" and not _blank(lines[1]):
920
+ for at in range(1, len(lines)):
921
+ if lines[at].rstrip(" \t") == "---":
922
+ frontmatter = [1, at + 1]
923
+ first = at + 1
924
+ break
925
+ parser = _Parser()
926
+ for at in range(first, len(lines)):
927
+ parser.incorporate(lines[at], at + 1)
928
+ root = parser.finish(len(lines))
929
+ for node in _tables(root):
930
+ siblings = node.parent.children
931
+ at = siblings.index(node)
932
+ if at + 1 < len(siblings) and siblings[at + 1].start == node.end + 1:
933
+ raise Refusal("table-not-blank-below", "a table directly above a non-blank line")
934
+ doc = Document(root, lines, frontmatter)
935
+ _refuse_shadowed_slugs(doc)
936
+ if not record:
937
+ _refuse_dangling_links(doc)
938
+ return doc
939
+
940
+
941
+ def _refuse_shadowed_slugs(doc: Document) -> None:
942
+ """A marker whose name is the slug of a heading it does not bind takes the
943
+ name from that heading (SPEC.md section 9; 0005 item 11)."""
944
+ for node in doc.nodes:
945
+ if node.type != "heading":
946
+ continue
947
+ slug = _slug(node.text)
948
+ target = doc.names.get(slug)
949
+ if target is None or target == node.ref():
950
+ continue
951
+ if "region" in target and node.ref() in target["region"]:
952
+ continue
953
+ siblings = node.parent.children
954
+ following = siblings[siblings.index(node) + 1 : siblings.index(node) + 2]
955
+ if following and following[0].marker == (slug, ":begin"):
956
+ continue
957
+ raise Refusal("name-shadows-slug", f"the marker {slug!r} reuses a heading's slug")
958
+
959
+
960
+ _ASCII_PUNCTUATION = set(string.punctuation)
961
+ _HTML_OTHER = re.compile(
962
+ r"<!-->|<!--->|<!--.*?-->|<\?.*?\?>|<![A-Za-z][^>]*>|<!\[CDATA\[.*?\]\]>|"
963
+ r"</[A-Za-z][A-Za-z0-9-]*[ \t\n]*>",
964
+ re.S,
965
+ )
966
+ _ATTRIBUTE = (
967
+ r"[ \t\n]+([A-Za-z_:][A-Za-z0-9_.:-]*)"
968
+ r"(?:[ \t\n]*=[ \t\n]*([^ \t\n\"'=<>`]+|'[^']*'|\"[^\"]*\"))?"
969
+ )
970
+ _OPEN_TAG = re.compile(
971
+ r"<[A-Za-z][A-Za-z0-9-]*(?:" + _ATTRIBUTE.replace("(", "(?:", 1) + r")*[ \t\n]*/?>"
972
+ )
973
+ _ONE_ATTRIBUTE = re.compile(_ATTRIBUTE)
974
+ _AUTOLINK = re.compile(
975
+ r"<[A-Za-z][A-Za-z0-9.+-]{1,31}:[^\x00-\x20<>]*>"
976
+ r"|<[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+@[A-Za-z0-9.-]+>"
977
+ )
978
+ _LINK_NAME = re.compile(r"[a-z0-9][a-z0-9-]*\Z")
979
+ _HEADING_ID = re.compile(r"\{#([^\s}]+)\}[ \t]*$")
980
+
981
+
982
+ def _link_tail(text: str, at: int) -> tuple[str, str | None, int] | None:
983
+ """The `(destination title)` at `text[at]`, as CommonMark reads one: the
984
+ destination's raw text, the title as written or None, and where it ends."""
985
+
986
+ def spaces(i: int) -> int:
987
+ while i < len(text) and text[i] in " \t\n":
988
+ i += 1
989
+ return i
990
+
991
+ if text[at : at + 1] != "(":
992
+ return None
993
+ i = spaces(at + 1)
994
+ if text[i : i + 1] == "<":
995
+ j = i + 1
996
+ while j < len(text) and text[j] not in "<>\n":
997
+ j += 2 if text[j] == "\\" else 1
998
+ if text[j : j + 1] != ">":
999
+ return None
1000
+ destination, i = text[i + 1 : j], j + 1
1001
+ else:
1002
+ j, depth = i, 0
1003
+ while j < len(text):
1004
+ c = text[j]
1005
+ if c == "\\" and text[j + 1 : j + 2] in _ASCII_PUNCTUATION and j + 1 < len(text):
1006
+ j += 2
1007
+ continue
1008
+ if c <= " " or c == "\x7f":
1009
+ break
1010
+ if c == "(":
1011
+ depth += 1
1012
+ elif c == ")":
1013
+ if depth == 0:
1014
+ break
1015
+ depth -= 1
1016
+ j += 1
1017
+ if depth:
1018
+ return None
1019
+ destination, i = text[i:j], j
1020
+ after = spaces(i)
1021
+ title = None
1022
+ if after > i and text[after : after + 1] in ('"', "'", "("):
1023
+ close = ")" if text[after] == "(" else text[after]
1024
+ j = after + 1
1025
+ while j < len(text) and text[j] != close:
1026
+ j += 2 if text[j] == "\\" else 1
1027
+ if j >= len(text):
1028
+ return None
1029
+ title, after = text[after : j + 1], spaces(j + 1)
1030
+ if text[after : after + 1] != ")":
1031
+ return None
1032
+ return destination, title, after + 1
1033
+
1034
+
1035
+ def _scan_inlines(text: str) -> tuple[list[dict], list[tuple[int, int, str]]]:
1036
+ """The inline links and images in `text`, and its inline HTML as `(start,
1037
+ end, text)`, by CommonMark's rules: code spans, autolinks and inline HTML
1038
+ bind tighter than links, and a link holds no link."""
1039
+ links: list[dict] = []
1040
+ html: list[tuple[int, int, str]] = []
1041
+ brackets: list[dict] = []
1042
+ i = 0
1043
+ while i < len(text):
1044
+ c = text[i]
1045
+ if c == "\\" and text[i + 1 : i + 2] in _ASCII_PUNCTUATION and i + 1 < len(text):
1046
+ i += 2
1047
+ elif c == "`":
1048
+ run = re.compile("`+").match(text, i).end() - i
1049
+ ticks = next(
1050
+ (m for m in re.finditer("`+", text[i + run :]) if len(m.group()) == run),
1051
+ None,
1052
+ )
1053
+ i = i + run + ticks.end() if ticks else i + run
1054
+ elif c == "<":
1055
+ auto = _AUTOLINK.match(text, i)
1056
+ tag = None if auto else (_OPEN_TAG.match(text, i) or _HTML_OTHER.match(text, i))
1057
+ if tag:
1058
+ html.append((tag.start(), tag.end(), tag.group()))
1059
+ i = (auto or tag).end() if (auto or tag) else i + 1
1060
+ elif c == "!" and text[i + 1 : i + 2] == "[":
1061
+ brackets.append({"image": True, "active": True, "at": i, "open": 2})
1062
+ i += 2
1063
+ elif c == "[":
1064
+ brackets.append({"image": False, "active": True, "at": i, "open": 1})
1065
+ i += 1
1066
+ elif c == "]":
1067
+ opener = brackets.pop() if brackets else None
1068
+ tail = _link_tail(text, i + 1) if opener and opener["active"] else None
1069
+ if tail is None:
1070
+ i += 1
1071
+ continue
1072
+ links.append(
1073
+ {
1074
+ "image": opener["image"],
1075
+ "destination": tail[0],
1076
+ "title": tail[1],
1077
+ "start": opener["at"],
1078
+ "open": opener["open"],
1079
+ "close": i,
1080
+ "end": tail[2],
1081
+ }
1082
+ )
1083
+ if not opener["image"]:
1084
+ for earlier in brackets:
1085
+ if not earlier["image"]:
1086
+ earlier["active"] = False
1087
+ i = tail[2]
1088
+ else:
1089
+ i += 1
1090
+ return links, html
1091
+
1092
+
1093
+ def _rendered_slugs(raw: str) -> set[str]:
1094
+ """The slugs a renderer could give a heading whose raw content is `raw`."""
1095
+ links, html = _scan_inlines(raw)
1096
+ drop = [False] * len(raw)
1097
+ for start, end, _tag in html:
1098
+ drop[start:end] = [True] * (end - start)
1099
+ for link in links:
1100
+ drop[link["start"] : link["start"] + link["open"]] = [True] * link["open"]
1101
+ drop[link["close"] : link["end"]] = [True] * (link["end"] - link["close"])
1102
+ text = "".join(c for c, gone in zip(raw, drop, strict=True) if not gone)
1103
+ found = set()
1104
+ for variant in (text, re.sub(r"[*`]", "", text), re.sub(r"[*_`]", "", text)):
1105
+ found.add(_SLUG_REMOVED.sub("", variant.lower()).replace(" ", "-"))
1106
+ return found
1107
+
1108
+
1109
+ def _tag_anchors(tag: str) -> list[str]:
1110
+ """The `id` and `name` attribute values of an inline HTML open tag."""
1111
+ if not _OPEN_TAG.fullmatch(tag):
1112
+ return []
1113
+ found = []
1114
+ for match in _ONE_ATTRIBUTE.finditer(tag):
1115
+ if match.group(1).lower() in ("id", "name") and match.group(2):
1116
+ found.append(match.group(2).strip("\"'"))
1117
+ return found
1118
+
1119
+
1120
+ def _possible_targets(doc: Document) -> set[str]:
1121
+ """Everything in the document a same-file link could mean (0005 item 10)."""
1122
+ targets = {"top", *doc.names}
1123
+ seen: dict[str, int] = {}
1124
+ for node in doc.nodes:
1125
+ if node.type == "heading":
1126
+ custom = _HEADING_ID.search(node.text)
1127
+ if custom:
1128
+ targets.add(custom.group(1))
1129
+ raws = {node.text, _HEADING_ID.sub("", node.text).rstrip()}
1130
+ for slug in {found for raw in raws for found in _rendered_slugs(raw)}:
1131
+ count = seen.get(slug, 0)
1132
+ seen[slug] = count + 1
1133
+ targets.add(slug if count == 0 else f"{slug}-{count}")
1134
+ for leaf in doc.leaves:
1135
+ if leaf["type"] in ("paragraph", "heading"):
1136
+ texts = [leaf["content"]]
1137
+ elif leaf["type"] == "table":
1138
+ texts = [cell for row in leaf["content"] for cell in row]
1139
+ else:
1140
+ continue
1141
+ for text in texts:
1142
+ for _start, _end, tag in _scan_inlines(text)[1]:
1143
+ targets.update(_tag_anchors(tag))
1144
+ return targets
1145
+
1146
+
1147
+ def _refuse_dangling_links(doc: Document) -> None:
1148
+ """A link to `#name` where nothing could be its target (SPEC.md section 9;
1149
+ 0005 item 10)."""
1150
+ targets = None
1151
+ for leaf in doc.leaves:
1152
+ if leaf["type"] in ("paragraph", "heading"):
1153
+ texts = [leaf["content"]]
1154
+ elif leaf["type"] == "table":
1155
+ texts = [cell for row in leaf["content"] for cell in row]
1156
+ else:
1157
+ continue
1158
+ for text in texts:
1159
+ for link in _scan_inlines(text)[0]:
1160
+ destination = link["destination"]
1161
+ if link["image"] or not destination.startswith("#"):
1162
+ continue
1163
+ if link["title"] == '"transclude"':
1164
+ continue
1165
+ name = destination[1:]
1166
+ if not _LINK_NAME.match(name):
1167
+ continue
1168
+ if targets is None:
1169
+ targets = _possible_targets(doc)
1170
+ if name not in targets:
1171
+ raise Refusal("dangling-link", f"a link to {name!r}, which names nothing")
1172
+
1173
+
1174
+ def _tables(node: Node):
1175
+ for child in node.children:
1176
+ if child.type == "table":
1177
+ yield child
1178
+ yield from _tables(child)
1179
+
1180
+
1181
+ def conform(text: str, queries: list[str]) -> dict:
1182
+ """The interface `conformance/run_cases.py --impl FILE` calls."""
1183
+ try:
1184
+ doc = parse(text)
1185
+ except Refusal as refusal:
1186
+ return {"accept": False, "refusal": refusal.code, "message": refusal.message}
1187
+ return {
1188
+ "accept": True,
1189
+ "frontmatter": doc.frontmatter,
1190
+ "blocks": doc.blocks,
1191
+ "leaves": doc.leaves,
1192
+ "names": doc.names,
1193
+ "resolve": {name: doc.resolve(name) for name in queries},
1194
+ }
1195
+
1196
+
1197
+ def resolve(repo: dict, queries: dict) -> dict:
1198
+ """The interface `conformance/run_cases.py --impl FILE` calls for the
1199
+ `anchor` kind. The resolver is `anchor.py`, beside this file, loaded by
1200
+ path as this file is, and handed this file's reader, so that a mutated
1201
+ copy of the reader is the one it runs."""
1202
+ path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "anchor.py")
1203
+ anchor = sys.modules.get("blockspec_anchor")
1204
+ if anchor is None or anchor.__file__ != path:
1205
+ spec = importlib.util.spec_from_file_location("blockspec_anchor", path)
1206
+ if spec is None or spec.loader is None:
1207
+ raise ImportError(f"cannot load {path!r}")
1208
+ anchor = importlib.util.module_from_spec(spec)
1209
+ sys.modules["blockspec_anchor"] = anchor
1210
+ spec.loader.exec_module(anchor)
1211
+ reader = SimpleNamespace(parse=parse, Refusal=Refusal, split_lines=_split_lines, slug=_slug)
1212
+ return anchor.resolve(repo, queries, reader)