marktip 0.7.1__tar.gz → 0.7.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {marktip-0.7.1 → marktip-0.7.3}/CMakeLists.txt +1 -1
- {marktip-0.7.1 → marktip-0.7.3}/PKG-INFO +68 -10
- {marktip-0.7.1 → marktip-0.7.3}/README.md +65 -8
- {marktip-0.7.1 → marktip-0.7.3}/pyproject.toml +5 -3
- marktip-0.7.3/scripts/build_cjk_map.py +100 -0
- {marktip-0.7.1 → marktip-0.7.3}/src/ast.cpp +274 -30
- {marktip-0.7.1 → marktip-0.7.3}/src/ast.h +18 -3
- {marktip-0.7.1 → marktip-0.7.3}/src/errors.h +3 -2
- {marktip-0.7.1 → marktip-0.7.3}/src/marktip.cpp +20 -11
- {marktip-0.7.1 → marktip-0.7.3}/src/parser.cpp +28 -1
- {marktip-0.7.1 → marktip-0.7.3}/src/parser.h +1 -1
- marktip-0.7.3/tests/test_cjk_friendly.py +94 -0
- marktip-0.7.3/tests/test_content_model.py +159 -0
- {marktip-0.7.1 → marktip-0.7.3}/tests/test_errors.py +91 -0
- marktip-0.7.3/tests/test_node_type_policy.py +100 -0
- marktip-0.7.3/tests/test_nul_normalization.py +176 -0
- {marktip-0.7.1 → marktip-0.7.3}/tests/test_strict.py +7 -4
- marktip-0.7.3/third_party/md4c/src/cjk.h +84 -0
- {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/src/md4c.c +116 -115
- {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/upstream_metadata/README.md +24 -6
- marktip-0.7.3/third_party/md4c/upstream_metadata/Unicode-LICENSE.txt +39 -0
- {marktip-0.7.1 → marktip-0.7.3}/.github/workflows/build-distributions.yml +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/.gitignore +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/LICENSE +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/python/marktip/__init__.py +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/scripts/benchmark.py +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/src/serializer.cpp +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/src/serializer.h +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/tests/test_document.py +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/tests/test_parse.py +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/tests/test_roundtrip.py +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/tests/test_serialize.py +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/tests/test_uri_policy.py +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/tests/test_validate.py +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/LICENSE.md +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/src/entity.c +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/src/entity.h +0 -0
- {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/src/md4c.h +0 -0
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: marktip
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.3
|
|
4
4
|
Summary: Fast C++/MD4C Markdown parser and canonical Markdown serializer for Tiptap-style JSON
|
|
5
5
|
Keywords: markdown,tiptap,md4c,parser
|
|
6
6
|
Author: Saneaven (Minjae Kyung)
|
|
7
|
-
License-Expression: MIT
|
|
7
|
+
License-Expression: MIT AND Unicode-3.0
|
|
8
8
|
License-File: LICENSE
|
|
9
9
|
License-File: third_party/md4c/LICENSE.md
|
|
10
|
+
License-File: third_party/md4c/upstream_metadata/Unicode-LICENSE.txt
|
|
10
11
|
Classifier: Development Status :: 3 - Alpha
|
|
11
12
|
Classifier: Intended Audience :: Developers
|
|
12
13
|
Classifier: Operating System :: MacOS
|
|
@@ -52,14 +53,20 @@ doc = tm.from_dict(ast)
|
|
|
52
53
|
```
|
|
53
54
|
|
|
54
55
|
`from_markdown` follows GFM/CommonMark by default.
|
|
55
|
-
Pass `cjk_friendly=True` to
|
|
56
|
+
Pass `cjk_friendly=True` to follow the [CommonMark CJK-friendly Amendments Specification](https://github.com/tats-u/markdown-cjk-friendly/blob/main/specification.md) for emphasis (`*`, `_`) and strikethrough (`~`). This opt-in extension recognizes CJK punctuation as well as letters, so `**《제목》**a`, `a**《제목》**`, and `**「約束」**a` retain their bold marks:
|
|
56
57
|
|
|
57
58
|
```python
|
|
58
59
|
doc = tm.from_markdown("**볼드**은 강조", cjk_friendly=True)
|
|
59
60
|
```
|
|
60
61
|
|
|
61
62
|
Strikethrough follows the GFM reference implementations (cmark-gfm/micromark): intra-word `~~` strikes (`a~~b~~c`), including next to CJK letters (`~~삭제~~은`).
|
|
62
|
-
`cjk_friendly` still matters for
|
|
63
|
+
`cjk_friendly` still matters for punctuation-adjacent cases such as `~~"인용"~~라고`.
|
|
64
|
+
|
|
65
|
+
The CJK classifier uses Unicode 17.0.0 East Asian Width `W`, `F`, or `H` (excluding default emoji presentation characters), plus the Hangul script. This includes fullwidth digits such as `1`. Before a delimiter, CJK characters followed by a non-emoji general-use variation selector (`U+FE00`–`U+FE0E`) are classified as one sequence; ideographic variation selectors (`U+E0100`–`U+E01EF`) and standardized fullwidth quotation-mark sequences are also recognized. `U+FE0F` is an emoji presentation selector and is excluded from that sequence rule.
|
|
66
|
+
|
|
67
|
+
Underscores retain CommonMark's additional restriction on emphasis inside words. The CJK-friendly specification changes delimiter flanking without removing that protection for names such as `snake_case`: `_《제목》_을` and `__《제목》__을` stay literal, even with `cjk_friendly=True`. Use `*` or `**` when emphasis must adjoin a word.
|
|
68
|
+
|
|
69
|
+
> **Changed in 0.7.3** — CJK-friendly character classification, variation-selector sequences, flanking, and underscore conditions follow the specification. The same flanking rules apply to `~~`. An editor document containing bold `《제목》` followed by `a` now survives serialization and reparsing with `cjk_friendly=True`.
|
|
63
70
|
|
|
64
71
|
Pass `html=False` to parse raw HTML as literal text instead of `htmlBlock`/`htmlInline` nodes.
|
|
65
72
|
Content is preserved (the serializer escapes it), and `<br>` inside table cells still maps to `hardBreak` so marktip's own table output round-trips:
|
|
@@ -104,9 +111,30 @@ doc = tm.from_markdown(untrusted, link_schemes=("https",), image_relative="rejec
|
|
|
104
111
|
Violations raise `InvalidNodeError` with `.field` set to `"href"` or `"src"`.
|
|
105
112
|
The options are keyword-only.
|
|
106
113
|
|
|
114
|
+
### Rejecting node types
|
|
115
|
+
|
|
116
|
+
marktip's schema includes every node it can serialize, but an application may use a smaller ProseMirror dialect.
|
|
117
|
+
Both write boundaries accept `reject_node_types`, so that dialect can be enforced after either Markdown parsing or dict conversion without a second application-side tree walk:
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
NO_RAW_HTML = ("htmlBlock", "htmlInline")
|
|
121
|
+
|
|
122
|
+
doc = tm.from_dict(editor_json, reject_node_types=NO_RAW_HTML)
|
|
123
|
+
doc = tm.from_markdown(imported_markdown, reject_node_types=NO_RAW_HTML)
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
`None` (the default) rejects nothing; an empty iterable has the same effect.
|
|
127
|
+
Every entry must name a node type marktip knows, and matching is exact and case-sensitive.
|
|
128
|
+
A bare string or a non-string entry raises `TypeError`; an unknown node name raises `ValueError`.
|
|
129
|
+
A document containing a configured type raises `InvalidNodeError` with `.code == "disallowed_node_type"`, `.field == "type"`, and `.path` locating the node.
|
|
130
|
+
The option is keyword-only.
|
|
131
|
+
|
|
132
|
+
This differs from `html=False`: `html=False` preserves raw markup as escaped literal text on Markdown input, while rejecting `htmlBlock`/`htmlInline` refuses markup that parsed as a raw-HTML node.
|
|
133
|
+
`reject_node_types` also works for dict input, where there is no parser flag.
|
|
134
|
+
|
|
107
135
|
### Refusing lossy conversions
|
|
108
136
|
|
|
109
|
-
Node and mark *types* are closed, but
|
|
137
|
+
Node and mark *types* are always closed, but fields, attrs and content models are only closed when `strict` is enabled.
|
|
110
138
|
An attr the serializer never reads is dropped without a word, so a `colspan: 2` cell quietly becomes a plain one:
|
|
111
139
|
|
|
112
140
|
```python
|
|
@@ -121,7 +149,7 @@ It is keyword-only and a `from_dict` option only — the parser emits nothing bu
|
|
|
121
149
|
| `strict` | Keeps |
|
|
122
150
|
| --- | --- |
|
|
123
151
|
| `"off"` (default) | nothing is checked |
|
|
124
|
-
| `"content"` | content and structure. Presentation an editor stamped on is type-checked and dropped |
|
|
152
|
+
| `"content"` | content and structure, including closed node fields and content models. Presentation an editor stamped on is type-checked and dropped |
|
|
125
153
|
| `"exact"` | every attr the JSON carries |
|
|
126
154
|
|
|
127
155
|
Both enabled levels also enforce the ProseMirror same-type self-exclusion rule on each node's marks.
|
|
@@ -129,6 +157,27 @@ The standard marks represented by marktip are self-exclusive, so a node cannot c
|
|
|
129
157
|
The conflicting mark is rejected with `InvalidNodeError` rather than being serialized into Markdown that reparses with different formatting or structure.
|
|
130
158
|
`strict="off"` skips this check because strict validation is disabled.
|
|
131
159
|
|
|
160
|
+
Both enabled levels reject a node or mark top-level key that its type does not declare (`unknown_field`), and enforce the following content models (`invalid_content`):
|
|
161
|
+
|
|
162
|
+
| Parent | Allowed children |
|
|
163
|
+
| --- | --- |
|
|
164
|
+
| `doc`, `blockquote`, `listItem`, `taskItem`, `tableCell`, `tableHeader` | block nodes |
|
|
165
|
+
| `paragraph`, `heading` | inline nodes (`text`, `hardBreak`, `image`, `htmlInline`) |
|
|
166
|
+
| `codeBlock` | unmarked `text` only |
|
|
167
|
+
| `bulletList` | `listItem` |
|
|
168
|
+
| `orderedList` | `listItem`, `taskItem` |
|
|
169
|
+
| `taskList` | `taskItem` |
|
|
170
|
+
| `table` | `tableRow` |
|
|
171
|
+
| `tableRow` | `tableHeader`, `tableCell` |
|
|
172
|
+
| `text`, `hardBreak`, `horizontalRule`, `image`, `htmlBlock`, `htmlInline` | none |
|
|
173
|
+
|
|
174
|
+
Block nodes are `paragraph`, `heading`, `blockquote`, `codeBlock`, `horizontalRule`, `bulletList`, `orderedList`, `taskList`, `table`, and `htmlBlock`.
|
|
175
|
+
A strict text node must carry a non-empty string `text` field.
|
|
176
|
+
This is marktip's canonical child grammar, not a verbatim copy of one editor's schema: in particular, ordered Markdown task items remain `orderedList` → `taskItem`, and empty parser-produced containers remain valid.
|
|
177
|
+
These rules prevent a nested block submitted inside a paragraph, content attached to a leaf, or an arbitrary child key from being silently flattened or omitted by the serializer.
|
|
178
|
+
|
|
179
|
+
> **Changed in 0.7.2** — `strict="content"` and `strict="exact"` now close node/mark top-level fields and per-node content models with `unknown_field` and `invalid_content`. `strict="off"` keeps the previous permissive converter behavior.
|
|
180
|
+
|
|
132
181
|
> **Changed in 0.7.1** — `strict="content"` and `strict="exact"` now reject duplicate same-type marks with `invalid_mark_set`. Markdown parsing collapses nested spans that map to the same self-exclusive mark, so parser output remains valid on the strict read path.
|
|
133
182
|
|
|
134
183
|
The two levels differ over one thing, and it is worth being precise about which:
|
|
@@ -234,6 +283,9 @@ For `from_markdown` it locates the node in the parsed document, since the input
|
|
|
234
283
|
| `disallowed_scheme` | `InvalidNodeError` | a link `href` / image `src` scheme is outside the allowlist |
|
|
235
284
|
| `disallowed_relative_url` | `InvalidNodeError` | a scheme-less `href`/`src` violates the relative policy |
|
|
236
285
|
| `invalid_uri_char` | `InvalidNodeError` | an `href`/`src` contains an ASCII control character |
|
|
286
|
+
| `disallowed_node_type` | `InvalidNodeError` | a node type appears in `reject_node_types` |
|
|
287
|
+
| `unknown_field` | `InvalidNodeError` | `strict`: a node/mark top-level key is not defined for that type |
|
|
288
|
+
| `invalid_content` | `InvalidNodeError` | `strict`: a node violates its content model or text payload contract |
|
|
237
289
|
| `unknown_attr` | `InvalidNodeError` | `strict`: the attr name is in neither dialect for that type |
|
|
238
290
|
| `invalid_attr_value` | `InvalidNodeError` | `strict`: the attr's value has the wrong type or is out of range |
|
|
239
291
|
| `unrepresentable` | `InvalidNodeError` | `strict`: the value is well-formed, but markdown cannot carry it |
|
|
@@ -243,7 +295,7 @@ For `from_markdown` it locates the node in the parsed document, since the input
|
|
|
243
295
|
| `parse_failed` | `ParseError` | MD4C could not parse the input |
|
|
244
296
|
|
|
245
297
|
`from_markdown` still raises a plain `TypeError` when the argument is not `str`/`bytes` — that is a caller bug, not a malformed document.
|
|
246
|
-
So is a bad option value: a non-iterable `link_schemes` raises `TypeError`, and `link_schemes=("https://",)`, `link_relative="nope"` or `strict="strict"` raises `ValueError`.
|
|
298
|
+
So is a bad option value: a non-iterable `link_schemes` or `reject_node_types` raises `TypeError`, and `link_schemes=("https://",)`, `reject_node_types=("callout",)`, `link_relative="nope"` or `strict="strict"` raises `ValueError`.
|
|
247
299
|
`strict` takes a level name, so `strict=True` raises `TypeError` rather than quietly meaning something.
|
|
248
300
|
|
|
249
301
|
> **Changed in 0.4.0** — these were previously `ValueError` (`from_dict`) and `RuntimeError` (`from_markdown`).
|
|
@@ -275,9 +327,10 @@ It guarantees, for the whole tree:
|
|
|
275
327
|
- **Depth** is at most 2048 levels.
|
|
276
328
|
- **No ASCII control character** in a link `href` or image `src`, entity references included (`	`, `	`).
|
|
277
329
|
- **The URI policy**, when `link_schemes`/`image_schemes`/`link_relative`/`image_relative` ask for it — see [Restricting link and image URIs](#restricting-link-and-image-uris).
|
|
330
|
+
- **The node-type policy**, when `reject_node_types` names a type — see [Rejecting node types](#rejecting-node-types).
|
|
278
331
|
|
|
279
332
|
A document that survives `from_dict` always serializes; callers do not need to re-check any of the above.
|
|
280
|
-
`from_markdown` enforces the same
|
|
333
|
+
`from_markdown` enforces the same URI and rejected-node rules on what it parses.
|
|
281
334
|
|
|
282
335
|
It deliberately does **not** validate, unless `strict` asks it to:
|
|
283
336
|
|
|
@@ -292,11 +345,15 @@ It deliberately does **not** validate, unless `strict` asks it to:
|
|
|
292
345
|
- **Mark sets.**
|
|
293
346
|
With strict validation disabled, two marks of the same type may appear on one node and are serialized independently.
|
|
294
347
|
`strict="content"` and `strict="exact"` reject the second mark because marktip's standard ProseMirror/Tiptap marks are self-exclusive.
|
|
348
|
+
- **Node and mark top-level fields.**
|
|
349
|
+
With strict validation disabled, a key the converter does not read is ignored.
|
|
350
|
+
Both enabled levels reject it with `unknown_field`, including `marks` on a block or `text` on a non-text node.
|
|
295
351
|
- **Content models.**
|
|
296
|
-
|
|
352
|
+
With strict validation disabled, which node may contain which is not checked — a `text` node directly under `doc` is accepted and serialized.
|
|
353
|
+
Both enabled levels enforce the content-model table above, the text payload shape, and the absence of marks inside a code block.
|
|
297
354
|
- **Raw HTML.**
|
|
298
355
|
The URI policy governs `link` marks and `image` nodes, not markup inside `htmlBlock`/`htmlInline`, so `<a href="javascript:...">` passes it.
|
|
299
|
-
|
|
356
|
+
Reject those node types with `reject_node_types=("htmlBlock", "htmlInline")`; on Markdown input, `html=False` is the alternative that retains the markup as literal text.
|
|
300
357
|
- **URI syntax beyond the scheme.**
|
|
301
358
|
Everything after the scheme is opaque, and the stored value is never rewritten — an `href` is checked in its entity-decoded form but stored exactly as given.
|
|
302
359
|
Under `link_relative="allow"` a protocol-relative `//evil.com/x` also passes, since it carries no scheme; `"path_only"` is the setting that rejects it.
|
|
@@ -318,6 +375,7 @@ except tm.InvalidNodeError as e:
|
|
|
318
375
|
Markdown cannot represent every Tiptap document exactly.
|
|
319
376
|
Rather than emitting or retaining a representation that changes on the next conversion, marktip applies these deterministic (and idempotent) normalizations during parsing and serialization:
|
|
320
377
|
|
|
378
|
+
- **NUL in content strings** — U+0000 in text or a non-URI string attr (`alt`, `title`, `language`, `html`, etc.) becomes U+FFFD. Markdown input is replaced once before tokenization, including inline/fenced/indented code and raw HTML, so reparsing cannot retain a NUL or accumulate replacement characters. On dict input, `link.href` and `image.src` are excluded and retain their unconditional `invalid_uri_char` rejection.
|
|
321
379
|
- **Hard breaks in headings** — ATX headings are single-line, so a `hardBreak` (or a literal newline carried over from setext input) inside a heading serializes as a single space: `heading("a", hardBreak, "b")` → `# a b`.
|
|
322
380
|
- **Multi-block table cells** — blocks inside a cell are joined with `<br>` and the cell is flattened to one line; two paragraphs in a cell reparse as one paragraph containing a `hardBreak`.
|
|
323
381
|
- **Headerless tables** — GFM tables require a header row, so the first row is always emitted as the header; a leading `tableCell` row reparses as `tableHeader`.
|
|
@@ -24,14 +24,20 @@ doc = tm.from_dict(ast)
|
|
|
24
24
|
```
|
|
25
25
|
|
|
26
26
|
`from_markdown` follows GFM/CommonMark by default.
|
|
27
|
-
Pass `cjk_friendly=True` to
|
|
27
|
+
Pass `cjk_friendly=True` to follow the [CommonMark CJK-friendly Amendments Specification](https://github.com/tats-u/markdown-cjk-friendly/blob/main/specification.md) for emphasis (`*`, `_`) and strikethrough (`~`). This opt-in extension recognizes CJK punctuation as well as letters, so `**《제목》**a`, `a**《제목》**`, and `**「約束」**a` retain their bold marks:
|
|
28
28
|
|
|
29
29
|
```python
|
|
30
30
|
doc = tm.from_markdown("**볼드**은 강조", cjk_friendly=True)
|
|
31
31
|
```
|
|
32
32
|
|
|
33
33
|
Strikethrough follows the GFM reference implementations (cmark-gfm/micromark): intra-word `~~` strikes (`a~~b~~c`), including next to CJK letters (`~~삭제~~은`).
|
|
34
|
-
`cjk_friendly` still matters for
|
|
34
|
+
`cjk_friendly` still matters for punctuation-adjacent cases such as `~~"인용"~~라고`.
|
|
35
|
+
|
|
36
|
+
The CJK classifier uses Unicode 17.0.0 East Asian Width `W`, `F`, or `H` (excluding default emoji presentation characters), plus the Hangul script. This includes fullwidth digits such as `1`. Before a delimiter, CJK characters followed by a non-emoji general-use variation selector (`U+FE00`–`U+FE0E`) are classified as one sequence; ideographic variation selectors (`U+E0100`–`U+E01EF`) and standardized fullwidth quotation-mark sequences are also recognized. `U+FE0F` is an emoji presentation selector and is excluded from that sequence rule.
|
|
37
|
+
|
|
38
|
+
Underscores retain CommonMark's additional restriction on emphasis inside words. The CJK-friendly specification changes delimiter flanking without removing that protection for names such as `snake_case`: `_《제목》_을` and `__《제목》__을` stay literal, even with `cjk_friendly=True`. Use `*` or `**` when emphasis must adjoin a word.
|
|
39
|
+
|
|
40
|
+
> **Changed in 0.7.3** — CJK-friendly character classification, variation-selector sequences, flanking, and underscore conditions follow the specification. The same flanking rules apply to `~~`. An editor document containing bold `《제목》` followed by `a` now survives serialization and reparsing with `cjk_friendly=True`.
|
|
35
41
|
|
|
36
42
|
Pass `html=False` to parse raw HTML as literal text instead of `htmlBlock`/`htmlInline` nodes.
|
|
37
43
|
Content is preserved (the serializer escapes it), and `<br>` inside table cells still maps to `hardBreak` so marktip's own table output round-trips:
|
|
@@ -76,9 +82,30 @@ doc = tm.from_markdown(untrusted, link_schemes=("https",), image_relative="rejec
|
|
|
76
82
|
Violations raise `InvalidNodeError` with `.field` set to `"href"` or `"src"`.
|
|
77
83
|
The options are keyword-only.
|
|
78
84
|
|
|
85
|
+
### Rejecting node types
|
|
86
|
+
|
|
87
|
+
marktip's schema includes every node it can serialize, but an application may use a smaller ProseMirror dialect.
|
|
88
|
+
Both write boundaries accept `reject_node_types`, so that dialect can be enforced after either Markdown parsing or dict conversion without a second application-side tree walk:
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
NO_RAW_HTML = ("htmlBlock", "htmlInline")
|
|
92
|
+
|
|
93
|
+
doc = tm.from_dict(editor_json, reject_node_types=NO_RAW_HTML)
|
|
94
|
+
doc = tm.from_markdown(imported_markdown, reject_node_types=NO_RAW_HTML)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
`None` (the default) rejects nothing; an empty iterable has the same effect.
|
|
98
|
+
Every entry must name a node type marktip knows, and matching is exact and case-sensitive.
|
|
99
|
+
A bare string or a non-string entry raises `TypeError`; an unknown node name raises `ValueError`.
|
|
100
|
+
A document containing a configured type raises `InvalidNodeError` with `.code == "disallowed_node_type"`, `.field == "type"`, and `.path` locating the node.
|
|
101
|
+
The option is keyword-only.
|
|
102
|
+
|
|
103
|
+
This differs from `html=False`: `html=False` preserves raw markup as escaped literal text on Markdown input, while rejecting `htmlBlock`/`htmlInline` refuses markup that parsed as a raw-HTML node.
|
|
104
|
+
`reject_node_types` also works for dict input, where there is no parser flag.
|
|
105
|
+
|
|
79
106
|
### Refusing lossy conversions
|
|
80
107
|
|
|
81
|
-
Node and mark *types* are closed, but
|
|
108
|
+
Node and mark *types* are always closed, but fields, attrs and content models are only closed when `strict` is enabled.
|
|
82
109
|
An attr the serializer never reads is dropped without a word, so a `colspan: 2` cell quietly becomes a plain one:
|
|
83
110
|
|
|
84
111
|
```python
|
|
@@ -93,7 +120,7 @@ It is keyword-only and a `from_dict` option only — the parser emits nothing bu
|
|
|
93
120
|
| `strict` | Keeps |
|
|
94
121
|
| --- | --- |
|
|
95
122
|
| `"off"` (default) | nothing is checked |
|
|
96
|
-
| `"content"` | content and structure. Presentation an editor stamped on is type-checked and dropped |
|
|
123
|
+
| `"content"` | content and structure, including closed node fields and content models. Presentation an editor stamped on is type-checked and dropped |
|
|
97
124
|
| `"exact"` | every attr the JSON carries |
|
|
98
125
|
|
|
99
126
|
Both enabled levels also enforce the ProseMirror same-type self-exclusion rule on each node's marks.
|
|
@@ -101,6 +128,27 @@ The standard marks represented by marktip are self-exclusive, so a node cannot c
|
|
|
101
128
|
The conflicting mark is rejected with `InvalidNodeError` rather than being serialized into Markdown that reparses with different formatting or structure.
|
|
102
129
|
`strict="off"` skips this check because strict validation is disabled.
|
|
103
130
|
|
|
131
|
+
Both enabled levels reject a node or mark top-level key that its type does not declare (`unknown_field`), and enforce the following content models (`invalid_content`):
|
|
132
|
+
|
|
133
|
+
| Parent | Allowed children |
|
|
134
|
+
| --- | --- |
|
|
135
|
+
| `doc`, `blockquote`, `listItem`, `taskItem`, `tableCell`, `tableHeader` | block nodes |
|
|
136
|
+
| `paragraph`, `heading` | inline nodes (`text`, `hardBreak`, `image`, `htmlInline`) |
|
|
137
|
+
| `codeBlock` | unmarked `text` only |
|
|
138
|
+
| `bulletList` | `listItem` |
|
|
139
|
+
| `orderedList` | `listItem`, `taskItem` |
|
|
140
|
+
| `taskList` | `taskItem` |
|
|
141
|
+
| `table` | `tableRow` |
|
|
142
|
+
| `tableRow` | `tableHeader`, `tableCell` |
|
|
143
|
+
| `text`, `hardBreak`, `horizontalRule`, `image`, `htmlBlock`, `htmlInline` | none |
|
|
144
|
+
|
|
145
|
+
Block nodes are `paragraph`, `heading`, `blockquote`, `codeBlock`, `horizontalRule`, `bulletList`, `orderedList`, `taskList`, `table`, and `htmlBlock`.
|
|
146
|
+
A strict text node must carry a non-empty string `text` field.
|
|
147
|
+
This is marktip's canonical child grammar, not a verbatim copy of one editor's schema: in particular, ordered Markdown task items remain `orderedList` → `taskItem`, and empty parser-produced containers remain valid.
|
|
148
|
+
These rules prevent a nested block submitted inside a paragraph, content attached to a leaf, or an arbitrary child key from being silently flattened or omitted by the serializer.
|
|
149
|
+
|
|
150
|
+
> **Changed in 0.7.2** — `strict="content"` and `strict="exact"` now close node/mark top-level fields and per-node content models with `unknown_field` and `invalid_content`. `strict="off"` keeps the previous permissive converter behavior.
|
|
151
|
+
|
|
104
152
|
> **Changed in 0.7.1** — `strict="content"` and `strict="exact"` now reject duplicate same-type marks with `invalid_mark_set`. Markdown parsing collapses nested spans that map to the same self-exclusive mark, so parser output remains valid on the strict read path.
|
|
105
153
|
|
|
106
154
|
The two levels differ over one thing, and it is worth being precise about which:
|
|
@@ -206,6 +254,9 @@ For `from_markdown` it locates the node in the parsed document, since the input
|
|
|
206
254
|
| `disallowed_scheme` | `InvalidNodeError` | a link `href` / image `src` scheme is outside the allowlist |
|
|
207
255
|
| `disallowed_relative_url` | `InvalidNodeError` | a scheme-less `href`/`src` violates the relative policy |
|
|
208
256
|
| `invalid_uri_char` | `InvalidNodeError` | an `href`/`src` contains an ASCII control character |
|
|
257
|
+
| `disallowed_node_type` | `InvalidNodeError` | a node type appears in `reject_node_types` |
|
|
258
|
+
| `unknown_field` | `InvalidNodeError` | `strict`: a node/mark top-level key is not defined for that type |
|
|
259
|
+
| `invalid_content` | `InvalidNodeError` | `strict`: a node violates its content model or text payload contract |
|
|
209
260
|
| `unknown_attr` | `InvalidNodeError` | `strict`: the attr name is in neither dialect for that type |
|
|
210
261
|
| `invalid_attr_value` | `InvalidNodeError` | `strict`: the attr's value has the wrong type or is out of range |
|
|
211
262
|
| `unrepresentable` | `InvalidNodeError` | `strict`: the value is well-formed, but markdown cannot carry it |
|
|
@@ -215,7 +266,7 @@ For `from_markdown` it locates the node in the parsed document, since the input
|
|
|
215
266
|
| `parse_failed` | `ParseError` | MD4C could not parse the input |
|
|
216
267
|
|
|
217
268
|
`from_markdown` still raises a plain `TypeError` when the argument is not `str`/`bytes` — that is a caller bug, not a malformed document.
|
|
218
|
-
So is a bad option value: a non-iterable `link_schemes` raises `TypeError`, and `link_schemes=("https://",)`, `link_relative="nope"` or `strict="strict"` raises `ValueError`.
|
|
269
|
+
So is a bad option value: a non-iterable `link_schemes` or `reject_node_types` raises `TypeError`, and `link_schemes=("https://",)`, `reject_node_types=("callout",)`, `link_relative="nope"` or `strict="strict"` raises `ValueError`.
|
|
219
270
|
`strict` takes a level name, so `strict=True` raises `TypeError` rather than quietly meaning something.
|
|
220
271
|
|
|
221
272
|
> **Changed in 0.4.0** — these were previously `ValueError` (`from_dict`) and `RuntimeError` (`from_markdown`).
|
|
@@ -247,9 +298,10 @@ It guarantees, for the whole tree:
|
|
|
247
298
|
- **Depth** is at most 2048 levels.
|
|
248
299
|
- **No ASCII control character** in a link `href` or image `src`, entity references included (`	`, `	`).
|
|
249
300
|
- **The URI policy**, when `link_schemes`/`image_schemes`/`link_relative`/`image_relative` ask for it — see [Restricting link and image URIs](#restricting-link-and-image-uris).
|
|
301
|
+
- **The node-type policy**, when `reject_node_types` names a type — see [Rejecting node types](#rejecting-node-types).
|
|
250
302
|
|
|
251
303
|
A document that survives `from_dict` always serializes; callers do not need to re-check any of the above.
|
|
252
|
-
`from_markdown` enforces the same
|
|
304
|
+
`from_markdown` enforces the same URI and rejected-node rules on what it parses.
|
|
253
305
|
|
|
254
306
|
It deliberately does **not** validate, unless `strict` asks it to:
|
|
255
307
|
|
|
@@ -264,11 +316,15 @@ It deliberately does **not** validate, unless `strict` asks it to:
|
|
|
264
316
|
- **Mark sets.**
|
|
265
317
|
With strict validation disabled, two marks of the same type may appear on one node and are serialized independently.
|
|
266
318
|
`strict="content"` and `strict="exact"` reject the second mark because marktip's standard ProseMirror/Tiptap marks are self-exclusive.
|
|
319
|
+
- **Node and mark top-level fields.**
|
|
320
|
+
With strict validation disabled, a key the converter does not read is ignored.
|
|
321
|
+
Both enabled levels reject it with `unknown_field`, including `marks` on a block or `text` on a non-text node.
|
|
267
322
|
- **Content models.**
|
|
268
|
-
|
|
323
|
+
With strict validation disabled, which node may contain which is not checked — a `text` node directly under `doc` is accepted and serialized.
|
|
324
|
+
Both enabled levels enforce the content-model table above, the text payload shape, and the absence of marks inside a code block.
|
|
269
325
|
- **Raw HTML.**
|
|
270
326
|
The URI policy governs `link` marks and `image` nodes, not markup inside `htmlBlock`/`htmlInline`, so `<a href="javascript:...">` passes it.
|
|
271
|
-
|
|
327
|
+
Reject those node types with `reject_node_types=("htmlBlock", "htmlInline")`; on Markdown input, `html=False` is the alternative that retains the markup as literal text.
|
|
272
328
|
- **URI syntax beyond the scheme.**
|
|
273
329
|
Everything after the scheme is opaque, and the stored value is never rewritten — an `href` is checked in its entity-decoded form but stored exactly as given.
|
|
274
330
|
Under `link_relative="allow"` a protocol-relative `//evil.com/x` also passes, since it carries no scheme; `"path_only"` is the setting that rejects it.
|
|
@@ -290,6 +346,7 @@ except tm.InvalidNodeError as e:
|
|
|
290
346
|
Markdown cannot represent every Tiptap document exactly.
|
|
291
347
|
Rather than emitting or retaining a representation that changes on the next conversion, marktip applies these deterministic (and idempotent) normalizations during parsing and serialization:
|
|
292
348
|
|
|
349
|
+
- **NUL in content strings** — U+0000 in text or a non-URI string attr (`alt`, `title`, `language`, `html`, etc.) becomes U+FFFD. Markdown input is replaced once before tokenization, including inline/fenced/indented code and raw HTML, so reparsing cannot retain a NUL or accumulate replacement characters. On dict input, `link.href` and `image.src` are excluded and retain their unconditional `invalid_uri_char` rejection.
|
|
293
350
|
- **Hard breaks in headings** — ATX headings are single-line, so a `hardBreak` (or a literal newline carried over from setext input) inside a heading serializes as a single space: `heading("a", hardBreak, "b")` → `# a b`.
|
|
294
351
|
- **Multi-block table cells** — blocks inside a cell are joined with `<br>` and the cell is flattened to one line; two paragraphs in a cell reparse as one paragraph containing a `hardBreak`.
|
|
295
352
|
- **Headerless tables** — GFM tables require a header row, so the first row is always emitted as the header; a leading `tableCell` row reparses as `tableHeader`.
|
|
@@ -4,12 +4,12 @@ build-backend = "scikit_build_core.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "marktip"
|
|
7
|
-
version = "0.7.
|
|
7
|
+
version = "0.7.3"
|
|
8
8
|
description = "Fast C++/MD4C Markdown parser and canonical Markdown serializer for Tiptap-style JSON"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
11
|
-
license = "MIT"
|
|
12
|
-
license-files = ["LICENSE", "third_party/md4c/LICENSE.md"]
|
|
11
|
+
license = "MIT AND Unicode-3.0"
|
|
12
|
+
license-files = ["LICENSE", "third_party/md4c/LICENSE.md", "third_party/md4c/upstream_metadata/Unicode-LICENSE.txt"]
|
|
13
13
|
authors = [{ name = "Saneaven (Minjae Kyung)" }]
|
|
14
14
|
keywords = ["markdown", "tiptap", "md4c", "parser"]
|
|
15
15
|
classifiers = [
|
|
@@ -38,9 +38,11 @@ wheel.packages = ["python/marktip"]
|
|
|
38
38
|
sdist.include = [
|
|
39
39
|
"third_party/md4c/src/md4c.c",
|
|
40
40
|
"third_party/md4c/src/md4c.h",
|
|
41
|
+
"third_party/md4c/src/cjk.h",
|
|
41
42
|
"third_party/md4c/src/entity.c",
|
|
42
43
|
"third_party/md4c/src/entity.h",
|
|
43
44
|
"third_party/md4c/LICENSE.md",
|
|
45
|
+
"third_party/md4c/upstream_metadata/Unicode-LICENSE.txt",
|
|
44
46
|
]
|
|
45
47
|
|
|
46
48
|
[tool.pytest.ini_options]
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""Generate the CJK-friendly classifier from Unicode 17.0.0 data files."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import hashlib
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
VERSION = "17.0.0"
|
|
10
|
+
SOURCES = {
|
|
11
|
+
"EastAsianWidth.txt": "EastAsianWidth.txt",
|
|
12
|
+
"Scripts.txt": "Scripts.txt",
|
|
13
|
+
"emoji-data.txt": "emoji/emoji-data.txt",
|
|
14
|
+
"StandardizedVariants.txt": "StandardizedVariants.txt",
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def records(path):
|
|
19
|
+
for line in path.read_text(encoding="utf-8").splitlines():
|
|
20
|
+
body = line.split("#", 1)[0].strip()
|
|
21
|
+
if body:
|
|
22
|
+
yield [field.strip() for field in body.split(";")]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def codepoints(value):
|
|
26
|
+
parts = value.split("..")
|
|
27
|
+
return range(int(parts[0], 16), int(parts[-1], 16) + 1)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def intervals(values):
|
|
31
|
+
start = previous = None
|
|
32
|
+
for value in sorted(values):
|
|
33
|
+
if previous is not None and value != previous + 1:
|
|
34
|
+
yield start, previous
|
|
35
|
+
start = None
|
|
36
|
+
if start is None:
|
|
37
|
+
start = value
|
|
38
|
+
previous = value
|
|
39
|
+
if previous is not None:
|
|
40
|
+
yield start, previous
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def generate(directory):
|
|
44
|
+
for filename in SOURCES:
|
|
45
|
+
header = (directory / filename).read_text(encoding="utf-8")[:1000]
|
|
46
|
+
if not re.search(r"17\.0(?:\.0)?\b", header):
|
|
47
|
+
raise ValueError(f"{filename} must contain Unicode {VERSION} data")
|
|
48
|
+
widths = {}
|
|
49
|
+
for bounds, width in records(directory / "EastAsianWidth.txt"):
|
|
50
|
+
widths.update(dict.fromkeys(codepoints(bounds), width))
|
|
51
|
+
hangul = set()
|
|
52
|
+
for bounds, script in records(directory / "Scripts.txt"):
|
|
53
|
+
if script == "Hangul":
|
|
54
|
+
hangul.update(codepoints(bounds))
|
|
55
|
+
emoji = set()
|
|
56
|
+
for bounds, property_name in records(directory / "emoji-data.txt"):
|
|
57
|
+
if property_name == "Emoji_Presentation":
|
|
58
|
+
emoji.update(codepoints(bounds))
|
|
59
|
+
cjk = {cp for cp, width in widths.items() if width in {"W", "F", "H"}} - emoji
|
|
60
|
+
cjk.update(hangul)
|
|
61
|
+
ambiguous = []
|
|
62
|
+
for sequence, description, *_ in records(directory / "StandardizedVariants.txt"):
|
|
63
|
+
base, selector = (int(cp, 16) for cp in sequence.split())
|
|
64
|
+
if (
|
|
65
|
+
widths.get(base) == "A"
|
|
66
|
+
and re.search(r"\bfullwidth form\b", description)
|
|
67
|
+
and "non-fullwidth" not in description
|
|
68
|
+
):
|
|
69
|
+
ambiguous.append((base, selector))
|
|
70
|
+
|
|
71
|
+
lines = [
|
|
72
|
+
"/* Generated by scripts/build_cjk_map.py; Unicode " + VERSION + ".",
|
|
73
|
+
" * Data license: upstream_metadata/Unicode-LICENSE.txt.",
|
|
74
|
+
]
|
|
75
|
+
for filename, source in SOURCES.items():
|
|
76
|
+
digest = hashlib.sha256((directory / filename).read_bytes()).hexdigest()
|
|
77
|
+
lines.append(" * https://www.unicode.org/Public/" + VERSION + "/ucd/" + source)
|
|
78
|
+
lines.append(" * SHA-256: " + digest)
|
|
79
|
+
lines.extend([
|
|
80
|
+
" */",
|
|
81
|
+
"#define R(cp_min, cp_max) ((cp_min) | 0x40000000), ((cp_max) | 0x80000000)",
|
|
82
|
+
"#define S(cp) (cp)",
|
|
83
|
+
"static const unsigned MD_CJK_MAP[] = {",
|
|
84
|
+
])
|
|
85
|
+
for start, end in intervals(cjk):
|
|
86
|
+
value = f"S(0x{start:04x})" if start == end else f"R(0x{start:04x},0x{end:04x})"
|
|
87
|
+
lines.append(" " + value + ",")
|
|
88
|
+
lines.extend(["};", "#undef R", "#undef S", "static const unsigned MD_CJK_AMBIGUOUS_PUNCT[][2] = {"])
|
|
89
|
+
for base, selector in sorted(ambiguous):
|
|
90
|
+
lines.append(f" {{ 0x{base:04x}, 0x{selector:04x} }},")
|
|
91
|
+
lines.extend(["};", ""])
|
|
92
|
+
return "\n".join(lines)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
if __name__ == "__main__":
|
|
96
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
97
|
+
parser.add_argument("ucd_directory", type=Path)
|
|
98
|
+
parser.add_argument("--output", type=Path, default=Path("third_party/md4c/src/cjk.h"))
|
|
99
|
+
args = parser.parse_args()
|
|
100
|
+
args.output.write_text(generate(args.ucd_directory), encoding="utf-8")
|