marktip 0.7.1__tar.gz → 0.7.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {marktip-0.7.1 → marktip-0.7.3}/CMakeLists.txt +1 -1
  2. {marktip-0.7.1 → marktip-0.7.3}/PKG-INFO +68 -10
  3. {marktip-0.7.1 → marktip-0.7.3}/README.md +65 -8
  4. {marktip-0.7.1 → marktip-0.7.3}/pyproject.toml +5 -3
  5. marktip-0.7.3/scripts/build_cjk_map.py +100 -0
  6. {marktip-0.7.1 → marktip-0.7.3}/src/ast.cpp +274 -30
  7. {marktip-0.7.1 → marktip-0.7.3}/src/ast.h +18 -3
  8. {marktip-0.7.1 → marktip-0.7.3}/src/errors.h +3 -2
  9. {marktip-0.7.1 → marktip-0.7.3}/src/marktip.cpp +20 -11
  10. {marktip-0.7.1 → marktip-0.7.3}/src/parser.cpp +28 -1
  11. {marktip-0.7.1 → marktip-0.7.3}/src/parser.h +1 -1
  12. marktip-0.7.3/tests/test_cjk_friendly.py +94 -0
  13. marktip-0.7.3/tests/test_content_model.py +159 -0
  14. {marktip-0.7.1 → marktip-0.7.3}/tests/test_errors.py +91 -0
  15. marktip-0.7.3/tests/test_node_type_policy.py +100 -0
  16. marktip-0.7.3/tests/test_nul_normalization.py +176 -0
  17. {marktip-0.7.1 → marktip-0.7.3}/tests/test_strict.py +7 -4
  18. marktip-0.7.3/third_party/md4c/src/cjk.h +84 -0
  19. {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/src/md4c.c +116 -115
  20. {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/upstream_metadata/README.md +24 -6
  21. marktip-0.7.3/third_party/md4c/upstream_metadata/Unicode-LICENSE.txt +39 -0
  22. {marktip-0.7.1 → marktip-0.7.3}/.github/workflows/build-distributions.yml +0 -0
  23. {marktip-0.7.1 → marktip-0.7.3}/.gitignore +0 -0
  24. {marktip-0.7.1 → marktip-0.7.3}/LICENSE +0 -0
  25. {marktip-0.7.1 → marktip-0.7.3}/python/marktip/__init__.py +0 -0
  26. {marktip-0.7.1 → marktip-0.7.3}/scripts/benchmark.py +0 -0
  27. {marktip-0.7.1 → marktip-0.7.3}/src/serializer.cpp +0 -0
  28. {marktip-0.7.1 → marktip-0.7.3}/src/serializer.h +0 -0
  29. {marktip-0.7.1 → marktip-0.7.3}/tests/test_document.py +0 -0
  30. {marktip-0.7.1 → marktip-0.7.3}/tests/test_parse.py +0 -0
  31. {marktip-0.7.1 → marktip-0.7.3}/tests/test_roundtrip.py +0 -0
  32. {marktip-0.7.1 → marktip-0.7.3}/tests/test_serialize.py +0 -0
  33. {marktip-0.7.1 → marktip-0.7.3}/tests/test_uri_policy.py +0 -0
  34. {marktip-0.7.1 → marktip-0.7.3}/tests/test_validate.py +0 -0
  35. {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/LICENSE.md +0 -0
  36. {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/src/entity.c +0 -0
  37. {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/src/entity.h +0 -0
  38. {marktip-0.7.1 → marktip-0.7.3}/third_party/md4c/src/md4c.h +0 -0
@@ -7,7 +7,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
7
7
  set(CMAKE_CXX_EXTENSIONS OFF)
8
8
 
9
9
  if(NOT DEFINED SKBUILD_PROJECT_VERSION)
10
- set(SKBUILD_PROJECT_VERSION "0.7.1")
10
+ set(SKBUILD_PROJECT_VERSION "0.7.3")
11
11
  endif()
12
12
 
13
13
  set(PYBIND11_FINDPYTHON ON)
@@ -1,12 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: marktip
3
- Version: 0.7.1
3
+ Version: 0.7.3
4
4
  Summary: Fast C++/MD4C Markdown parser and canonical Markdown serializer for Tiptap-style JSON
5
5
  Keywords: markdown,tiptap,md4c,parser
6
6
  Author: Saneaven (Minjae Kyung)
7
- License-Expression: MIT
7
+ License-Expression: MIT AND Unicode-3.0
8
8
  License-File: LICENSE
9
9
  License-File: third_party/md4c/LICENSE.md
10
+ License-File: third_party/md4c/upstream_metadata/Unicode-LICENSE.txt
10
11
  Classifier: Development Status :: 3 - Alpha
11
12
  Classifier: Intended Audience :: Developers
12
13
  Classifier: Operating System :: MacOS
@@ -52,14 +53,20 @@ doc = tm.from_dict(ast)
52
53
  ```
53
54
 
54
55
  `from_markdown` follows GFM/CommonMark by default.
55
- Pass `cjk_friendly=True` to relax the emphasis and strikethrough rules so delimiters next to CJK text still open and close (e.g. `**볼드**은` parses as bold), a non-standard extension that is off by default:
56
+ Pass `cjk_friendly=True` to follow the [CommonMark CJK-friendly Amendments Specification](https://github.com/tats-u/markdown-cjk-friendly/blob/main/specification.md) for emphasis (`*`, `_`) and strikethrough (`~`). This opt-in extension recognizes CJK punctuation as well as letters, so `**《제목》**a`, `a**《제목》**`, and `**「約束」**a` retain their bold marks:
56
57
 
57
58
  ```python
58
59
  doc = tm.from_markdown("**볼드**은 강조", cjk_friendly=True)
59
60
  ```
60
61
 
61
62
  Strikethrough follows the GFM reference implementations (cmark-gfm/micromark): intra-word `~~` strikes (`a~~b~~c`), including next to CJK letters (`~~삭제~~은`).
62
- `cjk_friendly` still matters for `*`/`_`, and for `~` in punctuation-adjacent cases such as `~~"인용"~~라고`.
63
+ `cjk_friendly` still matters for punctuation-adjacent cases such as `~~"인용"~~라고`.
64
+
65
+ The CJK classifier uses Unicode 17.0.0 East Asian Width `W`, `F`, or `H` (excluding default emoji presentation characters), plus the Hangul script. This includes fullwidth digits such as `1`. Before a delimiter, CJK characters followed by a non-emoji general-use variation selector (`U+FE00`–`U+FE0E`) are classified as one sequence; ideographic variation selectors (`U+E0100`–`U+E01EF`) and standardized fullwidth quotation-mark sequences are also recognized. `U+FE0F` is an emoji presentation selector and is excluded from that sequence rule.
66
+
67
+ Underscores retain CommonMark's additional restriction on emphasis inside words. The CJK-friendly specification changes delimiter flanking without removing that protection for names such as `snake_case`: `_《제목》_을` and `__《제목》__을` stay literal, even with `cjk_friendly=True`. Use `*` or `**` when emphasis must adjoin a word.
68
+
69
+ > **Changed in 0.7.3** — CJK-friendly character classification, variation-selector sequences, flanking, and underscore conditions follow the specification. The same flanking rules apply to `~~`. An editor document containing bold `《제목》` followed by `a` now survives serialization and reparsing with `cjk_friendly=True`.
63
70
 
64
71
  Pass `html=False` to parse raw HTML as literal text instead of `htmlBlock`/`htmlInline` nodes.
65
72
  Content is preserved (the serializer escapes it), and `<br>` inside table cells still maps to `hardBreak` so marktip's own table output round-trips:
@@ -104,9 +111,30 @@ doc = tm.from_markdown(untrusted, link_schemes=("https",), image_relative="rejec
104
111
  Violations raise `InvalidNodeError` with `.field` set to `"href"` or `"src"`.
105
112
  The options are keyword-only.
106
113
 
114
+ ### Rejecting node types
115
+
116
+ marktip's schema includes every node it can serialize, but an application may use a smaller ProseMirror dialect.
117
+ Both write boundaries accept `reject_node_types`, so that dialect can be enforced after either Markdown parsing or dict conversion without a second application-side tree walk:
118
+
119
+ ```python
120
+ NO_RAW_HTML = ("htmlBlock", "htmlInline")
121
+
122
+ doc = tm.from_dict(editor_json, reject_node_types=NO_RAW_HTML)
123
+ doc = tm.from_markdown(imported_markdown, reject_node_types=NO_RAW_HTML)
124
+ ```
125
+
126
+ `None` (the default) rejects nothing; an empty iterable has the same effect.
127
+ Every entry must name a node type marktip knows, and matching is exact and case-sensitive.
128
+ A bare string or a non-string entry raises `TypeError`; an unknown node name raises `ValueError`.
129
+ A document containing a configured type raises `InvalidNodeError` with `.code == "disallowed_node_type"`, `.field == "type"`, and `.path` locating the node.
130
+ The option is keyword-only.
131
+
132
+ This differs from `html=False`: `html=False` preserves raw markup as escaped literal text on Markdown input, while rejecting `htmlBlock`/`htmlInline` refuses markup that parsed as a raw-HTML node.
133
+ `reject_node_types` also works for dict input, where there is no parser flag.
134
+
107
135
  ### Refusing lossy conversions
108
136
 
109
- Node and mark *types* are closed, but `attrs` are not.
137
+ Node and mark *types* are always closed, but fields, attrs and content models are only closed when `strict` is enabled.
110
138
  An attr the serializer never reads is dropped without a word, so a `colspan: 2` cell quietly becomes a plain one:
111
139
 
112
140
  ```python
@@ -121,7 +149,7 @@ It is keyword-only and a `from_dict` option only — the parser emits nothing bu
121
149
  | `strict` | Keeps |
122
150
  | --- | --- |
123
151
  | `"off"` (default) | nothing is checked |
124
- | `"content"` | content and structure. Presentation an editor stamped on is type-checked and dropped |
152
+ | `"content"` | content and structure, including closed node fields and content models. Presentation an editor stamped on is type-checked and dropped |
125
153
  | `"exact"` | every attr the JSON carries |
126
154
 
127
155
  Both enabled levels also enforce the ProseMirror same-type self-exclusion rule on each node's marks.
@@ -129,6 +157,27 @@ The standard marks represented by marktip are self-exclusive, so a node cannot c
129
157
  The conflicting mark is rejected with `InvalidNodeError` rather than being serialized into Markdown that reparses with different formatting or structure.
130
158
  `strict="off"` skips this check because strict validation is disabled.
131
159
 
160
+ Both enabled levels reject a node or mark top-level key that its type does not declare (`unknown_field`), and enforce the following content models (`invalid_content`):
161
+
162
+ | Parent | Allowed children |
163
+ | --- | --- |
164
+ | `doc`, `blockquote`, `listItem`, `taskItem`, `tableCell`, `tableHeader` | block nodes |
165
+ | `paragraph`, `heading` | inline nodes (`text`, `hardBreak`, `image`, `htmlInline`) |
166
+ | `codeBlock` | unmarked `text` only |
167
+ | `bulletList` | `listItem` |
168
+ | `orderedList` | `listItem`, `taskItem` |
169
+ | `taskList` | `taskItem` |
170
+ | `table` | `tableRow` |
171
+ | `tableRow` | `tableHeader`, `tableCell` |
172
+ | `text`, `hardBreak`, `horizontalRule`, `image`, `htmlBlock`, `htmlInline` | none |
173
+
174
+ Block nodes are `paragraph`, `heading`, `blockquote`, `codeBlock`, `horizontalRule`, `bulletList`, `orderedList`, `taskList`, `table`, and `htmlBlock`.
175
+ A strict text node must carry a non-empty string `text` field.
176
+ This is marktip's canonical child grammar, not a verbatim copy of one editor's schema: in particular, ordered Markdown task items remain `orderedList` → `taskItem`, and empty parser-produced containers remain valid.
177
+ These rules prevent a nested block submitted inside a paragraph, content attached to a leaf, or an arbitrary child key from being silently flattened or omitted by the serializer.
178
+
179
+ > **Changed in 0.7.2** — `strict="content"` and `strict="exact"` now close node/mark top-level fields and per-node content models with `unknown_field` and `invalid_content`. `strict="off"` keeps the previous permissive converter behavior.
180
+
132
181
  > **Changed in 0.7.1** — `strict="content"` and `strict="exact"` now reject duplicate same-type marks with `invalid_mark_set`. Markdown parsing collapses nested spans that map to the same self-exclusive mark, so parser output remains valid on the strict read path.
133
182
 
134
183
  The two levels differ over one thing, and it is worth being precise about which:
@@ -234,6 +283,9 @@ For `from_markdown` it locates the node in the parsed document, since the input
234
283
  | `disallowed_scheme` | `InvalidNodeError` | a link `href` / image `src` scheme is outside the allowlist |
235
284
  | `disallowed_relative_url` | `InvalidNodeError` | a scheme-less `href`/`src` violates the relative policy |
236
285
  | `invalid_uri_char` | `InvalidNodeError` | an `href`/`src` contains an ASCII control character |
286
+ | `disallowed_node_type` | `InvalidNodeError` | a node type appears in `reject_node_types` |
287
+ | `unknown_field` | `InvalidNodeError` | `strict`: a node/mark top-level key is not defined for that type |
288
+ | `invalid_content` | `InvalidNodeError` | `strict`: a node violates its content model or text payload contract |
237
289
  | `unknown_attr` | `InvalidNodeError` | `strict`: the attr name is in neither dialect for that type |
238
290
  | `invalid_attr_value` | `InvalidNodeError` | `strict`: the attr's value has the wrong type or is out of range |
239
291
  | `unrepresentable` | `InvalidNodeError` | `strict`: the value is well-formed, but markdown cannot carry it |
@@ -243,7 +295,7 @@ For `from_markdown` it locates the node in the parsed document, since the input
243
295
  | `parse_failed` | `ParseError` | MD4C could not parse the input |
244
296
 
245
297
  `from_markdown` still raises a plain `TypeError` when the argument is not `str`/`bytes` — that is a caller bug, not a malformed document.
246
- So is a bad option value: a non-iterable `link_schemes` raises `TypeError`, and `link_schemes=("https://",)`, `link_relative="nope"` or `strict="strict"` raises `ValueError`.
298
+ So is a bad option value: a non-iterable `link_schemes` or `reject_node_types` raises `TypeError`, and `link_schemes=("https://",)`, `reject_node_types=("callout",)`, `link_relative="nope"` or `strict="strict"` raises `ValueError`.
247
299
  `strict` takes a level name, so `strict=True` raises `TypeError` rather than quietly meaning something.
248
300
 
249
301
  > **Changed in 0.4.0** — these were previously `ValueError` (`from_dict`) and `RuntimeError` (`from_markdown`).
@@ -275,9 +327,10 @@ It guarantees, for the whole tree:
275
327
  - **Depth** is at most 2048 levels.
276
328
  - **No ASCII control character** in a link `href` or image `src`, entity references included (`&Tab;`, `&#09;`).
277
329
  - **The URI policy**, when `link_schemes`/`image_schemes`/`link_relative`/`image_relative` ask for it — see [Restricting link and image URIs](#restricting-link-and-image-uris).
330
+ - **The node-type policy**, when `reject_node_types` names a type — see [Rejecting node types](#rejecting-node-types).
278
331
 
279
332
  A document that survives `from_dict` always serializes; callers do not need to re-check any of the above.
280
- `from_markdown` enforces the same two URI rules on what it parses.
333
+ `from_markdown` enforces the same URI and rejected-node rules on what it parses.
281
334
 
282
335
  It deliberately does **not** validate, unless `strict` asks it to:
283
336
 
@@ -292,11 +345,15 @@ It deliberately does **not** validate, unless `strict` asks it to:
292
345
  - **Mark sets.**
293
346
  With strict validation disabled, two marks of the same type may appear on one node and are serialized independently.
294
347
  `strict="content"` and `strict="exact"` reject the second mark because marktip's standard ProseMirror/Tiptap marks are self-exclusive.
348
+ - **Node and mark top-level fields.**
349
+ With strict validation disabled, a key the converter does not read is ignored.
350
+ Both enabled levels reject it with `unknown_field`, including `marks` on a block or `text` on a non-text node.
295
351
  - **Content models.**
296
- Which node may contain which is not checked — a `text` node directly under `doc` is accepted and serialized.
352
+ With strict validation disabled, which node may contain which is not checked — a `text` node directly under `doc` is accepted and serialized.
353
+ Both enabled levels enforce the content-model table above, the text payload shape, and the absence of marks inside a code block.
297
354
  - **Raw HTML.**
298
355
  The URI policy governs `link` marks and `image` nodes, not markup inside `htmlBlock`/`htmlInline`, so `<a href="javascript:...">` passes it.
299
- Pair the policy with `html=False` to close that.
356
+ Reject those node types with `reject_node_types=("htmlBlock", "htmlInline")`; on Markdown input, `html=False` is the alternative that retains the markup as literal text.
300
357
  - **URI syntax beyond the scheme.**
301
358
  Everything after the scheme is opaque, and the stored value is never rewritten — an `href` is checked in its entity-decoded form but stored exactly as given.
302
359
  Under `link_relative="allow"` a protocol-relative `//evil.com/x` also passes, since it carries no scheme; `"path_only"` is the setting that rejects it.
@@ -318,6 +375,7 @@ except tm.InvalidNodeError as e:
318
375
  Markdown cannot represent every Tiptap document exactly.
319
376
  Rather than emitting or retaining a representation that changes on the next conversion, marktip applies these deterministic (and idempotent) normalizations during parsing and serialization:
320
377
 
378
+ - **NUL in content strings** — U+0000 in text or a non-URI string attr (`alt`, `title`, `language`, `html`, etc.) becomes U+FFFD. Markdown input is replaced once before tokenization, including inline/fenced/indented code and raw HTML, so reparsing cannot retain a NUL or accumulate replacement characters. On dict input, `link.href` and `image.src` are excluded and retain their unconditional `invalid_uri_char` rejection.
321
379
  - **Hard breaks in headings** — ATX headings are single-line, so a `hardBreak` (or a literal newline carried over from setext input) inside a heading serializes as a single space: `heading("a", hardBreak, "b")` → `# a b`.
322
380
  - **Multi-block table cells** — blocks inside a cell are joined with `<br>` and the cell is flattened to one line; two paragraphs in a cell reparse as one paragraph containing a `hardBreak`.
323
381
  - **Headerless tables** — GFM tables require a header row, so the first row is always emitted as the header; a leading `tableCell` row reparses as `tableHeader`.
@@ -24,14 +24,20 @@ doc = tm.from_dict(ast)
24
24
  ```
25
25
 
26
26
  `from_markdown` follows GFM/CommonMark by default.
27
- Pass `cjk_friendly=True` to relax the emphasis and strikethrough rules so delimiters next to CJK text still open and close (e.g. `**볼드**은` parses as bold), a non-standard extension that is off by default:
27
+ Pass `cjk_friendly=True` to follow the [CommonMark CJK-friendly Amendments Specification](https://github.com/tats-u/markdown-cjk-friendly/blob/main/specification.md) for emphasis (`*`, `_`) and strikethrough (`~`). This opt-in extension recognizes CJK punctuation as well as letters, so `**《제목》**a`, `a**《제목》**`, and `**「約束」**a` retain their bold marks:
28
28
 
29
29
  ```python
30
30
  doc = tm.from_markdown("**볼드**은 강조", cjk_friendly=True)
31
31
  ```
32
32
 
33
33
  Strikethrough follows the GFM reference implementations (cmark-gfm/micromark): intra-word `~~` strikes (`a~~b~~c`), including next to CJK letters (`~~삭제~~은`).
34
- `cjk_friendly` still matters for `*`/`_`, and for `~` in punctuation-adjacent cases such as `~~"인용"~~라고`.
34
+ `cjk_friendly` still matters for punctuation-adjacent cases such as `~~"인용"~~라고`.
35
+
36
+ The CJK classifier uses Unicode 17.0.0 East Asian Width `W`, `F`, or `H` (excluding default emoji presentation characters), plus the Hangul script. This includes fullwidth digits such as `1`. Before a delimiter, CJK characters followed by a non-emoji general-use variation selector (`U+FE00`–`U+FE0E`) are classified as one sequence; ideographic variation selectors (`U+E0100`–`U+E01EF`) and standardized fullwidth quotation-mark sequences are also recognized. `U+FE0F` is an emoji presentation selector and is excluded from that sequence rule.
37
+
38
+ Underscores retain CommonMark's additional restriction on emphasis inside words. The CJK-friendly specification changes delimiter flanking without removing that protection for names such as `snake_case`: `_《제목》_을` and `__《제목》__을` stay literal, even with `cjk_friendly=True`. Use `*` or `**` when emphasis must adjoin a word.
39
+
40
+ > **Changed in 0.7.3** — CJK-friendly character classification, variation-selector sequences, flanking, and underscore conditions follow the specification. The same flanking rules apply to `~~`. An editor document containing bold `《제목》` followed by `a` now survives serialization and reparsing with `cjk_friendly=True`.
35
41
 
36
42
  Pass `html=False` to parse raw HTML as literal text instead of `htmlBlock`/`htmlInline` nodes.
37
43
  Content is preserved (the serializer escapes it), and `<br>` inside table cells still maps to `hardBreak` so marktip's own table output round-trips:
@@ -76,9 +82,30 @@ doc = tm.from_markdown(untrusted, link_schemes=("https",), image_relative="rejec
76
82
  Violations raise `InvalidNodeError` with `.field` set to `"href"` or `"src"`.
77
83
  The options are keyword-only.
78
84
 
85
+ ### Rejecting node types
86
+
87
+ marktip's schema includes every node it can serialize, but an application may use a smaller ProseMirror dialect.
88
+ Both write boundaries accept `reject_node_types`, so that dialect can be enforced after either Markdown parsing or dict conversion without a second application-side tree walk:
89
+
90
+ ```python
91
+ NO_RAW_HTML = ("htmlBlock", "htmlInline")
92
+
93
+ doc = tm.from_dict(editor_json, reject_node_types=NO_RAW_HTML)
94
+ doc = tm.from_markdown(imported_markdown, reject_node_types=NO_RAW_HTML)
95
+ ```
96
+
97
+ `None` (the default) rejects nothing; an empty iterable has the same effect.
98
+ Every entry must name a node type marktip knows, and matching is exact and case-sensitive.
99
+ A bare string or a non-string entry raises `TypeError`; an unknown node name raises `ValueError`.
100
+ A document containing a configured type raises `InvalidNodeError` with `.code == "disallowed_node_type"`, `.field == "type"`, and `.path` locating the node.
101
+ The option is keyword-only.
102
+
103
+ This differs from `html=False`: `html=False` preserves raw markup as escaped literal text on Markdown input, while rejecting `htmlBlock`/`htmlInline` refuses markup that parsed as a raw-HTML node.
104
+ `reject_node_types` also works for dict input, where there is no parser flag.
105
+
79
106
  ### Refusing lossy conversions
80
107
 
81
- Node and mark *types* are closed, but `attrs` are not.
108
+ Node and mark *types* are always closed, but fields, attrs and content models are only closed when `strict` is enabled.
82
109
  An attr the serializer never reads is dropped without a word, so a `colspan: 2` cell quietly becomes a plain one:
83
110
 
84
111
  ```python
@@ -93,7 +120,7 @@ It is keyword-only and a `from_dict` option only — the parser emits nothing bu
93
120
  | `strict` | Keeps |
94
121
  | --- | --- |
95
122
  | `"off"` (default) | nothing is checked |
96
- | `"content"` | content and structure. Presentation an editor stamped on is type-checked and dropped |
123
+ | `"content"` | content and structure, including closed node fields and content models. Presentation an editor stamped on is type-checked and dropped |
97
124
  | `"exact"` | every attr the JSON carries |
98
125
 
99
126
  Both enabled levels also enforce the ProseMirror same-type self-exclusion rule on each node's marks.
@@ -101,6 +128,27 @@ The standard marks represented by marktip are self-exclusive, so a node cannot c
101
128
  The conflicting mark is rejected with `InvalidNodeError` rather than being serialized into Markdown that reparses with different formatting or structure.
102
129
  `strict="off"` skips this check because strict validation is disabled.
103
130
 
131
+ Both enabled levels reject a node or mark top-level key that its type does not declare (`unknown_field`), and enforce the following content models (`invalid_content`):
132
+
133
+ | Parent | Allowed children |
134
+ | --- | --- |
135
+ | `doc`, `blockquote`, `listItem`, `taskItem`, `tableCell`, `tableHeader` | block nodes |
136
+ | `paragraph`, `heading` | inline nodes (`text`, `hardBreak`, `image`, `htmlInline`) |
137
+ | `codeBlock` | unmarked `text` only |
138
+ | `bulletList` | `listItem` |
139
+ | `orderedList` | `listItem`, `taskItem` |
140
+ | `taskList` | `taskItem` |
141
+ | `table` | `tableRow` |
142
+ | `tableRow` | `tableHeader`, `tableCell` |
143
+ | `text`, `hardBreak`, `horizontalRule`, `image`, `htmlBlock`, `htmlInline` | none |
144
+
145
+ Block nodes are `paragraph`, `heading`, `blockquote`, `codeBlock`, `horizontalRule`, `bulletList`, `orderedList`, `taskList`, `table`, and `htmlBlock`.
146
+ A strict text node must carry a non-empty string `text` field.
147
+ This is marktip's canonical child grammar, not a verbatim copy of one editor's schema: in particular, ordered Markdown task items remain `orderedList` → `taskItem`, and empty parser-produced containers remain valid.
148
+ These rules prevent a nested block submitted inside a paragraph, content attached to a leaf, or an arbitrary child key from being silently flattened or omitted by the serializer.
149
+
150
+ > **Changed in 0.7.2** — `strict="content"` and `strict="exact"` now close node/mark top-level fields and per-node content models with `unknown_field` and `invalid_content`. `strict="off"` keeps the previous permissive converter behavior.
151
+
104
152
  > **Changed in 0.7.1** — `strict="content"` and `strict="exact"` now reject duplicate same-type marks with `invalid_mark_set`. Markdown parsing collapses nested spans that map to the same self-exclusive mark, so parser output remains valid on the strict read path.
105
153
 
106
154
  The two levels differ over one thing, and it is worth being precise about which:
@@ -206,6 +254,9 @@ For `from_markdown` it locates the node in the parsed document, since the input
206
254
  | `disallowed_scheme` | `InvalidNodeError` | a link `href` / image `src` scheme is outside the allowlist |
207
255
  | `disallowed_relative_url` | `InvalidNodeError` | a scheme-less `href`/`src` violates the relative policy |
208
256
  | `invalid_uri_char` | `InvalidNodeError` | an `href`/`src` contains an ASCII control character |
257
+ | `disallowed_node_type` | `InvalidNodeError` | a node type appears in `reject_node_types` |
258
+ | `unknown_field` | `InvalidNodeError` | `strict`: a node/mark top-level key is not defined for that type |
259
+ | `invalid_content` | `InvalidNodeError` | `strict`: a node violates its content model or text payload contract |
209
260
  | `unknown_attr` | `InvalidNodeError` | `strict`: the attr name is in neither dialect for that type |
210
261
  | `invalid_attr_value` | `InvalidNodeError` | `strict`: the attr's value has the wrong type or is out of range |
211
262
  | `unrepresentable` | `InvalidNodeError` | `strict`: the value is well-formed, but markdown cannot carry it |
@@ -215,7 +266,7 @@ For `from_markdown` it locates the node in the parsed document, since the input
215
266
  | `parse_failed` | `ParseError` | MD4C could not parse the input |
216
267
 
217
268
  `from_markdown` still raises a plain `TypeError` when the argument is not `str`/`bytes` — that is a caller bug, not a malformed document.
218
- So is a bad option value: a non-iterable `link_schemes` raises `TypeError`, and `link_schemes=("https://",)`, `link_relative="nope"` or `strict="strict"` raises `ValueError`.
269
+ So is a bad option value: a non-iterable `link_schemes` or `reject_node_types` raises `TypeError`, and `link_schemes=("https://",)`, `reject_node_types=("callout",)`, `link_relative="nope"` or `strict="strict"` raises `ValueError`.
219
270
  `strict` takes a level name, so `strict=True` raises `TypeError` rather than quietly meaning something.
220
271
 
221
272
  > **Changed in 0.4.0** — these were previously `ValueError` (`from_dict`) and `RuntimeError` (`from_markdown`).
@@ -247,9 +298,10 @@ It guarantees, for the whole tree:
247
298
  - **Depth** is at most 2048 levels.
248
299
  - **No ASCII control character** in a link `href` or image `src`, entity references included (`&Tab;`, `&#09;`).
249
300
  - **The URI policy**, when `link_schemes`/`image_schemes`/`link_relative`/`image_relative` ask for it — see [Restricting link and image URIs](#restricting-link-and-image-uris).
301
+ - **The node-type policy**, when `reject_node_types` names a type — see [Rejecting node types](#rejecting-node-types).
250
302
 
251
303
  A document that survives `from_dict` always serializes; callers do not need to re-check any of the above.
252
- `from_markdown` enforces the same two URI rules on what it parses.
304
+ `from_markdown` enforces the same URI and rejected-node rules on what it parses.
253
305
 
254
306
  It deliberately does **not** validate, unless `strict` asks it to:
255
307
 
@@ -264,11 +316,15 @@ It deliberately does **not** validate, unless `strict` asks it to:
264
316
  - **Mark sets.**
265
317
  With strict validation disabled, two marks of the same type may appear on one node and are serialized independently.
266
318
  `strict="content"` and `strict="exact"` reject the second mark because marktip's standard ProseMirror/Tiptap marks are self-exclusive.
319
+ - **Node and mark top-level fields.**
320
+ With strict validation disabled, a key the converter does not read is ignored.
321
+ Both enabled levels reject it with `unknown_field`, including `marks` on a block or `text` on a non-text node.
267
322
  - **Content models.**
268
- Which node may contain which is not checked — a `text` node directly under `doc` is accepted and serialized.
323
+ With strict validation disabled, which node may contain which is not checked — a `text` node directly under `doc` is accepted and serialized.
324
+ Both enabled levels enforce the content-model table above, the text payload shape, and the absence of marks inside a code block.
269
325
  - **Raw HTML.**
270
326
  The URI policy governs `link` marks and `image` nodes, not markup inside `htmlBlock`/`htmlInline`, so `<a href="javascript:...">` passes it.
271
- Pair the policy with `html=False` to close that.
327
+ Reject those node types with `reject_node_types=("htmlBlock", "htmlInline")`; on Markdown input, `html=False` is the alternative that retains the markup as literal text.
272
328
  - **URI syntax beyond the scheme.**
273
329
  Everything after the scheme is opaque, and the stored value is never rewritten — an `href` is checked in its entity-decoded form but stored exactly as given.
274
330
  Under `link_relative="allow"` a protocol-relative `//evil.com/x` also passes, since it carries no scheme; `"path_only"` is the setting that rejects it.
@@ -290,6 +346,7 @@ except tm.InvalidNodeError as e:
290
346
  Markdown cannot represent every Tiptap document exactly.
291
347
  Rather than emitting or retaining a representation that changes on the next conversion, marktip applies these deterministic (and idempotent) normalizations during parsing and serialization:
292
348
 
349
+ - **NUL in content strings** — U+0000 in text or a non-URI string attr (`alt`, `title`, `language`, `html`, etc.) becomes U+FFFD. Markdown input is replaced once before tokenization, including inline/fenced/indented code and raw HTML, so reparsing cannot retain a NUL or accumulate replacement characters. On dict input, `link.href` and `image.src` are excluded and retain their unconditional `invalid_uri_char` rejection.
293
350
  - **Hard breaks in headings** — ATX headings are single-line, so a `hardBreak` (or a literal newline carried over from setext input) inside a heading serializes as a single space: `heading("a", hardBreak, "b")` → `# a b`.
294
351
  - **Multi-block table cells** — blocks inside a cell are joined with `<br>` and the cell is flattened to one line; two paragraphs in a cell reparse as one paragraph containing a `hardBreak`.
295
352
  - **Headerless tables** — GFM tables require a header row, so the first row is always emitted as the header; a leading `tableCell` row reparses as `tableHeader`.
@@ -4,12 +4,12 @@ build-backend = "scikit_build_core.build"
4
4
 
5
5
  [project]
6
6
  name = "marktip"
7
- version = "0.7.1"
7
+ version = "0.7.3"
8
8
  description = "Fast C++/MD4C Markdown parser and canonical Markdown serializer for Tiptap-style JSON"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
11
- license = "MIT"
12
- license-files = ["LICENSE", "third_party/md4c/LICENSE.md"]
11
+ license = "MIT AND Unicode-3.0"
12
+ license-files = ["LICENSE", "third_party/md4c/LICENSE.md", "third_party/md4c/upstream_metadata/Unicode-LICENSE.txt"]
13
13
  authors = [{ name = "Saneaven (Minjae Kyung)" }]
14
14
  keywords = ["markdown", "tiptap", "md4c", "parser"]
15
15
  classifiers = [
@@ -38,9 +38,11 @@ wheel.packages = ["python/marktip"]
38
38
  sdist.include = [
39
39
  "third_party/md4c/src/md4c.c",
40
40
  "third_party/md4c/src/md4c.h",
41
+ "third_party/md4c/src/cjk.h",
41
42
  "third_party/md4c/src/entity.c",
42
43
  "third_party/md4c/src/entity.h",
43
44
  "third_party/md4c/LICENSE.md",
45
+ "third_party/md4c/upstream_metadata/Unicode-LICENSE.txt",
44
46
  ]
45
47
 
46
48
  [tool.pytest.ini_options]
@@ -0,0 +1,100 @@
1
+ """Generate the CJK-friendly classifier from Unicode 17.0.0 data files."""
2
+
3
+ import argparse
4
+ import hashlib
5
+ from pathlib import Path
6
+ import re
7
+
8
+
9
+ VERSION = "17.0.0"
10
+ SOURCES = {
11
+ "EastAsianWidth.txt": "EastAsianWidth.txt",
12
+ "Scripts.txt": "Scripts.txt",
13
+ "emoji-data.txt": "emoji/emoji-data.txt",
14
+ "StandardizedVariants.txt": "StandardizedVariants.txt",
15
+ }
16
+
17
+
18
+ def records(path):
19
+ for line in path.read_text(encoding="utf-8").splitlines():
20
+ body = line.split("#", 1)[0].strip()
21
+ if body:
22
+ yield [field.strip() for field in body.split(";")]
23
+
24
+
25
+ def codepoints(value):
26
+ parts = value.split("..")
27
+ return range(int(parts[0], 16), int(parts[-1], 16) + 1)
28
+
29
+
30
+ def intervals(values):
31
+ start = previous = None
32
+ for value in sorted(values):
33
+ if previous is not None and value != previous + 1:
34
+ yield start, previous
35
+ start = None
36
+ if start is None:
37
+ start = value
38
+ previous = value
39
+ if previous is not None:
40
+ yield start, previous
41
+
42
+
43
+ def generate(directory):
44
+ for filename in SOURCES:
45
+ header = (directory / filename).read_text(encoding="utf-8")[:1000]
46
+ if not re.search(r"17\.0(?:\.0)?\b", header):
47
+ raise ValueError(f"{filename} must contain Unicode {VERSION} data")
48
+ widths = {}
49
+ for bounds, width in records(directory / "EastAsianWidth.txt"):
50
+ widths.update(dict.fromkeys(codepoints(bounds), width))
51
+ hangul = set()
52
+ for bounds, script in records(directory / "Scripts.txt"):
53
+ if script == "Hangul":
54
+ hangul.update(codepoints(bounds))
55
+ emoji = set()
56
+ for bounds, property_name in records(directory / "emoji-data.txt"):
57
+ if property_name == "Emoji_Presentation":
58
+ emoji.update(codepoints(bounds))
59
+ cjk = {cp for cp, width in widths.items() if width in {"W", "F", "H"}} - emoji
60
+ cjk.update(hangul)
61
+ ambiguous = []
62
+ for sequence, description, *_ in records(directory / "StandardizedVariants.txt"):
63
+ base, selector = (int(cp, 16) for cp in sequence.split())
64
+ if (
65
+ widths.get(base) == "A"
66
+ and re.search(r"\bfullwidth form\b", description)
67
+ and "non-fullwidth" not in description
68
+ ):
69
+ ambiguous.append((base, selector))
70
+
71
+ lines = [
72
+ "/* Generated by scripts/build_cjk_map.py; Unicode " + VERSION + ".",
73
+ " * Data license: upstream_metadata/Unicode-LICENSE.txt.",
74
+ ]
75
+ for filename, source in SOURCES.items():
76
+ digest = hashlib.sha256((directory / filename).read_bytes()).hexdigest()
77
+ lines.append(" * https://www.unicode.org/Public/" + VERSION + "/ucd/" + source)
78
+ lines.append(" * SHA-256: " + digest)
79
+ lines.extend([
80
+ " */",
81
+ "#define R(cp_min, cp_max) ((cp_min) | 0x40000000), ((cp_max) | 0x80000000)",
82
+ "#define S(cp) (cp)",
83
+ "static const unsigned MD_CJK_MAP[] = {",
84
+ ])
85
+ for start, end in intervals(cjk):
86
+ value = f"S(0x{start:04x})" if start == end else f"R(0x{start:04x},0x{end:04x})"
87
+ lines.append(" " + value + ",")
88
+ lines.extend(["};", "#undef R", "#undef S", "static const unsigned MD_CJK_AMBIGUOUS_PUNCT[][2] = {"])
89
+ for base, selector in sorted(ambiguous):
90
+ lines.append(f" {{ 0x{base:04x}, 0x{selector:04x} }},")
91
+ lines.extend(["};", ""])
92
+ return "\n".join(lines)
93
+
94
+
95
+ if __name__ == "__main__":
96
+ parser = argparse.ArgumentParser(description=__doc__)
97
+ parser.add_argument("ucd_directory", type=Path)
98
+ parser.add_argument("--output", type=Path, default=Path("third_party/md4c/src/cjk.h"))
99
+ args = parser.parse_args()
100
+ args.output.write_text(generate(args.ucd_directory), encoding="utf-8")