marktip 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {marktip-0.2.0 → marktip-0.4.0}/CMakeLists.txt +1 -1
- marktip-0.4.0/PKG-INFO +179 -0
- marktip-0.4.0/README.md +151 -0
- {marktip-0.2.0 → marktip-0.4.0}/pyproject.toml +1 -1
- {marktip-0.2.0 → marktip-0.4.0}/python/marktip/__init__.py +24 -2
- {marktip-0.2.0 → marktip-0.4.0}/src/ast.cpp +86 -21
- {marktip-0.2.0 → marktip-0.4.0}/src/ast.h +2 -0
- marktip-0.4.0/src/errors.h +70 -0
- marktip-0.4.0/src/marktip.cpp +111 -0
- {marktip-0.2.0 → marktip-0.4.0}/src/parser.cpp +54 -59
- {marktip-0.2.0 → marktip-0.4.0}/src/parser.h +1 -1
- {marktip-0.2.0 → marktip-0.4.0}/src/serializer.cpp +77 -33
- {marktip-0.2.0 → marktip-0.4.0}/tests/test_document.py +3 -3
- marktip-0.4.0/tests/test_errors.py +190 -0
- {marktip-0.2.0 → marktip-0.4.0}/tests/test_parse.py +66 -0
- {marktip-0.2.0 → marktip-0.4.0}/tests/test_roundtrip.py +162 -4
- {marktip-0.2.0 → marktip-0.4.0}/tests/test_serialize.py +1 -1
- marktip-0.4.0/tests/test_validate.py +170 -0
- {marktip-0.2.0 → marktip-0.4.0}/third_party/md4c/src/md4c.c +46 -10
- {marktip-0.2.0 → marktip-0.4.0}/third_party/md4c/upstream_metadata/README.md +9 -0
- marktip-0.2.0/PKG-INFO +0 -82
- marktip-0.2.0/README.md +0 -54
- marktip-0.2.0/src/marktip.cpp +0 -28
- {marktip-0.2.0 → marktip-0.4.0}/.github/workflows/build-distributions.yml +0 -0
- {marktip-0.2.0 → marktip-0.4.0}/.gitignore +0 -0
- {marktip-0.2.0 → marktip-0.4.0}/LICENSE +0 -0
- {marktip-0.2.0 → marktip-0.4.0}/scripts/benchmark.py +0 -0
- {marktip-0.2.0 → marktip-0.4.0}/src/serializer.h +0 -0
- {marktip-0.2.0 → marktip-0.4.0}/third_party/md4c/LICENSE.md +0 -0
- {marktip-0.2.0 → marktip-0.4.0}/third_party/md4c/src/entity.c +0 -0
- {marktip-0.2.0 → marktip-0.4.0}/third_party/md4c/src/entity.h +0 -0
- {marktip-0.2.0 → marktip-0.4.0}/third_party/md4c/src/md4c.h +0 -0
marktip-0.4.0/PKG-INFO
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: marktip
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: Fast C++/MD4C Markdown parser and canonical Markdown serializer for Tiptap-style JSON
|
|
5
|
+
Keywords: markdown,tiptap,md4c,parser
|
|
6
|
+
Author: Saneaven (Minjae Kyung)
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
License-File: third_party/md4c/LICENSE.md
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: MacOS
|
|
13
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
14
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
15
|
+
Classifier: Programming Language :: C++
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: pytest>=8; extra == "test"
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# marktip
|
|
30
|
+
|
|
31
|
+
Fast C++/MD4C Markdown conversion for Tiptap-style JSON.
|
|
32
|
+
|
|
33
|
+
## Installation
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
python -m pip install marktip
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Release wheels are built for common CPython versions on Linux, macOS, and Windows.
|
|
40
|
+
If a wheel is not available for a platform, pip can build from the source distribution with a C++17 compiler and standard Python build tooling.
|
|
41
|
+
|
|
42
|
+
## Usage
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
import marktip as tm
|
|
46
|
+
|
|
47
|
+
doc = tm.from_markdown("# Hello")
|
|
48
|
+
ast = doc.to_dict()
|
|
49
|
+
markdown = doc.to_markdown()
|
|
50
|
+
|
|
51
|
+
doc = tm.from_dict(ast)
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
`from_markdown` follows GFM/CommonMark by default.
|
|
55
|
+
Pass `cjk_friendly=True` to relax the emphasis and strikethrough rules so delimiters next to CJK text still open and close (e.g. `**볼드**은` parses as bold), a non-standard extension that is off by default:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
doc = tm.from_markdown("**볼드**은 강조", cjk_friendly=True)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Strikethrough follows the GFM reference implementations (cmark-gfm/micromark): intra-word `~~` strikes (`a~~b~~c`), including next to CJK letters (`~~삭제~~은`).
|
|
62
|
+
`cjk_friendly` still matters for `*`/`_`, and for `~` in punctuation-adjacent cases such as `~~"인용"~~라고`.
|
|
63
|
+
|
|
64
|
+
Pass `html=False` to parse raw HTML as literal text instead of `htmlBlock`/`htmlInline` nodes.
|
|
65
|
+
Content is preserved (the serializer escapes it), and `<br>` inside table cells still maps to `hardBreak` so marktip's own table output round-trips:
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
doc = tm.from_markdown("a <u>x</u> b", html=False)
|
|
69
|
+
# paragraph with the literal text "a <u>x</u> b"
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
marktip targets GFM core syntax and canonical Markdown output rather than byte-identical source preservation.
|
|
73
|
+
|
|
74
|
+
## Errors
|
|
75
|
+
|
|
76
|
+
Every rejection marktip raises derives from `marktip.MarktipError`, so "the input is malformed" is a single `except` — no need to catch a bare `ValueError` wide enough to swallow an unrelated bug:
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
try:
|
|
80
|
+
tm.from_dict(payload).to_markdown()
|
|
81
|
+
except tm.MarktipError as e:
|
|
82
|
+
return 422, {"code": e.code, "path": e.path, "detail": e.detail}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
```
|
|
86
|
+
Exception
|
|
87
|
+
└─ MarktipError .code, .path, .detail
|
|
88
|
+
├─ UnknownTypeError + .type, .kind — type outside the closed schema
|
|
89
|
+
├─ InvalidNodeError + .field — violates the node grammar
|
|
90
|
+
└─ ParseError — markdown could not be parsed
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
`.path` is a breadcrumb into the input, e.g. `content[1].content[0].marks[0]`; it is `""` when the failure is at the root or has no location.
|
|
94
|
+
`.detail` is the same string as `str(e)`.
|
|
95
|
+
`.code` is a stable machine key:
|
|
96
|
+
|
|
97
|
+
| `.code` | Class | Raised when |
|
|
98
|
+
| --- | --- | --- |
|
|
99
|
+
| `unknown_node_type` | `UnknownTypeError` | node `type` is outside the closed schema |
|
|
100
|
+
| `unknown_mark_type` | `UnknownTypeError` | mark `type` is outside the closed schema |
|
|
101
|
+
| `missing_type` | `InvalidNodeError` | a node or mark has no `type` key |
|
|
102
|
+
| `invalid_root` | `InvalidNodeError` | the root node's type is not `doc` |
|
|
103
|
+
| `wrong_type` | `InvalidNodeError` | `attrs`/`content`/`marks`/a child has the wrong Python type |
|
|
104
|
+
| `max_depth` | `InvalidNodeError` | dict nesting exceeds 2048 |
|
|
105
|
+
| `markdown_max_depth` | `ParseError` | markdown nesting exceeds 2048 |
|
|
106
|
+
| `parse_failed` | `ParseError` | MD4C could not parse the input |
|
|
107
|
+
|
|
108
|
+
`from_markdown` still raises a plain `TypeError` when the argument is not `str`/`bytes` — that is a caller bug, not a malformed document.
|
|
109
|
+
|
|
110
|
+
> **Changed in 0.4.0** — these were previously `ValueError` (`from_dict`) and `RuntimeError` (`from_markdown`).
|
|
111
|
+
> `MarktipError` derives from `Exception` directly, so `except ValueError` no longer catches them.
|
|
112
|
+
|
|
113
|
+
## What `from_dict` validates
|
|
114
|
+
|
|
115
|
+
The Tiptap-side schema is closed: every node/mark type must have a markdown mapping.
|
|
116
|
+
`from_dict` is the single enforcement point and rejects unknown types instead of silently dropping content.
|
|
117
|
+
It guarantees, for the whole tree:
|
|
118
|
+
|
|
119
|
+
- **Closed schema.**
|
|
120
|
+
Node types are `doc`, `paragraph`, `text`, `hardBreak`, `heading`, `blockquote`, `codeBlock`, `horizontalRule`, `bulletList`, `orderedList`, `listItem`, `taskList`, `taskItem`, `table`, `tableRow`, `tableHeader`, `tableCell`, `image`, `htmlBlock`, `htmlInline`.
|
|
121
|
+
Marks are `bold`, `italic`, `strike`, `code`, `link`.
|
|
122
|
+
Anything else → `UnknownTypeError`.
|
|
123
|
+
- **`type` is required** on every node and mark.
|
|
124
|
+
- **The root is a `doc`.**
|
|
125
|
+
- **Shapes**: `attrs` is a dict, `content` and `marks` are lists, and every entry of `content`/`marks` is a dict.
|
|
126
|
+
- **Depth** is at most 2048 levels.
|
|
127
|
+
|
|
128
|
+
A document that survives `from_dict` always serializes; callers do not need to re-check any of the above.
|
|
129
|
+
|
|
130
|
+
It deliberately does **not** validate:
|
|
131
|
+
|
|
132
|
+
- **`attrs` value types.**
|
|
133
|
+
Keys are free-form.
|
|
134
|
+
`str`, `int`, and `bool` round-trip unchanged; anything else is coerced to a string (`[1, 2]` → `"[1, 2]"`, `1.5` → `"1.5"`, `None` → `""`), so `to_dict()` will not always give back what you passed in.
|
|
135
|
+
- **Per-node required attrs.**
|
|
136
|
+
A `heading` without `level`, an `image` without `src`, or a `link` mark without `href` is accepted; the serializer substitutes a default rather than failing.
|
|
137
|
+
- **Content models.**
|
|
138
|
+
Which node may contain which is not checked — a `text` node directly under `doc` is accepted and serialized.
|
|
139
|
+
|
|
140
|
+
Structural violations report where they happened:
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
try:
|
|
144
|
+
tm.from_dict({"type": "doc", "content": [{"content": []}]})
|
|
145
|
+
except tm.InvalidNodeError as e:
|
|
146
|
+
e.code # "missing_type"
|
|
147
|
+
e.field # "type"
|
|
148
|
+
e.path # "content[0]"
|
|
149
|
+
e.detail # "node is missing required key 'type'"
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## Defined normalizations
|
|
153
|
+
|
|
154
|
+
Markdown cannot represent every Tiptap document exactly.
|
|
155
|
+
Rather than emitting markdown that reparses into a different structure, marktip applies these deterministic (and idempotent) normalizations at serialization time:
|
|
156
|
+
|
|
157
|
+
- **Hard breaks in headings** — ATX headings are single-line, so a `hardBreak` (or a literal newline carried over from setext input) inside a heading serializes as a single space: `heading("a", hardBreak, "b")` → `# a b`.
|
|
158
|
+
- **Multi-block table cells** — blocks inside a cell are joined with `<br>` and the cell is flattened to one line; two paragraphs in a cell reparse as one paragraph containing a `hardBreak`.
|
|
159
|
+
- **Headerless tables** — GFM tables require a header row, so the first row is always emitted as the header; a leading `tableCell` row reparses as `tableHeader`.
|
|
160
|
+
- **Heading levels** — clamped to 1–6 at serialization (`0` → `#`, `7` → `######`).
|
|
161
|
+
- **Emphasis boundary whitespace** — whitespace touching an emphasis delimiter has no valid markdown form and is expelled outside the marks (`bold("굵게 ")` → `**굵게** `), cf. prosemirror-markdown's `expelEnclosingWhitespace`.
|
|
162
|
+
- **Adjacent same-family lists** — consecutive lists of the same family alternate markers (`-`/`*`, `1.`/`1)`) so they stay separate lists on reparse instead of merging (which would renumber ordered items or spread task checkboxes).
|
|
163
|
+
|
|
164
|
+
## Development
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
python -m pip install .[test]
|
|
168
|
+
python -m pytest
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
For a direct local CMake build:
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \
|
|
175
|
+
-Dpybind11_DIR="$(python -m pybind11 --cmakedir)"
|
|
176
|
+
cmake --build build
|
|
177
|
+
PYTHONPATH=python python -m pytest
|
|
178
|
+
PYTHONPATH=python python scripts/benchmark.py
|
|
179
|
+
```
|
marktip-0.4.0/README.md
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
# marktip
|
|
2
|
+
|
|
3
|
+
Fast C++/MD4C Markdown conversion for Tiptap-style JSON.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
python -m pip install marktip
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Release wheels are built for common CPython versions on Linux, macOS, and Windows.
|
|
12
|
+
If a wheel is not available for a platform, pip can build from the source distribution with a C++17 compiler and standard Python build tooling.
|
|
13
|
+
|
|
14
|
+
## Usage
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
import marktip as tm
|
|
18
|
+
|
|
19
|
+
doc = tm.from_markdown("# Hello")
|
|
20
|
+
ast = doc.to_dict()
|
|
21
|
+
markdown = doc.to_markdown()
|
|
22
|
+
|
|
23
|
+
doc = tm.from_dict(ast)
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
`from_markdown` follows GFM/CommonMark by default.
|
|
27
|
+
Pass `cjk_friendly=True` to relax the emphasis and strikethrough rules so delimiters next to CJK text still open and close (e.g. `**볼드**은` parses as bold), a non-standard extension that is off by default:
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
doc = tm.from_markdown("**볼드**은 강조", cjk_friendly=True)
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Strikethrough follows the GFM reference implementations (cmark-gfm/micromark): intra-word `~~` strikes (`a~~b~~c`), including next to CJK letters (`~~삭제~~은`).
|
|
34
|
+
`cjk_friendly` still matters for `*`/`_`, and for `~` in punctuation-adjacent cases such as `~~"인용"~~라고`.
|
|
35
|
+
|
|
36
|
+
Pass `html=False` to parse raw HTML as literal text instead of `htmlBlock`/`htmlInline` nodes.
|
|
37
|
+
Content is preserved (the serializer escapes it), and `<br>` inside table cells still maps to `hardBreak` so marktip's own table output round-trips:
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
doc = tm.from_markdown("a <u>x</u> b", html=False)
|
|
41
|
+
# paragraph with the literal text "a <u>x</u> b"
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
marktip targets GFM core syntax and canonical Markdown output rather than byte-identical source preservation.
|
|
45
|
+
|
|
46
|
+
## Errors
|
|
47
|
+
|
|
48
|
+
Every rejection marktip raises derives from `marktip.MarktipError`, so "the input is malformed" is a single `except` — no need to catch a bare `ValueError` wide enough to swallow an unrelated bug:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
try:
|
|
52
|
+
tm.from_dict(payload).to_markdown()
|
|
53
|
+
except tm.MarktipError as e:
|
|
54
|
+
return 422, {"code": e.code, "path": e.path, "detail": e.detail}
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
Exception
|
|
59
|
+
└─ MarktipError .code, .path, .detail
|
|
60
|
+
├─ UnknownTypeError + .type, .kind — type outside the closed schema
|
|
61
|
+
├─ InvalidNodeError + .field — violates the node grammar
|
|
62
|
+
└─ ParseError — markdown could not be parsed
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
`.path` is a breadcrumb into the input, e.g. `content[1].content[0].marks[0]`; it is `""` when the failure is at the root or has no location.
|
|
66
|
+
`.detail` is the same string as `str(e)`.
|
|
67
|
+
`.code` is a stable machine key:
|
|
68
|
+
|
|
69
|
+
| `.code` | Class | Raised when |
|
|
70
|
+
| --- | --- | --- |
|
|
71
|
+
| `unknown_node_type` | `UnknownTypeError` | node `type` is outside the closed schema |
|
|
72
|
+
| `unknown_mark_type` | `UnknownTypeError` | mark `type` is outside the closed schema |
|
|
73
|
+
| `missing_type` | `InvalidNodeError` | a node or mark has no `type` key |
|
|
74
|
+
| `invalid_root` | `InvalidNodeError` | the root node's type is not `doc` |
|
|
75
|
+
| `wrong_type` | `InvalidNodeError` | `attrs`/`content`/`marks`/a child has the wrong Python type |
|
|
76
|
+
| `max_depth` | `InvalidNodeError` | dict nesting exceeds 2048 |
|
|
77
|
+
| `markdown_max_depth` | `ParseError` | markdown nesting exceeds 2048 |
|
|
78
|
+
| `parse_failed` | `ParseError` | MD4C could not parse the input |
|
|
79
|
+
|
|
80
|
+
`from_markdown` still raises a plain `TypeError` when the argument is not `str`/`bytes` — that is a caller bug, not a malformed document.
|
|
81
|
+
|
|
82
|
+
> **Changed in 0.4.0** — these were previously `ValueError` (`from_dict`) and `RuntimeError` (`from_markdown`).
|
|
83
|
+
> `MarktipError` derives from `Exception` directly, so `except ValueError` no longer catches them.
|
|
84
|
+
|
|
85
|
+
## What `from_dict` validates
|
|
86
|
+
|
|
87
|
+
The Tiptap-side schema is closed: every node/mark type must have a markdown mapping.
|
|
88
|
+
`from_dict` is the single enforcement point and rejects unknown types instead of silently dropping content.
|
|
89
|
+
It guarantees, for the whole tree:
|
|
90
|
+
|
|
91
|
+
- **Closed schema.**
|
|
92
|
+
Node types are `doc`, `paragraph`, `text`, `hardBreak`, `heading`, `blockquote`, `codeBlock`, `horizontalRule`, `bulletList`, `orderedList`, `listItem`, `taskList`, `taskItem`, `table`, `tableRow`, `tableHeader`, `tableCell`, `image`, `htmlBlock`, `htmlInline`.
|
|
93
|
+
Marks are `bold`, `italic`, `strike`, `code`, `link`.
|
|
94
|
+
Anything else → `UnknownTypeError`.
|
|
95
|
+
- **`type` is required** on every node and mark.
|
|
96
|
+
- **The root is a `doc`.**
|
|
97
|
+
- **Shapes**: `attrs` is a dict, `content` and `marks` are lists, and every entry of `content`/`marks` is a dict.
|
|
98
|
+
- **Depth** is at most 2048 levels.
|
|
99
|
+
|
|
100
|
+
A document that survives `from_dict` always serializes; callers do not need to re-check any of the above.
|
|
101
|
+
|
|
102
|
+
It deliberately does **not** validate:
|
|
103
|
+
|
|
104
|
+
- **`attrs` value types.**
|
|
105
|
+
Keys are free-form.
|
|
106
|
+
`str`, `int`, and `bool` round-trip unchanged; anything else is coerced to a string (`[1, 2]` → `"[1, 2]"`, `1.5` → `"1.5"`, `None` → `""`), so `to_dict()` will not always give back what you passed in.
|
|
107
|
+
- **Per-node required attrs.**
|
|
108
|
+
A `heading` without `level`, an `image` without `src`, or a `link` mark without `href` is accepted; the serializer substitutes a default rather than failing.
|
|
109
|
+
- **Content models.**
|
|
110
|
+
Which node may contain which is not checked — a `text` node directly under `doc` is accepted and serialized.
|
|
111
|
+
|
|
112
|
+
Structural violations report where they happened:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
try:
|
|
116
|
+
tm.from_dict({"type": "doc", "content": [{"content": []}]})
|
|
117
|
+
except tm.InvalidNodeError as e:
|
|
118
|
+
e.code # "missing_type"
|
|
119
|
+
e.field # "type"
|
|
120
|
+
e.path # "content[0]"
|
|
121
|
+
e.detail # "node is missing required key 'type'"
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Defined normalizations
|
|
125
|
+
|
|
126
|
+
Markdown cannot represent every Tiptap document exactly.
|
|
127
|
+
Rather than emitting markdown that reparses into a different structure, marktip applies these deterministic (and idempotent) normalizations at serialization time:
|
|
128
|
+
|
|
129
|
+
- **Hard breaks in headings** — ATX headings are single-line, so a `hardBreak` (or a literal newline carried over from setext input) inside a heading serializes as a single space: `heading("a", hardBreak, "b")` → `# a b`.
|
|
130
|
+
- **Multi-block table cells** — blocks inside a cell are joined with `<br>` and the cell is flattened to one line; two paragraphs in a cell reparse as one paragraph containing a `hardBreak`.
|
|
131
|
+
- **Headerless tables** — GFM tables require a header row, so the first row is always emitted as the header; a leading `tableCell` row reparses as `tableHeader`.
|
|
132
|
+
- **Heading levels** — clamped to 1–6 at serialization (`0` → `#`, `7` → `######`).
|
|
133
|
+
- **Emphasis boundary whitespace** — whitespace touching an emphasis delimiter has no valid markdown form and is expelled outside the marks (`bold("굵게 ")` → `**굵게** `), cf. prosemirror-markdown's `expelEnclosingWhitespace`.
|
|
134
|
+
- **Adjacent same-family lists** — consecutive lists of the same family alternate markers (`-`/`*`, `1.`/`1)`) so they stay separate lists on reparse instead of merging (which would renumber ordered items or spread task checkboxes).
|
|
135
|
+
|
|
136
|
+
## Development
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
python -m pip install .[test]
|
|
140
|
+
python -m pytest
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
For a direct local CMake build:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \
|
|
147
|
+
-Dpybind11_DIR="$(python -m pybind11 --cmakedir)"
|
|
148
|
+
cmake --build build
|
|
149
|
+
PYTHONPATH=python python -m pytest
|
|
150
|
+
PYTHONPATH=python python scripts/benchmark.py
|
|
151
|
+
```
|
|
@@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "marktip"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "Fast C++/MD4C Markdown parser and canonical Markdown serializer for Tiptap-style JSON"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
"""Fast Markdown conversion for Tiptap-style JSON."""
|
|
2
2
|
|
|
3
3
|
try:
|
|
4
|
-
from ._core import
|
|
4
|
+
from ._core import (
|
|
5
|
+
Document,
|
|
6
|
+
InvalidNodeError,
|
|
7
|
+
MarktipError,
|
|
8
|
+
ParseError,
|
|
9
|
+
UnknownTypeError,
|
|
10
|
+
__version__,
|
|
11
|
+
from_dict,
|
|
12
|
+
from_markdown,
|
|
13
|
+
)
|
|
5
14
|
except ImportError: # pragma: no cover - development-only CMake build fallback
|
|
6
15
|
import importlib.util
|
|
7
16
|
import pathlib
|
|
@@ -21,7 +30,20 @@ except ImportError: # pragma: no cover - development-only CMake build fallback
|
|
|
21
30
|
|
|
22
31
|
__version__ = _module.__version__
|
|
23
32
|
Document = _module.Document
|
|
33
|
+
MarktipError = _module.MarktipError
|
|
34
|
+
UnknownTypeError = _module.UnknownTypeError
|
|
35
|
+
InvalidNodeError = _module.InvalidNodeError
|
|
36
|
+
ParseError = _module.ParseError
|
|
24
37
|
from_dict = _module.from_dict
|
|
25
38
|
from_markdown = _module.from_markdown
|
|
26
39
|
|
|
27
|
-
__all__ = [
|
|
40
|
+
__all__ = [
|
|
41
|
+
"__version__",
|
|
42
|
+
"Document",
|
|
43
|
+
"MarktipError",
|
|
44
|
+
"UnknownTypeError",
|
|
45
|
+
"InvalidNodeError",
|
|
46
|
+
"ParseError",
|
|
47
|
+
"from_markdown",
|
|
48
|
+
"from_dict",
|
|
49
|
+
]
|
|
@@ -1,12 +1,57 @@
|
|
|
1
1
|
#include "ast.h"
|
|
2
2
|
|
|
3
|
-
#include <
|
|
3
|
+
#include <algorithm>
|
|
4
|
+
#include <iterator>
|
|
4
5
|
|
|
5
6
|
namespace py = pybind11;
|
|
6
7
|
|
|
7
8
|
namespace marktip {
|
|
8
9
|
namespace {
|
|
9
10
|
|
|
11
|
+
// The closed schema: every node/mark type the parser can emit and the
|
|
12
|
+
// serializer understands. Sorted byte-wise (uppercase sorts before lowercase,
|
|
13
|
+
// e.g. tableCell < tableHeader < tableRow) — required by binary_search; do not
|
|
14
|
+
// "alphabetize" case-insensitively.
|
|
15
|
+
constexpr std::string_view kKnownNodeTypes[] = {
|
|
16
|
+
"blockquote", "bulletList", "codeBlock", "doc", "hardBreak", "heading", "horizontalRule",
|
|
17
|
+
"htmlBlock", "htmlInline", "image", "listItem", "orderedList", "paragraph", "table",
|
|
18
|
+
"tableCell", "tableHeader", "tableRow", "taskItem", "taskList", "text",
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
constexpr std::string_view kKnownMarkTypes[] = {"bold", "code", "italic", "link", "strike"};
|
|
22
|
+
|
|
23
|
+
bool is_known_node_type(std::string_view type) {
|
|
24
|
+
return std::binary_search(std::begin(kKnownNodeTypes), std::end(kKnownNodeTypes), type);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
bool is_known_mark_type(std::string_view type) {
|
|
28
|
+
return std::binary_search(std::begin(kKnownMarkTypes), std::end(kKnownMarkTypes), type);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Breadcrumb into the input dict, e.g. content[0].content[2].marks[1].
|
|
32
|
+
// Segments are static strings so pushing a frame never allocates; the path
|
|
33
|
+
// string is only rendered when an error is actually thrown.
|
|
34
|
+
struct PathFrame {
|
|
35
|
+
const char* segment; // "content" | "marks"
|
|
36
|
+
std::size_t index;
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
using PathStack = std::vector<PathFrame>;
|
|
40
|
+
|
|
41
|
+
std::string format_path(const PathStack& path) {
|
|
42
|
+
std::string out;
|
|
43
|
+
for (std::size_t i = 0; i < path.size(); ++i) {
|
|
44
|
+
if (i != 0) {
|
|
45
|
+
out.push_back('.');
|
|
46
|
+
}
|
|
47
|
+
out += path[i].segment;
|
|
48
|
+
out.push_back('[');
|
|
49
|
+
out += std::to_string(path[i].index);
|
|
50
|
+
out.push_back(']');
|
|
51
|
+
}
|
|
52
|
+
return out;
|
|
53
|
+
}
|
|
54
|
+
|
|
10
55
|
bool dict_contains(py::dict dict, const char* key) {
|
|
11
56
|
return PyMapping_HasKeyString(dict.ptr(), key) == 1;
|
|
12
57
|
}
|
|
@@ -18,16 +63,16 @@ py::object dict_get(py::dict dict, const char* key) {
|
|
|
18
63
|
return dict[py::str(key)];
|
|
19
64
|
}
|
|
20
65
|
|
|
21
|
-
py::dict expect_dict(py::handle value, std::string_view context) {
|
|
66
|
+
py::dict expect_dict(py::handle value, std::string_view context, const char* field, const PathStack& path) {
|
|
22
67
|
if (!py::isinstance<py::dict>(value)) {
|
|
23
|
-
throw
|
|
68
|
+
throw InvalidNodeError("wrong_type", field, std::string(context) + " must be a dict", format_path(path));
|
|
24
69
|
}
|
|
25
70
|
return py::reinterpret_borrow<py::dict>(value);
|
|
26
71
|
}
|
|
27
72
|
|
|
28
|
-
py::list expect_list(py::handle value, std::string_view context) {
|
|
73
|
+
py::list expect_list(py::handle value, std::string_view context, const char* field, const PathStack& path) {
|
|
29
74
|
if (!py::isinstance<py::list>(value)) {
|
|
30
|
-
throw
|
|
75
|
+
throw InvalidNodeError("wrong_type", field, std::string(context) + " must be a list", format_path(path));
|
|
31
76
|
}
|
|
32
77
|
return py::reinterpret_borrow<py::list>(value);
|
|
33
78
|
}
|
|
@@ -39,10 +84,11 @@ std::string py_to_string(py::handle value) {
|
|
|
39
84
|
return py::cast<std::string>(py::str(value));
|
|
40
85
|
}
|
|
41
86
|
|
|
42
|
-
std::string required_type(py::dict value, std::string_view context) {
|
|
87
|
+
std::string required_type(py::dict value, std::string_view context, const PathStack& path) {
|
|
43
88
|
py::object type = dict_get(value, "type");
|
|
44
89
|
if (type.is_none()) {
|
|
45
|
-
throw
|
|
90
|
+
throw InvalidNodeError("missing_type", "type", std::string(context) + " is missing required key 'type'",
|
|
91
|
+
format_path(path));
|
|
46
92
|
}
|
|
47
93
|
return py_to_string(type);
|
|
48
94
|
}
|
|
@@ -124,26 +170,33 @@ py::dict node_to_py(const Document& document, std::size_t index) {
|
|
|
124
170
|
return out;
|
|
125
171
|
}
|
|
126
172
|
|
|
127
|
-
Mark mark_from_py(py::dict mark_dict) {
|
|
173
|
+
Mark mark_from_py(py::dict mark_dict, const PathStack& path) {
|
|
128
174
|
Mark mark;
|
|
129
|
-
mark.type = required_type(mark_dict, "mark");
|
|
175
|
+
mark.type = required_type(mark_dict, "mark", path);
|
|
176
|
+
if (!is_known_mark_type(mark.type)) {
|
|
177
|
+
throw UnknownTypeError("mark", mark.type, format_path(path));
|
|
178
|
+
}
|
|
130
179
|
|
|
131
180
|
py::object attrs = dict_get(mark_dict, "attrs");
|
|
132
181
|
if (!attrs.is_none()) {
|
|
133
|
-
mark.attrs = attrs_from_py(expect_dict(attrs, "mark attrs"));
|
|
182
|
+
mark.attrs = attrs_from_py(expect_dict(attrs, "mark attrs", "attrs", path));
|
|
134
183
|
}
|
|
135
184
|
|
|
136
185
|
return mark;
|
|
137
186
|
}
|
|
138
187
|
|
|
139
188
|
void fill_node_from_py(Document& document, std::size_t index, py::dict node_dict, std::string_view context,
|
|
140
|
-
std::size_t depth) {
|
|
189
|
+
std::size_t depth, PathStack& path) {
|
|
141
190
|
if (depth > kMaxNodeDepth) {
|
|
142
|
-
throw
|
|
191
|
+
throw InvalidNodeError("max_depth", "content", "node content nesting exceeds maximum depth",
|
|
192
|
+
format_path(path));
|
|
143
193
|
}
|
|
144
194
|
|
|
145
195
|
Node node;
|
|
146
|
-
node.type = required_type(node_dict, context);
|
|
196
|
+
node.type = required_type(node_dict, context, path);
|
|
197
|
+
if (!is_known_node_type(node.type)) {
|
|
198
|
+
throw UnknownTypeError("node", node.type, format_path(path));
|
|
199
|
+
}
|
|
147
200
|
|
|
148
201
|
if (node.type == "text") {
|
|
149
202
|
node.text = py_to_string(dict_get(node_dict, "text"));
|
|
@@ -151,14 +204,18 @@ void fill_node_from_py(Document& document, std::size_t index, py::dict node_dict
|
|
|
151
204
|
|
|
152
205
|
py::object attrs = dict_get(node_dict, "attrs");
|
|
153
206
|
if (!attrs.is_none()) {
|
|
154
|
-
node.attrs = attrs_from_py(expect_dict(attrs, "node attrs"));
|
|
207
|
+
node.attrs = attrs_from_py(expect_dict(attrs, "node attrs", "attrs", path));
|
|
155
208
|
}
|
|
156
209
|
|
|
157
210
|
py::object marks = dict_get(node_dict, "marks");
|
|
158
211
|
if (!marks.is_none()) {
|
|
159
|
-
py::list mark_list = expect_list(marks, "node marks");
|
|
212
|
+
py::list mark_list = expect_list(marks, "node marks", "marks", path);
|
|
213
|
+
std::size_t mark_index = 0;
|
|
160
214
|
for (py::handle mark_handle : mark_list) {
|
|
161
|
-
|
|
215
|
+
path.push_back({"marks", mark_index});
|
|
216
|
+
node.marks.push_back(mark_from_py(expect_dict(mark_handle, "mark", "marks", path), path));
|
|
217
|
+
path.pop_back();
|
|
218
|
+
++mark_index;
|
|
162
219
|
}
|
|
163
220
|
}
|
|
164
221
|
|
|
@@ -169,10 +226,15 @@ void fill_node_from_py(Document& document, std::size_t index, py::dict node_dict
|
|
|
169
226
|
return;
|
|
170
227
|
}
|
|
171
228
|
|
|
172
|
-
py::list content_list = expect_list(content, "node content");
|
|
229
|
+
py::list content_list = expect_list(content, "node content", "content", path);
|
|
230
|
+
std::size_t child_pos = 0;
|
|
173
231
|
for (py::handle child_handle : content_list) {
|
|
174
232
|
std::size_t child_index = document.append_child(index, Node{});
|
|
175
|
-
|
|
233
|
+
path.push_back({"content", child_pos});
|
|
234
|
+
fill_node_from_py(document, child_index, expect_dict(child_handle, "content child", "content", path), "node",
|
|
235
|
+
depth + 1, path);
|
|
236
|
+
path.pop_back();
|
|
237
|
+
++child_pos;
|
|
176
238
|
}
|
|
177
239
|
}
|
|
178
240
|
|
|
@@ -330,13 +392,16 @@ py::dict Document::to_dict() const {
|
|
|
330
392
|
}
|
|
331
393
|
|
|
332
394
|
Document from_dict_py(py::dict root) {
|
|
333
|
-
|
|
395
|
+
PathStack path;
|
|
396
|
+
path.reserve(32);
|
|
397
|
+
|
|
398
|
+
std::string type = required_type(root, "root node", path);
|
|
334
399
|
if (type != "doc") {
|
|
335
|
-
throw
|
|
400
|
+
throw InvalidNodeError("invalid_root", "type", "root node must have type 'doc'", format_path(path));
|
|
336
401
|
}
|
|
337
402
|
|
|
338
403
|
Document document;
|
|
339
|
-
fill_node_from_py(document, 0, root, "root node", 0);
|
|
404
|
+
fill_node_from_py(document, 0, root, "root node", 0, path);
|
|
340
405
|
return document;
|
|
341
406
|
}
|
|
342
407
|
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
#pragma once
|
|
2
|
+
|
|
3
|
+
#include <exception>
|
|
4
|
+
#include <string>
|
|
5
|
+
#include <utility>
|
|
6
|
+
|
|
7
|
+
namespace marktip {
|
|
8
|
+
|
|
9
|
+
// Every error marktip raises for bad input derives from MarktipError,
|
|
10
|
+
// so a caller can express "the document is malformed" as a single except clause.
|
|
11
|
+
//
|
|
12
|
+
// These types are thrown from parse_to_document() while the GIL is released,
|
|
13
|
+
// so they must never hold pybind11 objects — std::string members only.
|
|
14
|
+
class MarktipError : public std::exception {
|
|
15
|
+
public:
|
|
16
|
+
MarktipError(std::string code, std::string message, std::string path = {})
|
|
17
|
+
: code_(std::move(code)), message_(std::move(message)), path_(std::move(path)) {}
|
|
18
|
+
|
|
19
|
+
const char* what() const noexcept override { return message_.c_str(); }
|
|
20
|
+
|
|
21
|
+
const std::string& code() const noexcept { return code_; }
|
|
22
|
+
const std::string& path() const noexcept { return path_; }
|
|
23
|
+
|
|
24
|
+
private:
|
|
25
|
+
std::string code_; // stable machine key, e.g. "wrong_type"
|
|
26
|
+
std::string message_;
|
|
27
|
+
std::string path_; // "content[0].marks[1]"; empty = root / not applicable
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
// Raised by from_dict when a node or mark type is outside the closed schema.
|
|
31
|
+
class UnknownTypeError : public MarktipError {
|
|
32
|
+
public:
|
|
33
|
+
UnknownTypeError(std::string kind, std::string type_name, std::string path)
|
|
34
|
+
// path is passed by copy, not moved: argument evaluation order is unspecified,
|
|
35
|
+
// and the message argument above reads it.
|
|
36
|
+
: MarktipError("unknown_" + kind + "_type",
|
|
37
|
+
"unknown " + kind + " type '" + type_name + "' at " +
|
|
38
|
+
(path.empty() ? std::string("root") : path),
|
|
39
|
+
path),
|
|
40
|
+
kind_(std::move(kind)),
|
|
41
|
+
type_name_(std::move(type_name)) {}
|
|
42
|
+
|
|
43
|
+
const std::string& kind() const noexcept { return kind_; }
|
|
44
|
+
const std::string& type_name() const noexcept { return type_name_; }
|
|
45
|
+
|
|
46
|
+
private:
|
|
47
|
+
std::string kind_; // "node" | "mark"
|
|
48
|
+
std::string type_name_;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
// Raised when the input violates the node grammar itself: a missing 'type', a non-dict attrs,
|
|
52
|
+
// a non-list content, a root that is not a doc.
|
|
53
|
+
class InvalidNodeError : public MarktipError {
|
|
54
|
+
public:
|
|
55
|
+
InvalidNodeError(std::string code, std::string field, std::string message, std::string path)
|
|
56
|
+
: MarktipError(std::move(code), std::move(message), std::move(path)), field_(std::move(field)) {}
|
|
57
|
+
|
|
58
|
+
const std::string& field() const noexcept { return field_; }
|
|
59
|
+
|
|
60
|
+
private:
|
|
61
|
+
std::string field_; // "type" | "attrs" | "marks" | "content"
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
// Raised by from_markdown when the markdown itself cannot be parsed.
|
|
65
|
+
class ParseError : public MarktipError {
|
|
66
|
+
public:
|
|
67
|
+
ParseError(std::string code, std::string message) : MarktipError(std::move(code), std::move(message)) {}
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
} // namespace marktip
|