marktip 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {marktip-0.2.0 → marktip-0.3.0}/CMakeLists.txt +1 -1
- marktip-0.3.0/PKG-INFO +146 -0
- marktip-0.3.0/README.md +118 -0
- {marktip-0.2.0 → marktip-0.3.0}/pyproject.toml +1 -1
- {marktip-0.2.0 → marktip-0.3.0}/python/marktip/__init__.py +3 -2
- {marktip-0.2.0 → marktip-0.3.0}/src/ast.cpp +67 -5
- {marktip-0.2.0 → marktip-0.3.0}/src/ast.h +22 -0
- marktip-0.3.0/src/marktip.cpp +62 -0
- {marktip-0.2.0 → marktip-0.3.0}/src/parser.cpp +21 -8
- {marktip-0.2.0 → marktip-0.3.0}/src/parser.h +1 -1
- {marktip-0.2.0 → marktip-0.3.0}/src/serializer.cpp +74 -31
- {marktip-0.2.0 → marktip-0.3.0}/tests/test_parse.py +66 -0
- {marktip-0.2.0 → marktip-0.3.0}/tests/test_roundtrip.py +162 -4
- marktip-0.3.0/tests/test_validate.py +164 -0
- {marktip-0.2.0 → marktip-0.3.0}/third_party/md4c/src/md4c.c +46 -10
- {marktip-0.2.0 → marktip-0.3.0}/third_party/md4c/upstream_metadata/README.md +9 -0
- marktip-0.2.0/PKG-INFO +0 -82
- marktip-0.2.0/README.md +0 -54
- marktip-0.2.0/src/marktip.cpp +0 -28
- {marktip-0.2.0 → marktip-0.3.0}/.github/workflows/build-distributions.yml +0 -0
- {marktip-0.2.0 → marktip-0.3.0}/.gitignore +0 -0
- {marktip-0.2.0 → marktip-0.3.0}/LICENSE +0 -0
- {marktip-0.2.0 → marktip-0.3.0}/scripts/benchmark.py +0 -0
- {marktip-0.2.0 → marktip-0.3.0}/src/serializer.h +0 -0
- {marktip-0.2.0 → marktip-0.3.0}/tests/test_document.py +0 -0
- {marktip-0.2.0 → marktip-0.3.0}/tests/test_serialize.py +0 -0
- {marktip-0.2.0 → marktip-0.3.0}/third_party/md4c/LICENSE.md +0 -0
- {marktip-0.2.0 → marktip-0.3.0}/third_party/md4c/src/entity.c +0 -0
- {marktip-0.2.0 → marktip-0.3.0}/third_party/md4c/src/entity.h +0 -0
- {marktip-0.2.0 → marktip-0.3.0}/third_party/md4c/src/md4c.h +0 -0
marktip-0.3.0/PKG-INFO
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: marktip
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Fast C++/MD4C Markdown parser and canonical Markdown serializer for Tiptap-style JSON
|
|
5
|
+
Keywords: markdown,tiptap,md4c,parser
|
|
6
|
+
Author: Saneaven (Minjae Kyung)
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
License-File: third_party/md4c/LICENSE.md
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: MacOS
|
|
13
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
14
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
15
|
+
Classifier: Programming Language :: C++
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
23
|
+
Classifier: Topic :: Text Processing :: Markup :: Markdown
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: pytest>=8; extra == "test"
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# marktip
|
|
30
|
+
|
|
31
|
+
Fast C++/MD4C Markdown conversion for Tiptap-style JSON.
|
|
32
|
+
|
|
33
|
+
## Installation
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
python -m pip install marktip
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Release wheels are built for common CPython versions on Linux, macOS, and
|
|
40
|
+
Windows. If a wheel is not available for a platform, pip can build from the
|
|
41
|
+
source distribution with a C++17 compiler and standard Python build tooling.
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import marktip as tm
|
|
47
|
+
|
|
48
|
+
doc = tm.from_markdown("# Hello")
|
|
49
|
+
ast = doc.to_dict()
|
|
50
|
+
markdown = doc.to_markdown()
|
|
51
|
+
|
|
52
|
+
doc = tm.from_dict(ast)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`from_markdown` follows GFM/CommonMark by default. Pass `cjk_friendly=True` to
|
|
56
|
+
relax the emphasis and strikethrough rules so delimiters next to CJK text still
|
|
57
|
+
open and close (e.g. `**볼드**은` parses as bold), a non-standard extension that
|
|
58
|
+
is off by default:
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
doc = tm.from_markdown("**볼드**은 강조", cjk_friendly=True)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Strikethrough follows the GFM reference implementations (cmark-gfm/micromark):
|
|
65
|
+
intra-word `~~` strikes (`a~~b~~c`), including next to CJK letters
|
|
66
|
+
(`~~삭제~~은`). `cjk_friendly` still matters for `*`/`_`, and for `~` in
|
|
67
|
+
punctuation-adjacent cases such as `~~"인용"~~라고`.
|
|
68
|
+
|
|
69
|
+
Pass `html=False` to parse raw HTML as literal text instead of
|
|
70
|
+
`htmlBlock`/`htmlInline` nodes. Content is preserved (the serializer escapes
|
|
71
|
+
it), and `<br>` inside table cells still maps to `hardBreak` so marktip's own
|
|
72
|
+
table output round-trips:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
doc = tm.from_markdown("a <u>x</u> b", html=False)
|
|
76
|
+
# paragraph with the literal text "a <u>x</u> b"
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
marktip targets GFM core syntax and canonical Markdown output rather than
|
|
80
|
+
byte-identical source preservation.
|
|
81
|
+
|
|
82
|
+
## Schema enforcement
|
|
83
|
+
|
|
84
|
+
The Tiptap-side schema is closed: every node/mark type must have a markdown
|
|
85
|
+
mapping. `from_dict` rejects unknown types instead of silently dropping
|
|
86
|
+
content, raising `marktip.UnknownTypeError` — a `ValueError` subclass with
|
|
87
|
+
structured fields for programmatic relay:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
try:
|
|
91
|
+
tm.from_dict({"type": "doc", "content": [{"type": "callout"}]})
|
|
92
|
+
except tm.UnknownTypeError as e:
|
|
93
|
+
e.type # "callout"
|
|
94
|
+
e.kind # "node" or "mark"
|
|
95
|
+
e.path # "content[0]"
|
|
96
|
+
e.detail # "unknown node type 'callout' at content[0]"
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Known node types: `doc`, `paragraph`, `text`, `hardBreak`, `heading`,
|
|
100
|
+
`blockquote`, `codeBlock`, `horizontalRule`, `bulletList`, `orderedList`,
|
|
101
|
+
`listItem`, `taskList`, `taskItem`, `table`, `tableRow`, `tableHeader`,
|
|
102
|
+
`tableCell`, `image`, `htmlBlock`, `htmlInline`. Known marks: `bold`,
|
|
103
|
+
`italic`, `strike`, `code`, `link`.
|
|
104
|
+
|
|
105
|
+
## Defined normalizations
|
|
106
|
+
|
|
107
|
+
Markdown cannot represent every Tiptap document exactly. Rather than emitting
|
|
108
|
+
markdown that reparses into a different structure, marktip applies these
|
|
109
|
+
deterministic (and idempotent) normalizations at serialization time:
|
|
110
|
+
|
|
111
|
+
- **Hard breaks in headings** — ATX headings are single-line, so a `hardBreak`
|
|
112
|
+
(or a literal newline carried over from setext input) inside a heading
|
|
113
|
+
serializes as a single space: `heading("a", hardBreak, "b")` → `# a b`.
|
|
114
|
+
- **Multi-block table cells** — blocks inside a cell are joined with `<br>`
|
|
115
|
+
and the cell is flattened to one line; two paragraphs in a cell reparse as
|
|
116
|
+
one paragraph containing a `hardBreak`.
|
|
117
|
+
- **Headerless tables** — GFM tables require a header row, so the first row is
|
|
118
|
+
always emitted as the header; a leading `tableCell` row reparses as
|
|
119
|
+
`tableHeader`.
|
|
120
|
+
- **Heading levels** — clamped to 1–6 at serialization (`0` → `#`, `7` →
|
|
121
|
+
`######`).
|
|
122
|
+
- **Emphasis boundary whitespace** — whitespace touching an emphasis delimiter
|
|
123
|
+
has no valid markdown form and is expelled outside the marks
|
|
124
|
+
(`bold("굵게 ")` → `**굵게** `), cf. prosemirror-markdown's
|
|
125
|
+
`expelEnclosingWhitespace`.
|
|
126
|
+
- **Adjacent same-family lists** — consecutive lists of the same family
|
|
127
|
+
alternate markers (`-`/`*`, `1.`/`1)`) so they stay separate lists on
|
|
128
|
+
reparse instead of merging (which would renumber ordered items or spread
|
|
129
|
+
task checkboxes).
|
|
130
|
+
|
|
131
|
+
## Development
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
python -m pip install .[test]
|
|
135
|
+
python -m pytest
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
For a direct local CMake build:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \
|
|
142
|
+
-Dpybind11_DIR="$(python -m pybind11 --cmakedir)"
|
|
143
|
+
cmake --build build
|
|
144
|
+
PYTHONPATH=python python -m pytest
|
|
145
|
+
PYTHONPATH=python python scripts/benchmark.py
|
|
146
|
+
```
|
marktip-0.3.0/README.md
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# marktip
|
|
2
|
+
|
|
3
|
+
Fast C++/MD4C Markdown conversion for Tiptap-style JSON.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
python -m pip install marktip
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Release wheels are built for common CPython versions on Linux, macOS, and
|
|
12
|
+
Windows. If a wheel is not available for a platform, pip can build from the
|
|
13
|
+
source distribution with a C++17 compiler and standard Python build tooling.
|
|
14
|
+
|
|
15
|
+
## Usage
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
import marktip as tm
|
|
19
|
+
|
|
20
|
+
doc = tm.from_markdown("# Hello")
|
|
21
|
+
ast = doc.to_dict()
|
|
22
|
+
markdown = doc.to_markdown()
|
|
23
|
+
|
|
24
|
+
doc = tm.from_dict(ast)
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
`from_markdown` follows GFM/CommonMark by default. Pass `cjk_friendly=True` to
|
|
28
|
+
relax the emphasis and strikethrough rules so delimiters next to CJK text still
|
|
29
|
+
open and close (e.g. `**볼드**은` parses as bold), a non-standard extension that
|
|
30
|
+
is off by default:
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
doc = tm.from_markdown("**볼드**은 강조", cjk_friendly=True)
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Strikethrough follows the GFM reference implementations (cmark-gfm/micromark):
|
|
37
|
+
intra-word `~~` strikes (`a~~b~~c`), including next to CJK letters
|
|
38
|
+
(`~~삭제~~은`). `cjk_friendly` still matters for `*`/`_`, and for `~` in
|
|
39
|
+
punctuation-adjacent cases such as `~~"인용"~~라고`.
|
|
40
|
+
|
|
41
|
+
Pass `html=False` to parse raw HTML as literal text instead of
|
|
42
|
+
`htmlBlock`/`htmlInline` nodes. Content is preserved (the serializer escapes
|
|
43
|
+
it), and `<br>` inside table cells still maps to `hardBreak` so marktip's own
|
|
44
|
+
table output round-trips:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
doc = tm.from_markdown("a <u>x</u> b", html=False)
|
|
48
|
+
# paragraph with the literal text "a <u>x</u> b"
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
marktip targets GFM core syntax and canonical Markdown output rather than
|
|
52
|
+
byte-identical source preservation.
|
|
53
|
+
|
|
54
|
+
## Schema enforcement
|
|
55
|
+
|
|
56
|
+
The Tiptap-side schema is closed: every node/mark type must have a markdown
|
|
57
|
+
mapping. `from_dict` rejects unknown types instead of silently dropping
|
|
58
|
+
content, raising `marktip.UnknownTypeError` — a `ValueError` subclass with
|
|
59
|
+
structured fields for programmatic relay:
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
try:
|
|
63
|
+
tm.from_dict({"type": "doc", "content": [{"type": "callout"}]})
|
|
64
|
+
except tm.UnknownTypeError as e:
|
|
65
|
+
e.type # "callout"
|
|
66
|
+
e.kind # "node" or "mark"
|
|
67
|
+
e.path # "content[0]"
|
|
68
|
+
e.detail # "unknown node type 'callout' at content[0]"
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Known node types: `doc`, `paragraph`, `text`, `hardBreak`, `heading`,
|
|
72
|
+
`blockquote`, `codeBlock`, `horizontalRule`, `bulletList`, `orderedList`,
|
|
73
|
+
`listItem`, `taskList`, `taskItem`, `table`, `tableRow`, `tableHeader`,
|
|
74
|
+
`tableCell`, `image`, `htmlBlock`, `htmlInline`. Known marks: `bold`,
|
|
75
|
+
`italic`, `strike`, `code`, `link`.
|
|
76
|
+
|
|
77
|
+
## Defined normalizations
|
|
78
|
+
|
|
79
|
+
Markdown cannot represent every Tiptap document exactly. Rather than emitting
|
|
80
|
+
markdown that reparses into a different structure, marktip applies these
|
|
81
|
+
deterministic (and idempotent) normalizations at serialization time:
|
|
82
|
+
|
|
83
|
+
- **Hard breaks in headings** — ATX headings are single-line, so a `hardBreak`
|
|
84
|
+
(or a literal newline carried over from setext input) inside a heading
|
|
85
|
+
serializes as a single space: `heading("a", hardBreak, "b")` → `# a b`.
|
|
86
|
+
- **Multi-block table cells** — blocks inside a cell are joined with `<br>`
|
|
87
|
+
and the cell is flattened to one line; two paragraphs in a cell reparse as
|
|
88
|
+
one paragraph containing a `hardBreak`.
|
|
89
|
+
- **Headerless tables** — GFM tables require a header row, so the first row is
|
|
90
|
+
always emitted as the header; a leading `tableCell` row reparses as
|
|
91
|
+
`tableHeader`.
|
|
92
|
+
- **Heading levels** — clamped to 1–6 at serialization (`0` → `#`, `7` →
|
|
93
|
+
`######`).
|
|
94
|
+
- **Emphasis boundary whitespace** — whitespace touching an emphasis delimiter
|
|
95
|
+
has no valid markdown form and is expelled outside the marks
|
|
96
|
+
(`bold("굵게 ")` → `**굵게** `), cf. prosemirror-markdown's
|
|
97
|
+
`expelEnclosingWhitespace`.
|
|
98
|
+
- **Adjacent same-family lists** — consecutive lists of the same family
|
|
99
|
+
alternate markers (`-`/`*`, `1.`/`1)`) so they stay separate lists on
|
|
100
|
+
reparse instead of merging (which would renumber ordered items or spread
|
|
101
|
+
task checkboxes).
|
|
102
|
+
|
|
103
|
+
## Development
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
python -m pip install .[test]
|
|
107
|
+
python -m pytest
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
For a direct local CMake build:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \
|
|
114
|
+
-Dpybind11_DIR="$(python -m pybind11 --cmakedir)"
|
|
115
|
+
cmake --build build
|
|
116
|
+
PYTHONPATH=python python -m pytest
|
|
117
|
+
PYTHONPATH=python python scripts/benchmark.py
|
|
118
|
+
```
|
|
@@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "marktip"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Fast C++/MD4C Markdown parser and canonical Markdown serializer for Tiptap-style JSON"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Fast Markdown conversion for Tiptap-style JSON."""
|
|
2
2
|
|
|
3
3
|
try:
|
|
4
|
-
from ._core import Document, __version__, from_dict, from_markdown
|
|
4
|
+
from ._core import Document, UnknownTypeError, __version__, from_dict, from_markdown
|
|
5
5
|
except ImportError: # pragma: no cover - development-only CMake build fallback
|
|
6
6
|
import importlib.util
|
|
7
7
|
import pathlib
|
|
@@ -21,7 +21,8 @@ except ImportError: # pragma: no cover - development-only CMake build fallback
|
|
|
21
21
|
|
|
22
22
|
__version__ = _module.__version__
|
|
23
23
|
Document = _module.Document
|
|
24
|
+
UnknownTypeError = _module.UnknownTypeError
|
|
24
25
|
from_dict = _module.from_dict
|
|
25
26
|
from_markdown = _module.from_markdown
|
|
26
27
|
|
|
27
|
-
__all__ = ["__version__", "Document", "from_markdown", "from_dict"]
|
|
28
|
+
__all__ = ["__version__", "Document", "UnknownTypeError", "from_markdown", "from_dict"]
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
#include "ast.h"
|
|
2
2
|
|
|
3
|
+
#include <algorithm>
|
|
4
|
+
#include <iterator>
|
|
3
5
|
#include <stdexcept>
|
|
4
6
|
|
|
5
7
|
namespace py = pybind11;
|
|
@@ -7,6 +9,50 @@ namespace py = pybind11;
|
|
|
7
9
|
namespace marktip {
|
|
8
10
|
namespace {
|
|
9
11
|
|
|
12
|
+
// The closed schema: every node/mark type the parser can emit and the
|
|
13
|
+
// serializer understands. Sorted byte-wise (uppercase sorts before lowercase,
|
|
14
|
+
// e.g. tableCell < tableHeader < tableRow) — required by binary_search; do not
|
|
15
|
+
// "alphabetize" case-insensitively.
|
|
16
|
+
constexpr std::string_view kKnownNodeTypes[] = {
|
|
17
|
+
"blockquote", "bulletList", "codeBlock", "doc", "hardBreak", "heading", "horizontalRule",
|
|
18
|
+
"htmlBlock", "htmlInline", "image", "listItem", "orderedList", "paragraph", "table",
|
|
19
|
+
"tableCell", "tableHeader", "tableRow", "taskItem", "taskList", "text",
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
constexpr std::string_view kKnownMarkTypes[] = {"bold", "code", "italic", "link", "strike"};
|
|
23
|
+
|
|
24
|
+
bool is_known_node_type(std::string_view type) {
|
|
25
|
+
return std::binary_search(std::begin(kKnownNodeTypes), std::end(kKnownNodeTypes), type);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
bool is_known_mark_type(std::string_view type) {
|
|
29
|
+
return std::binary_search(std::begin(kKnownMarkTypes), std::end(kKnownMarkTypes), type);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Breadcrumb into the input dict, e.g. content[0].content[2].marks[1].
|
|
33
|
+
// Segments are static strings so pushing a frame never allocates; the path
|
|
34
|
+
// string is only rendered when an error is actually thrown.
|
|
35
|
+
struct PathFrame {
|
|
36
|
+
const char* segment; // "content" | "marks"
|
|
37
|
+
std::size_t index;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
using PathStack = std::vector<PathFrame>;
|
|
41
|
+
|
|
42
|
+
std::string format_path(const PathStack& path) {
|
|
43
|
+
std::string out;
|
|
44
|
+
for (std::size_t i = 0; i < path.size(); ++i) {
|
|
45
|
+
if (i != 0) {
|
|
46
|
+
out.push_back('.');
|
|
47
|
+
}
|
|
48
|
+
out += path[i].segment;
|
|
49
|
+
out.push_back('[');
|
|
50
|
+
out += std::to_string(path[i].index);
|
|
51
|
+
out.push_back(']');
|
|
52
|
+
}
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
|
|
10
56
|
bool dict_contains(py::dict dict, const char* key) {
|
|
11
57
|
return PyMapping_HasKeyString(dict.ptr(), key) == 1;
|
|
12
58
|
}
|
|
@@ -124,9 +170,12 @@ py::dict node_to_py(const Document& document, std::size_t index) {
|
|
|
124
170
|
return out;
|
|
125
171
|
}
|
|
126
172
|
|
|
127
|
-
Mark mark_from_py(py::dict mark_dict) {
|
|
173
|
+
Mark mark_from_py(py::dict mark_dict, const PathStack& path) {
|
|
128
174
|
Mark mark;
|
|
129
175
|
mark.type = required_type(mark_dict, "mark");
|
|
176
|
+
if (!is_known_mark_type(mark.type)) {
|
|
177
|
+
throw UnknownTypeError("mark", mark.type, format_path(path));
|
|
178
|
+
}
|
|
130
179
|
|
|
131
180
|
py::object attrs = dict_get(mark_dict, "attrs");
|
|
132
181
|
if (!attrs.is_none()) {
|
|
@@ -137,13 +186,16 @@ Mark mark_from_py(py::dict mark_dict) {
|
|
|
137
186
|
}
|
|
138
187
|
|
|
139
188
|
void fill_node_from_py(Document& document, std::size_t index, py::dict node_dict, std::string_view context,
|
|
140
|
-
std::size_t depth) {
|
|
189
|
+
std::size_t depth, PathStack& path) {
|
|
141
190
|
if (depth > kMaxNodeDepth) {
|
|
142
191
|
throw std::invalid_argument("node content nesting exceeds maximum depth");
|
|
143
192
|
}
|
|
144
193
|
|
|
145
194
|
Node node;
|
|
146
195
|
node.type = required_type(node_dict, context);
|
|
196
|
+
if (!is_known_node_type(node.type)) {
|
|
197
|
+
throw UnknownTypeError("node", node.type, format_path(path));
|
|
198
|
+
}
|
|
147
199
|
|
|
148
200
|
if (node.type == "text") {
|
|
149
201
|
node.text = py_to_string(dict_get(node_dict, "text"));
|
|
@@ -157,8 +209,12 @@ void fill_node_from_py(Document& document, std::size_t index, py::dict node_dict
|
|
|
157
209
|
py::object marks = dict_get(node_dict, "marks");
|
|
158
210
|
if (!marks.is_none()) {
|
|
159
211
|
py::list mark_list = expect_list(marks, "node marks");
|
|
212
|
+
std::size_t mark_index = 0;
|
|
160
213
|
for (py::handle mark_handle : mark_list) {
|
|
161
|
-
|
|
214
|
+
path.push_back({"marks", mark_index});
|
|
215
|
+
node.marks.push_back(mark_from_py(expect_dict(mark_handle, "mark"), path));
|
|
216
|
+
path.pop_back();
|
|
217
|
+
++mark_index;
|
|
162
218
|
}
|
|
163
219
|
}
|
|
164
220
|
|
|
@@ -170,9 +226,13 @@ void fill_node_from_py(Document& document, std::size_t index, py::dict node_dict
|
|
|
170
226
|
}
|
|
171
227
|
|
|
172
228
|
py::list content_list = expect_list(content, "node content");
|
|
229
|
+
std::size_t child_pos = 0;
|
|
173
230
|
for (py::handle child_handle : content_list) {
|
|
174
231
|
std::size_t child_index = document.append_child(index, Node{});
|
|
175
|
-
|
|
232
|
+
path.push_back({"content", child_pos});
|
|
233
|
+
fill_node_from_py(document, child_index, expect_dict(child_handle, "content child"), "node", depth + 1, path);
|
|
234
|
+
path.pop_back();
|
|
235
|
+
++child_pos;
|
|
176
236
|
}
|
|
177
237
|
}
|
|
178
238
|
|
|
@@ -336,7 +396,9 @@ Document from_dict_py(py::dict root) {
|
|
|
336
396
|
}
|
|
337
397
|
|
|
338
398
|
Document document;
|
|
339
|
-
|
|
399
|
+
PathStack path;
|
|
400
|
+
path.reserve(32);
|
|
401
|
+
fill_node_from_py(document, 0, root, "root node", 0, path);
|
|
340
402
|
return document;
|
|
341
403
|
}
|
|
342
404
|
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
#include <pybind11/pybind11.h>
|
|
4
4
|
|
|
5
5
|
#include <cstddef>
|
|
6
|
+
#include <stdexcept>
|
|
6
7
|
#include <string>
|
|
7
8
|
#include <string_view>
|
|
8
9
|
#include <utility>
|
|
@@ -46,6 +47,27 @@ struct Node {
|
|
|
46
47
|
std::vector<std::size_t> content;
|
|
47
48
|
};
|
|
48
49
|
|
|
50
|
+
// Raised by from_dict when a node or mark type is outside the closed schema.
|
|
51
|
+
// Subclasses std::invalid_argument so it surfaces as (a subclass of) ValueError.
|
|
52
|
+
class UnknownTypeError : public std::invalid_argument {
|
|
53
|
+
public:
|
|
54
|
+
UnknownTypeError(std::string kind, std::string type_name, std::string path)
|
|
55
|
+
: std::invalid_argument("unknown " + kind + " type '" + type_name + "' at " +
|
|
56
|
+
(path.empty() ? std::string("root") : path)),
|
|
57
|
+
kind_(std::move(kind)),
|
|
58
|
+
type_name_(std::move(type_name)),
|
|
59
|
+
path_(std::move(path)) {}
|
|
60
|
+
|
|
61
|
+
const std::string& kind() const noexcept { return kind_; }
|
|
62
|
+
const std::string& type_name() const noexcept { return type_name_; }
|
|
63
|
+
const std::string& path() const noexcept { return path_; }
|
|
64
|
+
|
|
65
|
+
private:
|
|
66
|
+
std::string kind_; // "node" | "mark"
|
|
67
|
+
std::string type_name_;
|
|
68
|
+
std::string path_; // "content[0].marks[1]"; empty = root
|
|
69
|
+
};
|
|
70
|
+
|
|
49
71
|
void set_attr(AttrList& attrs, std::string key, AttrValue value);
|
|
50
72
|
const AttrValue* find_attr(const AttrList& attrs, std::string_view key);
|
|
51
73
|
std::string attr_string(const AttrList& attrs, std::string_view key, std::string fallback = {});
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
#include <pybind11/gil_safe_call_once.h>
|
|
2
|
+
#include <pybind11/pybind11.h>
|
|
3
|
+
|
|
4
|
+
#include <exception>
|
|
5
|
+
#include <string>
|
|
6
|
+
|
|
7
|
+
#include "ast.h"
|
|
8
|
+
#include "parser.h"
|
|
9
|
+
#include "serializer.h"
|
|
10
|
+
|
|
11
|
+
namespace py = pybind11;
|
|
12
|
+
|
|
13
|
+
#ifndef MARKTIP_VERSION
|
|
14
|
+
#define MARKTIP_VERSION "0.3.0"
|
|
15
|
+
#endif
|
|
16
|
+
|
|
17
|
+
PYBIND11_MODULE(_core, module) {
|
|
18
|
+
module.doc() = "Fast MD4C-backed Markdown conversion with a C++ Document AST";
|
|
19
|
+
module.attr("__version__") = MARKTIP_VERSION;
|
|
20
|
+
|
|
21
|
+
// ValueError subclass carrying structured fields (.type, .path, .kind,
|
|
22
|
+
// .detail) so callers can relay schema violations programmatically.
|
|
23
|
+
PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store<py::object> unknown_type_error_storage;
|
|
24
|
+
unknown_type_error_storage.call_once_and_store_result([&]() -> py::object {
|
|
25
|
+
py::object exc = py::exception<marktip::UnknownTypeError>(module, "UnknownTypeError", PyExc_ValueError);
|
|
26
|
+
exc.attr("__module__") = "marktip._core";
|
|
27
|
+
return exc;
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
py::register_exception_translator([](std::exception_ptr p) {
|
|
31
|
+
if (!p) {
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
try {
|
|
35
|
+
std::rethrow_exception(p);
|
|
36
|
+
} catch (const marktip::UnknownTypeError& e) {
|
|
37
|
+
const py::object& exc_type = unknown_type_error_storage.get_stored();
|
|
38
|
+
py::object exc = exc_type(e.what());
|
|
39
|
+
exc.attr("type") = e.type_name();
|
|
40
|
+
exc.attr("path") = e.path();
|
|
41
|
+
exc.attr("kind") = e.kind();
|
|
42
|
+
exc.attr("detail") = std::string(e.what());
|
|
43
|
+
PyErr_SetObject(exc_type.ptr(), exc.ptr());
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
py::class_<marktip::Document>(module, "Document")
|
|
48
|
+
.def("to_dict", &marktip::Document::to_dict, "Convert the document AST to a Tiptap-style JSON dict.")
|
|
49
|
+
.def("to_markdown",
|
|
50
|
+
[](const marktip::Document& document) {
|
|
51
|
+
return marktip::to_markdown(document);
|
|
52
|
+
},
|
|
53
|
+
"Serialize the document AST to canonical Markdown.");
|
|
54
|
+
|
|
55
|
+
module.def("from_markdown", &marktip::from_markdown_py, py::arg("markdown"), py::arg("cjk_friendly") = false,
|
|
56
|
+
py::arg("html") = true,
|
|
57
|
+
"Parse Markdown into a Document. Set cjk_friendly=True to relax the emphasis rules around CJK text "
|
|
58
|
+
"(non-standard extension; the default follows GFM/CommonMark exactly). Set html=False to parse raw "
|
|
59
|
+
"HTML as literal text instead of htmlBlock/htmlInline nodes; <br> inside table cells still maps to "
|
|
60
|
+
"hardBreak.");
|
|
61
|
+
module.def("from_dict", &marktip::from_dict_py, py::arg("ast"), "Build a Document from a Tiptap-style JSON dict.");
|
|
62
|
+
}
|
|
@@ -147,8 +147,8 @@ std::string align_to_string(MD_ALIGN align) {
|
|
|
147
147
|
|
|
148
148
|
class AstBuilder {
|
|
149
149
|
public:
|
|
150
|
-
explicit AstBuilder(std::size_t input_size = 0)
|
|
151
|
-
: document_(std::max<std::size_t>(32, input_size / 24)) {
|
|
150
|
+
explicit AstBuilder(std::size_t input_size = 0, bool html = true)
|
|
151
|
+
: document_(std::max<std::size_t>(32, input_size / 24)), html_(html) {
|
|
152
152
|
stack_.push_back(0);
|
|
153
153
|
}
|
|
154
154
|
|
|
@@ -412,6 +412,7 @@ private:
|
|
|
412
412
|
std::vector<std::size_t> stack_;
|
|
413
413
|
std::vector<Mark> active_marks_;
|
|
414
414
|
std::string error_;
|
|
415
|
+
bool html_ = true;
|
|
415
416
|
|
|
416
417
|
Node& current() {
|
|
417
418
|
return document_.node(stack_.back());
|
|
@@ -517,12 +518,19 @@ private:
|
|
|
517
518
|
}
|
|
518
519
|
|
|
519
520
|
// Table cells cannot hold literal newlines, so hard breaks in cells are
|
|
520
|
-
// serialized as <br>; map them back to hardBreak when parsing.
|
|
521
|
+
// serialized as <br>; map them back to hardBreak when parsing. This
|
|
522
|
+
// must stay ahead of the html_ gate so marktip's own cell output
|
|
523
|
+
// round-trips even with html disabled.
|
|
521
524
|
if (is_br_tag(value) && (current().type == "tableCell" || current().type == "tableHeader")) {
|
|
522
525
|
add_hard_break();
|
|
523
526
|
return;
|
|
524
527
|
}
|
|
525
528
|
|
|
529
|
+
if (!html_) {
|
|
530
|
+
add_text(value);
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
|
|
526
534
|
std::size_t parent_index = stack_.back();
|
|
527
535
|
Node& parent = document_.node(parent_index);
|
|
528
536
|
if (!parent.content.empty()) {
|
|
@@ -704,11 +712,16 @@ int text_callback(MD_TEXTTYPE type, const MD_CHAR* text, MD_SIZE size, void* use
|
|
|
704
712
|
}
|
|
705
713
|
}
|
|
706
714
|
|
|
707
|
-
Document parse_to_document(const std::string& markdown, bool cjk_friendly) {
|
|
708
|
-
AstBuilder builder(markdown.size());
|
|
715
|
+
Document parse_to_document(const std::string& markdown, bool cjk_friendly, bool html) {
|
|
716
|
+
AstBuilder builder(markdown.size(), html);
|
|
709
717
|
MD_PARSER parser {};
|
|
710
718
|
parser.abi_version = 0;
|
|
711
|
-
|
|
719
|
+
// With html=false, block-level raw HTML degrades to paragraph text via
|
|
720
|
+
// MD_FLAG_NOHTMLBLOCKS. Inline HTML spans stay recognized (no
|
|
721
|
+
// MD_FLAG_NOHTMLSPANS) so the <br>-in-cell hardBreak mapping keeps
|
|
722
|
+
// working; AstBuilder turns the remaining spans into literal text.
|
|
723
|
+
parser.flags = MD_DIALECT_GITHUB | (cjk_friendly ? MD_FLAG_CJKFRIENDLYEMPHASIS : 0) |
|
|
724
|
+
(html ? 0 : MD_FLAG_NOHTMLBLOCKS);
|
|
712
725
|
parser.enter_block = enter_block_callback;
|
|
713
726
|
parser.leave_block = leave_block_callback;
|
|
714
727
|
parser.enter_span = enter_span_callback;
|
|
@@ -729,7 +742,7 @@ Document parse_to_document(const std::string& markdown, bool cjk_friendly) {
|
|
|
729
742
|
|
|
730
743
|
} // namespace
|
|
731
744
|
|
|
732
|
-
Document from_markdown_py(py::object markdown, bool cjk_friendly) {
|
|
745
|
+
Document from_markdown_py(py::object markdown, bool cjk_friendly, bool html) {
|
|
733
746
|
std::string input;
|
|
734
747
|
if (py::isinstance<py::str>(markdown) || py::isinstance<py::bytes>(markdown)) {
|
|
735
748
|
input = py::cast<std::string>(markdown);
|
|
@@ -740,7 +753,7 @@ Document from_markdown_py(py::object markdown, bool cjk_friendly) {
|
|
|
740
753
|
Document document;
|
|
741
754
|
{
|
|
742
755
|
py::gil_scoped_release release;
|
|
743
|
-
document = parse_to_document(input, cjk_friendly);
|
|
756
|
+
document = parse_to_document(input, cjk_friendly, html);
|
|
744
757
|
}
|
|
745
758
|
return document;
|
|
746
759
|
}
|