marktip 0.1.0__tar.gz → 0.2.0__tar.gz

{marktip-0.1.0 → marktip-0.2.0}/.github/workflows/build-distributions.yml

@@ -63,3 +63,27 @@ jobs:
         with:
           name: marktip-wheels-${{ matrix.os }}
           path: wheelhouse/*.whl
+
+  publish:
+    name: Publish to PyPI
+    needs: [sdist, wheels]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/marktip
+
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download built distributions
+        uses: actions/download-artifact@v6
+        with:
+          pattern: marktip-*
+          path: dist
+          merge-multiple: true
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

{marktip-0.1.0 → marktip-0.2.0}/CMakeLists.txt

@@ -7,7 +7,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
 if(NOT DEFINED SKBUILD_PROJECT_VERSION)
-  set(SKBUILD_PROJECT_VERSION "0.1.0")
+  set(SKBUILD_PROJECT_VERSION "0.2.0")
 endif()
 
 set(PYBIND11_FINDPYTHON ON)

{marktip-0.1.0 → marktip-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: marktip
-Version: 0.1.0
+Version: 0.2.0
 Summary: Fast C++/MD4C Markdown parser and canonical Markdown serializer for Tiptap-style JSON
 Keywords: markdown,tiptap,md4c,parser
 Author: Saneaven (Minjae Kyung)
@@ -52,8 +52,17 @@ markdown = doc.to_markdown()
 doc = tm.from_dict(ast)
 ```
 
-The first version targets GFM core syntax and canonical Markdown output rather
-than byte-identical source preservation.
+`from_markdown` follows GFM/CommonMark by default. Pass `cjk_friendly=True` to
+relax the emphasis and strikethrough rules so delimiters next to CJK text still
+open and close (e.g. `**볼드**은` parses as bold), a non-standard extension that
+is off by default:
+
+```python
+doc = tm.from_markdown("**볼드**은 강조", cjk_friendly=True)
+```
+
+marktip targets GFM core syntax and canonical Markdown output rather than
+byte-identical source preservation.
 
 ## Development
 

{marktip-0.1.0 → marktip-0.2.0}/README.md

@@ -24,8 +24,17 @@ markdown = doc.to_markdown()
 doc = tm.from_dict(ast)
 ```
 
-The first version targets GFM core syntax and canonical Markdown output rather
-than byte-identical source preservation.
+`from_markdown` follows GFM/CommonMark by default. Pass `cjk_friendly=True` to
+relax the emphasis and strikethrough rules so delimiters next to CJK text still
+open and close (e.g. `**볼드**은` parses as bold), a non-standard extension that
+is off by default:
+
+```python
+doc = tm.from_markdown("**볼드**은 강조", cjk_friendly=True)
+```
+
+marktip targets GFM core syntax and canonical Markdown output rather than
+byte-identical source preservation.
 
 ## Development
 

{marktip-0.1.0 → marktip-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"
 
 [project]
 name = "marktip"
-version = "0.1.0"
+version = "0.2.0"
 description = "Fast C++/MD4C Markdown parser and canonical Markdown serializer for Tiptap-style JSON"
 readme = "README.md"
 requires-python = ">=3.9"

{marktip-0.1.0 → marktip-0.2.0}/src/marktip.cpp

@@ -6,7 +6,7 @@
 namespace py = pybind11;
 
 #ifndef MARKTIP_VERSION
-#define MARKTIP_VERSION "0.1.0"
+#define MARKTIP_VERSION "0.2.0"
 #endif
 
 PYBIND11_MODULE(_core, module) {
@@ -21,6 +21,8 @@ PYBIND11_MODULE(_core, module) {
              },
              "Serialize the document AST to canonical Markdown.");
 
-    module.def("from_markdown", &marktip::from_markdown_py, py::arg("markdown"), "Parse Markdown into a Document.");
+    module.def("from_markdown", &marktip::from_markdown_py, py::arg("markdown"), py::arg("cjk_friendly") = false,
+               "Parse Markdown into a Document. Set cjk_friendly=True to relax the emphasis rules around CJK text "
+               "(non-standard extension; the default follows GFM/CommonMark exactly).");
     module.def("from_dict", &marktip::from_dict_py, py::arg("ast"), "Build a Document from a Tiptap-style JSON dict.");
 }

{marktip-0.1.0 → marktip-0.2.0}/src/parser.cpp

@@ -704,11 +704,11 @@ int text_callback(MD_TEXTTYPE type, const MD_CHAR* text, MD_SIZE size, void* use
     }
 }
 
-Document parse_to_document(const std::string& markdown) {
+Document parse_to_document(const std::string& markdown, bool cjk_friendly) {
     AstBuilder builder(markdown.size());
     MD_PARSER parser {};
     parser.abi_version = 0;
-    parser.flags = MD_DIALECT_GITHUB;
+    parser.flags = MD_DIALECT_GITHUB | (cjk_friendly ? MD_FLAG_CJKFRIENDLYEMPHASIS : 0);
     parser.enter_block = enter_block_callback;
     parser.leave_block = leave_block_callback;
     parser.enter_span = enter_span_callback;
@@ -729,7 +729,7 @@ Document parse_to_document(const std::string& markdown) {
 
 }  // namespace
 
-Document from_markdown_py(py::object markdown) {
+Document from_markdown_py(py::object markdown, bool cjk_friendly) {
     std::string input;
     if (py::isinstance<py::str>(markdown) || py::isinstance<py::bytes>(markdown)) {
         input = py::cast<std::string>(markdown);
@@ -740,7 +740,7 @@ Document from_markdown_py(py::object markdown) {
     Document document;
     {
         py::gil_scoped_release release;
-        document = parse_to_document(input);
+        document = parse_to_document(input, cjk_friendly);
     }
     return document;
 }

{marktip-0.1.0 → marktip-0.2.0}/src/parser.h

@@ -6,6 +6,6 @@
 
 namespace marktip {
 
-Document from_markdown_py(pybind11::object markdown);
+Document from_markdown_py(pybind11::object markdown, bool cjk_friendly);
 
 }  // namespace marktip

{marktip-0.1.0 → marktip-0.2.0}/src/serializer.cpp

@@ -211,6 +211,150 @@ std::string apply_marks(std::string rendered, const std::vector<Mark>& marks, bo
     return rendered;
 }
 
+// A run of consecutive inline nodes sharing the same marks, with its content
+// already rendered. `marks` excludes "code" (code spans are atomic per run).
+struct InlineRun {
+    std::vector<Mark> marks;
+    std::string content;
+};
+
+bool is_emphasis_mark(const Mark& mark) {
+    return mark.type == "bold" || mark.type == "italic" || mark.type == "strike";
+}
+
+std::size_t common_prefix(const std::vector<Mark>& a, const std::vector<Mark>& b) {
+    std::size_t limit = std::min(a.size(), b.size());
+    std::size_t i = 0;
+    while (i < limit && a[i] == b[i]) {
+        ++i;
+    }
+    return i;
+}
+
+std::size_t first_emphasis_at_or_after(const std::vector<Mark>& marks, std::size_t from) {
+    for (std::size_t i = from; i < marks.size(); ++i) {
+        if (is_emphasis_mark(marks[i])) {
+            return i;
+        }
+    }
+    return marks.size();
+}
+
+std::string open_delimiter(const Mark& mark) {
+    if (mark.type == "bold") {
+        return "**";
+    }
+    if (mark.type == "italic") {
+        return "*";
+    }
+    if (mark.type == "strike") {
+        return "~~";
+    }
+    if (mark.type == "link") {
+        return "[";
+    }
+    return "";
+}
+
+std::string close_delimiter(const Mark& mark) {
+    if (mark.type == "bold") {
+        return "**";
+    }
+    if (mark.type == "italic") {
+        return "*";
+    }
+    if (mark.type == "strike") {
+        return "~~";
+    }
+    if (mark.type == "link") {
+        std::string href = mark_attr_string(mark, "href");
+        std::string title = mark_attr_string(mark, "title");
+        return "](" + escape_link_destination(href) +
+               (title.empty() ? "" : " \"" + escape_title(title) + "\"") + ")";
+    }
+    return "";
+}
+
+// Whitespace adjacent to an emphasis delimiter prevents the delimiter from
+// opening/closing and no markdown syntax can express it, so such whitespace is
+// moved outside the emphasis marks (cf. prosemirror-markdown
+// expelEnclosingWhitespace). It stays inside enclosing non-emphasis marks
+// such as links, which have no whitespace restriction.
+void expel_boundary_whitespace(std::vector<InlineRun>& runs) {
+    static const char* const kWhitespace = " \t\n";
+    const std::vector<Mark> no_marks;
+    std::vector<InlineRun> result;
+
+    for (std::size_t i = 0; i < runs.size(); ++i) {
+        const std::vector<Mark>& prev = i > 0 ? runs[i - 1].marks : no_marks;
+        const std::vector<Mark>& next = i + 1 < runs.size() ? runs[i + 1].marks : no_marks;
+        InlineRun& run = runs[i];
+
+        std::size_t opening_emphasis = first_emphasis_at_or_after(run.marks, common_prefix(prev, run.marks));
+        if (opening_emphasis < run.marks.size()) {
+            std::size_t body = run.content.find_first_not_of(kWhitespace);
+            std::size_t lead_len = body == std::string::npos ? run.content.size() : body;
+            if (lead_len > 0) {
+                InlineRun lead;
+                lead.marks.assign(run.marks.begin(), run.marks.begin() + opening_emphasis);
+                lead.content = run.content.substr(0, lead_len);
+                run.content.erase(0, lead_len);
+                result.push_back(std::move(lead));
+            }
+        }
+
+        InlineRun tail;
+        bool has_tail = false;
+        std::size_t closing_emphasis = first_emphasis_at_or_after(run.marks, common_prefix(run.marks, next));
+        if (closing_emphasis < run.marks.size() && !run.content.empty()) {
+            std::size_t body = run.content.find_last_not_of(kWhitespace);
+            std::size_t keep = body == std::string::npos ? 0 : body + 1;
+            if (keep < run.content.size()) {
+                tail.marks.assign(run.marks.begin(), run.marks.begin() + closing_emphasis);
+                tail.content = run.content.substr(keep);
+                run.content.erase(keep);
+                has_tail = true;
+            }
+        }
+
+        if (!run.content.empty()) {
+            InlineRun kept;
+            kept.marks = run.marks;
+            kept.content = std::move(run.content);
+            result.push_back(std::move(kept));
+        }
+        if (has_tail) {
+            result.push_back(std::move(tail));
+        }
+    }
+
+    runs = std::move(result);
+}
+
+// Emit runs keeping shared marks open across run boundaries, so e.g.
+// [a: bold][b: bold+italic] becomes "**a*b***" rather than "**a*****b***".
+std::string emit_runs(const std::vector<InlineRun>& runs) {
+    std::string out;
+    std::vector<Mark> open;
+    for (const InlineRun& run : runs) {
+        std::size_t keep = common_prefix(open, run.marks);
+        while (open.size() > keep) {
+            out += close_delimiter(open.back());
+            open.pop_back();
+        }
+        for (std::size_t j = keep; j < run.marks.size(); ++j) {
+            out += open_delimiter(run.marks[j]);
+            open.push_back(run.marks[j]);
+        }
+        out += run.content;
+    }
+    while (!open.empty()) {
+        out += close_delimiter(open.back());
+        open.pop_back();
+    }
+    return out;
+}
+
 class MarkdownWriter {
 public:
     explicit MarkdownWriter(const Document& document) : document_(document) {}
@@ -483,9 +627,9 @@ private:
         return rendered;
     }
 
-    // Render one inline node without applying its marks. Marks are applied once
-    // around each run of consecutive nodes sharing the same marks, so a hard
-    // break inside e.g. bold yields "**a  \nb**" rather than "**a****  \n****b**".
+    // Render one inline node without applying its marks. Marks are emitted by
+    // emit_runs(), which keeps marks shared between consecutive runs open
+    // across the boundary.
     std::string render_inline_content(const Node& node, bool table_cell, bool in_code) {
         const std::string& type = node.type;
         if (type == "text") {
@@ -512,7 +656,13 @@ private:
             return render_inline(node, table_cell);
         }
 
-        std::string out;
+        std::vector<InlineRun> runs = build_runs(node, table_cell);
+        expel_boundary_whitespace(runs);
+        return emit_runs(runs);
+    }
+
+    std::vector<InlineRun> build_runs(const Node& node, bool table_cell) {
+        std::vector<InlineRun> runs;
         const std::vector<std::size_t>& content = node.content;
         std::size_t i = 0;
         while (i < content.size()) {
@@ -528,14 +678,18 @@ private:
                 group += render_inline_content(child, table_cell, in_code);
                 ++j;
             }
-            if (in_code) {
-                out += apply_marks(code_span(group), first.marks, true);
-            } else {
-                out += apply_marks(std::move(group), first.marks);
+
+            InlineRun run;
+            for (const Mark& mark : first.marks) {
+                if (mark.type != "code") {
+                    run.marks.push_back(mark);
+                }
             }
+            run.content = in_code ? code_span(group) : std::move(group);
+            runs.push_back(std::move(run));
             i = j;
         }
-        return out;
+        return runs;
     }
 
     std::string render_inline(const Node& node, bool table_cell = false) {

{marktip-0.1.0 → marktip-0.2.0}/tests/test_parse.py

@@ -81,5 +81,19 @@ def test_parse_maps_br_in_table_cell_to_hard_break():
     ]
 
 
+def test_parse_cjk_friendly_is_opt_in():
+    md = "**마크다운(Markdown)**은 표준이다"
+
+    relaxed = tm.from_markdown(md, cjk_friendly=True).to_dict()
+    assert relaxed["content"][0]["content"][0] == {
+        "type": "text",
+        "text": "마크다운(Markdown)",
+        "marks": [{"type": "bold"}],
+    }
+
+    standard = tm.from_markdown(md).to_dict()
+    assert standard["content"][0]["content"][0]["text"].startswith("**")
+
+
 def test_parse_accepts_bytes():
     assert tm.from_markdown(b"# Bytes").to_dict()["content"][0]["content"][0]["text"] == "Bytes"

marktip-0.2.0/tests/test_roundtrip.py

@@ -0,0 +1,246 @@
+import pytest
+
+import marktip as tm
+
+
+def assert_stable(markdown: str):
+    doc = tm.from_markdown(markdown)
+    serialized = doc.to_markdown()
+    assert tm.from_markdown(serialized).to_dict() == doc.to_dict()
+
+
+def test_roundtrip_core_blocks_and_marks():
+    assert_stable(
+        "# Title\n\n"
+        "A **bold** and *italic* paragraph with `code` and ~~strike~~.\n\n"
+        "> quoted\n\n"
+        "- one\n"
+        "- two\n\n"
+        "1. first\n"
+        "2. second\n"
+    )
+
+
+def test_roundtrip_gfm_table_task_list_and_code():
+    assert_stable(
+        "- [x] done\n"
+        "- [ ] todo\n\n"
+        "| A | B |\n"
+        "| :- | -: |\n"
+        "| x | y |\n\n"
+        "```python\n"
+        "print('ok')\n"
+        "```\n"
+    )
+
+
+def test_roundtrip_multi_block_list_items_and_loose_lists():
+    assert_stable("- first\n\n  second\n")
+    assert_stable("- a\n\n- b\n")
+
+
+def test_roundtrip_hard_break_inside_marks():
+    assert_stable("**a\\\nb**")
+    assert_stable("[a\\\nb](https://example.com)")
+
+
+def test_roundtrip_block_markers_escaped_in_text():
+    assert_stable("\\- not a list")
+    assert_stable("\\> not a quote")
+    assert_stable("1\\. not ordered")
+
+
+def test_roundtrip_ordered_task_list():
+    assert_stable("1. [x] done\n2. [ ] todo\n")
+
+
+def test_roundtrip_large_and_deep_smoke():
+    nested = "- root\n  - child\n    - grandchild\n"
+    large = "\n".join(f"Paragraph {i} with **bold** text." for i in range(200))
+    assert_stable(nested + "\n" + large)
+
+
+# ─────────────────────────────────────────────────────────────────────────
+# Adversarial round-trip regressions.
+#
+# The tests above check md -> AST -> md -> AST stability for markdown
+# input. The tests below cover the two remaining invariants of a canonical
+# pipeline:
+#
+#   1. AST identity:   from_dict -> to_markdown -> from_markdown == input
+#      (an editor document survives serialization; no mark silently drops)
+#   2. Byte idempotence: normalize(normalize(md)) == normalize(md)
+#      (re-saving an unchanged document never changes bytes)
+#
+# CJK text next to emphasis delimiters is not recognized by standard
+# CommonMark flanking rules; those cases round-trip only under the
+# non-standard `cjk_friendly=True` parsing extension, so the tests pass the
+# flag explicitly. Whitespace touching an emphasis boundary has no valid
+# markdown representation at all; the serializer expels it outside the
+# delimiters, and the tests assert that normalization.
+
+# ── helpers ──────────────────────────────────────────────────────────────
+
+
+def doc(*blocks):
+    return {"type": "doc", "content": list(blocks)}
+
+
+def p(*children):
+    return {"type": "paragraph", "content": list(children)}
+
+
+def text(value, *marks):
+    node = {"type": "text", "text": value}
+    if marks:
+        node["marks"] = [{"type": mark} for mark in marks]
+    return node
+
+
+def hard_break(*marks):
+    node = {"type": "hardBreak"}
+    if marks:
+        node["marks"] = [{"type": mark} for mark in marks]
+    return node
+
+
+def normalize(md):
+    return tm.from_markdown(md).to_markdown()
+
+
+def assert_ast_roundtrip(ast, cjk_friendly=False):
+    md = tm.from_dict(ast).to_markdown()
+    back = tm.from_markdown(md, cjk_friendly=cjk_friendly).to_dict()
+    assert back == ast, f"serialized markdown: {md!r}"
+
+
+# ── regression guards: currently passing, must stay green ───────────────
+
+
+def test_plain_cjk_bold():
+    assert_ast_roundtrip(doc(p(text("볼드", "bold"), text("입니다"))))
+
+
+def test_paren_ordered_marker_escaped():
+    # '1)' is a valid CommonMark ordered-list marker; global paren escaping
+    # currently prevents the reparse. Guards against relaxing escape_inline.
+    assert_ast_roundtrip(doc(p(text("1) 항목처럼 보이는 문장"))))
+
+
+def test_dot_ordered_marker_escaped():
+    assert_ast_roundtrip(doc(p(text("2026. 연간 회고"))))
+
+
+def test_thematic_break_lookalike():
+    assert_ast_roundtrip(doc(p(text("---"))))
+
+
+def test_setext_lookalike_after_hard_break():
+    assert_ast_roundtrip(doc(p(text("제목처럼"), hard_break(), text("==="))))
+
+
+def test_code_mark_with_backticks():
+    assert_ast_roundtrip(doc(p(text("a`b", "code"))))
+
+
+def test_hard_break_inside_bold():
+    assert_ast_roundtrip(
+        doc(p(text("첫 줄", "bold"), hard_break("bold"), text("둘째 줄", "bold")))
+    )
+
+
+# ── AST identity: CJK-adjacent emphasis (needs cjk_friendly parsing) ─────
+
+
+def test_bold_before_cjk_particle():
+    # Standard flanking rejects `)**은` (punctuation before, letter after);
+    # the cjk_friendly extension accepts it.
+    assert_ast_roundtrip(
+        doc(p(text("마크다운(Markdown)", "bold"), text("은 표준이다"))),
+        cjk_friendly=True,
+    )
+
+
+def test_italic_ending_with_fullwidth_stop():
+    # `*강조。*다음` — U+3002 is punctuation, '다' is a CJK letter.
+    assert_ast_roundtrip(
+        doc(p(text("강조。", "italic"), text("다음 문장"))), cjk_friendly=True
+    )
+
+
+def test_bold_quoted_word_before_particle():
+    # `**"인용"**라고` — '"' before the closing delimiter, CJK letter after.
+    assert_ast_roundtrip(
+        doc(p(text('"인용"', "bold"), text("라고 말했다"))), cjk_friendly=True
+    )
+
+
+def test_strike_before_cjk_particle():
+    # `~~삭제~~은` — the strikethrough closer is also blocked by standard rules.
+    assert_ast_roundtrip(
+        doc(p(text("삭제", "strike"), text("은 취소선"))), cjk_friendly=True
+    )
+
+
+def test_cjk_relaxation_is_opt_in():
+    # Default parsing must stay exactly GFM/CommonMark: the emphasis does not
+    # close, so the delimiters remain literal text.
+    md = tm.from_dict(
+        doc(p(text("마크다운(Markdown)", "bold"), text("은 표준이다")))
+    ).to_markdown()
+    reparsed = tm.from_markdown(md).to_dict()
+    assert reparsed["content"][0]["content"][0]["text"].startswith("**")
+
+
+# ── boundary whitespace: expelled outside the delimiters ─────────────────
+# Whitespace touching an emphasis delimiter has no valid markdown
+# representation, so the serializer moves it outside the marks
+# (cf. prosemirror-markdown expelEnclosingWhitespace). Text content and
+# rendering are unchanged; only the space's mark membership normalizes.
+
+
+def test_trailing_space_inside_bold_is_expelled():
+    ast = doc(p(text("굵게 ", "bold"), text("이후 텍스트")))
+    md = tm.from_dict(ast).to_markdown()
+    assert md == "**굵게** 이후 텍스트"
+    assert tm.from_markdown(md).to_dict() == doc(
+        p(text("굵게", "bold"), text(" 이후 텍스트"))
+    )
+
+
+def test_leading_space_inside_bold_is_expelled():
+    ast = doc(p(text("앞"), text(" 굵게", "bold")))
+    md = tm.from_dict(ast).to_markdown()
+    assert md == "앞 **굵게**"
+    assert tm.from_markdown(md).to_dict() == doc(p(text("앞 "), text("굵게", "bold")))
+
+
+# ── mark runs: shared marks stay open across run boundaries ──────────────
+
+
+def test_adjacent_bold_to_bold_italic():
+    # Serializes to `**a*b***`: the shared bold stays open, so the reparse
+    # restores the exact mark order.
+    assert_ast_roundtrip(doc(p(text("a", "bold"), text("b", "bold", "italic"))))
+
+
+# ── idempotence: normalize twice == normalize once ───────────────────────
+
+IDEMPOTENCE_CORPUS = [
+    "**a*****b***",
+    "これは**私のやりたかったこと。**だからするの。",
+    "**마크다운(Markdown)**은 표준이다",
+    "**굵게 **이후",
+    "\\*리터럴\\* 별표",
+    "- 항목 하나\n- 항목 둘",
+    "1. one\n1. two",
+    "> 인용\n>\n> 두 문단",
+    "`code` 와 **bold** 혼합",
+    "a  \nb",
+]
+
+
+@pytest.mark.parametrize("md", IDEMPOTENCE_CORPUS)
+def test_normalize_is_idempotent(md):
+    once = normalize(md)
+    assert normalize(once) == once, f"first pass produced: {once!r}"

{marktip-0.1.0 → marktip-0.2.0}/third_party/md4c/src/md4c.c

@@ -678,6 +678,42 @@ struct MD_UNICODE_FOLD_INFO_tag {
         return (md_unicode_bsearch__(codepoint, PUNCT_MAP, SIZEOF_ARRAY(PUNCT_MAP)) >= 0);
     }
 
+    /* MARKTIP LOCAL PATCH (MD_FLAG_CJKFRIENDLYEMPHASIS):
+     * CJK letters (Hangul, kana, Han ideographs, fullwidth forms). Used to
+     * relax the emphasis flanking rules around CJK text. */
+    static int
+    md_is_cjk_char__(unsigned codepoint)
+    {
+#define R(cp_min, cp_max)   ((cp_min) | 0x40000000), ((cp_max) | 0x80000000)
+#define S(cp)               (cp)
+        static const unsigned CJK_MAP[] = {
+            R(0x1100,0x11ff),   /* Hangul jamo */
+            S(0x3005), S(0x3007), S(0x303b),
+            R(0x3040,0x30fa),   /* hiragana, katakana (except middle dot) */
+            R(0x30fc,0x30ff),
+            R(0x3105,0x312f),   /* bopomofo */
+            R(0x3130,0x318f),   /* Hangul compatibility jamo */
+            R(0x31f0,0x31ff),   /* katakana phonetic extensions */
+            R(0x3400,0x4dbf),   /* CJK ideographs extension A */
+            R(0x4e00,0x9fff),   /* CJK unified ideographs */
+            R(0xa960,0xa97f),   /* Hangul jamo extended-A */
+            R(0xac00,0xd7a3),   /* Hangul syllables */
+            R(0xd7b0,0xd7ff),   /* Hangul jamo extended-B */
+            R(0xf900,0xfaff),   /* CJK compatibility ideographs */
+            R(0xff21,0xff3a),   /* fullwidth Latin capitals */
+            R(0xff41,0xff5a),   /* fullwidth Latin small letters */
+            R(0xff66,0xff9f),   /* halfwidth katakana */
+            R(0x20000,0x3ffff)  /* CJK ideographs extensions B..H */
+        };
+#undef R
+#undef S
+
+        if(codepoint < 0x1100)
+            return FALSE;
+
+        return (md_unicode_bsearch__(codepoint, CJK_MAP, SIZEOF_ARRAY(CJK_MAP)) >= 0);
+    }
+
     static void
     md_get_unicode_fold_info(unsigned codepoint, MD_UNICODE_FOLD_INFO* info)
     {
@@ -871,6 +907,10 @@ struct MD_UNICODE_FOLD_INFO_tag {
     #define ISUNICODEPUNCT(off)             md_is_unicode_punct__(md_decode_utf16le__(STR(off), ctx->size - (off), NULL))
     #define ISUNICODEPUNCTBEFORE(off)       md_is_unicode_punct__(md_decode_utf16le_before__(ctx, off))
 
+    /* MARKTIP LOCAL PATCH (MD_FLAG_CJKFRIENDLYEMPHASIS) */
+    #define ISCJK(off)                      md_is_cjk_char__(md_decode_utf16le__(STR(off), ctx->size - (off), NULL))
+    #define ISCJKBEFORE(off)                md_is_cjk_char__(md_decode_utf16le_before__(ctx, off))
+
     static inline int
     md_decode_unicode(const CHAR* str, OFF off, SZ str_size, SZ* p_char_size)
     {
@@ -952,6 +992,10 @@ struct MD_UNICODE_FOLD_INFO_tag {
     #define ISUNICODEPUNCT(off)             md_is_unicode_punct__(md_decode_utf8__(STR(off), ctx->size - (off), NULL))
     #define ISUNICODEPUNCTBEFORE(off)       md_is_unicode_punct__(md_decode_utf8_before__(ctx, off))
 
+    /* MARKTIP LOCAL PATCH (MD_FLAG_CJKFRIENDLYEMPHASIS) */
+    #define ISCJK(off)                      md_is_cjk_char__(md_decode_utf8__(STR(off), ctx->size - (off), NULL))
+    #define ISCJKBEFORE(off)                md_is_cjk_char__(md_decode_utf8_before__(ctx, off))
+
     static inline unsigned
     md_decode_unicode(const CHAR* str, OFF off, SZ str_size, SZ* p_char_size)
     {
@@ -965,6 +1009,10 @@ struct MD_UNICODE_FOLD_INFO_tag {
     #define ISUNICODEPUNCT(off)             ISPUNCT(off)
     #define ISUNICODEPUNCTBEFORE(off)       ISPUNCT((off)-1)
 
+    /* MARKTIP LOCAL PATCH (MD_FLAG_CJKFRIENDLYEMPHASIS): no CJK in ASCII mode. */
+    #define ISCJK(off)                      FALSE
+    #define ISCJKBEFORE(off)                FALSE
+
     static inline void
     md_get_unicode_fold_info(unsigned codepoint, MD_UNICODE_FOLD_INFO* info)
     {
@@ -3044,6 +3092,8 @@ md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, MD_SIZE n_lines, int table_m
                 OFF tmp = off+1;
                 int left_level;     /* What precedes: 0 = whitespace; 1 = punctuation; 2 = other char. */
                 int right_level;    /* What follows: 0 = whitespace; 1 = punctuation; 2 = other char. */
+                int left_is_cjk = FALSE;
+                int right_is_cjk = FALSE;
 
                 while(tmp < line->end  &&  CH(tmp) == ch)
                     tmp++;
@@ -3062,6 +3112,16 @@ md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, MD_SIZE n_lines, int table_m
                 else
                     right_level = 2;
 
+                /* MARKTIP LOCAL PATCH: a CJK character next to the delimiter
+                 * does not block it from being an opener/closer on the other
+                 * side (cf. markdown-it-cjk-friendly). */
+                if(ctx->parser.flags & MD_FLAG_CJKFRIENDLYEMPHASIS) {
+                    if(off > line->beg)
+                        left_is_cjk = ISCJKBEFORE(off);
+                    if(tmp < line->end)
+                        right_is_cjk = ISCJK(tmp);
+                }
+
                 /* Intra-word underscore doesn't have special meaning. */
                 if(ch == _T('_')  &&  left_level == 2  &&  right_level == 2) {
                     left_level = 0;
@@ -3071,9 +3131,9 @@ md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, MD_SIZE n_lines, int table_m
                 if(left_level != 0  ||  right_level != 0) {
                     unsigned flags = 0;
 
-                    if(left_level > 0  &&  left_level >= right_level)
+                    if(left_level > 0  &&  (left_level >= right_level  ||  right_is_cjk))
                         flags |= MD_MARK_POTENTIAL_CLOSER;
-                    if(right_level > 0  &&  right_level >= left_level)
+                    if(right_level > 0  &&  (right_level >= left_level  ||  left_is_cjk))
                         flags |= MD_MARK_POTENTIAL_OPENER;
                     if(flags == (MD_MARK_POTENTIAL_OPENER | MD_MARK_POTENTIAL_CLOSER))
                         flags |= MD_MARK_EMPH_OC;
@@ -3295,10 +3355,14 @@ md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, MD_SIZE n_lines, int table_m
 
                 if(tmp - off <= 2) {
                     unsigned flags = MD_MARK_POTENTIAL_OPENER | MD_MARK_POTENTIAL_CLOSER;
+                    /* MARKTIP LOCAL PATCH: CJK-friendly strikethrough. */
+                    int cjk_friendly = (ch == _T('~')  &&  (ctx->parser.flags & MD_FLAG_CJKFRIENDLYEMPHASIS));
 
-                    if(off > line->beg  &&  !ISUNICODEWHITESPACEBEFORE(off)  &&  !ISUNICODEPUNCTBEFORE(off))
+                    if(off > line->beg  &&  !ISUNICODEWHITESPACEBEFORE(off)  &&  !ISUNICODEPUNCTBEFORE(off)
+                        &&  !(cjk_friendly  &&  ISCJKBEFORE(off)))
                         flags &= ~MD_MARK_POTENTIAL_OPENER;
-                    if(tmp < line->end  &&  !ISUNICODEWHITESPACE(tmp)  &&  !ISUNICODEPUNCT(tmp))
+                    if(tmp < line->end  &&  !ISUNICODEWHITESPACE(tmp)  &&  !ISUNICODEPUNCT(tmp)
+                        &&  !(cjk_friendly  &&  ISCJK(tmp)))
                         flags &= ~MD_MARK_POTENTIAL_CLOSER;
                     if(flags != 0)
                         ADD_MARK(ch, off, tmp, flags);

{marktip-0.1.0 → marktip-0.2.0}/third_party/md4c/src/md4c.h

@@ -319,6 +319,11 @@ typedef struct MD_SPAN_WIKILINK {
 #define MD_FLAG_UNDERLINE                   0x4000  /* Enable underline extension (and disables '_' for normal emphasis). */
 #define MD_FLAG_HARD_SOFT_BREAKS            0x8000  /* Force all soft breaks to act as hard breaks. */
 
+/* MARKTIP LOCAL PATCH (not upstream): relax emphasis/strikethrough flanking
+ * rules so a CJK character next to a delimiter does not prevent it from
+ * opening/closing (cf. markdown-it-cjk-friendly). */
+#define MD_FLAG_CJKFRIENDLYEMPHASIS         0x10000
+
 #define MD_FLAG_PERMISSIVEAUTOLINKS         (MD_FLAG_PERMISSIVEEMAILAUTOLINKS | MD_FLAG_PERMISSIVEURLAUTOLINKS | MD_FLAG_PERMISSIVEWWWAUTOLINKS)
 #define MD_FLAG_NOHTML                      (MD_FLAG_NOHTMLBLOCKS | MD_FLAG_NOHTMLSPANS)
 

marktip-0.2.0/third_party/md4c/upstream_metadata/README.md

@@ -0,0 +1,21 @@
+# MD4C vendored source
+
+This directory vendors the parser-only MD4C 0.5.3 source files:
+
+- `src/md4c.c`
+- `src/md4c.h`
+- `src/entity.c` (HTML entity table, used to decode `MD_TEXT_ENTITY`)
+- `src/entity.h`
+- `LICENSE.md`
+
+Upstream: https://github.com/mity/md4c
+Pinned tag: `release-0.5.3`
+
+## Local patches (deviations from upstream)
+
+- `MD_FLAG_CJKFRIENDLYEMPHASIS` (md4c.h, md4c.c): opt-in flag that relaxes the
+  emphasis (`*`, `_`) and strikethrough (`~`) flanking rules so that a CJK
+  character next to a delimiter does not prevent it from opening/closing
+  (cf. markdown-it-cjk-friendly). Adds `md_is_cjk_char__()` and the
+  `ISCJK`/`ISCJKBEFORE` macros. All patched hunks are marked with
+  `MARKTIP LOCAL PATCH` comments. Behavior is unchanged unless the flag is set.

marktip-0.1.0/tests/test_roundtrip.py

@@ -1,58 +0,0 @@
-import marktip as tm
-
-
-def assert_stable(markdown: str):
-    doc = tm.from_markdown(markdown)
-    serialized = doc.to_markdown()
-    assert tm.from_markdown(serialized).to_dict() == doc.to_dict()
-
-
-def test_roundtrip_core_blocks_and_marks():
-    assert_stable(
-        "# Title\n\n"
-        "A **bold** and *italic* paragraph with `code` and ~~strike~~.\n\n"
-        "> quoted\n\n"
-        "- one\n"
-        "- two\n\n"
-        "1. first\n"
-        "2. second\n"
-    )
-
-
-def test_roundtrip_gfm_table_task_list_and_code():
-    assert_stable(
-        "- [x] done\n"
-        "- [ ] todo\n\n"
-        "| A | B |\n"
-        "| :- | -: |\n"
-        "| x | y |\n\n"
-        "```python\n"
-        "print('ok')\n"
-        "```\n"
-    )
-
-
-def test_roundtrip_multi_block_list_items_and_loose_lists():
-    assert_stable("- first\n\n  second\n")
-    assert_stable("- a\n\n- b\n")
-
-
-def test_roundtrip_hard_break_inside_marks():
-    assert_stable("**a\\\nb**")
-    assert_stable("[a\\\nb](https://example.com)")
-
-
-def test_roundtrip_block_markers_escaped_in_text():
-    assert_stable("\\- not a list")
-    assert_stable("\\> not a quote")
-    assert_stable("1\\. not ordered")
-
-
-def test_roundtrip_ordered_task_list():
-    assert_stable("1. [x] done\n2. [ ] todo\n")
-
-
-def test_roundtrip_large_and_deep_smoke():
-    nested = "- root\n  - child\n    - grandchild\n"
-    large = "\n".join(f"Paragraph {i} with **bold** text." for i in range(200))
-    assert_stable(nested + "\n" + large)

marktip-0.1.0/third_party/md4c/upstream_metadata/README.md

@@ -1,12 +0,0 @@
-# MD4C vendored source
-
-This directory vendors the parser-only MD4C 0.5.3 source files:
-
-- `src/md4c.c`
-- `src/md4c.h`
-- `src/entity.c` (HTML entity table, used to decode `MD_TEXT_ENTITY`)
-- `src/entity.h`
-- `LICENSE.md`
-
-Upstream: https://github.com/mity/md4c
-Pinned tag: `release-0.5.3`