markdown-flow 0.3.3__tar.gz → 0.4.0__tar.gz

{markdown_flow-0.3.3 → markdown_flow-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: markdown-flow
-Version: 0.3.3
+Version: 0.4.0
 Summary: An agent library designed to parse and process MarkdownFlow documents
 Project-URL: Homepage, https://github.com/ai-shifu/markdown-flow-agent-py
 Project-URL: Bug Tracker, https://github.com/ai-shifu/markdown-flow-agent-py/issues
@@ -691,6 +691,40 @@ builder.start_interaction()
 
 ### Variable System Deep Dive
 
+#### Text variable names
+
+Names are opaque Unicode text. Python `str.strip()` removes surrounding whitespace;
+the remaining name must be nonempty. Internal spaces, punctuation, emoji, single
+braces, brackets, `|`, `//`, and `...` stay exactly as written. Internal control
+characters (U+0000–001F and U+007F–009F) and U+2028/U+2029 are invalid. Case,
+repeated spaces, and Unicode sequences are never normalized; U+FEFF remains part
+of the name.
+
+The first literal `}}` closes a marker. A name cannot contain that pair. Quotes
+and backslashes are literal characters with no decoding rule, and nested `{{`
+inside a name is not another marker. Every complete `{{ ... }}` or `%{{ ... }}`
+is read before interaction delimiters, so internal `]`, `|`, `//`, and `...` stay
+literal. Only a leading assignment marker sets the interaction's variable;
+markers elsewhere in its body remain literal option text. An unterminated
+marker keeps the remaining text opaque, without restarting at an inner `?[`.
+
+At document level, complete inline code and HTML are read before markers inside
+them. Complete variable examples there still support collection and replacement,
+but an unclosed example cannot consume text beyond that markup span. Escaped
+document openers stay literal. A true raw marker owns its entire name before any
+code or HTML inside that name is considered; interaction bodies keep the raw
+marker grammar.
+
+Use `format_variable_reference(name)` and `format_variable_assignment(name)` from
+`markdown_flow.variable_names` to emit padded markers such as
+`{{ What is your gender? }}` and `%{{ a} }}`. Padding lets a name end in a single
+brace. Extraction and replacement use the same trimmed key; existing stored keys
+are not renamed or migrated.
+
+The shared conformance corpus is `scripts/variable-name-contract.json`. Run its
+offline parser, document, replacement, translation, and answer round trips with
+`python -m pytest scripts/test_variable_names.py -q`.
+
 ```python
 from markdown_flow import extract_variables_from_text, replace_variables_in_text
 

{markdown_flow-0.3.3 → markdown_flow-0.4.0}/README.md

@@ -673,6 +673,40 @@ builder.start_interaction()
 
 ### Variable System Deep Dive
 
+#### Text variable names
+
+Names are opaque Unicode text. Python `str.strip()` removes surrounding whitespace;
+the remaining name must be nonempty. Internal spaces, punctuation, emoji, single
+braces, brackets, `|`, `//`, and `...` stay exactly as written. Internal control
+characters (U+0000–001F and U+007F–009F) and U+2028/U+2029 are invalid. Case,
+repeated spaces, and Unicode sequences are never normalized; U+FEFF remains part
+of the name.
+
+The first literal `}}` closes a marker. A name cannot contain that pair. Quotes
+and backslashes are literal characters with no decoding rule, and nested `{{`
+inside a name is not another marker. Every complete `{{ ... }}` or `%{{ ... }}`
+is read before interaction delimiters, so internal `]`, `|`, `//`, and `...` stay
+literal. Only a leading assignment marker sets the interaction's variable;
+markers elsewhere in its body remain literal option text. An unterminated
+marker keeps the remaining text opaque, without restarting at an inner `?[`.
+
+At document level, complete inline code and HTML are read before markers inside
+them. Complete variable examples there still support collection and replacement,
+but an unclosed example cannot consume text beyond that markup span. Escaped
+document openers stay literal. A true raw marker owns its entire name before any
+code or HTML inside that name is considered; interaction bodies keep the raw
+marker grammar.
+
+Use `format_variable_reference(name)` and `format_variable_assignment(name)` from
+`markdown_flow.variable_names` to emit padded markers such as
+`{{ What is your gender? }}` and `%{{ a} }}`. Padding lets a name end in a single
+brace. Extraction and replacement use the same trimmed key; existing stored keys
+are not renamed or migrated.
+
+The shared conformance corpus is `scripts/variable-name-contract.json`. Run its
+offline parser, document, replacement, translation, and answer round trips with
+`python -m pytest scripts/test_variable_names.py -q`.
+
 ```python
 from markdown_flow import extract_variables_from_text, replace_variables_in_text
 

{markdown_flow-0.3.3 → markdown_flow-0.4.0}/markdown_flow/__init__.py

@@ -75,6 +75,13 @@ from .parser import (
     generate_smart_validation_template,
     replace_variables_in_text,
 )
+from .variable_names import (
+    find_variable_markers,
+    format_variable_assignment,
+    format_variable_reference,
+    normalize_variable_name,
+    read_variable_marker,
+)
 
 
 # Public API
@@ -104,6 +111,11 @@ __all__ = [
     "extract_interaction_question",
     "extract_variables_from_text",
     "replace_variables_in_text",
+    "normalize_variable_name",
+    "read_variable_marker",
+    "find_variable_markers",
+    "format_variable_reference",
+    "format_variable_assignment",
 ]
 
-__version__ = "0.3.3"
+__version__ = "0.4.0"

{markdown_flow-0.3.3 → markdown_flow-0.4.0}/markdown_flow/constants.py

@@ -6,11 +6,15 @@ Constants for document parsing, variable matching, validation, and other core fu
 
 import re
 
+from .variable_names import (
+    VARIABLE_ASSIGNMENT_SOURCE,
+    VARIABLE_REFERENCE_SOURCE,
+    VARIABLE_REFERENCE_UNIT_SOURCE,
+)
+
 
 # Pre-compiled regex patterns
-COMPILED_PERCENT_VARIABLE_REGEX = re.compile(
-    r"%\{\{([^}]+)\}\}"  # Match %{{variable}} format for preserved variables
-)
+COMPILED_PERCENT_VARIABLE_REGEX = re.compile(VARIABLE_ASSIGNMENT_SOURCE)
 
 # Interaction regex base patterns
 #
@@ -21,7 +25,7 @@ COMPILED_PERCENT_VARIABLE_REGEX = re.compile(
 # a left-to-right scanner disagree about `?[a\\]`, and two parsers of one grammar must not.
 # A backslash before anything else -- LaTeX such as `$\pi$`, or `\\` as a line break -- stays
 # ordinary text, exactly as it was before escapes existed. See markdown_flow.escaping.
-_INTERACTION_CONTENT = r"(?:\\[|/.\]]|\\(?![|/.\]])|[^\]\\])*"
+_INTERACTION_CONTENT = rf"(?:{VARIABLE_REFERENCE_UNIT_SOURCE}|\\[|/.\]]|\\(?![|/.\]])|(?!\{{\{{)[^\]\\])*"
 INTERACTION_PATTERN = rf"(?<!\\)\?\[({_INTERACTION_CONTENT})\](?!\()"  # Base pattern with capturing group for content extraction, excludes escaped \?[]
 INTERACTION_PATTERN_NON_CAPTURING = rf"(?<!\\)\?\[{_INTERACTION_CONTENT}\](?!\()"  # Non-capturing version for block splitting, excludes escaped \?[]
 INTERACTION_PATTERN_SPLIT = rf"((?<!\\)\?\[{_INTERACTION_CONTENT}\](?!\())"  # Pattern for re.split() with outer capturing group, excludes escaped \?[]
@@ -29,13 +33,21 @@ INTERACTION_PATTERN_SPLIT = rf"((?<!\\)\?\[{_INTERACTION_CONTENT}\](?!\())"  # P
 # InteractionParser specific regex patterns
 COMPILED_INTERACTION_REGEX = re.compile(INTERACTION_PATTERN)  # Main interaction pattern matcher
 COMPILED_LAYER1_INTERACTION_REGEX = COMPILED_INTERACTION_REGEX  # Layer 1: Basic format validation (alias)
-COMPILED_LAYER2_VARIABLE_REGEX = re.compile(r"^%\{\{([^}]+)\}\}(.*)$")  # Layer 2: Variable detection
+COMPILED_LAYER2_VARIABLE_REGEX = re.compile(r"^\s*" + VARIABLE_ASSIGNMENT_SOURCE + r"(.*)$")
 COMPILED_LAYER3_ELLIPSIS_REGEX = re.compile(r"^(.*)\.\.\.(.*)")  # Layer 3: Split content around ellipsis
 COMPILED_LAYER3_BUTTON_VALUE_REGEX = re.compile(r"^(.+)//(.+)$")  # Layer 3: Parse Button//value format
-COMPILED_BRACE_VARIABLE_REGEX = re.compile(
-    r"(?<!%)\{\{([^}]+)\}\}"  # Match {{variable}} format for replaceable variables
-)
+COMPILED_BRACE_VARIABLE_REGEX = re.compile(r"(?<!%)" + VARIABLE_REFERENCE_SOURCE)
 COMPILED_SINGLE_PIPE_SPLIT_REGEX = re.compile(r"(?<!\|)\|(?!\|)")  # Split on single | but not ||
+COMPILED_VARIABLE_NAME_PLACEHOLDER_REGEX = re.compile(r"\x00mdf-name:+[0-9]+\x00")
+
+# Inline HTML owns braces in its tag source before document-level MDF scanning.
+_HTML_SPACE = r"[ \t\r\n]"
+_HTML_ATTRIBUTE = r"[A-Za-z_:][A-Za-z0-9_.:-]*(?:" + _HTML_SPACE + r"*=" + _HTML_SPACE + r"*(?:[^ \t\r\n\"'=<>`]+|'[^']*'|\"[^\"]*\"))?"
+COMPILED_INLINE_HTML_REGEX = re.compile(
+    r"(?:<[A-Za-z][A-Za-z0-9-]*(?:" + _HTML_SPACE + r"+" + _HTML_ATTRIBUTE + r")*" + _HTML_SPACE + r"*/?>"
+    r"|</[A-Za-z][A-Za-z0-9-]*" + _HTML_SPACE + r"*>"
+    r"|<!--[\s\S]*?-->|<\?[\s\S]*?\?>|<![A-Z]+" + _HTML_SPACE + r"+[^>]*>|<!\[CDATA\[[\s\S]*?\]\]>)"
+)
 
 # Canonical user_input key for non-assignment interactions (?[A|B] without a
 # variable). Clients should submit {DEFAULT_NON_ASSIGNMENT_INPUT_KEY: values};
@@ -135,6 +147,7 @@ INTERACTION_PROMPT_WITH_TRANSLATION = """
 
 - Translate the text in `buttons` and `question` into the target language
 - Translate only the display text (the Display part); do not change the JSON structure
+- Tokens such as `__MDFLOW_VARIABLE_0__` are opaque placeholders: copy every token occurrence exactly as given in its original order and translate only the surrounding text
 - If `display//value` separation exists (for example `Yes//1`), translate only the Display part before `//` and keep value unchanged
 - Use the target language 100%; language mixing is not allowed. Translate all non-target-language words before outputting
 - If a text segment's original language is already the same as the target language, return that text unchanged (for example, when the target language is English and the original text is also English, do not rewrite it)

{markdown_flow-0.3.3 → markdown_flow-0.4.0}/markdown_flow/core.py

@@ -13,6 +13,7 @@ from typing import Any
 from .constants import (
     BLOCK_INDEX_OUT_OF_RANGE_ERROR,
     BLOCK_SEPARATOR,
+    COMPILED_LAYER2_VARIABLE_REGEX,
     CONTEXT_BUTTON_OPTIONS_TEMPLATE,
     CONTEXT_QUESTION_TEMPLATE,
     DEFAULT_INTERACTION_ERROR_PROMPT,
@@ -21,8 +22,6 @@ from .constants import (
     INPUT_EMPTY_ERROR,
     INTERACTION_ERROR_RENDER_INSTRUCTIONS,
     INTERACTION_PARSE_ERROR,
-    INTERACTION_PATTERN_NON_CAPTURING,
-    INTERACTION_PATTERN_SPLIT,
     INTERACTION_PROMPT_BASE,
     INTERACTION_PROMPT_WITH_TRANSLATION,
     LLM_PROVIDER_REQUIRED_ERROR,
@@ -62,7 +61,76 @@ from .parser import (
     process_output_instructions,
     replace_variables_in_text,
 )
+from .parser.document_context import find_document_interaction_spans, find_document_protected_spans
 from .tag_filter import StreamTagFilter, strip_preserve_tags
+from .variable_names import find_variable_markers
+
+
+def _translation_marker_spans(text: str) -> list[tuple[int, int]]:
+    """Include the preserved-variable prefix in each complete marker's source span."""
+    return [(marker.start - (marker.start > 0 and text[marker.start - 1] == "%"), marker.end) for marker in find_variable_markers(text)]
+
+
+def _mask_translation_markers(text: str, markers: dict[str, str]) -> str:
+    """Replace whole markers with collision-free tokens, without reading inside keys."""
+    tokens = {raw: token for token, raw in markers.items()}
+    parts: list[str] = []
+    at = 0
+    for start, end in _translation_marker_spans(text):
+        parts.extend((text[at:start], tokens[text[start:end]]))
+        at = end
+    parts.append(text[at:])
+    return "".join(parts)
+
+
+def _restore_translation_markers(original: str, translated: Any, markers: dict[str, str]) -> str:
+    """Accept a translated field only when all its marker tokens survive in order."""
+
+    def token_sequence(text: str) -> list[str]:
+        positions = []
+        for token in markers:
+            at = 0
+            while (at := text.find(token, at)) >= 0:
+                positions.append((at, token))
+                at += len(token)
+        return [token for _, token in sorted(positions)]
+
+    if not isinstance(translated, str) or token_sequence(original) != token_sequence(translated):
+        translated = original
+    for token, raw in markers.items():
+        translated = translated.replace(token, raw)
+    return translated
+
+
+def _find_translation_text(text: str, old: str) -> int:
+    """Locate a display span without matching a substring inside an opaque marker."""
+    if not old:
+        return -1
+    spans = _translation_marker_spans(text)
+    at = 0
+    while (at := text.find(old, at)) >= 0:
+        end = at + len(old)
+        if not any(at < marker_end and end > marker_start and (at > marker_start or end < marker_end) for marker_start, marker_end in spans):
+            return at
+        at += 1
+    return -1
+
+
+def _replace_translation_text(text: str, old: str, new: str) -> str:
+    """Replace only a complete display span, leaving marker internals untouched."""
+    at = _find_translation_text(text, old)
+    return text[:at] + new + text[at + len(old) :] if at >= 0 else text
+
+
+def _strip_translation_tags(text: str) -> str:
+    """Filter generated tags while retaining literal tag text inside variable names."""
+    parts: list[str] = []
+    at = 0
+    for marker in find_variable_markers(text):
+        parts.extend((strip_preserve_tags(text[at : marker.start]), text[marker.start : marker.end]))
+        at = marker.end
+    parts.append(strip_preserve_tags(text[at:]))
+    return "".join(parts)
 
 
 class MarkdownFlow:
@@ -468,18 +536,32 @@ class MarkdownFlow:
 
         # Parse the preprocessed document (code blocks already replaced with placeholders)
         # The preprocessing was done once during initialization
-        segments = re.split(BLOCK_SEPARATOR, self._processed_document)
+        source = self._processed_document
+        protected = list(find_document_protected_spans(source))
+        segments = []
+        at = 0
+        protected_at = 0
+        for separator in re.finditer(BLOCK_SEPARATOR, source):
+            while protected_at < len(protected) and protected[protected_at][1] <= separator.start():
+                protected_at += 1
+            if protected_at < len(protected) and protected[protected_at][0] < separator.end():
+                continue
+            segments.append(source[at : separator.start()])
+            at = separator.end()
+        segments.append(source[at:])
         final_blocks: list[Block] = []
 
         for segment in segments:
-            # Use dedicated split pattern to avoid duplicate blocks from capturing groups
-            parts = re.split(INTERACTION_PATTERN_SPLIT, segment)
-
-            for part in parts:
+            at = 0
+            parts: list[tuple[str, bool]] = []
+            for start, end in find_document_interaction_spans(segment):
+                parts.extend(((segment[at:start], False), (segment[start:end], True)))
+                at = end
+            parts.append((segment[at:], False))
+            for part, interaction in parts:
                 part = part.strip()
                 if part:
-                    # Use non-capturing pattern for matching
-                    if re.match(INTERACTION_PATTERN_NON_CAPTURING, part):
+                    if interaction:
                         block = Block(
                             content=part,
                             block_type=BlockType.INTERACTION,
@@ -763,7 +845,7 @@ class MarkdownFlow:
             translated_content = self._reconstruct_with_translation(processed_block.content, translatable_json, translated_json, interaction_info)
 
             # 过滤 LLM 输出中可能泄露的 原样输出 标签
-            translated_content = strip_preserve_tags(translated_content)
+            translated_content = _strip_translation_tags(translated_content)
 
             return LLMResult(
                 content=translated_content,
@@ -805,7 +887,7 @@ class MarkdownFlow:
                 translated_content = self._reconstruct_with_translation(processed_block.content, translatable_json, full_response, interaction_info)
 
                 # 过滤 LLM 输出中可能泄露的 原样输出 标签
-                translated_content = strip_preserve_tags(translated_content)
+                translated_content = _strip_translation_tags(translated_content)
 
                 # Return complete content once (not incremental)
                 yield LLMResult(
@@ -1453,16 +1535,24 @@ class MarkdownFlow:
         if not interaction_info:
             return "{}", None
 
-        translatable = {}
+        # Keep marker source out of the translation request. The prefix cannot
+        # collide with author text, and equal raw markers share the same token.
+        prefix = "__MDFLOW_VARIABLE_"
+        while prefix in interaction_content:
+            prefix = "_" + prefix
+        raw_markers = dict.fromkeys(interaction_content[start:end] for start, end in _translation_marker_spans(interaction_content))
+        markers = {f"{prefix}{i}__": raw for i, raw in enumerate(raw_markers)}
+        interaction_info["_translation_markers"] = markers
+        translatable: dict[str, str | list[str]] = {}
 
         # Extract button display text
         if interaction_info.get("buttons"):
-            button_texts = [btn["display"] for btn in interaction_info["buttons"]]
+            button_texts = [_mask_translation_markers(btn["display"], markers) for btn in interaction_info["buttons"]]
             translatable["buttons"] = button_texts
 
         # Extract question text
         if interaction_info.get("question"):
-            translatable["question"] = interaction_info["question"]
+            translatable["question"] = _mask_translation_markers(interaction_info["question"], markers)
 
         # Convert to JSON
 
@@ -1536,19 +1626,26 @@ class MarkdownFlow:
         except (json.JSONDecodeError, ValueError):
             return original_content
 
+        markers = interaction_info.get("_translation_markers", {})
+        prefix = ""
         reconstructed = original_content
+        marker = COMPILED_LAYER2_VARIABLE_REGEX.match(original_content[2:-1]) if original_content.startswith("?[") else None
+        if marker:
+            body_at = 2 + marker.start(2)
+            prefix, reconstructed = original_content[:body_at], original_content[body_at:]
 
         # Replace button display text (smart value handling)
         if "buttons" in translated and interaction_info.get("buttons"):
             for i, button in enumerate(interaction_info["buttons"]):
                 if i < len(translated["buttons"]):
                     old_display = button["display"]
-                    new_display = translated["buttons"][i]
+                    original_display = original.get("buttons", [])[i]
+                    new_display = _restore_translation_markers(original_display, translated["buttons"][i], markers)
 
                     # Detect if translation happened
                     translation_happened = False
                     if "buttons" in original and i < len(original["buttons"]):
-                        if original["buttons"][i] != new_display:
+                        if old_display != new_display:
                             translation_happened = True
 
                     # If value separation exists (display//value format), preserve value
@@ -1557,36 +1654,36 @@ class MarkdownFlow:
                         # Replace format: oldDisplay//value -> newDisplay//value
                         old_pattern = f"{old_display}//{button['value']}"
                         new_pattern = f"{new_display}//{button['value']}"
-                        reconstructed = reconstructed.replace(old_pattern, new_pattern, 1)
+                        reconstructed = _replace_translation_text(reconstructed, old_pattern, new_pattern)
                     elif translation_happened:
                         # Display == Value, but need to check if display//value format exists in original text
                         # In this case, original text is like "小兔子//小兔子", both Display and Value are "小兔子"
                         old_pattern_with_separator = f"{old_display}//"
-                        if old_pattern_with_separator in reconstructed:
+                        if _find_translation_text(reconstructed, old_pattern_with_separator) >= 0:
                             # Original text has display//value format (e.g., "小兔子//小兔子")
                             # Only replace display part, preserve original value
                             # Example: 小兔子//小兔子 -> 토끼//小兔子
                             old_pattern = f"{old_display}//{button['value']}"
                             new_pattern = f"{new_display}//{button['value']}"
-                            reconstructed = reconstructed.replace(old_pattern, new_pattern, 1)
+                            reconstructed = _replace_translation_text(reconstructed, old_pattern, new_pattern)
                         else:
                             # Original text has no // separator (e.g., "小兔子")
                             # Auto-add value: translated//original
                             # Example: 小兔子 -> 토끼//小兔子
                             old_pattern = old_display
                             new_pattern = f"{new_display}//{old_display}"
-                            reconstructed = reconstructed.replace(old_pattern, new_pattern, 1)
+                            reconstructed = _replace_translation_text(reconstructed, old_pattern, new_pattern)
                     else:
                         # No translation, keep as is
-                        reconstructed = reconstructed.replace(old_display, new_display, 1)
+                        reconstructed = _replace_translation_text(reconstructed, old_display, new_display)
 
         # Replace question text
         if "question" in translated and interaction_info.get("question"):
             old_question = interaction_info["question"]
-            new_question = translated["question"]
-            reconstructed = reconstructed.replace(f"...{old_question}", f"...{new_question}", 1)
+            new_question = _restore_translation_markers(original["question"], translated["question"], markers)
+            reconstructed = _replace_translation_text(reconstructed, f"...{old_question}", f"...{new_question}")
 
-        return reconstructed
+        return prefix + reconstructed
 
     def _build_validation_messages(
         self,
@@ -1630,10 +1727,10 @@ class MarkdownFlow:
         # 2. Validation task template (choose based on language requirement)
         if has_language_requirement:
             # Has language requirement: use Base + WithLanguage
-            task_template = (VALIDATION_TASK_BASE + VALIDATION_TASK_WITH_LANGUAGE).replace("{target_variable}", target_variable)
+            task_template = (VALIDATION_TASK_BASE + VALIDATION_TASK_WITH_LANGUAGE).replace('"{target_variable}"', json.dumps(target_variable, ensure_ascii=False))
         else:
             # No language requirement: use default (Base + NoLanguage)
-            task_template = VALIDATION_TASK_TEMPLATE.replace("{target_variable}", target_variable)
+            task_template = VALIDATION_TASK_TEMPLATE.replace('"{target_variable}"', json.dumps(target_variable, ensure_ascii=False))
 
         system_parts.append(task_template)
 
@@ -1662,7 +1759,7 @@ class MarkdownFlow:
 
         # Add extraction requirements (using template)
         system_parts.append("")
-        system_parts.append(VALIDATION_REQUIREMENTS_TEMPLATE)
+        system_parts.append(VALIDATION_REQUIREMENTS_TEMPLATE.replace('"{target_variable}"', json.dumps(target_variable, ensure_ascii=False)))
 
         # 3. Output language anchoring (bottom - if output language is set)
         if has_language_requirement:

{markdown_flow-0.3.3 → markdown_flow-0.4.0}/markdown_flow/escaping.py

@@ -18,6 +18,9 @@ The cost is that a literal backslash cannot sit immediately before a literal del
 an escaped bar, not a backslash and a separator. Nothing in the measured corpus writes that.
 """
 
+from .variable_names import read_variable_marker
+
+
 # The characters a backslash may make ordinary. `/` covers `//`, `.` covers `...`.
 ESCAPABLE = "|/.]"
 
@@ -45,10 +48,22 @@ def escape_interaction_text(text: str) -> str:
     out: list[str] = []
     index = 0
     while index < len(text):
+        if text.startswith("{{", index):
+            marker = read_variable_marker(text, index)
+            end = marker.end if marker is not None else len(text)
+            out.append(text[index:end])
+            index = end
+            continue
         char = text[index]
         if char == ".":
             run = len(text) - index - len(text[index:].lstrip("."))
-            if run >= 3:
+            # A run of three or more is the `...` delimiter. A shorter one still needs escaping
+            # when a literal backslash precedes it, because the parser would read that backslash
+            # and the dot as an escape pair and drop the backslash -- `a\\.b` would come back as
+            # `a.b`, the round trip would fail, and the question would be refused. Escaping the
+            # dot puts a second backslash in front of it, which the parser reads as one literal
+            # backslash followed by an escaped dot: exactly the text that went in.
+            if run >= 3 or (index > 0 and text[index - 1] == "\\"):
                 out.append("\\." * run)
             else:
                 out.append("." * run)
@@ -66,6 +81,12 @@ def unescape_interaction_text(text: str) -> str:
     out: list[str] = []
     index = 0
     while index < len(text):
+        if text.startswith("{{", index):
+            marker = read_variable_marker(text, index)
+            end = marker.end if marker is not None else len(text)
+            out.append(text[index:end])
+            index = end
+            continue
         if is_escape(text, index):
             out.append(text[index + 1])
             index += 2
@@ -84,6 +105,12 @@ def find_unescaped(text: str, needle: str, start: int = 0) -> int:
     """
     index = start
     while index < len(text):
+        if text.startswith("{{", index):
+            marker = read_variable_marker(text, index)
+            if marker is None:
+                return -1
+            index = marker.end
+            continue
         if is_escape(text, index):
             index += 2
             continue
@@ -112,6 +139,10 @@ def split_on_single_pipe(text: str) -> list[str]:
     start = 0
     index = 0
     while index < len(text):
+        if text.startswith("{{", index):
+            marker = read_variable_marker(text, index)
+            index = marker.end if marker is not None else len(text)
+            continue
         if is_escape(text, index):
             index += 2
             continue
@@ -154,7 +185,8 @@ def split_on_ellipsis(content: str) -> tuple[str, str] | None:
 
     What is new is only that an escaped ellipsis no longer counts.
     """
-    head = content.split("\n", 1)[0]
+    newline = find_unescaped(content, "\n")
+    head = content[:newline] if newline >= 0 else content
     at = rfind_unescaped(head, "...")
     if at < 0:
         return None

markdown_flow-0.4.0/markdown_flow/parser/document_context.py

@@ -0,0 +1,131 @@
+"""Document-stage boundaries without changing the raw variable-marker grammar."""
+
+from collections.abc import Iterator
+from string import punctuation
+
+from ..constants import COMPILED_INLINE_HTML_REGEX
+from ..variable_names import VariableMarker, find_interaction_end, find_variable_markers, read_variable_marker
+from .code_fence_utils import is_code_fence_end, parse_code_fence_start
+
+
+def _fenced_code_end(text: str, start: int) -> int | None:
+    if start and text[start - 1] != "\n":
+        return None
+    line_end = text.find("\n", start)
+    line_end = line_end if line_end >= 0 else len(text)
+    fence = parse_code_fence_start(text[start:line_end])
+    if fence is None:
+        return None
+    at = line_end + 1
+    while at < len(text):
+        end = text.find("\n", at)
+        end = end if end >= 0 else len(text)
+        if is_code_fence_end(text[at:end], fence):
+            return end
+        at = end + 1
+    return len(text)
+
+
+def read_document_literal_end(text: str, start: int) -> int | None:
+    """Read a Markdown literal region; unmatched inline tick runs stop locally."""
+    fenced_end = _fenced_code_end(text, start)
+    if fenced_end is not None:
+        return fenced_end
+    if text[start] == "\\":
+        # Markdown escapes ASCII punctuation, never the following line ending.
+        return start + 2 if start + 1 < len(text) and text[start + 1] in punctuation else None
+    if text[start] == "<":
+        html = COMPILED_INLINE_HTML_REGEX.match(text, start)
+        return html.end() if html is not None else None
+    if text[start] != "`":
+        return None
+
+    opening_end = start + 1
+    while opening_end < len(text) and text[opening_end] == "`":
+        opening_end += 1
+    size = opening_end - start
+    at = opening_end
+    while at < len(text):
+        if text[at] == "\n":
+            line_end = text.find("\n", at + 1)
+            line = text[at + 1 : line_end if line_end >= 0 else len(text)]
+            # Inline code cannot absorb a later fenced block or a new paragraph.
+            if not line.strip() or parse_code_fence_start(line) is not None:
+                break
+        if text[at] == "`":
+            end = at + 1
+            while end < len(text) and text[end] == "`":
+                end += 1
+            if end - at == size:
+                return end
+            at = end
+        else:
+            at += 1
+    # An unmatched run is literal punctuation; its following braces stay active.
+    return opening_end
+
+
+def _document_units(text: str) -> Iterator[tuple[int, int, bool]]:
+    """Yield literal markup and raw-marker regions, keeping interaction bodies raw."""
+    at = 0
+    interaction_until = -1
+    while at < len(text):
+        if at > interaction_until:
+            literal_end = read_document_literal_end(text, at)
+            if literal_end is not None:
+                yield at, literal_end, True
+                at = literal_end
+                continue
+            if text.startswith("?[", at):
+                end = find_interaction_end(text, at + 2)
+                interaction_until = end if end >= 0 else len(text)
+        if text.startswith("{{", at):
+            marker = read_variable_marker(text, at)
+            end = marker.end if marker is not None else len(text)
+            yield at, end, False
+            at = end
+        else:
+            at += 1
+
+
+def find_document_variable_markers(text: str) -> Iterator[VariableMarker]:
+    """Collect complete examples inside markup, without letting them cross its edge."""
+    for start, end, literal in _document_units(text):
+        if literal:
+            if text[start] == "\\":
+                continue
+            for marker in find_variable_markers(text[start:end]):
+                yield VariableMarker(start + marker.start, start + marker.end, marker.raw_name, marker.name)
+        else:
+            marker = read_variable_marker(text, start)
+            if marker is not None:
+                yield marker
+
+
+def find_document_protected_spans(text: str) -> Iterator[tuple[int, int]]:
+    """Keep document separators outside marker names and already-owned markup."""
+    for start, end, _ in _document_units(text):
+        yield start, end
+
+
+def find_document_interaction_spans(text: str) -> Iterator[tuple[int, int]]:
+    """Find controls outside markup while retaining raw ownership inside controls."""
+    at = 0
+    while at < len(text):
+        literal_end = read_document_literal_end(text, at)
+        if literal_end is not None:
+            at = literal_end
+        elif text.startswith("{{", at):
+            marker = read_variable_marker(text, at)
+            if marker is None:
+                return
+            at = marker.end
+        elif text.startswith("?[", at):
+            end = find_interaction_end(text, at + 2)
+            if end < 0:
+                return
+            if text[end + 1 : end + 2] != "(":
+                yield at, end + 1
+            at = end + 1
+        else:
+            at += 1