markdown-flow 0.3.4__tar.gz → 0.4.1__tar.gz

{markdown_flow-0.3.4 → markdown_flow-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: markdown-flow
-Version: 0.3.4
+Version: 0.4.1
 Summary: An agent library designed to parse and process MarkdownFlow documents
 Project-URL: Homepage, https://github.com/ai-shifu/markdown-flow-agent-py
 Project-URL: Bug Tracker, https://github.com/ai-shifu/markdown-flow-agent-py/issues
@@ -691,6 +691,44 @@ builder.start_interaction()
 
 ### Variable System Deep Dive
 
+#### Text variable names
+
+Names are opaque Unicode text. Python `str.strip()` removes surrounding whitespace;
+the remaining name must be nonempty. Internal spaces, punctuation, emoji, single
+braces, brackets, `|`, `//`, and `...` stay exactly as written. Internal control
+characters (U+0000–001F and U+007F–009F) and U+2028/U+2029 are invalid. Case,
+repeated spaces, and Unicode sequences are never normalized; U+FEFF remains part
+of the name.
+
+The first literal `}}` closes a marker. A name cannot contain that pair. Quotes
+and backslashes are literal characters with no decoding rule, and nested `{{`
+inside a name is not another marker. Every complete `{{ ... }}` or `%{{ ... }}`
+is read before interaction delimiters, so internal `]`, `|`, `//`, and `...` stay
+literal. Only a leading assignment marker sets the interaction's variable;
+markers elsewhere in its body remain literal option text. An unterminated
+marker keeps the remaining text opaque, without restarting at an inner `?[`.
+
+At document level, complete inline code, inline math, link resources, indented
+code, and raw-text HTML bodies (`script`, `style`, `pre`, `textarea`) bound local
+variable examples. Quote/list-contained fences and ordinary HTML blocks also keep
+unfinished examples local. Reference-definition destinations and titles retain
+their complete variable examples. Physical CRLF, LF and CR boundaries preserve
+original offsets and marker text. Complete variable examples there still support collection and replacement,
+but an unclosed example cannot consume text beyond that markup span. Escaped
+document openers stay literal. A true raw marker owns its entire name before any
+code or HTML inside that name is considered; interaction bodies keep the raw
+marker grammar.
+
+Use `format_variable_reference(name)` and `format_variable_assignment(name)` from
+`markdown_flow.variable_names` to emit padded markers such as
+`{{ What is your gender? }}` and `%{{ a} }}`. Padding lets a name end in a single
+brace. Extraction and replacement use the same trimmed key; existing stored keys
+are not renamed or migrated.
+
+The shared conformance corpus is `scripts/variable-name-contract.json`. Run its
+offline parser, document, replacement, translation, and answer round trips with
+`python -m pytest scripts/test_variable_names.py -q`.
+
 ```python
 from markdown_flow import extract_variables_from_text, replace_variables_in_text
 

{markdown_flow-0.3.4 → markdown_flow-0.4.1}/README.md

@@ -673,6 +673,44 @@ builder.start_interaction()
 
 ### Variable System Deep Dive
 
+#### Text variable names
+
+Names are opaque Unicode text. Python `str.strip()` removes surrounding whitespace;
+the remaining name must be nonempty. Internal spaces, punctuation, emoji, single
+braces, brackets, `|`, `//`, and `...` stay exactly as written. Internal control
+characters (U+0000–001F and U+007F–009F) and U+2028/U+2029 are invalid. Case,
+repeated spaces, and Unicode sequences are never normalized; U+FEFF remains part
+of the name.
+
+The first literal `}}` closes a marker. A name cannot contain that pair. Quotes
+and backslashes are literal characters with no decoding rule, and nested `{{`
+inside a name is not another marker. Every complete `{{ ... }}` or `%{{ ... }}`
+is read before interaction delimiters, so internal `]`, `|`, `//`, and `...` stay
+literal. Only a leading assignment marker sets the interaction's variable;
+markers elsewhere in its body remain literal option text. An unterminated
+marker keeps the remaining text opaque, without restarting at an inner `?[`.
+
+At document level, complete inline code, inline math, link resources, indented
+code, and raw-text HTML bodies (`script`, `style`, `pre`, `textarea`) bound local
+variable examples. Quote/list-contained fences and ordinary HTML blocks also keep
+unfinished examples local. Reference-definition destinations and titles retain
+their complete variable examples. Physical CRLF, LF and CR boundaries preserve
+original offsets and marker text. Complete variable examples there still support collection and replacement,
+but an unclosed example cannot consume text beyond that markup span. Escaped
+document openers stay literal. A true raw marker owns its entire name before any
+code or HTML inside that name is considered; interaction bodies keep the raw
+marker grammar.
+
+Use `format_variable_reference(name)` and `format_variable_assignment(name)` from
+`markdown_flow.variable_names` to emit padded markers such as
+`{{ What is your gender? }}` and `%{{ a} }}`. Padding lets a name end in a single
+brace. Extraction and replacement use the same trimmed key; existing stored keys
+are not renamed or migrated.
+
+The shared conformance corpus is `scripts/variable-name-contract.json`. Run its
+offline parser, document, replacement, translation, and answer round trips with
+`python -m pytest scripts/test_variable_names.py -q`.
+
 ```python
 from markdown_flow import extract_variables_from_text, replace_variables_in_text
 

{markdown_flow-0.3.4 → markdown_flow-0.4.1}/markdown_flow/__init__.py

@@ -75,6 +75,13 @@ from .parser import (
     generate_smart_validation_template,
     replace_variables_in_text,
 )
+from .variable_names import (
+    find_variable_markers,
+    format_variable_assignment,
+    format_variable_reference,
+    normalize_variable_name,
+    read_variable_marker,
+)
 
 
 # Public API
@@ -104,6 +111,11 @@ __all__ = [
     "extract_interaction_question",
     "extract_variables_from_text",
     "replace_variables_in_text",
+    "normalize_variable_name",
+    "read_variable_marker",
+    "find_variable_markers",
+    "format_variable_reference",
+    "format_variable_assignment",
 ]
 
-__version__ = "0.3.4"
+__version__ = "0.4.1"

{markdown_flow-0.3.4 → markdown_flow-0.4.1}/markdown_flow/constants.py

@@ -6,11 +6,15 @@ Constants for document parsing, variable matching, validation, and other core fu
 
 import re
 
+from .variable_names import (
+    VARIABLE_ASSIGNMENT_SOURCE,
+    VARIABLE_REFERENCE_SOURCE,
+    VARIABLE_REFERENCE_UNIT_SOURCE,
+)
+
 
 # Pre-compiled regex patterns
-COMPILED_PERCENT_VARIABLE_REGEX = re.compile(
-    r"%\{\{([^}]+)\}\}"  # Match %{{variable}} format for preserved variables
-)
+COMPILED_PERCENT_VARIABLE_REGEX = re.compile(VARIABLE_ASSIGNMENT_SOURCE)
 
 # Interaction regex base patterns
 #
@@ -21,7 +25,7 @@ COMPILED_PERCENT_VARIABLE_REGEX = re.compile(
 # a left-to-right scanner disagree about `?[a\\]`, and two parsers of one grammar must not.
 # A backslash before anything else -- LaTeX such as `$\pi$`, or `\\` as a line break -- stays
 # ordinary text, exactly as it was before escapes existed. See markdown_flow.escaping.
-_INTERACTION_CONTENT = r"(?:\\[|/.\]]|\\(?![|/.\]])|[^\]\\])*"
+_INTERACTION_CONTENT = rf"(?:{VARIABLE_REFERENCE_UNIT_SOURCE}|\\[|/.\]]|\\(?![|/.\]])|(?!\{{\{{)[^\]\\])*"
 INTERACTION_PATTERN = rf"(?<!\\)\?\[({_INTERACTION_CONTENT})\](?!\()"  # Base pattern with capturing group for content extraction, excludes escaped \?[]
 INTERACTION_PATTERN_NON_CAPTURING = rf"(?<!\\)\?\[{_INTERACTION_CONTENT}\](?!\()"  # Non-capturing version for block splitting, excludes escaped \?[]
 INTERACTION_PATTERN_SPLIT = rf"((?<!\\)\?\[{_INTERACTION_CONTENT}\](?!\())"  # Pattern for re.split() with outer capturing group, excludes escaped \?[]
@@ -29,13 +33,34 @@ INTERACTION_PATTERN_SPLIT = rf"((?<!\\)\?\[{_INTERACTION_CONTENT}\](?!\())"  # P
 # InteractionParser specific regex patterns
 COMPILED_INTERACTION_REGEX = re.compile(INTERACTION_PATTERN)  # Main interaction pattern matcher
 COMPILED_LAYER1_INTERACTION_REGEX = COMPILED_INTERACTION_REGEX  # Layer 1: Basic format validation (alias)
-COMPILED_LAYER2_VARIABLE_REGEX = re.compile(r"^%\{\{([^}]+)\}\}(.*)$")  # Layer 2: Variable detection
+COMPILED_LAYER2_VARIABLE_REGEX = re.compile(r"^\s*" + VARIABLE_ASSIGNMENT_SOURCE + r"(.*)$")
 COMPILED_LAYER3_ELLIPSIS_REGEX = re.compile(r"^(.*)\.\.\.(.*)")  # Layer 3: Split content around ellipsis
 COMPILED_LAYER3_BUTTON_VALUE_REGEX = re.compile(r"^(.+)//(.+)$")  # Layer 3: Parse Button//value format
-COMPILED_BRACE_VARIABLE_REGEX = re.compile(
-    r"(?<!%)\{\{([^}]+)\}\}"  # Match {{variable}} format for replaceable variables
-)
+COMPILED_BRACE_VARIABLE_REGEX = re.compile(r"(?<!%)" + VARIABLE_REFERENCE_SOURCE)
 COMPILED_SINGLE_PIPE_SPLIT_REGEX = re.compile(r"(?<!\|)\|(?!\|)")  # Split on single | but not ||
+COMPILED_VARIABLE_NAME_PLACEHOLDER_REGEX = re.compile(r"\x00mdf-name:+[0-9]+\x00")
+
+# Inline HTML owns braces in its tag source before document-level MDF scanning.
+_HTML_SPACE = r"[ \t\r\n]"
+_HTML_ATTRIBUTE = r"[A-Za-z_:][A-Za-z0-9_.:-]*(?:" + _HTML_SPACE + r"*=" + _HTML_SPACE + r"*(?:[^ \t\r\n\"'=<>`]+|'[^']*'|\"[^\"]*\"))?"
+COMPILED_INLINE_HTML_REGEX = re.compile(
+    r"(?:<[A-Za-z][A-Za-z0-9-]*(?:" + _HTML_SPACE + r"+" + _HTML_ATTRIBUTE + r")*" + _HTML_SPACE + r"*/?>"
+    r"|</[A-Za-z][A-Za-z0-9-]*" + _HTML_SPACE + r"*>"
+    r"|<!--[\s\S]*?-->|<\?[\s\S]*?\?>|<![A-Z]+" + _HTML_SPACE + r"+[^>]*>|<!\[CDATA\[[\s\S]*?\]\]>)"
+)
+COMPILED_INDENTED_CODE_REGEX = re.compile(r"(?: {4}| {0,3}\t)")
+COMPILED_RAW_HTML_START_REGEX = re.compile(r" {0,3}<(script|style|pre|textarea)(?=[ \t\r\n\f>])", re.IGNORECASE)
+COMPILED_RAW_HTML_END_REGEX = {name: re.compile(r"</" + name + r"[ \t\r\n]*>", re.IGNORECASE) for name in ("script", "style", "pre", "textarea")}
+COMPILED_DOLLAR_RUN_REGEX = re.compile(r"\$+")
+COMPILED_BLANK_LINE_REGEX = re.compile(r"(?:\r\n|\r(?!\n)|(?<!\r)\n)[ \t]*(?:\r\n|\r(?!\n)|(?<!\r)\n)")
+COMPILED_ATX_START_REGEX = re.compile(r"^ {0,3}#{1,6}(?=[ \t\r\n]|$)", re.MULTILINE)
+COMPILED_PHYSICAL_LINE_END_REGEX = re.compile(r"\r\n|\r|\n")
+COMPILED_CONTAINER_LIST_REGEX = re.compile(r"(?:[-+*]|[0-9]{1,9}[.)])(?=[ \t])")
+COMPILED_DOCUMENT_FENCE_REGEX = re.compile(r"(`{3,})([^`]*$)|(~{3,})(.*$)")
+COMPILED_DOCUMENT_TAG_REGEX = re.compile(r" {0,3}</?([A-Za-z][A-Za-z0-9-]*)(?=[ \t/>]|$)")
+COMPILED_REFERENCE_PREFIX_REGEX = re.compile(r" {0,3}\[((?:\\[^\r\n]|[^\]\\\r\n])+)]\:[ \t]*")
+COMPILED_DOCUMENT_BREAK_REGEX = re.compile(r" {0,3}(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,})$")
+COMPILED_SETEXT_REGEX = re.compile(r" {0,3}(?:=+|-+)[ \t]*$")
 
 # Canonical user_input key for non-assignment interactions (?[A|B] without a
 # variable). Clients should submit {DEFAULT_NON_ASSIGNMENT_INPUT_KEY: values};
@@ -135,6 +160,7 @@ INTERACTION_PROMPT_WITH_TRANSLATION = """
 
 - Translate the text in `buttons` and `question` into the target language
 - Translate only the display text (the Display part); do not change the JSON structure
+- Tokens such as `__MDFLOW_VARIABLE_0__` are opaque placeholders: copy every token occurrence exactly as given in its original order and translate only the surrounding text
 - If `display//value` separation exists (for example `Yes//1`), translate only the Display part before `//` and keep value unchanged
 - Use the target language 100%; language mixing is not allowed. Translate all non-target-language words before outputting
 - If a text segment's original language is already the same as the target language, return that text unchanged (for example, when the target language is English and the original text is also English, do not rewrite it)

{markdown_flow-0.3.4 → markdown_flow-0.4.1}/markdown_flow/core.py

@@ -13,6 +13,7 @@ from typing import Any
 from .constants import (
     BLOCK_INDEX_OUT_OF_RANGE_ERROR,
     BLOCK_SEPARATOR,
+    COMPILED_LAYER2_VARIABLE_REGEX,
     CONTEXT_BUTTON_OPTIONS_TEMPLATE,
     CONTEXT_QUESTION_TEMPLATE,
     DEFAULT_INTERACTION_ERROR_PROMPT,
@@ -21,8 +22,6 @@ from .constants import (
     INPUT_EMPTY_ERROR,
     INTERACTION_ERROR_RENDER_INSTRUCTIONS,
     INTERACTION_PARSE_ERROR,
-    INTERACTION_PATTERN_NON_CAPTURING,
-    INTERACTION_PATTERN_SPLIT,
     INTERACTION_PROMPT_BASE,
     INTERACTION_PROMPT_WITH_TRANSLATION,
     LLM_PROVIDER_REQUIRED_ERROR,
@@ -62,7 +61,76 @@ from .parser import (
     process_output_instructions,
     replace_variables_in_text,
 )
+from .parser.document_context import find_document_interaction_spans, find_document_protected_spans
 from .tag_filter import StreamTagFilter, strip_preserve_tags
+from .variable_names import find_variable_markers
+
+
+def _translation_marker_spans(text: str) -> list[tuple[int, int]]:
+    """Include the preserved-variable prefix in each complete marker's source span."""
+    return [(marker.start - (marker.start > 0 and text[marker.start - 1] == "%"), marker.end) for marker in find_variable_markers(text)]
+
+
+def _mask_translation_markers(text: str, markers: dict[str, str]) -> str:
+    """Replace whole markers with collision-free tokens, without reading inside keys."""
+    tokens = {raw: token for token, raw in markers.items()}
+    parts: list[str] = []
+    at = 0
+    for start, end in _translation_marker_spans(text):
+        parts.extend((text[at:start], tokens[text[start:end]]))
+        at = end
+    parts.append(text[at:])
+    return "".join(parts)
+
+
+def _restore_translation_markers(original: str, translated: Any, markers: dict[str, str]) -> str:
+    """Accept a translated field only when all its marker tokens survive in order."""
+
+    def token_sequence(text: str) -> list[str]:
+        positions = []
+        for token in markers:
+            at = 0
+            while (at := text.find(token, at)) >= 0:
+                positions.append((at, token))
+                at += len(token)
+        return [token for _, token in sorted(positions)]
+
+    if not isinstance(translated, str) or token_sequence(original) != token_sequence(translated):
+        translated = original
+    for token, raw in markers.items():
+        translated = translated.replace(token, raw)
+    return translated
+
+
+def _find_translation_text(text: str, old: str) -> int:
+    """Locate a display span without matching a substring inside an opaque marker."""
+    if not old:
+        return -1
+    spans = _translation_marker_spans(text)
+    at = 0
+    while (at := text.find(old, at)) >= 0:
+        end = at + len(old)
+        if not any(at < marker_end and end > marker_start and (at > marker_start or end < marker_end) for marker_start, marker_end in spans):
+            return at
+        at += 1
+    return -1
+
+
+def _replace_translation_text(text: str, old: str, new: str) -> str:
+    """Replace only a complete display span, leaving marker internals untouched."""
+    at = _find_translation_text(text, old)
+    return text[:at] + new + text[at + len(old) :] if at >= 0 else text
+
+
+def _strip_translation_tags(text: str) -> str:
+    """Filter generated tags while retaining literal tag text inside variable names."""
+    parts: list[str] = []
+    at = 0
+    for marker in find_variable_markers(text):
+        parts.extend((strip_preserve_tags(text[at : marker.start]), text[marker.start : marker.end]))
+        at = marker.end
+    parts.append(strip_preserve_tags(text[at:]))
+    return "".join(parts)
 
 
 class MarkdownFlow:
@@ -468,18 +536,32 @@ class MarkdownFlow:
 
         # Parse the preprocessed document (code blocks already replaced with placeholders)
         # The preprocessing was done once during initialization
-        segments = re.split(BLOCK_SEPARATOR, self._processed_document)
+        source = self._processed_document
+        protected = list(find_document_protected_spans(source))
+        segments = []
+        at = 0
+        protected_at = 0
+        for separator in re.finditer(BLOCK_SEPARATOR, source):
+            while protected_at < len(protected) and protected[protected_at][1] <= separator.start():
+                protected_at += 1
+            if protected_at < len(protected) and protected[protected_at][0] < separator.end():
+                continue
+            segments.append(source[at : separator.start()])
+            at = separator.end()
+        segments.append(source[at:])
         final_blocks: list[Block] = []
 
         for segment in segments:
-            # Use dedicated split pattern to avoid duplicate blocks from capturing groups
-            parts = re.split(INTERACTION_PATTERN_SPLIT, segment)
-
-            for part in parts:
+            at = 0
+            parts: list[tuple[str, bool]] = []
+            for start, end in find_document_interaction_spans(segment):
+                parts.extend(((segment[at:start], False), (segment[start:end], True)))
+                at = end
+            parts.append((segment[at:], False))
+            for part, interaction in parts:
                 part = part.strip()
                 if part:
-                    # Use non-capturing pattern for matching
-                    if re.match(INTERACTION_PATTERN_NON_CAPTURING, part):
+                    if interaction:
                         block = Block(
                             content=part,
                             block_type=BlockType.INTERACTION,
@@ -763,7 +845,7 @@ class MarkdownFlow:
             translated_content = self._reconstruct_with_translation(processed_block.content, translatable_json, translated_json, interaction_info)
 
             # 过滤 LLM 输出中可能泄露的 原样输出 标签
-            translated_content = strip_preserve_tags(translated_content)
+            translated_content = _strip_translation_tags(translated_content)
 
             return LLMResult(
                 content=translated_content,
@@ -805,7 +887,7 @@ class MarkdownFlow:
                 translated_content = self._reconstruct_with_translation(processed_block.content, translatable_json, full_response, interaction_info)
 
                 # 过滤 LLM 输出中可能泄露的 原样输出 标签
-                translated_content = strip_preserve_tags(translated_content)
+                translated_content = _strip_translation_tags(translated_content)
 
                 # Return complete content once (not incremental)
                 yield LLMResult(
@@ -1453,16 +1535,24 @@ class MarkdownFlow:
         if not interaction_info:
             return "{}", None
 
-        translatable = {}
+        # Keep marker source out of the translation request. The prefix cannot
+        # collide with author text, and equal raw markers share the same token.
+        prefix = "__MDFLOW_VARIABLE_"
+        while prefix in interaction_content:
+            prefix = "_" + prefix
+        raw_markers = dict.fromkeys(interaction_content[start:end] for start, end in _translation_marker_spans(interaction_content))
+        markers = {f"{prefix}{i}__": raw for i, raw in enumerate(raw_markers)}
+        interaction_info["_translation_markers"] = markers
+        translatable: dict[str, str | list[str]] = {}
 
         # Extract button display text
         if interaction_info.get("buttons"):
-            button_texts = [btn["display"] for btn in interaction_info["buttons"]]
+            button_texts = [_mask_translation_markers(btn["display"], markers) for btn in interaction_info["buttons"]]
             translatable["buttons"] = button_texts
 
         # Extract question text
         if interaction_info.get("question"):
-            translatable["question"] = interaction_info["question"]
+            translatable["question"] = _mask_translation_markers(interaction_info["question"], markers)
 
         # Convert to JSON
 
@@ -1536,19 +1626,26 @@ class MarkdownFlow:
         except (json.JSONDecodeError, ValueError):
             return original_content
 
+        markers = interaction_info.get("_translation_markers", {})
+        prefix = ""
         reconstructed = original_content
+        marker = COMPILED_LAYER2_VARIABLE_REGEX.match(original_content[2:-1]) if original_content.startswith("?[") else None
+        if marker:
+            body_at = 2 + marker.start(2)
+            prefix, reconstructed = original_content[:body_at], original_content[body_at:]
 
         # Replace button display text (smart value handling)
         if "buttons" in translated and interaction_info.get("buttons"):
             for i, button in enumerate(interaction_info["buttons"]):
                 if i < len(translated["buttons"]):
                     old_display = button["display"]
-                    new_display = translated["buttons"][i]
+                    original_display = original.get("buttons", [])[i]
+                    new_display = _restore_translation_markers(original_display, translated["buttons"][i], markers)
 
                     # Detect if translation happened
                     translation_happened = False
                     if "buttons" in original and i < len(original["buttons"]):
-                        if original["buttons"][i] != new_display:
+                        if old_display != new_display:
                             translation_happened = True
 
                     # If value separation exists (display//value format), preserve value
@@ -1557,36 +1654,36 @@ class MarkdownFlow:
                         # Replace format: oldDisplay//value -> newDisplay//value
                         old_pattern = f"{old_display}//{button['value']}"
                         new_pattern = f"{new_display}//{button['value']}"
-                        reconstructed = reconstructed.replace(old_pattern, new_pattern, 1)
+                        reconstructed = _replace_translation_text(reconstructed, old_pattern, new_pattern)
                     elif translation_happened:
                         # Display == Value, but need to check if display//value format exists in original text
                         # In this case, original text is like "小兔子//小兔子", both Display and Value are "小兔子"
                         old_pattern_with_separator = f"{old_display}//"
-                        if old_pattern_with_separator in reconstructed:
+                        if _find_translation_text(reconstructed, old_pattern_with_separator) >= 0:
                             # Original text has display//value format (e.g., "小兔子//小兔子")
                             # Only replace display part, preserve original value
                             # Example: 小兔子//小兔子 -> 토끼//小兔子
                             old_pattern = f"{old_display}//{button['value']}"
                             new_pattern = f"{new_display}//{button['value']}"
-                            reconstructed = reconstructed.replace(old_pattern, new_pattern, 1)
+                            reconstructed = _replace_translation_text(reconstructed, old_pattern, new_pattern)
                         else:
                             # Original text has no // separator (e.g., "小兔子")
                             # Auto-add value: translated//original
                             # Example: 小兔子 -> 토끼//小兔子
                             old_pattern = old_display
                             new_pattern = f"{new_display}//{old_display}"
-                            reconstructed = reconstructed.replace(old_pattern, new_pattern, 1)
+                            reconstructed = _replace_translation_text(reconstructed, old_pattern, new_pattern)
                     else:
                         # No translation, keep as is
-                        reconstructed = reconstructed.replace(old_display, new_display, 1)
+                        reconstructed = _replace_translation_text(reconstructed, old_display, new_display)
 
         # Replace question text
         if "question" in translated and interaction_info.get("question"):
             old_question = interaction_info["question"]
-            new_question = translated["question"]
-            reconstructed = reconstructed.replace(f"...{old_question}", f"...{new_question}", 1)
+            new_question = _restore_translation_markers(original["question"], translated["question"], markers)
+            reconstructed = _replace_translation_text(reconstructed, f"...{old_question}", f"...{new_question}")
 
-        return reconstructed
+        return prefix + reconstructed
 
     def _build_validation_messages(
         self,
@@ -1630,10 +1727,10 @@ class MarkdownFlow:
         # 2. Validation task template (choose based on language requirement)
         if has_language_requirement:
             # Has language requirement: use Base + WithLanguage
-            task_template = (VALIDATION_TASK_BASE + VALIDATION_TASK_WITH_LANGUAGE).replace("{target_variable}", target_variable)
+            task_template = (VALIDATION_TASK_BASE + VALIDATION_TASK_WITH_LANGUAGE).replace('"{target_variable}"', json.dumps(target_variable, ensure_ascii=False))
         else:
             # No language requirement: use default (Base + NoLanguage)
-            task_template = VALIDATION_TASK_TEMPLATE.replace("{target_variable}", target_variable)
+            task_template = VALIDATION_TASK_TEMPLATE.replace('"{target_variable}"', json.dumps(target_variable, ensure_ascii=False))
 
         system_parts.append(task_template)
 
@@ -1662,7 +1759,7 @@ class MarkdownFlow:
 
         # Add extraction requirements (using template)
         system_parts.append("")
-        system_parts.append(VALIDATION_REQUIREMENTS_TEMPLATE)
+        system_parts.append(VALIDATION_REQUIREMENTS_TEMPLATE.replace('"{target_variable}"', json.dumps(target_variable, ensure_ascii=False)))
 
         # 3. Output language anchoring (bottom - if output language is set)
         if has_language_requirement:

{markdown_flow-0.3.4 → markdown_flow-0.4.1}/markdown_flow/escaping.py

@@ -18,6 +18,9 @@ The cost is that a literal backslash cannot sit immediately before a literal del
 an escaped bar, not a backslash and a separator. Nothing in the measured corpus writes that.
 """
 
+from .variable_names import read_variable_marker
+
+
 # The characters a backslash may make ordinary. `/` covers `//`, `.` covers `...`.
 ESCAPABLE = "|/.]"
 
@@ -45,6 +48,12 @@ def escape_interaction_text(text: str) -> str:
     out: list[str] = []
     index = 0
     while index < len(text):
+        if text.startswith("{{", index):
+            marker = read_variable_marker(text, index)
+            end = marker.end if marker is not None else len(text)
+            out.append(text[index:end])
+            index = end
+            continue
         char = text[index]
         if char == ".":
             run = len(text) - index - len(text[index:].lstrip("."))
@@ -72,6 +81,12 @@ def unescape_interaction_text(text: str) -> str:
     out: list[str] = []
     index = 0
     while index < len(text):
+        if text.startswith("{{", index):
+            marker = read_variable_marker(text, index)
+            end = marker.end if marker is not None else len(text)
+            out.append(text[index:end])
+            index = end
+            continue
         if is_escape(text, index):
             out.append(text[index + 1])
             index += 2
@@ -90,6 +105,12 @@ def find_unescaped(text: str, needle: str, start: int = 0) -> int:
     """
     index = start
     while index < len(text):
+        if text.startswith("{{", index):
+            marker = read_variable_marker(text, index)
+            if marker is None:
+                return -1
+            index = marker.end
+            continue
         if is_escape(text, index):
             index += 2
             continue
@@ -118,6 +139,10 @@ def split_on_single_pipe(text: str) -> list[str]:
     start = 0
     index = 0
     while index < len(text):
+        if text.startswith("{{", index):
+            marker = read_variable_marker(text, index)
+            index = marker.end if marker is not None else len(text)
+            continue
         if is_escape(text, index):
             index += 2
             continue
@@ -160,7 +185,8 @@ def split_on_ellipsis(content: str) -> tuple[str, str] | None:
 
     What is new is only that an escaped ellipsis no longer counts.
     """
-    head = content.split("\n", 1)[0]
+    newline = find_unescaped(content, "\n")
+    head = content[:newline] if newline >= 0 else content
     at = rfind_unescaped(head, "...")
     if at < 0:
         return None