markdown-flow 0.4.0__tar.gz → 0.4.2.dev1__tar.gz

{markdown_flow-0.4.0 → markdown_flow-0.4.2.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: markdown-flow
-Version: 0.4.0
+Version: 0.4.2.dev1
 Summary: An agent library designed to parse and process MarkdownFlow documents
 Project-URL: Homepage, https://github.com/ai-shifu/markdown-flow-agent-py
 Project-URL: Bug Tracker, https://github.com/ai-shifu/markdown-flow-agent-py/issues
@@ -708,8 +708,12 @@ literal. Only a leading assignment marker sets the interaction's variable;
 markers elsewhere in its body remain literal option text. An unterminated
 marker keeps the remaining text opaque, without restarting at an inner `?[`.
 
-At document level, complete inline code and HTML are read before markers inside
-them. Complete variable examples there still support collection and replacement,
+At document level, complete inline code, inline math, link resources, URI/email
+angle autolinks, indented code, and raw-text HTML bodies (`script`, `style`, `pre`, `textarea`) bound local
+variable examples. Quote/list-contained fences and ordinary HTML blocks also keep
+unfinished examples local. Reference-definition destinations and titles retain
+their complete variable examples. Physical CRLF, LF and CR boundaries preserve
+original offsets and marker text. Complete variable examples there still support collection and replacement,
 but an unclosed example cannot consume text beyond that markup span. Escaped
 document openers stay literal. A true raw marker owns its entire name before any
 code or HTML inside that name is considered; interaction bodies keep the raw

{markdown_flow-0.4.0 → markdown_flow-0.4.2.dev1}/README.md

@@ -690,8 +690,12 @@ literal. Only a leading assignment marker sets the interaction's variable;
 markers elsewhere in its body remain literal option text. An unterminated
 marker keeps the remaining text opaque, without restarting at an inner `?[`.
 
-At document level, complete inline code and HTML are read before markers inside
-them. Complete variable examples there still support collection and replacement,
+At document level, complete inline code, inline math, link resources, URI/email
+angle autolinks, indented code, and raw-text HTML bodies (`script`, `style`, `pre`, `textarea`) bound local
+variable examples. Quote/list-contained fences and ordinary HTML blocks also keep
+unfinished examples local. Reference-definition destinations and titles retain
+their complete variable examples. Physical CRLF, LF and CR boundaries preserve
+original offsets and marker text. Complete variable examples there still support collection and replacement,
 but an unclosed example cannot consume text beyond that markup span. Escaped
 document openers stay literal. A true raw marker owns its entire name before any
 code or HTML inside that name is considered; interaction bodies keep the raw

{markdown_flow-0.4.0 → markdown_flow-0.4.2.dev1}/markdown_flow/__init__.py

@@ -118,4 +118,4 @@ __all__ = [
     "format_variable_assignment",
 ]
 
-__version__ = "0.4.0"
+__version__ = "0.4.2.dev1"

{markdown_flow-0.4.0 → markdown_flow-0.4.2.dev1}/markdown_flow/constants.py

@@ -48,6 +48,19 @@ COMPILED_INLINE_HTML_REGEX = re.compile(
     r"|</[A-Za-z][A-Za-z0-9-]*" + _HTML_SPACE + r"*>"
     r"|<!--[\s\S]*?-->|<\?[\s\S]*?\?>|<![A-Z]+" + _HTML_SPACE + r"+[^>]*>|<!\[CDATA\[[\s\S]*?\]\]>)"
 )
+COMPILED_INDENTED_CODE_REGEX = re.compile(r"(?: {4}| {0,3}\t)")
+COMPILED_RAW_HTML_START_REGEX = re.compile(r" {0,3}<(script|style|pre|textarea)(?=[ \t\r\n\f>])", re.IGNORECASE)
+COMPILED_RAW_HTML_END_REGEX = {name: re.compile(r"</" + name + r"[ \t\r\n]*>", re.IGNORECASE) for name in ("script", "style", "pre", "textarea")}
+COMPILED_DOLLAR_RUN_REGEX = re.compile(r"\$+")
+COMPILED_BLANK_LINE_REGEX = re.compile(r"(?:\r\n|\r(?!\n)|(?<!\r)\n)[ \t]*(?:\r\n|\r(?!\n)|(?<!\r)\n)")
+COMPILED_ATX_START_REGEX = re.compile(r"^ {0,3}#{1,6}(?=[ \t\r\n]|$)", re.MULTILINE)
+COMPILED_PHYSICAL_LINE_END_REGEX = re.compile(r"\r\n|\r|\n")
+COMPILED_CONTAINER_LIST_REGEX = re.compile(r"(?:[-+*]|[0-9]{1,9}[.)])(?=[ \t])")
+COMPILED_DOCUMENT_FENCE_REGEX = re.compile(r"(`{3,})([^`]*$)|(~{3,})(.*$)")
+COMPILED_DOCUMENT_TAG_REGEX = re.compile(r" {0,3}</?([A-Za-z][A-Za-z0-9-]*)(?=[ \t/>]|$)")
+COMPILED_REFERENCE_PREFIX_REGEX = re.compile(r" {0,3}\[((?:\\[^\r\n]|[^\]\\\r\n])+)]\:[ \t]*")
+COMPILED_DOCUMENT_BREAK_REGEX = re.compile(r" {0,3}(?:(?:\*[ \t]*){3,}|(?:_[ \t]*){3,}|(?:-[ \t]*){3,})$")
+COMPILED_SETEXT_REGEX = re.compile(r" {0,3}(?:=+|-+)[ \t]*$")
 
 # Canonical user_input key for non-assignment interactions (?[A|B] without a
 # variable). Clients should submit {DEFAULT_NON_ASSIGNMENT_INPUT_KEY: values};

{markdown_flow-0.4.0 → markdown_flow-0.4.2.dev1}/markdown_flow/core.py

@@ -71,9 +71,8 @@ def _translation_marker_spans(text: str) -> list[tuple[int, int]]:
     return [(marker.start - (marker.start > 0 and text[marker.start - 1] == "%"), marker.end) for marker in find_variable_markers(text)]
 
 
-def _mask_translation_markers(text: str, markers: dict[str, str]) -> str:
-    """Replace whole markers with collision-free tokens, without reading inside keys."""
-    tokens = {raw: token for token, raw in markers.items()}
+def _mask_translation_markers(text: str, tokens: dict[str, str]) -> str:
+    """Replace whole markers using the interaction's shared raw-to-token index."""
     parts: list[str] = []
     at = 0
     for start, end in _translation_marker_spans(text):
@@ -83,23 +82,29 @@ def _mask_translation_markers(text: str, markers: dict[str, str]) -> str:
     return "".join(parts)
 
 
-def _restore_translation_markers(original: str, translated: Any, markers: dict[str, str]) -> str:
-    """Accept a translated field only when all its marker tokens survive in order."""
+def _translation_token_pattern(markers: dict[str, str]) -> re.Pattern[str] | None:
+    """Match only the generated token namespace, without enumerating every token."""
+    if not markers:
+        return None
+    prefix = next(iter(markers)).rsplit("_", 3)[0] + "_"
+    return re.compile(re.escape(prefix) + r"[0-9]+__")
+
+
+def _restore_translation_markers(original: str, translated: Any, markers: dict[str, str], token_pattern: re.Pattern[str] | None = None) -> str:
+    """Accept a translated field only when its known marker tokens survive in order."""
+    if not isinstance(translated, str):
+        translated = original
+    if not markers:
+        return translated
+    token_pattern = token_pattern if token_pattern is not None else _translation_token_pattern(markers)
+    assert token_pattern is not None
 
     def token_sequence(text: str) -> list[str]:
-        positions = []
-        for token in markers:
-            at = 0
-            while (at := text.find(token, at)) >= 0:
-                positions.append((at, token))
-                at += len(token)
-        return [token for _, token in sorted(positions)]
+        return [match.group() for match in token_pattern.finditer(text) if match.group() in markers]
 
-    if not isinstance(translated, str) or token_sequence(original) != token_sequence(translated):
+    if token_sequence(original) != token_sequence(translated):
         translated = original
-    for token, raw in markers.items():
-        translated = translated.replace(token, raw)
-    return translated
+    return token_pattern.sub(lambda match: markers.get(match.group(), match.group()), translated)
 
 
 def _find_translation_text(text: str, old: str) -> int:
@@ -1537,22 +1542,33 @@ class MarkdownFlow:
 
         # Keep marker source out of the translation request. The prefix cannot
         # collide with author text, and equal raw markers share the same token.
-        prefix = "__MDFLOW_VARIABLE_"
-        while prefix in interaction_content:
-            prefix = "_" + prefix
+        base = "__MDFLOW_VARIABLE_"
+        used_nonces: set[str] = set()
+        collision = False
+        for match in re.finditer(re.escape(base) + r"(?:N([0-9]+)_)?", interaction_content):
+            collision = True
+            if match.group(1) is not None:
+                used_nonces.add(match.group(1))
+        prefix = base
+        if collision:
+            nonce = 0
+            while str(nonce) in used_nonces:
+                nonce += 1
+            prefix = f"{base}N{nonce}_"
         raw_markers = dict.fromkeys(interaction_content[start:end] for start, end in _translation_marker_spans(interaction_content))
         markers = {f"{prefix}{i}__": raw for i, raw in enumerate(raw_markers)}
         interaction_info["_translation_markers"] = markers
+        tokens_by_raw = {raw: token for token, raw in markers.items()}
         translatable: dict[str, str | list[str]] = {}
 
         # Extract button display text
         if interaction_info.get("buttons"):
-            button_texts = [_mask_translation_markers(btn["display"], markers) for btn in interaction_info["buttons"]]
+            button_texts = [_mask_translation_markers(btn["display"], tokens_by_raw) for btn in interaction_info["buttons"]]
             translatable["buttons"] = button_texts
 
         # Extract question text
         if interaction_info.get("question"):
-            translatable["question"] = _mask_translation_markers(interaction_info["question"], markers)
+            translatable["question"] = _mask_translation_markers(interaction_info["question"], tokens_by_raw)
 
         # Convert to JSON
 
@@ -1627,6 +1643,7 @@ class MarkdownFlow:
             return original_content
 
         markers = interaction_info.get("_translation_markers", {})
+        token_pattern = _translation_token_pattern(markers)
         prefix = ""
         reconstructed = original_content
         marker = COMPILED_LAYER2_VARIABLE_REGEX.match(original_content[2:-1]) if original_content.startswith("?[") else None
@@ -1640,7 +1657,7 @@ class MarkdownFlow:
                 if i < len(translated["buttons"]):
                     old_display = button["display"]
                     original_display = original.get("buttons", [])[i]
-                    new_display = _restore_translation_markers(original_display, translated["buttons"][i], markers)
+                    new_display = _restore_translation_markers(original_display, translated["buttons"][i], markers, token_pattern)
 
                     # Detect if translation happened
                     translation_happened = False
@@ -1680,7 +1697,7 @@ class MarkdownFlow:
         # Replace question text
         if "question" in translated and interaction_info.get("question"):
             old_question = interaction_info["question"]
-            new_question = _restore_translation_markers(original["question"], translated["question"], markers)
+            new_question = _restore_translation_markers(original["question"], translated["question"], markers, token_pattern)
             reconstructed = _replace_translation_text(reconstructed, f"...{old_question}", f"...{new_question}")
 
         return prefix + reconstructed