markdown-flow 0.2.16__py3-none-any.whl → 0.2.26__py3-none-any.whl

markdown_flow/core.py

@@ -18,6 +18,7 @@ from .constants import (
     COMPILED_INTERACTION_CONTENT_RECONSTRUCT_REGEX,
     COMPILED_VARIABLE_REFERENCE_CLEANUP_REGEX,
     COMPILED_WHITESPACE_CLEANUP_REGEX,
+    DEFAULT_BASE_SYSTEM_PROMPT,
     DEFAULT_INTERACTION_ERROR_PROMPT,
     DEFAULT_INTERACTION_PROMPT,
     DEFAULT_VALIDATION_SYSTEM_MESSAGE,
@@ -28,13 +29,14 @@ from .constants import (
     INTERACTION_PATTERN_SPLIT,
     INTERACTION_RENDER_INSTRUCTIONS,
     LLM_PROVIDER_REQUIRED_ERROR,
+    OUTPUT_INSTRUCTION_EXPLANATION,
     UNSUPPORTED_PROMPT_TYPE_ERROR,
 )
 from .enums import BlockType
 from .exceptions import BlockIndexError
 from .llm import LLMProvider, LLMResult, ProcessMode
 from .models import Block, InteractionValidationConfig
-from .utils import (
+from .parser import (
     InteractionParser,
     InteractionType,
     extract_interaction_question,
@@ -59,48 +61,105 @@ class MarkdownFlow:
     _document_prompt: str | None
     _interaction_prompt: str | None
     _interaction_error_prompt: str | None
+    _max_context_length: int
     _blocks: list[Block] | None
     _interaction_configs: dict[int, InteractionValidationConfig]
+    _model: str | None
+    _temperature: float | None
 
     def __init__(
         self,
         document: str,
         llm_provider: LLMProvider | None = None,
+        base_system_prompt: str | None = None,
         document_prompt: str | None = None,
         interaction_prompt: str | None = None,
         interaction_error_prompt: str | None = None,
+        max_context_length: int = 0,
     ):
         """
         Initialize MarkdownFlow instance.
 
         Args:
             document: Markdown document content
-            llm_provider: LLM provider, if None only PROMPT_ONLY mode is available
+            llm_provider: LLM provider (required for COMPLETE and STREAM modes)
+            base_system_prompt: MarkdownFlow base system prompt (framework-level, content blocks only)
             document_prompt: Document-level system prompt
             interaction_prompt: Interaction content rendering prompt
             interaction_error_prompt: Interaction error rendering prompt
+            max_context_length: Maximum number of context messages to keep (0 = unlimited)
         """
         self._document = document
         self._llm_provider = llm_provider
+        self._base_system_prompt = base_system_prompt or DEFAULT_BASE_SYSTEM_PROMPT
         self._document_prompt = document_prompt
         self._interaction_prompt = interaction_prompt or DEFAULT_INTERACTION_PROMPT
         self._interaction_error_prompt = interaction_error_prompt or DEFAULT_INTERACTION_ERROR_PROMPT
+        self._max_context_length = max_context_length
         self._blocks = None
         self._interaction_configs: dict[int, InteractionValidationConfig] = {}
+        self._model: str | None = None
+        self._temperature: float | None = None
 
     def set_llm_provider(self, provider: LLMProvider) -> None:
         """Set LLM provider."""
         self._llm_provider = provider
 
+    def set_model(self, model: str) -> "MarkdownFlow":
+        """
+        Set model name for this instance.
+
+        Args:
+            model: Model name to use
+
+        Returns:
+            Self for method chaining
+        """
+        self._model = model
+        return self
+
+    def set_temperature(self, temperature: float) -> "MarkdownFlow":
+        """
+        Set temperature for this instance.
+
+        Args:
+            temperature: Temperature value (typically 0.0-2.0)
+
+        Returns:
+            Self for method chaining
+        """
+        self._temperature = temperature
+        return self
+
+    def get_model(self) -> str | None:
+        """
+        Get model name for this instance.
+
+        Returns:
+            Model name if set, None otherwise
+        """
+        return self._model
+
+    def get_temperature(self) -> float | None:
+        """
+        Get temperature for this instance.
+
+        Returns:
+            Temperature value if set, None otherwise
+        """
+        return self._temperature
+
     def set_prompt(self, prompt_type: str, value: str | None) -> None:
         """
         Set prompt template.
 
         Args:
-            prompt_type: Prompt type ('document', 'interaction', 'interaction_error')
+            prompt_type: Prompt type ('base_system', 'document', 'interaction', 'interaction_error')
             value: Prompt content
         """
-        if prompt_type == "document":
+        if prompt_type == "base_system":
+            self._base_system_prompt = value or DEFAULT_BASE_SYSTEM_PROMPT
+        elif prompt_type == "document":
             self._document_prompt = value
         elif prompt_type == "interaction":
             self._interaction_prompt = value or DEFAULT_INTERACTION_PROMPT
@@ -109,6 +168,44 @@ class MarkdownFlow:
         else:
             raise ValueError(UNSUPPORTED_PROMPT_TYPE_ERROR.format(prompt_type=prompt_type))
 
+    def _truncate_context(
+        self,
+        context: list[dict[str, str]] | None,
+    ) -> list[dict[str, str]] | None:
+        """
+        Filter and truncate context to specified maximum length.
+
+        Processing steps:
+        1. Filter out messages with empty content (empty string or whitespace only)
+        2. Truncate to max_context_length if configured (0 = unlimited)
+
+        Args:
+            context: Original context list
+
+        Returns:
+            Filtered and truncated context. Returns None if no valid messages remain.
+        """
+        if not context:
+            return None
+
+        # Step 1: Filter out messages with empty or whitespace-only content
+        filtered_context = [msg for msg in context if msg.get("content", "").strip()]
+
+        # Return None if no valid messages remain after filtering
+        if not filtered_context:
+            return None
+
+        # Step 2: Truncate to max_context_length if configured
+        if self._max_context_length == 0:
+            # No limit, return all filtered messages
+            return filtered_context
+
+        # Keep the most recent N messages
+        if len(filtered_context) > self._max_context_length:
+            return filtered_context[-self._max_context_length :]
+
+        return filtered_context
+
     @property
     def document(self) -> str:
         """Get document content."""
@@ -183,8 +280,7 @@ class MarkdownFlow:
         context: list[dict[str, str]] | None = None,
         variables: dict[str, str | list[str]] | None = None,
         user_input: dict[str, list[str]] | None = None,
-        dynamic_interaction_format: str | None = None,
-    ) -> LLMResult | Generator[LLMResult, None, None]:
+    ):
         """
         Unified block processing interface.
 
@@ -194,11 +290,14 @@ class MarkdownFlow:
             context: Context message list
             variables: Variable mappings
             user_input: User input (for interaction blocks)
-            dynamic_interaction_format: Dynamic interaction format for validation
 
         Returns:
             LLMResult or Generator[LLMResult, None, None]
         """
+        # Process base_system_prompt variable replacement
+        if self._base_system_prompt:
+            self._base_system_prompt = replace_variables_in_text(self._base_system_prompt, variables or {})
+
         # Process document_prompt variable replacement
         if self._document_prompt:
             self._document_prompt = replace_variables_in_text(self._document_prompt, variables or {})
@@ -206,16 +305,12 @@ class MarkdownFlow:
         block = self.get_block(block_index)
 
         if block.block_type == BlockType.CONTENT:
-            # Check if this is dynamic interaction validation
-            if dynamic_interaction_format and user_input:
-                return self._process_dynamic_interaction_validation(block_index, dynamic_interaction_format, user_input, mode, context, variables)
-            # Normal content processing (possibly with dynamic conversion)
             return self._process_content(block_index, mode, context, variables)
 
         if block.block_type == BlockType.INTERACTION:
             if user_input is None:
                 # Render interaction content
-                return self._process_interaction_render(block_index, mode, variables)
+                return self._process_interaction_render(block_index, mode, context, variables)
             # Process user input
             return self._process_interaction_input(block_index, user_input, mode, context, variables)
 
@@ -234,37 +329,27 @@ class MarkdownFlow:
         mode: ProcessMode,
         context: list[dict[str, str]] | None,
         variables: dict[str, str | list[str]] | None,
-    ) -> LLMResult | Generator[LLMResult, None, None]:
+    ):
         """Process content block."""
+        # Truncate context to configured maximum length
+        truncated_context = self._truncate_context(context)
 
-        # For PROMPT_ONLY mode, use standard content processing
-        if mode == ProcessMode.PROMPT_ONLY:
-            messages = self._build_content_messages(block_index, variables)
-            return LLMResult(prompt=messages[-1]["content"], metadata={"messages": messages})
-
-        # For COMPLETE and STREAM modes with LLM provider, use dynamic interaction check
-        # LLM will decide whether content needs to be converted to interaction block
-        if self._llm_provider:
-            block = self.get_block(block_index)
-            if block.block_type == BlockType.CONTENT:
-                return self._process_with_dynamic_check(block_index, mode, context, variables)
-
-        # Fallback: Build messages using standard content processing
-        messages = self._build_content_messages(block_index, variables)
+        # Build messages with context
+        messages = self._build_content_messages(block_index, variables, truncated_context)
 
         if mode == ProcessMode.COMPLETE:
             if not self._llm_provider:
                 raise ValueError(LLM_PROVIDER_REQUIRED_ERROR)
 
-            result = self._llm_provider.complete(messages)
-            return LLMResult(content=result.content, prompt=messages[-1]["content"], metadata=result.metadata)
+            content = self._llm_provider.complete(messages, model=self._model, temperature=self._temperature)
+            return LLMResult(content=content, prompt=messages[-1]["content"])
 
         if mode == ProcessMode.STREAM:
             if not self._llm_provider:
                 raise ValueError(LLM_PROVIDER_REQUIRED_ERROR)
 
             def stream_generator():
-                for chunk in self._llm_provider.stream(messages):
+                for chunk in self._llm_provider.stream(messages, model=self._model, temperature=self._temperature):  # type: ignore[attr-defined]
                     yield LLMResult(content=chunk, prompt=messages[-1]["content"])
 
             return stream_generator()
@@ -281,7 +366,13 @@ class MarkdownFlow:
 
         return LLMResult(content=content)
 
-    def _process_interaction_render(self, block_index: int, mode: ProcessMode, variables: dict[str, str | list[str]] | None = None) -> LLMResult | Generator[LLMResult, None, None]:
+    def _process_interaction_render(
+        self,
+        block_index: int,
+        mode: ProcessMode,
+        context: list[dict[str, str]] | None = None,
+        variables: dict[str, str | list[str]] | None = None,
+    ):
         """Process interaction content rendering."""
         block = self.get_block(block_index)
 
@@ -298,24 +389,17 @@ class MarkdownFlow:
             # Unable to extract, return processed content
             return LLMResult(content=processed_block.content)
 
-        # Build render messages
-        messages = self._build_interaction_render_messages(question_text)
+        # Truncate context to configured maximum length
+        truncated_context = self._truncate_context(context)
 
-        if mode == ProcessMode.PROMPT_ONLY:
-            return LLMResult(
-                prompt=messages[-1]["content"],
-                metadata={
-                    "original_content": processed_block.content,
-                    "question_text": question_text,
-                },
-            )
+        # Build render messages with context
+        messages = self._build_interaction_render_messages(question_text, truncated_context)
 
         if mode == ProcessMode.COMPLETE:
             if not self._llm_provider:
                 return LLMResult(content=processed_block.content)  # Fallback processing
 
-            result = self._llm_provider.complete(messages)
-            rendered_question = result.content
+            rendered_question = self._llm_provider.complete(messages, model=self._model, temperature=self._temperature)
             rendered_content = self._reconstruct_interaction_content(processed_block.content, rendered_question)
 
             return LLMResult(
@@ -343,7 +427,7 @@ class MarkdownFlow:
             # With LLM provider, collect full response then return once
             def stream_generator():
                 full_response = ""
-                for chunk in self._llm_provider.stream(messages):
+                for chunk in self._llm_provider.stream(messages, model=self._model, temperature=self._temperature):  # type: ignore[attr-defined]
                     full_response += chunk
 
                 # Reconstruct final interaction content
@@ -366,14 +450,13 @@ class MarkdownFlow:
         variables: dict[str, str | list[str]] | None = None,
     ) -> LLMResult | Generator[LLMResult, None, None]:
         """Process interaction user input."""
-        _ = context  # Mark as intentionally unused
         block = self.get_block(block_index)
         target_variable = block.variables[0] if block.variables else "user_input"
 
         # Basic validation
         if not user_input or not any(values for values in user_input.values()):
             error_msg = INPUT_EMPTY_ERROR
-            return self._render_error(error_msg, mode)
+            return self._render_error(error_msg, mode, context)
 
         # Get the target variable value from user_input
         target_values = user_input.get(target_variable, [])
@@ -387,24 +470,99 @@ class MarkdownFlow:
 
         if "error" in parse_result:
             error_msg = INTERACTION_PARSE_ERROR.format(error=parse_result["error"])
-            return self._render_error(error_msg, mode)
+            return self._render_error(error_msg, mode, context)
 
         interaction_type = parse_result.get("type")
 
         # Process user input based on interaction type
         if interaction_type in [
-            InteractionType.BUTTONS_ONLY,
             InteractionType.BUTTONS_WITH_TEXT,
-            InteractionType.BUTTONS_MULTI_SELECT,
             InteractionType.BUTTONS_MULTI_WITH_TEXT,
         ]:
-            # All button types: validate user input against available buttons
+            # Buttons with text input: smart validation (match buttons first, then LLM validate custom text)
+            buttons = parse_result.get("buttons", [])
+
+            # Step 1: Match button values
+            matched_values, unmatched_values = self._match_button_values(buttons, target_values)
+
+            # Step 2: If there are unmatched values (custom text), validate with LLM
+            if unmatched_values:
+                # Create user_input for LLM validation (only custom text)
+                custom_input = {target_variable: unmatched_values}
+
+                validation_result = self._process_llm_validation(
+                    block_index=block_index,
+                    user_input=custom_input,
+                    target_variable=target_variable,
+                    mode=mode,
+                    context=context,
+                )
+
+                # Handle validation result based on mode
+                if mode == ProcessMode.COMPLETE:
+                    # Check if validation passed
+                    if isinstance(validation_result, LLMResult) and validation_result.variables:
+                        validated_values = validation_result.variables.get(target_variable, [])
+                        # Merge matched button values + validated custom text
+                        all_values = matched_values + validated_values
+                        return LLMResult(
+                            content="",
+                            variables={target_variable: all_values},
+                            metadata={
+                                "interaction_type": str(interaction_type),
+                                "matched_button_values": matched_values,
+                                "validated_custom_values": validated_values,
+                            },
+                        )
+                    else:
+                        # Validation failed, return error
+                        return validation_result
+
+                if mode == ProcessMode.STREAM:
+                    # For stream mode, collect validation result
+                    def stream_merge_generator():
+                        # Consume the validation stream
+                        for result in validation_result:  # type: ignore[attr-defined]
+                            if isinstance(result, LLMResult) and result.variables:
+                                validated_values = result.variables.get(target_variable, [])
+                                all_values = matched_values + validated_values
+                                yield LLMResult(
+                                    content="",
+                                    variables={target_variable: all_values},
+                                    metadata={
+                                        "interaction_type": str(interaction_type),
+                                        "matched_button_values": matched_values,
+                                        "validated_custom_values": validated_values,
+                                    },
+                                )
+                            else:
+                                # Validation failed
+                                yield result
+
+                    return stream_merge_generator()
+            else:
+                # All values matched buttons, return directly
+                return LLMResult(
+                    content="",
+                    variables={target_variable: matched_values},
+                    metadata={
+                        "interaction_type": str(interaction_type),
+                        "all_matched_buttons": True,
+                    },
+                )
+
+        if interaction_type in [
+            InteractionType.BUTTONS_ONLY,
+            InteractionType.BUTTONS_MULTI_SELECT,
+        ]:
+            # Pure button types: only basic button validation (no LLM)
             return self._process_button_validation(
                 parse_result,
                 target_values,
                 target_variable,
                 mode,
                 interaction_type,
+                context,
             )
 
         if interaction_type == InteractionType.NON_ASSIGNMENT_BUTTON:
@@ -420,19 +578,50 @@ class MarkdownFlow:
             )
 
         # Text-only input type: ?[%{{sys_user_nickname}}...question]
-        # For text-only inputs, directly use the target variable values
+        # Use LLM validation to check if input is relevant to the question
         if target_values:
-            return LLMResult(
-                content="",
-                variables={target_variable: target_values},
-                metadata={
-                    "interaction_type": "text_only",
-                    "target_variable": target_variable,
-                    "values": target_values,
-                },
+            return self._process_llm_validation(
+                block_index=block_index,
+                user_input=user_input,
+                target_variable=target_variable,
+                mode=mode,
+                context=context,
             )
         error_msg = f"No input provided for variable '{target_variable}'"
-        return self._render_error(error_msg, mode)
+        return self._render_error(error_msg, mode, context)
+
+    def _match_button_values(
+        self,
+        buttons: list[dict[str, str]],
+        target_values: list[str],
+    ) -> tuple[list[str], list[str]]:
+        """
+        Match user input values against button options.
+
+        Args:
+            buttons: List of button dictionaries with 'display' and 'value' keys
+            target_values: User input values to match
+
+        Returns:
+            Tuple of (matched_values, unmatched_values)
+            - matched_values: Values that match button options (using button value)
+            - unmatched_values: Values that don't match any button
+        """
+        matched_values = []
+        unmatched_values = []
+
+        for value in target_values:
+            matched = False
+            for button in buttons:
+                if value in [button["display"], button["value"]]:
+                    matched_values.append(button["value"])  # Use button value
+                    matched = True
+                    break
+
+            if not matched:
+                unmatched_values.append(value)
+
+        return matched_values, unmatched_values
 
     def _process_button_validation(
         self,
@@ -441,6 +630,7 @@ class MarkdownFlow:
         target_variable: str,
         mode: ProcessMode,
         interaction_type: InteractionType,
+        context: list[dict[str, str]] | None = None,
     ) -> LLMResult | Generator[LLMResult, None, None]:
         """
         Simplified button validation with new input format.
@@ -451,6 +641,7 @@ class MarkdownFlow:
             target_variable: Target variable name
             mode: Processing mode
             interaction_type: Type of interaction
+            context: Conversation history context (optional)
         """
         buttons = parse_result.get("buttons", [])
         is_multi_select = interaction_type in [
@@ -476,9 +667,9 @@ class MarkdownFlow:
             # Pure button mode requires input
             button_displays = [btn["display"] for btn in buttons]
             error_msg = f"Please select from: {', '.join(button_displays)}"
-            return self._render_error(error_msg, mode)
+            return self._render_error(error_msg, mode, context)
 
-        # First, check if user input matches available buttons
+        # Validate input values against available buttons
         valid_values = []
         invalid_values = []
 
@@ -491,30 +682,19 @@ class MarkdownFlow:
                     break
 
             if not matched:
-                invalid_values.append(value)
-
-        # If there are invalid values and this interaction allows text input, use LLM validation
-        if invalid_values and allow_text_input:
-            # Use LLM validation for text input interactions
-            button_displays = [btn["display"] for btn in buttons]
-            question = parse_result.get("question", "")
-
-            return self._process_llm_validation_with_options(
-                block_index=0,  # Not used in the method
-                user_input={target_variable: target_values},
-                target_variable=target_variable,
-                options=button_displays,
-                question=question,
-                mode=mode,
-            )
-
-        # Check for validation errors in pure button mode or when text input not allowed
-        if invalid_values:
+                if allow_text_input:
+                    # Allow custom text in buttons+text mode
+                    valid_values.append(value)
+                else:
+                    invalid_values.append(value)
+
+        # Check for validation errors
+        if invalid_values and not allow_text_input:
             button_displays = [btn["display"] for btn in buttons]
             error_msg = f"Invalid options: {', '.join(invalid_values)}. Please select from: {', '.join(button_displays)}"
-            return self._render_error(error_msg, mode)
+            return self._render_error(error_msg, mode, context)
 
-        # Success: return validated button values
+        # Success: return validated values
         return LLMResult(
             content="",
             variables={target_variable: valid_values},
@@ -524,7 +704,6 @@ class MarkdownFlow:
                 "valid_values": valid_values,
                 "invalid_values": invalid_values,
                 "total_input_count": len(target_values),
-                "llm_validated": False,
             },
         )
 
@@ -534,27 +713,18 @@ class MarkdownFlow:
         user_input: dict[str, list[str]],
         target_variable: str,
         mode: ProcessMode,
+        context: list[dict[str, str]] | None = None,
     ) -> LLMResult | Generator[LLMResult, None, None]:
         """Process LLM validation."""
         # Build validation messages
-        messages = self._build_validation_messages(block_index, user_input, target_variable)
-
-        if mode == ProcessMode.PROMPT_ONLY:
-            return LLMResult(
-                prompt=messages[-1]["content"],
-                metadata={
-                    "validation_target": user_input,
-                    "target_variable": target_variable,
-                },
-            )
+        messages = self._build_validation_messages(block_index, user_input, target_variable, context)
 
         if mode == ProcessMode.COMPLETE:
             if not self._llm_provider:
                 # Fallback processing, return variables directly
                 return LLMResult(content="", variables=user_input)  # type: ignore[arg-type]
 
-            result = self._llm_provider.complete(messages)
-            llm_response = result.content
+            llm_response = self._llm_provider.complete(messages, model=self._model, temperature=self._temperature)
 
             # Parse validation response and convert to LLMResult
             # Use joined target values for fallback; avoids JSON string injection
@@ -568,7 +738,7 @@ class MarkdownFlow:
 
             def stream_generator():
                 full_response = ""
-                for chunk in self._llm_provider.stream(messages):
+                for chunk in self._llm_provider.stream(messages, model=self._model, temperature=self._temperature):  # type: ignore[attr-defined]
                     full_response += chunk
 
                 # Parse complete response and convert to LLMResult
@@ -592,28 +762,15 @@ class MarkdownFlow:
         mode: ProcessMode,
     ) -> LLMResult | Generator[LLMResult, None, None]:
         """Process LLM validation with button options (third case)."""
-        _ = block_index  # Mark as intentionally unused
         # Build special validation messages containing button option information
         messages = self._build_validation_messages_with_options(user_input, target_variable, options, question)
 
-        if mode == ProcessMode.PROMPT_ONLY:
-            return LLMResult(
-                prompt=messages[-1]["content"],
-                metadata={
-                    "validation_target": user_input,
-                    "target_variable": target_variable,
-                    "options": options,
-                    "question": question,
-                },
-            )
-
         if mode == ProcessMode.COMPLETE:
             if not self._llm_provider:
                 # Fallback processing, return variables directly
                 return LLMResult(content="", variables=user_input)  # type: ignore[arg-type]
 
-            result = self._llm_provider.complete(messages)
-            llm_response = result.content
+            llm_response = self._llm_provider.complete(messages, model=self._model, temperature=self._temperature)
 
             # Parse validation response and convert to LLMResult
             # Use joined target values for fallback; avoids JSON string injection
@@ -627,7 +784,7 @@ class MarkdownFlow:
 
             def stream_generator():
                 full_response = ""
-                for chunk in self._llm_provider.stream(messages):
+                for chunk in self._llm_provider.stream(messages, model=self._model, temperature=self._temperature):  # type: ignore[attr-defined]
                     full_response += chunk
                     # For validation scenario, don't output chunks in real-time, only final result
 
@@ -644,22 +801,24 @@ class MarkdownFlow:
 
             return stream_generator()
 
-    def _render_error(self, error_message: str, mode: ProcessMode) -> LLMResult | Generator[LLMResult, None, None]:
+    def _render_error(
+        self,
+        error_message: str,
+        mode: ProcessMode,
+        context: list[dict[str, str]] | None = None,
+    ) -> LLMResult | Generator[LLMResult, None, None]:
         """Render user-friendly error message."""
-        messages = self._build_error_render_messages(error_message)
+        # Truncate context to configured maximum length
+        truncated_context = self._truncate_context(context)
 
-        if mode == ProcessMode.PROMPT_ONLY:
-            return LLMResult(
-                prompt=messages[-1]["content"],
-                metadata={"original_error": error_message},
-            )
+        # Build error messages with context
+        messages = self._build_error_render_messages(error_message, truncated_context)
 
         if mode == ProcessMode.COMPLETE:
             if not self._llm_provider:
                 return LLMResult(content=error_message)  # Fallback processing
 
-            result = self._llm_provider.complete(messages)
-            friendly_error = result.content
+            friendly_error = self._llm_provider.complete(messages, model=self._model, temperature=self._temperature)
             return LLMResult(content=friendly_error, prompt=messages[-1]["content"])
 
         if mode == ProcessMode.STREAM:
@@ -667,7 +826,7 @@ class MarkdownFlow:
                 return LLMResult(content=error_message)
 
             def stream_generator():
-                for chunk in self._llm_provider.stream(messages):
+                for chunk in self._llm_provider.stream(messages, model=self._model, temperature=self._temperature):  # type: ignore[attr-defined]
                     yield LLMResult(content=chunk, prompt=messages[-1]["content"])
 
             return stream_generator()
@@ -678,13 +837,15 @@ class MarkdownFlow:
         self,
         block_index: int,
         variables: dict[str, str | list[str]] | None,
+        context: list[dict[str, str]] | None = None,
     ) -> list[dict[str, str]]:
         """Build content block messages."""
         block = self.get_block(block_index)
         block_content = block.content
 
-        # Process output instructions
-        block_content = process_output_instructions(block_content)
+        # Process output instructions and detect if preserved content exists
+        # Returns: (processed_content, has_preserved_content)
+        block_content, has_preserved_content = process_output_instructions(block_content)
 
         # Replace variables
         block_content = replace_variables_in_text(block_content, variables or {})
@@ -692,22 +853,43 @@ class MarkdownFlow:
         # Build message array
         messages = []
 
-        # Add document prompt
+        # Build system message with XML tags
+        system_parts = []
+
+        # 1. Base system prompt (if exists and non-empty)
+        if self._base_system_prompt:
+            system_parts.append(f"<base_system>\n{self._base_system_prompt}\n</base_system>")
+
+        # 2. Document prompt (if exists and non-empty)
         if self._document_prompt:
-            messages.append({"role": "system", "content": self._document_prompt})
+            system_parts.append(f"<document_prompt>\n{self._document_prompt}\n</document_prompt>")
+
+        # 3. Output instruction (if preserved content exists)
+        # Note: OUTPUT_INSTRUCTION_EXPLANATION already contains <preserve_or_translate_instruction> tags
+        if has_preserved_content:
+            system_parts.append(OUTPUT_INSTRUCTION_EXPLANATION.strip())
 
-        # For most content blocks, historical conversation context is not needed
-        # because each document block is an independent instruction
-        # If future specific scenarios need context, logic can be added here
-        # if context:
-        #     messages.extend(context)
+        # Combine all parts and add as system message
+        if system_parts:
+            system_msg = "\n\n".join(system_parts)
+            messages.append({"role": "system", "content": system_msg})
+
+        # Add conversation history context if provided
+        # Context is inserted after system message and before current user message
+        truncated_context = self._truncate_context(context)
+        if truncated_context:
+            messages.extend(truncated_context)
 
         # Add processed content as user message (as instruction to LLM)
         messages.append({"role": "user", "content": block_content})
 
         return messages
 
-    def _build_interaction_render_messages(self, question_text: str) -> list[dict[str, str]]:
+    def _build_interaction_render_messages(
+        self,
+        question_text: str,
+        context: list[dict[str, str]] | None = None,
+    ) -> list[dict[str, str]]:
         """Build interaction rendering messages."""
         # Check if using custom interaction prompt
         if self._interaction_prompt != DEFAULT_INTERACTION_PROMPT:
@@ -721,15 +903,32 @@ class MarkdownFlow:
         messages = []
 
         messages.append({"role": "system", "content": render_prompt})
+
+        # NOTE: Context is temporarily disabled for interaction rendering
+        # Mixing conversation history with interaction content rewriting can cause issues
+        # The context parameter is kept in the signature for future use
+        # truncated_context = self._truncate_context(context)
+        # if truncated_context:
+        #     messages.extend(truncated_context)
+
         messages.append({"role": "user", "content": question_text})
 
         return messages
 
-    def _build_validation_messages(self, block_index: int, user_input: dict[str, list[str]], target_variable: str) -> list[dict[str, str]]:
+    def _build_validation_messages(
+        self,
+        block_index: int,
+        user_input: dict[str, list[str]],
+        target_variable: str,
+        context: list[dict[str, str]] | None = None,
+    ) -> list[dict[str, str]]:
         """Build validation messages."""
         block = self.get_block(block_index)
         config = self.get_interaction_validation_config(block_index)
 
+        # Truncate context to configured maximum length
+        truncated_context = self._truncate_context(context)
+
         if config and config.validation_template:
             # Use custom validation template
             validation_prompt = config.validation_template
@@ -740,7 +939,8 @@ class MarkdownFlow:
             system_message = DEFAULT_VALIDATION_SYSTEM_MESSAGE
         else:
             # Use smart default validation template
-            from .utils import (
+            from .parser import (
+                InteractionParser,
                 extract_interaction_question,
                 generate_smart_validation_template,
             )
@@ -748,11 +948,17 @@ class MarkdownFlow:
             # Extract interaction question
             interaction_question = extract_interaction_question(block.content)
 
-            # Generate smart validation template
+            # Parse interaction to extract button information
+            parser = InteractionParser()
+            parse_result = parser.parse(block.content)
+            buttons = parse_result.get("buttons") if "buttons" in parse_result else None
+
+            # Generate smart validation template with context and buttons
             validation_template = generate_smart_validation_template(
                 target_variable,
-                context=None,  # Could consider passing context here
+                context=truncated_context,
                 interaction_question=interaction_question,
+                buttons=buttons,
             )
 
             # Replace template variables
@@ -765,6 +971,11 @@ class MarkdownFlow:
         messages = []
 
         messages.append({"role": "system", "content": system_message})
+
+        # Add conversation history context if provided (only if not using custom template)
+        if truncated_context and not (config and config.validation_template):
+            messages.extend(truncated_context)
+
         messages.append({"role": "user", "content": validation_prompt})
 
         return messages
@@ -795,7 +1006,11 @@ class MarkdownFlow:
 
         return messages
 
-    def _build_error_render_messages(self, error_message: str) -> list[dict[str, str]]:
+    def _build_error_render_messages(
+        self,
+        error_message: str,
+        context: list[dict[str, str]] | None = None,
+    ) -> list[dict[str, str]]:
         """Build error rendering messages."""
         render_prompt = f"""{self._interaction_error_prompt}
 
@@ -808,6 +1023,12 @@ Original Error: {error_message}
             messages.append({"role": "system", "content": self._document_prompt})
 
         messages.append({"role": "system", "content": render_prompt})
+
+        # Add conversation history context if provided
+        truncated_context = self._truncate_context(context)
+        if truncated_context:
+            messages.extend(truncated_context)
+
         messages.append({"role": "user", "content": error_message})
 
         return messages
@@ -827,411 +1048,5 @@ Original Error: {error_message}
         if match:
             prefix = match.group(1)
             suffix = match.group(2)
-            # Extract only the closing bracket from suffix, remove original question
-            # suffix format is "original_question]", we only want "]"
-            if suffix.endswith("]"):
-                clean_suffix = "]"
-            else:
-                clean_suffix = suffix
-
-            return f"{prefix}{cleaned_question}{clean_suffix}"
+            return f"{prefix}{cleaned_question}{suffix}"
         return original_content  # type: ignore[unreachable]
-
-    # Dynamic Interaction Methods
-
-    def _process_with_dynamic_check(
-        self,
-        block_index: int,
-        mode: ProcessMode,
-        context: list[dict[str, str]] | None,
-        variables: dict[str, str | list[str]] | None,
-    ) -> LLMResult | Generator[LLMResult, None, None]:
-        """Process content with dynamic interaction detection and conversion."""
-
-        block = self.get_block(block_index)
-        messages = self._build_dynamic_check_messages(block, context, variables)
-
-        # Define Function Calling tools with structured approach
-        tools = [
-            {
-                "type": "function",
-                "function": {
-                    "name": "create_interaction_block",
-                    "description": "Convert content to interaction block with structured data when it needs to collect user input",
-                    "parameters": {
-                        "type": "object",
-                        "properties": {
-                            "needs_interaction": {"type": "boolean", "description": "Whether this content needs to be converted to interaction block"},
-                            "variable_name": {"type": "string", "description": "Name of the variable to collect (without {{}} brackets)"},
-                            "interaction_type": {
-                                "type": "string",
-                                "enum": ["single_select", "multi_select", "text_input", "mixed"],
-                                "description": "Type of interaction: single_select (|), multi_select (||), text_input (...), mixed (options + text)",
-                            },
-                            "options": {"type": "array", "items": {"type": "string"}, "description": "List of selectable options (3-4 specific options based on context)"},
-                            "allow_text_input": {"type": "boolean", "description": "Whether to include a text input option for 'Other' cases"},
-                            "text_input_prompt": {"type": "string", "description": "Prompt text for the text input option (e.g., '其他请输入', 'Other, please specify')"},
-                        },
-                        "required": ["needs_interaction"],
-                    },
-                },
-            }
-        ]
-
-        if not self._llm_provider:
-            raise ValueError(LLM_PROVIDER_REQUIRED_ERROR)
-
-        # Call LLM with tools
-        result = self._llm_provider.complete(messages, tools)
-
-        # If interaction was generated through Function Calling, construct the MarkdownFlow format
-        if result.transformed_to_interaction and result.metadata and "tool_args" in result.metadata:
-            tool_args = result.metadata["tool_args"]
-            if tool_args.get("needs_interaction"):
-                # Construct MarkdownFlow format from structured data
-                interaction_content = self._build_interaction_format(tool_args)
-                result.content = interaction_content
-
-        # If transformed to interaction, return as is
-        if result.transformed_to_interaction:
-            return result
-
-        # If not transformed, continue with normal processing using standard content messages
-        normal_messages = self._build_content_messages(block_index, variables)
-
-        if mode == ProcessMode.STREAM:
-
-            def stream_wrapper():
-                stream_generator = self._llm_provider.stream(normal_messages)
-                for chunk in stream_generator:
-                    yield LLMResult(content=chunk)
-
-            return stream_wrapper()
-
-        # Complete mode - use normal content processing
-        normal_result = self._llm_provider.complete(normal_messages)
-        return LLMResult(content=normal_result.content, prompt=normal_messages[-1]["content"], metadata=normal_result.metadata)
-
-    def _build_dynamic_check_messages(
-        self,
-        block: "Block",
-        context: list[dict[str, str]] | None,
-        variables: dict[str, str | list[str]] | None,
-    ) -> list[dict[str, str]]:
-        """Build messages for dynamic interaction detection."""
-
-        import json
-
-        # System prompt for detection
-        system_prompt = """You are an intelligent document processing assistant specializing in creating interactive forms.
-
-Task: Analyze the given content block and determine if it needs to be converted to an interaction block to collect user information.
-
-**ABSOLUTE RULE**: Convert ONLY when ALL THREE mandatory elements are explicitly present:
-1. Storage action word + target connector + variable
-2. No exceptions, no implications, no assumptions
-
-**MANDATORY TRIPLE PATTERN (ALL REQUIRED):**
-
-**Element 1: Storage Action Words**
-- Chinese: "记录", "保存", "存储", "收集", "采集"
-- English: "save", "store", "record", "collect", "gather"
-
-**Element 2: Target Connection Words**
-- Chinese: "到", "为", "在", "至"
-- English: "to", "as", "in", "into"
-
-**Element 3: Target Variable**
-- Must contain {{variable_name}} syntax for NEW data storage
-- Variable must be for collecting NEW information, not using existing data
-
-**VALID CONVERSION FORMULA:**
-[Storage Word] + [Connector] + {{new_variable}}
-
-Examples of VALID patterns:
-- "...记录到{{姓名}}"
-- "...保存为{{偏好}}"
-- "...存储在{{选择}}"
-- "...save to {{preference}}"
-- "...collect as {{user_input}}"
-
-**STRICT EXCLUSION RULES:**
-
-❌ NEVER convert if missing ANY element:
-- No storage action word = NO conversion
-- No target connector = NO conversion
-- No {{variable}} = NO conversion
-- Using existing {{variable}} instead of collecting new = NO conversion
-
-❌ NEVER convert casual conversation:
-- Simple questions without storage intent
-- Introduction requests without persistence
-- General inquiries without data collection
-- Educational or exploratory content
-
-❌ NEVER infer or assume storage intent:
-- Don't assume "询问姓名" means "保存姓名"
-- Don't assume "了解偏好" means "记录偏好"
-- Don't assume data collection without explicit storage words
-
-**PATTERN ANALYSIS METHOD:**
-1. **Exact Pattern Match**: Search for [Storage Word] + [Connector] + {{variable}}
-2. **No Pattern = No Conversion**: If exact pattern not found, return needs_interaction: false
-3. **Zero Tolerance**: No partial matches, no similar meanings, no interpretations
-
-**ULTRA-CONSERVATIVE APPROACH:**
-- If there's ANY doubt about storage intent = DON'T convert
-- If storage pattern is not 100% explicit = DON'T convert
-- If you need to "interpret" or "infer" storage intent = DON'T convert
-- Prefer false negatives over false positives
-
-When exact pattern is found, generate structured interaction data. Otherwise, always return needs_interaction: false."""
-
-        # User message with content and context
-        # Build user prompt with document context
-        user_prompt_parts = []
-
-        # Add document-level prompt context if exists
-        if self._document_prompt:
-            user_prompt_parts.append(f"""Document-level instructions:
-{self._document_prompt}
-
-(Note: The above are the user's document-level instructions that provide context and requirements for processing.)
-""")
-
-        # Prepare content analysis with both original and resolved versions
-        original_content = block.content
-
-        # Create resolved content with variable substitution for better context
-        resolved_content = original_content
-        if variables:
-            from .utils import replace_variables_in_text
-
-            resolved_content = replace_variables_in_text(original_content, variables)
-
-        content_analysis = f"""Current content block to analyze:
-
-**Original content (shows variable structure):**
-{original_content}
-
-**Resolved content (with current variable values):**
-{resolved_content}
-
-**Existing variable values:**
-{json.dumps(variables, ensure_ascii=False) if variables else "None"}"""
-
-        # Add different analysis based on whether content has variables
-        if "{{" in original_content and "}}" in original_content:
-            from .utils import extract_variables_from_text
-
-            content_variables = set(extract_variables_from_text(original_content))
-
-            # Find new variables (not yet collected)
-            new_variables = content_variables - (set(variables.keys()) if variables else set())
-            existing_used_variables = content_variables & (set(variables.keys()) if variables else set())
-
-            content_analysis += f"""
-
-**Variable analysis:**
-- Variables used from previous steps: {list(existing_used_variables) if existing_used_variables else "None"}
-- New variables to collect: {list(new_variables) if new_variables else "None"}
-
-**Context guidance:**
-- Use the resolved content to understand the actual context and requirements
-- Generate options based on the real variable values shown in the resolved content
-- Collect user input for the new variables identified above"""
-
-        user_prompt_parts.append(content_analysis)
-
-        # Add analysis requirements and structured output guide
-        user_prompt_parts.append("""## Analysis Task:
-1. Determine if this content needs to be converted to an interaction block
-2. If conversion is needed, provide structured interaction data
-
-## Context-based Analysis:
-- Use the "Resolved content" to understand actual context (e.g., if it shows "川菜", generate Sichuan dish options)
-- Extract the "New variables to collect" identified in the variable analysis above
-- Generate 3-4 specific options based on the resolved context and document-level instructions
-- Follow ALL document-level instruction requirements (language, domain, terminology)
-
-## Selection Type Decision Logic:
-Ask: "Can a user realistically want/choose multiple of these options simultaneously?"
-
-**Use MULTI_SELECT when:**
-- Food dishes (can order multiple: 宫保鸡丁, 麻婆豆腐)
-- Programming skills (can know multiple: Python, JavaScript)
-- Interests/hobbies (can have multiple: 读书, 运动, 旅游)
-- Product features (can want multiple: 定制颜色, 个性化logo)
-- Exercise types (can do multiple: 跑步, 游泳, 瑜伽)
-
-**Use SINGLE_SELECT when:**
-- Job positions (usually apply for one: 软件工程师 OR 产品经理)
-- Experience levels (have one current level: Beginner OR Advanced)
-- Budget ranges (have one range: 5-10万 OR 10-20万)
-- Education levels (have one highest: Bachelor's OR Master's)
-
-## Output Instructions:
-If this content needs interaction, use the create_interaction_block function with:
-- `needs_interaction`: true/false
-- `variable_name`: the variable to collect (from "New variables" above)
-- `interaction_type`: "single_select", "multi_select", "text_input", or "mixed"
-- `options`: array of 3-4 specific options based on context
-- `allow_text_input`: true if you want to include "other" option
-- `text_input_prompt`: text for the "other" option (in appropriate language)
-
-Analyze the content and provide the structured interaction data.""")
-
-        user_prompt = "\n\n".join(user_prompt_parts)
-
-        messages = [{"role": "system", "content": system_prompt}]
-
-        # Add context if provided
-        if context:
-            messages.extend(context)
-
-        messages.append({"role": "user", "content": user_prompt})
-
-        return messages
-
-    def _build_interaction_format(self, tool_args: dict) -> str:
-        """Build MarkdownFlow interaction format from structured Function Calling data."""
-        variable_name = tool_args.get("variable_name", "")
-        interaction_type = tool_args.get("interaction_type", "single_select")
-        options = tool_args.get("options", [])
-        allow_text_input = tool_args.get("allow_text_input", False)
-        text_input_prompt = tool_args.get("text_input_prompt", "...请输入")
-
-        if not variable_name:
-            return ""
-
-        # For text_input type, options can be empty
-        if interaction_type != "text_input" and not options:
-            return ""
-
-        # Choose separator based on interaction type
-        if interaction_type in ["multi_select", "mixed"]:
-            separator = "||"
-        else:
-            separator = "|"
-
-        # Build options string
-        if interaction_type == "text_input":
-            # Text input only
-            options_str = f"...{text_input_prompt}"
-        else:
-            # Options with potential text input
-            options_str = separator.join(options)
-
-            if allow_text_input and text_input_prompt:
-                # Ensure text input has ... prefix
-                text_option = text_input_prompt if text_input_prompt.startswith("...") else f"...{text_input_prompt}"
-                options_str += f"{separator}{text_option}"
-
-        return f"?[%{{{{{variable_name}}}}} {options_str}]"
-
-    def _process_dynamic_interaction_validation(
-        self,
-        block_index: int,
-        interaction_format: str,
-        user_input: dict[str, list[str]],
-        mode: ProcessMode,
-        context: list[dict[str, str]] | None,
-        variables: dict[str, str | list[str]] | None,
-    ) -> LLMResult:
-        """Validate user input for dynamically generated interaction blocks using same logic as normal interactions."""
-        _ = block_index  # Mark as intentionally unused
-        _ = context  # Mark as intentionally unused
-
-        from .utils import InteractionParser
-
-        # Parse the interaction format using the same parser as normal interactions
-        parser = InteractionParser()
-        parse_result = parser.parse(interaction_format)
-
-        if "error" in parse_result:
-            error_msg = f"Invalid interaction format: {parse_result['error']}"
-            return self._render_error(error_msg, mode)
-
-        # Extract variable name and interaction type
-        variable_name = parse_result.get("variable")
-        interaction_type = parse_result.get("type")
-
-        if not variable_name:
-            error_msg = f"No variable found in interaction format: {interaction_format}"
-            return self._render_error(error_msg, mode)
-
-        # Get user input for the target variable
-        target_values = user_input.get(variable_name, [])
-
-        # Basic validation - check if input is provided when required
-        if not target_values:
-            # Check if this is a text input or allows empty input
-            allow_text_input = interaction_type in [
-                InteractionType.BUTTONS_WITH_TEXT,
-                InteractionType.BUTTONS_MULTI_WITH_TEXT,
-            ]
-
-            if allow_text_input:
-                # Allow empty input for buttons+text mode - merge with existing variables
-                merged_variables = dict(variables or {})
-                merged_variables[variable_name] = []
-                return LLMResult(
-                    content="",
-                    variables=merged_variables,
-                    metadata={
-                        "interaction_type": "dynamic_interaction",
-                        "empty_input": True,
-                    },
-                )
-            error_msg = f"No input provided for variable '{variable_name}'"
-            return self._render_error(error_msg, mode)
-
-        # Use the same validation logic as normal interactions
-        if interaction_type in [
-            InteractionType.BUTTONS_ONLY,
-            InteractionType.BUTTONS_WITH_TEXT,
-            InteractionType.BUTTONS_MULTI_SELECT,
-            InteractionType.BUTTONS_MULTI_WITH_TEXT,
-        ]:
-            # Button validation - reuse the existing button validation logic
-            button_result = self._process_button_validation(
-                parse_result,
-                target_values,
-                variable_name,
-                mode,
-                interaction_type,
-            )
-
-            # Merge with existing variables for dynamic interactions
-            if hasattr(button_result, "variables") and button_result.variables is not None and variables:
-                merged_variables = dict(variables)
-                merged_variables.update(button_result.variables)
-                return LLMResult(
-                    content=button_result.content,
-                    variables=merged_variables,
-                    metadata=button_result.metadata,
-                )
-            return button_result
-
-        if interaction_type == InteractionType.NON_ASSIGNMENT_BUTTON:
-            # Non-assignment buttons: don't set variables, keep existing ones
-            return LLMResult(
-                content="",
-                variables=dict(variables or {}),
-                metadata={
-                    "interaction_type": "non_assignment_button",
-                    "user_input": user_input,
-                },
-            )
-        # Text-only input type - merge with existing variables
-        merged_variables = dict(variables or {})
-        merged_variables[variable_name] = target_values
-        return LLMResult(
-            content="",
-            variables=merged_variables,
-            metadata={
-                "interaction_type": "text_only",
-                "target_variable": variable_name,
-                "values": target_values,
-            },
-        )