markdown-flow 0.2.18__py3-none-any.whl → 0.2.26__py3-none-any.whl

markdown_flow/parser/validation.py

@@ -0,0 +1,121 @@
+"""
+Validation Parser Module
+
+Provides validation template generation and response parsing for user input validation.
+"""
+
+import json
+from typing import Any
+
+from ..constants import (
+    CONTEXT_BUTTON_OPTIONS_TEMPLATE,
+    CONTEXT_CONVERSATION_TEMPLATE,
+    CONTEXT_QUESTION_MARKER,
+    CONTEXT_QUESTION_TEMPLATE,
+    SMART_VALIDATION_TEMPLATE,
+    VALIDATION_ILLEGAL_DEFAULT_REASON,
+    VALIDATION_RESPONSE_ILLEGAL,
+    VALIDATION_RESPONSE_OK,
+)
+from .json_parser import parse_json_response
+
+
+def generate_smart_validation_template(
+    target_variable: str,
+    context: list[dict[str, Any]] | None = None,
+    interaction_question: str | None = None,
+    buttons: list[dict[str, str]] | None = None,
+) -> str:
+    """
+    Generate smart validation template based on context and question.
+
+    Args:
+        target_variable: Target variable name
+        context: Context message list with role and content fields
+        interaction_question: Question text from interaction block
+        buttons: Button options list with display and value fields
+
+    Returns:
+        Generated validation template
+    """
+    # Build context information
+    context_info = ""
+    if interaction_question or context or buttons:
+        context_parts = []
+
+        # Add question information (most important, put first)
+        if interaction_question:
+            context_parts.append(CONTEXT_QUESTION_TEMPLATE.format(question=interaction_question))
+
+        # Add button options information
+        if buttons:
+            button_displays = [btn.get("display", "") for btn in buttons if btn.get("display")]
+            if button_displays:
+                button_options_str = ", ".join(button_displays)
+                button_info = CONTEXT_BUTTON_OPTIONS_TEMPLATE.format(button_options=button_options_str)
+                context_parts.append(button_info)
+
+        # Add conversation context
+        if context:
+            for msg in context:
+                if msg.get("role") == "assistant" and CONTEXT_QUESTION_MARKER not in msg.get("content", ""):
+                    # Other assistant messages as context (exclude extracted questions)
+                    context_parts.append(CONTEXT_CONVERSATION_TEMPLATE.format(content=msg.get("content", "")))
+
+        if context_parts:
+            context_info = "\n\n".join(context_parts)
+
+    # Use template from constants
+    # Note: {sys_user_input} will be replaced later in _build_validation_messages
+    return SMART_VALIDATION_TEMPLATE.format(
+        target_variable=target_variable,
+        context_info=context_info,
+        sys_user_input="{sys_user_input}",  # Keep placeholder for later replacement
+    ).strip()
+
+
+def parse_validation_response(llm_response: str, original_input: str, target_variable: str) -> dict[str, Any]:
+    """
+    Parse LLM validation response, returning standard format.
+
+    Supports JSON format and natural language text responses.
+
+    Args:
+        llm_response: LLM's raw response
+        original_input: User's original input
+        target_variable: Target variable name
+
+    Returns:
+        Standardized parsing result with content and variables fields
+    """
+    try:
+        # Try to parse JSON response
+        parsed_response = parse_json_response(llm_response)
+
+        if isinstance(parsed_response, dict):
+            result = parsed_response.get("result", "").lower()
+
+            if result == VALIDATION_RESPONSE_OK:
+                # Validation successful
+                parse_vars = parsed_response.get("parse_vars", {})
+                if target_variable not in parse_vars:
+                    parse_vars[target_variable] = original_input.strip()
+
+                return {"content": "", "variables": parse_vars}
+
+            if result == VALIDATION_RESPONSE_ILLEGAL:
+                # Validation failed
+                reason = parsed_response.get("reason", VALIDATION_ILLEGAL_DEFAULT_REASON)
+                return {"content": reason, "variables": None}
+
+    except (json.JSONDecodeError, ValueError, KeyError):
+        # JSON parsing failed, fallback to text mode
+        pass
+
+    # Text response parsing (fallback processing)
+    response_lower = llm_response.lower()
+
+    # Check against standard response format
+    if "ok" in response_lower or "valid" in response_lower:
+        return {"content": "", "variables": {target_variable: original_input.strip()}}
+    return {"content": llm_response, "variables": None}

markdown_flow/parser/variable.py

@@ -0,0 +1,95 @@
+"""
+Variable Parser Module
+
+Provides variable extraction and replacement functionality for MarkdownFlow documents.
+"""
+
+import re
+
+from ..constants import (
+    COMPILED_BRACE_VARIABLE_REGEX,
+    COMPILED_PERCENT_VARIABLE_REGEX,
+    VARIABLE_DEFAULT_VALUE,
+)
+
+
+def extract_variables_from_text(text: str) -> list[str]:
+    """
+    Extract all variable names from text.
+
+    Recognizes two variable formats:
+    - %{{variable_name}} format (preserved variables)
+    - {{variable_name}} format (replaceable variables)
+
+    Args:
+        text: Text content to analyze
+
+    Returns:
+        Sorted list of unique variable names
+    """
+    variables = set()
+
+    # Match %{{...}} format variables using pre-compiled regex
+    matches = COMPILED_PERCENT_VARIABLE_REGEX.findall(text)
+    for match in matches:
+        variables.add(match.strip())
+
+    # Match {{...}} format variables (excluding %) using pre-compiled regex
+    matches = COMPILED_BRACE_VARIABLE_REGEX.findall(text)
+    for match in matches:
+        variables.add(match.strip())
+
+    return sorted(list(variables))
+
+
+def replace_variables_in_text(text: str, variables: dict[str, str | list[str]]) -> str:
+    """
+    Replace variables in text, undefined or empty variables are auto-assigned "UNKNOWN".
+
+    Args:
+        text: Text containing variables
+        variables: Variable name to value mapping
+
+    Returns:
+        Text with variables replaced
+    """
+    if not text or not isinstance(text, str):
+        return text or ""
+
+    # Check each variable for null or empty values, assign "UNKNOWN" if so
+    if variables:
+        for key, value in variables.items():
+            if value is None or value == "" or (isinstance(value, list) and not value):
+                variables[key] = VARIABLE_DEFAULT_VALUE
+
+    # Initialize variables as empty dict (if None)
+    if not variables:
+        variables = {}
+
+    # Find all {{variable}} format variable references
+    variable_pattern = r"\{\{([^{}]+)\}\}"
+    matches = re.findall(variable_pattern, text)
+
+    # Assign "UNKNOWN" to undefined variables
+    for var_name in matches:
+        var_name = var_name.strip()
+        if var_name not in variables:
+            variables[var_name] = "UNKNOWN"
+
+    # Use updated replacement logic, preserve %{{var_name}} format variables
+    result = text
+    for var_name, var_value in variables.items():
+        # Convert value to string based on type
+        if isinstance(var_value, list):
+            # Multiple values - join with comma
+            value_str = ", ".join(str(v) for v in var_value if v is not None and str(v).strip())
+            if not value_str:
+                value_str = VARIABLE_DEFAULT_VALUE
+        else:
+            value_str = str(var_value) if var_value is not None else VARIABLE_DEFAULT_VALUE
+
+        # Use negative lookbehind assertion to exclude %{{var_name}} format
+        pattern = f"(?<!%){{{{{re.escape(var_name)}}}}}"
+        result = re.sub(pattern, value_str, result)
+
+    return result

markdown_flow/providers/__init__.py

@@ -0,0 +1,15 @@
+"""
+Markdown-Flow LLM Providers Module
+
+Provides built-in LLM provider implementations.
+"""
+
+from .config import ProviderConfig
+from .openai import OpenAIProvider, create_provider, create_default_provider
+
+__all__ = [
+    "ProviderConfig",
+    "OpenAIProvider",
+    "create_provider",
+    "create_default_provider",
+]

markdown_flow/providers/config.py

@@ -0,0 +1,51 @@
+"""
+Provider Configuration Module
+
+Provides configuration classes for LLM providers.
+"""
+
+import os
+from dataclasses import dataclass, field
+
+
+@dataclass
+class ProviderConfig:
+    """
+    Configuration for LLM providers.
+
+    Supports environment variable defaults for easy configuration.
+    """
+
+    api_key: str = field(default_factory=lambda: os.getenv("LLM_API_KEY", ""))
+    """API key for the LLM service. Default: LLM_API_KEY environment variable."""
+
+    base_url: str = field(default_factory=lambda: os.getenv("LLM_BASE_URL", "https://api.openai.com/v1"))
+    """Base URL for the API endpoint. Default: LLM_BASE_URL environment variable or OpenAI default."""
+
+    model: str = field(default_factory=lambda: os.getenv("LLM_MODEL", "gpt-3.5-turbo"))
+    """Default model name. Default: LLM_MODEL environment variable or gpt-3.5-turbo."""
+
+    temperature: float = field(default_factory=lambda: float(os.getenv("LLM_TEMPERATURE", "0.7")))
+    """Default temperature (0.0-2.0). Default: LLM_TEMPERATURE environment variable or 0.7."""
+
+    debug: bool = field(default_factory=lambda: os.getenv("LLM_DEBUG", "false").lower() in ("true", "1", "yes"))
+    """Enable debug mode (colorized console output). Default: LLM_DEBUG environment variable or False."""
+
+    timeout: float | None = field(
+        default_factory=lambda: float(os.getenv("LLM_TIMEOUT")) if os.getenv("LLM_TIMEOUT") else None
+    )
+    """Request timeout in seconds. None means no timeout. Default: LLM_TIMEOUT environment variable or None."""
+
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if not self.api_key:
+            raise ValueError(
+                "API key is required. Set it via ProviderConfig(api_key='...') "
+                "or LLM_API_KEY environment variable."
+            )
+
+        if self.temperature < 0.0 or self.temperature > 2.0:
+            raise ValueError(f"Temperature must be between 0.0 and 2.0, got {self.temperature}")
+
+        if self.timeout is not None and self.timeout <= 0:
+            raise ValueError(f"Timeout must be positive or None, got {self.timeout}")

markdown_flow/providers/openai.py

@@ -0,0 +1,371 @@
+"""
+OpenAI-Compatible Provider Implementation
+
+Provides a production-ready OpenAI-compatible LLM provider with debug mode,
+token tracking, and comprehensive metadata.
+"""
+
+import time
+from collections.abc import Generator
+from typing import Any
+
+from ..llm import LLMProvider
+from .config import ProviderConfig
+
+try:
+    from openai import OpenAI
+except ImportError:
+    OpenAI = None  # type: ignore[misc, assignment]
+
+
+class OpenAIProvider(LLMProvider):
+    """
+    OpenAI-compatible LLM provider implementation.
+
+    Features:
+    - Debug mode with colorized console output
+    - Automatic token usage tracking
+    - Comprehensive metadata (model, temperature, processing time, tokens, timestamp)
+    - Instance-level model/temperature override support
+    - Streaming and non-streaming modes
+    """
+
+    def __init__(self, config: ProviderConfig):
+        """
+        Initialize OpenAI provider.
+
+        Args:
+            config: Provider configuration
+
+        Raises:
+            ImportError: If openai package is not installed
+            ValueError: If configuration is invalid
+        """
+        if OpenAI is None:
+            raise ImportError(
+                "The 'openai' package is required for OpenAIProvider. "
+                "Install it with: pip install openai"
+            )
+
+        self.config = config
+        self.client = OpenAI(
+            api_key=config.api_key,
+            base_url=config.base_url,
+            timeout=config.timeout,
+        )
+        self._last_metadata: dict[str, Any] = {}
+
+    def complete(
+        self,
+        messages: list[dict[str, str]],
+        model: str | None = None,
+        temperature: float | None = None,
+    ) -> str:
+        """
+        Non-streaming LLM call.
+
+        Args:
+            messages: Message list
+            model: Optional model override
+            temperature: Optional temperature override
+
+        Returns:
+            LLM response content
+
+        Raises:
+            Exception: If API call fails
+        """
+        # Determine actual model and temperature (instance override > provider default)
+        actual_model = model if model is not None else self.config.model
+        actual_temperature = temperature if temperature is not None else self.config.temperature
+
+        # Debug output: Request info
+        if self.config.debug:
+            self._print_request_info(messages, actual_model, actual_temperature)
+
+        # Format messages
+        formatted_messages = self._format_messages(messages)
+
+        # Record start time
+        start_time = time.time()
+
+        try:
+            # Make API call
+            response = self.client.chat.completions.create(
+                model=actual_model,
+                messages=formatted_messages,
+                temperature=actual_temperature,
+            )
+
+            # Calculate processing time
+            processing_time_ms = int((time.time() - start_time) * 1000)
+
+            # Extract content
+            if not response.choices or len(response.choices) == 0:
+                raise Exception("API response error: no choices returned")
+
+            choice = response.choices[0]
+            if not choice.message:
+                raise Exception("Response has no message field")
+
+            content = choice.message.content or ""
+
+            # Extract token usage
+            usage = response.usage
+            metadata = {
+                "model": actual_model,
+                "temperature": actual_temperature,
+                "provider": "openai-compatible",
+                "processing_time": processing_time_ms,
+                "timestamp": int(time.time()),
+            }
+
+            if usage:
+                metadata.update(
+                    {
+                        "prompt_tokens": usage.prompt_tokens,
+                        "output_tokens": usage.completion_tokens,
+                        "total_tokens": usage.total_tokens,
+                    }
+                )
+
+            # Save metadata for retrieval by MarkdownFlow
+            self._last_metadata = metadata
+
+            # Debug output: Response metadata
+            if self.config.debug:
+                self._print_response_metadata(metadata)
+
+            return content
+
+        except Exception as e:
+            raise Exception(f"API request failed: {str(e)}") from e
+
+    def stream(
+        self,
+        messages: list[dict[str, str]],
+        model: str | None = None,
+        temperature: float | None = None,
+    ) -> Generator[str, None, None]:
+        """
+        Streaming LLM call.
+
+        Args:
+            messages: Message list
+            model: Optional model override
+            temperature: Optional temperature override
+
+        Yields:
+            Incremental LLM response content
+
+        Raises:
+            Exception: If API call fails
+        """
+        # Determine actual model and temperature
+        actual_model = model if model is not None else self.config.model
+        actual_temperature = temperature if temperature is not None else self.config.temperature
+
+        # Debug output: Request info
+        if self.config.debug:
+            self._print_request_info(messages, actual_model, actual_temperature)
+
+        # Format messages
+        formatted_messages = self._format_messages(messages)
+
+        # Record start time
+        start_time = time.time()
+
+        try:
+            # Create streaming response
+            stream = self.client.chat.completions.create(
+                model=actual_model,
+                messages=formatted_messages,
+                temperature=actual_temperature,
+                stream=True,
+            )
+ 
+            for chunk in stream:
+                if chunk.choices and chunk.choices[0].delta.content:
+                    yield chunk.choices[0].delta.content
+
+            # Calculate processing time after stream completes
+            processing_time_ms = int((time.time() - start_time) * 1000)
+
+            # Save metadata for retrieval by MarkdownFlow
+            metadata = {
+                "model": actual_model,
+                "temperature": actual_temperature,
+                "provider": "openai-compatible",
+                "processing_time": processing_time_ms,
+                "timestamp": int(time.time()),
+                "stream_done": True,
+            }
+            self._last_metadata = metadata
+
+            # Debug output: Stream completion info
+            if self.config.debug:
+                self._print_response_metadata(metadata)
+
+        except Exception as e:
+            raise ValueError(f"Streaming request failed: {str(e)}") from e
+
+    def get_last_metadata(self) -> dict[str, Any]:
+        """
+        Get metadata from the last LLM call.
+
+        This method allows MarkdownFlow to retrieve comprehensive metadata including
+        token usage, processing time, and other information from the most recent
+        complete() or stream() call.
+
+        Returns:
+            Dictionary containing metadata:
+            - model: Model name used
+            - temperature: Temperature value used
+            - provider: Provider identifier
+            - processing_time: Processing time in milliseconds
+            - timestamp: Unix timestamp
+            - prompt_tokens: Number of input tokens (if available)
+            - output_tokens: Number of output tokens (if available)
+            - total_tokens: Total tokens (if available)
+            - stream_done: True if this was a completed stream (stream mode only)
+
+        Example:
+            >>> provider = create_default_provider()
+            >>> content = provider.complete(messages)
+            >>> metadata = provider.get_last_metadata()
+            >>> print(f"Used {metadata['total_tokens']} tokens")
+        """
+        return self._last_metadata.copy()
+
+    def _format_messages(self, messages: list[dict[str, str]]) -> list[dict[str, str]]:
+        """
+        Format messages for API call.
+
+        Args:
+            messages: Raw message list
+
+        Returns:
+            Formatted message list
+        """
+        formatted = []
+        for msg in messages:
+            if isinstance(msg, dict) and "role" in msg and "content" in msg:
+                formatted.append(
+                    {
+                        "role": msg["role"],
+                        "content": str(msg["content"]),
+                    }
+                )
+            else:
+                # Fallback for non-standard format
+                formatted.append(
+                    {
+                        "role": "user",
+                        "content": str(msg),
+                    }
+                )
+        return formatted
+
+    def _print_request_info(self, messages: list[dict[str, str]], model: str, temperature: float) -> None:
+        """
+        Print colorized request information to console (debug mode).
+
+        Args:
+            messages: Message list
+            model: Model name
+            temperature: Temperature value
+        """
+        print("\033[97m\033[44m[ ====== LLM Request Start ====== ]\033[0m")
+        print(f"\033[30m\033[42mmodel\033[0m: {model}")
+        print(f"\033[30m\033[42mtemperature\033[0m: {temperature}")
+
+        for message in messages:
+            role = message.get("role", "user")
+            content = message.get("content", "")
+            # Truncate long content for readability
+            display_content = content if len(content) <= 200 else content[:200] + "..."
+            print(f"\033[30m\033[43m{role}\033[0m: {display_content}")
+
+        print("\033[97m\033[44m[ ====== LLM Request End ====== ]\033[0m")
+
+    def _print_response_metadata(self, metadata: dict[str, Any]) -> None:
+        """
+        Print colorized response metadata to console (debug mode).
+
+        Args:
+            metadata: Response metadata dictionary
+        """
+        print("\033[97m\033[42m[ ====== LLM Response Metadata ====== ]\033[0m")
+
+        # Essential fields
+        print(f"\033[36mmodel:\033[0m {metadata.get('model', 'N/A')}")
+        print(f"\033[36mtemperature:\033[0m {metadata.get('temperature', 'N/A')}")
+        print(f"\033[36mprovider:\033[0m {metadata.get('provider', 'N/A')}")
+        print(f"\033[36mprocessing_time:\033[0m {metadata.get('processing_time', 'N/A')} ms")
+
+        # Token usage (if available)
+        if "prompt_tokens" in metadata:
+            print(
+                f"\033[36mprompt_tokens:\033[0m \033[33m{metadata['prompt_tokens']}\033[0m  "
+                f"\033[36moutput_tokens:\033[0m \033[33m{metadata['output_tokens']}\033[0m  "
+                f"\033[36mtotal_tokens:\033[0m \033[32m{metadata['total_tokens']}\033[0m"
+            )
+
+        print(f"\033[36mtimestamp:\033[0m {metadata.get('timestamp', 'N/A')}")
+
+        if metadata.get("stream_done"):
+            print("\033[36mstream:\033[0m completed")
+
+        print("\033[97m\033[42m[ ====== ======================= ====== ]\033[0m")
+
+
+def create_provider(config: ProviderConfig | None = None) -> OpenAIProvider:
+    """
+    Create an OpenAI provider instance.
+
+    Args:
+        config: Optional provider configuration. If None, uses default config
+                (reads from environment variables).
+
+    Returns:
+        OpenAIProvider instance
+
+    Raises:
+        ValueError: If configuration is invalid
+        ImportError: If openai package is not installed
+
+    Example:
+        >>> config = ProviderConfig(api_key="sk-...", model="gpt-4")
+        >>> provider = create_provider(config)
+    """
+    if config is None:
+        config = ProviderConfig()
+    return OpenAIProvider(config)
+
+
+def create_default_provider() -> OpenAIProvider:
+    """
+    Create an OpenAI provider with default configuration.
+
+    Reads configuration from environment variables:
+    - LLM_API_KEY: API key (required)
+    - LLM_BASE_URL: Base URL (default: https://api.openai.com/v1)
+    - LLM_MODEL: Model name (default: gpt-3.5-turbo)
+    - LLM_TEMPERATURE: Temperature (default: 0.7)
+    - LLM_DEBUG: Debug mode (default: false)
+    - LLM_TIMEOUT: Request timeout in seconds (default: None, no timeout)
+
+    Returns:
+        OpenAIProvider instance with default config
+
+    Raises:
+        ValueError: If LLM_API_KEY is not set
+        ImportError: If openai package is not installed
+
+    Example:
+        >>> # Set environment variable first
+        >>> import os
+        >>> os.environ["LLM_API_KEY"] = "sk-..."
+        >>> provider = create_default_provider()
+    """
+    return create_provider()