unrealon 1.0.9__py3-none-any.whl → 1.1.0__py3-none-any.whl

unrealon_llm/src/modules/html_processor/models/universal_model.py

@@ -1,56 +0,0 @@
-"""
-Universal HTML Processing Model
-
-Single simplified Pydantic model for any HTML page extraction with markdown documentation.
-"""
-
-from typing import Dict, List, Union
-from pydantic import BaseModel, Field, ConfigDict, field_validator
-
-
-class UniversalExtractionSchema(BaseModel):
-    """Universal HTML page extraction schema with markdown documentation"""
-    
-    model_config = ConfigDict(
-        validate_assignment=True,
-        extra="allow",  # Allow extra fields for flexibility
-        title="Universal Extraction Schema"
-    )
-    
-    # LLM analysis results
-    detected_item_type: str = Field(
-        ...,
-        description="Auto-detected type of page (product, listing, article, service, etc.)"
-    )
-    extraction_strategy: str = Field(
-        ...,
-        description="Brief description of extraction strategy"
-    )
-    confidence: float = Field(
-        ...,
-        ge=0,
-        le=1,
-        description="Overall extraction confidence"
-    )
-    
-    # Simple CSS selectors organized by field
-    selectors: Dict[str, List[str]] = Field(
-        ...,
-        description="CSS selectors organized by field name (title, price, description, items_container, etc.)"
-    )
-    
-    # Comprehensive markdown documentation
-    documentation: str = Field(
-        ...,
-        description="Markdown documentation with examples, explanations, and extraction guidance"
-    )
-    
-    @field_validator('selectors', mode='before')
-    @classmethod
-    def convert_strings_to_lists(cls, v):
-        """Convert string selectors to lists automatically"""
-        if isinstance(v, dict):
-            for key, value in v.items():
-                if isinstance(value, str):
-                    v[key] = [value]
-        return v

unrealon_llm/src/modules/html_processor/processor.py

@@ -1,102 +0,0 @@
-"""
-HTML Processor Factory
-
-Factory class for creating HTML processors.
-"""
-
-from unrealon_llm.src.core import SmartLLMClient
-from unrealon_llm.src.dto import LLMConfig
-
-from .listing_processor import ListingProcessor
-from .details_processor import DetailsProcessor
-
-
-class UnrealOnLLM:
-    """Factory class for creating UnrealOn LLM components"""
-
-    @staticmethod
-    def create_client(
-        openrouter_api_key: str,
-        default_model: str = "anthropic/claude-3.5-sonnet",
-        daily_cost_limit: float = 5.0,
-        enable_caching: bool = True,
-        cache_ttl_minutes: int = 30,
-    ) -> SmartLLMClient:
-        """
-        Create LLM client
-
-        Args:
-            openrouter_api_key: OpenRouter API key
-            default_model: Default model to use
-            daily_cost_limit: Daily cost limit in USD
-            enable_caching: Enable response caching
-            cache_ttl_minutes: Cache TTL in minutes
-
-        Returns:
-            Configured SmartLLMClient instance
-        """
-        config = LLMConfig(
-            openrouter_api_key=openrouter_api_key,
-            default_model=default_model,
-            daily_cost_limit_usd=daily_cost_limit,
-            request_timeout_seconds=60,
-            max_retries=3,
-            enable_global_cache=enable_caching,
-            cache_ttl_hours=max(1, int(cache_ttl_minutes / 60)),
-        )
-
-        return SmartLLMClient(config)
-
-    @staticmethod
-    def create_listing_processor(
-        openrouter_api_key: str,
-        default_model: str = "anthropic/claude-3.5-sonnet",
-        daily_cost_limit: float = 1.0,
-        enable_caching: bool = False,  # Disable cache for HTML processors
-    ) -> ListingProcessor:
-        """
-        Create listing processor
-
-        Args:
-            openrouter_api_key: OpenRouter API key
-            default_model: Default model to use
-            daily_cost_limit: Daily cost limit in USD
-            enable_caching: Enable response caching (disabled by default for HTML processing)
-
-        Returns:
-            Configured ListingProcessor instance
-        """
-        llm_client = UnrealOnLLM.create_client(
-            openrouter_api_key=openrouter_api_key,
-            default_model=default_model,
-            daily_cost_limit=daily_cost_limit,
-            enable_caching=enable_caching,
-        )
-        return ListingProcessor(llm_client)
-
-    @staticmethod
-    def create_details_processor(
-        openrouter_api_key: str,
-        default_model: str = "anthropic/claude-3.5-sonnet",
-        daily_cost_limit: float = 1.0,
-        enable_caching: bool = False,  # Disable cache for HTML processors
-    ) -> DetailsProcessor:
-        """
-        Create details processor
-
-        Args:
-            openrouter_api_key: OpenRouter API key
-            default_model: Default model to use
-            daily_cost_limit: Daily cost limit in USD
-            enable_caching: Enable response caching (disabled by default for HTML processing)
-
-        Returns:
-            Configured DetailsProcessor instance
-        """
-        llm_client = UnrealOnLLM.create_client(
-            openrouter_api_key=openrouter_api_key,
-            default_model=default_model,
-            daily_cost_limit=daily_cost_limit,
-            enable_caching=enable_caching,
-        )
-        return DetailsProcessor(llm_client)

unrealon_llm/src/provider.py

@@ -1,116 +0,0 @@
-"""
-UnrealOn LLM Provider
-
-Simple provider module for UnrealOn LLM functionality.
-"""
-
-# Core LLM functionality
-from unrealon_llm.src.core import SmartLLMClient
-
-# HTML parsing and analysis
-from unrealon_llm.src.utils.html_cleaner import SmartHTMLCleaner
-from unrealon_llm.src.modules.html_processor import (
-    ListingProcessor,
-    DetailsProcessor,
-    UnrealOnLLM,
-)
-
-# Logging and configuration
-from unrealon_llm.src.llm_config import setup_llm_logging, configure_llm_logging
-from unrealon_llm.src.llm_logging import (
-    get_llm_logger,
-    LLMEventType,
-    LLMContext,
-    initialize_llm_logger,
-)
-
-# Configuration and DTOs
-from unrealon_llm.src.dto import (
-    # Core configuration
-    LLMConfig,
-    AnalysisConfig,
-    TranslationConfig,
-    # Enums
-    LLMProvider,
-    OptimizationLevel,
-    CacheStrategy,
-    MessageRole,
-    PatternType,
-    SelectorType,
-    LanguageCode,
-    DataType,
-    SchemaFormat,
-    ProcessingStage,
-    # Core models
-    TokenUsage,
-    ChatMessage,
-    LLMResponse,
-    LanguageDetection,
-    CostBreakdown,
-    HealthStatus,
-    ProcessingMetrics,
-    # HTML Analysis models
-    DetectedPattern,
-    SelectorInfo,
-    HTMLAnalysisRequest,
-    HTMLAnalysisResult,
-    SelectorValidationResult,
-    CompleteAnalysisResult,
-)
-
-# Utilities
-from unrealon_llm.src.utils.data_extractor import SmartDataExtractor
-from unrealon_llm.src.utils.smart_counter import SmartTokenCounter
-from unrealon_llm.src.utils.language_detector import LanguageDetector
-
-
-# Direct exports for convenience
-__all__ = [
-    # Factory class
-    "UnrealOnLLM",
-    # Core classes
-    "SmartLLMClient",
-    "SmartHTMLCleaner",
-    "SmartDataExtractor",
-    "SmartTokenCounter",
-    "LanguageDetector",
-    "ListingProcessor",
-    "DetailsProcessor",
-    # Configuration classes
-    "LLMConfig",
-    "AnalysisConfig",
-    "TranslationConfig",
-    # Enums
-    "LLMProvider",
-    "OptimizationLevel",
-    "CacheStrategy",
-    "MessageRole",
-    "PatternType",
-    "SelectorType",
-    "LanguageCode",
-    "DataType",
-    "SchemaFormat",
-    "ProcessingStage",
-    # Core models
-    "TokenUsage",
-    "ChatMessage",
-    "LLMResponse",
-    "LanguageDetection",
-    "CostBreakdown",
-    "HealthStatus",
-    "ProcessingMetrics",
-    # HTML Analysis models
-    "DetectedPattern",
-    "SelectorInfo",
-    "HTMLAnalysisRequest",
-    "HTMLAnalysisResult",
-    "SelectorValidationResult",
-    "CompleteAnalysisResult",
-    # Logging
-    "setup_llm_logging",
-    "configure_llm_logging",
-    "get_llm_logger",
-    "initialize_llm_logger",
-    "LLMEventType",
-    "LLMContext",
-]

unrealon_llm/src/utils/__init__.py

@@ -1,95 +0,0 @@
-"""
-UnrealOn LLM Utilities
-
-Utility functions and helpers for UnrealOn LLM platform including
-language detection, token counting, and model caching.
-"""
-
-# Language detection utilities
-from .language_detector import (
-    LanguageDetector,
-    detect_language,
-    detect_multiple_languages,
-    is_language,
-)
-
-# Token counting utilities (legacy)
-from .token_counter import (
-    TokenCounter,
-    count_tokens,
-    count_message_tokens,
-    optimize_for_tokens,
-)
-
-# Smart counting utilities (new approach)
-from .smart_counter import (
-    SmartTokenCounter,
-    smart_count_tokens,
-    smart_count_messages,
-)
-
-# Models cache utilities
-from .models_cache import (
-    ModelInfo,
-    ModelsCache,
-)
-
-# HTML cleaning utilities
-from .html_cleaner import (
-    SmartHTMLCleaner,
-    clean_html_for_llm,
-    extract_js_data_only,
-)
-
-# Common utilities
-from .common import (
-    generate_correlation_id,
-    generate_request_id,
-)
-
-# Data extraction utilities
-from .data_extractor import (
-    SmartDataExtractor,
-    safe_extract_json,
-    extract_llm_response_data,
-    create_data_extractor,
-)
-
-# Exports
-__all__ = [
-    # Language Detection
-    "LanguageDetector",
-    "detect_language", 
-    "detect_multiple_languages",
-    "is_language",
-    
-    # Token Counting (Legacy)
-    "TokenCounter",
-    "count_tokens",
-    "count_message_tokens", 
-    "optimize_for_tokens",
-    
-    # Smart Counting (New)
-    "SmartTokenCounter",
-    "smart_count_tokens",
-    "smart_count_messages",
-    
-    # Models Cache
-    "ModelInfo",
-    "ModelsCache",
-    
-    # HTML Cleaning
-    "SmartHTMLCleaner",
-    "clean_html_for_llm",
-    "extract_js_data_only",
-    
-    # Common Utilities
-    "generate_correlation_id",
-    "generate_request_id",
-    
-    # Data Extraction
-    "SmartDataExtractor",
-    "safe_extract_json",
-    "extract_llm_response_data",
-    "create_data_extractor",
-]

unrealon_llm/src/utils/common.py

@@ -1,64 +0,0 @@
-"""
-Common Utilities
-
-General-purpose utility functions for UnrealOn LLM including
-ID generation, validation, and other common helpers.
-"""
-
-import uuid
-import secrets
-from typing import Optional
-
-
-def generate_correlation_id() -> str:
-    """Generate a unique correlation ID for tracking operations."""
-    return f"llm_{uuid.uuid4().hex[:16]}"
-
-
-def generate_request_id() -> str:
-    """Generate a unique request ID for API calls."""
-    return f"req_{secrets.token_hex(8)}"
-
-
-def generate_session_id() -> str:
-    """Generate a unique session ID."""
-    return f"sess_{uuid.uuid4().hex[:12]}"
-
-
-def sanitize_model_name(model_name: str) -> str:
-    """Sanitize model name for logging and metrics."""
-    return model_name.replace("/", "_").replace(":", "_").replace("-", "_")
-
-
-def truncate_text(text: str, max_length: int = 100, suffix: str = "...") -> str:
-    """Truncate text for logging purposes."""
-    if len(text) <= max_length:
-        return text
-    return text[:max_length - len(suffix)] + suffix
-
-
-def format_bytes(bytes_count: int) -> str:
-    """Format bytes into human readable format."""
-    for unit in ['B', 'KB', 'MB', 'GB']:
-        if bytes_count < 1024.0:
-            return f"{bytes_count:.1f} {unit}"
-        bytes_count /= 1024.0
-    return f"{bytes_count:.1f} TB"
-
-
-def format_duration_ms(duration_ms: float) -> str:
-    """Format duration in milliseconds to human readable format."""
-    if duration_ms < 1000:
-        return f"{duration_ms:.1f}ms"
-    elif duration_ms < 60000:
-        return f"{duration_ms / 1000:.1f}s"
-    else:
-        minutes = int(duration_ms / 60000)
-        seconds = (duration_ms % 60000) / 1000
-        return f"{minutes}m{seconds:.1f}s"
-
-
-def safe_get_env(key: str, default: Optional[str] = None) -> Optional[str]:
-    """Safely get environment variable with optional default."""
-    import os
-    return os.getenv(key, default)

unrealon_llm/src/utils/data_extractor.py

@@ -1,188 +0,0 @@
-"""
-Data Extractor
-
-Simple wrapper around json_extractor lib for extracting JSON from text.
-KISS methodology - just extract JSON, nothing more.
-"""
-
-import json
-import logging
-import re
-from typing import Any, Dict, List, Optional, Type, TypeVar, Union
-from pydantic import BaseModel, ValidationError
-
-from unrealon_llm.src.exceptions import ResponseParsingError
-
-logger = logging.getLogger(__name__)
-
-T = TypeVar("T", bound=BaseModel)
-
-
-def _extract_json_smart(content: str) -> Optional[str]:
-    """
-    Smart JSON extraction from text - finds valid JSON objects/arrays.
-
-    Args:
-        content: Text content that may contain JSON
-
-    Returns:
-        First valid JSON string found or None
-    """
-    # Try to find JSON objects {} or arrays []
-    patterns = [
-        r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}",  # Simple nested objects
-        r"\[[^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*\]",  # Simple nested arrays
-    ]
-
-    for pattern in patterns:
-        matches = re.finditer(pattern, content, re.DOTALL)
-        for match in matches:
-            json_candidate = match.group()
-            try:
-                # Test if it's valid JSON
-                json.loads(json_candidate)
-                return json_candidate
-            except json.JSONDecodeError:
-                continue
-
-    # Fallback: find between outermost braces
-    first_brace = content.find("{")
-    if first_brace == -1:
-        return None
-
-    # Find matching closing brace
-    brace_count = 0
-    for i, char in enumerate(content[first_brace:], first_brace):
-        if char == "{":
-            brace_count += 1
-        elif char == "}":
-            brace_count -= 1
-            if brace_count == 0:
-                return content[first_brace : i + 1]
-
-    return None
-
-
-def extract_json(
-    content: str,
-    expected_schema: Optional[Type[T]] = None,
-    fallback_value: Optional[Any] = None,
-    strict_mode: bool = True,
-) -> Union[T, Dict[str, Any], None]:
-    """
-    Extract JSON from text content.
-
-    Args:
-        content: Text content containing JSON
-        expected_schema: Pydantic model for validation
-        fallback_value: Return value if extraction fails (non-strict mode)
-        strict_mode: Raise exception on failure if True
-
-    Returns:
-        Extracted JSON data
-
-    Raises:
-        ResponseParsingError: If extraction fails in strict mode
-    """
-    if not content:
-        if strict_mode:
-            raise ResponseParsingError("Empty content", "json")
-        return fallback_value
-
-    try:
-        # Smart JSON extraction
-        json_content = _extract_json_smart(content)
-
-        if json_content is None:
-            if strict_mode:
-                raise ResponseParsingError("No valid JSON found", "json")
-            return fallback_value
-
-        # Parse the extracted JSON
-        extracted_data = json.loads(json_content)
-        logger.info(f"Successfully extracted and parsed JSON: {type(extracted_data)}")
-
-        # Validate with schema if provided
-        if expected_schema and issubclass(expected_schema, BaseModel):
-            return expected_schema.model_validate(extracted_data)
-
-        return extracted_data
-
-    except ValidationError as e:
-        if strict_mode:
-            raise e
-        logger.warning(f"Schema validation failed: {e}")
-        return fallback_value
-
-    except Exception as e:
-        if strict_mode:
-            raise ResponseParsingError(content[:200], "json")
-        logger.warning(f"JSON extraction failed: {e}")
-        return fallback_value
-
-
-def safe_extract_json(
-    content: str,
-    expected_schema: Optional[Type[T]] = None,
-    fallback_value: Optional[Any] = None,
-) -> Union[T, Dict[str, Any], None]:
-    """
-    Safe JSON extraction (non-strict mode).
-
-    Args:
-        content: Text content
-        expected_schema: Optional schema
-        fallback_value: Fallback value
-
-    Returns:
-        Extracted JSON or fallback
-    """
-    return extract_json(content, expected_schema, fallback_value, strict_mode=False)
-
-
-def extract_llm_response_data(
-    response_content: str,
-    expected_schema: Optional[Type[T]] = None,
-    required_fields: Optional[List[str]] = None,
-) -> Union[T, Dict[str, Any]]:
-    """
-    Extract data from LLM response (strict mode).
-
-    Args:
-        response_content: LLM response text
-        expected_schema: Pydantic model
-        required_fields: Required fields to check
-
-    Returns:
-        Extracted and validated data
-
-    Raises:
-        ResponseParsingError: If extraction fails
-    """
-    result = extract_json(response_content, expected_schema, strict_mode=True)
-
-    # Check required fields if no schema
-    if not expected_schema and required_fields and isinstance(result, dict):
-        missing_fields = [field for field in required_fields if field not in result]
-        if missing_fields:
-            raise ResponseParsingError(
-                f"Missing required fields: {missing_fields}", "json"
-            )
-
-    return result
-
-
-# Legacy compatibility
-class SmartDataExtractor:
-    """Simple wrapper for compatibility."""
-
-    def __init__(self, strict_mode: bool = True):
-        self.strict_mode = strict_mode
-
-    def extract_json(self, content: str, expected_schema=None, fallback_value=None):
-        return extract_json(content, expected_schema, fallback_value, self.strict_mode)
-
-
-def create_data_extractor(strict_mode: bool = True) -> SmartDataExtractor:
-    """Create data extractor instance."""
-    return SmartDataExtractor(strict_mode)