unrealon 1.0.9__py3-none-any.whl → 1.1.0__py3-none-any.whl

unrealon_llm/src/utils/smart_counter.py

@@ -1,229 +0,0 @@
-"""
-Smart Token Counter
-
-Intelligent token counting and cost estimation using ModelsCache.
-Combines pure token counting with live model data.
-"""
-
-import logging
-from typing import List, Optional, Tuple
-
-from unrealon_llm.src.dto import ChatMessage
-from unrealon_llm.src.exceptions import ValidationError
-from unrealon_llm.src.utils.models_cache import ModelsCache, ModelInfo
-from unrealon_llm.src.utils.token_counter import TokenCounter
-
-logger = logging.getLogger(__name__)
-
-
-class SmartTokenCounter:
-    """
-    Intelligent token counter with model-aware cost estimation
-    """
-    
-    def __init__(self, models_cache: ModelsCache):
-        """
-        Initialize smart counter
-        
-        Args:
-            models_cache: Models cache for pricing information
-        """
-        self.models_cache = models_cache
-        self.token_counter = TokenCounter()
-    
-    async def count_and_estimate(
-        self, 
-        text: str, 
-        model: str,
-        estimated_output_tokens: int = 0
-    ) -> Tuple[int, float]:
-        """
-        Count tokens and estimate cost
-        
-        Args:
-            text: Input text
-            model: Model name
-            estimated_output_tokens: Expected output tokens
-            
-        Returns:
-            Tuple of (input_tokens, estimated_cost_usd)
-        """
-        # Count input tokens
-        input_tokens = self.token_counter.count_tokens(text, model)
-        
-        # Get model info for cost estimation
-        if self.models_cache:
-            await self.models_cache.fetch_all_models()
-            model_info = self.models_cache.get_model(model)
-        else:
-            model_info = None
-        
-        if model_info:
-            cost = model_info.estimate_cost(input_tokens, estimated_output_tokens)
-        else:
-            # Fallback estimation
-            cost = self._fallback_cost_estimation(input_tokens, estimated_output_tokens)
-            logger.warning(f"Model {model} not found in cache, using fallback pricing")
-        
-        return input_tokens, cost
-    
-    async def count_messages_and_estimate(
-        self,
-        messages: List[ChatMessage],
-        model: str,
-        estimated_output_tokens: int = 0
-    ) -> Tuple[int, float]:
-        """
-        Count message tokens and estimate cost
-        
-        Args:
-            messages: Chat messages
-            model: Model name
-            estimated_output_tokens: Expected output tokens
-            
-        Returns:
-            Tuple of (total_tokens, estimated_cost_usd)
-        """
-        # Count tokens including message overhead
-        total_tokens = self.token_counter.count_message_tokens(messages, model)
-        
-        # Get model info for cost estimation
-        if self.models_cache:
-            await self.models_cache.fetch_all_models()
-            model_info = self.models_cache.get_model(model)
-        else:
-            model_info = None
-        
-        if model_info:
-            cost = model_info.estimate_cost(total_tokens, estimated_output_tokens)
-        else:
-            # Fallback estimation
-            cost = self._fallback_cost_estimation(total_tokens, estimated_output_tokens)
-            logger.warning(f"Model {model} not found in cache, using fallback pricing")
-        
-        return total_tokens, cost
-    
-    async def optimize_for_budget(
-        self,
-        text: str,
-        model: str,
-        max_cost_usd: float,
-        estimated_output_tokens: int = 0
-    ) -> str:
-        """
-        Optimize text to fit within budget
-        
-        Args:
-            text: Input text
-            model: Model name
-            max_cost_usd: Maximum allowed cost
-            estimated_output_tokens: Expected output tokens
-            
-        Returns:
-            Optimized text that fits budget
-        """
-        # Get model pricing - first fetch models, then get model info
-        if self.models_cache:
-            await self.models_cache.fetch_all_models()
-            model_info = self.models_cache.get_model(model)
-        else:
-            model_info = None
-            
-        if not model_info:
-            logger.warning(f"Model {model} not found, using fallback optimization")
-            return self.token_counter.optimize_text_for_tokens(text, 1000, model)
-        
-        # Calculate max input tokens for budget
-        output_cost = (estimated_output_tokens / 1_000_000) * model_info.completion_price
-        remaining_budget = max_cost_usd - output_cost
-        
-        if remaining_budget <= 0:
-            raise ValidationError("Budget too low even for output tokens")
-        
-        max_input_tokens = int((remaining_budget / model_info.prompt_price) * 1_000_000)
-        
-        # Optimize text for token limit
-        return self.token_counter.optimize_text_for_tokens(text, max_input_tokens, model)
-    
-    async def get_model_context_limit(self, model: str) -> Optional[int]:
-        """Get context length limit for model"""
-        if self.models_cache:
-            await self.models_cache.fetch_all_models()
-            model_info = self.models_cache.get_model(model)
-            return model_info.context_length if model_info else None
-        return None
-    
-    async def suggest_cheaper_alternative(
-        self,
-        current_model: str,
-        input_tokens: int,
-        output_tokens: int
-    ) -> Optional[Tuple[str, float, float]]:
-        """
-        Suggest cheaper model alternative
-        
-        Returns:
-            Tuple of (model_name, current_cost, suggested_cost) or None
-        """
-        if self.models_cache:
-            await self.models_cache.fetch_all_models()
-            current_info = self.models_cache.get_model(current_model)
-        else:
-            current_info = None
-            
-        if not current_info:
-            return None
-        
-        current_cost = current_info.estimate_cost(input_tokens, output_tokens)
-        
-        # Find cheaper models with similar context length
-        min_context = min(current_info.context_length, input_tokens + output_tokens)
-        
-        # Note: get_budget_models is not an async method, let's check if models_cache has this method
-        # For now, we'll implement a simple fallback
-        budget_models = []
-        if self.models_cache:
-            # Find models with lower pricing
-            all_models = list(self.models_cache.models.values())
-            max_price = current_info.prompt_price * 0.8  # 20% cheaper
-            budget_models = [
-                m for m in all_models 
-                if m.prompt_price <= max_price and m.context_length >= min_context and m.is_available
-            ]
-        
-        for model_info in budget_models:
-            if model_info.context_length >= min_context:
-                suggested_cost = model_info.estimate_cost(input_tokens, output_tokens)
-                if suggested_cost < current_cost:
-                    return model_info.id, current_cost, suggested_cost
-        
-        return None
-    
-    def _fallback_cost_estimation(self, input_tokens: int, output_tokens: int) -> float:
-        """Fallback cost estimation when model not found"""
-        # Conservative estimation: $0.002 per 1K tokens (GPT-3.5-turbo-like pricing)
-        total_tokens = input_tokens + output_tokens
-        return (total_tokens / 1000) * 0.002
-
-
-# Convenience functions for backward compatibility
-async def smart_count_tokens(
-    text: str, 
-    model: str, 
-    models_cache: ModelsCache,
-    estimate_output: int = 0
-) -> Tuple[int, float]:
-    """Quick smart token counting with cost estimation"""
-    counter = SmartTokenCounter(models_cache)
-    return await counter.count_and_estimate(text, model, estimate_output)
-
-
-async def smart_count_messages(
-    messages: List[ChatMessage],
-    model: str,
-    models_cache: ModelsCache,
-    estimate_output: int = 0
-) -> Tuple[int, float]:
-    """Quick smart message counting with cost estimation"""
-    counter = SmartTokenCounter(models_cache)
-    return await counter.count_messages_and_estimate(messages, model, estimate_output)

unrealon_llm/src/utils/token_counter.py

@@ -1,189 +0,0 @@
-"""
-Token Counting Utilities
-
-Pure token counting with tiktoken - no hardcoded models.
-Works with ModelsCache for model information.
-"""
-
-import re
-from typing import Dict, List, Optional
-
-import tiktoken
-
-from unrealon_llm.src.dto import ChatMessage
-from unrealon_llm.src.exceptions import ValidationError
-
-
-class TokenCounter:
-    """Pure token counting utility - no model knowledge"""
-    
-    def __init__(self):
-        """Initialize token counter"""
-        # Cache for encoding objects
-        self._encoding_cache: Dict[str, any] = {}
-        
-        # Universal encoding for all models
-        self._universal_encoding = "cl100k_base"
-        
-        # Language ratios for character-based estimation
-        self._language_ratios = {
-            "english": 0.25,      # ~4 chars per token
-            "korean": 0.33,       # ~3 chars per token  
-            "chinese": 0.5,       # ~2 chars per token
-            "japanese": 0.4,      # ~2.5 chars per token
-            "russian": 0.25,      # ~4 chars per token
-            "default": 0.25       # Conservative estimate
-        }
-    
-    def count_tokens(self, text: str, model: str = "gpt-3.5-turbo") -> int:
-        """
-        Count tokens in text for specified model
-        
-        Args:
-            text: Text to count tokens for
-            model: Model name
-            
-        Returns:
-            Number of tokens
-        """
-        if not isinstance(text, str):
-            raise ValidationError("Text must be a string")
-        
-        if not text:
-            return 0
-        
-        # Try tiktoken first
-        try:
-            return self._count_with_tiktoken(text, model)
-        except Exception:
-            # Fallback to estimation
-            return self._estimate_tokens(text)
-    
-    def count_message_tokens(self, messages: List[ChatMessage], model: str = "gpt-3.5-turbo") -> int:
-        """
-        Count tokens for chat messages including formatting overhead
-        
-        Args:
-            messages: List of chat messages
-            model: Model name
-            
-        Returns:
-            Total tokens including overhead
-        """
-        if not messages:
-            return 0
-        
-        total_tokens = 0
-        
-        # Count tokens for each message
-        for message in messages:
-            # Content tokens
-            total_tokens += self.count_tokens(message.content, model)
-            
-            # Role tokens
-            total_tokens += self.count_tokens(message.role.value, model)
-            
-            # Name tokens if present
-            if message.name:
-                total_tokens += self.count_tokens(message.name, model)
-            
-            # Message formatting overhead
-            total_tokens += self._get_message_overhead(model)
-        
-        # Conversation overhead
-        total_tokens += self._get_conversation_overhead(model)
-        
-        return total_tokens
-    
-    def optimize_text_for_tokens(self, text: str, max_tokens: int, model: str = "gpt-3.5-turbo") -> str:
-        """
-        Truncate text to fit within token limit
-        
-        Args:
-            text: Input text
-            max_tokens: Maximum allowed tokens
-            model: Model name
-            
-        Returns:
-            Truncated text
-        """
-        current_tokens = self.count_tokens(text, model)
-        
-        if current_tokens <= max_tokens:
-            return text
-        
-        # Binary search for optimal length
-        left, right = 0, len(text)
-        best_text = ""
-        
-        while left <= right:
-            mid = (left + right) // 2
-            candidate = text[:mid]
-            
-            if self.count_tokens(candidate, model) <= max_tokens:
-                best_text = candidate
-                left = mid + 1
-            else:
-                right = mid - 1
-        
-        return best_text
-    
-    def _count_with_tiktoken(self, text: str, model: str) -> int:
-        """Count tokens using tiktoken (universal encoding)"""
-        encoding_name = self._universal_encoding
-        
-        if encoding_name not in self._encoding_cache:
-            self._encoding_cache[encoding_name] = tiktoken.get_encoding(encoding_name)
-        
-        encoding = self._encoding_cache[encoding_name]
-        return len(encoding.encode(text))
-    
-    def _estimate_tokens(self, text: str) -> int:
-        """Fallback token estimation"""
-        language = self._detect_language(text)
-        ratio = self._language_ratios.get(language, self._language_ratios["default"])
-        
-        char_count = len(re.sub(r'\s+', '', text))
-        estimated_tokens = int(char_count * ratio)
-        
-        return max(1, estimated_tokens)
-    
-    def _detect_language(self, text: str) -> str:
-        """Simple language detection for estimation"""
-        if re.search(r'[가-힣]', text):
-            return "korean"
-        elif re.search(r'[\u4e00-\u9fff]', text):
-            return "chinese"
-        elif re.search(r'[ひらがなカタカナ\u3040-\u309f\u30a0-\u30ff]', text):
-            return "japanese"
-        elif re.search(r'[а-яё]', text, re.IGNORECASE):
-            return "russian"
-        else:
-            return "english"
-    
-    def _get_message_overhead(self, model: str) -> int:
-        """Message formatting overhead (universal)"""
-        return 3  # Universal overhead for all models
-    
-    def _get_conversation_overhead(self, model: str) -> int:
-        """Conversation formatting overhead (universal)"""
-        return 2  # Universal overhead for all models
-
-
-# Convenience functions
-def count_tokens(text: str, model: str = "gpt-3.5-turbo") -> int:
-    """Quick token counting"""
-    counter = TokenCounter()
-    return counter.count_tokens(text, model)
-
-
-def count_message_tokens(messages: List[ChatMessage], model: str = "gpt-3.5-turbo") -> int:
-    """Quick message token counting"""
-    counter = TokenCounter()
-    return counter.count_message_tokens(messages, model)
-
-
-def optimize_for_tokens(text: str, max_tokens: int, model: str = "gpt-3.5-turbo") -> str:
-    """Quick text optimization for token limits"""
-    counter = TokenCounter()
-    return counter.optimize_text_for_tokens(text, max_tokens, model)

unrealon_sdk/README.md

@@ -1,25 +0,0 @@
-# UnrealOn SDK
-
-Core SDK for UnrealOn parser development with WebSocket communication and enterprise services.
-
-## Features
-
-- WebSocket client for real-time communication
-- Auto-generated API models
-- Enterprise services (proxy, monitoring, etc.)
-- CLI tools for SDK management
-
-## Installation
-
-```bash
-pip install unrealon-sdk
-```
-
-## Usage
-
-```python
-from unrealon_sdk import AdapterClient, AdapterConfig
-
-config = AdapterConfig()
-client = AdapterClient(config)
-```

unrealon_sdk/__init__.py

@@ -1,30 +0,0 @@
-"""
-🚀 UnrealOn SDK v1.0 - Enterprise Parsing Platform
-
-The most advanced SDK for building enterprise-grade parsing solutions with:
-- 90% code reduction through intelligent automation
-- Type-safe operations with auto-generated models  
-- Production-ready features out of the box
-- Real-time communication and monitoring
-"""
-
-# Import everything from src
-from .src import *
-
-# Description
-__description__ = "Enterprise Parsing Platform SDK for UnrealOn"
-
-
-# Import from centralized config
-from unrealon.sdk_config import (
-    VERSION as __version__,
-    AUTHOR as __author__,
-    AUTHOR_EMAIL as __email__,
-    LICENSE as __license__,
-    PROJECT_URL as __url__,
-    VERSION_INFO,
-    PROJECT_INFO,
-    get_version,
-    is_debug_mode,
-    check_compatibility,
-)

unrealon_sdk/pyproject.toml

@@ -1,231 +0,0 @@
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
-
-[tool.poetry]
-name = "unrealon-sdk"
-version = "1.0.0"
-description = "UnrealOn SDK Core - WebSocket, enterprise services, auto-generated models"
-authors = ["UnrealOn SDK Team <sdk@unrealon.com>"]
-readme = "README.md"
-license = "MIT"
-homepage = "https://github.com/unrealon/unrealon-sdk"
-repository = "https://github.com/unrealon/unrealon-sdk"
-documentation = "https://docs.unrealon.com/sdk"
-keywords = ["websocket", "enterprise", "sdk", "parsing", "automation"]
-classifiers = [
-    "Development Status :: 5 - Production/Stable",
-    "Intended Audience :: Developers",
-    "Topic :: Software Development :: Libraries :: Python Modules",
-    "Topic :: System :: Networking",
-    "Topic :: Internet :: WWW/HTTP",
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11", 
-    "Programming Language :: Python :: 3.12",
-    "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent",
-]
-packages = [{include = "*", from = "src"}]
-
-[tool.poetry.dependencies]
-# CORE SDK DEPENDENCIES
-python = "^3.9"
-
-# Core Pydantic v2 requirement (MANDATORY)
-pydantic = "^2.0.0"
-pydantic-settings = "^2.0.0"
-
-# WebSocket and Network Core
-python-socketio = "^5.0"
-websockets = "^12.0"
-aiohttp = "^3.8.0"
-httpx = ">=0.23.0,<0.24.0"
-
-# Type System
-typing-extensions = "^4.0"
-
-# Configuration and Data
-PyYAML = "^6.0"
-python-dateutil = "^2.8"
-
-[tool.poetry.group.dev.dependencies]
-# Testing Framework (SDK Core)
-pytest = "^7.4.0"
-pytest-asyncio = "^0.21.0"
-pytest-cov = "^4.1.0"
-pytest-mock = "^3.11.0"
-pytest-benchmark = "^4.0.0"
-pytest-html = "^4.0.0"
-pytest-xdist = "^3.3.0"
-pytest-timeout = "^2.1.0"
-
-# Type Checking (SDK Core)
-mypy = "^1.5.0"
-types-PyYAML = "^6.0"
-types-python-dateutil = "^2.8"
-
-# Security Testing
-bandit = "^1.7.5"
-safety = "^2.3.0"
-
-# Code Quality
-black = "^23.7.0"
-isort = "^5.12.0"
-flake8 = "^6.0.0"
-pylint = "^2.17.0"
-
-# Performance Testing
-locust = "^2.0.0"
-
-# Mock and Test Utilities
-responses = "^0.23.0"
-aioresponses = "^0.7.4"
-
-[tool.poetry.scripts]
-# SDK CLI entry points (when used standalone)
-unrealon-sdk = "unrealon_sdk.cli.main:cli"
-
-# SDK management utilities
-sdk-health = "unrealon_sdk.cli.commands.health:main"
-sdk-benchmark = "unrealon_sdk.cli.commands.benchmark:main"
-sdk-diagnostics = "unrealon_sdk.cli.commands.diagnostics:main"
-sdk-integration = "unrealon_sdk.cli.commands.integration:main"
-sdk-security = "unrealon_sdk.cli.commands.security:main"
-sdk-server = "unrealon_sdk.cli.commands.server:main"
-
-[tool.mypy]
-# SDK Core type checking (Strictest)
-python_version = "3.9"
-strict = true
-warn_return_any = true
-warn_unused_configs = true
-disallow_any_generics = true
-disallow_subclassing_any = true
-disallow_untyped_calls = true
-disallow_untyped_defs = true
-disallow_incomplete_defs = true
-check_untyped_defs = true
-disallow_untyped_decorators = true
-no_implicit_optional = true
-warn_redundant_casts = true
-warn_unused_ignores = true
-warn_no_return = true
-warn_unreachable = true
-strict_equality = true
-show_error_codes = true
-
-[[tool.mypy.overrides]]
-module = [
-    "socketio.*",
-    "aiohttp.*",
-    "locust.*",
-    "responses.*",
-    "aioresponses.*",
-]
-ignore_missing_imports = true
-
-[tool.black]
-# SDK Core formatting
-line-length = 100
-target-version = ['py39']
-include = '\.pyi?$'
-
-[tool.isort]
-# SDK Core import sorting
-profile = "black"
-line_length = 100
-multi_line_output = 3
-include_trailing_comma = true
-force_grid_wrap = 0
-use_parentheses = true
-ensure_newline_before_comments = true
-known_first_party = ["unrealon_sdk"]
-known_third_party = ["socketio", "aiohttp", "httpx", "pydantic", "yaml"]
-
-[tool.pytest.ini_options]
-# SDK Core testing (Strictest requirements)
-minversion = "7.0"
-testpaths = ["tests"]
-python_files = ["test_*.py", "*_test.py"]
-python_classes = ["Test*"]
-python_functions = ["test_*"]
-addopts = [
-    "--strict-markers",
-    "--strict-config",
-    "--cov=unrealon_sdk",
-    "--cov-report=term-missing",
-    "--cov-report=html:htmlcov_sdk",
-    "--cov-report=xml:coverage_sdk.xml",
-    "--cov-fail-under=100",  # SDK requires 100% coverage
-    "--cov-branch",
-    "--benchmark-disable",
-    "--timeout=30",
-    "-v",
-    "-ra",
-]
-markers = [
-    "unit: Unit tests (fast, no external dependencies)",
-    "integration: Integration tests (slower, external dependencies)",
-    "websocket: WebSocket communication tests",
-    "enterprise: Enterprise features tests",
-    "performance: Performance and benchmark tests",
-    "security: Security and vulnerability tests",
-    "slow: Slow tests (> 5 seconds)",
-    "network: Tests requiring network access",
-]
-asyncio_mode = "auto"
-timeout = 30
-filterwarnings = [
-    "error",
-    "ignore::UserWarning",
-    "ignore::DeprecationWarning",
-]
-
-[tool.coverage.run]
-# SDK Core coverage (100% requirement)
-source = ["unrealon_sdk"]
-branch = true
-omit = [
-    "*/tests/*",
-    "*/test_*",
-    "*/__pycache__/*",
-    "*/examples/*",
-]
-
-[tool.coverage.report]
-# SDK Core coverage reporting
-precision = 2
-show_missing = true
-skip_covered = false
-exclude_lines = [
-    "pragma: no cover",
-    "def __repr__",
-    "if self.debug:",
-    "if settings.DEBUG",
-    "raise AssertionError",
-    "raise NotImplementedError",
-    "if 0:",
-    "if __name__ == .__main__.:",
-    "class .*\\bProtocol\\):",
-    "@(abc\\.)?abstractmethod",
-]
-
-[tool.bandit]
-# SDK Core security scanning
-exclude_dirs = ["tests", "examples"]
-skips = ["B101", "B601"]
-
-[tool.pylint.messages_control]
-# SDK Core code quality
-disable = [
-    "too-few-public-methods",
-    "too-many-arguments",
-    "too-many-instance-attributes", 
-    "line-too-long",
-    "missing-docstring",
-]
-
-[tool.pylint.format]
-max-line-length = 100