speedy-utils 1.1.22__py3-none-any.whl → 1.1.23__py3-none-any.whl

llm_utils/__init__.py

@@ -1,5 +1,5 @@
 from llm_utils.lm.openai_memoize import MOpenAI
-from llm_utils.lm import  LLMTask, AsyncLM, AsyncLLMTask
+from llm_utils.lm import  LLMTask, AsyncLM, AsyncLLMTask, LLMJudgeBase, ChainOfThought, TranslationEvaluatorJudge, Signature, InputField, OutputField, Input, Output
 from llm_utils.vector_cache import VectorCache
 from llm_utils.lm.lm_base import get_model_name
 from llm_utils.lm.base_prompt_builder import BasePromptBuilder
@@ -15,7 +15,7 @@ def kill_vllm_on_port(port: int) -> bool:
     """Kill VLLM server on specific port. Returns True if server was killed."""
     return LLMTask.kill_vllm_on_port(port)
 
-from .chat_format import (
+from llm_utils.chat_format import (
     build_chatml_input,
     display_chat_messages_as_html,
     display_conversations,
@@ -46,5 +46,13 @@ __all__ = [
     "BasePromptBuilder",
     "LLM",
     "kill_all_vllm",
-    "kill_vllm_on_port"
+    "kill_vllm_on_port",
+    "LLMJudgeBase",
+    "ChainOfThought",
+    "TranslationEvaluatorJudge",
+    "Signature",
+    "InputField",
+    "OutputField",
+    "Input",
+    "Output",
 ]

llm_utils/lm/__init__.py

@@ -3,6 +3,8 @@ from .async_lm.async_llm_task import AsyncLLMTask
 from .lm_base import LMBase, get_model_name
 from .llm_task import  LLMTask
 from .base_prompt_builder import BasePromptBuilder
+from .llm_as_a_judge import LLMJudgeBase, ChainOfThought, TranslationEvaluatorJudge
+from .signature import Signature, InputField, OutputField, Input, Output
 
 __all__ = [
     "LMBase",
@@ -10,4 +12,12 @@ __all__ = [
     "AsyncLM",
     "AsyncLLMTask",
     "BasePromptBuilder",
+    "LLMJudgeBase",
+    "ChainOfThought",
+    "TranslationEvaluatorJudge",
+    "Signature",
+    "InputField",
+    "OutputField",
+    "Input",
+    "Output",
 ]

llm_utils/lm/llm_as_a_judge.py

@@ -0,0 +1,390 @@
+"""
+LLM-as-a-Judge implementation with template support and SFT export utilities.
+
+This module provides a base class for creating LLM judges with structured
+prompts, variable substitution, and export capabilities for fine-tuning.
+"""
+
+import json
+from typing import Any, Dict, List, Optional, Type, Union
+from pydantic import BaseModel
+from ..chat_format import get_conversation_one_turn
+from .llm_task import LLMTask
+from .signature import Signature
+
+
+class LLMJudgeBase(LLMTask):
+    """Base class for LLM judges with template support and SFT export."""
+    
+    def __init__(
+        self,
+        system_prompt_template: str,
+        signature: Optional[Type[Signature]] = None,
+        **kwargs
+    ):
+        """
+        Initialize LLMJudgeBase.
+        
+        Args:
+            system_prompt_template: System prompt template with {variable} placeholders
+            signature: Optional Signature class for structured I/O
+            **kwargs: Additional arguments passed to LLMTask
+        """
+        self.system_prompt_template = system_prompt_template
+        self.signature = signature
+        self.sft_data: List[Dict[str, Any]] = []  # Store SFT training examples
+        
+        # Set instruction from signature if available
+        if signature is not None:
+            instruction = signature.get_instruction()
+            kwargs.setdefault('instruction', instruction)
+            kwargs.setdefault('output_model', signature.get_output_model())
+        else:
+            kwargs.setdefault('instruction', system_prompt_template)
+        
+        super().__init__(**kwargs)
+    
+    def format_system_prompt(self, variables: Dict[str, Any]) -> str:
+        """Format system prompt template with provided variables."""
+        try:
+            return self.system_prompt_template.format(**variables)
+        except KeyError as e:
+            missing_var = str(e).strip("'")
+            raise ValueError(f"Missing required variable '{missing_var}' for system prompt template")
+    
+    def judge(
+        self,
+        input_data: Union[str, Dict[str, Any], BaseModel],
+        variables: Optional[Dict[str, Any]] = None,
+        **runtime_kwargs
+    ) -> List[Dict[str, Any]]:
+        """
+        Execute judgment with variable substitution in system prompt.
+        
+        Args:
+            input_data: Input data for the judge
+            variables: Variables to substitute in system prompt template
+            **runtime_kwargs: Additional runtime arguments
+            
+        Returns:
+            List of judgment results
+        """
+        variables = variables or {}
+        
+        # Format system prompt with variables
+        formatted_prompt = self.format_system_prompt(variables)
+        
+        # Temporarily override instruction
+        original_instruction = self.instruction
+        self.instruction = formatted_prompt
+        
+        try:
+            # Handle different input types
+            if isinstance(input_data, dict):
+                processed_input = json.dumps(input_data)
+            else:
+                processed_input = input_data
+            results = self(processed_input, **runtime_kwargs)
+            
+            # Store for SFT if needed
+            self._store_sft_example(input_data, results, variables, formatted_prompt)
+            
+            return results
+        finally:
+            # Restore original instruction
+            self.instruction = original_instruction
+    
+    def _store_sft_example(
+        self,
+        input_data: Union[str, Dict[str, Any], BaseModel],
+        results: List[Dict[str, Any]],
+        variables: Dict[str, Any],
+        formatted_prompt: str
+    ) -> None:
+        """Store example for SFT export."""
+        for result in results:
+            # Create input text
+            if isinstance(input_data, str):
+                input_text = input_data
+            elif isinstance(input_data, BaseModel):
+                input_text = input_data.model_dump_json()
+            elif isinstance(input_data, dict):
+                input_text = json.dumps(input_data)
+            else:
+                input_text = str(input_data)
+            
+            # Extract output
+            output_text = result['parsed']
+            if isinstance(output_text, BaseModel):
+                output_text = output_text.model_dump_json()
+            elif not isinstance(output_text, str):
+                output_text = str(output_text)
+            
+            # Create conversation format
+            messages = get_conversation_one_turn(
+                formatted_prompt,
+                input_text,
+                output_text
+            )
+            
+            sft_example = {
+                'messages': messages,
+                'variables': variables,
+                'input_data': input_data,
+                'output': result['parsed']
+            }
+            
+            self.sft_data.append(sft_example)
+    
+    def export_sft_data(self, format: str = 'messages') -> List[Dict[str, Any]]:
+        """
+        Export stored examples in SFT format.
+        
+        Args:
+            format: Export format ('messages', 'full', or 'sharegpt')
+            
+        Returns:
+            List of SFT training examples
+        """
+        if format == 'messages':
+            return [{'messages': example['messages']} for example in self.sft_data]
+        elif format == 'full':
+            return self.sft_data
+        elif format == 'sharegpt':
+            # Convert to ShareGPT format
+            sharegpt_data = []
+            for example in self.sft_data:
+                conversations = []
+                for msg in example['messages']:
+                    conversations.append({
+                        'from': 'human' if msg['role'] == 'user' else 'gpt' if msg['role'] == 'assistant' else 'system',
+                        'value': msg['content']
+                    })
+                sharegpt_data.append({'conversations': conversations})
+            return sharegpt_data
+        else:
+            raise ValueError(f"Unsupported format: {format}. Choose from 'messages', 'full', or 'sharegpt'")
+    
+    def save_sft_data(self, filepath: str, format: str = 'messages') -> None:
+        """Save SFT data to file."""
+        sft_data = self.export_sft_data(format)
+        with open(filepath, 'w', encoding='utf-8') as f:
+            json.dump(sft_data, f, indent=2, ensure_ascii=False)
+    
+    def clear_sft_data(self) -> None:
+        """Clear stored SFT examples."""
+        self.sft_data.clear()
+
+
+class ChainOfThought:
+    """DSPy-like ChainOfThought wrapper for signatures."""
+    
+    def __init__(self, signature: Type[Signature], **llm_kwargs):
+        """
+        Initialize ChainOfThought with a signature.
+        
+        Args:
+            signature: Signature class defining input/output structure
+            **llm_kwargs: Arguments passed to LLMJudgeBase
+        """
+        self.signature = signature
+        
+        # Create system prompt from signature
+        system_prompt = signature.get_instruction()
+        
+        # Add reasoning instruction
+        system_prompt += "\n\nThink step by step before providing your final answer."
+        
+        self.llm = LLMJudgeBase(
+            system_prompt_template=system_prompt,
+            signature=signature,
+            **llm_kwargs
+        )
+    
+    def __call__(self, **kwargs) -> Any:
+        """Execute chain of thought reasoning."""
+        # Format input using signature
+        signature_instance = self.signature(**kwargs)
+        input_text = signature_instance.format_input(**kwargs)
+        
+        results = self.llm.judge(input_text)
+        
+        # Return the parsed output
+        if results:
+            return results[0]['parsed']
+        return None
+
+
+# Example usage classes based on the raw code
+class TranslationOutput(BaseModel):
+    """Output schema for translation evaluation."""
+    structure_score: int  # 0 = wrong, 1 = partially correct, 2 = correct
+    translation_score: int  # 0 = not faithful, 1 = somewhat faithful, 2 = fully faithful
+    term_score: int  # 0 = glossary not followed, 1 = partially followed, 2 = fully followed or no glossary provided
+
+
+class TranslationEvaluatorJudge(LLMJudgeBase):
+    """Translation evaluator judge based on the raw code example."""
+    
+    def __init__(self, **kwargs):
+        system_prompt = """You are a careful **translation evaluator**.
+
+You are given five inputs:
+
+* **Source Prompt** (the original text & any constraints)
+* **AI Translation** (the machine translation to evaluate)
+* **Human Reference** (a reference rendering; use only for guidance, not as ground truth)
+* **System Message** (an automated hint about a possible structural error)
+* **Glossaries** (optional terminology constraints; may be empty)
+
+## Your tasks
+
+1. **Check structure correctness**:
+   - Use the System Message as a hint.
+   - Assign a `structure_score`:
+     * `0` = structure is clearly wrong or the error flagged is correct.
+     * `1` = partially correct but flawed.
+     * `2` = structure is correct; the system error is invalid.
+
+2. **Check translation quality**:
+   - Compare AI Translation with Source Prompt and Human Reference.
+   - Assign a `translation_score`:
+     * `0` = unfaithful (major omissions/additions/distortions/repetitions).
+     * `1` = somewhat faithful (mostly correct but noticeable issues).
+     * `2` = faithful (preserves meaning, scope, nuance; only minor style differences).
+
+3. **Check glossary/terminology adherence**:
+   - If no glossary is provided → `term_score = 2`.
+   - If glossary exists but only partially followed → `term_score = 1`.
+   - If glossary exists but not followed at all → `term_score = 0`.
+
+## Output format (JSON only; no commentary)
+
+Return exactly one JSON object with the three scores.
+Do not output any explanations.
+
+---
+
+### Inputs
+
+Source Prompt: {SOURCE_PROMPT}
+
+AI Translation: {AI_TRANSLATION}
+
+Human Reference: {HUMAN_REFERENCE}
+
+System Message: {SYSTEM_MESSAGE}
+
+Glossaries: {GLOSSARIES}
+"""
+        
+        super().__init__(
+            system_prompt_template=system_prompt,
+            output_model=TranslationOutput,
+            **kwargs
+        )
+    
+    def evaluate_translation(
+        self,
+        source_prompt: str,
+        ai_translation: str,
+        human_reference: str,
+        system_message: str,
+        glossaries: str
+    ) -> TranslationOutput:
+        """
+        Evaluate a translation with all required parameters.
+        
+        Returns:
+            TranslationOutput with the three scores
+        """
+        variables = {
+            'SOURCE_PROMPT': source_prompt,
+            'AI_TRANSLATION': ai_translation,
+            'HUMAN_REFERENCE': human_reference,
+            'SYSTEM_MESSAGE': system_message,
+            'GLOSSARIES': glossaries
+        }
+        
+        input_data = {
+            'source': source_prompt,
+            'target': human_reference,
+            'glossaries': glossaries,
+            'translation': ai_translation
+        }
+        
+        results = self.judge(json.dumps(input_data), variables=variables)
+        return results[0]['parsed']
+
+
+# Example usage and testing
+if __name__ == "__main__":
+    # Test the Signature system
+    from .signature import Signature, InputField, OutputField
+    
+    # Example 1: Using Signature with ChainOfThought (like DSPy)
+    class FactJudge(Signature):
+        """Judge if the answer is factually correct based on the context."""
+        
+        context: str = InputField(desc="Context for the prediction")  # type: ignore
+        question: str = InputField(desc="Question to be answered")  # type: ignore 
+        answer: str = InputField(desc="Answer for the question")  # type: ignore
+        factually_correct: bool = OutputField(desc="Is the answer factually correct based on the context?")  # type: ignore
+    
+    print("=== Testing Signature System ===")
+    print("Instruction:")
+    print(FactJudge.get_instruction())
+    
+    # Example 2: Using LLMJudgeBase directly
+    judge_prompt = """You are a factual accuracy judge.
+    
+Given:
+- Context: {context}
+- Question: {question}  
+- Answer: {answer}
+
+Determine if the answer is factually correct based on the context.
+Respond with true if correct, false if incorrect."""
+    
+    print("\n=== Testing LLMJudgeBase ===")
+    print("System prompt template:")
+    print(judge_prompt)
+    
+    # Example 3: Translation evaluator from raw code
+    print("\n=== Translation Evaluator Example ===")
+    evaluator = TranslationEvaluatorJudge()
+    print("Translation evaluator initialized with structured output schema.")
+    print("Output schema:", TranslationOutput.model_json_schema())
+    
+    # Test SFT export functionality
+    print("\n=== SFT Export Test ===")
+    # Create a mock judge with some example data
+    mock_judge = LLMJudgeBase("Rate the quality: {text}")
+    mock_judge.sft_data = [
+        {
+            'messages': [
+                {'role': 'system', 'content': 'Rate the quality: This is good text'},
+                {'role': 'user', 'content': 'Please rate this text'},
+                {'role': 'assistant', 'content': '{"quality": "good"}'}
+            ],
+            'variables': {'text': 'This is good text'},
+            'input_data': 'Please rate this text',
+            'output': '{"quality": "good"}'
+        }
+    ]
+    
+    sft_formats = ['messages', 'sharegpt']
+    for format_name in sft_formats:
+        exported = mock_judge.export_sft_data(format_name)
+        print(f"SFT export ({format_name} format): {len(exported)} examples")
+        if exported:
+            print(f"Sample structure: {list(exported[0].keys())}")
+    
+    print("\n=== All Tests Completed ===")
+    print("The LLMJudgeBase system is ready for use!")
+    print("\nKey features:")
+    print("- System prompt templating with variables")
+    print("- DSPy-like Signature system")
+    print("- Automatic SFT data collection")
+    print("- Multiple export formats (messages, sharegpt, full)")
+    print("- Chain of Thought reasoning support")

llm_utils/lm/signature.py

@@ -0,0 +1,282 @@
+"""
+DSPy-like signature system for structured LLM interactions.
+
+This module provides a declarative way to define LLM input/output schemas
+with field descriptions and type annotations.
+"""
+
+from typing import Any, Dict, List, Type, Union, get_type_hints, Annotated, get_origin, get_args
+from pydantic import BaseModel, Field
+import inspect
+
+
+class InputField:
+    """Represents an input field in a signature."""
+    
+    def __init__(self, desc: str = "", **kwargs):
+        self.desc = desc
+        self.kwargs = kwargs
+    
+    def __class_getitem__(cls, item):
+        """Support for InputField[type] syntax."""
+        return item
+
+
+class OutputField:
+    """Represents an output field in a signature."""
+    
+    def __init__(self, desc: str = "", **kwargs):
+        self.desc = desc
+        self.kwargs = kwargs
+    
+    def __class_getitem__(cls, item):
+        """Support for OutputField[type] syntax."""
+        return item
+
+
+# Type aliases for cleaner syntax
+def Input(desc: str = "", **kwargs) -> Any:
+    """Create an input field descriptor."""
+    return InputField(desc=desc, **kwargs)
+
+
+def Output(desc: str = "", **kwargs) -> Any:
+    """Create an output field descriptor."""
+    return OutputField(desc=desc, **kwargs)
+
+
+class SignatureMeta(type):
+    """Metaclass for Signature that processes field annotations."""
+    
+    def __new__(cls, name, bases, namespace, **kwargs):
+        # Get type hints for this class
+        annotations = namespace.get('__annotations__', {})
+        
+        # Store field information
+        input_fields = {}
+        output_fields = {}
+        
+        for field_name, field_type in annotations.items():
+            field_value = namespace.get(field_name)
+            field_desc = None
+            
+            # Handle Annotated[Type, Field(...)] syntax using get_origin/get_args
+            if get_origin(field_type) is Annotated:
+                # Extract args from Annotated type
+                args = get_args(field_type)
+                if args:
+                    # First arg is the actual type
+                    field_type = args[0]
+                    # Look for InputField or OutputField in the metadata
+                    for metadata in args[1:]:
+                        if isinstance(metadata, (InputField, OutputField)):
+                            field_desc = metadata
+                            break
+            
+            # Handle old syntax with direct assignment
+            if field_desc is None and isinstance(field_value, (InputField, OutputField)):
+                field_desc = field_value
+            
+            # Store field information
+            if isinstance(field_desc, InputField):
+                input_fields[field_name] = {
+                    'type': field_type,
+                    'desc': field_desc.desc,
+                    **field_desc.kwargs
+                }
+            elif isinstance(field_desc, OutputField):
+                output_fields[field_name] = {
+                    'type': field_type,
+                    'desc': field_desc.desc,
+                    **field_desc.kwargs
+                }
+        
+        # Store in class attributes
+        namespace['_input_fields'] = input_fields
+        namespace['_output_fields'] = output_fields
+        
+        return super().__new__(cls, name, bases, namespace)
+
+
+class Signature(metaclass=SignatureMeta):
+    """Base class for defining LLM signatures with input and output fields."""
+    
+    _input_fields: Dict[str, Dict[str, Any]] = {}
+    _output_fields: Dict[str, Dict[str, Any]] = {}
+    
+    def __init__(self, **kwargs):
+        """Initialize signature with field values."""
+        for field_name, value in kwargs.items():
+            setattr(self, field_name, value)
+    
+    @classmethod
+    def get_instruction(cls) -> str:
+        """Generate instruction text from docstring and field descriptions."""
+        instruction = cls.__doc__ or "Complete the following task."
+        instruction = instruction.strip()
+        
+        # Add input field descriptions
+        if cls._input_fields:
+            instruction += "\n\n**Input Fields:**\n"
+            for field_name, field_info in cls._input_fields.items():
+                desc = field_info.get('desc', '')
+                field_type = field_info['type']
+                type_str = getattr(field_type, '__name__', str(field_type))
+                instruction += f"- {field_name} ({type_str}): {desc}\n"
+        
+        # Add output field descriptions
+        if cls._output_fields:
+            instruction += "\n**Output Fields:**\n"
+            for field_name, field_info in cls._output_fields.items():
+                desc = field_info.get('desc', '')
+                field_type = field_info['type']
+                type_str = getattr(field_type, '__name__', str(field_type))
+                instruction += f"- {field_name} ({type_str}): {desc}\n"
+        
+        return instruction
+    
+    @classmethod
+    def get_input_model(cls) -> Union[Type[BaseModel], type[str]]:
+        """Generate Pydantic input model from input fields."""
+        if not cls._input_fields:
+            return str
+        
+        fields = {}
+        annotations = {}
+        
+        for field_name, field_info in cls._input_fields.items():
+            field_type = field_info['type']
+            desc = field_info.get('desc', '')
+            
+            # Create Pydantic field
+            field_kwargs = {k: v for k, v in field_info.items() 
+                          if k not in ['type', 'desc']}
+            if desc:
+                field_kwargs['description'] = desc
+            
+            fields[field_name] = Field(**field_kwargs) if field_kwargs else Field()
+            annotations[field_name] = field_type
+        
+        # Create dynamic Pydantic model
+        input_model = type(
+            f"{cls.__name__}Input",
+            (BaseModel,),
+            {
+                '__annotations__': annotations,
+                **fields
+            }
+        )
+        
+        return input_model
+    
+    @classmethod
+    def get_output_model(cls) -> Union[Type[BaseModel], type[str]]:
+        """Generate Pydantic output model from output fields."""
+        if not cls._output_fields:
+            return str
+        
+        fields = {}
+        annotations = {}
+        
+        for field_name, field_info in cls._output_fields.items():
+            field_type = field_info['type']
+            desc = field_info.get('desc', '')
+            
+            # Create Pydantic field
+            field_kwargs = {k: v for k, v in field_info.items() 
+                          if k not in ['type', 'desc']}
+            if desc:
+                field_kwargs['description'] = desc
+                
+            fields[field_name] = Field(**field_kwargs) if field_kwargs else Field()
+            annotations[field_name] = field_type
+        
+        # Create dynamic Pydantic model
+        output_model = type(
+            f"{cls.__name__}Output",
+            (BaseModel,),
+            {
+                '__annotations__': annotations,
+                **fields
+            }
+        )
+        
+        return output_model
+    
+    def format_input(self, **kwargs) -> str:
+        """Format input fields as a string."""
+        input_data = {}
+        
+        # Collect input field values
+        for field_name in self._input_fields:
+            if field_name in kwargs:
+                input_data[field_name] = kwargs[field_name]
+            elif hasattr(self, field_name):
+                input_data[field_name] = getattr(self, field_name)
+        
+        # Format as key-value pairs
+        formatted_lines = []
+        for field_name, value in input_data.items():
+            field_info = self._input_fields[field_name]
+            desc = field_info.get('desc', '')
+            if desc:
+                formatted_lines.append(f"{field_name} ({desc}): {value}")
+            else:
+                formatted_lines.append(f"{field_name}: {value}")
+        
+        return '\n'.join(formatted_lines)
+
+
+# Export functions for easier importing
+__all__ = ['Signature', 'InputField', 'OutputField', 'Input', 'Output']
+
+
+# Example usage for testing
+if __name__ == "__main__":
+    # Define a signature like DSPy - using Annotated approach
+    class FactJudge(Signature):
+        """Judge if the answer is factually correct based on the context."""
+        
+        context: Annotated[str, Input("Context for the prediction")]
+        question: Annotated[str, Input("Question to be answered")]
+        answer: Annotated[str, Input("Answer for the question")]
+        factually_correct: Annotated[bool, Output("Is the answer factually correct based on the context?")]
+    
+    # Alternative syntax still works but will show type warnings
+    class FactJudgeOldSyntax(Signature):
+        """Judge if the answer is factually correct based on the context."""
+        
+        context: str = InputField(desc="Context for the prediction")  # type: ignore
+        question: str = InputField(desc="Question to be answered")  # type: ignore
+        answer: str = InputField(desc="Answer for the question")  # type: ignore
+        factually_correct: bool = OutputField(desc="Is the answer factually correct based on the context?")  # type: ignore
+    
+    # Test both signatures
+    for judge_class in [FactJudge, FactJudgeOldSyntax]:
+        print(f"\n=== Testing {judge_class.__name__} ===")
+        print("Instruction:")
+        print(judge_class.get_instruction())
+        
+        print("\nInput Model:")
+        input_model = judge_class.get_input_model()
+        if input_model is not str and hasattr(input_model, 'model_json_schema'):
+            print(input_model.model_json_schema())  # type: ignore
+        else:
+            print("String input model")
+        
+        print("\nOutput Model:")
+        output_model = judge_class.get_output_model()
+        if output_model is not str and hasattr(output_model, 'model_json_schema'):
+            print(output_model.model_json_schema())  # type: ignore
+        else:
+            print("String output model")
+        
+        # Test instance usage
+        judge = judge_class()
+        input_text = judge.format_input(
+            context="The sky is blue during daytime.",
+            question="What color is the sky?",
+            answer="Blue"
+        )
+        print("\nFormatted Input:")
+        print(input_text)

{speedy_utils-1.1.22.dist-info → speedy_utils-1.1.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: speedy-utils
-Version: 1.1.22
+Version: 1.1.23
 Summary: Fast and easy-to-use package for data science
 Project-URL: Homepage, https://github.com/anhvth/speedy
 Project-URL: Repository, https://github.com/anhvth/speedy

{speedy_utils-1.1.22.dist-info → speedy_utils-1.1.23.dist-info}/RECORD

@@ -1,15 +1,16 @@
-llm_utils/__init__.py,sha256=uhKDNctjJFbfzoxYjLvR0nRtl33QxHBsD1TCYvcoySg,1351
+llm_utils/__init__.py,sha256=gUmdXk6DF7dTKgYr23LdnkXLNT6x8bUZoacKQJ9pi8I,1625
 llm_utils/group_messages.py,sha256=Oe2tlhg-zRodG1-hodYebddrR77j9UdE05LzJw0EvYI,3622
 llm_utils/chat_format/__init__.py,sha256=8dBIUqFJvkgQYedxBtcyxt-4tt8JxAKVap2JlTXmgaM,737
 llm_utils/chat_format/display.py,sha256=3jKDm4OTrvytK1qBhSOjRLltUIObHsYFdBLgm8SVDE8,14159
 llm_utils/chat_format/transform.py,sha256=eU0c3PdAHCNLuGP1UqPwln0B34Lv3bt_uV9v9BrlCN4,5402
 llm_utils/chat_format/utils.py,sha256=xTxN4HrLHcRO2PfCTR43nH1M5zCa7v0kTTdzAcGkZg0,1229
-llm_utils/lm/__init__.py,sha256=totIZnq1P8eNlfVco0OfdGdTNt1-wSXDSRReRRzYYxw,319
+llm_utils/lm/__init__.py,sha256=znjUTzke2tmCRWkR46sbOQPcRNe5oEbLo5zqg6Vxud0,632
 llm_utils/lm/base_prompt_builder.py,sha256=OLqyxbA8QeYIVFzB9EqxUiE_P2p4_MD_Lq4WSwxFtKU,12136
+llm_utils/lm/llm_as_a_judge.py,sha256=LwqzlIMSBbpv6A2Qq8-fhVO2CGO7_BtU6j0PXoIWFOA,14022
 llm_utils/lm/llm_task.py,sha256=gawOtoP-LOH-8iaUI_2-TXvhFAVuj2fWGxeQZ1xytAo,25288
-llm_utils/lm/lm.py,sha256=8TaLuU7naPQbOFmiS2NQyWVLG0jUUzRRBQsR0In7GVo,7249
 llm_utils/lm/lm_base.py,sha256=pqbHZOdR7yUMpvwt8uBG1dZnt76SY_Wk8BkXQQ-mpWs,9557
 llm_utils/lm/openai_memoize.py,sha256=KToCcB_rhyrULxolnwMfQgl5GNrAeykePxuLS4hBjtc,3442
+llm_utils/lm/signature.py,sha256=s3Zjxjs6AU97jbz1LZ2BTGw-F9aFCF3G346gSMtyEpE,10370
 llm_utils/lm/utils.py,sha256=25oOznZhbBWfen1-X1PXQfO09kQZgP5V9CDuqLrf_ZU,12440
 llm_utils/lm/async_lm/__init__.py,sha256=PUBbCuf5u6-0GBUu-2PI6YAguzsyXj-LPkU6vccqT6E,121
 llm_utils/lm/async_lm/_utils.py,sha256=P1-pUDf_0pDmo8WTIi43t5ARlyGA1RIJfpAhz-gfA5g,6105
@@ -44,7 +45,7 @@ speedy_utils/multi_worker/thread.py,sha256=UniMl8mw-Xw1y3aU9bKGMtBSlQj05QhFouWb5
 speedy_utils/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 speedy_utils/scripts/mpython.py,sha256=IvywP7Y0_V6tWfMP-4MjPvN5_KfxWF21xaLJsCIayCk,3821
 speedy_utils/scripts/openapi_client_codegen.py,sha256=f2125S_q0PILgH5dyzoKRz7pIvNEjCkzpi4Q4pPFRZE,9683
-speedy_utils-1.1.22.dist-info/METADATA,sha256=3inYiDAe7IfOJrGHozb6NxtRCEzcgh1OY1de46PFF4I,8028
-speedy_utils-1.1.22.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-speedy_utils-1.1.22.dist-info/entry_points.txt,sha256=1rrFMfqvaMUE9hvwGiD6vnVh98kmgy0TARBj-v0Lfhs,244
-speedy_utils-1.1.22.dist-info/RECORD,,
+speedy_utils-1.1.23.dist-info/METADATA,sha256=xmQbqlIBS8fHw9ZZFG2kLftUiZX37cCt0hMmFc7putk,8028
+speedy_utils-1.1.23.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+speedy_utils-1.1.23.dist-info/entry_points.txt,sha256=1rrFMfqvaMUE9hvwGiD6vnVh98kmgy0TARBj-v0Lfhs,244
+speedy_utils-1.1.23.dist-info/RECORD,,

llm_utils/lm/lm.py

@@ -1,207 +0,0 @@
-# # from ._utils import *
-# from typing import (
-#     Any,
-#     List,
-#     Literal,
-#     Optional,
-#     Type,
-#     Union,
-#     cast,
-# )
-
-# from loguru import logger
-# from openai import AuthenticationError, BadRequestError, OpenAI, RateLimitError
-# from pydantic import BaseModel
-# from speedy_utils import jloads
-
-# # from llm_utils.lm.async_lm.async_llm_task import OutputModelType
-# from llm_utils.lm.lm_base import LMBase
-
-# from .async_lm._utils import (
-#     LegacyMsgs,
-#     Messages,
-#     OutputModelType,
-#     ParsedOutput,
-#     RawMsgs,
-# )
-
-
-# class LM(LMBase):
-#     """Unified **sync** language‑model wrapper with optional JSON parsing."""
-
-#     def __init__(
-#         self,
-#         *,
-#         model: Optional[str] = None,
-#         response_model: Optional[type[BaseModel]] = None,
-#         temperature: float = 0.0,
-#         max_tokens: int = 2_000,
-#         base_url: Optional[str] = None,
-#         api_key: Optional[str] = None,
-#         cache: bool = True,
-#         ports: Optional[List[int]] = None,
-#         top_p: float = 1.0,
-#         presence_penalty: float = 0.0,
-#         top_k: int = 1,
-#         repetition_penalty: float = 1.0,
-#         frequency_penalty: Optional[float] = None,
-#     ) -> None:
-
-#         if model is None:
-#             if base_url is None:
-#                 raise ValueError("Either model or base_url must be provided")
-#             models = OpenAI(base_url=base_url, api_key=api_key or 'abc').models.list().data
-#             assert len(models) == 1, f"Found {len(models)} models, please specify one."
-#             model = models[0].id
-#             print(f"Using model: {model}")
-
-#         super().__init__(
-#             ports=ports,
-#             base_url=base_url,
-#             cache=cache,
-#             api_key=api_key,
-#         )
-
-#         # Model behavior options
-#         self.response_model = response_model
-
-#         # Store all model-related parameters in model_kwargs
-#         self.model_kwargs = dict(
-#             model=model,
-#             temperature=temperature,
-#             max_tokens=max_tokens,
-#             top_p=top_p,
-#             presence_penalty=presence_penalty,
-#         )
-#         self.extra_body = dict(
-#             top_k=top_k,
-#             repetition_penalty=repetition_penalty,
-#             frequency_penalty=frequency_penalty,
-#         )
-
-#     def _unified_client_call(
-#         self,
-#         messages: RawMsgs,
-#         extra_body: Optional[dict] = None,
-#         max_tokens: Optional[int] = None,
-#     ) -> dict:
-#         """Unified method for all client interactions (caching handled by MOpenAI)."""
-#         converted_messages: Messages = (
-#             self._convert_messages(cast(LegacyMsgs, messages))
-#             if messages and isinstance(messages[0], dict)
-#             else cast(Messages, messages)
-#         )
-#         if max_tokens is not None:
-#             self.model_kwargs["max_tokens"] = max_tokens
-
-#         try:
-#             # Get completion from API (caching handled by MOpenAI)
-#             call_kwargs = {
-#                 "messages": converted_messages,
-#                 **self.model_kwargs,
-#             }
-#             if extra_body:
-#                 call_kwargs["extra_body"] = extra_body
-
-#             completion = self.client.chat.completions.create(**call_kwargs)
-
-#             if hasattr(completion, "model_dump"):
-#                 completion = completion.model_dump()
-
-#         except (AuthenticationError, RateLimitError, BadRequestError) as exc:
-#             error_msg = f"OpenAI API error ({type(exc).__name__}): {exc}"
-#             logger.error(error_msg)
-#             raise
-
-#         return completion
-
-#     def __call__(
-#         self,
-#         prompt: Optional[str] = None,
-#         messages: Optional[RawMsgs] = None,
-#         max_tokens: Optional[int] = None,
-#     ):  # -> tuple[Any | dict[Any, Any], list[ChatCompletionMessagePar...:# -> tuple[Any | dict[Any, Any], list[ChatCompletionMessagePar...:
-#         """Unified sync call for language model, returns (assistant_message.model_dump(), messages)."""
-#         if (prompt is None) == (messages is None):
-#             raise ValueError("Provide *either* `prompt` or `messages` (but not both).")
-
-#         if prompt is not None:
-#             messages = [{"role": "user", "content": prompt}]
-
-#         assert messages is not None
-
-#         openai_msgs: Messages = (
-#             self._convert_messages(cast(LegacyMsgs, messages))
-#             if isinstance(messages[0], dict)
-#             else cast(Messages, messages)
-#         )
-
-#         assert self.model_kwargs["model"] is not None, (
-#             "Model must be set before making a call."
-#         )
-
-#         # Use unified client call
-#         raw_response = self._unified_client_call(
-#             list(openai_msgs), max_tokens=max_tokens
-#         )
-
-#         if hasattr(raw_response, "model_dump"):
-#             raw_response = raw_response.model_dump()  # type: ignore
-
-#         # Extract the assistant's message
-#         assistant_msg = raw_response["choices"][0]["message"]
-#         # Build the full messages list (input + assistant reply)
-#         full_messages = list(messages) + [
-#             {"role": assistant_msg["role"], "content": assistant_msg["content"]}
-#         ]
-#         # Return the OpenAI message as model_dump (if available) and the messages list
-#         if hasattr(assistant_msg, "model_dump"):
-#             msg_dump = assistant_msg.model_dump()
-#         else:
-#             msg_dump = dict(assistant_msg)
-#         return msg_dump, full_messages
-
-#     def parse(
-#         self,
-#         messages: Messages,
-#         response_model: Optional[type[BaseModel]] = None,
-#     ) -> ParsedOutput[BaseModel]:
-#         """Parse response using OpenAI's native parse API."""
-#         # Use provided response_model or fall back to instance default
-#         model_to_use = response_model or self.response_model
-#         assert model_to_use is not None, "response_model must be provided or set at init."
-
-#         # Use OpenAI's native parse API directly
-#         response = self.client.chat.completions.parse(
-#             model=self.model_kwargs["model"],
-#             messages=messages,
-#             response_format=model_to_use,
-#             **{k: v for k, v in self.model_kwargs.items() if k != "model"}
-#         )
-        
-#         parsed = response.choices[0].message.parsed
-#         completion = response.model_dump() if hasattr(response, "model_dump") else {}
-#         full_messages = list(messages) + [
-#             {"role": "assistant", "content": parsed}
-#         ]
-
-#         return ParsedOutput(
-#             messages=full_messages,
-#             parsed=cast(BaseModel, parsed),
-#             completion=completion,
-#             model_kwargs=self.model_kwargs,
-#         )
-
-
-
-#     def __enter__(self):
-#         return self
-
-#     def __exit__(self, exc_type, exc_val, exc_tb):
-#         if hasattr(self, "_last_client"):
-#             last_client = self._last_client  # type: ignore
-#             if hasattr(last_client, "close"):
-#                 last_client.close()
-#         else:
-#             logger.warning("No last client to close")
-LM = None