speedy-utils 1.1.23__py3-none-any.whl → 1.1.25__py3-none-any.whl

{speedy_utils-1.1.23.dist-info → speedy_utils-1.1.25.dist-info}/RECORD

@@ -1,17 +1,18 @@
-llm_utils/__init__.py,sha256=gUmdXk6DF7dTKgYr23LdnkXLNT6x8bUZoacKQJ9pi8I,1625
+llm_utils/__init__.py,sha256=pbnOQddU5KnhP8uqMqN9E87BeDeCxFrgta2m2P89LmM,1591
 llm_utils/group_messages.py,sha256=Oe2tlhg-zRodG1-hodYebddrR77j9UdE05LzJw0EvYI,3622
-llm_utils/chat_format/__init__.py,sha256=8dBIUqFJvkgQYedxBtcyxt-4tt8JxAKVap2JlTXmgaM,737
-llm_utils/chat_format/display.py,sha256=3jKDm4OTrvytK1qBhSOjRLltUIObHsYFdBLgm8SVDE8,14159
+llm_utils/chat_format/__init__.py,sha256=MCNT8o-BZWmoOFE5VLyhJJOqHg8lJGqHXEKSXU08fK0,775
+llm_utils/chat_format/display.py,sha256=HiAOAC8FY7956gNuwE7rxii1MCCebn0avbXi1iIcDSc,17178
 llm_utils/chat_format/transform.py,sha256=eU0c3PdAHCNLuGP1UqPwln0B34Lv3bt_uV9v9BrlCN4,5402
 llm_utils/chat_format/utils.py,sha256=xTxN4HrLHcRO2PfCTR43nH1M5zCa7v0kTTdzAcGkZg0,1229
-llm_utils/lm/__init__.py,sha256=znjUTzke2tmCRWkR46sbOQPcRNe5oEbLo5zqg6Vxud0,632
+llm_utils/lm/__init__.py,sha256=FBe8wVNWDMpvJ2kQYedJ3HH5L2BCAZBQVE0zEjND0Vo,729
 llm_utils/lm/base_prompt_builder.py,sha256=OLqyxbA8QeYIVFzB9EqxUiE_P2p4_MD_Lq4WSwxFtKU,12136
-llm_utils/lm/llm_as_a_judge.py,sha256=LwqzlIMSBbpv6A2Qq8-fhVO2CGO7_BtU6j0PXoIWFOA,14022
-llm_utils/lm/llm_task.py,sha256=gawOtoP-LOH-8iaUI_2-TXvhFAVuj2fWGxeQZ1xytAo,25288
+llm_utils/lm/llm.py,sha256=uk45JhVcWDMaqezn9Yn_K5hehFSmQ4txU901fn_PcQg,16262
+llm_utils/lm/llm_signature.py,sha256=SP72cWXaVGcZs3m2V361DcLk_St7aYJamNapUiFBB6Q,1242
 llm_utils/lm/lm_base.py,sha256=pqbHZOdR7yUMpvwt8uBG1dZnt76SY_Wk8BkXQQ-mpWs,9557
-llm_utils/lm/openai_memoize.py,sha256=KToCcB_rhyrULxolnwMfQgl5GNrAeykePxuLS4hBjtc,3442
-llm_utils/lm/signature.py,sha256=s3Zjxjs6AU97jbz1LZ2BTGw-F9aFCF3G346gSMtyEpE,10370
-llm_utils/lm/utils.py,sha256=25oOznZhbBWfen1-X1PXQfO09kQZgP5V9CDuqLrf_ZU,12440
+llm_utils/lm/mixins.py,sha256=Sn5KyPKGCT_HVJmmosmy3XSlZ0_k5Kds0VvSJqeUDpI,13695
+llm_utils/lm/openai_memoize.py,sha256=PDs3YCXKgHXaHlegkhouzPtf2Gom_o7pvzChCT-NQyQ,3870
+llm_utils/lm/signature.py,sha256=16QOHnGc-p7H8rR3j1dPg8AokdV_rEGUYCGGkIHIghE,10240
+llm_utils/lm/utils.py,sha256=oiJ50b8WV6oktnW4BByr1gRaGc55VJeF3IyhHqoofp4,12193
 llm_utils/lm/async_lm/__init__.py,sha256=PUBbCuf5u6-0GBUu-2PI6YAguzsyXj-LPkU6vccqT6E,121
 llm_utils/lm/async_lm/_utils.py,sha256=P1-pUDf_0pDmo8WTIi43t5ARlyGA1RIJfpAhz-gfA5g,6105
 llm_utils/lm/async_lm/async_llm_task.py,sha256=-BVOk18ZD8eC2obTLgiPq39f2PP3cji17Ku-Gb7c7Xo,18683
@@ -26,8 +27,8 @@ llm_utils/vector_cache/cli.py,sha256=DMXTj8nZ2_LRjprbYPb4uzq04qZtOfBbmblmaqDcCuM
 llm_utils/vector_cache/core.py,sha256=J8ocRX9sBfzboQkf5vFF2cx0SK-nftmKWJUa91WUBy8,31134
 llm_utils/vector_cache/types.py,sha256=ru8qmUZ8_lNd3_oYpjCMtpXTsqmwsSBe56Z4hTWm3xI,435
 llm_utils/vector_cache/utils.py,sha256=dwbbXlRrARrpmS4YqSlYQqrTURg0UWe8XvaAWcX05MM,1458
-speedy_utils/__init__.py,sha256=QBvGIbrC5yczQwh4T8iu9KQx6w9u-v_JdoQfA67hLUg,5780
-speedy_utils/all.py,sha256=t-HKzDmhF1MTFnmq7xRnPs5nFG_aZaLH9Ua0RM6nQ9Y,4855
+speedy_utils/__init__.py,sha256=wPz1MNAicV7skqqZloUFt5QrJcAhxtPQ4jFXk2lz6YA,6190
+speedy_utils/all.py,sha256=gXXRlBLvU8AON7XqO6iFQ8LCIQEIcP_2CDumd_U1ppI,5171
 speedy_utils/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 speedy_utils/common/clock.py,sha256=3n4FkCW0dz46O8By09V5Pve1DSMgpLDRbWEVRryryeQ,7423
 speedy_utils/common/function_decorator.py,sha256=BspJ0YuGL6elS7lWBAgELZ-sCfED_1N2P5fgH-fCRUQ,2132
@@ -35,17 +36,17 @@ speedy_utils/common/logger.py,sha256=a2iZx0eWyfi2-2X_H2QmfuA3tfR7_XSM7Nd0GdUnUOs
 speedy_utils/common/notebook_utils.py,sha256=-97kehJ_Gg3TzDLubsLIYJcykqX1NXhbvBO6nniZSYM,2063
 speedy_utils/common/patcher.py,sha256=VCmdxyTF87qroggQkQklRPhAOPJbeBqhcJoTsLcDxNw,2303
 speedy_utils/common/report_manager.py,sha256=eBiw5KY6bWUhwki3B4lK5o8bFsp7L5x28X9GCI-Sd1w,3899
-speedy_utils/common/utils_cache.py,sha256=NCwILnhsK86sDPkkriDTCyuM-qUKFxYOo1Piww1ED0g,22381
-speedy_utils/common/utils_io.py,sha256=-RkQjYGa3zVqpgVInsdp8dbS5oLwdJdUsRz1XIUSJzg,14257
-speedy_utils/common/utils_misc.py,sha256=cdEuBBpiB1xpuzj0UBDHDuTIerqsMIw37ENq6EXliOw,1795
+speedy_utils/common/utils_cache.py,sha256=h3JbIi0V5pTaFNJDjfwORSN63bc0SrRq_dm8KZJiL94,27023
+speedy_utils/common/utils_io.py,sha256=E7mbxB_OpLvNWoFM2Qpxi1jaD8VwF-tvNOpGbf7swuU,14849
+speedy_utils/common/utils_misc.py,sha256=yYlyP0eXQuapY1dn5O8-UDePPq5bb6FxKFjb1kfZy5o,2354
 speedy_utils/common/utils_print.py,sha256=syRrnSFtguxrV-elx6DDVcSGu4Qy7D_xVNZhPwbUY4A,4864
 speedy_utils/multi_worker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 speedy_utils/multi_worker/process.py,sha256=RGGGnbZXCbEbdmxFVmnNfyccClAlflzRPE0d1C3CeeE,11385
-speedy_utils/multi_worker/thread.py,sha256=UniMl8mw-Xw1y3aU9bKGMtBSlQj05QhFouWb5aVkATI,21679
+speedy_utils/multi_worker/thread.py,sha256=bRjxUHkBjbXHQ2KSsf-Zao28zbSId-8mqMFHwSG1l1s,25206
 speedy_utils/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 speedy_utils/scripts/mpython.py,sha256=IvywP7Y0_V6tWfMP-4MjPvN5_KfxWF21xaLJsCIayCk,3821
 speedy_utils/scripts/openapi_client_codegen.py,sha256=f2125S_q0PILgH5dyzoKRz7pIvNEjCkzpi4Q4pPFRZE,9683
-speedy_utils-1.1.23.dist-info/METADATA,sha256=xmQbqlIBS8fHw9ZZFG2kLftUiZX37cCt0hMmFc7putk,8028
-speedy_utils-1.1.23.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-speedy_utils-1.1.23.dist-info/entry_points.txt,sha256=1rrFMfqvaMUE9hvwGiD6vnVh98kmgy0TARBj-v0Lfhs,244
-speedy_utils-1.1.23.dist-info/RECORD,,
+speedy_utils-1.1.25.dist-info/METADATA,sha256=P99_Ej4mZJfouYjbGDsAjqodW4IJz7qBnvfR_V8RZ_k,8028
+speedy_utils-1.1.25.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+speedy_utils-1.1.25.dist-info/entry_points.txt,sha256=1rrFMfqvaMUE9hvwGiD6vnVh98kmgy0TARBj-v0Lfhs,244
+speedy_utils-1.1.25.dist-info/RECORD,,

llm_utils/lm/llm_as_a_judge.py

@@ -1,390 +0,0 @@
-"""
-LLM-as-a-Judge implementation with template support and SFT export utilities.
-
-This module provides a base class for creating LLM judges with structured
-prompts, variable substitution, and export capabilities for fine-tuning.
-"""
-
-import json
-from typing import Any, Dict, List, Optional, Type, Union
-from pydantic import BaseModel
-from ..chat_format import get_conversation_one_turn
-from .llm_task import LLMTask
-from .signature import Signature
-
-
-class LLMJudgeBase(LLMTask):
-    """Base class for LLM judges with template support and SFT export."""
-    
-    def __init__(
-        self,
-        system_prompt_template: str,
-        signature: Optional[Type[Signature]] = None,
-        **kwargs
-    ):
-        """
-        Initialize LLMJudgeBase.
-        
-        Args:
-            system_prompt_template: System prompt template with {variable} placeholders
-            signature: Optional Signature class for structured I/O
-            **kwargs: Additional arguments passed to LLMTask
-        """
-        self.system_prompt_template = system_prompt_template
-        self.signature = signature
-        self.sft_data: List[Dict[str, Any]] = []  # Store SFT training examples
-        
-        # Set instruction from signature if available
-        if signature is not None:
-            instruction = signature.get_instruction()
-            kwargs.setdefault('instruction', instruction)
-            kwargs.setdefault('output_model', signature.get_output_model())
-        else:
-            kwargs.setdefault('instruction', system_prompt_template)
-        
-        super().__init__(**kwargs)
-    
-    def format_system_prompt(self, variables: Dict[str, Any]) -> str:
-        """Format system prompt template with provided variables."""
-        try:
-            return self.system_prompt_template.format(**variables)
-        except KeyError as e:
-            missing_var = str(e).strip("'")
-            raise ValueError(f"Missing required variable '{missing_var}' for system prompt template")
-    
-    def judge(
-        self,
-        input_data: Union[str, Dict[str, Any], BaseModel],
-        variables: Optional[Dict[str, Any]] = None,
-        **runtime_kwargs
-    ) -> List[Dict[str, Any]]:
-        """
-        Execute judgment with variable substitution in system prompt.
-        
-        Args:
-            input_data: Input data for the judge
-            variables: Variables to substitute in system prompt template
-            **runtime_kwargs: Additional runtime arguments
-            
-        Returns:
-            List of judgment results
-        """
-        variables = variables or {}
-        
-        # Format system prompt with variables
-        formatted_prompt = self.format_system_prompt(variables)
-        
-        # Temporarily override instruction
-        original_instruction = self.instruction
-        self.instruction = formatted_prompt
-        
-        try:
-            # Handle different input types
-            if isinstance(input_data, dict):
-                processed_input = json.dumps(input_data)
-            else:
-                processed_input = input_data
-            results = self(processed_input, **runtime_kwargs)
-            
-            # Store for SFT if needed
-            self._store_sft_example(input_data, results, variables, formatted_prompt)
-            
-            return results
-        finally:
-            # Restore original instruction
-            self.instruction = original_instruction
-    
-    def _store_sft_example(
-        self,
-        input_data: Union[str, Dict[str, Any], BaseModel],
-        results: List[Dict[str, Any]],
-        variables: Dict[str, Any],
-        formatted_prompt: str
-    ) -> None:
-        """Store example for SFT export."""
-        for result in results:
-            # Create input text
-            if isinstance(input_data, str):
-                input_text = input_data
-            elif isinstance(input_data, BaseModel):
-                input_text = input_data.model_dump_json()
-            elif isinstance(input_data, dict):
-                input_text = json.dumps(input_data)
-            else:
-                input_text = str(input_data)
-            
-            # Extract output
-            output_text = result['parsed']
-            if isinstance(output_text, BaseModel):
-                output_text = output_text.model_dump_json()
-            elif not isinstance(output_text, str):
-                output_text = str(output_text)
-            
-            # Create conversation format
-            messages = get_conversation_one_turn(
-                formatted_prompt,
-                input_text,
-                output_text
-            )
-            
-            sft_example = {
-                'messages': messages,
-                'variables': variables,
-                'input_data': input_data,
-                'output': result['parsed']
-            }
-            
-            self.sft_data.append(sft_example)
-    
-    def export_sft_data(self, format: str = 'messages') -> List[Dict[str, Any]]:
-        """
-        Export stored examples in SFT format.
-        
-        Args:
-            format: Export format ('messages', 'full', or 'sharegpt')
-            
-        Returns:
-            List of SFT training examples
-        """
-        if format == 'messages':
-            return [{'messages': example['messages']} for example in self.sft_data]
-        elif format == 'full':
-            return self.sft_data
-        elif format == 'sharegpt':
-            # Convert to ShareGPT format
-            sharegpt_data = []
-            for example in self.sft_data:
-                conversations = []
-                for msg in example['messages']:
-                    conversations.append({
-                        'from': 'human' if msg['role'] == 'user' else 'gpt' if msg['role'] == 'assistant' else 'system',
-                        'value': msg['content']
-                    })
-                sharegpt_data.append({'conversations': conversations})
-            return sharegpt_data
-        else:
-            raise ValueError(f"Unsupported format: {format}. Choose from 'messages', 'full', or 'sharegpt'")
-    
-    def save_sft_data(self, filepath: str, format: str = 'messages') -> None:
-        """Save SFT data to file."""
-        sft_data = self.export_sft_data(format)
-        with open(filepath, 'w', encoding='utf-8') as f:
-            json.dump(sft_data, f, indent=2, ensure_ascii=False)
-    
-    def clear_sft_data(self) -> None:
-        """Clear stored SFT examples."""
-        self.sft_data.clear()
-
-
-class ChainOfThought:
-    """DSPy-like ChainOfThought wrapper for signatures."""
-    
-    def __init__(self, signature: Type[Signature], **llm_kwargs):
-        """
-        Initialize ChainOfThought with a signature.
-        
-        Args:
-            signature: Signature class defining input/output structure
-            **llm_kwargs: Arguments passed to LLMJudgeBase
-        """
-        self.signature = signature
-        
-        # Create system prompt from signature
-        system_prompt = signature.get_instruction()
-        
-        # Add reasoning instruction
-        system_prompt += "\n\nThink step by step before providing your final answer."
-        
-        self.llm = LLMJudgeBase(
-            system_prompt_template=system_prompt,
-            signature=signature,
-            **llm_kwargs
-        )
-    
-    def __call__(self, **kwargs) -> Any:
-        """Execute chain of thought reasoning."""
-        # Format input using signature
-        signature_instance = self.signature(**kwargs)
-        input_text = signature_instance.format_input(**kwargs)
-        
-        results = self.llm.judge(input_text)
-        
-        # Return the parsed output
-        if results:
-            return results[0]['parsed']
-        return None
-
-
-# Example usage classes based on the raw code
-class TranslationOutput(BaseModel):
-    """Output schema for translation evaluation."""
-    structure_score: int  # 0 = wrong, 1 = partially correct, 2 = correct
-    translation_score: int  # 0 = not faithful, 1 = somewhat faithful, 2 = fully faithful
-    term_score: int  # 0 = glossary not followed, 1 = partially followed, 2 = fully followed or no glossary provided
-
-
-class TranslationEvaluatorJudge(LLMJudgeBase):
-    """Translation evaluator judge based on the raw code example."""
-    
-    def __init__(self, **kwargs):
-        system_prompt = """You are a careful **translation evaluator**.
-
-You are given five inputs:
-
-* **Source Prompt** (the original text & any constraints)
-* **AI Translation** (the machine translation to evaluate)
-* **Human Reference** (a reference rendering; use only for guidance, not as ground truth)
-* **System Message** (an automated hint about a possible structural error)
-* **Glossaries** (optional terminology constraints; may be empty)
-
-## Your tasks
-
-1. **Check structure correctness**:
-   - Use the System Message as a hint.
-   - Assign a `structure_score`:
-     * `0` = structure is clearly wrong or the error flagged is correct.
-     * `1` = partially correct but flawed.
-     * `2` = structure is correct; the system error is invalid.
-
-2. **Check translation quality**:
-   - Compare AI Translation with Source Prompt and Human Reference.
-   - Assign a `translation_score`:
-     * `0` = unfaithful (major omissions/additions/distortions/repetitions).
-     * `1` = somewhat faithful (mostly correct but noticeable issues).
-     * `2` = faithful (preserves meaning, scope, nuance; only minor style differences).
-
-3. **Check glossary/terminology adherence**:
-   - If no glossary is provided → `term_score = 2`.
-   - If glossary exists but only partially followed → `term_score = 1`.
-   - If glossary exists but not followed at all → `term_score = 0`.
-
-## Output format (JSON only; no commentary)
-
-Return exactly one JSON object with the three scores.
-Do not output any explanations.
-
----
-
-### Inputs
-
-Source Prompt: {SOURCE_PROMPT}
-
-AI Translation: {AI_TRANSLATION}
-
-Human Reference: {HUMAN_REFERENCE}
-
-System Message: {SYSTEM_MESSAGE}
-
-Glossaries: {GLOSSARIES}
-"""
-        
-        super().__init__(
-            system_prompt_template=system_prompt,
-            output_model=TranslationOutput,
-            **kwargs
-        )
-    
-    def evaluate_translation(
-        self,
-        source_prompt: str,
-        ai_translation: str,
-        human_reference: str,
-        system_message: str,
-        glossaries: str
-    ) -> TranslationOutput:
-        """
-        Evaluate a translation with all required parameters.
-        
-        Returns:
-            TranslationOutput with the three scores
-        """
-        variables = {
-            'SOURCE_PROMPT': source_prompt,
-            'AI_TRANSLATION': ai_translation,
-            'HUMAN_REFERENCE': human_reference,
-            'SYSTEM_MESSAGE': system_message,
-            'GLOSSARIES': glossaries
-        }
-        
-        input_data = {
-            'source': source_prompt,
-            'target': human_reference,
-            'glossaries': glossaries,
-            'translation': ai_translation
-        }
-        
-        results = self.judge(json.dumps(input_data), variables=variables)
-        return results[0]['parsed']
-
-
-# Example usage and testing
-if __name__ == "__main__":
-    # Test the Signature system
-    from .signature import Signature, InputField, OutputField
-    
-    # Example 1: Using Signature with ChainOfThought (like DSPy)
-    class FactJudge(Signature):
-        """Judge if the answer is factually correct based on the context."""
-        
-        context: str = InputField(desc="Context for the prediction")  # type: ignore
-        question: str = InputField(desc="Question to be answered")  # type: ignore 
-        answer: str = InputField(desc="Answer for the question")  # type: ignore
-        factually_correct: bool = OutputField(desc="Is the answer factually correct based on the context?")  # type: ignore
-    
-    print("=== Testing Signature System ===")
-    print("Instruction:")
-    print(FactJudge.get_instruction())
-    
-    # Example 2: Using LLMJudgeBase directly
-    judge_prompt = """You are a factual accuracy judge.
-    
-Given:
-- Context: {context}
-- Question: {question}  
-- Answer: {answer}
-
-Determine if the answer is factually correct based on the context.
-Respond with true if correct, false if incorrect."""
-    
-    print("\n=== Testing LLMJudgeBase ===")
-    print("System prompt template:")
-    print(judge_prompt)
-    
-    # Example 3: Translation evaluator from raw code
-    print("\n=== Translation Evaluator Example ===")
-    evaluator = TranslationEvaluatorJudge()
-    print("Translation evaluator initialized with structured output schema.")
-    print("Output schema:", TranslationOutput.model_json_schema())
-    
-    # Test SFT export functionality
-    print("\n=== SFT Export Test ===")
-    # Create a mock judge with some example data
-    mock_judge = LLMJudgeBase("Rate the quality: {text}")
-    mock_judge.sft_data = [
-        {
-            'messages': [
-                {'role': 'system', 'content': 'Rate the quality: This is good text'},
-                {'role': 'user', 'content': 'Please rate this text'},
-                {'role': 'assistant', 'content': '{"quality": "good"}'}
-            ],
-            'variables': {'text': 'This is good text'},
-            'input_data': 'Please rate this text',
-            'output': '{"quality": "good"}'
-        }
-    ]
-    
-    sft_formats = ['messages', 'sharegpt']
-    for format_name in sft_formats:
-        exported = mock_judge.export_sft_data(format_name)
-        print(f"SFT export ({format_name} format): {len(exported)} examples")
-        if exported:
-            print(f"Sample structure: {list(exported[0].keys())}")
-    
-    print("\n=== All Tests Completed ===")
-    print("The LLMJudgeBase system is ready for use!")
-    print("\nKey features:")
-    print("- System prompt templating with variables")
-    print("- DSPy-like Signature system")
-    print("- Automatic SFT data collection")
-    print("- Multiple export formats (messages, sharegpt, full)")
-    print("- Chain of Thought reasoning support")