speedy-utils 1.1.22__tar.gz → 1.1.23__tar.gz

{speedy_utils-1.1.22 → speedy_utils-1.1.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: speedy-utils
-Version: 1.1.22
+Version: 1.1.23
 Summary: Fast and easy-to-use package for data science
 Project-URL: Homepage, https://github.com/anhvth/speedy
 Project-URL: Repository, https://github.com/anhvth/speedy

speedy_utils-1.1.23/notebooks/llm_utils/llm_as_a_judge.ipynb

@@ -0,0 +1,642 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "136ff273",
+   "metadata": {},
+   "source": [
+    "# LLM-as-a-Judge Tutorial\n",
+    "\n",
+    "This notebook demonstrates how to use the LLM-as-a-Judge system with structured prompts, variable substitution, and SFT export capabilities."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "60c1bf59",
+   "metadata": {},
+   "source": [
+    "## Setup and Imports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "bb8f8e2b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "import os\n",
+    "# sys.path.append('../../src')  # Add src to path for imports\n",
+    "\n",
+    "from llm_utils import (\n",
+    "    LLMJudgeBase, \n",
+    "    ChainOfThought, \n",
+    "    TranslationEvaluatorJudge,\n",
+    "    Signature, \n",
+    "    InputField, \n",
+    "    OutputField\n",
+    ")\n",
+    "from pydantic import BaseModel\n",
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "eaceb8bd",
+   "metadata": {},
+   "source": [
+    "## Example 1: DSPy-like Signature System\n",
+    "\n",
+    "First, let's create a simple factual accuracy judge using the Signature system:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5b5e2123",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generated Instruction:\n",
+      "Judge if the answer is factually correct based on the context.\n",
+      "\n",
+      "**Input Fields:**\n",
+      "- context (str): Context for the prediction\n",
+      "- question (str): Question to be answered\n",
+      "- answer (str): Answer for the question\n",
+      "\n",
+      "**Output Fields:**\n",
+      "- factually_correct (bool): Is the answer factually correct based on the context?\n",
+      "\n",
+      "\n",
+      "==================================================\n",
+      "\n",
+      "Input Schema:\n",
+      "{\n",
+      "  \"properties\": {\n",
+      "    \"context\": {\n",
+      "      \"description\": \"Context for the prediction\",\n",
+      "      \"title\": \"Context\",\n",
+      "      \"type\": \"string\"\n",
+      "    },\n",
+      "    \"question\": {\n",
+      "      \"description\": \"Question to be answered\",\n",
+      "      \"title\": \"Question\",\n",
+      "      \"type\": \"string\"\n",
+      "    },\n",
+      "    \"answer\": {\n",
+      "      \"description\": \"Answer for the question\",\n",
+      "      \"title\": \"Answer\",\n",
+      "      \"type\": \"string\"\n",
+      "    }\n",
+      "  },\n",
+      "  \"required\": [\n",
+      "    \"context\",\n",
+      "    \"question\",\n",
+      "    \"answer\"\n",
+      "  ],\n",
+      "  \"title\": \"FactJudgeInput\",\n",
+      "  \"type\": \"object\"\n",
+      "}\n",
+      "\n",
+      "Output Schema:\n",
+      "{\n",
+      "  \"properties\": {\n",
+      "    \"factually_correct\": {\n",
+      "      \"description\": \"Is the answer factually correct based on the context?\",\n",
+      "      \"title\": \"Factually Correct\",\n",
+      "      \"type\": \"boolean\"\n",
+      "    }\n",
+      "  },\n",
+      "  \"required\": [\n",
+      "    \"factually_correct\"\n",
+      "  ],\n",
+      "  \"title\": \"FactJudgeOutput\",\n",
+      "  \"type\": \"object\"\n",
+      "}\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Define a signature like DSPy (original syntax - shows type warnings)\n",
+    "class FactJudge(Signature):\n",
+    "    \"\"\"Judge if the answer is factually correct based on the context.\"\"\"\n",
+    "    \n",
+    "    # Note: The assignments below will show type warnings, but work correctly\n",
+    "    context: str = InputField(desc=\"Context for the prediction\")  # type: ignore\n",
+    "    question: str = InputField(desc=\"Question to be answered\")  # type: ignore\n",
+    "    answer: str = InputField(desc=\"Answer for the question\")  \n",
+    "    factually_correct: bool = OutputField(desc=\"Is the answer factually correct based on the context?\")  # type: ignore\n",
+    "\n",
+    "# Show the generated instruction\n",
+    "print(\"Generated Instruction:\")\n",
+    "print(FactJudge.get_instruction())\n",
+    "print(\"\\n\" + \"=\"*50 + \"\\n\")\n",
+    "\n",
+    "# Show the input/output models\n",
+    "input_model = FactJudge.get_input_model()\n",
+    "output_model = FactJudge.get_output_model()\n",
+    "\n",
+    "if input_model is not str:\n",
+    "    print(\"Input Schema:\")\n",
+    "    print(json.dumps(input_model.model_json_schema(), indent=2))\n",
+    "\n",
+    "if output_model is not str:\n",
+    "    print(\"\\nOutput Schema:\")\n",
+    "    print(json.dumps(output_model.model_json_schema(), indent=2))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "13db83d4",
+   "metadata": {},
+   "source": [
+    "## Type-Safe Alternative Syntax\n",
+    "\n",
+    "The signature system now supports type-safe syntax using `typing.Annotated` to avoid type checker warnings:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "48d7aef5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Import the new type-safe helper functions\n",
+    "from typing import Annotated\n",
+    "from llm_utils import Input, Output\n",
+    "\n",
+    "# Type-safe syntax - no warnings!\n",
+    "class FactJudgeTypeSafe(Signature):\n",
+    "    \"\"\"Judge if the answer is factually correct based on the context.\"\"\"\n",
+    "    \n",
+    "    context: Annotated[str, Input(\"Context for the prediction\")]\n",
+    "    question: Annotated[str, Input(\"Question to be answered\")]\n",
+    "    answer: Annotated[str, Input(\"Answer for the question\")]\n",
+    "    factually_correct: Annotated[bool, Output(\"Is the answer factually correct?\")]\n",
+    "\n",
+    "print(\"Type-Safe Signature Instruction:\")\n",
+    "print(FactJudgeTypeSafe.get_instruction())\n",
+    "print(\"\\n\" + \"=\"*50 + \"\\n\")\n",
+    "\n",
+    "# Both approaches generate the same schemas\n",
+    "type_safe_input = FactJudgeTypeSafe.get_input_model()\n",
+    "type_safe_output = FactJudgeTypeSafe.get_output_model()\n",
+    "\n",
+    "if type_safe_input is not str:\n",
+    "    print(\"Type-Safe Input Schema:\")\n",
+    "    print(json.dumps(type_safe_input.model_json_schema(), indent=2))\n",
+    "\n",
+    "if type_safe_output is not str:\n",
+    "    print(\"\\nType-Safe Output Schema:\")\n",
+    "    print(json.dumps(type_safe_output.model_json_schema(), indent=2))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "795b1e2c",
+   "metadata": {},
+   "source": [
+    "## Example 2: Using ChainOfThought with Mock Client\n",
+    "\n",
+    "Note: In a real scenario, you would provide an actual OpenAI client or VLLM endpoint."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1a225fc4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Chain of Thought Usage Pattern:\n",
+      "\n",
+      "# With actual LLM client:\n",
+      "judge = ChainOfThought(FactJudge, client='http://localhost:8000/v1')\n",
+      "\n",
+      "# Execute judgment\n",
+      "result = judge(\n",
+      "    context=\"The sky is blue during daytime due to light scattering.\",\n",
+      "    question=\"What color is the sky?\",\n",
+      "    answer=\"Blue\"\n",
+      ")\n",
+      "\n",
+      "print(f\"Is factually correct: {result.factually_correct}\")\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "# For demonstration purposes, let's see how you would use ChainOfThought\n",
+    "# (This requires an actual LLM client to run)\n",
+    "\n",
+    "print(\"Chain of Thought Usage Pattern:\")\n",
+    "print(\"\"\"\n",
+    "# With actual LLM client:\n",
+    "judge = ChainOfThought(FactJudge, client='http://localhost:8000/v1')\n",
+    "\n",
+    "# Execute judgment\n",
+    "result = judge(\n",
+    "    context=\"The sky is blue during daytime due to light scattering.\",\n",
+    "    question=\"What color is the sky?\",\n",
+    "    answer=\"Blue\"\n",
+    ")\n",
+    "\n",
+    "print(f\"Is factually correct: {result.factually_correct}\")\n",
+    "\"\"\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8af947a4",
+   "metadata": {},
+   "source": [
+    "## Example 3: Custom Judge with Template Variables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "0ba925e6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Template Variables Required:\n",
+      "  - criteria\n",
+      "  - content\n",
+      "  - judge_type\n",
+      "  - categories\n",
+      "  - content_type\n",
+      "\n",
+      "Template Preview:\n",
+      "You are a {judge_type} evaluating {content_type}.\n",
+      "\n",
+      "Evaluation Criteria:\n",
+      "- {criteria}\n",
+      "\n",
+      "Rate the following on a scale of 1-10 and provide reasoning.\n",
+      "Also identify relevant categories from: {categories}\n",
+      "...\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Define a custom output model\n",
+    "class QualityScore(BaseModel):\n",
+    "    score: int  # 1-10 rating\n",
+    "    reasoning: str\n",
+    "    categories: list[str]\n",
+    "\n",
+    "# Create a judge with template variables\n",
+    "quality_prompt = \"\"\"\n",
+    "You are a {judge_type} evaluating {content_type}.\n",
+    "\n",
+    "Evaluation Criteria:\n",
+    "- {criteria}\n",
+    "\n",
+    "Rate the following on a scale of 1-10 and provide reasoning.\n",
+    "Also identify relevant categories from: {categories}\n",
+    "\n",
+    "Content to evaluate:\n",
+    "{content}\n",
+    "\"\"\".strip()\n",
+    "\n",
+    "# Show the template structure\n",
+    "print(\"Template Variables Required:\")\n",
+    "import re\n",
+    "variables = re.findall(r'\\{([^}]+)\\}', quality_prompt)\n",
+    "for var in set(variables):\n",
+    "    print(f\"  - {var}\")\n",
+    "\n",
+    "print(\"\\nTemplate Preview:\")\n",
+    "print(quality_prompt[:200] + \"...\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7f3518cf",
+   "metadata": {},
+   "source": [
+    "## Example 4: Translation Evaluator from Raw Code\n",
+    "\n",
+    "This demonstrates the TranslationEvaluatorJudge based on your raw code example:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "de4ef463",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create translation evaluator\n",
+    "evaluator = TranslationEvaluatorJudge()\n",
+    "\n",
+    "print(\"Translation Evaluator System Prompt:\")\n",
+    "print(evaluator.system_prompt_template[:300] + \"...\")\n",
+    "\n",
+    "print(\"\\nOutput Schema:\")\n",
+    "from llm_utils.lm.llm_as_a_judge import TranslationOutput\n",
+    "print(json.dumps(TranslationOutput.model_json_schema(), indent=2))\n",
+    "\n",
+    "print(\"\\nUsage Pattern:\")\n",
+    "print(\"\"\"\n",
+    "# With actual LLM client:\n",
+    "result = evaluator.evaluate_translation(\n",
+    "    source_prompt=\"Translate this to French: Hello world\",\n",
+    "    ai_translation=\"Bonjour le monde\",\n",
+    "    human_reference=\"Bonjour tout le monde\",\n",
+    "    system_message=\"NONE\",\n",
+    "    glossaries=\"\"\n",
+    ")\n",
+    "\n",
+    "print(f\"Structure Score: {result.structure_score}\")\n",
+    "print(f\"Translation Score: {result.translation_score}\")\n",
+    "print(f\"Term Score: {result.term_score}\")\n",
+    "\"\"\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "168a06d3",
+   "metadata": {},
+   "source": [
+    "## Example 5: SFT Data Collection and Export"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8fae83e4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create a mock judge for demonstration\n",
+    "mock_judge = LLMJudgeBase(\n",
+    "    system_prompt_template=\"Rate the sentiment of this text: {text}. Scale: {scale}\",\n",
+    "    output_model=str  # Simple string output for demo\n",
+    ")\n",
+    "\n",
+    "# Simulate some SFT data\n",
+    "mock_judge.sft_data = [\n",
+    "    {\n",
+    "        'messages': [\n",
+    "            {\n",
+    "                'role': 'system',\n",
+    "                'content': 'Rate the sentiment of this text: I love sunny days! Scale: 1-10'\n",
+    "            },\n",
+    "            {\n",
+    "                'role': 'user',\n",
+    "                'content': 'Please rate the sentiment'\n",
+    "            },\n",
+    "            {\n",
+    "                'role': 'assistant',\n",
+    "                'content': '9 - Very positive sentiment'\n",
+    "            }\n",
+    "        ],\n",
+    "        'variables': {'text': 'I love sunny days!', 'scale': '1-10'},\n",
+    "        'input_data': 'Please rate the sentiment',\n",
+    "        'output': '9 - Very positive sentiment'\n",
+    "    },\n",
+    "    {\n",
+    "        'messages': [\n",
+    "            {\n",
+    "                'role': 'system', \n",
+    "                'content': 'Rate the sentiment of this text: This is terrible. Scale: 1-10'\n",
+    "            },\n",
+    "            {\n",
+    "                'role': 'user',\n",
+    "                'content': 'Please rate the sentiment'\n",
+    "            },\n",
+    "            {\n",
+    "                'role': 'assistant',\n",
+    "                'content': '2 - Very negative sentiment'\n",
+    "            }\n",
+    "        ],\n",
+    "        'variables': {'text': 'This is terrible', 'scale': '1-10'},\n",
+    "        'input_data': 'Please rate the sentiment',\n",
+    "        'output': '2 - Very negative sentiment'\n",
+    "    }\n",
+    "]\n",
+    "\n",
+    "print(f\"Collected {len(mock_judge.sft_data)} training examples\")\n",
+    "\n",
+    "# Test different export formats\n",
+    "formats = ['messages', 'sharegpt', 'full']\n",
+    "\n",
+    "for format_name in formats:\n",
+    "    exported = mock_judge.export_sft_data(format_name)\n",
+    "    print(f\"\\n=== {format_name.upper()} Format ===\")\n",
+    "    print(f\"Exported {len(exported)} examples\")\n",
+    "    print(\"Sample structure:\", list(exported[0].keys()))\n",
+    "    \n",
+    "    if format_name == 'sharegpt':\n",
+    "        print(\"ShareGPT sample:\")\n",
+    "        print(json.dumps(exported[0], indent=2)[:300] + \"...\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "87fcd4d4",
+   "metadata": {},
+   "source": [
+    "## Example 6: Batch Processing Pattern"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8afba99b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Demonstrate how to use the judge system for batch processing\n",
+    "print(\"Batch Processing Pattern:\")\n",
+    "print(\"\"\"\n",
+    "# Example: Process multiple translations\n",
+    "evaluator = TranslationEvaluatorJudge(client='your-llm-endpoint')\n",
+    "\n",
+    "# Sample data\n",
+    "translations = [\n",
+    "    {\n",
+    "        'source': 'Hello world',\n",
+    "        'ai_translation': 'Bonjour le monde', \n",
+    "        'human_reference': 'Bonjour tout le monde',\n",
+    "        'system_message': 'NONE',\n",
+    "        'glossaries': ''\n",
+    "    },\n",
+    "    # ... more translations\n",
+    "]\n",
+    "\n",
+    "# Process in batch\n",
+    "results = []\n",
+    "for item in translations:\n",
+    "    result = evaluator.evaluate_translation(**item)\n",
+    "    results.append(result)\n",
+    "    \n",
+    "# Export all collected SFT data\n",
+    "evaluator.save_sft_data('translation_judge_training_data.json')\n",
+    "\n",
+    "# Analyze results\n",
+    "avg_structure = sum(r.structure_score for r in results) / len(results)\n",
+    "avg_translation = sum(r.translation_score for r in results) / len(results)\n",
+    "avg_term = sum(r.term_score for r in results) / len(results)\n",
+    "\n",
+    "print(f\"Average Scores:\")\n",
+    "print(f\"  Structure: {avg_structure:.2f}\")\n",
+    "print(f\"  Translation: {avg_translation:.2f}\")\n",
+    "print(f\"  Terms: {avg_term:.2f}\")\n",
+    "\"\"\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9ba497a2",
+   "metadata": {},
+   "source": [
+    "## Example 7: Creating Custom Judge Classes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1d2ba4f9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Example of creating a custom judge class\n",
+    "class CodeQualityJudge(LLMJudgeBase):\n",
+    "    \"\"\"Judge code quality with multiple criteria.\"\"\"\n",
+    "    \n",
+    "    def __init__(self, **kwargs):\n",
+    "        system_prompt = \"\"\"\n",
+    "You are an expert code reviewer evaluating {language} code.\n",
+    "\n",
+    "Criteria:\n",
+    "- Readability: How easy is it to understand?\n",
+    "- Performance: Are there obvious performance issues?\n",
+    "- Best Practices: Does it follow {language} best practices?\n",
+    "- Security: Are there security concerns?\n",
+    "\n",
+    "Code to review:\n",
+    "```{language}\n",
+    "{code}\n",
+    "```\n",
+    "\n",
+    "Additional context: {context}\n",
+    "\"\"\".strip()\n",
+    "        \n",
+    "        # Define output model\n",
+    "        class CodeReview(BaseModel):\n",
+    "            readability_score: int  # 1-10\n",
+    "            performance_score: int  # 1-10\n",
+    "            best_practices_score: int  # 1-10\n",
+    "            security_score: int  # 1-10\n",
+    "            overall_rating: str  # \"excellent\", \"good\", \"fair\", \"poor\"\n",
+    "            recommendations: list[str]\n",
+    "            \n",
+    "        super().__init__(\n",
+    "            system_prompt_template=system_prompt,\n",
+    "            output_model=CodeReview,\n",
+    "            **kwargs\n",
+    "        )\n",
+    "    \n",
+    "    def review_code(self, code: str, language: str = 'python', context: str = '') -> dict:\n",
+    "        \"\"\"Review code with structured output.\"\"\"\n",
+    "        variables = {\n",
+    "            'code': code,\n",
+    "            'language': language,\n",
+    "            'context': context\n",
+    "        }\n",
+    "        \n",
+    "        results = self.judge(f\"Please review this {language} code\", variables=variables)\n",
+    "        return results[0]['parsed']\n",
+    "\n",
+    "# Show the system prompt template\n",
+    "judge = CodeQualityJudge()\n",
+    "print(\"Code Quality Judge System Prompt Template:\")\n",
+    "print(judge.system_prompt_template)\n",
+    "\n",
+    "print(\"\\nUsage Example:\")\n",
+    "print(\"\"\"\n",
+    "# With actual LLM client:\n",
+    "code_judge = CodeQualityJudge(client='your-endpoint')\n",
+    "\n",
+    "result = code_judge.review_code(\n",
+    "    code=\"def add(a, b): return a + b\",\n",
+    "    language=\"python\",\n",
+    "    context=\"Simple utility function\"\n",
+    ")\n",
+    "\n",
+    "print(f\"Overall Rating: {result.overall_rating}\")\n",
+    "print(f\"Readability: {result.readability_score}/10\")\n",
+    "\"\"\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3cea50a9",
+   "metadata": {},
+   "source": [
+    "## Summary\n",
+    "\n",
+    "This notebook demonstrated the key features of the LLM-as-a-Judge system:\n",
+    "\n",
+    "1. **Signature System**: DSPy-like declarative interface for defining input/output schemas\n",
+    "2. **Template Variables**: System prompts with variable substitution\n",
+    "3. **SFT Export**: Automatic collection and export of training data\n",
+    "4. **Chain of Thought**: Built-in reasoning support\n",
+    "5. **Custom Judges**: Easy creation of domain-specific evaluation classes\n",
+    "6. **Multiple Formats**: Support for various export formats (messages, ShareGPT, etc.)\n",
+    "\n",
+    "### Next Steps:\n",
+    "\n",
+    "1. Set up your LLM endpoint (OpenAI API or VLLM server)\n",
+    "2. Create your own Signature classes for your specific use cases\n",
+    "3. Collect evaluation data and export for fine-tuning smaller models\n",
+    "4. Experiment with different prompt templates and evaluation criteria\n",
+    "\n",
+    "### Key Classes:\n",
+    "\n",
+    "- `Signature`: Define structured input/output schemas\n",
+    "- `LLMJudgeBase`: Core judge class with template support\n",
+    "- `ChainOfThought`: DSPy-like reasoning wrapper\n",
+    "- `TranslationEvaluatorJudge`: Ready-to-use translation evaluator\n",
+    "\n",
+    "The system is designed to be flexible and extensible for various evaluation tasks!"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "speedy_utils",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

{speedy_utils-1.1.22 → speedy_utils-1.1.23}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "speedy-utils"
-version = "1.1.22"
+version = "1.1.23"
 description = "Fast and easy-to-use package for data science"
 authors = [{ name = "AnhVTH", email = "anhvth.226@gmail.com" }]
 readme = "README.md"

{speedy_utils-1.1.22 → speedy_utils-1.1.23}/src/llm_utils/__init__.py

@@ -1,5 +1,5 @@
 from llm_utils.lm.openai_memoize import MOpenAI
-from llm_utils.lm import  LLMTask, AsyncLM, AsyncLLMTask
+from llm_utils.lm import  LLMTask, AsyncLM, AsyncLLMTask, LLMJudgeBase, ChainOfThought, TranslationEvaluatorJudge, Signature, InputField, OutputField, Input, Output
 from llm_utils.vector_cache import VectorCache
 from llm_utils.lm.lm_base import get_model_name
 from llm_utils.lm.base_prompt_builder import BasePromptBuilder
@@ -15,7 +15,7 @@ def kill_vllm_on_port(port: int) -> bool:
     """Kill VLLM server on specific port. Returns True if server was killed."""
     return LLMTask.kill_vllm_on_port(port)
 
-from .chat_format import (
+from llm_utils.chat_format import (
     build_chatml_input,
     display_chat_messages_as_html,
     display_conversations,
@@ -46,5 +46,13 @@ __all__ = [
     "BasePromptBuilder",
     "LLM",
     "kill_all_vllm",
-    "kill_vllm_on_port"
+    "kill_vllm_on_port",
+    "LLMJudgeBase",
+    "ChainOfThought",
+    "TranslationEvaluatorJudge",
+    "Signature",
+    "InputField",
+    "OutputField",
+    "Input",
+    "Output",
 ]