llm-evaluation-toolkit 0.1.0__py3-none-any.whl

llm_eval/__init__.py

@@ -0,0 +1,30 @@
+from llm_eval.evaluators import BaseEvaluator
+from llm_eval.metrics import (
+    BaseMetric,
+    BLEUMetric,
+    ROUGEMetric,
+    SemanticSimilarityMetric,
+    LLMJudgeMetric,
+)
+from llm_eval.providers import BaseProvider, GenerationConfig
+from llm_eval.providers.openai_provider import OpenAIProvider
+from llm_eval.providers.anthropic_provider import AnthropicProvider
+from llm_eval.types import EvalResult
+from llm_eval.datasets import DatasetLoader, EvalDataset
+
+__version__ = "0.1.0"
+__all__ = [
+    "BaseMetric",
+    "BaseEvaluator",
+    "EvalResult",
+    "BLEUMetric",
+    "ROUGEMetric",
+    "SemanticSimilarityMetric",
+    "LLMJudgeMetric",
+    "BaseProvider",
+    "GenerationConfig",
+    "OpenAIProvider",
+    "AnthropicProvider",
+    "DatasetLoader",
+    "EvalDataset",
+]

llm_eval/datasets.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass
+class EvalDataset:
+    """評価用データセットを格納するクラス"""
+    name: str
+    questions: list[str]
+    references: list[str]
+    metadata: dict = None
+
+    def __post_init__(self):
+        if self.metadata is None:
+            self.metadata = {}
+        if len(self.questions) != len(self.references):
+            raise ValueError(
+                f"questionsとreferencesの数が一致しません: "
+                f"{len(self.questions)} != {len(self.references)}"
+            )
+
+    def __len__(self) -> int:
+        return len(self.questions)
+
+    def __repr__(self) -> str:
+        return f"EvalDataset(name={self.name}, size={len(self)})"
+
+    def subset(self, n: int) -> "EvalDataset":
+        """先頭n件だけ取り出す"""
+        return EvalDataset(
+            name=f"{self.name}(subset={n})",
+            questions=self.questions[:n],
+            references=self.references[:n],
+            metadata=self.metadata,
+        )
+
+
+class DatasetLoader:
+    """HuggingFace datasetsからベンチマークを読み込むクラス"""
+
+    @staticmethod
+    def _import_datasets():
+        try:
+            import datasets
+            return datasets
+        except ImportError:
+            raise ImportError(
+                "datasetsパッケージが必要です: pip install datasets"
+            )
+
+    @classmethod
+    def load_squad(cls, split: str = "validation", max_samples: int = 100) -> EvalDataset:
+        """
+        SQuAD (Stanford Question Answering Dataset) を読み込む
+
+        質問応答タスクの定番ベンチマーク。
+        questions: 質問文
+        references: 正解テキスト
+        """
+        ds = cls._import_datasets()
+        dataset = ds.load_dataset("squad", split=split)
+
+        questions = []
+        references = []
+
+        for item in dataset.select(range(min(max_samples, len(dataset)))):
+            questions.append(item["question"])
+            references.append(item["answers"]["text"][0])
+
+        return EvalDataset(
+            name="squad",
+            questions=questions,
+            references=references,
+            metadata={"split": split, "max_samples": max_samples},
+        )
+
+    @classmethod
+    def load_cnn_dailymail(cls, split: str = "validation", max_samples: int = 50) -> EvalDataset:
+        """
+        CNN/DailyMail を読み込む
+
+        要約タスクの定番ベンチマーク。
+        questions: 元記事
+        references: 正解要約
+        """
+        ds = cls._import_datasets()
+        dataset = ds.load_dataset("cnn_dailymail", "3.0.0", split=split)
+
+        questions = []
+        references = []
+
+        for item in dataset.select(range(min(max_samples, len(dataset)))):
+            questions.append(item["article"][:512])
+            references.append(item["highlights"])
+
+        return EvalDataset(
+            name="cnn_dailymail",
+            questions=questions,
+            references=references,
+            metadata={"split": split, "max_samples": max_samples},
+        )
+
+    @classmethod
+    def from_dict(
+        cls,
+        name: str,
+        questions: list[str],
+        references: list[str],
+    ) -> EvalDataset:
+        """
+        自前のデータからEvalDatasetを作る
+
+        APIキーなしでテストしたいときに便利。
+        """
+        return EvalDataset(
+            name=name,
+            questions=questions,
+            references=references,
+        )

llm_eval/evaluators/__init__.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from abc import ABC
+
+from llm_eval.metrics import BaseMetric
+from llm_eval.types import EvalResult
+
+
+class BaseEvaluator(ABC):
+    """複数のMetricをまとめて実行する基底クラス"""
+
+    def __init__(self, metrics: list[BaseMetric]):
+        self.metrics = metrics
+
+    def evaluate(
+        self,
+        predictions: list[str],
+        references: list[str],
+    ) -> dict[str, EvalResult]:
+        results = {}
+        for metric in self.metrics:
+            results[metric.name] = metric.compute(predictions, references)
+        return results
+
+    def add_metric(self, metric: BaseMetric) -> None:
+        self.metrics.append(metric)
+
+    def __repr__(self) -> str:
+        metric_names = [m.name for m in self.metrics]
+        return f"{self.__class__.__name__}(metrics={metric_names})"

llm_eval/metrics/__init__.py

@@ -0,0 +1,13 @@
+from llm_eval.metrics.base import BaseMetric
+from llm_eval.metrics.bleu import BLEUMetric
+from llm_eval.metrics.rouge import ROUGEMetric
+from llm_eval.metrics.semantic import SemanticSimilarityMetric
+from llm_eval.metrics.judge import LLMJudgeMetric
+
+__all__ = [
+    "BaseMetric",
+    "BLEUMetric",
+    "ROUGEMetric",
+    "SemanticSimilarityMetric",
+    "LLMJudgeMetric",
+]

llm_eval/metrics/base.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from llm_eval.types import EvalResult
+
+
+class BaseMetric(ABC):
+    """すべての評価指標の基底クラス"""
+
+    def __init__(self, name: str):
+        self.name = name
+
+    @abstractmethod
+    def compute(
+        self,
+        predictions: list[str],
+        references: list[str],
+    ) -> EvalResult:
+        ...
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(name={self.name})"

llm_eval/metrics/bleu.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import nltk
+from nltk.translate.bleu_score import corpus_bleu, SmoothingFunction
+
+from llm_eval.metrics.base import BaseMetric
+from llm_eval.types import EvalResult
+
+
+class BLEUMetric(BaseMetric):
+    """
+    BLEU (Bilingual Evaluation Understudy) スコアを計算する
+
+    機械翻訳や要約タスクでLLM出力と正解文の一致度を測る。
+    スコアは0.0〜1.0で、1.0が完全一致。
+    """
+
+    def __init__(self, max_n: int = 4):
+        super().__init__(name="bleu")
+        self.max_n = max_n
+        self._ensure_nltk_data()
+
+    def _ensure_nltk_data(self) -> None:
+        try:
+            nltk.data.find("tokenizers/punkt")
+        except LookupError:
+            nltk.download("punkt", quiet=True)
+        try:
+            nltk.data.find("tokenizers/punkt_tab")
+        except LookupError:
+            nltk.download("punkt_tab", quiet=True)
+
+    def compute(
+        self,
+        predictions: list[str],
+        references: list[str],
+    ) -> EvalResult:
+        if len(predictions) != len(references):
+            raise ValueError(
+                f"predictionsとreferencesの数が一致しません: "
+                f"{len(predictions)} != {len(references)}"
+            )
+
+        tokenized_predictions = [p.lower().split() for p in predictions]
+        tokenized_references = [[r.lower().split()] for r in references]
+
+        smoothing = SmoothingFunction().method1
+        score = corpus_bleu(
+            tokenized_references,
+            tokenized_predictions,
+            smoothing_function=smoothing,
+        )
+
+        return EvalResult(
+            metric_name=self.name,
+            score=float(score),
+            details={
+                "num_samples": len(predictions),
+                "max_n": self.max_n,
+            },
+        )

llm_eval/metrics/judge.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import os
+import re
+
+from llm_eval.metrics.base import BaseMetric
+from llm_eval.types import EvalResult
+
+
+JUDGE_PROMPT_TEMPLATE = """You are an expert evaluator for AI-generated text.
+
+Evaluate the following response based on these criteria:
+- Accuracy: Is the information correct?
+- Clarity: Is the response clear and easy to understand?
+- Completeness: Does it fully address the question?
+- Relevance: Is it relevant to the question asked?
+
+Question: {question}
+Response: {response}
+
+Provide your evaluation in the following format:
+Score: [0-10]
+Reasoning: [brief explanation]
+
+Be strict but fair. A score of 10 means perfect."""
+
+
+class LLMJudgeMetric(BaseMetric):
+    """
+    LLM自身が審判として出力を採点するメトリクス
+
+    正解テキストなしで出力の品質を評価できる。
+    スコアは0.0〜1.0に正規化される（元スコアは0〜10）。
+    """
+
+    def __init__(
+        self,
+        judge_model: str = "gpt-4o-mini",
+        api_key: str | None = None,
+        prompt_template: str | None = None,
+    ):
+        super().__init__(name="llm_judge")
+        self.judge_model = judge_model
+        self.api_key = api_key or os.environ.get("OPENAI_API_KEY")
+        self.prompt_template = prompt_template or JUDGE_PROMPT_TEMPLATE
+        self._client = None
+
+    def _get_client(self):
+        """OpenAIクライアントの遅延初期化"""
+        if self._client is None:
+            try:
+                from openai import OpenAI
+            except ImportError:
+                raise ImportError(
+                    "openaiパッケージが必要です: "
+                    "pip install llm-evaluation-toolkit[openai]"
+                )
+            self._client = OpenAI(api_key=self.api_key)
+        return self._client
+
+    def _parse_score(self, judge_response: str) -> float:
+        """LLMの採点レスポンスからスコアを抽出する"""
+        pattern = r"Score:\s*(\d+(?:\.\d+)?)"
+        match = re.search(pattern, judge_response, re.IGNORECASE)
+        if match:
+            raw_score = float(match.group(1))
+            return min(max(raw_score / 10.0, 0.0), 1.0)
+        return 0.5
+
+    def _judge_single(self, question: str, response: str) -> tuple[float, str]:
+        """1つのレスポンスを採点する"""
+        client = self._get_client()
+        prompt = self.prompt_template.format(
+            question=question,
+            response=response,
+        )
+        result = client.chat.completions.create(
+            model=self.judge_model,
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=256,
+            temperature=0.0,
+        )
+        judge_response = result.choices[0].message.content or ""
+        score = self._parse_score(judge_response)
+        return score, judge_response
+
+    def compute(
+        self,
+        predictions: list[str],
+        references: list[str],
+    ) -> EvalResult:
+        """
+        Args:
+            predictions: LLMの出力テキストのリスト
+            references:  対応する質問文のリスト（このMetricでは正解ではなく質問を渡す）
+        """
+        if len(predictions) != len(references):
+            raise ValueError(
+                f"predictionsとreferencesの数が一致しません: "
+                f"{len(predictions)} != {len(references)}"
+            )
+
+        scores = []
+        judge_responses = []
+
+        for question, response in zip(references, predictions):
+            score, judge_response = self._judge_single(question, response)
+            scores.append(score)
+            judge_responses.append(judge_response)
+            print(f"採点完了: {score:.2f} / 1.0")
+
+        avg_score = sum(scores) / len(scores)
+
+        return EvalResult(
+            metric_name=self.name,
+            score=avg_score,
+            details={
+                "num_samples": len(predictions),
+                "judge_model": self.judge_model,
+                "individual_scores": scores,
+                "judge_responses": judge_responses,
+            },
+        )

llm_eval/metrics/rouge.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from rouge_score import rouge_scorer
+
+from llm_eval.metrics.base import BaseMetric
+from llm_eval.types import EvalResult
+
+
+class ROUGEMetric(BaseMetric):
+    """
+    ROUGE (Recall-Oriented Understudy for Gisting Evaluation) スコアを計算する
+
+    要約タスクでよく使われる。
+    ROUGE-1: 単語の一致率
+    ROUGE-2: 2単語フレーズの一致率
+    ROUGE-L: 最長共通部分列ベースの一致率
+    """
+
+    def __init__(self, rouge_types: list[str] | None = None):
+        super().__init__(name="rouge")
+        self.rouge_types = rouge_types or ["rouge1", "rouge2", "rougeL"]
+        self._scorer = rouge_scorer.RougeScorer(
+            self.rouge_types,
+            use_stemmer=True,
+        )
+
+    def compute(
+        self,
+        predictions: list[str],
+        references: list[str],
+    ) -> EvalResult:
+        if len(predictions) != len(references):
+            raise ValueError(
+                f"predictionsとreferencesの数が一致しません: "
+                f"{len(predictions)} != {len(references)}"
+            )
+
+        aggregated: dict[str, list[float]] = {t: [] for t in self.rouge_types}
+
+        for pred, ref in zip(predictions, references):
+            scores = self._scorer.score(ref, pred)
+            for rouge_type in self.rouge_types:
+                aggregated[rouge_type].append(scores[rouge_type].fmeasure)
+
+        details = {}
+        for rouge_type, values in aggregated.items():
+            details[rouge_type] = sum(values) / len(values)
+
+        primary_score = details.get("rougeL", details[self.rouge_types[0]])
+
+        return EvalResult(
+            metric_name=self.name,
+            score=primary_score,
+            details=details,
+        )

llm_eval/metrics/semantic.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import numpy as np
+
+from llm_eval.metrics.base import BaseMetric
+from llm_eval.types import EvalResult
+
+
+class SemanticSimilarityMetric(BaseMetric):
+    """
+    埋め込みベクトルのコサイン類似度で意味的な近さを評価する
+
+    BLEUやROUGEでは捉えられない言い換えや同義語を考慮できる。
+    スコアは0.0〜1.0で、1.0が意味的に完全一致。
+    """
+
+    def __init__(self, model_name: str = "all-MiniLM-L6-v2"):
+        super().__init__(name="semantic_similarity")
+        self.model_name = model_name
+        self._model = None
+
+    def _load_model(self):
+        """モデルの遅延読み込み（初回呼び出し時にダウンロード）"""
+        if self._model is None:
+            try:
+                from sentence_transformers import SentenceTransformer
+            except ImportError:
+                raise ImportError(
+                    "sentence-transformersが必要です: "
+                    "pip install llm-evaluation-toolkit[semantic]"
+                )
+            print(f"モデルを読み込み中: {self.model_name}")
+            self._model = SentenceTransformer(self.model_name)
+        return self._model
+
+    def _cosine_similarity(self, a: np.ndarray, b: np.ndarray) -> float:
+        """2つのベクトルのコサイン類似度を計算する"""
+        dot_product = np.dot(a, b)
+        norm_a = np.linalg.norm(a)
+        norm_b = np.linalg.norm(b)
+        if norm_a == 0 or norm_b == 0:
+            return 0.0
+        return float(dot_product / (norm_a * norm_b))
+
+    def compute(
+        self,
+        predictions: list[str],
+        references: list[str],
+    ) -> EvalResult:
+        if len(predictions) != len(references):
+            raise ValueError(
+                f"predictionsとreferencesの数が一致しません: "
+                f"{len(predictions)} != {len(references)}"
+            )
+
+        model = self._load_model()
+
+        pred_embeddings = model.encode(predictions, convert_to_numpy=True)
+        ref_embeddings = model.encode(references, convert_to_numpy=True)
+
+        similarities = [
+            self._cosine_similarity(pred_embeddings[i], ref_embeddings[i])
+            for i in range(len(predictions))
+        ]
+
+        avg_score = float(np.mean(similarities))
+
+        return EvalResult(
+            metric_name=self.name,
+            score=avg_score,
+            details={
+                "num_samples": len(predictions),
+                "model": self.model_name,
+                "individual_scores": similarities,
+                "min_score": float(np.min(similarities)),
+                "max_score": float(np.max(similarities)),
+            },
+        )

llm_eval/providers/__init__.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+@dataclass
+class GenerationConfig:
+    """テキスト生成の設定"""
+    max_tokens: int = 512
+    temperature: float = 0.0
+    system_prompt: str = "You are a helpful assistant."
+
+
+class BaseProvider(ABC):
+    """すべてのLLMプロバイダの基底クラス"""
+
+    def __init__(self, model: str, config: GenerationConfig | None = None):
+        self.model = model
+        self.config = config or GenerationConfig()
+
+    @abstractmethod
+    def generate(self, prompt: str) -> str:
+        ...
+
+    def generate_batch(self, prompts: list[str]) -> list[str]:
+        return [self.generate(p) for p in prompts]
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(model={self.model})"
+
+
+from llm_eval.providers.openai_provider import OpenAIProvider  # noqa: E402
+from llm_eval.providers.anthropic_provider import AnthropicProvider  # noqa: E402
+
+__all__ = ["BaseProvider", "GenerationConfig", "OpenAIProvider", "AnthropicProvider"]

llm_eval/providers/anthropic_provider.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import os
+
+from llm_eval.providers import BaseProvider, GenerationConfig
+
+
+class AnthropicProvider(BaseProvider):
+    """Anthropic API プロバイダ"""
+
+    def __init__(
+        self,
+        model: str = "claude-haiku-4-5-20251001",
+        config: GenerationConfig | None = None,
+        api_key: str | None = None,
+    ):
+        super().__init__(model=model, config=config)
+        try:
+            import anthropic
+        except ImportError:
+            raise ImportError(
+                "anthropicパッケージが必要です: pip install llm-evaluation-toolkit[anthropic]"
+            )
+        self._client = anthropic.Anthropic(
+            api_key=api_key or os.environ.get("ANTHROPIC_API_KEY")
+        )
+
+    def generate(self, prompt: str) -> str:
+        response = self._client.messages.create(
+            model=self.model,
+            max_tokens=self.config.max_tokens,
+            system=self.config.system_prompt,
+            messages=[
+                {"role": "user", "content": prompt},
+            ],
+        )
+        return response.content[0].text

llm_eval/providers/openai_provider.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import os
+
+from llm_eval.providers import BaseProvider, GenerationConfig
+
+
+class OpenAIProvider(BaseProvider):
+    """OpenAI API プロバイダ"""
+
+    def __init__(
+        self,
+        model: str = "gpt-4o-mini",
+        config: GenerationConfig | None = None,
+        api_key: str | None = None,
+    ):
+        super().__init__(model=model, config=config)
+        try:
+            from openai import OpenAI
+        except ImportError:
+            raise ImportError(
+                "openaiパッケージが必要です: pip install llm-evaluation-toolkit[openai]"
+            )
+        self._client = OpenAI(
+            api_key=api_key or os.environ.get("OPENAI_API_KEY")
+        )
+
+    def generate(self, prompt: str) -> str:
+        response = self._client.chat.completions.create(
+            model=self.model,
+            messages=[
+                {"role": "system", "content": self.config.system_prompt},
+                {"role": "user", "content": prompt},
+            ],
+            max_tokens=self.config.max_tokens,
+            temperature=self.config.temperature,
+        )
+        return response.choices[0].message.content or ""

llm_eval/types.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class EvalResult:
+    """評価結果を格納するデータクラス"""
+
+    metric_name: str
+    score: float
+    details: dict[str, Any] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def __repr__(self) -> str:
+        return f"EvalResult(metric={self.metric_name}, score={self.score:.4f})"

llm_evaluation_toolkit-0.1.0.dist-info/METADATA

@@ -0,0 +1,279 @@
+Metadata-Version: 2.4
+Name: llm-evaluation-toolkit
+Version: 0.1.0
+Summary: A lightweight toolkit for evaluating LLM outputs
+Project-URL: Homepage, https://github.com/swoswoyuu1156/llm-evaluation-toolkit
+Project-URL: Repository, https://github.com/swoswoyuu1156/llm-evaluation-toolkit
+Project-URL: Issues, https://github.com/swoswoyuu1156/llm-evaluation-toolkit/issues
+Author-email: swoswoyuu1156 <174082729+swoswoyuu1156@users.noreply.github.com>
+License: MIT License
+        
+        Copyright (c) 2026 swoswoyuu1156
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: ai,evaluation,llm,machine-learning,nlp
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Requires-Dist: datasets>=2.14
+Requires-Dist: nltk>=3.8
+Requires-Dist: numpy>=1.24
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: requests>=2.28
+Requires-Dist: rouge-score>=0.1.2
+Requires-Dist: sentence-transformers>=2.2
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.20; extra == 'anthropic'
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.0; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0; extra == 'openai'
+Provides-Extra: semantic
+Requires-Dist: sentence-transformers>=2.2; extra == 'semantic'
+Description-Content-Type: text/markdown
+
+# llm-evaluation-toolkit
+
+[![CI](https://github.com/swoswoyuu1156/llm-evaluation-toolkit/actions/workflows/ci.yml/badge.svg)](https://github.com/swoswoyuu1156/llm-evaluation-toolkit/actions/workflows/ci.yml)
+[![Python](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![PyPI version](https://badge.fury.io/py/llm-evaluation-toolkit.svg)](https://badge.fury.io/py/llm-evaluation-toolkit)
+
+LLMの出力を評価するための軽量Pythonライブラリです。BLEU・ROUGE・意味的類似度・LLM-as-a-Judgeの4種類の評価指標を、統一されたAPIで利用できます。
+
+*A lightweight Python toolkit for evaluating LLM outputs. Supports BLEU, ROUGE, semantic similarity, and LLM-as-a-Judge — all with a unified API.*
+
+---
+
+## なぜ llm-evaluation-toolkit なのか？ / Why llm-evaluation-toolkit?
+
+既存の評価ライブラリは研究用途に特化していたり、セットアップが複雑すぎる問題がありました。このライブラリは、実際の開発ワークフローでLLMの出力を素早く評価したい開発者のために設計されています。
+
+*Existing evaluation libraries are either too research-focused or require complex setup. This toolkit is designed for developers who need to evaluate LLM outputs quickly in real workflows.*
+
+- **統一API** — すべての評価指標が同じインターフェースを持つ / *Unified API — all metrics share the same interface*
+- **複数プロバイダ対応** — OpenAI・Anthropicをすぐに利用可能 / *Multiple providers — OpenAI and Anthropic out of the box*
+- **正解テキスト不要** — LLM-as-a-Judgeは参照テキストなしで評価可能 / *No reference needed — LLM-as-a-Judge works without ground truth*
+- **軽量設計** — 必要な機能だけインストールできる / *Lightweight — install only what you need*
+
+---
+
+## インストール / Installation
+
+```bash
+# 基本インストール / Base installation
+pip install llm-evaluation-toolkit
+
+# OpenAI対応 / With OpenAI support
+pip install llm-evaluation-toolkit[openai]
+
+# Anthropic対応 / With Anthropic support
+pip install llm-evaluation-toolkit[anthropic]
+
+# 意味的類似度対応 / With semantic similarity support
+pip install llm-evaluation-toolkit[semantic]
+
+# 全機能 / All features
+pip install llm-evaluation-toolkit[openai,anthropic,semantic]
+```
+
+---
+
+## クイックスタート / Quick Start
+
+### 基本的な評価 / Basic evaluation
+
+```python
+from llm_eval import BLEUMetric, ROUGEMetric, SemanticSimilarityMetric
+
+predictions = [
+    "The cat is on the mat",
+    "A dog was running in the park",
+]
+references = [
+    "A cat is sitting on a mat",
+    "The dog ran through the park",
+]
+
+bleu = BLEUMetric()
+rouge = ROUGEMetric()
+semantic = SemanticSimilarityMetric()
+
+print(bleu.compute(predictions, references))
+# EvalResult(metric=bleu, score=0.4231)
+
+print(rouge.compute(predictions, references))
+# EvalResult(metric=rouge, score=0.6842)
+
+print(semantic.compute(predictions, references))
+# EvalResult(metric=semantic_similarity, score=0.8923)
+```
+
+### 正解テキストなしで評価（LLM-as-a-Judge） / Evaluate without reference texts
+
+```python
+from llm_eval import LLMJudgeMetric
+
+# 正解テキストなしで出力品質を評価できる
+# Evaluate output quality without any reference text
+judge = LLMJudgeMetric(judge_model="gpt-4o-mini")
+
+questions = ["日本の首都はどこですか？"]
+answers = ["日本の首都は東京です。政治・経済・文化の中心地として機能しています。"]
+
+result = judge.compute(answers, questions)
+print(result)
+# EvalResult(metric=llm_judge, score=0.9)
+```
+
+### 複数の評価指標を一括実行 / Run multiple metrics at once
+
+```python
+from llm_eval import BaseEvaluator, BLEUMetric, ROUGEMetric, SemanticSimilarityMetric
+
+evaluator = BaseEvaluator(metrics=[
+    BLEUMetric(),
+    ROUGEMetric(),
+    SemanticSimilarityMetric(),
+])
+
+results = evaluator.evaluate(predictions, references)
+for metric_name, result in results.items():
+    print(f"{metric_name}: {result.score:.4f}")
+```
+
+### LLMプロバイダとの連携 / Use with LLM providers
+
+```python
+from llm_eval import OpenAIProvider, GenerationConfig, BLEUMetric
+
+# LLMで回答を生成してそのまま評価する
+# Generate predictions from LLM and evaluate directly
+provider = OpenAIProvider(
+    model="gpt-4o-mini",
+    config=GenerationConfig(temperature=0.0, max_tokens=256),
+)
+
+questions = ["機械学習とは何ですか？", "ニューラルネットワークを説明してください。"]
+predictions = provider.generate_batch(questions)
+
+references = [
+    "機械学習はデータから学習するAIの一分野です。",
+    "ニューラルネットワークは人間の脳を模倣した計算システムです。",
+]
+
+result = BLEUMetric().compute(predictions, references)
+print(result)
+```
+
+### ベンチマークデータセットの利用 / Load benchmark datasets
+
+```python
+from llm_eval import DatasetLoader
+
+# 組み込みベンチマークを使う / Use built-in benchmarks
+squad = DatasetLoader.load_squad(max_samples=50)
+print(squad)
+# EvalDataset(name=squad, size=50)
+
+# 自前データを使う / Use your own data
+dataset = DatasetLoader.from_dict(
+    name="my_dataset",
+    questions=["質問1", "質問2"],
+    references=["回答1", "回答2"],
+)
+```
+
+---
+
+## 対応評価指標 / Supported Metrics
+
+| 指標 / Metric | 正解テキスト / Reference | 適したタスク / Best for |
+|--------------|------------------------|------------------------|
+| BLEU | 必要 / Yes | 翻訳・固定フォーマット生成 / Translation, fixed-format generation |
+| ROUGE | 必要 / Yes | 要約 / Summarization |
+| Semantic Similarity | 必要 / Yes | 言い換えを含むタスク / Paraphrase-heavy tasks |
+| LLM-as-a-Judge | 不要 / No | 自由記述生成 / Open-ended generation |
+
+---
+
+## 対応プロバイダ / Supported Providers
+
+| プロバイダ / Provider | インストール / Install | モデル例 / Models |
+|----------------------|----------------------|------------------|
+| OpenAI | `pip install llm-evaluation-toolkit[openai]` | gpt-4o, gpt-4o-mini |
+| Anthropic | `pip install llm-evaluation-toolkit[anthropic]` | claude-opus-4-6, claude-haiku-4-5 |
+
+---
+
+## 環境変数 / Environment Variables
+
+```bash
+OPENAI_API_KEY=your_openai_api_key
+ANTHROPIC_API_KEY=your_anthropic_api_key
+```
+
+---
+
+## 開発環境のセットアップ / Development Setup
+
+```bash
+git clone https://github.com/swoswoyuu1156/llm-evaluation-toolkit.git
+cd llm-evaluation-toolkit
+python -m venv .venv
+
+# Mac/Linux
+source .venv/bin/activate
+# Windows
+.venv\Scripts\activate
+
+pip install -e ".[dev]"
+
+# テスト実行 / Run tests
+pytest tests/ -v --cov=src/llm_eval
+
+# リント実行 / Run linter
+ruff check src/ tests/
+```
+
+---
+
+## コントリビューション / Contributing
+
+コントリビューションを歓迎します！まず [CONTRIBUTING.md](CONTRIBUTING.md) をご確認ください。
+
+*Contributions are welcome! Please read [CONTRIBUTING.md](CONTRIBUTING.md) first.*
+
+---
+
+## ライセンス / License
+
+MIT License — 詳細は [LICENSE](LICENSE) をご確認ください。
+
+*MIT License — see [LICENSE](LICENSE) for details.*

llm_evaluation_toolkit-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+llm_eval/__init__.py,sha256=WZGF2gsReDmuW8tTozYzRP2CsW32wxVSoh9FXGpEHrE,780
+llm_eval/datasets.py,sha256=stNm2vvSZf9y5sz6qNYxGXowAe458levjSy35Qg4gOc,3528
+llm_eval/types.py,sha256=J1nFoWtPsi1fwRmxF_UpIKHzXOkYcNlW1-mybLa30FI,418
+llm_eval/evaluators/__init__.py,sha256=P6Ayk7a3DkB8YPSBm1v2IvWJQszqNwIVbxJ4xxSH0Eg,841
+llm_eval/metrics/__init__.py,sha256=k57U6Y1JBDf6-uS5tgTL6W0ggEAotMh4VbYumJ6-PyM,373
+llm_eval/metrics/base.py,sha256=Z4bFvy9n6LpG78-UXUaL4SpO8x73q95TotLecEIVuKs,491
+llm_eval/metrics/bleu.py,sha256=X7YqDYZcS2lHESuIhb8FX-r9K3hHZ3zDQNn-uJQvIDs,1826
+llm_eval/metrics/judge.py,sha256=edwJxALUNtwu0LABS3F-QeM0MQNpUF-Y8phLnEFKPm0,4115
+llm_eval/metrics/rouge.py,sha256=CEt-bAvKQTAXdxlU1z76bxt9wePlKDSluyaUZ05EQ-o,1762
+llm_eval/metrics/semantic.py,sha256=FtqrWYQkr_juOezlOs-p4aYvaKuf6Z4CHZkZt5PDxjI,2729
+llm_eval/providers/__init__.py,sha256=O_ommltS7L6GyI__ff8DDDVCXhWY5kxDp2eI4aAu42o,1067
+llm_eval/providers/anthropic_provider.py,sha256=Qhh0bgmzr3-kTG0PVr4POuBWTfGbF17ch_oloDh4esQ,1115
+llm_eval/providers/openai_provider.py,sha256=XYhUed104Vfvrlo1vHqXJbi_pZG6buzXqdN3bqgpQuc,1184
+llm_evaluation_toolkit-0.1.0.dist-info/METADATA,sha256=06QyqOU_lJsq-9EJAQESmcRCjjO2GaMiiDD6AIFUHbg,10275
+llm_evaluation_toolkit-0.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+llm_evaluation_toolkit-0.1.0.dist-info/licenses/LICENSE,sha256=o38RZRbQzW-aKIgfjXQ3tNCZoSDUNvY3-OC15qE6baE,1070
+llm_evaluation_toolkit-0.1.0.dist-info/RECORD,,

llm_evaluation_toolkit-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

llm_evaluation_toolkit-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 swoswoyuu1156
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.