tesserakit-evals 0.4.0__tar.gz

tesserakit_evals-0.4.0/.gitignore

@@ -0,0 +1,8 @@
+.venv/
+__pycache__/
+*.pyc
+dist/
+build/
+*.egg-info/
+out/
+.DS_Store

tesserakit_evals-0.4.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.4
+Name: tesserakit-evals
+Version: 0.4.0
+Summary: Eval job pack for Tessera: compile messy data into eval-ready assets.
+Project-URL: Homepage, https://github.com/ShaileshRawat1403/tessera
+Project-URL: Repository, https://github.com/ShaileshRawat1403/tessera
+Project-URL: Issues, https://github.com/ShaileshRawat1403/tessera/issues
+Author: Shailesh Rawat
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Requires-Dist: pydantic>=2.7
+Requires-Dist: rich>=13.7
+Requires-Dist: tesserakit-core>=0.1.0
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# tessera-evals
+
+First Tessera job pack. It compiles messy CSV records into an eval pack with dataset, golden candidates, rubric, and quality reports.

tesserakit_evals-0.4.0/README.md

@@ -0,0 +1,3 @@
+# tessera-evals
+
+First Tessera job pack. It compiles messy CSV records into an eval pack with dataset, golden candidates, rubric, and quality reports.

tesserakit_evals-0.4.0/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["hatchling>=1.25"]
+build-backend = "hatchling.build"
+
+[project]
+name = "tesserakit-evals"
+version = "0.4.0"
+description = "Eval job pack for Tessera: compile messy data into eval-ready assets."
+readme = "README.md"
+requires-python = ">=3.10"
+authors = [{ name = "Shailesh Rawat" }]
+dependencies = [
+  "tesserakit-core>=0.1.0",
+  "typer>=0.12",
+  "rich>=13.7",
+  "pydantic>=2.7",
+]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Environment :: Console",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+]
+
+[project.urls]
+Homepage = "https://github.com/ShaileshRawat1403/tessera"
+Repository = "https://github.com/ShaileshRawat1403/tessera"
+Issues = "https://github.com/ShaileshRawat1403/tessera/issues"
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0"]
+
+[project.entry-points."tessera.commands"]
+evals = "tessera_evals.cli:register"
+
+[project.entry-points."tessera.jobpacks"]
+evals = "tessera_evals.pack:create_pack"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/tessera_evals"]

tesserakit_evals-0.4.0/src/tessera_evals/__init__.py

@@ -0,0 +1,3 @@
+"""Eval job pack for Tessera."""
+
+__version__ = "0.3.1"

tesserakit_evals-0.4.0/src/tessera_evals/adapters.py

@@ -0,0 +1,131 @@
+"""Export canonical eval records to framework-native interchange files.
+
+Tessera stays framework-independent: it emits each target's documented file
+format rather than importing the framework. A canonical ``EvalRecord`` maps to:
+
+  input    <- input.user_message
+  expected <- expected.reference_answer
+  context  <- context.source_text (optional)
+
+Targets:
+  deepeval      -> goldens JSON (list of {input, expected_output, context})
+  ragas         -> JSONL of {question, ground_truth, contexts}
+  openai-evals  -> JSONL of {input: [chat messages], ideal}
+  langsmith     -> JSONL of {inputs, outputs}
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Callable
+
+from tessera_evals.schema import EvalRecord
+
+TARGETS = ("deepeval", "ragas", "openai-evals", "langsmith")
+
+
+def _input_text(rec: EvalRecord) -> str:
+    return str(rec.input.get("user_message", "")).strip()
+
+
+def _expected_text(rec: EvalRecord) -> str:
+    return str(rec.expected.get("reference_answer", "")).strip()
+
+
+def _context_list(rec: EvalRecord) -> list[str]:
+    src = str(rec.context.get("source_text", "")).strip()
+    return [src] if src else []
+
+
+def to_deepeval(records: list[EvalRecord]) -> str:
+    """DeepEval goldens JSON: a list of golden objects."""
+    goldens = [
+        {
+            "input": _input_text(r),
+            "expected_output": _expected_text(r) or None,
+            "context": _context_list(r) or None,
+            "additional_metadata": {
+                "id": r.id,
+                "task_type": r.task_type,
+                "review_status": r.expected.get("review_status", ""),
+            },
+        }
+        for r in records
+    ]
+    return json.dumps({"goldens": goldens}, ensure_ascii=False, indent=2) + "\n"
+
+
+def to_ragas(records: list[EvalRecord]) -> str:
+    """RAGAS evaluation dataset as JSONL: question / ground_truth / contexts."""
+    lines = []
+    for r in records:
+        lines.append(
+            json.dumps(
+                {
+                    "question": _input_text(r),
+                    "ground_truth": _expected_text(r),
+                    "contexts": _context_list(r),
+                },
+                ensure_ascii=False,
+            )
+        )
+    return "\n".join(lines) + ("\n" if lines else "")
+
+
+def to_openai_evals(records: list[EvalRecord]) -> str:
+    """OpenAI Evals samples.jsonl: {input: [messages], ideal}."""
+    lines = []
+    for r in records:
+        sample: dict[str, Any] = {
+            "input": [{"role": "user", "content": _input_text(r)}],
+            "ideal": _expected_text(r),
+        }
+        lines.append(json.dumps(sample, ensure_ascii=False))
+    return "\n".join(lines) + ("\n" if lines else "")
+
+
+def to_langsmith(records: list[EvalRecord]) -> str:
+    """LangSmith dataset examples JSONL: {inputs, outputs}."""
+    lines = []
+    for r in records:
+        example = {
+            "inputs": {"input": _input_text(r)},
+            "outputs": {"expected": _expected_text(r)},
+            "metadata": {"id": r.id, "task_type": r.task_type},
+        }
+        lines.append(json.dumps(example, ensure_ascii=False))
+    return "\n".join(lines) + ("\n" if lines else "")
+
+
+_ADAPTERS: dict[str, tuple[Callable[[list[EvalRecord]], str], str]] = {
+    "deepeval": (to_deepeval, "deepeval_goldens.json"),
+    "ragas": (to_ragas, "ragas_dataset.jsonl"),
+    "openai-evals": (to_openai_evals, "openai_evals_samples.jsonl"),
+    "langsmith": (to_langsmith, "langsmith_examples.jsonl"),
+}
+
+
+def load_dataset(path: Path) -> list[EvalRecord]:
+    """Read a canonical dataset.jsonl (the evals pack's output) into EvalRecords."""
+    records: list[EvalRecord] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if line:
+            records.append(EvalRecord.model_validate_json(line))
+    return records
+
+
+def export(records: list[EvalRecord], target: str, output_dir: Path) -> Path:
+    """Write one target's interchange file; return its path."""
+    if target not in _ADAPTERS:
+        raise ValueError(f"unknown target '{target}'; choose from {', '.join(TARGETS)}")
+    render, filename = _ADAPTERS[target]
+    output_dir.mkdir(parents=True, exist_ok=True)
+    out = output_dir / filename
+    out.write_text(render(records), encoding="utf-8")
+    return out
+
+
+def export_all(records: list[EvalRecord], output_dir: Path) -> list[Path]:
+    return [export(records, t, output_dir) for t in TARGETS]

tesserakit_evals-0.4.0/src/tessera_evals/cli.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from tessera_core.models import RunContext
+
+from tessera_evals.pack import EvalsPack
+
+console = Console()
+evals_app = typer.Typer(help="Compile messy data into eval-ready assets.")
+
+
+@evals_app.command("compile")
+def compile_cmd(
+    input: Path = typer.Option(..., "--input", "-i", exists=True, readable=True, help="Input CSV path, or a prompts examples.jsonl / prompt-pack directory with --from-prompts."),
+    task: str = typer.Option(..., "--task", help="Task type, for example customer_support, rag_qa, classification."),
+    output: Path = typer.Option(Path("eval_pack"), "--output", "-o", help="Output directory."),
+    input_column: str | None = typer.Option(None, "--input-column", help="Override input/question column."),
+    expected_column: str | None = typer.Option(None, "--expected-column", help="Override expected/golden-answer column."),
+    context_column: str | None = typer.Option(None, "--context-column", help="Override context/source column."),
+    from_prompts: bool = typer.Option(False, "--from-prompts", help="Treat input as a prompts-pack examples.jsonl (or directory) instead of a CSV."),
+    enrich: bool = typer.Option(False, "--enrich", help="LLM-enriched rubric (not available in v0.1)."),
+) -> None:
+    """Create dataset, golden candidates, rubric, and quality reports from a CSV or a prompts examples.jsonl."""
+    if enrich:
+        console.print(
+            "[yellow]LLM enrichment is not available in v0.1. Using deterministic rubric templates.[/yellow]"
+        )
+
+    ctx = RunContext(job_name="evals", output_dir=output)
+    options = {
+        "task_type": task,
+        "input_column": input_column,
+        "expected_column": expected_column,
+        "context_column": context_column,
+    }
+    if from_prompts:
+        options["source"] = "prompts"
+
+    pack = EvalsPack()
+    artifacts = pack.run(input_path=input, ctx=ctx, options=options)
+
+    table = Table(title="Eval Pack Created")
+    table.add_column("Artifact")
+    table.add_column("Path")
+    table.add_column("Kind")
+    for art in artifacts:
+        table.add_row(art.name, str(art.path), art.kind)
+    console.print(table)
+
+    summary = Table(title="Run Summary")
+    summary.add_column("Metric")
+    summary.add_column("Value")
+    summary.add_row("run_id", ctx.run_id)
+    summary.add_row("records", str(ctx.metadata.get("record_count", 0)))
+    summary.add_row("findings", str(ctx.metadata.get("finding_count", 0)))
+    console.print(summary)
+
+
+@evals_app.command("export")
+def export_cmd(
+    input: Path = typer.Option(..., "--input", "-i", exists=True, readable=True, help="A canonical dataset.jsonl (the output of `evals compile`)."),
+    target: str = typer.Option("all", "--target", help="deepeval | ragas | openai-evals | langsmith | all."),
+    output: Path = typer.Option(Path("eval_export"), "--output", "-o", help="Output directory."),
+) -> None:
+    """Export a canonical eval dataset to framework-native interchange files."""
+    from tessera_evals.adapters import TARGETS, export, export_all, load_dataset
+
+    records = load_dataset(input)
+
+    target = target.lower()
+    if target == "all":
+        paths = export_all(records, output)
+    elif target in TARGETS:
+        paths = [export(records, target, output)]
+    else:
+        console.print(f"[red]Unknown target '{target}'. Choose from: {', '.join(TARGETS)}, all.[/red]")
+        raise typer.Exit(code=2)
+
+    table = Table(title="Eval Export")
+    table.add_column("Target file")
+    table.add_column("Records")
+    for p in paths:
+        table.add_row(str(p), str(len(records)))
+    console.print(table)
+
+
+def register(root_app: typer.Typer) -> None:
+    root_app.add_typer(evals_app, name="evals")