paperlab-space 0.1.0__py3-none-any.whl

paperlab_space/__init__.py

@@ -0,0 +1,51 @@
+"""paperlab-space: evaluate LLMs on spreadsheet data and produce paper-ready results.
+
+>>> from paperlab_space import RunConfig, load_dataset, run, export
+>>> data = load_dataset("data.xlsx")                          # doctest: +SKIP
+>>> config = RunConfig.from_yaml("run.yaml")                  # doctest: +SKIP
+>>> results = run(config, data)                               # doctest: +SKIP
+>>> export.write_all(results, "results/")                     # doctest: +SKIP
+"""
+
+__version__ = "0.1.0"
+
+from paperlab_space import export
+from paperlab_space.config import DatasetSpec, MetricSpec, ModelSpec, RetrySettings, RunConfig
+from paperlab_space.data import ColumnMapping, Dataset, MappedDataset, list_sheets, load_dataset
+from paperlab_space.errors import ConfigError, DataError, PaperLabError, TemplateError
+from paperlab_space.metrics import available_metrics, register, score
+from paperlab_space.prompt import PromptTemplate
+from paperlab_space.results import MetricResult, ModelRun, RowOutput, RunResults
+from paperlab_space.runner import LiteLLMBackend, Progress, arun, estimate, run
+
+__all__ = [
+    "ColumnMapping",
+    "ConfigError",
+    "DataError",
+    "Dataset",
+    "DatasetSpec",
+    "LiteLLMBackend",
+    "MappedDataset",
+    "MetricResult",
+    "MetricSpec",
+    "ModelRun",
+    "ModelSpec",
+    "PaperLabError",
+    "Progress",
+    "PromptTemplate",
+    "RetrySettings",
+    "RowOutput",
+    "RunConfig",
+    "RunResults",
+    "TemplateError",
+    "__version__",
+    "arun",
+    "available_metrics",
+    "estimate",
+    "export",
+    "list_sheets",
+    "load_dataset",
+    "register",
+    "run",
+    "score",
+]

paperlab_space/_cli.py

@@ -0,0 +1,230 @@
+"""The ``paperlab-space`` command (needs the extra: ``pip install 'paperlab-space[cli]'``)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+import yaml
+from rich.console import Console
+from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn, TimeRemainingColumn
+from rich.table import Table
+
+from paperlab_space import __version__
+from paperlab_space.config import RunConfig
+from paperlab_space.data import ColumnMapping, list_sheets, load_dataset
+from paperlab_space.errors import PaperLabError
+from paperlab_space.export import write_all
+from paperlab_space.metrics import available_metrics
+from paperlab_space.results import RunResults
+from paperlab_space.runner import Progress as RunProgress
+from paperlab_space.runner import estimate, prepare_data, run
+
+app = typer.Typer(
+    help="Evaluate LLMs on spreadsheet data and get paper-ready results.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+# emoji=False: model IDs like `x:free:` must not turn into emoji.
+console = Console(emoji=False)
+err = Console(stderr=True, emoji=False)
+
+_EXPECTED_HINTS = ("expected", "gold", "answer", "label", "reference", "target", "output", "ref")
+_ID_HINTS = ("id", "row_id", "index", "no", "no.", "#")
+
+
+def _fail(exc: Exception) -> typer.Exit:
+    err.print(f"[bold red]Error:[/] {exc}")
+    return typer.Exit(1)
+
+
+@app.callback(invoke_without_command=True)
+def _main(
+    version: Annotated[bool, typer.Option("--version", help="Show the version and exit.")] = False,
+) -> None:
+    if version:
+        console.print(f"paperlab-space {__version__}")
+        raise typer.Exit()
+
+
+@app.command()
+def init(
+    data: Annotated[Path, typer.Argument(help="Excel or CSV file to evaluate on.")],
+    output: Annotated[Path, typer.Option("--output", "-o", help="Config file to write.")] = Path(
+        "run.yaml"
+    ),
+    sheet: Annotated[str | None, typer.Option(help="Sheet name for Excel files.")] = None,
+    force: Annotated[bool, typer.Option(help="Overwrite an existing config.")] = False,
+) -> None:
+    """Write a starter run config for DATA, guessing the column roles."""
+    try:
+        dataset = load_dataset(data, sheet)
+    except PaperLabError as exc:
+        raise _fail(exc) from exc
+    if output.exists() and not force:
+        raise _fail(FileExistsError(f"{output} exists; use --force to overwrite."))
+
+    columns = dataset.columns
+    lower = {c.lower(): c for c in columns}
+    id_col = next((lower[h] for h in _ID_HINTS if h in lower), None)
+    expected = next(
+        (c for c in columns if any(h in c.lower() for h in _EXPECTED_HINTS) and c != id_col), None
+    )
+    inputs = [c for c in columns if c not in (id_col, expected)][:1] or columns[:1]
+
+    try:
+        relative = data.resolve().relative_to(output.resolve().parent)
+    except ValueError:
+        relative = data.resolve()
+    config: dict[str, object] = {
+        "dataset": {"path": str(relative), "sheet": dataset.sheet},
+        "mapping": {"input": inputs, "context": [], "expected": expected, "id": id_col},
+        "prompt": f"{{{inputs[0]}}}",
+        "models": ["openrouter/nvidia/nemotron-3-super-120b-a12b:free", "gemini/gemini-2.5-flash"],
+        "metrics": ["exact_match"] if expected else [],
+        "seed": 42,
+    }
+    header = (
+        f"# PaperLab run config for {data.name}\n"
+        f"# Columns: {', '.join(columns)}\n"
+        "# Edit the mapping, prompt, models, and metrics, then run:\n"
+        f"#   paperlab-space estimate {output.name}\n"
+        f"#   paperlab-space run {output.name}\n"
+        f"# Metrics: {', '.join(sorted(available_metrics()))}\n"
+        "# API keys are read from OPENROUTER_API_KEY, GEMINI_API_KEY, OPENAI_API_KEY, ...\n"
+    )
+    output.write_text(
+        header + yaml.safe_dump(config, allow_unicode=True, sort_keys=False), encoding="utf-8"
+    )
+    console.print(f"Wrote [bold]{output}[/] (input: {inputs}, expected: {expected}, id: {id_col})")
+
+
+@app.command()
+def validate(
+    data: Annotated[Path, typer.Argument(help="Excel or CSV file.")],
+    sheet: Annotated[str | None, typer.Option(help="Sheet name for Excel files.")] = None,
+    input: Annotated[
+        list[str] | None, typer.Option("--input", "-i", help="Input column(s).")
+    ] = None,
+    expected: Annotated[str | None, typer.Option("--expected", "-e")] = None,
+    id: Annotated[str | None, typer.Option("--id")] = None,
+    delimiter: Annotated[str | None, typer.Option(help="Separator for multiple answers.")] = None,
+    preview: Annotated[int, typer.Option(help="Rows to preview as JSONL.")] = 3,
+) -> None:
+    """Check a data file for problems and preview the converted JSONL."""
+    try:
+        dataset = load_dataset(data, sheet)
+        console.print(
+            f"[bold]{data.name}[/]: {len(dataset)} rows, sheet {dataset.sheet!r}"
+            if dataset.sheet
+            else f"[bold]{data.name}[/]: {len(dataset)} rows"
+        )
+        if data.suffix.lower() in (".xlsx", ".xlsm"):
+            console.print(f"Sheets: {', '.join(list_sheets(data))}")
+        console.print(f"Columns: {', '.join(dataset.columns)}")
+        if not input:
+            console.print("Pass --input (and --expected) to check a column mapping.")
+            return
+        mapping = ColumnMapping(input=input, expected=expected, id=id, answer_delimiter=delimiter)
+        mapped = dataset.map(mapping)
+    except (PaperLabError, ValueError) as exc:
+        raise _fail(exc) from exc
+    report = mapped.validate()
+    console.print(str(report), markup=False, style="red" if not report.ok else None)
+    if preview:
+        console.print(f"\nFirst {min(preview, len(mapped))} records:")
+        for line in mapped.to_jsonl().splitlines()[:preview]:
+            console.print(line, markup=False)
+    if not report.ok:
+        raise typer.Exit(1)
+
+
+@app.command("estimate")
+def estimate_cmd(
+    config: Annotated[Path, typer.Argument(help="Run config (YAML).")],
+) -> None:
+    """Estimate tokens, cost, and time for a run without calling any model."""
+    try:
+        cfg = RunConfig.from_yaml(config)
+        console.print(str(estimate(cfg, prepare_data(cfg, None))), markup=False)
+    except PaperLabError as exc:
+        raise _fail(exc) from exc
+
+
+@app.command("run")
+def run_cmd(
+    config: Annotated[Path, typer.Argument(help="Run config (YAML).")],
+    out: Annotated[Path | None, typer.Option("--out", "-o", help="Output folder.")] = None,
+    yes: Annotated[bool, typer.Option("--yes", "-y", help="Skip the confirmation.")] = False,
+    resume: Annotated[
+        Path | None, typer.Option(help="results.json of an interrupted run to continue.")
+    ] = None,
+) -> None:
+    """Run an evaluation and write Excel, LaTeX, YAML, and JSON results."""
+    out = out or config.parent / f"results-{config.stem}"
+    try:
+        cfg = RunConfig.from_yaml(config)
+        data = prepare_data(cfg, None)
+        previous = RunResults.load_json(resume) if resume else None
+    except PaperLabError as exc:
+        raise _fail(exc) from exc
+
+    console.print(str(estimate(cfg, data)), markup=False)
+    if not yes and not typer.confirm("Start the run?", default=True):
+        raise typer.Exit()
+
+    with Progress(
+        TextColumn("{task.description}"),
+        BarColumn(),
+        MofNCompleteColumn(),
+        TextColumn("[red]{task.fields[failed]} failed"),
+        TimeRemainingColumn(),
+        console=console,
+    ) as bar:
+        tasks = {m.name: bar.add_task(m.name, total=len(data), failed=0) for m in cfg.models}
+
+        def on_progress(p: RunProgress) -> None:
+            bar.update(tasks[p.model], completed=p.done, failed=p.failed)
+
+        try:
+            results = run(cfg, data, on_progress=on_progress, previous=previous)
+        except PaperLabError as exc:
+            raise _fail(exc) from exc
+
+    paths = write_all(results, out)
+    _print_scores(results)
+    failed = sum(r.n_failed for r in results.runs.values())
+    if failed:
+        console.print(
+            f"[yellow]{failed} request(s) failed.[/] Re-run with "
+            f"--resume {paths['json']} to retry only those rows."
+        )
+    console.print(
+        f"Results written to [bold]{out}[/]: " + ", ".join(p.name for p in paths.values())
+    )
+
+
+@app.command()
+def metrics() -> None:
+    """List the available metrics."""
+    table = Table("Metric", "Needs answer key", "Description")
+    for name, cls in sorted(available_metrics().items()):
+        table.add_row(name, "yes" if cls.requires_reference else "no", cls.description)
+    console.print(table)
+
+
+def _print_scores(results: RunResults) -> None:
+    keys = [s.key for s in results.config.metrics]
+    table = Table("Model", *keys, "Failed", "Cost (USD)")
+    for name, model_run in results.runs.items():
+        scores = results.scores.get(name, {})
+        cells = [
+            f"{scores[k].value:.4f}" if k in scores and scores[k].value is not None else "--"
+            for k in keys
+        ]
+        cost = model_run.cost_usd
+        table.add_row(
+            name, *cells, str(model_run.n_failed), "--" if cost is None else f"{cost:.4f}"
+        )
+    console.print(table)

paperlab_space/cli.py

@@ -0,0 +1,22 @@
+"""Entry point for the ``paperlab-space`` command."""
+
+from __future__ import annotations
+
+import sys
+
+
+def main() -> None:
+    try:
+        from paperlab_space._cli import app
+    except ImportError as exc:
+        if getattr(exc, "name", None) not in ("typer", "rich"):
+            raise
+        sys.stderr.write(
+            "The paperlab-space command needs the CLI extra:\n  pip install 'paperlab-space[cli]'\n"
+        )
+        raise SystemExit(1) from exc
+    app()
+
+
+if __name__ == "__main__":
+    main()

paperlab_space/config.py

@@ -0,0 +1,196 @@
+"""The run configuration: one object that fully describes an evaluation.
+
+The same :class:`RunConfig` drives a run and is written out as the reproducibility
+config, so a run can be repeated from its exported config alone.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator
+
+from paperlab_space.data.mapping import ColumnMapping
+from paperlab_space.errors import ConfigError
+from paperlab_space.prompt import PromptTemplate
+
+
+class _Strict(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+
+class ModelSpec(_Strict):
+    """A model to evaluate, addressed by its LiteLLM ID (``provider/model``)."""
+
+    id: str
+    label: str | None = None
+    """Display name in tables. Defaults to the model ID without the provider prefix."""
+    temperature: float = Field(0.0, ge=0)
+    max_tokens: int | None = Field(None, gt=0)
+    top_p: float | None = Field(None, gt=0, le=1)
+    seed: int | None = None
+    requests_per_minute: float | None = Field(None, gt=0)
+    """Override the default rate limit for this model."""
+    concurrency: int | None = Field(None, gt=0)
+    """Override the run-level number of parallel requests for this model."""
+    params: dict[str, Any] = {}
+    """Extra provider parameters passed through to LiteLLM."""
+
+    @model_validator(mode="before")
+    @classmethod
+    def _from_string(cls, value: Any) -> Any:
+        return {"id": value} if isinstance(value, str) else value
+
+    @field_validator("id")
+    @classmethod
+    def _check_id(cls, value: str) -> str:
+        value = value.strip()
+        if "/" not in value:
+            raise ValueError(
+                f"Model ID {value!r} needs a provider prefix, e.g. 'openrouter/{value}' or "
+                f"'gemini/{value}'."
+            )
+        return value
+
+    @property
+    def provider(self) -> str:
+        return self.id.split("/", 1)[0]
+
+    @property
+    def name(self) -> str:
+        """The label used in outputs."""
+        return self.label or self.id.split("/", 1)[1]
+
+
+class MetricSpec(_Strict):
+    """A metric name plus its options, e.g. ``{"name": "exact_match", "ignore_case": false}``."""
+
+    name: str
+    label: str | None = None
+    """Column name in outputs; needed to use the same metric twice with different options."""
+    params: dict[str, Any] = {}
+
+    @model_validator(mode="before")
+    @classmethod
+    def _from_string(cls, value: Any) -> Any:
+        if isinstance(value, str):
+            return {"name": value}
+        if isinstance(value, dict) and "name" in value and "params" not in value:
+            # Allow the flat form ``{name: f1, average: macro}``.
+            keys = {"name", "label"}
+            return {
+                **{k: v for k, v in value.items() if k in keys},
+                "params": {k: v for k, v in value.items() if k not in keys},
+            }
+        return value
+
+    @property
+    def key(self) -> str:
+        return self.label or self.name
+
+
+class DatasetSpec(_Strict):
+    """Where the data comes from. Optional when the dataset is passed in directly."""
+
+    path: str
+    sheet: str | int | None = None
+
+
+class RetrySettings(_Strict):
+    max_attempts: int = Field(5, ge=1)
+    base_delay: float = Field(2.0, ge=0)
+    """Seconds before the first retry; doubles on each attempt."""
+    max_delay: float = Field(60.0, ge=0)
+    timeout: float = Field(120.0, gt=0)
+    """Seconds to wait for one model response."""
+
+
+class RunConfig(_Strict):
+    """Everything needed to run (and re-run) an evaluation."""
+
+    dataset: DatasetSpec | None = None
+    mapping: ColumnMapping
+    prompt: PromptTemplate
+    models: list[ModelSpec]
+    metrics: list[MetricSpec] = []
+    seed: int | None = None
+    """Default seed for models that do not set their own."""
+    concurrency: int = Field(4, gt=0)
+    """Parallel requests per model (rate limits still apply)."""
+    retry: RetrySettings = RetrySettings()
+
+    @model_validator(mode="after")
+    def _check(self) -> RunConfig:
+        if not self.models:
+            raise ValueError("Pick at least one model.")
+        names = [m.name for m in self.models]
+        dupes = sorted({n for n in names if names.count(n) > 1})
+        if dupes:
+            raise ValueError(
+                f"Models must have distinct labels; duplicated: {', '.join(dupes)}. "
+                "Set `label` to tell them apart (e.g. the same model at two temperatures)."
+            )
+        keys = [m.key for m in self.metrics]
+        dupes = sorted({k for k in keys if keys.count(k) > 1})
+        if dupes:
+            raise ValueError(
+                f"Metrics must have distinct labels; duplicated: {', '.join(dupes)}. "
+                "Set `label` to use the same metric with different options."
+            )
+        try:
+            self.prompt.check(self.mapping.fields)
+        except Exception as exc:
+            raise ValueError(str(exc)) from exc
+        return self
+
+    def model_seed(self, model: ModelSpec) -> int | None:
+        return model.seed if model.seed is not None else self.seed
+
+    # --- YAML -------------------------------------------------------------------------
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> RunConfig:
+        """Build from a dict; also accepts a reproducibility file (config under ``config``)."""
+        if "config" in data and "paperlab_space" in data:
+            data = data["config"]
+        try:
+            return cls.model_validate(data)
+        except ValidationError as exc:
+            raise ConfigError(_format_errors(exc)) from exc
+
+    @classmethod
+    def from_yaml(cls, path: str | Path) -> RunConfig:
+        """Load a YAML config. A relative dataset path is resolved against the file's folder."""
+        path = Path(path)
+        try:
+            data = yaml.safe_load(path.read_text(encoding="utf-8"))
+        except (OSError, yaml.YAMLError) as exc:
+            raise ConfigError(f"Could not read {path}: {exc}") from exc
+        if not isinstance(data, dict):
+            raise ConfigError(f"{path} does not contain a YAML mapping.")
+        config = cls.from_dict(data)
+        if config.dataset is not None and not Path(config.dataset.path).is_absolute():
+            resolved = (path.parent / config.dataset.path).resolve()
+            config = config.model_copy(
+                update={"dataset": config.dataset.model_copy(update={"path": str(resolved)})}
+            )
+        return config
+
+    def to_dict(self) -> dict[str, Any]:
+        return self.model_dump(mode="json", exclude_defaults=False)
+
+    def to_yaml(self, path: str | Path | None = None) -> str:
+        text = yaml.safe_dump(self.to_dict(), allow_unicode=True, sort_keys=False)
+        if path is not None:
+            Path(path).write_text(text, encoding="utf-8")
+        return text
+
+
+def _format_errors(exc: ValidationError) -> str:
+    lines = ["Invalid run configuration:"]
+    for error in exc.errors():
+        where = ".".join(str(part) for part in error["loc"]) or "config"
+        lines.append(f"  {where}: {error['msg']}")
+    return "\n".join(lines)

paperlab_space/data/__init__.py

@@ -0,0 +1,17 @@
+"""Loading, mapping, and validating evaluation data."""
+
+from paperlab_space.data.load import Dataset, list_sheets, load_dataset
+from paperlab_space.data.mapping import ColumnMapping, MappedDataset, Record
+from paperlab_space.data.validate import Issue, ValidationReport, validate
+
+__all__ = [
+    "ColumnMapping",
+    "Dataset",
+    "Issue",
+    "MappedDataset",
+    "Record",
+    "ValidationReport",
+    "list_sheets",
+    "load_dataset",
+    "validate",
+]