promptuna-cli 1.24.0__tar.gz

promptuna_cli-1.24.0/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.3
+Name: promptuna-cli
+Version: 1.24.0
+Summary: Typer CLI for on-disk promptuna run / evaluate / optimize jobs
+Requires-Dist: promptuna==1.24.0
+Requires-Dist: typer>=0.12
+Requires-Python: >=3.13

promptuna_cli-1.24.0/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "promptuna-cli"
+version = "1.24.0"
+description = "Typer CLI for on-disk promptuna run / evaluate / optimize jobs"
+requires-python = ">=3.13"
+dependencies = [
+    "promptuna==1.24.0",
+    "typer>=0.12",
+]
+
+[project.scripts]
+promptuna = "promptuna_cli.main:run_cli"
+
+[tool.uv.sources]
+promptuna = { workspace = true }
+
+[build-system]
+requires = ["uv_build>=0.11.1,<0.12.0"]
+build-backend = "uv_build"

promptuna_cli-1.24.0/src/promptuna_cli/__init__.py

@@ -0,0 +1,5 @@
+"""Typer CLI for on-disk promptuna projects."""
+
+from promptuna_cli.main import app, run_cli
+
+__all__ = ["app", "run_cli"]

promptuna_cli-1.24.0/src/promptuna_cli/_common.py

@@ -0,0 +1,151 @@
+"""Shared helpers for CLI job commands."""
+
+from __future__ import annotations
+
+import json
+import uuid
+from collections.abc import Callable, Iterator
+from pathlib import Path
+from typing import Any, Literal
+
+import typer
+
+from promptuna.evaluate import RunInfo, RunResults, Scoring
+from promptuna.jobs import JobArchive, JobConfig, JobKind, get_jobs_root, load_job, stream_job
+from promptuna.optimize import Step
+from promptuna.program import Experiment
+from promptuna.projects import ProjectValidationError, set_projects_root
+from promptuna.report import render_history, render_run
+from promptuna.run import FailedTrial, SuccessfulTrial
+
+OutputFormat = Literal["human", "json"]
+
+
+def apply_projects_root(projects_root: Path | None) -> None:
+    """Apply a CLI override for the active projects root."""
+    if projects_root is not None:
+        set_projects_root(projects_root.expanduser().resolve())
+
+
+def parse_metric_names(values: list[str]) -> list[str]:
+    """Flatten ``--metric`` values, splitting on commas when present."""
+    names: list[str] = []
+    for value in values:
+        for part in value.split(","):
+            stripped = part.strip()
+            if stripped:
+                names.append(stripped)
+    if not names:
+        raise typer.BadParameter("at least one --metric is required")
+    return names
+
+
+def _collecting_source[T](
+    source: Callable[[], Iterator[T]],
+) -> tuple[Callable[[], Iterator[T]], list[T]]:
+    collected: list[T] = []
+
+    def wrapped() -> Iterator[T]:
+        for item in source():
+            collected.append(item)
+            yield item
+
+    return wrapped, collected
+
+
+def execute_job(
+    *,
+    config: JobConfig,
+    source: Callable[[], Iterator[Any]],
+    experiment: Experiment,
+    render_human: Callable[[list[Any]], str],
+    output_format: OutputFormat,
+) -> None:
+    """Run one blocking job, persist it on disk, and print the result."""
+    job_id = str(uuid.uuid4())
+    archive = JobArchive.open(get_jobs_root(), job_id, config)
+    source_fn, collected = _collecting_source(source)
+
+    try:
+        for _ in stream_job(archive, source_fn()):
+            pass
+    except Exception:
+        _exit_on_failed_job(job_id)
+        raise
+
+    record = load_job(get_jobs_root(), job_id)
+    if record.manifest["status"] == "error":
+        _exit_on_failed_job(job_id)
+
+    typer.echo(f"job_id: {job_id}", err=True)
+    if output_format == "json":
+        typer.echo(json.dumps(record.summary, indent=2, sort_keys=True))
+        return
+
+    typer.echo(render_human(collected))
+
+
+def _exit_on_failed_job(job_id: str) -> None:
+    record = load_job(get_jobs_root(), job_id)
+    error = record.manifest.get("error") or "unknown error"
+    typer.echo(f"job failed: {error}", err=True)
+    raise typer.Exit(code=1)
+
+
+def render_run_human(experiment: Experiment, items: list[Any]) -> str:
+    """Render a run or evaluate job as markdown."""
+    trials = [item for item in items if isinstance(item, (SuccessfulTrial, FailedTrial))]
+    scorings = [item for item in items if isinstance(item, Scoring)]
+    results = RunResults(
+        experiment=experiment,
+        run=RunInfo(),
+        trials=trials,
+        scorings=scorings,
+    )
+    error_format = None if not scorings else "inputs"
+    return render_run(results, error_format=error_format)
+
+
+def render_optimize_human(items: list[Any]) -> str:
+    """Render an optimize job trajectory as markdown."""
+    steps = [item for item in items if isinstance(item, Step)]
+    return render_history(steps)
+
+
+def handle_project_error(exc: ProjectValidationError) -> None:
+    """Map project validation failures to a CLI exit."""
+    typer.echo(str(exc), err=True)
+    raise typer.Exit(code=2) from exc
+
+
+def build_job_config(
+    *,
+    kind: JobKind,
+    project: str,
+    program: str,
+    prompt: str,
+    examples: str,
+    dataset_path: Path,
+    model: str,
+    workers: int,
+    metrics: tuple[str, ...] | None = None,
+    steps: int | None = None,
+    proposer_model: str | None = None,
+) -> JobConfig:
+    """Build a :class:`JobConfig` for the active projects root."""
+    from promptuna.projects import get_projects_root
+
+    return JobConfig(
+        kind=kind,
+        projects_root=get_projects_root(),
+        project=project,
+        program=program,
+        prompt=prompt,
+        examples=examples,
+        dataset_path=dataset_path,
+        model=model,
+        workers=workers,
+        metrics=metrics,
+        steps=steps,
+        proposer_model=proposer_model,
+    )

promptuna_cli-1.24.0/src/promptuna_cli/main.py

@@ -0,0 +1,266 @@
+"""Typer entry point for the promptuna CLI."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from promptuna.evaluate import stream_evaluate
+from promptuna.jobs import get_jobs_root, load_job
+from promptuna.optimize import stream_optimize
+from promptuna.projects import (
+    ProjectValidationError,
+    build_experiment,
+    resolve_dataset_path,
+    resolve_project_dir,
+)
+from promptuna.run import stream_run
+from promptuna_cli._common import (
+    OutputFormat,
+    apply_projects_root,
+    build_job_config,
+    execute_job,
+    handle_project_error,
+    parse_metric_names,
+    render_optimize_human,
+    render_run_human,
+)
+
+app = typer.Typer(
+    name="promptuna",
+    no_args_is_help=True,
+    add_completion=False,
+    help="Run, evaluate, and optimize on-disk promptuna projects.",
+)
+
+
+@app.callback()
+def main(
+    projects_root: Annotated[
+        Path | None,
+        typer.Option(
+            "--projects-root",
+            help="Directory containing project folders (overrides PROMPTUNA_PROJECTS_ROOT).",
+            dir_okay=True,
+            file_okay=False,
+            resolve_path=True,
+        ),
+    ] = None,
+) -> None:
+    """Configure workspace paths shared by every subcommand."""
+    apply_projects_root(projects_root)
+
+
+@app.command()
+def run(
+    project: Annotated[str, typer.Option("--project", "-p", help="Project directory name.")],
+    program: Annotated[str, typer.Option("--program", help="Program function in programs.py.")],
+    prompt: Annotated[str, typer.Option("--prompt", help="Prompt template name.")],
+    examples: Annotated[str, typer.Option("--examples", help="Dataset name under data/.")],
+    model: Annotated[str, typer.Option("--model", "-m", help="LM id for program execution.")],
+    workers: Annotated[
+        int,
+        typer.Option("--workers", "-w", min=1, help="Parallel trial workers."),
+    ] = 1,
+    output_format: Annotated[
+        OutputFormat,
+        typer.Option("--format", "-f", help="Output format for the finished job."),
+    ] = "human",
+) -> None:
+    """Execute a program over a dataset."""
+    try:
+        experiment, example_rows, _ = build_experiment(
+            project=project,
+            program=program,
+            prompt=prompt,
+            model=model,
+            examples=examples,
+        )
+        project_dir = resolve_project_dir(project)
+        dataset_path = resolve_dataset_path(project_dir, examples)
+        config = build_job_config(
+            kind="run",
+            project=project,
+            program=program,
+            prompt=prompt,
+            examples=examples,
+            dataset_path=dataset_path,
+            model=model,
+            workers=workers,
+        )
+    except ProjectValidationError as exc:
+        handle_project_error(exc)
+
+    execute_job(
+        config=config,
+        source=lambda: stream_run(experiment, example_rows, workers=workers),
+        experiment=experiment,
+        render_human=lambda items: render_run_human(experiment, items),
+        output_format=output_format,
+    )
+
+
+@app.command()
+def evaluate(
+    project: Annotated[str, typer.Option("--project", "-p", help="Project directory name.")],
+    program: Annotated[str, typer.Option("--program", help="Program function in programs.py.")],
+    prompt: Annotated[str, typer.Option("--prompt", help="Prompt template name.")],
+    examples: Annotated[str, typer.Option("--examples", help="Dataset name under data/.")],
+    model: Annotated[str, typer.Option("--model", "-m", help="LM id for program execution.")],
+    metric: Annotated[
+        list[str],
+        typer.Option("--metric", "-M", help="Metric name from metrics.py (repeatable)."),
+    ],
+    workers: Annotated[
+        int,
+        typer.Option("--workers", "-w", min=1, help="Parallel trial workers."),
+    ] = 1,
+    output_format: Annotated[
+        OutputFormat,
+        typer.Option("--format", "-f", help="Output format for the finished job."),
+    ] = "human",
+) -> None:
+    """Execute a program and score it with one or more metrics."""
+    metric_names = parse_metric_names(metric)
+    try:
+        experiment, example_rows, metrics = build_experiment(
+            project=project,
+            program=program,
+            prompt=prompt,
+            model=model,
+            examples=examples,
+            metrics=metric_names,
+        )
+        project_dir = resolve_project_dir(project)
+        dataset_path = resolve_dataset_path(project_dir, examples)
+        config = build_job_config(
+            kind="evaluate",
+            project=project,
+            program=program,
+            prompt=prompt,
+            examples=examples,
+            dataset_path=dataset_path,
+            model=model,
+            workers=workers,
+            metrics=tuple(metric_names),
+        )
+    except ProjectValidationError as exc:
+        handle_project_error(exc)
+
+    assert metrics is not None
+
+    def source():
+        return stream_evaluate(experiment, example_rows, metrics, workers=workers)
+
+    execute_job(
+        config=config,
+        source=source,
+        experiment=experiment,
+        render_human=lambda items: render_run_human(experiment, items),
+        output_format=output_format,
+    )
+
+
+@app.command()
+def optimize(
+    project: Annotated[str, typer.Option("--project", "-p", help="Project directory name.")],
+    program: Annotated[str, typer.Option("--program", help="Program function in programs.py.")],
+    prompt: Annotated[str, typer.Option("--prompt", help="Prompt template name.")],
+    examples: Annotated[str, typer.Option("--examples", help="Dataset name under data/.")],
+    model: Annotated[str, typer.Option("--model", "-m", help="LM id for program execution.")],
+    metric: Annotated[
+        list[str],
+        typer.Option("--metric", "-M", help="Metric name from metrics.py (repeatable)."),
+    ],
+    steps: Annotated[int, typer.Option("--steps", min=0, help="Proposer steps after baseline.")],
+    proposer_model: Annotated[
+        str,
+        typer.Option("--proposer-model", help="LM id for prompt-template proposals."),
+    ],
+    workers: Annotated[
+        int,
+        typer.Option("--workers", "-w", min=1, help="Parallel trial workers."),
+    ] = 1,
+    output_format: Annotated[
+        OutputFormat,
+        typer.Option("--format", "-f", help="Output format for the finished job."),
+    ] = "human",
+) -> None:
+    """Search for a better prompt template."""
+    metric_names = parse_metric_names(metric)
+    try:
+        experiment, example_rows, metrics = build_experiment(
+            project=project,
+            program=program,
+            prompt=prompt,
+            model=model,
+            examples=examples,
+            metrics=metric_names,
+        )
+        project_dir = resolve_project_dir(project)
+        dataset_path = resolve_dataset_path(project_dir, examples)
+        config = build_job_config(
+            kind="optimize",
+            project=project,
+            program=program,
+            prompt=prompt,
+            examples=examples,
+            dataset_path=dataset_path,
+            model=model,
+            workers=workers,
+            metrics=tuple(metric_names),
+            steps=steps,
+            proposer_model=proposer_model,
+        )
+    except ProjectValidationError as exc:
+        handle_project_error(exc)
+
+    assert metrics is not None
+
+    def source():
+        return stream_optimize(
+            experiment,
+            example_rows,
+            metrics,
+            proposer_model=proposer_model,
+            steps=steps,
+            workers=workers,
+        )
+
+    execute_job(
+        config=config,
+        source=source,
+        experiment=experiment,
+        render_human=render_optimize_human,
+        output_format=output_format,
+    )
+
+
+@app.command()
+def report(
+    job_id: Annotated[str, typer.Argument(help="Job id under <projects_root>/jobs/.")],
+) -> None:
+    """Print ``summary.json`` for a finished on-disk job."""
+    try:
+        record = load_job(get_jobs_root(), job_id)
+    except FileNotFoundError:
+        typer.echo(f"job {job_id!r} not found", err=True)
+        raise typer.Exit(code=2) from None
+
+    if record.summary is None:
+        typer.echo(f"job {job_id!r} has no summary yet", err=True)
+        raise typer.Exit(code=1)
+
+    typer.echo(json.dumps(record.summary, indent=2, sort_keys=True))
+
+
+def run_cli() -> None:
+    """Console entry point."""
+    app()
+
+
+if __name__ == "__main__":
+    run_cli()