bathos 0.1.0__py3-none-any.whl

bathos/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

bathos/archive.py

@@ -0,0 +1,161 @@
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+import duckdb
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+@dataclass
+class ArchiveResult:
+    """Result of an archive operation."""
+
+    runs_archived: int
+    partitions_created: int
+    archive_size_bytes: int
+    manifest_path: Path
+    duration_s: float
+
+
+def archive(
+    catalog_dir: Path,
+    archive_root: Path | None = None,
+    project_slug: str | None = None,
+    dry_run: bool = False,
+) -> ArchiveResult:
+    """Export warm-tier runs to cold-tier partitioned Parquet.
+
+    Strategy: Partition by project_slug, year, month for efficient bulk queries.
+
+    Args:
+        catalog_dir: Path to catalog root
+        archive_root: Root for archive (default ~/.bth/archive/)
+        project_slug: Filter to specific project (default: all)
+        dry_run: If True, show what would be archived without writing
+
+    Returns:
+        ArchiveResult with counts, size, manifest path, duration
+    """
+    start_time = time.time()
+
+    if archive_root is None:
+        archive_root = Path.home() / ".bth" / "archive"
+
+    if not dry_run:
+        archive_root.mkdir(parents=True, exist_ok=True)
+
+    # Check for warm DB
+    db_path = catalog_dir / "bathos.db"
+    if not db_path.exists():
+        raise RuntimeError("No warm catalog. Run `bth compact` first.")
+
+    # Read runs from warm DB
+    con = duckdb.connect(str(db_path), read_only=True)
+
+    query = "SELECT * FROM runs ORDER BY id"
+    params = []
+    if project_slug:
+        query = "SELECT * FROM runs WHERE project_slug = ? ORDER BY id"
+        params = [project_slug]
+
+    results = con.execute(query, params).to_arrow_table()
+    con.close()
+
+    if len(results) == 0:
+        duration_s = time.time() - start_time
+        return ArchiveResult(
+            runs_archived=0,
+            partitions_created=0,
+            archive_size_bytes=0,
+            manifest_path=archive_root / "manifest.json",
+            duration_s=duration_s,
+        )
+
+    # Partition by project, year, month
+    partitions: dict[tuple[str, int, str], list[int]] = {}
+
+    timestamp_col = results.column("timestamp")
+    project_col = results.column("project_slug")
+
+    for i in range(len(results)):
+        # Extract project and timestamp
+        project = project_col[i].as_py()
+        ts = timestamp_col[i].as_py()
+
+        # Handle both int (microseconds) and datetime
+        dt = datetime.fromtimestamp(ts / 1000000.0) if isinstance(ts, int) else ts
+
+        year = dt.year
+        month = f"{dt.month:02d}"
+
+        key = (project, year, month)
+        if key not in partitions:
+            partitions[key] = []
+        partitions[key].append(i)
+
+    # Write partitions
+    total_archived = 0
+    partitions_created = 0
+    total_size = 0
+    manifest_entries = []
+
+    for (project, year, month), indices in sorted(partitions.items()):
+        partition_path = archive_root / f"project={project}" / f"year={year}" / f"month={month}"
+        output_file = partition_path / "runs.parquet"
+
+        if not dry_run:
+            partition_path.mkdir(parents=True, exist_ok=True)
+
+            # Filter results to just these rows
+            indices_array = pa.array(indices)
+            mask = pa.compute.is_in(pa.array(list(range(len(results)))), indices_array)
+            filtered = results.filter(mask)
+
+            # Write atomically: write to temp, then rename
+            temp_file = output_file.parent / f".{output_file.name}.tmp"
+            pq.write_table(filtered, str(temp_file))
+            temp_file.rename(output_file)
+
+            file_size = output_file.stat().st_size
+            total_size += file_size
+        else:
+            file_size = 0
+
+        manifest_entries.append(
+            {
+                "partition": f"project={project}/year={year}/month={month}",
+                "rows": len(indices),
+                "size_bytes": file_size if not dry_run else 0,
+            }
+        )
+
+        total_archived += len(indices)
+        partitions_created += 1
+
+    # Write manifest
+    manifest_path = archive_root / "manifest.json"
+    if not dry_run:
+        manifest = {
+            "timestamp": datetime.now(UTC).isoformat(),
+            "runs_archived": total_archived,
+            "partitions": partitions_created,
+            "total_size_bytes": total_size,
+            "entries": manifest_entries,
+        }
+        with open(manifest_path, "w") as f:
+            json.dump(manifest, f, indent=2)
+
+    duration_s = time.time() - start_time
+
+    return ArchiveResult(
+        runs_archived=total_archived,
+        partitions_created=partitions_created,
+        archive_size_bytes=total_size,
+        manifest_path=manifest_path,
+        duration_s=duration_s,
+    )

bathos/catalog.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.parquet as pq
+
+from bathos.schema import Run
+
+
+def init_catalog(catalog_dir: Path) -> None:
+    (catalog_dir / "runs").mkdir(parents=True, exist_ok=True)
+
+
+def write_run(run: Run, catalog_dir: Path) -> None:
+    """Write (or overwrite) a run record atomically."""
+    runs_dir = catalog_dir / "runs"
+    runs_dir.mkdir(parents=True, exist_ok=True)
+    target = runs_dir / f"run_{run.id}.parquet"
+    tmp = runs_dir / f"run_{run.id}.tmp.parquet"
+    pq.write_table(run.to_arrow(), tmp)
+    tmp.rename(target)  # atomic on POSIX
+
+
+def read_runs(catalog_dir: Path) -> list[Run]:
+    """Read all runs from Parquet fragments, sorted by timestamp DESC."""
+    runs_dir = catalog_dir / "runs"
+    if not runs_dir.exists():
+        return []
+    parquet_files = list(runs_dir.glob("run_*.parquet"))
+    if not parquet_files:
+        return []
+    tables = [pq.read_table(f) for f in parquet_files]
+    combined = pa.concat_tables(tables)
+    order = pc.sort_indices(combined, sort_keys=[("timestamp", "descending")])
+    combined = combined.take(order)
+    pydict = combined.to_pydict()
+    return [Run.from_arrow_row(pydict, i) for i in range(combined.num_rows)]

bathos/checker.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from bathos.git import capture_git_state
+from bathos.query import list_runs
+from bathos.schema import Run
+
+
+@dataclass
+class CheckResult:
+    """Result of checking a single run's git-drift validity."""
+
+    run_id: str
+    status: Literal["OK", "STALE", "DIRTY_RUN", "UNKNOWN_CODE"]
+    run_git_hash: str
+    current_hash: str
+
+
+@dataclass
+class OutputCheckResult:
+    """Result of checking a single output file."""
+
+    path: str
+    status: str  # "present", "missing", "unreadable"
+    size_bytes: int = 0
+
+
+def check_runs(
+    catalog_dir: Path,
+    project_root: Path,
+    status_filter: str | None = None,
+) -> list[CheckResult]:
+    """Check all runs in catalog for git-drift validity.
+
+    For each run:
+    - STALE: run's git_hash != current HEAD and run's git_dirty was False
+    - DIRTY_RUN: run's git_dirty was True
+    - UNKNOWN_CODE: run's git_hash == "unknown"
+    - OK: otherwise (hash matches current or dirty was True)
+
+    Args:
+        catalog_dir: Path to catalog directory
+        project_root: Path to project root (used to get current git state)
+        status_filter: Optional filter; return only results with this status
+
+    Returns:
+        List of CheckResult objects
+    """
+    # Get current git state
+    current_state = capture_git_state(project_root)
+    current_hash = current_state.hash
+
+    # Get all runs from catalog
+    all_runs = list_runs(catalog_dir)
+
+    results = []
+    for run in all_runs:
+        if run.git_hash == "unknown":
+            status = "UNKNOWN_CODE"
+        elif run.git_dirty:
+            status = "DIRTY_RUN"
+        elif run.git_hash != current_hash:
+            status = "STALE"
+        else:
+            status = "OK"
+
+        result = CheckResult(
+            run_id=run.id,
+            status=status,
+            run_git_hash=run.git_hash,
+            current_hash=current_hash,
+        )
+        results.append(result)
+
+    # Apply filter if provided
+    if status_filter:
+        results = [r for r in results if r.status == status_filter]
+
+    return results
+
+
+def check_output_files(run: Run) -> list[OutputCheckResult]:
+    """Verify output files exist and are readable.
+
+    Args:
+        run: Run object with output_paths
+
+    Returns:
+        List of OutputCheckResult for each file
+    """
+    from bathos.compact import _collect_output_metadata
+
+    results = []
+    for path in run.output_paths:
+        meta = _collect_output_metadata(path)
+        results.append(
+            OutputCheckResult(
+                path=path,
+                status=meta["status"],
+                size_bytes=meta.get("size_bytes", 0),
+            )
+        )
+
+    return results

bathos/cli.py

@@ -0,0 +1,337 @@
+from __future__ import annotations
+
+import os
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+import typer
+
+from bathos.archive import archive
+
+app = typer.Typer(help="bathos — local-first experiment tracking")
+
+
+def _catalog_dir() -> Path:
+    override = os.environ.get("BTH_CATALOG_DIR")
+    if override:
+        return Path(override)
+    from bathos.config import default_catalog_dir
+
+    return default_catalog_dir()
+
+
+def _require_project_slug() -> str:
+    slug_env = os.environ.get("BTH_PROJECT_SLUG")
+    if slug_env:
+        return slug_env
+    from bathos.config import find_project_config, load_project_config
+
+    cfg_path = find_project_config()
+    if cfg_path is None:
+        typer.echo("No .bth.toml found. Run `bth init` first.", err=True)
+        raise typer.Exit(1)
+    return load_project_config(cfg_path).slug
+
+
+@app.callback(invoke_without_command=True)
+def main(
+    version: bool = typer.Option(False, "--version", "-V", is_eager=True),
+):
+    if version:
+        from bathos import __version__
+
+        typer.echo(f"bathos {__version__}")
+        raise typer.Exit()
+
+
+@app.command()
+def init(
+    slug: str = typer.Option(..., "--slug", "-s", help="Project slug"),
+    remote: str | None = typer.Option(None, "--remote", help="host:remote_path"),
+    slurm_partition: str | None = typer.Option(None, "--slurm-partition"),
+):
+    """Register project, scaffold scripts/ dirs, write .bth.toml."""
+    from bathos.init import init_project
+
+    init_project(
+        Path.cwd(),
+        slug=slug,
+        catalog_dir=_catalog_dir(),
+        remote=remote,
+        slurm_partition=slurm_partition,
+    )
+    typer.echo(f"Initialized bathos project '{slug}'")
+
+
+@app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True})
+def run(
+    argv: list[str] = typer.Argument(...),
+    out: list[str] = typer.Option([], "--out", help="Output path to register"),
+    tag: list[str] = typer.Option([], "--tag", "-t"),
+):
+    """Run a script and record provenance."""
+    from bathos.runner import run_script
+
+    slug = _require_project_slug()
+    exit_code = run_script(
+        argv=argv,
+        project_slug=slug,
+        catalog_dir=_catalog_dir(),
+        output_paths=out,
+        tags=tag,
+    )
+    raise typer.Exit(exit_code)
+
+
+@app.command("ls")
+def ls_cmd(
+    project: str | None = typer.Option(None, "--project", "-p"),
+    since: str | None = typer.Option(None, "--since", help="e.g. 7d, 24h"),
+    status: str | None = typer.Option(None, "--status"),
+    limit: int = typer.Option(20, "--limit", "-n"),
+):
+    """List recent runs."""
+    from bathos.compact import _fragment_count, should_compact
+    from bathos.query import find_runs
+
+    since_dt = _parse_since(since)
+    catalog_dir = _catalog_dir()
+    runs = find_runs(catalog_dir, since=since_dt, project=project, status=status)
+    runs = runs[:limit]
+    if not runs:
+        typer.echo("No runs found.")
+        return
+    header = f"{'ID':38} {'PROJECT':12} {'STATUS':10} {'EXIT':5} {'DURATION':8} COMMAND"
+    typer.echo(header)
+    typer.echo("-" * len(header))
+    for r in runs:
+        typer.echo(
+            f"{r.id:38} {r.project_slug:12} {r.status:10} {r.exit_code:5} "
+            f"{r.duration_s:7.1f}s {r.command[:40]}"
+        )
+
+    # Check if compaction is recommended and show banner if needed
+    if should_compact(catalog_dir):
+        frag_count = _fragment_count(catalog_dir)
+        typer.echo()
+        typer.echo(f"⚠  {frag_count} uncompacted runs — run 'bth compact' to speed up queries")
+
+
+@app.command()
+def show(run_id: str = typer.Argument(...)):
+    """Show full details of a run."""
+    from bathos.query import get_run
+
+    r = get_run(run_id, _catalog_dir())
+    if r is None:
+        typer.echo(f"Run not found: {run_id}", err=True)
+        raise typer.Exit(1)
+    typer.echo(f"id:           {r.id}")
+    typer.echo(f"project:      {r.project_slug}")
+    typer.echo(f"status:       {r.status}")
+    typer.echo(f"exit_code:    {r.exit_code}")
+    typer.echo(f"duration:     {r.duration_s:.2f}s")
+    typer.echo(f"git_hash:     {r.git_hash}")
+    typer.echo(f"git_branch:   {r.git_branch}")
+    typer.echo(f"git_dirty:    {r.git_dirty}")
+    typer.echo(f"timestamp:    {r.timestamp.isoformat()}")
+    typer.echo(f"command:      {r.command}")
+    typer.echo(f"output_paths: {r.output_paths}")
+    typer.echo(f"tags:         {r.tags}")
+
+
+@app.command()
+def find(
+    project: str | None = typer.Option(None, "--project", "-p"),
+    since: str | None = typer.Option(None, "--since"),
+    status: str | None = typer.Option(None, "--status"),
+    tag: list[str] = typer.Option([], "--tag"),
+    slurm_job: str | None = typer.Option(None, "--slurm-job", help="SLURM job ID"),
+    output_file: str | None = typer.Option(
+        None, "--output-file", help="Filter to runs with matching output file path (glob pattern)"
+    ),
+):
+    """Find runs matching filters."""
+    from bathos.query import _filter_runs_by_output_file, find_runs
+
+    runs = find_runs(
+        _catalog_dir(),
+        since=_parse_since(since),
+        project=project,
+        status=status,
+        tags=tag or None,
+        slurm_job_id=slurm_job,
+    )
+    runs = _filter_runs_by_output_file(runs, pattern=output_file)
+    for r in runs:
+        typer.echo(f"{r.id}  {r.project_slug}  {r.status}  {r.command[:60]}")
+
+
+@app.command()
+def sql(query: str = typer.Argument(...)):
+    """Run raw DuckDB SQL against the catalog."""
+    from bathos.query import run_sql
+
+    try:
+        rows = run_sql(query, catalog_dir=_catalog_dir())
+        for row in rows:
+            typer.echo("\t".join(str(v) for v in row))
+    except RuntimeError as e:
+        typer.echo(str(e), err=True)
+        raise typer.Exit(1)
+
+
+@app.command()
+def compact():
+    """Compact cool fragments into warm DuckDB catalog."""
+    from bathos.compact import compact as compact_catalog
+
+    catalog_dir = _catalog_dir()
+    result = compact_catalog(catalog_dir)
+    typer.echo(f"Compacted {result.ingested} runs into bathos.db in {result.duration_s:.1f}s")
+
+
+@app.command("archive")
+def archive_cmd(
+    project: str | None = typer.Option(
+        None, "--project", "-p", help="Filter to specific project (default: all)"
+    ),
+    archive_dir: Path | None = typer.Option(
+        None, "--archive-dir", "-d", help="Archive root directory (default: ~/.bth/archive)"
+    ),
+    dry_run: bool = typer.Option(
+        False, "--dry-run", help="Show what would be archived without writing"
+    ),
+):
+    """Archive old runs to partitioned cold storage."""
+    catalog_dir = _catalog_dir()
+
+    if not catalog_dir.exists():
+        typer.secho("✗ Catalog not found", fg="red")
+        raise typer.Exit(1)
+
+    try:
+        result = archive(
+            catalog_dir,
+            archive_root=archive_dir,
+            project_slug=project,
+            dry_run=dry_run,
+        )
+
+        status = "[dry-run] " if dry_run else ""
+        typer.secho(
+            f"✓ {status}Archived {result.runs_archived} runs into "
+            f"{result.partitions_created} partitions in {result.duration_s:.1f}s",
+            fg="green",
+        )
+
+        if not dry_run:
+            typer.secho(f"  Manifest: {result.manifest_path}", fg="cyan")
+    except RuntimeError as e:
+        typer.secho(f"✗ {str(e)}", fg="red")
+        raise typer.Exit(1)
+
+
+@app.command()
+def check(
+    status: str | None = typer.Option(
+        None, "--status", help="Filter by status (OK, STALE, DIRTY_RUN, UNKNOWN_CODE)"
+    ),
+    check_outputs: bool = typer.Option(
+        False, "--check-outputs", help="Also verify output files exist and are readable"
+    ),
+):
+    """Check runs for git-drift validity against current HEAD."""
+    from bathos.checker import check_output_files, check_runs
+    from bathos.query import get_run
+
+    catalog_dir = _catalog_dir()
+    project_root = Path.cwd()
+
+    results = check_runs(catalog_dir, project_root, status_filter=status)
+
+    if not results:
+        if status:
+            typer.echo(f"No runs found with status={status}")
+        else:
+            typer.echo("No runs found in catalog")
+        return
+
+    # Print header
+    header = f"{'RUN_ID':38} {'STATUS':12} {'RUN_HASH':40} {'CURRENT_HASH':40}"
+    typer.echo(header)
+    typer.echo("-" * len(header))
+
+    # Print results
+    stale_count = 0
+    for result in results:
+        typer.echo(
+            f"{result.run_id:38} {result.status:12} {result.run_git_hash:40} {result.current_hash:40}"
+        )
+        if result.status == "STALE":
+            stale_count += 1
+
+    # Check output files if requested
+    if check_outputs:
+        typer.secho("\n[Output File Status]", fg="cyan")
+        for result in results:
+            run = get_run(result.run_id, catalog_dir)
+            if run is None:
+                continue
+            output_results = check_output_files(run)
+            if output_results:
+                for out_result in output_results:
+                    status_color = "green" if out_result.status == "present" else "red"
+                    typer.secho(
+                        f"  {result.run_id}: {out_result.path} ({out_result.status})",
+                        fg=status_color,
+                    )
+            else:
+                typer.secho(f"  {result.run_id}: no output files", fg="dim")
+
+    # Exit with error if any STALE runs found
+    if stale_count > 0:
+        typer.echo()
+        typer.echo(f"Warning: {stale_count} stale run(s) detected", err=True)
+        raise typer.Exit(1)
+
+
+@app.command()
+def sync(
+    remote: str = typer.Argument(..., help="Remote name from .bth.toml"),
+    pull: bool = typer.Option(False, "--pull", help="Pull from remote (default: push)"),
+):
+    """Sync cool-tier catalog to/from remote."""
+    from bathos.config import find_project_config, load_project_config
+    from bathos.sync import sync_catalog
+
+    cfg_path = find_project_config()
+    if cfg_path is None:
+        typer.echo("No .bth.toml found. Run `bth init` first.", err=True)
+        raise typer.Exit(1)
+
+    config = load_project_config(cfg_path)
+    catalog_dir = _catalog_dir()
+
+    try:
+        result = sync_catalog(remote, config, catalog_dir, pull=pull)
+        direction = "pulled" if pull else "pushed"
+        typer.echo(
+            f"{direction.capitalize()} {result.transferred} runs to/from '{result.remote}' in {result.duration_s:.1f}s"
+        )
+    except ValueError as e:
+        typer.echo(str(e), err=True)
+        raise typer.Exit(1)
+    except RuntimeError as e:
+        typer.echo(str(e), err=True)
+        raise typer.Exit(1)
+
+
+def _parse_since(since: str | None) -> datetime | None:
+    if since is None:
+        return None
+    if since.endswith("d"):
+        return datetime.now(UTC) - timedelta(days=float(since[:-1]))
+    if since.endswith("h"):
+        return datetime.now(UTC) - timedelta(hours=float(since[:-1]))
+    return None