cosmosforge 0.1.0__py3-none-any.whl

cosmosforge/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

cosmosforge/client/cli.py

@@ -0,0 +1,105 @@
+"""Client CLI (`cf`)."""
+
+from __future__ import annotations
+
+import tarfile
+import tempfile
+from pathlib import Path
+
+import typer
+
+from cosmosforge.client import ssh
+from cosmosforge.client.config import CONFIG_PATH, HostConfig, load_config
+from cosmosforge.config import BUNDLES_DIR, RESULTS_DIR
+
+app = typer.Typer(no_args_is_help=True)
+
+
+@app.command()
+def init(
+    host: str = typer.Option(..., help="Server hostname or IP."),
+    user: str = typer.Option(..., help="SSH user on the server."),
+):
+    """Write the server connection config."""
+    CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    CONFIG_PATH.write_text(f'host = "{host}"\nuser = "{user}"\n')
+    typer.echo(f"config written to {CONFIG_PATH}")
+
+
+@app.command()
+def submit(
+    task_dir: Path = typer.Argument(..., help="Local harbor task directory."),
+    host: str = typer.Option(None),
+    user: str = typer.Option(None),
+):
+    """Upload a task bundle to the server."""
+    cfg = load_config(host, user)
+    from cosmosforge.core.structure import check_structure
+
+    errors = check_structure(task_dir)
+    if errors:
+        for e in errors:
+            typer.echo(f"structure error: {e}", err=True)
+        raise typer.Exit(1)
+
+    bundle_id = task_dir.resolve().name
+    remote_tar = f"{BUNDLES_DIR}/incoming/{bundle_id}.tar.gz"
+    with tempfile.TemporaryDirectory(prefix="cf-submit-") as tmp:
+        tar_path = Path(tmp) / f"{bundle_id}.tar.gz"
+        with tarfile.open(tar_path, "w:gz") as tar:
+            tar.add(task_dir, arcname=bundle_id)
+        rc = ssh.upload(cfg, tar_path, remote_tar)
+        if rc != 0:
+            raise typer.Exit(rc)
+    rc = ssh.run_remote(
+        cfg, f"mkdir -p {BUNDLES_DIR}/incoming && cf-server receive {bundle_id} {remote_tar}"
+    )
+    if rc == 0:
+        typer.echo(f"submitted bundle: {bundle_id}")
+    raise typer.Exit(rc)
+
+
+@app.command()
+def eval(
+    bundle_id: str = typer.Argument(..., help="Bundle ID from `cf submit`."),
+    host: str = typer.Option(None),
+    user: str = typer.Option(None),
+):
+    """Run oracle/nop validation on the server and print the report."""
+    cfg = load_config(host, user)
+    rc = ssh.run_remote(cfg, f"cf-server eval {bundle_id}")
+    raise typer.Exit(rc)
+
+
+@app.command()
+def status(
+    host: str = typer.Option(None),
+    user: str = typer.Option(None),
+):
+    """List bundles and results on the server."""
+    cfg = load_config(host, user)
+    rc = ssh.run_remote(
+        cfg,
+        f"ls -1 {BUNDLES_DIR} 2>/dev/null; echo ---; ls -1 {RESULTS_DIR} 2>/dev/null",
+    )
+    raise typer.Exit(rc)
+
+
+@app.command()
+def results(
+    bundle_id: str = typer.Argument(...),
+    host: str = typer.Option(None),
+    user: str = typer.Option(None),
+):
+    """Fetch and print the stored result JSON for a bundle."""
+    cfg = load_config(host, user)
+    rc = ssh.run_remote(cfg, f"cat {RESULTS_DIR}/{bundle_id}.json")
+    raise typer.Exit(rc)
+
+
+def main():
+    app()
+
+
+if __name__ == "__main__":
+    main()

cosmosforge/client/config.py

@@ -0,0 +1,37 @@
+"""Client-side connection config (~/.cosmosforge/config.toml)."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+CONFIG_PATH = Path.home() / ".cosmosforge" / "config.toml"
+
+
+@dataclass
+class HostConfig:
+    host: str
+    user: str
+
+    def ssh_target(self) -> str:
+        return f"{self.user}@{self.host}"
+
+
+def load_config(host: str | None = None, user: str | None = None) -> HostConfig:
+    host = host or os.environ.get("COSMOSFORGE_HOST")
+    user = user or os.environ.get("COSMOSFORGE_USER")
+    if (not host or not user) and CONFIG_PATH.is_file():
+        try:
+            import tomllib
+            data = tomllib.loads(CONFIG_PATH.read_text())
+        except Exception:
+            data = {}
+        host = host or data.get("host")
+        user = user or data.get("user")
+    if not host or not user:
+        raise SystemExit(
+            "No server configured. Run `cf init --host <host> --user <user>`, "
+            "or set COSMOSFORGE_HOST / COSMOSFORGE_USER."
+        )
+    return HostConfig(host=host, user=user)

cosmosforge/client/ssh.py

@@ -0,0 +1,23 @@
+"""Thin ssh/scp wrappers. Password auth passes through interactively."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+from cosmosforge.client.config import HostConfig
+
+
+def _run(cmd: list[str]) -> subprocess.CompletedProcess:
+    # stdin/stdout are inherited so ssh password prompts reach the user.
+    return subprocess.run(cmd, check=False)
+
+
+def run_remote(cfg: HostConfig, command: str) -> int:
+    proc = _run(["ssh", "-o", "BatchMode=no", cfg.ssh_target(), command])
+    return proc.returncode
+
+
+def upload(cfg: HostConfig, local_path: Path, remote_path: str) -> int:
+    proc = _run(["scp", str(local_path), f"{cfg.ssh_target()}:{remote_path}"])
+    return proc.returncode

cosmosforge/config.py

@@ -0,0 +1,14 @@
+import os
+from pathlib import Path
+
+SERVER_ROOT = Path(os.environ.get("COSMOSFORGE_ROOT", "/data/cosmosforge"))
+BUNDLES_DIR = SERVER_ROOT / "bundles"
+RESULTS_DIR = SERVER_ROOT / "results"
+
+
+def bundle_dir(bundle_id: str) -> Path:
+    return BUNDLES_DIR / bundle_id
+
+
+def result_path(bundle_id: str) -> Path:
+    return RESULTS_DIR / f"{bundle_id}.json"

cosmosforge/core/oracle_nop.py

@@ -0,0 +1,108 @@
+"""Oracle/nop validation via the harbor framework.
+
+Runs both agents in a single harbor job using a job config
+(`harbor run -c`), then parses the job's result.json with harbor's own
+models to get per-agent rewards.
+"""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import yaml
+
+from cosmosforge.core.report import RunResult
+from cosmosforge.core.structure import check_structure
+
+EXPECTED_REWARDS = {"oracle": 1.0, "nop": 0.0}
+
+
+def _write_job_config(task_dir: Path, config_path: Path) -> None:
+    config = {
+        "path": str(task_dir),
+        "agents": [{"name": name} for name in EXPECTED_REWARDS],
+        "job_name": "cosmosforge-eval",
+        "n_concurrent": 1,
+        "quiet": True,
+    }
+    config_path.write_text(yaml.safe_dump(config))
+
+
+def _mean_reward(trial_result) -> float | None:
+    rewards = trial_result.verifier_result.rewards if trial_result.verifier_result else None
+    if not rewards:
+        return None
+    return sum(float(v) for v in rewards.values()) / len(rewards)
+
+
+def run_oracle_nop(task_dir: Path, jobs_dir: Path) -> tuple[list[RunResult], str | None]:
+    """Run oracle/nop in one harbor job. Returns (runs, error)."""
+    from harbor.models.job.result import JobResult
+
+    jobs_dir.mkdir(parents=True, exist_ok=True)
+    config_path = jobs_dir / "job-config.yaml"
+    _write_job_config(task_dir, config_path)
+
+    cmd = [
+        "harbor", "run",
+        "-c", str(config_path),
+        "-o", str(jobs_dir),
+    ]
+    proc = subprocess.run(cmd, capture_output=True, text=True, timeout=3600, check=False)
+
+    result_path = jobs_dir / "cosmosforge-eval" / "result.json"
+    if not result_path.exists():
+        return [], f"harbor run produced no result.json (rc={proc.returncode}): {proc.stderr[-2000:]}"
+
+    job_result = JobResult.model_validate_json(result_path.read_text())
+
+    runs: dict[str, RunResult] = {}
+    for trial in job_result.trial_results:
+        agent = trial.agent_info.name
+        reward = _mean_reward(trial)
+        error = None
+        if trial.exception_info is not None:
+            error = f"{trial.exception_info.exception_type}: {trial.exception_info.exception_message}"
+        entry = runs.setdefault(agent, RunResult(agent=agent, reward=None, passed=False))
+        if reward is not None:
+            entry.reward = reward
+        if error and not entry.error:
+            entry.error = error
+
+    missing = [a for a in EXPECTED_REWARDS if a not in runs]
+    for agent in missing:
+        runs[agent] = RunResult(agent=agent, reward=None, passed=False,
+                                error="no trial result for this agent")
+
+    error = None
+    if proc.returncode != 0:
+        error = f"harbor run exited with rc={proc.returncode}"
+    return [runs[a] for a in EXPECTED_REWARDS], error
+
+
+def evaluate_oracle_nop(bundle_id: str, task_dir: Path, jobs_dir: Path):
+    from cosmosforge.core.report import EvaluationReport
+
+    errors = check_structure(task_dir)
+    report = EvaluationReport(
+        bundle_id=bundle_id,
+        task_name=task_dir.name,
+        structure_ok=not errors,
+        structure_errors=errors,
+    )
+    if errors:
+        return report
+
+    runs, error = run_oracle_nop(task_dir, jobs_dir)
+    report.runs = runs
+    for run in runs:
+        expected = EXPECTED_REWARDS.get(run.agent)
+        if run.reward == expected and run.error is None:
+            run.passed = True
+        else:
+            run.passed = False
+            if run.reward != expected and run.error is None:
+                run.error = f"expected reward {expected}, got {run.reward}"
+    report.passed = error is None and all(r.passed for r in report.runs)
+    return report

cosmosforge/core/report.py

@@ -0,0 +1,33 @@
+"""Evaluation report models shared by client and server."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+
+
+@dataclass
+class RunResult:
+    agent: str  # "oracle" | "nop"
+    reward: float | None
+    passed: bool
+    error: str | None = None
+
+
+@dataclass
+class EvaluationReport:
+    bundle_id: str
+    task_name: str
+    structure_ok: bool
+    structure_errors: list[str] = field(default_factory=list)
+    runs: list[RunResult] = field(default_factory=list)
+    passed: bool = False
+
+    def to_json(self) -> str:
+        return json.dumps(asdict(self), indent=2)
+
+    @classmethod
+    def from_json(cls, text: str) -> "EvaluationReport":
+        data = json.loads(text)
+        data["runs"] = [RunResult(**r) for r in data["runs"]]
+        return cls(**data)

cosmosforge/core/structure.py

@@ -0,0 +1,23 @@
+"""Harbor task structure validation."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+REQUIRED_FILES = [
+    "task.toml",
+    "instruction.md",
+    "environment/Dockerfile",
+    "solution/solve.sh",
+    "tests/test.sh",
+]
+
+
+def check_structure(task_dir: Path) -> list[str]:
+    errors = []
+    if not task_dir.is_dir():
+        return [f"not a directory: {task_dir}"]
+    for rel in REQUIRED_FILES:
+        if not (task_dir / rel).is_file():
+            errors.append(f"missing required file: {rel}")
+    return errors

cosmosforge/server/cli.py

@@ -0,0 +1,69 @@
+"""Server-side CLI (`cf-server`), invoked over SSH by the client."""
+
+from __future__ import annotations
+
+import shutil
+import tempfile
+from pathlib import Path
+
+import typer
+
+from cosmosforge import config
+from cosmosforge.core.oracle_nop import evaluate_oracle_nop
+from cosmosforge.server import store
+
+app = typer.Typer(no_args_is_help=True)
+
+INCOMING_DIR = config.BUNDLES_DIR / "incoming"
+
+
+@app.command()
+def receive(
+    bundle_id: str = typer.Argument(..., help="Bundle ID (directory name under bundles/)."),
+    tar_path: Path = typer.Argument(..., help="Path to uploaded tar.gz."),
+):
+    """Extract an uploaded bundle and validate its structure."""
+    task_dir = store.extract_bundle(tar_path, bundle_id)
+    from cosmosforge.core.structure import check_structure
+
+    errors = check_structure(task_dir)
+    if errors:
+        for e in errors:
+            typer.echo(f"structure error: {e}", err=True)
+        shutil.rmtree(config.bundle_dir(bundle_id))
+        raise typer.Exit(1)
+    typer.echo(f"bundle {bundle_id} received, task dir: {task_dir}")
+
+
+@app.command()
+def eval(
+    bundle_id: str = typer.Argument(..., help="Bundle ID to evaluate."),
+):
+    """Run oracle/nop validation and store the report JSON."""
+    task_dir = _find_task_dir(bundle_id)
+    with tempfile.TemporaryDirectory(prefix="cf-jobs-") as jobs_dir:
+        report = evaluate_oracle_nop(bundle_id, task_dir, Path(jobs_dir))
+    out = store.save_result(bundle_id, report.to_json())
+    typer.echo(report.to_json())
+    typer.echo(f"report saved to {out}", err=True)
+    if not report.passed:
+        raise typer.Exit(1)
+
+
+def _find_task_dir(bundle_id: str) -> Path:
+    bundle = config.bundle_dir(bundle_id)
+    if not bundle.is_dir():
+        typer.echo(f"bundle not found: {bundle_id}", err=True)
+        raise typer.Exit(1)
+    entries = [p for p in bundle.iterdir() if p.is_dir()]
+    if len(entries) == 1:
+        return entries[0]
+    return bundle
+
+
+def main():
+    app()
+
+
+if __name__ == "__main__":
+    main()

cosmosforge/server/store.py

@@ -0,0 +1,33 @@
+"""Server-side bundle/results storage."""
+
+from __future__ import annotations
+
+import tarfile
+from pathlib import Path
+
+from cosmosforge import config
+
+
+def extract_bundle(tar_path: Path, bundle_id: str) -> Path:
+    dest = config.bundle_dir(bundle_id)
+    dest.mkdir(parents=True, exist_ok=True)
+    with tarfile.open(tar_path, "r:gz") as tar:
+        tar.extractall(dest, filter="data")
+    # Support both "bundle-id/task-dir/..." and bare "task-dir/..." layouts.
+    entries = [p for p in dest.iterdir() if not p.name.startswith(".")]
+    if len(entries) == 1 and entries[0].is_dir():
+        return entries[0]
+    return dest
+
+
+def save_result(bundle_id: str, report_json: str) -> Path:
+    config.RESULTS_DIR.mkdir(parents=True, exist_ok=True)
+    out = config.result_path(bundle_id)
+    out.write_text(report_json)
+    return out
+
+
+def list_bundles() -> list[str]:
+    if not config.BUNDLES_DIR.exists():
+        return []
+    return sorted(p.name for p in config.BUNDLES_DIR.iterdir() if p.is_dir())

cosmosforge-0.1.0.dist-info/METADATA

@@ -0,0 +1,11 @@
+Metadata-Version: 2.5
+Name: cosmosforge
+Version: 0.1.0
+Summary: CLI and evaluation service for Harbor-format terminal tasks.
+Requires-Python: >=3.10
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: typer>=0.12
+Description-Content-Type: text/markdown
+
+# CosmosForge

cosmosforge-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+cosmosforge/__init__.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
+cosmosforge/config.py,sha256=wmUZnKCMJIPzve3BQuk6skRYCWXpj0jBOXjlaNYx0uo,353
+cosmosforge/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cosmosforge/client/cli.py,sha256=tZtjWCZ-mxudmFpCeXRSnOXWXbwabVHTmtpILpIDWzI,2974
+cosmosforge/client/config.py,sha256=rcxM0DnQqtBmeVHDpOxpbs_5emPeV2A6yE1ST5A5Its,1088
+cosmosforge/client/ssh.py,sha256=RQFwY0cBlA5oKbhy_L_F7HhTCU1E86fM-RAmid_1rpA,716
+cosmosforge/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cosmosforge/core/oracle_nop.py,sha256=xWq1RJtsmWUGzZL5Dm7S8ZCKy2zATYjrVzNsQAb7Dzk,3685
+cosmosforge/core/report.py,sha256=weP8SpSqNNbz74lKh3Z7QFQ-_g1jLa9k0W6f1SW4WaY,812
+cosmosforge/core/structure.py,sha256=CL_9Iu5ZPxbei1sh6nDbR-YL03snhWpxtI55KpPRQzU,536
+cosmosforge/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cosmosforge/server/cli.py,sha256=luTe8Z9bWHOXV0RDi7vxx5s6aKNeknYXqasNPiRQClM,2006
+cosmosforge/server/store.py,sha256=_CaJui4cFrdKvCU8CaxyktgmOrrIpv5y4tNaOu4N03Y,1007
+cosmosforge-0.1.0.dist-info/METADATA,sha256=VBE8X8iZ2uRe75GGbT5YH5LSSTj7Id0ItM8ezwI-4rs,284
+cosmosforge-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+cosmosforge-0.1.0.dist-info/entry_points.txt,sha256=hAIdwh6vamn8V5VHJCZso8PlrcA6UR3sYww_xhmDx8Y,89
+cosmosforge-0.1.0.dist-info/RECORD,,

cosmosforge-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

cosmosforge-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+cf = cosmosforge.client.cli:app
+cf-server = cosmosforge.server.cli:app