clawbench-cli 0.1.2__py3-none-any.whl

clawbench/hf_upload.py

@@ -0,0 +1,78 @@
+"""Optional HuggingFace dataset upload for ClawBench runs."""
+
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def hf_upload_enabled(env: dict[str, str]) -> bool:
+    """Check if HF_TOKEN and HF_REPO_ID are configured."""
+    return bool(env.get("HF_TOKEN")) and bool(env.get("HF_REPO_ID"))
+
+
+def upload_run(output_dir: Path, repo_path_prefix: str, env: dict[str, str]) -> None:
+    """Upload a run's output directory to HuggingFace, then replace local data/ with a marker.
+
+    Args:
+        output_dir: Local output path (contains run-meta.json, data/).
+        repo_path_prefix: Path inside the HF repo, e.g. "model/case-model-ts".
+        env: Dict with HF_TOKEN and HF_REPO_ID.
+    """
+    try:
+        from huggingface_hub import HfApi
+    except ImportError:
+        print("  WARNING: huggingface_hub not installed, skipping upload")
+        return
+
+    token = env["HF_TOKEN"]
+    repo_id = env["HF_REPO_ID"]
+    api = HfApi(token=token)
+
+    try:
+        commit_info = api.upload_folder(
+            folder_path=str(output_dir),
+            repo_id=repo_id,
+            repo_type="dataset",
+            path_in_repo=repo_path_prefix,
+            ignore_patterns=[".my-info-tmp/**"],
+            commit_message=f"Add run: {repo_path_prefix}",
+        )
+        commit_url = getattr(commit_info, "commit_url", None) or ""
+        print(f"  Uploaded to HF: {repo_id}/{repo_path_prefix}")
+
+        # Replace local data/ with a lightweight marker
+        marker = {
+            "repo_id": repo_id,
+            "path_in_repo": repo_path_prefix,
+            "commit_url": commit_url,
+            "uploaded_at": datetime.now(timezone.utc).isoformat(),
+        }
+        (output_dir / "uploaded.json").write_text(json.dumps(marker, indent=2))
+
+    except Exception as e:
+        print(f"  WARNING: HuggingFace upload failed: {e}")
+
+
+def upload_file(local_path: Path, path_in_repo: str, env: dict[str, str]) -> None:
+    """Upload a single file to HuggingFace (e.g. batch-summary.json)."""
+    try:
+        from huggingface_hub import HfApi
+    except ImportError:
+        print("  WARNING: huggingface_hub not installed, skipping upload")
+        return
+
+    token = env["HF_TOKEN"]
+    repo_id = env["HF_REPO_ID"]
+    api = HfApi(token=token)
+
+    try:
+        api.upload_file(
+            path_or_fileobj=str(local_path),
+            path_in_repo=path_in_repo,
+            repo_id=repo_id,
+            repo_type="dataset",
+            commit_message=f"Add {path_in_repo}",
+        )
+        print(f"  Uploaded to HF: {repo_id}/{path_in_repo}")
+    except Exception as e:
+        print(f"  WARNING: HuggingFace upload failed: {e}")

clawbench/image.py

@@ -0,0 +1,127 @@
+"""Container image lifecycle helpers.
+
+The single responsibility here is answering the question: *is the
+``clawbench`` image available locally and is it the right version?* —
+pulling from the registry when it isn't, and falling back to a local
+build when pull fails (offline, rate-limited, arch mismatch).
+
+Why pull-first, build-fallback:
+
+- A first-time ``docker build`` takes 5-10 minutes on a fresh system.
+  For users who just typed ``pip install claw-bench``, that is an awful
+  first impression. A prebuilt image on GHCR is an order of magnitude
+  faster and already exists on the release pipeline.
+- But pulls can fail in ways builds cannot (behind an enterprise proxy,
+  no GHCR auth, unsupported arch). Silently falling back to build keeps
+  the package usable in those environments instead of hard-erroring.
+
+Version-label check:
+
+- The release CI tags images with ``LABEL org.clawbench.version=<v>``
+  matching the pypi version. We warn loudly (but keep going) if the
+  local image's label diverges from ``clawbench.__version__`` — the
+  single most common post-release footgun is "works locally because I
+  have a stale hand-built image that nobody else has."
+"""
+
+from __future__ import annotations
+
+import subprocess
+
+from clawbench import __version__
+from clawbench.engine import detect_engine
+
+IMAGE_NAME = "clawbench"
+REGISTRY_REF = "ghcr.io/reacher-z/claw-bench"
+VERSION_LABEL = "org.clawbench.version"
+
+
+def _engine_or_fail() -> str:
+    eng = detect_engine()
+    if eng is None:
+        raise RuntimeError(
+            "No container engine (podman or docker) found on PATH. "
+            "Install podman: https://podman.io/docs/installation"
+        )
+    return eng
+
+
+def image_exists(engine: str | None = None, ref: str = IMAGE_NAME) -> bool:
+    """Return True if ``ref`` is present in the local image store."""
+    eng = engine or _engine_or_fail()
+    return subprocess.run(
+        [eng, "image", "inspect", ref],
+        capture_output=True,
+    ).returncode == 0
+
+
+def image_label(engine: str | None = None, ref: str = IMAGE_NAME) -> str | None:
+    """Return the ``org.clawbench.version`` label from the local image,
+    or ``None`` if the image isn't present or has no label."""
+    eng = engine or _engine_or_fail()
+    r = subprocess.run(
+        [eng, "image", "inspect", "--format",
+         "{{ index .Config.Labels \"" + VERSION_LABEL + "\" }}", ref],
+        capture_output=True, text=True,
+    )
+    if r.returncode != 0:
+        return None
+    label = r.stdout.strip()
+    return label or None
+
+
+def pull_image(
+    engine: str | None = None,
+    tag: str | None = None,
+) -> tuple[bool, str]:
+    """Attempt to pull ``ghcr.io/reacher-z/claw-bench:<tag>`` and retag it
+    locally as ``clawbench`` so the rest of the code keeps working.
+
+    Returns ``(success, detail)``. ``detail`` is a diagnostic string with
+    the pull command's stderr on failure, empty on success.
+
+    ``tag`` defaults to the installed package version; callers that want
+    ``:latest`` explicitly can pass it.
+    """
+    eng = engine or _engine_or_fail()
+    use_tag = tag or __version__
+    ref = f"{REGISTRY_REF}:{use_tag}"
+    r = subprocess.run(
+        [eng, "pull", ref],
+        capture_output=True, text=True,
+    )
+    if r.returncode != 0:
+        return False, r.stderr.strip() or r.stdout.strip()
+    # Retag so the existing run.py / tui.py code paths that say
+    # ``clawbench`` (un-prefixed) keep working.
+    tag_r = subprocess.run(
+        [eng, "tag", ref, IMAGE_NAME],
+        capture_output=True, text=True,
+    )
+    if tag_r.returncode != 0:
+        return False, tag_r.stderr.strip()
+    return True, ""
+
+
+def verify_image_version(engine: str | None = None) -> tuple[bool, str]:
+    """Check whether the local image's version label matches the installed
+    wheel's version. Returns ``(matches, detail)``:
+
+    - ``(True, "")``   when the label equals ``__version__`` (or when the
+      image has no label at all — we treat unlabeled legacy images as OK
+      since they predate this scheme and warning on them would be noisy
+      for existing users).
+    - ``(False, msg)`` when labels mismatch; ``msg`` is user-facing.
+    """
+    eng = engine or _engine_or_fail()
+    if not image_exists(eng):
+        return False, f"image '{IMAGE_NAME}' not present locally"
+    label = image_label(eng)
+    if label is None:
+        return True, ""  # legacy image, no label — accept
+    if label == __version__:
+        return True, ""
+    return False, (
+        f"image version label '{label}' != package version '{__version__}'. "
+        f"Consider `claw-bench build --no-cache` to rebuild."
+    )

clawbench/paths.py

@@ -0,0 +1,150 @@
+"""Path helpers for the installed package.
+
+Three kinds of locations:
+
+1. **Bundled read-only data** inside the wheel — test cases, chrome extension,
+   dockerfile set, personal-info templates. Always accessed via
+   :func:`bundled_data_dir` (returns a real ``Path`` so it can be handed to
+   subprocess / ``docker build`` without further juggling).
+
+2. **User config** — per-user mutable state. Chosen via :mod:`platformdirs` so
+   macOS gets ``~/Library/Application Support/claw-bench`` and Linux gets
+   ``~/.config/claw-bench`` (respecting ``XDG_CONFIG_HOME``). Contains
+   ``models.yaml``, ``config.json``, optional ``secrets.env``.
+
+3. **Output directory** — where run artifacts land. Defaults to
+   ``./claw-output/`` in the caller's current directory, overridable via
+   ``--output-dir`` or ``CLAWBENCH_OUTPUT_DIR``.
+
+We also migrate from the pre-package legacy dir ``~/.config/clawbench/`` on
+first access so users coming from source installs keep their preferences.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+from importlib import resources
+from pathlib import Path
+
+from platformdirs import PlatformDirs
+
+_APP_NAME = "claw-bench"
+_LEGACY_CONFIG_DIR = Path.home() / ".config" / "clawbench"
+
+_dirs = PlatformDirs(_APP_NAME, appauthor=False)
+
+
+def bundled_data_dir() -> Path:
+    """Return the on-disk path to read-only bundled assets.
+
+    Uses ``importlib.resources.files("clawbench")`` which resolves to a real
+    filesystem path when the package is installed normally (wheel or editable).
+    We need a real ``Path`` rather than a ``Traversable`` because the
+    ``docker build`` context and ``--load-extension`` need a real directory.
+    """
+    root = resources.files("clawbench") / "data"
+    # ``files()`` returns a MultiplexedPath in rare cases (namespace packages);
+    # for single-package layouts it yields a PosixPath/WindowsPath directly.
+    return Path(str(root))
+
+
+def test_cases_dir() -> Path:
+    return bundled_data_dir() / "test-cases"
+
+
+def chrome_extension_dir() -> Path:
+    return bundled_data_dir() / "chrome-extension"
+
+
+def extension_server_dir() -> Path:
+    return bundled_data_dir() / "extension-server"
+
+
+def shared_dir() -> Path:
+    return bundled_data_dir() / "shared"
+
+
+def docker_build_dir() -> Path:
+    """Directory containing Dockerfile + entrypoint.sh + setup-openclaw.sh."""
+    return bundled_data_dir() / "docker"
+
+
+def bundled_models_yaml() -> Path:
+    """Seed template copied into the user config dir on first run.
+
+    We intentionally ship the *example* file, not the developer's live
+    ``models.yaml``. The live file in the repo may contain real API keys
+    (OpenRouter et al.) committed for local convenience — those must not
+    land on PyPI where every wheel is permanently indexed."""
+    return bundled_data_dir() / "models" / "models.example.yaml"
+
+
+def user_config_dir() -> Path:
+    """Platform-appropriate per-user config directory (created if missing)."""
+    d = Path(_dirs.user_config_dir)
+    d.mkdir(parents=True, exist_ok=True)
+    _migrate_legacy_config(d)
+    return d
+
+
+def user_models_yaml() -> Path:
+    """Path to the user's editable models config. Seeded from the bundled
+    template on first access so the file always exists for the TUI editor."""
+    dst = user_config_dir() / "models.yaml"
+    if not dst.exists():
+        src = bundled_models_yaml()
+        if src.exists():
+            shutil.copyfile(src, dst)
+        else:
+            dst.write_text("# ClawBench models.yaml\nmodels: {}\n", encoding="utf-8")
+    return dst
+
+
+def user_config_json() -> Path:
+    """TUI preferences (theme, last-used options)."""
+    return user_config_dir() / "config.json"
+
+
+def user_secrets_path() -> Path:
+    """Optional persisted secrets file (PURELYMAIL_API_KEY etc).
+
+    Not created automatically — the CLI's ``configure --secrets`` writes it
+    with chmod 600. ``run`` / ``batch`` load it via python-dotenv if present.
+    """
+    return user_config_dir() / "secrets.env"
+
+
+def default_output_dir() -> Path:
+    """Default run output directory.
+
+    Order of precedence:
+    1. ``CLAWBENCH_OUTPUT_DIR`` environment variable.
+    2. ``./claw-output`` in the caller's current working directory.
+    """
+    env = os.environ.get("CLAWBENCH_OUTPUT_DIR")
+    if env:
+        return Path(env).expanduser().resolve()
+    return Path.cwd() / "claw-output"
+
+
+def _migrate_legacy_config(new_dir: Path) -> None:
+    """One-shot migration from ``~/.config/clawbench/`` to the platformdirs
+    location. Copies files that don't already exist at the new location and
+    leaves the legacy dir alone so source installs keep working."""
+    if not _LEGACY_CONFIG_DIR.is_dir() or new_dir == _LEGACY_CONFIG_DIR:
+        return
+    for name in ("tui.json", "config.json", "models.yaml"):
+        src = _LEGACY_CONFIG_DIR / name
+        if not src.exists():
+            continue
+        # Normalize legacy tui.json filename to config.json going forward.
+        dst_name = "config.json" if name == "tui.json" else name
+        dst = new_dir / dst_name
+        if dst.exists():
+            continue
+        try:
+            shutil.copyfile(src, dst)
+        except OSError:
+            # Migration is best-effort; the CLI still works without it.
+            pass

clawbench/resume_template.json

@@ -0,0 +1,104 @@
+{
+  "// NOTE": "All overlapping fields MUST match personal_info.json exactly.",
+  "header": {
+    "name": "Alex Green",
+    "title": "Senior Software Engineer",
+    "email": "dummy_email",
+    "location": "Toronto, ON, Canada"
+  },
+  "summary": "Senior Software Engineer with 23+ years of experience in full-stack development, distributed systems, and cloud infrastructure. PhD in Computer Science from the University of Toronto. Currently leading a backend team at Pinecrest Technologies Inc., building enterprise data pipeline solutions. Previously built real-time transaction processing systems in FinTech. AWS and Kubernetes certified.",
+  "experience": [
+    {
+      "title": "Senior Software Engineer",
+      "company": "Pinecrest Technologies Inc.",
+      "location": "Toronto, ON",
+      "dates": "Mar 2019 – Present",
+      "bullets": [
+        "Lead backend team of 5 engineers building distributed data pipelines for enterprise SaaS platform",
+        "Design and implement RESTful APIs serving 2M+ daily requests with sub-100ms p99 latency",
+        "Mentor junior developers and conduct code reviews, improving team velocity by 30%"
+      ]
+    },
+    {
+      "title": "Software Engineer",
+      "company": "Crestridge Digital Corp.",
+      "location": "Toronto, ON",
+      "dates": "Jun 2012 - Feb 2019",
+      "bullets": [
+        "Developed real-time transaction processing systems handling $50M+ daily volume in FinTech",
+        "Built automated testing frameworks reducing QA cycle by 40%",
+        "Collaborated with product team on mobile banking features serving 500K+ users"
+      ]
+    },
+    {
+      "title": "Software Developer",
+      "company": "Cedarbrook Solutions Ltd.",
+      "location": "Toronto, ON",
+      "dates": "Sep 2002 - May 2012",
+      "bullets": [
+        "Full-stack web development for enterprise clients across multiple industries",
+        "Database administration and performance optimization for high-traffic applications",
+        "Part-time during graduate studies (2002-2010); full-time from 2010"
+      ]
+    }
+  ],
+  "education": [
+    {
+      "degree": "Ph.D. in Computer Science",
+      "institution": "University of Toronto",
+      "dates": "2004 - 2010",
+      "detail": "Dissertation: Scalable Real-Time Data Pipeline Architectures for High-Throughput Transaction Processing"
+    },
+    {
+      "degree": "M.Sc. in Computer Science",
+      "institution": "University of Toronto",
+      "dates": "2002 - 2004",
+      "detail": "GPA: 3.8/4.0 | Thesis: Efficient Query Processing in Distributed Database Systems"
+    },
+    {
+      "degree": "B.Sc. in Computer Science",
+      "institution": "University of Toronto",
+      "dates": "1998 - 2002",
+      "detail": "GPA: 3.6/4.0 | Dean's List (2001, 2002)"
+    }
+  ],
+  "skills": {
+    "languages": [
+      "Python",
+      "Java",
+      "TypeScript",
+      "Go"
+    ],
+    "databases": [
+      "PostgreSQL",
+      "Redis"
+    ],
+    "cloud_devops": [
+      "AWS",
+      "Docker",
+      "Kubernetes",
+      "Terraform",
+      "CI/CD"
+    ],
+    "frameworks": [
+      "React",
+      "Node.js",
+      "GraphQL",
+      "REST API Design"
+    ]
+  },
+  "certifications": [
+    "AWS Solutions Architect – Associate (2024)",
+    "Certified Kubernetes Administrator – CKA (2025)"
+  ],
+  "languages": [
+    {
+      "language": "English",
+      "proficiency": "Native"
+    },
+    {
+      "language": "French",
+      "proficiency": "Intermediate (B1)"
+    }
+  ]
+}