clousight-bench 0.2.0__tar.gz

clousight_bench-0.2.0/.dockerignore

@@ -0,0 +1,37 @@
+# Exclude large/irrelevant paths from the Docker build context.
+# The cb-probe image needs: pyproject.toml, README.md, src/
+# Everything else can be excluded to keep the context small.
+
+.git
+.venv
+venv
+__pycache__
+*.pyc
+*.pyo
+.pytest_cache
+.mypy_cache
+.ruff_cache
+.coverage
+htmlcov/
+
+# Benchmark results and run artifacts
+results/
+_vendor_cache/
+
+# Tests (not needed for pip install)
+tests/
+
+# Superpowers / dev tooling
+.superpowers/
+
+# Deploy infra (not needed inside the image)
+deploy/
+
+# Docs
+docs/
+site/
+
+# IDE / OS
+.idea/
+.vscode/
+*.DS_Store

clousight_bench-0.2.0/.github/CODEOWNERS

@@ -0,0 +1,16 @@
+# Code owners for Clousight Bench.
+# These owners are requested for review on every pull request that touches
+# matching paths. See https://docs.github.com/articles/about-code-owners
+
+# Default owner for everything in the repo.
+*                       @clousight-dev
+
+# Result schema, plugin API and lifecycle are the load-bearing contracts —
+# changes here ripple into every plugin and into clousight-bench-pro.
+/src/clousight_bench/core/schema.py     @clousight-dev
+/src/clousight_bench/core/plugin.py     @clousight-dev
+/src/clousight_bench/core/orchestrator.py   @clousight-dev
+
+# Packaging, CI and community health.
+/pyproject.toml         @clousight-dev
+/.github/               @clousight-dev

clousight_bench-0.2.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,54 @@
+name: Bug report
+description: Report something that is broken or produces a wrong / non-reproducible result.
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for the report. For **security** issues, do not use this form —
+        see [SECURITY.md](https://github.com/clousight/clousight-bench/blob/main/SECURITY.md).
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: What did you expect, and what happened instead?
+    validations:
+      required: true
+  - type: textarea
+    id: repro
+    attributes:
+      label: Steps to reproduce
+      description: The exact `csbench` command(s) and any config. Redact secrets.
+      placeholder: |
+        csbench run --domain agent-runtime --task T1.3 --platform local-sim
+      render: shell
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: Output of `csbench --version` (or the installed `clousight-bench` version).
+    validations:
+      required: true
+  - type: input
+    id: python
+    attributes:
+      label: Python version
+      placeholder: "3.12"
+    validations:
+      required: true
+  - type: input
+    id: platform-adapter
+    attributes:
+      label: Adapter / platform
+      description: e.g. local-sim, local-process, aliyun-agentrun (mock)
+  - type: textarea
+    id: result-record
+    attributes:
+      label: Result record / logs
+      description: >
+        If a run produced a wrong number, paste the relevant `ResultRecord`
+        JSON — especially `config_hash`, `runner_version`, `evidence_layer`.
+        Redact any secrets.
+      render: json

clousight_bench-0.2.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Security vulnerability
+    url: https://github.com/clousight/clousight-bench/security/advisories/new
+    about: Do NOT open a public issue for security problems. Report privately via GitHub Security Advisories.
+  - name: Question / discussion
+    url: https://github.com/clousight/clousight-bench/discussions
+    about: Ask usage questions or discuss ideas here rather than opening an issue.

clousight_bench-0.2.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,40 @@
+name: Feature request
+description: Propose a new platform, dimension, domain pack, or capability.
+labels: ["enhancement"]
+body:
+  - type: dropdown
+    id: kind
+    attributes:
+      label: What kind of addition is this?
+      options:
+        - New platform (ProviderAdapter)
+        - New dimension (Task)
+        - New domain pack
+        - New workload / load generator
+        - Core capability
+        - Other
+    validations:
+      required: true
+  - type: textarea
+    id: problem
+    attributes:
+      label: What problem does it solve?
+      description: What can't you measure or reproduce today?
+    validations:
+      required: true
+  - type: textarea
+    id: proposal
+    attributes:
+      label: Proposed approach
+      description: >
+        If it is a dimension, what does it measure and what evidence layer
+        (A/B/C/D) would it carry? If a platform, which cloud/service and what is
+        its own runtime behavior you want surfaced?
+  - type: checkboxes
+    id: principles
+    attributes:
+      label: Fits the project principles
+      options:
+        - label: Reports per-dimension (no blended score) and carries an evidence layer
+        - label: Does not require secrets in configs/results
+        - label: I'm willing to help implement / test it

clousight_bench-0.2.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,26 @@
+<!-- Thanks for contributing to Clousight Bench! -->
+
+## What & why
+
+<!-- What does this change do, and why? Link any related issue: Closes #123 -->
+
+## Type of change
+
+- [ ] Bug fix
+- [ ] New platform (one `ProviderAdapter` + one `configs/*.example.yaml`)
+- [ ] New dimension (one `Task` with `config()`, scoring, declared `evidence_layer`)
+- [ ] New domain pack / workload
+- [ ] Docs / packaging / CI
+- [ ] Other:
+
+## Checklist
+
+- [ ] Commits are signed off (`git commit -s`) per the [DCO](https://developercertificate.org/)
+- [ ] `ruff check src tests` passes
+- [ ] `mypy src` passes
+- [ ] `pytest -q` passes
+- [ ] Ran the local no-cloud smoke (`csbench run --domain agent-runtime --task T1.3 --platform local-sim`)
+- [ ] If packaging changed: built the wheel and ran the smoke **outside** the checkout
+- [ ] New adapters declare a status (`reference` / `experimental` / `wired` / `skeleton`); no skeleton is presented as runnable
+- [ ] Every result still carries `config_hash` + `runner_version` + `evidence_layer`; no blended scores; no secrets in configs/results
+- [ ] Changing task/scoring for a **shipped** dimension: bumped the version and added a `CHANGELOG.md` entry

clousight_bench-0.2.0/.github/dependabot.yml

@@ -0,0 +1,25 @@
+version: 2
+updates:
+  # Python dependencies declared in pyproject.toml.
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      python-deps:
+        patterns: ["*"]
+    labels: ["dependencies"]
+    commit-message:
+      prefix: "deps"
+
+  # GitHub Actions used by the CI / release workflows.
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      actions:
+        patterns: ["*"]
+    labels: ["dependencies", "ci"]
+    commit-message:
+      prefix: "ci"

clousight_bench-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,216 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+# Cancel superseded runs on the same ref (e.g. a new push to an open PR) so we
+# don't burn runners finishing a run whose result no longer matters.
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      # Don't cancel the whole matrix when one cell fails — a single flaky cell
+      # shouldn't hide the other versions' results or block the PR by itself.
+      # (The 4 cells run in parallel, so this costs no extra wall-clock.)
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev,store,validate,probe,agent,aliyun]"
+
+      - name: Lint (ruff)
+        run: ruff check src tests
+
+      - name: Format check (ruff)
+        run: ruff format --check src tests
+
+      - name: Type-check (mypy)
+        run: mypy
+
+      - name: Docs inventory up to date
+        run: python scripts/gen_docs.py --check
+
+      - name: Test (pytest)
+        # --durations surfaces the slowest tests in the log so CI-time regressions
+        # (e.g. a test doing real network I/O with long timeouts) are easy to spot.
+        run: pytest -q --durations=15 --cov --cov-report=term-missing --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        if: matrix.python-version == '3.12'
+        continue-on-error: true  # coverage upload is best-effort; a Codecov token/outage must never gate merges
+        uses: codecov/codecov-action@v7
+        with:
+          files: coverage.xml
+          fail_ci_if_error: false
+
+      - name: Conformance smoke (built-in domains)
+        run: |
+          csbench conformance --domain agent-runtime --platform local-sim
+          csbench conformance --domain bigdata-emr --platform local-process
+
+      - name: Local baseline smoke (no cloud account)
+        run: |
+          csbench list --verbose
+          for task in T1.1 T1.2 T1.3 T2.1 T4.1 T4.2 T5.1 T5.2; do
+            csbench run --domain agent-runtime --task "$task" --platform local-sim
+          done
+          csbench run --domain agent-runtime --task T1.3 --platform local-sim \
+            --config configs/local-sim.fail-fast.yaml
+          csbench run --domain bigdata-emr --task J1.1 --platform local-process
+          csbench report
+
+      - name: Assert every emitted record is schema 0.2
+        run: |
+          python - <<'PY'
+          import json, pathlib
+          paths = [p for p in pathlib.Path("results").rglob("*.json")
+                   if p.name not in {"comparison.json", "migration-manifest.json"}]
+          assert paths, "the smoke wrote no records"
+          for path in paths:
+              data = json.loads(path.read_text(encoding="utf-8"))
+              assert data["schema_version"] == "0.2", (path, data["schema_version"])
+              assert data["status"] in {"completed", "unsupported"}, (path, data["status"])
+              assert data["fingerprints"]["record_digest"].startswith("sha256:"), path
+              for gone in ("ok", "metrics", "evidence_layer", "config_hash"):
+                  assert gone not in data, (path, gone)
+          print(f"{len(paths)} records verified at schema 0.2")
+          PY
+
+      - name: Migration round-trip from schema 1.0
+        run: |
+          rm -rf /tmp/legacy /tmp/migrated /tmp/migrated-again
+          mkdir -p /tmp/legacy/agent-runtime/local-sim
+          cat > /tmp/legacy/agent-runtime/local-sim/T1.3-run-legacy.json <<'JSON'
+          {"domain":"agent-runtime","task_id":"T1.3","platform":"local-sim",
+           "run_id":"run-legacy","started_at":"2026-01-01T00:00:00Z",
+           "finished_at":"2026-01-01T00:00:05Z","config_hash":"sha256:dead",
+           "evidence_layer":"C","metrics":{"recovery_mode":"fail-fast"},
+           "ok":true,"runner_version":"1.0.0","raw":{},"notes":"legacy",
+           "schema_version":"1.0","series":{},"artifacts":[],"error":null}
+          JSON
+          csbench migrate-results /tmp/legacy --output /tmp/migrated
+          csbench migrate-results /tmp/legacy --output /tmp/migrated-again
+          diff -r /tmp/migrated /tmp/migrated-again
+          csbench report --results /tmp/migrated
+          python - <<'PY'
+          import json, pathlib
+          data = json.loads(pathlib.Path(
+              "/tmp/migrated/agent-runtime/local-sim/T1.3-run-legacy.json"
+          ).read_text(encoding="utf-8"))
+          assert data["schema_version"] == "0.2"
+          assert data["fingerprints"]["benchmark"] == "unknown"
+          assert data["extensions"]["legacy"]["config_hash"] == "sha256:dead"
+          print("migration is deterministic and idempotent")
+          PY
+
+      - name: Run plan aggregates repeats and drops warmups
+        run: |
+          rm -rf /tmp/plan-results
+          csbench run --domain agent-runtime --task T1.3 --platform local-sim \
+            --results /tmp/plan-results --repeat 3 --warmup 1
+          python - <<'PY'
+          import json, pathlib
+          aggs = list(pathlib.Path("/tmp/plan-results/aggregates").rglob("*.json"))
+          assert len(aggs) == 1, aggs
+          data = json.loads(aggs[0].read_text(encoding="utf-8"))
+          assert data["kind"] == "run_plan_aggregate"
+          assert len(data["runs"]["measured"]) == 3, data["runs"]
+          assert len(data["runs"]["warmup"]) == 1, data["runs"]
+          assert data["measurements"]["observed_attempts"]["n"] == 3
+          assert data["digest"].startswith("sha256:")
+          # Four run records + one aggregate; warmups are excluded from the stats.
+          recs = [p for p in pathlib.Path("/tmp/plan-results").rglob("*.json")
+                  if "aggregates" not in p.parts]
+          assert len(recs) == 4, recs
+          print("run plan: 4 records, 3 measured pooled, 1 warmup dropped")
+          PY
+
+  no-validate:
+    # Prove the hand-written validation floor still rejects bad input when the
+    # optional [validate] extra (jsonschema) is absent, so the fallback can't rot.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: Install without [validate]
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev,store]"
+      - name: Assert jsonschema is not installed
+        run: |
+          python - <<'PY'
+          import importlib.util
+          assert importlib.util.find_spec("jsonschema") is None, "jsonschema must be absent here"
+          print("jsonschema absent: exercising the hand-written fallback")
+          PY
+      - name: Test (pytest, fallback path)
+        run: pytest -q
+
+  wheel-smoke:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: Build wheel
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip wheel . --wheel-dir dist --no-deps
+      - name: Install wheel into isolated environment
+        run: |
+          python -m venv /tmp/csbench-wheel
+          /tmp/csbench-wheel/bin/pip install dist/*.whl
+      - name: Smoke installed wheel outside checkout
+        working-directory: /tmp
+        run: |
+          /tmp/csbench-wheel/bin/python -c "import importlib.metadata as m, importlib.resources as r, clousight_bench as cb; assert cb.__version__ == m.version('clousight-bench'), (cb.__version__, m.version('clousight-bench')); assert cb.RESULT_SCHEMA_VERSION == '0.2', cb.RESULT_SCHEMA_VERSION; root = r.files('clousight_bench.resources.workloads'); reqs = [('wordcount-py', 'run.py'), ('gsm8k-stats', 'run.py'), ('ycsb-wrapper', 'run.sh')]; missing = [n for n, e in reqs if not (root.joinpath(n, 'manifest.yaml').is_file() and root.joinpath(n, e).is_file())]; assert not missing, missing; assert r.files('clousight_bench').joinpath('py.typed').is_file(), 'py.typed marker missing from wheel'; schemas = r.files('clousight_bench.resources').joinpath('schemas'); need = ['runspec.schema.json', 'workload-manifest.schema.json', 'result-record-0.2.schema.json']; smissing = [s for s in need if not schemas.joinpath(s).is_file()]; assert not smissing, smissing; print('wheel version + workloads + py.typed + schemas + schema 0.2 verified')"
+          /tmp/csbench-wheel/bin/csbench list --verbose
+          /tmp/csbench-wheel/bin/csbench run \
+            --domain agent-runtime --task T1.3 --platform local-sim \
+            --results /tmp/csbench-wheel-results
+          /tmp/csbench-wheel/bin/csbench run \
+            --domain bigdata-emr --task J1.1 --platform local-process \
+            --results /tmp/csbench-wheel-results
+          /tmp/csbench-wheel/bin/csbench report \
+            --results /tmp/csbench-wheel-results
+
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: Install docs dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[docs]"
+      - name: Build docs
+        run: mkdocs build --strict

clousight_bench-0.2.0/.github/workflows/codeql.yml

@@ -0,0 +1,31 @@
+name: codeql
+
+# Static security analysis (CodeQL) for the Python source. Runs on pushes/PRs to
+# main and on a weekly schedule so newly-published queries catch older code.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "27 3 * * 1" # Mondays 03:27 UTC
+
+permissions:
+  contents: read
+  security-events: write
+
+jobs:
+  analyze:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: python
+          queries: security-and-quality
+      - name: Perform CodeQL analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:python"

clousight_bench-0.2.0/.github/workflows/dependency-review.yml

@@ -0,0 +1,21 @@
+name: dependency-review
+
+# Flag known-vulnerable or license-incompatible dependencies introduced by a PR,
+# before they land on main. Complements Dependabot (which updates existing deps).
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  review:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - name: Dependency review
+        uses: actions/dependency-review-action@v4
+        with:
+          fail-on-severity: high

clousight_bench-0.2.0/.github/workflows/docs.yml

@@ -0,0 +1,52 @@
+name: docs-deploy
+
+# Publish the MkDocs site to GitHub Pages on a version tag (the same trigger as
+# the PyPI release), so the docs a release ships match the published package.
+# CI already builds the docs with --strict on every PR (see ci.yml); this only
+# deploys. One-time setup: repo Settings → Pages → Source = "GitHub Actions".
+
+on:
+  push:
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Only one concurrent deploy; don't cancel an in-progress one mid-publish.
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - name: Install docs dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[docs]"
+      - name: Build docs (strict)
+        run: mkdocs build --strict --site-dir _site
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: _site
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v4

clousight_bench-0.2.0/.github/workflows/release.yml

@@ -0,0 +1,57 @@
+name: release
+
+# Publish to PyPI when a version tag is pushed (e.g. v0.2.0).
+# Uses PyPI Trusted Publishing (OIDC) — no API token stored as a secret.
+# One-time setup on PyPI: add this repo + workflow as a trusted publisher for
+# the `clousight-bench` project. See docs/RELEASING.md.
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - name: Verify tag matches package version
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .
+          TAG="${GITHUB_REF_NAME#v}"
+          PKG="$(python -c 'import importlib.metadata as m; print(m.version("clousight-bench"))')"
+          echo "tag=$TAG package=$PKG"
+          test "$TAG" = "$PKG" || { echo "::error::tag v$TAG != package version $PKG"; exit 1; }
+      - name: Build sdist + wheel
+        run: |
+          pip install build
+          python -m build
+      - name: Check metadata
+        run: |
+          pip install twine
+          twine check dist/*
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write   # required for Trusted Publishing (OIDC)
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

clousight_bench-0.2.0/.gitignore

@@ -0,0 +1,31 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+dist/
+build/
+.venv/
+venv/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+results/
+# Agent-bundle vendored deps (LangChain + OTel) — regenerated at build time, never commit
+_vendor_cache/
+.coverage
+coverage.xml
+htmlcov/
+site/
+*.local.yaml
+.env
+.gitsync.env
+.DS_Store
+.terraform/
+*.tfstate
+*.tfstate.backup
+.worktrees/
+
+# Terraform / infra build artifacts
+*.tfvars
+!*.tfvars.example
+mock-tools-fc.zip

clousight_bench-0.2.0/.gitsync.env.example

@@ -0,0 +1,6 @@
+# Template for scripts/gitsync.sh. Copy to `.gitsync.env` (which is gitignored):
+#     cp .gitsync.env.example .gitsync.env
+# The script reads $CSBENCH_REPO from the environment, or sources this file's
+# copy, to know which GitHub repo to push/PR against. Auth uses your machine's
+# `gh` config; no token is stored here.
+CSBENCH_REPO=clousight/clousight-bench

clousight_bench-0.2.0/AGENTS.md

@@ -0,0 +1,10 @@
+# Clousight Bench — Agent 入口
+
+本仓库是 Clousight Bench 开源测评核心。开发与贡献约定见 `README.md`。
+
+## 仓库身份（强制）
+
+- **GitHub 操作账号**：`clousight-dev`。`scripts/gitsync.sh` 会拒绝其它 `gh` 账号，并把本仓 `user.name` / `user.email` 写成 `Clousight` / `306954191+clousight-dev@users.noreply.github.com`（GitHub 按邮箱归属账号；个人 Gmail 会显示成个人 GitHub 用户）。
+- 未走 gitsync 时，向 GitHub 提交、推送、创建 PR、Issue 或 Release 前先用 `gh auth status` 确认；不是该账号时执行 `gh auth switch --user clousight-dev`。
+- 禁止直推 `main`。本地改动推到功能分支，`gitsync pr` 后 `gitsync merge`（默认 squash）。
+- 提交继续遵守仓库 DCO 要求（`git commit -s`）。`gitsync commit` 会自动加 `-s`。