iclayout-bench 0.4.0a1__tar.gz

iclayout_bench-0.4.0a1/.dockerignore

@@ -0,0 +1,8 @@
+**
+!pyproject.toml
+!uv.lock
+!scripts/
+!scripts/build_ngspice.sh
+!scripts/build_magic.sh
+!scripts/patches/
+!scripts/patches/magic-stable-device-order.patch

iclayout_bench-0.4.0a1/.gitattributes

@@ -0,0 +1,2 @@
+# Unified patches retain upstream context prefixes and tab indentation.
+scripts/patches/*.patch whitespace=-blank-at-eol,-space-before-tab

iclayout_bench-0.4.0a1/.github/workflows/cd.yml

@@ -0,0 +1,134 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+  workflow_dispatch:
+
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  package:
+    name: Build Python distributions
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@v7
+        with:
+          version: '0.11.2'
+          python-version: '3.12'
+          enable-cache: true
+      - run: uv sync --locked --extra results
+      - run: uv run --locked ruff check .
+      - run: uv run --locked python scripts/check_docs.py
+      - run: uv run --locked python scripts/check_repository.py
+      - run: uv run --locked pytest -m unit
+      - name: Check release tag
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        run: |
+          version="$(uv run --locked python -c 'from importlib.metadata import version; print(version("iclayout-bench"))')"
+          test "$GITHUB_REF_NAME" = "v${version}" || {
+            echo "Tag $GITHUB_REF_NAME does not match project version $version" >&2
+            exit 1
+          }
+      - name: Build distributions
+        run: uv build --out-dir build/dist
+      - name: Install wheel smoke test
+        run: |
+          package_venv="$RUNNER_TEMP/iclayout-bench-package"
+          python -m venv "$package_venv"
+          for bench_wheel in build/dist/*.whl; do
+            "$package_venv/bin/python" -m pip install "${bench_wheel}[results]"
+          done
+          "$package_venv/bin/python" -I - <<'PY'
+          import importlib.util
+          import subprocess
+          import sys
+          from importlib.resources import files
+          from pathlib import Path
+
+          for name in ("benchmarking", "benchmarking.engine", "benchmarking.service", "benchmarking.participants"):
+              module = importlib.import_module(name)
+              assert Path(module.__file__).resolve().is_relative_to(Path(sys.prefix).resolve())
+          assert importlib.util.find_spec("iclayout_bench_private") is None
+          assert files("benchmarking").joinpath("model_configs.yaml").is_file()
+          from importlib.metadata import version
+          from benchmarking.version import package_version
+          identity = package_version()
+          assert identity["version"] == version("iclayout-bench")
+          assert identity["commit"] == subprocess.check_output(
+              ["git", "rev-parse", "HEAD"], text=True).strip()
+          assert importlib.util.find_spec("layout_eval") is None
+          assert importlib.util.find_spec("layout_service") is None
+          from benchmarking.dataset import Dataset, process_manifest
+          from benchmarking.engine.qualification import acceptance_summary
+          assert importlib.util.find_spec("benchmarking.web") is None
+          from benchmarking.engine.source import implementation_identity
+          assert implementation_identity().startswith("sha256:")
+          for name in ("benchmarking.client", "benchmarking.observe", "benchmarking.run", "benchmarking.engine.cli",
+                       "benchmarking.engine.preview", "benchmarking.service", "benchmarking.results"):
+              subprocess.run([sys.executable, "-I", "-m", name, "--help"], check=True)
+          PY
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: iclayout-bench-release-${{ github.sha }}
+          path: build/dist/*
+          if-no-files-found: error
+          retention-days: 14
+
+  pypi:
+    name: Publish Python distributions to PyPI
+    needs: package
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    environment:
+      name: pypi
+      url: https://pypi.org/project/iclayout-bench/
+    permissions:
+      id-token: write
+    steps:
+      - name: Download checked distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: iclayout-bench-release-${{ github.sha }}
+          path: build/dist
+      - name: Publish with Trusted Publishing
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: build/dist/
+          skip-existing: true
+
+  publish:
+    name: Publish GitHub release
+    needs: [package, pypi]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+      - name: Download distributions
+        uses: actions/download-artifact@v4
+        with:
+          name: iclayout-bench-release-${{ github.sha }}
+          path: build/dist
+      - name: Create GitHub release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh release create "$GITHUB_REF_NAME" build/dist/* --verify-tag --generate-notes

iclayout_bench-0.4.0a1/.github/workflows/checks.yml

@@ -0,0 +1,98 @@
+name: Framework checks
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: checks-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  checks:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    strategy:
+      matrix:
+        python: ['3.12', '3.13']
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@v7
+        with:
+          version: '0.11.2'
+          python-version: ${{ matrix.python }}
+          enable-cache: true
+      - run: uv sync --locked --extra results --group analysis
+      - run: uv run --locked ruff check .
+      - run: uv run --locked python scripts/check_docs.py
+      - run: uv run --locked python scripts/check_repository.py
+      - run: uv run --locked --group analysis pytest -m unit
+      - run: git diff --exit-code
+
+  package:
+    name: Build package
+    needs: checks
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@v7
+        with:
+          version: '0.11.2'
+          python-version: '3.12'
+          enable-cache: true
+      - run: uv sync --locked --extra results
+      - name: Build distributions
+        run: uv build --out-dir build/dist
+      - name: Install wheel smoke test
+        run: |
+          package_venv="$RUNNER_TEMP/iclayout-bench-package"
+          python -m venv "$package_venv"
+          for bench_wheel in build/dist/*.whl; do
+            "$package_venv/bin/python" -m pip install "${bench_wheel}[results]"
+          done
+          "$package_venv/bin/python" -I - <<'PY'
+          import importlib.util
+          import subprocess
+          import sys
+          from importlib.resources import files
+          from pathlib import Path
+
+          for name in ("benchmarking", "benchmarking.engine", "benchmarking.service", "benchmarking.participants"):
+              module = importlib.import_module(name)
+              assert Path(module.__file__).resolve().is_relative_to(Path(sys.prefix).resolve())
+          assert importlib.util.find_spec("iclayout_bench_private") is None
+          assert files("benchmarking").joinpath("model_configs.yaml").is_file()
+          from importlib.metadata import version
+          from benchmarking.version import package_version
+          identity = package_version()
+          assert identity["version"] == version("iclayout-bench")
+          assert identity["commit"] == subprocess.check_output(
+              ["git", "rev-parse", "HEAD"], text=True).strip()
+          assert importlib.util.find_spec("layout_eval") is None
+          assert importlib.util.find_spec("layout_service") is None
+          from benchmarking.dataset import Dataset, process_manifest
+          from benchmarking.engine.qualification import acceptance_summary
+          assert importlib.util.find_spec("benchmarking.web") is None
+          from benchmarking.engine.source import implementation_identity
+          assert implementation_identity().startswith("sha256:")
+          for name in ("benchmarking.client", "benchmarking.observe", "benchmarking.run", "benchmarking.engine.cli",
+                       "benchmarking.engine.preview", "benchmarking.service", "benchmarking.results"):
+              subprocess.run([sys.executable, "-I", "-m", name, "--help"], check=True)
+          PY
+      - name: Upload distributions
+        uses: actions/upload-artifact@v4
+        with:
+          name: iclayout-bench-distributions-${{ github.sha }}
+          path: build/dist/*
+          if-no-files-found: error
+          retention-days: 14

iclayout_bench-0.4.0a1/.github/workflows/eda.yml

@@ -0,0 +1,39 @@
+name: Public evaluator checks
+on:
+  workflow_dispatch:
+    inputs:
+      dataset:
+        description: Hugging Face Dataset repository ID
+        required: true
+        type: string
+      revision:
+        description: Dataset Git commit
+        required: true
+        type: string
+permissions:
+  contents: read
+jobs:
+  evaluator:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 180
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+      - uses: astral-sh/setup-uv@v7
+        with:
+          version: '0.11.2'
+          python-version: '3.12'
+      - run: uv sync --locked --group analysis --group eda
+      - run: docker build -t iclayout-bench-tools:dev .
+      - env:
+          ICLAYOUT_BENCH_TEST_IMAGE: iclayout-bench-tools:dev
+          DATASET_REPO: ${{ inputs.dataset }}
+          DATASET_REVISION: ${{ inputs.revision }}
+        run: |
+          export ICLAYOUT_BENCH_DATASET="$(uv run --locked python -c 'import os; from huggingface_hub import snapshot_download; print(snapshot_download(repo_id=os.environ["DATASET_REPO"], repo_type="dataset", revision=os.environ["DATASET_REVISION"]))')"
+          uv run --locked pytest tests/unit/test_catalogs.py
+          uv run --locked pytest -m acceptance_container
+          uv run --locked pytest tests/test_http.py
+          uv run --locked --group eda pytest tests/integration/test_public_references.py -k 'cell_6t or NAND2_X1'

iclayout_bench-0.4.0a1/.github/workflows/image.yml

@@ -0,0 +1,77 @@
+name: Tools image
+
+on:
+  push:
+    branches: [main]
+    tags: ['v*.*.*']
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: tools-image-publish
+  cancel-in-progress: false
+
+jobs:
+  image:
+    if: vars.DOCKERHUB_IMAGE != ''
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - name: Validate Docker build context
+        run: bash tests/integration/test_build_context.sh
+      - uses: docker/setup-buildx-action@v4
+      - name: Extract image metadata
+        id: meta
+        uses: docker/metadata-action@v6
+        with:
+          images: ${{ vars.DOCKERHUB_IMAGE }}
+          flavor: latest=false
+          tags: |
+            type=sha,format=long
+            type=ref,event=tag
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+      - name: Build candidate
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          target: tools
+          platforms: linux/amd64
+          load: true
+          tags: iclayout-bench-tools:ci
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=tools
+          cache-to: type=gha,scope=tools,mode=max,ignore-error=true
+      - name: Check installed tools
+        run: |
+          docker run --rm --network none iclayout-bench-tools:ci bash -euc '
+            test "$(id -u)" != 0
+            klayout -v
+            ngspice --version
+            magic --version
+            openvaf --version
+            for tool in xschem qucs-s qucsator qucsator_rf qucsconv; do
+              command -v "$tool"
+            done
+            python -I -c "import klayout.db, klayout_pex, gdsfactory, psutil"
+          '
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Publish checked image
+        uses: docker/build-push-action@v7
+        with:
+          context: .
+          target: tools
+          platforms: linux/amd64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          provenance: mode=min
+          sbom: true

iclayout_bench-0.4.0a1/.github/workflows/pages.yml

@@ -0,0 +1,58 @@
+name: Publish reviewed website
+
+# The site is built and reviewed externally. This workflow never imports an
+# operator checkout, builds website sources, or participates in Python package CI.
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: github-pages
+  cancel-in-progress: false
+
+jobs:
+  upload:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: gh-pages
+          persist-credentials: false
+      - name: Verify the reviewed artifact
+        run: |
+          python - <<'PY'
+          import hashlib
+          import json
+          from pathlib import Path
+          root = Path('.')
+          manifest = json.loads(Path('release-manifest.json').read_text())
+          paths = [p for p in root.rglob('*') if '.git' not in p.parts]
+          assert not any(p.is_symlink() for p in paths), 'Symlinks are not publishable'
+          files = {p.as_posix() for p in paths if p.is_file()}
+          assert files == set(manifest['files']) | {'release-manifest.json'}, 'Unexpected release files'
+          assert {'index.html', '404.html', '.nojekyll', 'site-data/snapshot.json'} <= files
+          assert not any(p.split('/')[0] in {'admin', 'account', 'api', '.github'} for p in files)
+          assert sum(Path(p).stat().st_size for p in files) <= 1024 ** 3
+          for name, digest in manifest['files'].items():
+              path = Path(name)
+              assert not path.is_absolute() and '..' not in path.parts
+              assert hashlib.sha256(path.read_bytes()).hexdigest() == digest, name
+          PY
+      - uses: actions/upload-pages-artifact@v4
+        with:
+          path: .
+  deploy:
+    needs: upload
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

iclayout_bench-0.4.0a1/.gitignore

@@ -0,0 +1,23 @@
+# Python environments and repository-local generated outputs
+.venv/
+__pycache__/
+*.py[cod]
+*.egg-info/
+/build/
+/dist/
+
+# Legacy and tool caches
+/.cache/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+htmlcov/
+
+# Local credentials and settings
+.env
+.env.*
+auth.json
+
+# Generated run artifacts
+/results/

iclayout_bench-0.4.0a1/AGENTS.md

@@ -0,0 +1,56 @@
+# ICLayout-Bench Repository Conventions
+
+This repository owns the evaluation observation layer, generic client, public protocol, scoring specifications, Dataset loading, result archives/visualization and portable result delivery. It also owns the shared evaluation engine and local HTTP service. Operators own operator-controlled admission, frozen reruns, the official website and publication (one-way dependency operator applications → Public). See the architecture guide for the current package boundary. Write all repository Markdown in English, except `README_CN.md`.
+
+## Starting Work
+
+Run `git status --short --branch` and preserve unrelated changes. Use the [README](README.md) as the entry point and [CONTRIBUTING](CONTRIBUTING.md) for verification rules. Read the guide matching the change:
+
+| Work | Read first |
+|---|---|
+| Module boundaries and extension interfaces | [Architecture](docs/architecture.md) |
+| Tasks, inputs, sources, constraints, metrics, or judges | [Task contract](docs/tasks.md) and [qualification](CONTRIBUTING.md#task-qualification); [input isolation](docs/tasks.md#input-isolation) defines solver visibility |
+| Agents, submissions, events, batches, or statistics | [Running guide](docs/running.md) |
+| Result archives or presentation data | [Result storage](docs/running.md#result-archive), [task presentation bindings](docs/running.md#task-presentations-and-interactive-layouts), and [archive verification](CONTRIBUTING.md#result-archive-verification) |
+| Result packaging or website delivery | [Handoff boundaries](docs/architecture.md#result-handoff-and-storage-ownership) and [delivery contract](docs/running.md#website-result-delivery) |
+| Images, dependencies, PDKs, EDA backends, or support bundles | [Tools guide](docs/tools.md); dependencies are declared in `pyproject.toml`, synchronized in `uv.lock` |
+| Qualification admission, resource or endpoint approval, or report export | [Admission and export](docs/architecture.md#verified-reruns-and-disclosure) |
+
+PDK implementation and signoff rules belong to the upstream PDK. Resource manifests
+identify pinned sources; the framework resolves installations and derived resources
+through the external cache described in the tools guide.
+
+## Execution and Completion
+
+- Keep protocol, client, observation and analysis in `benchmarking/`, shared preparation/execution/EDA in `benchmarking/engine/`, the local HTTP adapter in `benchmarking/service/`, and reusable participant launch/configuration/MCP adapters in `benchmarking/participants/`. `benchmarking.run` schedules experiments; Agent harnesses retain their own decision loops. External applications consume the installed public interfaces; the package must not import consumer code. Static tasks and reference solutions belong in the independent Dataset repository; design sources, acceptance summaries and full retained validation belong in the design authoring workspace. Read local directories or Hugging Face snapshots directly; do not generate prepared case copies. Public installation and CI must not depend on operator applications.
+- Shared archive/import/query/presentation code belongs in `benchmarking.results`, and portable packaging/upload in `benchmarking.transfer`. Operators own the unified Web frontend and HTTP routes, website accounts, storage integration, publication permissions and leaderboard policy. Keep the local archive separate from the operator database; importing or publishing never changes recorded scores or verification levels.
+- Maintained case schematics and their provenance follow the presentation contract above. Keep solver inputs explicit; drawings and a later presentation revision do not rewrite a historical task or attest its evaluation.
+- Default to a compatible existing tool image or a thin derived image for development validation. Keep the public recipe/lock current and run affected checks; passing these checks completes development validation without a full rebuild. Before changing container dependencies or starting a full build, apply the triggers in [image development](docs/tools.md#image-development). Report the image used and actual checks.
+- A standard solve materializes only declared inputs and reviewed resource bundles. Public reference solutions may be downloaded for debugging but are never mounted for a standard solver Agent. Follow the [task guide](docs/tasks.md#asset-rights) for sources and distribution.
+- Evaluate frozen candidates independently with trusted materials, without Agent credentials or writable directories. DRC/LVS establishes physical validity; task success also requires the declared geometry and post-layout metrics.
+- Rerun affected case validation and shared evaluator regressions when the judge, rules, or task changes; use the task guide to determine calibration scope. Deterministic protocol tests are not model scores.
+- Update the owning guide as the single source of truth when behavior changes; keep onboarding in the README and verification guidance in CONTRIBUTING. Do not create `CONTEXT.md` or duplicate protocol documents.
+
+When complete: select the affected checks from the [verification matrix](CONTRIBUTING.md#verification), check documentation links against the commands they describe, and report actual test scope and Git status. Verify and commit cross-repository changes separately; retain reviewable evidence for architecture and upstream-reference changes.
+
+## Website publication branch
+
+The `gh-pages` branch may hold reviewed, generated website artifacts. Keep the
+main branch limited to package sources, documentation and the optional Pages
+workflow. Website authoring and export stay with the operator. Follow
+[static hosting](docs/architecture.md#static-website-hosting) for publication;
+package builds and CI remain independent of website artifacts.
+
+## Standalone boundary
+
+Participant usage examples are documented in [examples/README.md](examples/README.md).
+They run the installed public wheel in an isolated environment. Reusable configs
+and documentation are tracked; local dotenv files, environments, repository
+archives and persistent experiment results remain ignored and outside releases.
+
+Ship a reusable installed package and public interface documentation. Accept data
+through explicit Dataset paths/IDs and operator/participant integrations through
+public interfaces. The checkout, wheel, tests and CI must work with no sibling
+repositories present. Keep concrete case catalogs, design recipes, experiment
+results, website code and internal operating policies with their respective owners.
+Run the repository boundary check and isolated wheel checks before completion.