ncp-olmo-eval 0.1.0a16__tar.gz

ncp_olmo_eval-0.1.0a16/.github/workflows/ci.yml

@@ -0,0 +1,107 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  host-contracts:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.10", "3.12"]
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+      - name: Install host-only dependencies
+        run: python -m pip install -e '.[dev,scoring]'
+      - name: Lint
+        run: ruff check src tests scripts
+      - name: Compile
+        run: python -m compileall -q src
+      - name: Test contracts
+        run: pytest -q
+      - name: Check portable adapters
+        run: |
+          find scripts -name '*.sh' -print0 | xargs -0 -n1 bash -n
+          python -m py_compile scripts/*.py
+      - name: Build distributions
+        if: matrix.python == '3.12'
+        run: python -m build
+      - name: Verify Core88 summary from installed wheel path
+        if: matrix.python == '3.12'
+        run: |
+          python -m venv .ci/installed-package
+          .ci/installed-package/bin/python -m pip install --no-deps dist/*.whl
+          .ci/installed-package/bin/ncp-olmo-eval-dflash-benchmark --help
+          NCP_OLMO_SOURCE_REVISION="$(git rev-parse HEAD)" \
+            .ci/installed-package/bin/python tests/installed_package_summary_smoke.py
+
+  docker-scorer-contracts:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+      - name: Fetch pinned OLMo-Eval scorer source
+        env:
+          OLMO_EVAL_COMMIT: f8816eea36563f27b4a9dd2533d68d34f3c67d3f
+        run: |
+          git clone --filter=blob:none https://github.com/allenai/OLMo-Eval.git .ci/olmo-eval
+          git -C .ci/olmo-eval checkout --detach "$OLMO_EVAL_COMMIT"
+          test "$(git -C .ci/olmo-eval rev-parse HEAD)" = "$OLMO_EVAL_COMMIT"
+      - name: Build release runtime scorer slice and run Math smoke
+        run: |
+          docker build \
+            -f docker/runtime.Dockerfile \
+            --build-arg BASE_IMAGE=python:3.12.11-slim-bookworm \
+            --build-arg INSTALL_EXTRAS=scoring \
+            --build-arg SOURCE_REVISION="$(git rev-parse HEAD)" \
+            -t ncp-olmo-eval:ci-runtime .
+      - name: Prepare locked BigCodeBench parser wheels
+        run: |
+          mkdir -p .ci/wheels
+          python -m pip download --only-binary=:all: --no-deps \
+            --dest .ci/wheels \
+            --python-version 310 \
+            --implementation cp \
+            --abi cp310 \
+            --abi abi3 \
+            --platform manylinux_2_28_x86_64 \
+            tree-sitter==0.25.2 tree-sitter-python==0.25.0
+          tree_sitter_wheel="$(basename "$(find .ci/wheels -maxdepth 1 -name 'tree_sitter-*.whl' -print -quit)")"
+          tree_sitter_python_wheel="$(basename "$(find .ci/wheels -maxdepth 1 -name 'tree_sitter_python-*.whl' -print -quit)")"
+          test -n "$tree_sitter_wheel"
+          test -n "$tree_sitter_python_wheel"
+          {
+            echo "TREE_SITTER_WHEEL=.ci/wheels/$tree_sitter_wheel"
+            echo "TREE_SITTER_SHA256=$(sha256sum ".ci/wheels/$tree_sitter_wheel" | awk '{print $1}')"
+            echo "TREE_SITTER_PYTHON_WHEEL=.ci/wheels/$tree_sitter_python_wheel"
+            echo "TREE_SITTER_PYTHON_SHA256=$(sha256sum ".ci/wheels/$tree_sitter_python_wheel" | awk '{print $1}')"
+          } >> "$GITHUB_ENV"
+      - name: Build and execute the formal BigCodeBench sandbox image
+        env:
+          OLMO_EVAL_COMMIT: f8816eea36563f27b4a9dd2533d68d34f3c67d3f
+        run: |
+          docker build \
+            -f docker/core88_bigcodebench_sandbox.Dockerfile \
+            --build-arg CORE88_UPSTREAM_REFERENCE="allenai/OLMo-Eval@$OLMO_EVAL_COMMIT" \
+            --build-arg CORE88_SCORER_COMMIT="$(git rev-parse HEAD)" \
+            --build-arg TREE_SITTER_WHEEL="$TREE_SITTER_WHEEL" \
+            --build-arg TREE_SITTER_SHA256="$TREE_SITTER_SHA256" \
+            --build-arg TREE_SITTER_PYTHON_WHEEL="$TREE_SITTER_PYTHON_WHEEL" \
+            --build-arg TREE_SITTER_PYTHON_SHA256="$TREE_SITTER_PYTHON_SHA256" \
+            -t ncp-olmo-eval:ci-bigcodebench-sandbox .
+          docker run --rm \
+            -e OLMO_EVAL_COMMIT="$OLMO_EVAL_COMMIT" \
+            -e OLMO_EVAL_ROOT=/opt/olmo-eval \
+            -v "$PWD/.ci/olmo-eval:/opt/olmo-eval:ro" \
+            ncp-olmo-eval:ci-bigcodebench-sandbox \
+            python3 -m ncp_olmo_eval.runtime_smoke bigcodebench

ncp_olmo_eval-0.1.0a16/.github/workflows/publish-images.yml

@@ -0,0 +1,163 @@
+name: publish-public-images
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: Existing tag or full commit SHA to build
+        required: true
+      version:
+        description: Existing package version used as the image tag
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        image_key:
+          - NCP_OLMO_EVAL_IMAGE
+          - CORE88_PYTHON_IMAGE
+          - CORE88_BIGCODEBENCH_IMAGE
+          - CORE88_DS1000_IMAGE
+          - CORE88_MULTIPLE_IMAGE
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+        with:
+          ref: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.ref }}
+      - name: Free runner disk for the CUDA runtime image
+        if: matrix.image_key == 'NCP_OLMO_EVAL_IMAGE'
+        run: |
+          df -h /
+          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/share/boost
+          docker system prune --all --force
+          df -h /
+      - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
+      - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Verify version and source identity
+        id: identity
+        env:
+          REQUESTED_VERSION: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.version }}
+        run: |
+          package_version="$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
+          requested="${REQUESTED_VERSION#v}"
+          test "$requested" = "$package_version"
+          revision="$(git rev-parse HEAD)"
+          test "${#revision}" = 40
+          echo "version=$package_version" >> "$GITHUB_OUTPUT"
+          echo "revision=$revision" >> "$GITHUB_OUTPUT"
+      - name: Build, smoke, and push release images
+        env:
+          IMAGE_KEY: ${{ matrix.image_key }}
+          OUTPUT_DIR: dist/${{ matrix.image_key }}
+          PUSH: "1"
+          VERSION: ${{ steps.identity.outputs.version }}
+          SOURCE_REVISION: ${{ steps.identity.outputs.revision }}
+          REGISTRY: ghcr.io/luckysjtu
+        run: scripts/build-public-images.sh
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: public-image-${{ matrix.image_key }}
+          path: |
+            dist/${{ matrix.image_key }}/public-images-${{ matrix.image_key }}.env
+            dist/${{ matrix.image_key }}/public-images-${{ matrix.image_key }}.json
+          if-no-files-found: error
+
+  manifest:
+    needs: publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0
+        with:
+          pattern: public-image-*
+          path: dist/fragments
+          merge-multiple: true
+      - name: Merge and verify immutable image manifests
+        run: |
+          python3 - <<'PY'
+          import json
+          import re
+          from pathlib import Path
+
+          expected = (
+              "NCP_OLMO_EVAL_IMAGE",
+              "CORE88_PYTHON_IMAGE",
+              "CORE88_BIGCODEBENCH_IMAGE",
+              "CORE88_DS1000_IMAGE",
+              "CORE88_MULTIPLE_IMAGE",
+          )
+          fragments = sorted(Path("dist/fragments").glob("public-images-*.json"))
+          if len(fragments) != len(expected):
+              raise SystemExit(f"expected {len(expected)} image fragments, found {len(fragments)}")
+
+          version = None
+          revision = None
+          images = {}
+          for path in fragments:
+              payload = json.loads(path.read_text(encoding="utf-8"))
+              if payload.get("schema_version") != "ncp-olmo-eval-published-images-v1":
+                  raise SystemExit(f"unexpected schema in {path}")
+              if payload.get("immutable") is not True:
+                  raise SystemExit(f"mutable image fragment: {path}")
+              version = version or payload["release_version"]
+              revision = revision or payload["source_revision"]
+              if payload["release_version"] != version or payload["source_revision"] != revision:
+                  raise SystemExit(f"release identity mismatch in {path}")
+              for key, value in payload["images"].items():
+                  if key in images:
+                      raise SystemExit(f"duplicate image key: {key}")
+                  if not re.search(r"@sha256:[0-9a-f]{64}$", value):
+                      raise SystemExit(f"missing immutable digest for {key}")
+                  images[key] = value
+
+          if tuple(key for key in expected if key in images) != expected or len(images) != len(expected):
+              raise SystemExit(f"image keys do not match release contract: {sorted(images)}")
+
+          output = Path("dist")
+          combined = {
+              "schema_version": "ncp-olmo-eval-published-images-v1",
+              "release_version": version,
+              "source_revision": revision,
+              "immutable": True,
+              "images": {key: images[key] for key in expected},
+          }
+          (output / "public-images.json").write_text(
+              json.dumps(combined, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+          )
+          (output / "public-images.env").write_text(
+              "\n".join(f"{key}={images[key]}" for key in expected) + "\n",
+              encoding="utf-8",
+          )
+          PY
+          test "$(grep -Ec '@sha256:[0-9a-f]{64}$' dist/public-images.env)" = 5
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: public-image-digests
+          path: |
+            dist/public-images.env
+            dist/public-images.json
+          if-no-files-found: error
+      - name: Attach digest manifests to GitHub release
+        if: github.event_name == 'release'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_TAG: ${{ github.event.release.tag_name }}
+        run: |
+          gh release upload "$RELEASE_TAG" \
+            --repo "$GITHUB_REPOSITORY" \
+            dist/public-images.env dist/public-images.json --clobber

ncp_olmo_eval-0.1.0a16/.github/workflows/release.yml

@@ -0,0 +1,74 @@
+name: publish-python-release
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+        with:
+          ref: ${{ github.event.release.tag_name }}
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Verify release identity
+        env:
+          RELEASE_TAG: ${{ github.event.release.tag_name }}
+        run: |
+          python - <<'PY'
+          import os
+          import tomllib
+          from pathlib import Path
+
+          version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]
+          assert os.environ["RELEASE_TAG"] == f"v{version}"
+          PY
+          test "$(git rev-parse HEAD)" = "${{ github.sha }}"
+      - name: Build and inspect distributions
+        run: |
+          python -m pip install --upgrade build twine
+          python -m build
+          python -m twine check dist/*
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: python-package-distributions
+          path: dist/
+          if-no-files-found: error
+
+  publish-pypi:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/ncp-olmo-eval
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0
+        with:
+          name: python-package-distributions
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33
+
+  attach-distributions:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0
+        with:
+          name: python-package-distributions
+          path: dist/
+      - name: Attach immutable distributions to GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RELEASE_TAG: ${{ github.event.release.tag_name }}
+        run: gh release upload "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" dist/* --clobber

ncp_olmo_eval-0.1.0a16/CHANGELOG.md

@@ -0,0 +1,214 @@
+# Changelog
+
+## Unreleased
+
+## 0.1.0a16
+
+- Make checkout-free release jobs pass the repository explicitly when attaching
+  Python distributions and immutable OCI digest manifests to GitHub Releases.
+- Replace the unpublished `lm-evaluation-harness` Git direct reference with
+  the official `lm-eval==0.4.13` PyPI release. The pinned tag retains the same
+  GSM8K task files as the validated development commit, while making wheel and
+  source-distribution metadata acceptable to public package indexes.
+- Reject direct URL dependencies in the release metadata contract so PyPI
+  publication failures are caught before a tag is created.
+
+## 0.1.0a15
+
+- Accept NCP DFlash checkpoints published as Hugging Face sharded SafeTensors,
+  validate every file referenced by `model.safetensors.index.json`, and seal
+  the full shard identity while retaining one-file checkpoint compatibility.
+- Adopt **NCP-ArchPreview** as the public model-family name throughout the
+  documentation and package metadata while retaining existing distribution,
+  import, CLI, environment-variable, schema, and model-config identifiers for
+  backward compatibility.
+- Document the public Stage1, Stage2 v1/v2/v3, and Stage2 DFlash2 NCPFlash
+  checkpoints published by the ArchSpace Collection on Hugging Face.
+- Re-license original repository code under Apache-2.0 and separate upstream
+  attribution and asset terms into `NOTICE` and `THIRD_PARTY_NOTICES.md`.
+- Pin the public upstream OCI bases used by the validated Core88 sandboxes and
+  add release automation that publishes runtime and scorer images to GHCR with
+  an immutable digest manifest.
+- Add a PyPI Trusted Publishing workflow that builds once, validates the wheel
+  and source distribution, and publishes through GitHub OIDC without a stored
+  API token.
+- Build each public OCI image on an isolated matrix runner, then merge and
+  verify their immutable digest fragments without exhausting a shared runner.
+- Reclaim preinstalled hosted-runner toolchains before building the large CUDA
+  runtime image, and bootstrap pinned modern packaging tools in the legacy
+  MultiPL-E base so the release package is installed by its real identity.
+- Replace asset and image placeholders with pinned public references or an
+  explicit gated-asset boundary, and document the one-time publisher setup.
+
+## 0.1.0a14
+
+- Publish the a14 validation record for the sealed NCP DFlash
+  continuous-batching operating point introduced in `0.1.0a13`, without
+  changing the inference or scoring protocols.
+- Record both the fixed active-batch/draft-width sweep and a mixed-length
+  32-request continuous queue. The tested pair reached `1.43x` steady-state
+  continuous throughput and preserved the adaptive `1:8,2:8,4:4,8:2` policy.
+- Document cold-start cost, pair-specific acceptance, and matched downstream
+  quality A/B results so the synthetic speed result cannot be presented as a
+  universal or target-exact guarantee.
+- Add release-metadata consistency coverage and package the sanitized
+  validation evidence in the source distribution.
+
+## 0.1.0a13
+
+- Seal the complete target-plus-DFlash operating point into every comparison
+  artifact and registration, including active batch size, scheduler queue,
+  draft width policy, model-runner selection, cache/mixer settings, and target
+  runtime controls. Legacy speculative comparison artifacts must be regenerated.
+- Preserve the verified batch-to-draft-width policy when emitting Core88 and
+  GSM8K tasks instead of replacing it with a hard-coded width of 16 or planner
+  environment variables.
+- Require continuous-batching A/B runs to submit at least one full scheduler
+  queue, and bind the same batch, queue, and runtime contract into formal
+  inference manifests and GSM8K aggregates.
+
+## 0.1.0a12
+
+- Add the correctness-gated NCP DFlash proposer and vLLM 0.13 model-runner
+  adapter without patching installed vLLM files.
+- Add target/draft A/B benchmarking with exact and explicitly approximate
+  verification modes, immutable artifact binding, speed gates, GPU-memory
+  evidence, and proposal/rejection telemetry.
+- Extend model registration with a sealed draft identity and comparison
+  artifact. Formal speculative inference fails closed until the artifact
+  covers at least eight prompts and 1,024 forced tokens at seed 42.
+- Route verified speculative registrations through GSM8K and Core88 task specs,
+  propagate their proven generation batch size, and reject SciQ/RULER/HELMET
+  speculative use.
+- Add continuous scheduler-queue support and request-local target state
+  transactions needed for batched DFlash proposal and rejection.
+- Document the source integration's known exact-parity limitation: exact modes
+  remain fail-closed diagnostic gates, while the evidenced tuned path is an
+  explicitly approximate registration requiring matched downstream quality
+  A/B.
+
+## 0.1.0a11
+
+- Add standalone SciQ inference, scoring, status validation, and final result
+  materialization to the scheduler-neutral unified entry.
+- Seal SciQ to the OLMo `all_supported_local` task 348 export: 1,000 test
+  examples, four continuation likelihoods, zero-shot `Question:/Answer:`
+  prompts, seed 42, one eight-GPU vLLM job, and official raw `acc`.
+- Normalize OLMo's ordered metric definitions without losing their source
+  evidence, selecting raw `acc` ahead of `acc_norm` for SciQ.
+- Add subset-aware Core-native planning and aggregation while retaining strict
+  full-profile order checks, exact source SHA-256 validation, complete unique
+  prediction coverage, and CPU-only rescoring.
+
+## 0.1.0a10
+
+- Separate the immutable evaluator identity recorded in a Core88 prediction-score
+  snapshot from the installed release that only materializes the final tables.
+- Preserve and validate the cached scorer commit/tree against the workflow even
+  when the clean finalizer is installed at another path or comes from a newer
+  release; record the finalizer identity independently in the output report.
+- Extend the clean-wheel regression smoke to cover cached Core88 finalization
+  across distinct workflow, scorer, and installed-package paths.
+
+## 0.1.0a9
+
+- Preserve the installed evaluator package tree SHA-256 when the fixed Core88
+  30-column summary revalidates a workflow-bound GSM8K companion.
+- Add a regression contract for matching evaluator packages installed at a
+  different absolute path from the workflow checkout, including a clean-wheel
+  installation smoke in CI.
+
+## 0.1.0a8
+
+- Expose the locked BigCodeBench parser wheels at
+  `/opt/core88/olmo-eval-deps` through the formal sandbox image's
+  `PYTHONPATH`, and verify both `tree_sitter` modules during the image build.
+- Give copied parser wheels fixed container-side names, allowing formal builds
+  to use either root-level files or paths nested inside the build context.
+- Preserve that image-local dependency directory after task-spec environment
+  merging, so an explicit OLMo-Eval source `PYTHONPATH` cannot hide the
+  sandbox's parser modules.
+- Replace the synthetic BigCodeBench CI layer with a build and execution of
+  the published `core88_bigcodebench_sandbox.Dockerfile`, using the same
+  locked-wheel extraction layout as the release image.
+
+## 0.1.0a7
+
+- Pin the mutually compatible and previously GPU-validated dependency pair
+  `transformers==4.57.6` and `huggingface-hub==0.36.2`; the old public
+  `transformers>=5` declaration conflicted with vLLM 0.13.0's `<5` bound.
+- Pin the release Math scorer runtime to `sympy==1.14.0` and
+  `antlr4-python3-runtime==4.11`, install it in the default runtime image, and
+  execute a real Minerva/MATH equivalence smoke during the Docker build.
+- Load BigCodeBench's two required helpers from SHA-verified files at the
+  pinned OLMo-Eval commit instead of importing its eager task registry. This
+  removes the accidental dependency on the deleted
+  `huggingface_hub.utils.silent_tqdm` API without weakening source fidelity.
+- Add Docker CI that builds the runtime scorer slice, runs the official Math
+  runtime smoke, and runs a BigCodeBench helper preflight against the pinned
+  upstream source.
+- Allow site executors to replace only a Python task's interpreter through the
+  explicit `NCP_OLMO_TASK_PYTHON` boundary, so heterogeneous sandbox images do
+  not inherit an unrelated inference virtualenv.
+
+## 0.1.0a6
+
+- Gate LMDeploy argument validation on the selected backend in the Core88
+  worker pool and standalone GSM8K evaluator. Public vLLM jobs no longer enter
+  the intentionally unavailable LMDeploy compatibility boundary.
+- Make the omitted-LMDeploy validator itself a no-op for non-LMDeploy
+  backends, while continuing to fail closed when LMDeploy is explicitly
+  selected.
+- Add a regression test for both sides of that backend boundary.
+
+## 0.1.0a5
+
+- Remove the unpublished LMDeploy verification flag from portable vLLM
+  Core88, RULER, and HELMET commands. The public package intentionally omits
+  LMDeploy, so passing its internal-only flag made the public parsers reject
+  otherwise valid jobs before model loading.
+- Add release-integrity coverage that rejects future command-generation leaks
+  of the removed flag; release validation also parses the generated Core88
+  worker command in the pinned GPU runtime.
+
+## 0.1.0a4
+
+- Pin the official `lm-evaluation-harness` commit required by GSM8K/Core88.
+- Include that runtime in the `vllm` and CPU-only `gsm8k` installation extras.
+- Fail early on a missing or version-drifted harness and record its immutable
+  identity in prepared GSM8K manifests.
+
+## 0.1.0a3
+
+- Restored the CUDA device-layout helper required when Core88 starts its GPU
+  worker pool from an installed package.
+- Made the unpublished native-Megatron reference side of the optional parity
+  harness fail early with an actionable boundary instead of importing modules
+  that are not part of the public distribution.
+- Added release-integrity tests for internal package imports and worker device
+  mapping so source extraction omissions fail in CPU-only CI.
+- Kept the GSM8K, Core88, RULER, and HELMET vLLM protocol contracts unchanged.
+
+## 0.1.0a2
+
+- Replaced the site-specific submission layer with portable JSON task specs,
+  atomic task status files, and a local task runner.
+- Added example local, Slurm, and Kubernetes adapters with no site defaults.
+- Added a reproducible CUDA runtime image, explicit Core88 sandbox image
+  contracts, and sealed/verified Hugging Face asset preparation.
+- Made formal execution offline by default and rejected serialized secrets.
+- Preserved strict artifact validation and raw-rescore fallback for Core88.
+- Replaced Core88's worker-side `.git` requirement with an exact evaluator
+  source-tree SHA-256 while retaining Git revision evidence when available.
+
+## 0.1.0a1
+
+- First standalone alpha release.
+- Added a unified interactive and non-interactive CLI for model registration,
+  inference, scoring, status, and finalization.
+- Added vLLM support for stock OLMo and the experimental NCP-ArchPreview plugin.
+- Added Core88 plus standard GSM8K, RULER 4K-64K, and HELMET 8K-64K.
+- Added tokenizer-bound sealed long-context preparation, fail-closed resume and
+  finalization contracts and explicit seed 42.
+- Removed private paths, registry names, mounts, accounts, proxies, and model
+  defaults from the published code.

ncp_olmo_eval-0.1.0a16/CONTRIBUTING.md

@@ -0,0 +1,26 @@
+# Contributing
+
+Please keep benchmark protocol changes explicit and reviewable. A protocol
+change must update tests, result metadata/schema where necessary,
+`docs/PROTOCOLS.md`, and the changelog. Do not mix predictions from different
+protocol revisions under resume.
+
+Before submitting a change, run:
+
+```bash
+python -m compileall -q src tests
+python -m pytest
+ruff check src tests scripts
+git diff --check
+for script in scripts/*.sh; do bash -n "$script"; done
+python -m py_compile scripts/*.py
+```
+
+Release candidates must also follow
+[`docs/RELEASE_CHECKLIST.md`](docs/RELEASE_CHECKLIST.md), including clean-wheel,
+distribution-content, public-tree, and formal sandbox-image validation. Version
+changes, tags, and publication are explicit release actions and should not be
+mixed into routine cleanup.
+
+Never commit model weights, benchmark data, credentials, private registry
+names, internal mounts, or judge API keys.