corpus-assay 0.1.0__tar.gz

corpus_assay-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,98 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@v10.2.0
+        with:
+          enable-cache: true
+      - name: Install (builds the extension)
+        run: uv sync --locked
+      - run: cargo fmt --all --check
+      - run: cargo clippy --locked --all-targets -- -D warnings
+      - run: uv run --no-sync ruff check .
+      - run: uv run --no-sync ruff format --check .
+      - run: uv run --no-sync pyright
+
+  test:
+    name: Test (${{ matrix.os }}, Python ${{ matrix.python }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-14, windows-latest]
+        python: ["3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: astral-sh/setup-uv@v10.2.0
+        with:
+          enable-cache: true
+      - name: Install (builds the extension)
+        run: uv sync --locked --python ${{ matrix.python }}
+      - name: Test
+        run: uv run --no-sync pytest --cov=corpus_assay --cov-report=term-missing
+
+  wheel:
+    name: Wheel contents + offline e2e
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@v10.2.0
+      - name: Build wheel
+        run: uvx maturin build --release --locked --out dist
+      - name: Check wheel contents
+        run: |
+          python - <<'EOF'
+          import glob, zipfile
+          (wheel,) = glob.glob("dist/*.whl")
+          names = zipfile.ZipFile(wheel).namelist()
+          print("\n".join(names))
+          leaked = [n for n in names if n.endswith(".rs") or "Cargo." in n
+                    or n.startswith(("target/", "tests/", "src/"))]
+          assert not leaked, f"unexpected files in wheel: {leaked}"
+          assert "corpus_assay/ngram_config.json" in names
+          assert "corpus_assay/py.typed" in names
+          native = [n for n in names
+                    if n.startswith("corpus_assay/_native") and not n.endswith(".pyi")]
+          assert len(native) == 1 and ".abi3." in native[0], native
+          EOF
+      - name: Install the wheel into a clean venv
+        run: |
+          uv venv "$RUNNER_TEMP/wheel-venv" --python 3.12
+          uv pip install --python "$RUNNER_TEMP/wheel-venv" dist/*.whl pytest
+      - name: Offline e2e against the installed wheel
+        run: |
+          "$RUNNER_TEMP/wheel-venv/bin/python" -c "import corpus_assay; print(corpus_assay.__file__)"
+          "$RUNNER_TEMP/wheel-venv/bin/python" -m pytest tests -m e2e -p no:cacheprovider

corpus_assay-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,213 @@
+name: Release
+
+# Tag push (v*): build -> smoke-test -> TestPyPI -> verify -> PyPI -> GitHub Release.
+# Manual run: build + smoke-test, optionally publishing to TestPyPI only.
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+    inputs:
+      publish:
+        description: "Publish the built distributions to"
+        type: choice
+        options: [none, testpypi]
+        default: none
+
+permissions:
+  contents: read
+
+jobs:
+  version-check:
+    name: Version check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - name: Tag matches Cargo.toml and CITATION.cff versions
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          version=$(python3 -c "import tomllib; print(tomllib.load(open('Cargo.toml', 'rb'))['package']['version'])")
+          if [ "v${version}" != "${GITHUB_REF_NAME}" ]; then
+            echo "Tag ${GITHUB_REF_NAME} does not match Cargo.toml version ${version}" >&2
+            exit 1
+          fi
+          # Zenodo takes the archived release's metadata from CITATION.cff.
+          cff_version=$(sed -n 's/^version: *"\{0,1\}\([^"]*\)"\{0,1\} *$/\1/p' CITATION.cff)
+          if [ "${cff_version}" != "${version}" ]; then
+            echo "CITATION.cff version '${cff_version}' does not match ${version}" >&2
+            exit 1
+          fi
+
+  sdist:
+    name: Build sdist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist-sdist
+          path: dist
+          if-no-files-found: error
+
+  wheels:
+    name: Build wheel (${{ matrix.target }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+          - os: ubuntu-24.04-arm
+            target: aarch64-unknown-linux-gnu
+          - os: macos-14
+            target: aarch64-apple-darwin
+          - os: macos-14
+            target: x86_64-apple-darwin
+          - os: windows-latest
+            target: x86_64-pc-windows-msvc
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      # abi3 (see Cargo.toml): one cp312-abi3 wheel per platform covers CPython >= 3.12.
+      - uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          manylinux: auto
+          args: --release --locked --out dist
+          sccache: true
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist-${{ matrix.target }}
+          path: dist
+          if-no-files-found: error
+
+  smoke:
+    name: Smoke test (${{ matrix.os }}, Python ${{ matrix.python }})
+    needs: [wheels]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, ubuntu-24.04-arm, macos-14, macos-15-intel, windows-latest]
+        python: ["3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+      - uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install the built wheel
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --no-index --no-deps --only-binary :all: --find-links dist corpus-assay
+          python -m pip install corpus-assay pytest
+      - name: Smoke
+        shell: bash
+        run: |
+          corpus-assay --version
+          python -c "import corpus_assay, corpus_assay._native as n; print(corpus_assay.__file__, n.__version__)"
+      - name: Test suite against the installed wheel
+        run: python -m pytest tests -q -p no:cacheprovider
+
+  publish-testpypi:
+    name: Publish to TestPyPI
+    needs: [version-check, sdist, smoke]
+    if: startsWith(github.ref, 'refs/tags/v') || inputs.publish == 'testpypi'
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/corpus-assay
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
+
+  verify-testpypi:
+    name: Verify TestPyPI install
+    needs: [publish-testpypi]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - name: Install corpus-assay from TestPyPI (dependencies from PyPI)
+        shell: bash
+        run: |
+          version=$(python -c "import tomllib; print(tomllib.load(open('Cargo.toml', 'rb'))['package']['version'])")
+          for attempt in 1 2 3 4 5 6 7 8 9 10; do
+            if python -m pip install --no-deps --only-binary :all: \
+                --index-url https://test.pypi.org/simple/ "corpus-assay==${version}"; then
+              break
+            fi
+            echo "TestPyPI not ready yet (attempt ${attempt}); retrying in 30s"
+            sleep 30
+          done
+          python -m pip show corpus-assay
+          # Dependencies come from PyPI; corpus-assay itself is already satisfied.
+          python -m pip install "corpus-assay==${version}" pytest
+          python -c "import corpus_assay; assert corpus_assay.__version__ == '${version}', corpus_assay.__version__"
+      - name: Offline e2e
+        run: python -m pytest tests -m e2e -q -p no:cacheprovider
+
+  publish-pypi:
+    name: Publish to PyPI
+    needs: [verify-testpypi]
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/corpus-assay
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+      - uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: GitHub Release
+    needs: [publish-pypi]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+      - name: Create release with CHANGELOG notes
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          version="${GITHUB_REF_NAME#v}"
+          awk -v ver="$version" '
+            $0 ~ "^## \\[" ver "\\]" { found = 1; next }
+            found && /^## \[/ { exit }
+            found { print }
+          ' CHANGELOG.md > notes.md
+          gh release create "$GITHUB_REF_NAME" dist/* --title "corpus-assay $version" --notes-file notes.md

corpus_assay-0.1.0/.gitignore

@@ -0,0 +1,30 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+.python-version.local
+
+# Build outputs
+/target/
+/dist/
+/build/
+python/corpus_assay/*.so
+python/corpus_assay/*.pyd
+python/corpus_assay/*.dll
+
+# Tooling caches
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+.coverage.*
+coverage.xml
+htmlcov/
+.cache/
+
+# Editors / OS
+.idea/
+.vscode/
+*.swp
+.DS_Store

corpus_assay-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

corpus_assay-0.1.0/CHANGELOG.md

@@ -0,0 +1,32 @@
+# Changelog
+
+All notable changes to this project are documented here. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses
+[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-10-03
+
+Initial public release.
+
+### Added
+
+- `build-index`: hashed word n-gram index from Hugging Face or local benchmark
+  splits, with per-benchmark and per-item attribution sidecars, subtraction splits,
+  and provenance metadata (dataset revisions, fingerprints, normalization hashes).
+- `scan`, `run`, `run-hf`: multi-process Rust scanner over Parquet and JSONL
+  (`.gz`/`.zst`) shards, with an index-wide (`union`) or single-item (`item`) gate,
+  resumable per-shard outputs, and sampled per-document evidence.
+- `report`: leak tables by benchmark, shard, and item (JSON, CSV, Markdown).
+- `build-stopgrams-direct`: stop-grams learned from a background corpus, applied
+  with `--stopgrams`.
+- `init-config`, `list-benchmarks`, `describe-benchmark`, `validate-config`,
+  `inspect-index`, `inspect-run`, `reproduce`.
+- Global `--ngram-config` to override the bundled normalization config. Scans refuse
+  an index built with a different config unless `--allow-config-mismatch` is passed.
+- Experimental: spaced-seed recall channel (`build-spaced-index`, `--spaced-seeds`).
+- Prebuilt abi3 wheels for CPython 3.12+ on Linux, macOS, and Windows.
+
+[Unreleased]: https://github.com/mrcabbage972/corpus-assay/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/mrcabbage972/corpus-assay/releases/tag/v0.1.0

corpus_assay-0.1.0/CITATION.cff

@@ -0,0 +1,23 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+type: software
+title: "corpus-assay: benchmark contamination detection for LLM training corpora"
+abstract: >-
+  Fast, auditable n-gram detection of evaluation-benchmark contamination in LLM
+  training corpora, with a Rust scanner and per-benchmark and per-item attribution.
+authors:
+  - family-names: May
+    given-names: Victor
+repository-code: "https://github.com/mrcabbage972/corpus-assay"
+license: Apache-2.0
+version: 0.1.0
+date-released: "2026-10-03"
+# Zenodo archives every GitHub Release of this repository. Each release gets its own
+# version DOI, and one concept DOI covers all versions (it resolves to the latest).
+# The concept DOI goes in `doi:` once Zenodo has archived the first release (v0.1.0).
+keywords:
+  - large language models
+  - evaluation
+  - data contamination
+  - decontamination
+  - benchmarks

corpus_assay-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,59 @@
+# Contributing to corpus-assay
+
+Thanks for your interest! Bug reports, feature requests, and pull requests are welcome.
+
+## Development setup
+
+You need [uv](https://docs.astral.sh/uv/) and a stable [Rust toolchain](https://rustup.rs).
+
+```bash
+git clone https://github.com/mrcabbage972/corpus-assay
+cd corpus-assay
+make dev        # uv sync + build the Rust extension in place
+make check      # lint (ruff, pyright, rustfmt, clippy) + tests
+```
+
+The package lives in `python/corpus_assay/` and the Rust extension in `src/`.
+`uv sync` rebuilds the extension when Rust sources change. You can also rebuild
+explicitly with `make dev` (or `make release-dev` for an optimized build). Tests always
+run against the built extension, so rebuild after editing Rust.
+
+Useful targets:
+
+| Target | What it does |
+| --- | --- |
+| `make test` | Full test suite |
+| `make e2e` | Offline end-to-end CLI tests only (`pytest -m e2e`) |
+| `make test-cov` | Tests with coverage |
+| `make lint` | Checks only; never modifies files |
+| `make fmt` | Apply ruff and rustfmt formatting and safe fixes |
+| `make wheel` | Build a release wheel into `dist/` |
+
+## Pull requests
+
+- Keep changes focused, and add or update tests for behavior changes.
+- `make check` must pass. CI runs it on Linux, macOS, and Windows for Python 3.12 and 3.13.
+- Update `CHANGELOG.md` under *Unreleased* for user-visible changes.
+- Changing the normalization pipeline, hashing, or a binary format changes what existing
+  indexes mean. Call it out in the PR and the changelog.
+- The spaced-seed golden vectors (`tests/fixtures/spaced_golden.json`) should only change
+  when spaced-seed behavior is meant to change. Regenerate them with
+  `uv run python tests/test_spaced_golden.py`.
+
+## Releasing (maintainers)
+
+1. Bump `version` in `Cargo.toml` (the single source of the package version), set
+   `version` and `date-released` in `CITATION.cff`, and date the CHANGELOG section. The
+   release workflow refuses a tag that doesn't match `Cargo.toml` and `CITATION.cff`.
+2. Optionally dry-run: run the *Release* workflow manually with `publish: testpypi`.
+3. Tag and push: `git tag -a vX.Y.Z -m "corpus-assay X.Y.Z" && git push origin vX.Y.Z`.
+   The workflow then:
+   1. builds the wheels and sdist;
+   2. smoke-tests them on every platform;
+   3. publishes to TestPyPI and verifies the install;
+   4. publishes to PyPI after approval of the `pypi` environment;
+   5. creates the GitHub Release.
+
+   Zenodo's GitHub integration archives the GitHub Release and mints a DOI for that
+   version. The concept DOI in `CITATION.cff` and the README badge covers all versions,
+   so it doesn't change between releases.