scaneval 2.0.0a1__tar.gz

scaneval-2.0.0a1/.github/workflows/ci.yml

@@ -0,0 +1,155 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Python ${{ matrix.python }} and TypeScript
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ['3.11', '3.13', '3.14']
+    env:
+      PYTHONDONTWRITEBYTECODE: '1'
+      SEMGREP_SEND_METRICS: 'off'
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '24'
+          cache: npm
+          cache-dependency-path: sdk/typescript/package-lock.json
+      - name: Install test dependencies
+        run: |
+          python -m pip install -e '.[dev]' build 'semgrep==1.177.0'
+          npm ci --prefix sdk/typescript
+      - name: TypeScript tests and build
+        run: npm test --prefix sdk/typescript
+      - name: Python tests including Observer parity
+        # Tests use fixtures and mock runners, never paid models. Tests requiring
+        # external harness checkouts or preserved private bundles report skips.
+        run: python -m pytest -q -rs
+      - name: Build wheel
+        run: python -m build --wheel --outdir "$RUNNER_TEMP/scaneval-dist"
+      - name: Installed-wheel demo and replay
+        working-directory: ${{ runner.temp }}
+        env:
+          PYTHONPATH: ''
+        run: |
+          python -m venv "$RUNNER_TEMP/scaneval-wheel-env"
+          "$RUNNER_TEMP/scaneval-wheel-env/bin/python" -m pip install "$RUNNER_TEMP"/scaneval-dist/*.whl
+          "$RUNNER_TEMP/scaneval-wheel-env/bin/scaneval" demo "$RUNNER_TEMP/scaneval-demo"
+          "$RUNNER_TEMP/scaneval-wheel-env/bin/scaneval" replay "$RUNNER_TEMP/scaneval-demo" --output "$RUNNER_TEMP/scaneval-replay.json"
+          cmp "$RUNNER_TEMP/scaneval-demo/evaluation.json" "$RUNNER_TEMP/scaneval-replay.json"
+
+  runtime-dependencies:
+    name: Audited SV-Agent runtime dependencies
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    env:
+      PI_OFFLINE: '1'
+      PI_SKIP_VERSION_CHECK: '1'
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '26.7.0'
+          cache: npm
+          cache-dependency-path: containers/sv-agent/package-lock.json
+      - name: Install the image's runtime dependencies
+        run: npm ci --prefix containers/sv-agent --omit=dev --ignore-scripts --engine-strict
+      - name: Audit both dependency locks
+        run: |
+          npm audit --prefix containers/sv-agent --omit=dev --audit-level=low
+          npm audit --prefix sdk/typescript --package-lock-only --audit-level=low
+      - name: Exercise the native CLI, glob selection, and proxy transport
+        # Real installed Pi and dependencies against local fixtures, with no model calls.
+        run: node --test tests/node/sv_runtime_dependencies.test.mjs
+
+  oci_path_gate:
+    name: Decide whether OCI integration is relevant
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      run_oci: ${{ steps.filter.outputs.run_oci }}
+    steps:
+      # pull_request checks out GitHub's merge ref. Depth two contains the merge commit and its
+      # base parent, so HEAD^..HEAD is the complete PR change, not merely its last commit.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 2
+          persist-credentials: false
+      - name: Match OCI-relevant paths
+        id: filter
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+        run: |
+          set -euo pipefail
+          decision="$(python3 scripts/oci_ci_path_gate.py --event-name "$EVENT_NAME")"
+          case "$decision" in
+            true|false) ;;
+            *) echo "OCI path gate returned an invalid decision: $decision" >&2; exit 1 ;;
+          esac
+          echo "run_oci=$decision" >> "$GITHUB_OUTPUT"
+
+  oci-integration:
+    name: OCI backend against a real Docker engine
+    needs: oci_path_gate
+    # A failed filter must surface as a failed required check, never as a silently skipped Docker
+    # run. Pushes to main and workflow_dispatch return true without consulting the diff.
+    if: ${{ always() && (needs.oci_path_gate.result != 'success' || needs.oci_path_gate.outputs.run_oci != 'false') }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      PYTHONDONTWRITEBYTECODE: '1'
+      # Without this the module skips when no engine or image is present; here that is a failure.
+      SCANEVAL_REQUIRE_DOCKER: '1'
+    steps:
+      - name: Require a successful path decision
+        if: ${{ needs.oci_path_gate.result != 'success' }}
+        run: |
+          echo "The OCI path gate failed; refusing to hide the Docker check." >&2
+          exit 1
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.13'
+      - name: Install test dependencies
+        run: python -m pip install -e '.[dev]'
+      - name: Describe the engine
+        # model_provider_only needs an engine that applies gateway_mode_ipv4=isolated; the record
+        # of which engine ran is here when that test says it did not.
+        run: |
+          docker version
+          docker info
+      - name: Pull the pinned images
+        # The backend never pulls an image, so the tests use only what is already present.
+        run: |
+          docker pull python:3.12-slim@sha256:2f17fc044b579bab302c2e8054d3a686e2cb9a83de48e70534b94cd8ebbe06a9
+          docker pull semgrep/semgrep:1.177.0@sha256:acaac22ffc7b7cc5926de0751b223bce0b2491c33d18422fa72f632c78d81198
+      - name: OCI integration tests
+        # Real containers, local fixture services, and benign example.com retrieval; no model
+        # or target application is called. Native Linux also checks cross-UID capture reads.
+        run: python -m pytest -q -rs tests/test_v2_isolation_docker.py tests/test_v2_open_context_docker.py tests/test_v2_experiment_docker.py

scaneval-2.0.0a1/.github/workflows/publish-packages.yml

@@ -0,0 +1,116 @@
+name: Python alpha package
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/publish-packages.yml'
+      - 'pyproject.toml'
+      - 'MANIFEST.in'
+      - 'README_PYPI.md'
+      - 'LICENSE'
+      - 'src/**'
+      - 'scripts/check_python_package.py'
+  push:
+    branches: [main]
+    paths:
+      - '.github/workflows/publish-packages.yml'
+      - 'pyproject.toml'
+      - 'MANIFEST.in'
+      - 'README_PYPI.md'
+      - 'LICENSE'
+      - 'src/**'
+      - 'scripts/check_python_package.py'
+  workflow_dispatch:
+    inputs:
+      publish:
+        description: Publish the verified Python artifacts (requires PyPI trusted-publisher setup)
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+concurrency:
+  group: python-alpha-package-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  python-artifacts:
+    name: Python wheel and source archive
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.11'
+      - name: Build and validate distribution metadata
+        run: |
+          python -m pip install build twine
+          python - <<'PY'
+          import tomllib
+          from packaging.version import Version
+          with open('pyproject.toml', 'rb') as source:
+              project = tomllib.load(source)['project']
+          assert project['name'] == 'scaneval'
+          assert Version(project['version']).is_prerelease, 'This workflow publishes alpha releases only'
+          PY
+          python -m build --outdir "$RUNNER_TEMP/python-dist"
+          python -m twine check --strict "$RUNNER_TEMP"/python-dist/*
+      - name: Install and exercise the exact wheel and source archive
+        run: |
+          for artifact in "$RUNNER_TEMP"/python-dist/*; do
+            python scripts/check_python_package.py "$artifact"
+          done
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: python-dist-${{ github.sha }}
+          path: ${{ runner.temp }}/python-dist/*
+          if-no-files-found: error
+          retention-days: 14
+
+  verified-main:
+    name: Require successful main CI before publishing
+    if: github.event_name == 'workflow_dispatch' && inputs.publish
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Refuse publication outside main or before its CI passes
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPOSITORY: ${{ github.repository }}
+          RELEASE_SHA: ${{ github.sha }}
+          RELEASE_REF: ${{ github.ref }}
+        run: |
+          test "$RELEASE_REF" = refs/heads/main
+          gh api --method GET "repos/$REPOSITORY/actions/workflows/ci.yml/runs" \
+            -f head_sha="$RELEASE_SHA" -f status=success -f event=push > "$RUNNER_TEMP/ci.json"
+          python3 - <<'PY'
+          import json, os
+          from pathlib import Path
+          runs = json.loads(Path(os.environ['RUNNER_TEMP'], 'ci.json').read_text())['workflow_runs']
+          assert any(run['head_sha'] == os.environ['RELEASE_SHA'] and run['conclusion'] == 'success'
+                     and run['event'] == 'push' for run in runs), 'Current main CI has not passed'
+          PY
+
+  publish-pypi:
+    name: Publish Python alpha to PyPI
+    needs: [python-artifacts, verified-main]
+    if: github.event_name == 'workflow_dispatch' && inputs.publish
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    environment: pypi
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: python-dist-${{ github.sha }}
+          path: dist
+      - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3

scaneval-2.0.0a1/.gitignore

@@ -0,0 +1,35 @@
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+build/
+dist/
+.venv/
+venv/
+
+# Rust
+target/
+
+# Node
+node_modules/
+
+# IDE
+.vscode/
+.idea/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# ScanEval
+.repos/
+results/
+*.html
+
+# Scanner artifacts
+.securevibes/
+.claude/
+
+# case-build scratch dirs
+.*_build/

scaneval-2.0.0a1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ScanEval contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scaneval-2.0.0a1/MANIFEST.in

@@ -0,0 +1,29 @@
+include LICENSE README.md README_PYPI.md pyproject.toml .gitignore
+recursive-include src *.py *.json *.mts *.mjs
+recursive-include docs *.md *.json
+recursive-include examples *.py *.mts *.md *.json *.toml
+recursive-include schema *.json *.jsonl
+recursive-include corpus *.md *.json
+recursive-include scripts *.py
+recursive-include tools *.py
+recursive-include tests *.py *.json *.sarif *.mjs
+recursive-include sdk/typescript *.json *.ts *.mjs *.md LICENSE
+include sdk/typescript/.gitignore
+recursive-include containers/sv-agent Dockerfile Dockerfile.dockerignore *.json *.mjs *.md *.sh
+recursive-include skills *.md *.yaml
+recursive-include .github *.yml
+
+# Source archives carry runnable examples and test fixtures, never local runs or caches.
+prune .git
+prune .repos
+prune results
+prune build
+prune dist
+prune .venv
+prune .pytest_cache
+prune .claude
+prune .securevibes
+prune sdk/typescript/node_modules
+prune sdk/typescript/dist
+prune containers/sv-agent/node_modules
+global-exclude __pycache__ *.py[cod] .DS_Store

scaneval-2.0.0a1/PKG-INFO

@@ -0,0 +1,145 @@
+Metadata-Version: 2.4
+Name: scaneval
+Version: 2.0.0a1
+Summary: Alpha framework for reproducible security-scanner evaluation and observability.
+Author: ScanEval contributors
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Har1sh-k/scaneval
+Project-URL: Repository, https://github.com/Har1sh-k/scaneval
+Project-URL: Documentation, https://github.com/Har1sh-k/scaneval/blob/main/docs/INITIAL_BUILD.md
+Project-URL: Issues, https://github.com/Har1sh-k/scaneval/issues
+Keywords: security,scanner,evaluation,benchmark,observability
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: POSIX
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Security
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: jsonschema<5,>=4.23
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: PyYAML>=6.0; extra == "dev"
+Requires-Dist: ruamel.yaml>=0.18; extra == "dev"
+Provides-Extra: official-adapters
+Requires-Dist: semgrep>=1.77.0; extra == "official-adapters"
+Dynamic: license-file
+
+# ScanEval
+
+ScanEval evaluates structured security-scanner results against a versioned plan and
+supplied review decisions. Import the Python package and call
+`scaneval.evaluate(plan, result, decisions)` to produce an evaluation report offline.
+The same distribution includes a CLI, versioned data contracts, replay, local reports,
+and optional tracing through `scaneval.observer`.
+
+**Alpha version: `2.0.0a1`.** No benchmark results or scanner comparisons are published.
+The selected corpus contains 50 CVEs, but fixed controls remain unvalidated and scanner
+miss rates remain unmeasured. Live scanner integrations need separate installations and
+configuration. The default SV-Agent harness pin cannot yet produce a completed
+source-scan score. See the
+[current capabilities and limits](https://github.com/Har1sh-k/scaneval/blob/main/README.md#current-limits).
+
+## Install and evaluate
+
+Requires Python 3.11 or newer. Source materialization also requires Git. From a virtual
+environment on macOS or Linux:
+
+```bash
+python -m pip install "scaneval==2.0.0a1"
+```
+
+Load your saved ScanEval evaluation plan, normalized scan result, and review decisions:
+
+```python
+import json
+from pathlib import Path
+import scaneval
+
+plan = json.loads(Path("plan.json").read_text(encoding="utf-8"))
+result = json.loads(Path("result.json").read_text(encoding="utf-8"))
+decisions = json.loads(Path("decisions.json").read_text(encoding="utf-8"))
+
+report = scaneval.evaluate(plan, result, decisions)
+print(json.dumps(report, indent=2))
+```
+
+Scoring requires structured saved records and supplied finding-review decisions bound to
+those records. The plan assigns targets and controls; the decisions say which findings
+establish their alleged issues. Unreviewed findings remain unresolved. The evaluation API
+validates these inputs and computes scores; producing scanner findings and adjudicating
+natural-language allegations are separate steps.
+
+## Try the offline demo
+
+```bash
+scaneval demo demo-run
+scaneval replay demo-run --output demo-replay.json
+```
+
+Open `demo-run/report.html` locally. The demo uses fabricated evaluator fixtures and
+makes no scanner or model calls. Output paths must be new. Run `scaneval --help` for
+commands; the
+[execution guide](https://github.com/Har1sh-k/scaneval/blob/main/docs/INITIAL_BUILD.md)
+explains scanner configuration, review, and saved evidence.
+
+## Instrument a Python harness
+
+The public Python SDK is available from `scaneval.observer`. Emit events at the model,
+tool, context, and finding boundaries your harness exposes:
+
+```python
+from scaneval.observer import Observer
+
+events = []
+observer = Observer(
+    mode="metadata",
+    sink=events.append,
+    run_id="example-run",
+    producer_id="my-harness",
+)
+observer.emit(
+    type="model.request",
+    capture_status="complete",
+    call_id="call-1",
+    metadata={"model": "example-model"},
+)
+observer.flush()
+observer.close()
+```
+
+This example records one event in memory; it calls no model. Recording defaults to
+`off` unless the caller chooses a mode. Capture gaps remain explicit, and traces cannot
+prove hidden reasoning. The
+[Observer SDK guide](https://github.com/Har1sh-k/scaneval/blob/main/docs/OBSERVER_SDK.md)
+describes sinks, recording modes, and capture states. This Observer is part of `scaneval`;
+no separate Python Observer distribution is needed. A
+[TypeScript SDK](https://github.com/Har1sh-k/scaneval/tree/main/sdk/typescript)
+shares the trace contract and remains available as source. This release publishes only
+the Python `scaneval` distribution.
+
+## Cases, datasets, and execution limits
+
+Start with the
+[selected CVE corpus](https://github.com/Har1sh-k/scaneval/blob/main/corpus/diverse/README.md),
+[bring your own corpus](https://github.com/Har1sh-k/scaneval/blob/main/docs/BRING_YOUR_OWN_CORPUS.md),
+or [dataset adapters](https://github.com/Har1sh-k/scaneval/blob/main/docs/DATASET_ADAPTERS.md).
+Scoring uses supplied review decisions; unreviewed findings remain unresolved.
+
+The local execution backend runs scanners as the operator and enforces no isolation.
+The OCI backend supports specific adapters and configurations; read the
+[threat model](https://github.com/Har1sh-k/scaneval/blob/main/docs/THREAT_MODEL.md)
+before live runs. Keep scanner outputs, private findings, credentials, and content traces
+local.
+
+[Source and issues](https://github.com/Har1sh-k/scaneval) are on GitHub. ScanEval is
+[MIT licensed](https://github.com/Har1sh-k/scaneval/blob/main/LICENSE); external scanners
+and source snapshots retain their own licenses.

scaneval-2.0.0a1/README.md

@@ -0,0 +1,149 @@
+# ScanEval
+
+ScanEval evaluates security scanners on versioned cases and records how instrumented harnesses reach their findings. Scoring uses structured outputs and recorded review decisions. Static corpus labels can be assessed separately through evidence-bound machine review.
+
+> Alpha (`2.0.0a1`). No benchmark results are published. The [selected CVE corpus](corpus/diverse/README.md) contains 50 CVEs from 32 repositories across seven target languages: 9 Critical, 13 High, 20 Medium and 8 Low by the documented published-CVSS policy. Its 48 pinned vulnerable exports have mechanical checks and a separate machine-review workflow for static labels. Selection emphasizes contextual security reasoning; scanner miss rates remain unmeasured. Fixed controls remain unvalidated. Scanner outputs and traces stay local.
+
+## Python alpha package
+
+The planned PyPI release is `scaneval==2.0.0a1`; publication is pending. The Python distribution includes the evaluation API, CLI, and optional tracing through `scaneval.observer`. After publication, install the alpha:
+
+```bash
+python -m pip install scaneval==2.0.0a1
+```
+
+```python
+import scaneval
+
+report = scaneval.evaluate(plan, result, decisions)
+```
+
+The arguments are structured saved ScanEval records: an evaluation plan, normalized scanner result, and supplied review decisions bound to those records. Scoring is offline; scanner execution and finding adjudication are separate steps. See the [Python package guide](README_PYPI.md) for loading the records and the [Observer guide](docs/OBSERVER_SDK.md) for optional instrumentation. The TypeScript Observer remains available as source; this release publishes only the Python `scaneval` distribution. [Release instructions](docs/RELEASING.md) cover artifact verification and PyPI publishing. Use a source checkout for the corpus, examples, and development workflow below.
+
+## Quick start
+
+Requires Git and Python 3.11+. Live runs also need the configured scanner.
+From a terminal on macOS or Linux:
+
+```bash
+git clone https://github.com/Har1sh-k/scaneval.git
+cd scaneval
+python3 -m venv .venv
+source .venv/bin/activate
+python -m pip install -e ".[dev]"
+scaneval demo results/diagnostic-demo
+scaneval replay results/diagnostic-demo --output results/diagnostic-replay.json
+# Open results/diagnostic-demo/report.html locally.
+```
+
+The demo uses fabricated evaluator fixtures, not a live scanner or admitted CVEs. It exercises scoring without model calls. Output paths must be new.
+
+Start with the [CVE corpus](corpus/diverse/README.md) for the 50 curated cases, severity sources, selection rationale and machine review. [Dataset adapters](docs/DATASET_ADAPTERS.md) add separate PR-review and CVE-detection tracks for AACR-Bench, ReviewBench, Martian, OpenSSF, and your own datasets. The [execution guide](docs/INITIAL_BUILD.md#run-configuration) explains scanner configuration and review. To supply your own security fixes, findings, or documents as case packs, follow [bring your own corpus](docs/BRING_YOUR_OWN_CORPUS.md).
+
+## Evaluation
+
+One scan can cover several targets on the same input. A finding needs an accepted security allegation, not just a matching line and vulnerability category. Unreviewed findings remain unresolved; they are not automatically false positives.
+
+The scorer reports known-target detection, recall within review budgets, and false allegations against assigned fixed/safe controls. Reviewed precision is estimated separately, from a seeded, human-reviewed sample of delivered claims. Unavailable controls do not become a zero false-alarm rate. Errors and incomplete scans remain distinct from successful scans with no findings. Recorded decisions can be replayed offline.
+
+| Command | Purpose |
+|---|---|
+| `scaneval validate <kind> <path>` | Validate a versioned contract. |
+| `scaneval demo <new dir>` | Create the fabricated conformance bundle. No scanner or model runs. |
+| `scaneval score --plan --result --decisions` | Score separately stored records. |
+| `scaneval replay <bundle>` | Recompute a saved bundle offline. |
+| `scaneval report <bundle> --output` | Render a standalone HTML report. |
+| `scaneval investigate <run> --output <new-directory>` | Export a local investigation view and JSON with judgments, trace links, capture states, and artifact bindings ([guide](docs/INVESTIGATION.md)). |
+| `scaneval corpus init\|add-snapshot\|import\|validate\|approve\|admit\|disposition` | Prepare case packs, run mechanical checks, and record explicit reviews and admissions. |
+| `scaneval corpus machine-prepare\|machine-record\|machine-check` | Prepare evidence packets and validate machine judgments of static CVE labels without HITL. |
+| `scaneval corpus add-change-set\|pr-scope\|canonical` | Declare a base/head change set, state which targets and controls are scored in its PR review, and group records by canonical root cause. |
+| `scaneval plan --pack --snapshot-id --tree-hash --output` | Build an evaluation plan for one materialized input, or with `--mode pr --change-set-id` and the hashes that identify it, the PR plan of a change set. |
+| `scaneval run <config> --output <new dir>` | Run a frozen configuration and save its artifacts. |
+| `scaneval matrix freeze\|run\|report` | Freeze and execute the three evaluation conditions, retaining missingness and paired effects ([guide and diagnostic fixture](docs/EXPERIMENT_MATRIX.md)). |
+| `scaneval dataset list\|fetch\|import\|validate\|inspect` | Load a pinned public dataset, native JSON, or an installed Python dataset adapter ([guide](docs/DATASET_ADAPTERS.md)). |
+| `scaneval benchmark plan\|run\|judge-packet\|score` | Evaluate imported datasets with existing scanner adapters, frozen schedules, and supplied machine judgments. |
+| `scaneval blinding check\|review` | Dry-run a metadata blinding map against its snapshots, and record a named review of it. |
+| `scaneval import sarif <log> --pack --snapshot-id --tree-hash --system-id --output` | Import one run of a saved SARIF 2.1.0 log into a new bundle for review, offline. |
+| `scaneval review init\|record\|approve\|status` | Prepare and record the review of an invocation bundle. |
+| `scaneval diagnose context-coverage <bundle>` | Compare captured context spans with labeled targets. Does not change scores. |
+| `scaneval condition <bundle>... [--output]` | Inspect saved condition evidence and separate system/policy/status coverage, offline. |
+| `scaneval aggregate <run dir>... --output` | Weight every scheduled assignment of saved runs into corpus metrics with cluster-bootstrap intervals. |
+| `scaneval severity <run dir>... --pack --output [--html]` | Compare adjudicated claimed severity with reference CVSS, with coverage, provenance and offline replay. See [severity calibration](docs/SEVERITY.md). |
+| `scaneval aggregate <run dir>... --by-condition --output` | Report separate condition/status/policy strata, complete missingness, and descriptive target outcomes. |
+| `scaneval compare <run dir>... --baseline --candidate --output` | Compare two systems assigned the same frozen work, with paired intervals. Decides no promotion. |
+| `scaneval precision sample\|queue\|record\|estimate` | Sample delivered claims from saved runs, record human reviews, and estimate reviewed precision. Does not change scores. |
+| `scaneval gate --policy --comparison --output` | Hold a comparison, and optionally precision estimates, to a gate policy and write a pass, fail, or inconclusive decision with every requirement explained. Promotes nothing. |
+
+Run `scaneval <command> --help` for required arguments. `corpus validate --snapshot-id`, `blinding check`, `run`, `dataset fetch`, and `benchmark run` can reach the network; contacts depend on the configured sources and scanners. Exit code `2` means the command could not be carried out, `1` means it ran and reports a negative result, and `0` means it ran and reports nothing wrong. None certifies that a security label is correct.
+
+## Scanners and visibility
+
+The current adapters are `semgrep` with pinned local rules, `llm-harness` for an own-harness integration that drives the Fieldglass/SV-Agent engine directly, `deepsec` for the third-party scanner, and an initial `runsortie` integration that retains the evidence contract for full RunSortie orchestration: its Fieldglass engine, a primary agent's packet review, a separate challenger model, and RunSortie's own evidence commit. The production RunSortie route is currently fail-closed, as described below. `semgrep` (`--baseline-commit`) and `deepsec` (`process --diff`) also review a native PR request that names a base and a head commit in the scanner's workspace; [Current capabilities](docs/INITIAL_BUILD.md) says what each does with it. Scanner CLIs are installed separately. The optional `official-adapters` extra installs Semgrep; a reproducible comparison must also pin its version and rules. To build or review another execution integration, use the [scanner adapter builder skill](skills/scanner-adapter-builder/SKILL.md); it covers native workflow evidence and verification, not dataset import or corpus-label review.
+
+The **ScanEval Observer SDK**, available in Python and TypeScript, records model calls, tool use, supplied context, and finding lifecycle events at instrumented boundaries. It is opt-in. Native collectors can import Claude Code and Codex records without claiming to see decisions the harness never exposed. Capture gaps remain explicit; traces cannot prove hidden reasoning or absence of memorization.
+
+For the TypeScript SDK and harness driver:
+
+```bash
+npm ci --prefix sdk/typescript
+npm run build --prefix sdk/typescript
+```
+
+On its default `local` runtime the `llm-harness` adapter needs a local harness checkout and the built SDK; its `oci_image` runtime uses the harness and observer SDK in its pinned image instead ([SV-Agent image runtime](docs/OPEN_CONTEXT.md#sv-agent-image-runtime)). The `deepsec` adapter needs an installed scanner workspace; it runs the scanner in a separate private workspace. The initial `runsortie` adapter parses the exact RunSortie `af8f240…`/`0.1.0` configuration and retains a deeply checked native evidence/completion implementation, but secure v1 has no runnable direct-agent boundary. Preparation records the requested routes as not runnable, and scan returns `unsupported` before checkout identity/content inspection or launch. Its deterministic fake and fabricated-agent conformance paths exercise the retained commit/tree, packet, prompt, journal, budget, snapshot, and reconciliation checks only; they are not live readiness ([details and limits](docs/INITIAL_BUILD.md#adapters)).
+
+Guides: [Observer SDK](docs/OBSERVER_SDK.md), [native CLI collectors](docs/COLLECTORS.md), [context-coverage diagnostics](docs/DIAGNOSTICS.md).
+
+Implementation review: [Inspect AI visibility and observability comparison](docs/INSPECT_OBSERVABILITY_COMPARISON.md).
+
+## Current limits
+
+- Full-scan execution, native PR execution (a case pack declares a change set and states which targets and controls each PR review scores), case packs, source export, review, scoring, replay, and reports are implemented.
+- Each run freezes its evaluation schedule (every assignment, pre-registered plan, and vulnerable/fixed pair) before it prepares an input. An input it cannot prepare is recorded with its skipped assignments, and the other inputs still run.
+- The `metadata_blinded` profile applies reviewed token edits to documentation and display metadata. The `identity_reduced` profile adds hash-pinned identity-document/instruction removals and narrowly verified package URL leaf edits. An explicit strict policy validates answer coverage and records skipped transformations and residual cues; a profile name alone does not establish an experimental condition. Runtime code, dependency declarations, licenses, build/CI/security configuration, and source paths stay intact. Originals and transformation manifests remain evaluator-side. See [identity reduction](docs/IDENTITY_REDUCTION.md) for the reviewed workflow and limits.
+- A saved SARIF 2.1.0 log imports offline into a bundle that review, score, and replay read; its execution report is recorded unverified ([SARIF import](docs/SARIF_IMPORT.md)). Other saved vendor formats have no importer.
+- A system whose 2.1 run configuration selects the `oci` execution backend runs each scanner process in a locked-down Docker container from a digest-pinned image, with its network policy enforced (`unrestricted` is recorded as unenforced). `semgrep` can run under it, and so can the SV-Agent (`securevibes-agent`) harness through `llm-harness` with `config.runtime: "oci_image"`. That image runtime carries out full scans only, and only under the `oci` backend with the `public_research` policy; it is refused anywhere else. `deepsec`, `runsortie`, and `llm-harness` on `config.runtime: "local"` (a host checkout), are refused under `oci`. Under the `local` execution backend, systems run as the operator, unenforced. Implemented isolation does not show that a live-model scan has been validated or completed: at the default harness pin a scan still cannot reach a completed source-scan score, and no broad benchmark readiness is claimed. See [the threat model](docs/THREAT_MODEL.md) and [SV-Agent image runtime](docs/OPEN_CONTEXT.md#sv-agent-image-runtime).
+- [Closed-book standard](docs/CLOSED_BOOK.md) preserves repository identity and classifies saved history, retrieval, network, and provider evidence as attested, declared-only, failed, or unavailable. A local policy declaration never attests isolation.
+- The explicit `open_context` condition supplies bounded original ancestor history and controlled, recorded public retrieval under OCI. It measures assisted investigation; known-vulnerability retrieval may contribute to detection. History/capture coverage and provider visibility gaps stay visible, and its results remain separate from closed-book and identity-reduced observations ([guide and runnable fixture](docs/OPEN_CONTEXT.md)).
+- `runsortie` has been validated only deterministically: by a stand-in that follows RunSortie's scan control flow, and by an opt-in run of the real pinned CLI with fabricated engine and agents. It has not run against a live model. Secure v1 has no runnable native direct-agent route: preparation records `runnable: false`, and invocation returns `unsupported` with an empty command before checkout identity/content inspection, credentials, probes, adapter-private state, or any RunSortie/agent process. Codex persists an API-key credential file readable by workspace tools; Claude 2.1.310 can fetch and apply remote managed helpers and additional filesystem roots; Cursor and Gemini likewise lack the externally attestable process/filesystem sandbox plus endpoint-specific egress control required here. The retained fake/conformance path tests native evidence and completion logic only and cannot be enabled by configuration. Unsupported assignments remain visible in evaluation accounting.
+- Native PR review runs each scanner's own diff workflow over a neutral two-commit history of a declared change set: the harness's pr mode for `llm-harness`, `--baseline-commit` for `semgrep`, and `process --diff` for `deepsec`. An adapter without a PR mode is recorded as unsupported and stays in every denominator, and a full scan of the head never stands in for a PR review ([native PR review](docs/INITIAL_BUILD.md#native-pr-review)).
+- Corpus aggregation and paired comparison read saved, frozen run directories ([aggregation](docs/AGGREGATION.md)). They compute no reviewed precision and decide no promotion.
+- Reviewed precision comes only from people reviewing a seeded probability sample of delivered claims ([precision guide](docs/PRECISION.md)). It is not a repository false-positive rate.
+- A gate decision holds a saved comparison to a policy you write and says whether its requirements held ([gate guide](docs/GATE.md)). It promotes nothing: the workflow that owns the harness decides.
+- Fifty CVEs are selected in the [diverse corpus](corpus/diverse/README.md), including GitLab, Kubernetes, Airflow, Langflow, LangChain and Twenty. The [Astra-authored review skill](skills/cve-corpus-review/SKILL.md) supports machine review without a human gate; shared deployment context, claim-specific runtime validation and scanner preflight remain separate work. Java and Ruby are accepted by the corpus contract, but the current adapters still record them as unsupported. No representative benchmark or scanner comparison is claimed.
+- External [dataset tracks](corpus/benchmarks/README.md) provide 469 PR cases and 223 additional OpenSSF CVE cases. Their publisher labels remain separate from the curated corpus. Dataset benchmark runs use the local execution backend; scoring measures reference coverage and named fixed controls, not true precision or an official upstream leaderboard score. Missing or uncertain semantic judgments remain explicit.
+
+[Current capabilities](docs/INITIAL_BUILD.md) describes the implementation. [Design decisions](docs/DESIGN_DECISIONS.md) and [evaluation math](docs/EVALUATION_MATH.md) describe the broader contract and planned work.
+
+## Development
+
+```bash
+python -m pytest -q
+npm test --prefix sdk/typescript
+```
+
+Tests use local fixtures and mock model runners. They do not launch paid model evaluations. CI checks Python 3.11, 3.13, and 3.14, the TypeScript SDK, and demo/replay from an installed wheel.
+
+| Path | Contents |
+|---|---|
+| `src/scaneval/` | CLI, corpus, runner, evaluator, adapters, collectors, and Python Observer |
+| `src/scaneval/schemas/` | Versioned evaluation contracts |
+| `sdk/typescript/` | TypeScript Observer SDK |
+| `schema/v2/` | Shared trace contract and conformance fixtures |
+| `corpus/diverse/` | 50 selected CVEs, sourced severity distribution, pinned sources and machine review |
+| `scripts/sanitize_native_trace.py` | Scrub native records for collector fixtures |
+| `tests/` | Contract, scoring, runner, adapter, Observer, and collector tests |
+| `docs/` | Public design and usage guides |
+
+Generated runs belong under ignored `results/`; source caches belong under ignored `.repos/`. Do not commit scanner outputs, private findings, credentials, or content traces.
+
+## License
+
+[MIT](LICENSE). Source snapshots and external scanners retain their own licenses.
+
+### Model cutoffs and recent CVE research
+
+Inspect sourced provider cutoffs and check reviewed public-evidence chronology with
+`scaneval model-cutoffs list` and `scaneval model-cutoffs check`. The read-only
+helper preserves unknown dates and routes, distinguishes training from knowledge
+cutoffs, and reports the 2 Low / 3 Medium / 3 High / 2 Critical research target.
+See [model cutoffs](docs/MODEL_CUTOFFS.md) for usage, evidence limits, and maintenance.