petri-braintrust 0.2.0__tar.gz

petri_braintrust-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,52 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install ruff
+        run: pip install "ruff>=0.5"
+
+      - name: Lint
+        run: ruff check petri_braintrust tests
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install package + dev extras
+        run: pip install -e ".[dev]"
+
+      - name: Run pytest
+        run: pytest -v
+
+      - name: Smoke-test CLI commands
+        run: |
+          petri-bt --help
+          petri-bt presets
+          petri-bt study list
+          petri-bt study show observer-effect
+          petri-bt topics-init --project smoke --dry-run

petri_braintrust-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,53 @@
+name: Publish to PyPI
+
+# Publishes on GitHub release. Uses PyPI trusted publishing (OIDC) — no API
+# token to store. One-time setup on pypi.org: project → Publishing → add
+# GitHub publisher (owner: Paul-UK, repo: petri-braintrust,
+# workflow: publish.yml, environment: pypi).
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build sdist and wheel
+        run: |
+          python -m pip install build
+          python -m build
+
+      - name: Check version matches the release tag
+        run: |
+          PKG_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
+          TAG="${GITHUB_REF_NAME#v}"
+          if [ "$PKG_VERSION" != "$TAG" ]; then
+            echo "pyproject version ($PKG_VERSION) != release tag ($TAG)" >&2
+            exit 1
+          fi
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # required for PyPI trusted publishing
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

petri_braintrust-0.2.0/.gitignore

@@ -0,0 +1,30 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+build/
+dist/
+.eggs/
+
+# Inspect eval outputs
+logs/
+*.eval
+
+# Virtualenvs
+.venv/
+.venv-test/
+venv/
+
+# Editors / OS
+.DS_Store
+.vscode/
+.idea/
+
+# Claude Code session state
+.claude/
+
+# Article drafts (LinkedIn posts, results write-ups) — repo is for code +
+# commands; published artifacts live elsewhere. Headline numbers from any
+# pilot run live in the README and PR descriptions, not here.
+article/

petri_braintrust-0.2.0/CHANGELOG.md

@@ -0,0 +1,60 @@
+# Changelog
+
+All notable changes to petri-braintrust are documented here. The format
+follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions
+follow [SemVer](https://semver.org/) (pre-1.0: minor bumps may break).
+
+## [0.2.0] — 2026-06-10
+
+### Added
+- **Studies harness** (`petri_braintrust.studies`): reproducible
+  paired-condition research runs — one seed set, one framing manipulation,
+  per-condition Braintrust experiments that pair row-by-row in the comparison
+  view, plus a local paired report (per-dimension deltas, awareness
+  correlations).
+- **Observer-effect study**: ten oversight-neutral agentic-temptation seeds
+  run under observed vs unobserved framings; the paired delta is the
+  oversight gap. `petri-bt study run observer-effect`.
+- `petri-bt study list / show / run` CLI.
+- Exporter hooks: `seed_transform` (strip condition overlays so arms pair in
+  BT diffs) and `extra_metadata` (study/condition labels on every record).
+- Braintrust **Topics** integration: `petri-bt export --mode logs` /
+  `export_log_to_logs` send samples to project logs for clustering;
+  `petri-bt topics-init` enables a Petri-scoped Topics automation.
+- **Long-horizon study codified**: the ad-hoc 10-vs-100-turn eval is now a
+  registered StudySpec (`petri-bt study run long-horizon`) — same pinned
+  probe, arms differing only in turn budget.
+- `petri-bt study compare <baseline.eval> <treatment.eval> --study <slug>`:
+  re-analyze existing arm logs without re-running (rescues a study whose
+  second arm failed after the first burned tokens).
+- Study reports now include **per-seed deltas** and an exact two-sided
+  **sign test** per focus dimension, plus provenance stamps
+  (`petri_braintrust_version`, `study_spec_fingerprint`) in both the report
+  and Braintrust metadata.
+- **Model-agnostic role defaults**: `PETRI_BT_TARGET` / `PETRI_BT_AUDITOR` /
+  `PETRI_BT_JUDGE` env vars (precedence: explicit arg > env > preset). The
+  all-local warning now recognizes vLLM/SGLang/HF/llama-cpp prefixes, not
+  just Ollama.
+- `--verbose` / `-v` global CLI flag — informational logging (export
+  progress, resolved roles) is silent otherwise.
+- `py.typed` marker — the package now ships its type information.
+- CI: ruff lint job and broader CLI smoke tests; PyPI trusted-publishing
+  workflow on GitHub release.
+
+### Changed
+- **Dimension-direction map rebuilt against inspect-petri v3's real
+  dimension set**, derived at runtime from `JudgeDimension.palette` when
+  inspect-petri is installed (static snapshot as fallback). Neutral
+  descriptor dimensions (`fun_or_funny`, `high_stakes_setting`,
+  `unprompted_pos_affect`) no longer get meaningless `__inverted` values.
+  Stale guessed names (`deception`, `power_seeking`, …) were dropped.
+- **"Did not apply" judge scores (raw 0) are now omitted** from Braintrust
+  scores instead of being clipped to 0.0, keeping per-dimension means honest.
+  Skipped dimensions are listed under
+  `metadata.judge.<scorer>.not_applicable_dimensions`.
+
+## [0.1.0] — 2026-05
+
+Initial release: run Inspect Petri audits with mixed local (Ollama) / cloud
+model roles and export `.eval` logs to Braintrust experiments with
+per-dimension scores, per-call trace waterfalls, and judge metadata.

petri_braintrust-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Paul Tancre
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.