touchstone-dqi 0.0.1__tar.gz

touchstone_dqi-0.0.1/.env.example

@@ -0,0 +1,3 @@
+# Touchstone reads nothing from the network by default.
+# Pack credentials go here and are passed only to packs that declare them.
+# OPENAI_API_KEY=

touchstone_dqi-0.0.1/.githooks/commit-msg

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec python3 "$(git rev-parse --show-toplevel)/scripts/check_commit_msg.py" "$1"

touchstone_dqi-0.0.1/.github/workflows/README.md

@@ -0,0 +1,8 @@
+`ci.yml` runs three jobs.
+
+- `test`: lint, format check, unit tests.
+- `commit-messages`: every commit in a pull request is checked against
+  `scripts/check_commit_msg.py`. See CONTRIBUTING.md.
+- `offline-install`: builds a wheel, vendors its dependencies, then installs and runs
+  with no package index. Touchstone has to work in an environment with no egress, so
+  that claim is tested on every push rather than asserted in the README.

touchstone_dqi-0.0.1/.github/workflows/ci.yml

@@ -0,0 +1,57 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.12"
+      - run: uv sync --all-extras --dev
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run pytest -q
+
+  commit-messages:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - name: check every commit message in the pull request
+        run: |
+          set -e
+          base="${{ github.event.pull_request.base.sha }}"
+          head="${{ github.event.pull_request.head.sha }}"
+          status=0
+          for sha in $(git rev-list "$base".."$head"); do
+            git log -1 --format=%B "$sha" > /tmp/msg
+            echo "--- $(git log -1 --format=%h\ %s "$sha")"
+            python3 scripts/check_commit_msg.py /tmp/msg || status=1
+          done
+          exit $status
+
+  offline-install:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - name: build a wheel and vendor its dependencies
+        run: |
+          pip install build
+          python -m build --wheel --outdir dist
+          pip download -d vendor dist/*.whl
+      - name: install and run with networking disabled
+        run: |
+          python -m venv .airgap
+          .airgap/bin/pip install --no-index --find-links vendor touchstone-dqi
+          .airgap/bin/touchstone version

touchstone_dqi-0.0.1/.github/workflows/publish.yml

@@ -0,0 +1,43 @@
+name: publish
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      package:
+        description: "Which package to publish"
+        required: true
+        type: choice
+        options: [touchstone-dqi, dqi]
+
+jobs:
+  touchstone:
+    if: github.event_name == 'release' || inputs.package == 'touchstone-dqi'
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - run: pip install build
+      - run: python -m build
+      - uses: pypa/gh-action-pypi-publish@release/v1
+
+  dqi:
+    if: inputs.package == 'dqi'
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - run: pip install build
+      - run: python -m build packaging/dqi --outdir dist
+      - uses: pypa/gh-action-pypi-publish@release/v1

touchstone_dqi-0.0.1/.gitignore

@@ -0,0 +1,41 @@
+# Internal planning documents. Never commit these.
+# They live outside this repo, in products/touchstone/. This is a second line of defence.
+*-NIGERIA-FIRST.md
+*-ASQI-TEARDOWN.md
+*-DESIGN.md
+*-BUILD-PLAN.md
+*-NAMING.md
+PLAN-*.md
+CONTEXT.md
+HANDBOOK.md
+productPlan.md
+DECISIONS.md
+RUNLOG.md
+notes/
+internal/
+
+# Python
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+
+# Environment
+.env
+.env.*
+!.env.example
+
+# Runtime output
+runs/
+bundles/
+logs/
+
+# OS
+.DS_Store

touchstone_dqi-0.0.1/CONTRIBUTING.md

@@ -0,0 +1,153 @@
+# Contributing
+
+Read this before your first commit. The rules here are enforced by CI, not by review.
+
+## Commit messages
+
+Write like an engineer with work to do. Short, factual, imperative.
+
+```
+add wilson interval to rate estimator
+fix digest resolution for multi-arch images
+drop unused seed parameter from plan loader
+```
+
+**Rules, all enforced by `scripts/check_commit_msg.py`:**
+
+1. Subject is 50 characters or fewer. Hard limit 72.
+2. Imperative mood. `add`, not `added` or `adds`.
+3. Lowercase first word. No trailing full stop.
+4. Body optional. Add one only to explain *why*, never to restate the diff.
+   Wrap at 72. Blank line between subject and body.
+5. **No em dashes or en dashes.** Anywhere. Use a comma or a full stop.
+6. No marketing adjectives. The linter rejects: comprehensive, robust, seamless,
+   powerful, cutting-edge, state-of-the-art, elegant, blazing, leverage, delve,
+   utilise, holistic, streamline, unlock, empower.
+7. No AI attribution. No `Co-Authored-By` for a tool, no `Generated with`, no emoji.
+8. No issue-tracker noise in the subject. Put refs in the body.
+
+**Why 50 characters.** `git log --oneline` is how the history gets read. A subject
+that wraps is a subject nobody reads.
+
+**Why the adjective list.** Those words carry no information about the change. If a
+commit needs to say the work is good, the work is not good enough.
+
+### Bad, and why
+
+```
+feat: Implement comprehensive and robust validation logic for the plan
+loader module - this seamlessly handles all edge cases
+
+Generated with Claude Code
+Co-Authored-By: Claude <noreply@anthropic.com>
+```
+
+Too long. Past tense marketing. Em dash. Claims coverage it cannot prove.
+Attribution nobody needs.
+
+```
+validate plan against pack manifests
+```
+
+That is the same commit.
+
+## Code
+
+Same principle. Clarity first, volume last.
+
+1. **No comment that restates the code.** `# increment counter` above `i += 1` gets
+   deleted in review.
+2. **Comment only what the code cannot say**: a why, a constraint, a reference to a
+   spec or a paper, a known sharp edge.
+3. **Docstrings on public functions only**, one line, saying what it returns. Skip the
+   Args and Returns blocks unless the types are genuinely unclear.
+4. **No defensive noise.** Do not catch an exception you cannot handle. Do not check
+   for a condition the type system already rules out.
+5. **Functions do one thing.** If you need a section comment inside a function, that
+   section is a function.
+6. **No abbreviations that need a glossary.** `manifest`, not `mfst`.
+7. **Fail closed.** A missing file, an unresolvable digest or a stale hash is an error
+   with a non-zero exit. Never a warning, never a silent default. This tool produces
+   evidence, and evidence that degraded quietly is worse than no evidence.
+8. **No country, language or regulator logic in `src/`.** All of it belongs in packs
+   and in `mappings/`. CI fails the build on a two-letter country code in `src/`.
+
+### Bad
+
+```python
+def process_items(items: list[dict]) -> dict:
+    """
+    Process the items.
+
+    Args:
+        items: A list of items to process.
+
+    Returns:
+        A dictionary containing the processed results.
+    """
+    # Initialise the results dictionary
+    results = {}
+    # Loop over each item in the list
+    for item in items:
+        # Check if the item is valid
+        if item is not None:
+            results[item["id"]] = item
+    return results
+```
+
+### Good
+
+```python
+def index_by_id(items: list[ItemRecord]) -> dict[str, ItemRecord]:
+    """Index records by item_id. Raises on a duplicate."""
+    out = {}
+    for item in items:
+        if item.item_id in out:
+            raise DuplicateItemError(item.item_id)
+        out[item.item_id] = item
+    return out
+```
+
+Shorter, typed, named for what it does, and it fails closed on the case the first
+version silently swallowed.
+
+## Documentation
+
+The README is the only document most people read. Optimise it for someone who has
+five minutes and a system they want to test.
+
+1. What it is, in two sentences.
+2. What it needs to run.
+3. A command that produces output.
+4. Everything else in `docs/`.
+
+No badges beyond build and licence. No feature list written as marketing. No emoji.
+
+## Naming
+
+The instrument is **Touchstone**. The index it computes is **DQI**.
+
+- Tool versions: `Touchstone 1.0`.
+- Specification versions: `DQI Specification v1.0`.
+- Never `DQI 1.0` on its own. It is ambiguous and the ambiguity is permanent.
+
+## Before you push
+
+```bash
+uv run ruff check . && uv run ruff format --check .
+uv run pytest
+uv run python scripts/check_commit_msg.py .git/COMMIT_EDITMSG
+```
+
+Install the hooks once and the last one runs itself:
+
+```bash
+git config core.hooksPath .githooks
+```
+
+## Pull requests
+
+- One change per pull request.
+- The description says what changed and why. Not how, the diff says how.
+- A pull request that changes a contract in `src/touchstone/contracts/` is a major
+  version and needs a note saying what breaks.

touchstone_dqi-0.0.1/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

touchstone_dqi-0.0.1/PKG-INFO

@@ -0,0 +1,153 @@
+Metadata-Version: 2.5
+Name: touchstone-dqi
+Version: 0.0.1
+Summary: Seal evaluation output into an evidence bundle anyone can re-check offline.
+Project-URL: Homepage, https://github.com/Quantile-Labs/touchstone
+Project-URL: Source, https://github.com/Quantile-Labs/touchstone
+Author: Quantile Labs
+Maintainer: Quantile Labs
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: ai-assurance,audit,evaluation,reproducibility
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.12
+Requires-Dist: pydantic>=2.11
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: typer>=0.16
+Description-Content-Type: text/markdown
+
+# Touchstone
+
+Touchstone seals evaluation output into an evidence bundle: the plan, the per-item
+observations, and a SHA-256 over every file. Anyone can re-check that bundle offline,
+without this tool and without trusting whoever produced it.
+
+## Status
+
+Early. Three of the seven pipeline commands work. The rest are stubs that exit 2, and
+nothing is published to PyPI yet.
+
+## Requirements
+
+Python 3.12 or later. `run` will need Docker once it exists; nothing today does.
+
+## Install
+
+```bash
+git clone https://github.com/Quantile-Labs/touchstone
+cd touchstone
+python3 -m venv .venv
+.venv/bin/pip install -e .
+```
+
+## Check a plan
+
+A plan names the packs to run and the systems to run them against. `validate` reads it
+against each pack's `manifest.yaml` and reports what does not line up:
+
+```console
+$ touchstone validate plan.yaml
+plan.yaml: ok, 1 pack(s)
+
+$ touchstone validate broken.yaml
+broken.yaml: 1 problem(s)
+  example_pack: pack does not accept parameter 'max_itemz'
+```
+
+It resolves packs from `./packs`. Pass `--manifests` to point somewhere else. No
+container runs, and it exits 1 if anything is wrong.
+
+## Seal and verify a bundle
+
+`bundle` hashes every file under a directory and writes `MANIFEST.json`:
+
+```console
+$ touchstone bundle run-004
+run-004: sealed 3 file(s)
+sha256 f57c02f1af4a277d404c29af41cb8953a513a1b0ca38884fcacaf0cbf3359d19
+```
+
+The printed hash covers the file list, so identical content seals to the same value on
+any machine. Sealing a directory that already has a manifest is an error.
+
+`verify` re-checks the bundle and names what moved:
+
+```console
+$ touchstone verify run-004
+run-004: verified
+
+$ touchstone verify tampered
+tampered: 1 failure(s)
+  hash mismatch: items.jsonl
+```
+
+Exit 0 means every file matches its recorded hash and no unrecorded file is present.
+Exit 1 means it does not. There is no network call, no database and no config file.
+
+## Verify without Touchstone
+
+The bundle outlives the tool, so nothing in it needs the tool to read. Check any single
+file against its recorded hash:
+
+```console
+$ shasum -a 256 run-004/items.jsonl
+63a6e0f7ba35198b686265cab8c8b389599c8bb6c3fcff06234d5b68cda0d82f  run-004/items.jsonl
+```
+
+Recompute the bundle hash from the manifest alone:
+
+```console
+$ jq -cS '.files' run-004/MANIFEST.json | tr -d '\n' | shasum -a 256
+f57c02f1af4a277d404c29af41cb8953a513a1b0ca38884fcacaf0cbf3359d19  -
+```
+
+That hash detects a file edited after sealing. It does not detect a forger who reseals
+the whole bundle, which is what external anchoring is for. Anchoring is not built yet.
+
+## The pipeline
+
+```
+validate -> freeze -> run -> estimate -> grade -> bundle -> verify
+```
+
+| Command | Does | State |
+|---|---|---|
+| `validate` | check a plan against the manifests of the packs it names | works |
+| `freeze` | pin image digests, hash the plan, fix seeds and thresholds | exits 2 |
+| `run` | execute packs, write per-item observations | exits 2 |
+| `estimate` | compute rates and intervals, by stratum | exits 2 |
+| `grade` | apply a score card, produce DQI indicators | exits 2 |
+| `bundle` | hash every file in a directory, write `MANIFEST.json` | works |
+| `verify` | re-check a bundle against its manifest, offline | works |
+
+Only `run` needs a container. Everything after it is a function over files.
+
+## Design
+
+Three choices shape the rest:
+
+- Packs emit one observation per item, not a summary. Touchstone computes the rates and
+  intervals, so a reader can recompute them from the sample in the bundle.
+- Runs are pinned before they start. `freeze` resolves image tags to digests and hashes
+  the plan; `run` refuses a plan that has changed since.
+- A missing file, an unresolvable digest or a stale hash is an error with a non-zero
+  exit, never a warning.
+
+To write a pack, see [docs/packs.md](docs/packs.md).
+
+## Licence
+
+Apache 2.0. See [LICENSE](LICENSE).
+
+## Contributing
+
+Read [CONTRIBUTING.md](CONTRIBUTING.md) first. The commit message and code style rules
+are enforced by CI.
+
+Maintained by [Quantile Labs](https://quantilelabs.com).