touchstone-us 0.1.0__py3-none-any.whl

touchstone/__init__.py

@@ -0,0 +1 @@
+"""Local, evidence-first release qualification."""

touchstone/_assets/PROTOCOL.md

@@ -0,0 +1,92 @@
+# Workload contract
+
+The runner executes trusted OCI images with a manifest command (an argv array;
+there is no implicit host shell). The frozen workload file and fixture are
+mounted read-only at `/workload`. Images are resolved to local immutable IDs
+before the first measurement. Registry digests are recorded when available.
+
+## Start from the included example
+
+1. Keep `manifest.json`, `workload.py` and `fixtures.json` together.
+2. Replace the baseline and candidate images/commands with your own trusted
+   test implementations. The command must emit the JSON contract below.
+3. Supply case IDs and expected numeric vectors in the fixture. The `inputs`
+   object carries your workload's data; the controller does not interpret it.
+4. Import the three files in the local workspace, review the policy and run.
+
+File basenames in the manifest must match the workload/fixture filenames.
+Input files are limited to 1 MB each. One comparison executes three to ten
+paired trials, alternating baseline-first and candidate-first order. Workloads
+need no network and should not download models during measurement. Put their
+dependencies and model data in your approved image or the bounded fixture.
+
+## Manifest
+
+Use the starter's manifest. `baseline` and `candidate` each contain `image`
+and `command`. `device` is `cpu` or `cuda`. `repeats` is 3-10,
+`timeout_seconds` is 5-180, `memory_mb` is 256-8192, and `cpus` is 1-4.
+The policy covers throughput retention, amortized time ratio, absolute output
+error, reported memory, failed samples, minimum samples and timing CV.
+The manifest schema rejects unknown fields and invalid/non-finite values.
+
+## Fixture
+
+```json
+{
+  "schema_version": 1,
+  "cases": [{"id": "case-01", "values": [0.25, 0.75]}],
+  "inputs": {"your_workload_data": "goes here"}
+}
+```
+
+Case IDs must be unique. There are at most 256 cases and 256 numeric values per
+case, with at most 4096 values in total. Values are finite. The controller requires exact ID and vector-length
+coverage from each execution and calculates absolute errors against this file.
+
+## Standard output
+
+Emit exactly one JSON object; send diagnostic text to stderr. The example
+below is a schema illustration, not a measurement to reuse.
+
+```json
+{
+  "schema_version": 1,
+  "device": "cpu",
+  "samples": 1000,
+  "duration_seconds": 1.5,
+  "peak_memory_mb": 28.0,
+  "memory_kind": "process_rss",
+  "failures": 0,
+  "outputs": [{"id": "case-01", "values": [0.25, 0.75]}],
+  "runtime": {"python": "3.12", "workload": "your-suite-v1"}
+}
+```
+
+Use `cuda_allocated` for CUDA memory and `process_rss` for CPU memory. CUDA
+workloads must fail if CUDA is unavailable; silent CPU fallback cannot qualify
+the requested contract. Synchronize CUDA before and after the measured region.
+Keep initialization/warmup outside that region. `samples / duration_seconds`
+gives throughput; its reciprocal gives amortized processing time per sample.
+For a batched workload this is not request latency, nor p95 or p99. The v1
+JSON fields `latency_ms` and `max_latency_ratio` retain their original names;
+they represent this derived amortized time and its candidate/baseline ratio.
+These are trusted test-instrumentation measurements, not tamper-proof telemetry.
+
+## Results and boundaries
+
+- `eligible`: declared checks pass; a human release owner must still decide.
+- `hold`: at least one policy check fails; inspect the evidence and fix the candidate.
+- `inconclusive`: execution, reference coverage, comparability or repeatability
+  is incomplete. A crash, missing output or timeout never becomes a pass.
+- Cancelled/interrupted/error executions retain diagnostic evidence separately.
+
+The included examples are small real classifier inference, with deliberately
+injected faults in named variants. They are not a discovered upstream bug,
+representative LLM benchmark, convergence test or customer validation.
+
+This MVP uses rootless Podman, a non-root workload user, no container network,
+no capabilities, resource/process limits and bounded logs. CPU root filesystems
+are read-only. NVIDIA CDI needs a temporary writable image layer to initialize
+its loader on this WSL host; GPU workloads still run non-root and inputs remain
+read-only. CUDA GPU memory is not limited by the CPU cgroup memory setting.
+Only run trusted operator code. Containers are not a hostile-code security boundary.

touchstone/_assets/app.js

@@ -0,0 +1,151 @@
+import { checks, scenarios, evaluate, disposition } from './fixtures.js';
+const get = id => document.getElementById(id);
+const requestedScenario = new URL(location.href).searchParams.get('scenario');
+let selected = Object.hasOwn(scenarios, requestedScenario) ? requestedScenario : 'throughput';
+const checkList = get('check-list');
+const decision = get('decision');
+const status = get('review-status');
+const format = new Intl.NumberFormat('en-US', { maximumFractionDigits: 1 });
+function checkDetail(key, scenario) {
+  const { baseline, candidate } = scenario.fixture;
+  if (key === 'hardware') return `${baseline.samples} baseline / ${candidate.samples} candidate requests`;
+  if (key === 'throughput') return `${format.format(baseline.throughput)} → ${format.format(candidate.throughput)} tokens/s`;
+  if (key === 'fidelity') return `Observed max error: ${candidate.error}`;
+  return `${format.format(candidate.memoryGb)} GB peak / ${candidate.failures} failed requests`;
+}
+function renderChecks(results) {
+  checkList.replaceChildren(...checks.map(check => {
+    const row = document.createElement('div');
+    const outcome = !results ? 'pending' : results[check.key] === null ? 'inconclusive' : results[check.key] ? 'pass' : 'fail';
+    row.className = `check-row ${outcome}`;
+    const symbol = document.createElement('span');
+    symbol.className = 'check-symbol';
+    symbol.setAttribute('aria-hidden', 'true');
+    symbol.textContent = { pass: '✓', fail: '!', pending: '○', inconclusive: '—' }[outcome];
+    const label = document.createElement('span');
+    label.className = 'check-label';
+    const name = document.createElement('strong');
+    name.textContent = check.label;
+    label.append(name);
+    if (results && outcome !== 'inconclusive') {
+      const detail = document.createElement('small');
+      detail.textContent = checkDetail(check.key, scenarios[selected]);
+      label.append(detail);
+    }
+    const value = document.createElement('span');
+    value.textContent = { pass: 'Pass', fail: 'Fail', pending: 'Not run', inconclusive: 'Not qualified' }[outcome];
+    row.append(symbol, label, value);
+    return row;
+  }));
+}
+function selectScenario(key) {
+  selected = key;
+  const url = new URL(location.href);
+  url.searchParams.set('scenario', key);
+  history.replaceState(null, '', url);
+  const scenario = scenarios[key];
+  for (const button of document.querySelectorAll('[data-scenario]')) {
+    const active = button.dataset.scenario === key;
+    button.classList.toggle('active', active);
+    button.setAttribute('aria-pressed', String(active));
+  }
+  get('release-id').textContent = `Candidate stack ${scenario.release}`;
+  get('scenario-title').textContent = scenario.title;
+  get('baseline-name').textContent = scenario.baselineName;
+  get('candidate-name').textContent = scenario.candidateName;
+  get('sample-context').textContent = selected === 'hardware' ? 'H100 baseline / A100 candidate • 100 sampled requests per stack' : 'Matched H100 80 GB host profile • 100 sampled requests per stack';
+  status.className = 'status idle';
+  status.textContent = 'Not checked';
+  decision.className = 'decision';
+  decision.querySelector('.decision-symbol').textContent = '○';
+  decision.querySelector('strong').textContent = 'Choose a candidate. Run its checks.';
+  decision.querySelector('p').textContent = 'The decision and supporting evidence will appear here.';
+  get('evidence-details').hidden = true;
+  get('evidence-details').open = false;
+  get('compare-fixed').hidden = true;
+  get('run-checks').textContent = 'Run sample checks ↗';
+  renderChecks();
+}
+get('run-checks').addEventListener('click', () => {
+  const scenario = scenarios[selected];
+  const results = evaluate(scenario.fixture);
+  const result = disposition(results);
+  renderChecks(results);
+  status.className = `status ${result}`;
+  status.textContent = { ready: 'Eligible for review', blocked: 'Hold', inconclusive: 'Rerun required' }[result];
+  decision.className = `decision ${result}`;
+  decision.querySelector('.decision-symbol').textContent = { ready: '✓', blocked: '!', inconclusive: '↺' }[result];
+  decision.querySelector('strong').textContent = {
+    ready: 'Policy checks pass. Owner approval is next.',
+    blocked: selected === 'fidelity' ? 'Hold. The faster candidate fails output fidelity.' : 'Hold. Throughput falls below the release threshold.',
+    inconclusive: 'Rerun required. The hardware profiles do not match.',
+  }[result];
+  decision.querySelector('p').textContent = scenario.explanation;
+  get('evidence-details').hidden = false;
+  get('evidence-output').textContent = JSON.stringify(scenario.evidence, null, 2);
+  get('evidence-explanation').textContent = 'Illustrative evidence only. These image identities, bundles and results are synthetic; no live infrastructure is connected.';
+  get('compare-fixed').hidden = selected !== 'throughput' && selected !== 'fidelity';
+  get('run-checks').textContent = 'Run sample checks again ↗';
+});
+for (const button of document.querySelectorAll('[data-scenario]')) button.addEventListener('click', () => selectScenario(button.dataset.scenario));
+get('compare-fixed').addEventListener('click', () => {
+  selectScenario('ready');
+  get('run-checks').click();
+  get('run-checks').focus({ preventScroll: true });
+});
+const dialog = get('pilot-dialog');
+get('pilot-brief').addEventListener('click', () => dialog.showModal());
+get('close-pilot').addEventListener('click', () => dialog.close());
+get('pilot-done').addEventListener('click', () => {
+  dialog.close();
+  get('sample').scrollIntoView({ behavior: scrollBehavior() });
+  get('run-checks').focus({ preventScroll: true });
+});
+dialog.addEventListener('click', event => {
+  if (event.target === dialog) {
+    const bounds = dialog.getBoundingClientRect();
+    if (event.clientX < bounds.left || event.clientX > bounds.right || event.clientY < bounds.top || event.clientY > bounds.bottom) dialog.close();
+  }
+});
+const scrollBehavior = () => matchMedia('(prefers-reduced-motion: reduce)').matches ? 'instant' : 'smooth';
+for (const link of document.querySelectorAll('[data-jump]')) link.addEventListener('click', event => {
+  event.preventDefault();
+  selectScenario(link.dataset.jump);
+  get('sample').scrollIntoView({ behavior: scrollBehavior() });
+  get('run-checks').focus({ preventScroll: true });
+});
+const tabs = [...document.querySelectorAll('[role=tab]')];
+function selectTab(tab) {
+  for (const other of tabs) {
+    const active = other === tab;
+    other.setAttribute('aria-selected', String(active));
+    other.tabIndex = active ? 0 : -1;
+    get(other.getAttribute('aria-controls')).hidden = !active;
+  }
+}
+for (const tab of tabs) {
+  tab.addEventListener('click', () => selectTab(tab));
+  tab.addEventListener('keydown', event => {
+    const step = { ArrowRight: 1, ArrowLeft: -1 }[event.key];
+    if (!step) return;
+    const next = tabs[(tabs.indexOf(tab) + step + tabs.length) % tabs.length];
+    selectTab(next);
+    next.focus();
+  });
+}
+get('copy-code').addEventListener('click', async () => {
+  const panel = document.querySelector('[role=tabpanel]:not([hidden])');
+  const note = get('copy-status');
+  try {
+    await navigator.clipboard.writeText(panel.textContent);
+    note.textContent = 'Copied to the clipboard.';
+  } catch {
+    getSelection().selectAllChildren(panel);
+    note.textContent = 'Copy is unavailable here; the command is selected.';
+  }
+});
+const header = document.querySelector('.site-header');
+const markScrolled = () => header.classList.toggle('scrolled', scrollY > 8);
+addEventListener('scroll', markScrolled, { passive: true });
+markScrolled();
+selectScenario(selected);

touchstone/_assets/base.css

@@ -0,0 +1,128 @@
+@font-face {
+  font-family:TouchStone Sans;
+  src:local("Bahnschrift"),local("Arial");
+  font-display:swap;
+}
+:root {
+  --ink:#142338;
+  --muted:#58687c;
+  --cobalt:#254f9b;
+  --cobalt-deep:#213caf;
+  --ice:#f1f5fa;
+  --line:#dce3ec;
+  --pale:#e9edff;
+  --green:#227b59;
+  --amber:#9c6411;
+  --white:#fff;
+  --radius:12px;
+}
+* {
+  box-sizing:border-box;
+}
+html {
+  scroll-behavior:smooth;
+  scroll-padding-top:96px;
+}
+body {
+  margin:0;
+  background:var(--white);
+  color:var(--ink);
+  font-family:"TouchStone Sans","Segoe UI",Arial,sans-serif;
+  font-size:16px;
+  line-height:1.65;
+  -webkit-font-smoothing:antialiased;
+}
+button,a {
+  -webkit-tap-highlight-color:transparent;
+}
+button {
+  font:inherit;
+  cursor:pointer;
+}
+a {
+  color:inherit;
+  text-decoration:none;
+}
+a,button,summary {
+  touch-action:manipulation;
+}
+a:focus-visible,button:focus-visible,summary:focus-visible {
+  outline:3px solid #b25816;
+  outline-offset:5px;
+}
+h1,h2,h3,p {
+  margin:0;
+}
+h1,h2,h3 {
+  text-wrap:balance;
+}
+p {
+  max-width:68ch;
+}
+h1 {
+  font-size:clamp(48px,5.2vw,78px);
+  line-height:1.06;
+  letter-spacing:-.055em;
+  font-weight:650;
+}
+h2 {
+  font-size:clamp(34px,3.1vw,48px);
+  line-height:1.15;
+  letter-spacing:-.04em;
+  font-weight:650;
+}
+h3 {
+  font-size:19px;
+  line-height:1.35;
+  font-weight:650;
+  letter-spacing:-.015em;
+}
+.wrap {
+  width:min(1200px,calc(100% - 96px));
+  margin-inline:auto;
+}
+.site-header {
+  display:flex;
+  align-items:center;
+  justify-content:space-between;
+  min-height:104px;
+}
+.brand {
+  display:inline-flex;
+  align-items:center;
+  gap:10px;
+  font-size:24px;
+  font-weight:680;
+  letter-spacing:-.055em;
+  line-height:1;
+}
+.brand svg {
+  height:34px;
+  width:34px;
+  stroke:var(--cobalt);
+  fill:none;
+  stroke-width:3.4;
+  stroke-linejoin:round;
+  stroke-linecap:round;
+}
+.site-header nav {
+  display:flex;
+  align-items:center;
+  gap:31px;
+  font-size:14px;
+  font-weight:550;
+}
+.site-header nav a {
+  padding:8px 0;
+}
+.site-header nav a:hover,.text-link:hover,.site-footer>a:last-child:hover {
+  color:var(--cobalt);
+}
+.site-header nav .nav-cta {
+  padding:10px 17px;
+  border:1px solid var(--line);
+  border-radius:6px;
+}
+.nav-cta:hover {
+  background:var(--ice);
+}

touchstone/_assets/examples/cpu-correct.json

@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "name": "CPU / Reference match",
+  "description": "Measured inference with identical correct implementations. Performance can still be noisy.",
+  "demonstration": true,
+  "device": "cpu",
+  "baseline": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "correct"
+    ]
+  },
+  "candidate": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "correct"
+    ]
+  },
+  "workload_file": "workload.py",
+  "fixture_file": "fixtures.json",
+  "repeats": 3,
+  "timeout_seconds": 60,
+  "memory_mb": 512,
+  "cpus": 2,
+  "policy": {
+    "min_throughput_ratio": 0.8,
+    "max_latency_ratio": 1.3,
+    "max_abs_error": 1e-05,
+    "max_peak_memory_mb": 1024.0,
+    "max_failures": 0,
+    "min_samples": 100,
+    "max_timing_cv": 0.3
+  }
+}

touchstone/_assets/examples/cpu-crash.json

@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "name": "CPU / Injected execution failure",
+  "description": "Deliberately exits with status 3. The result must remain inconclusive.",
+  "demonstration": true,
+  "device": "cpu",
+  "baseline": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "correct"
+    ]
+  },
+  "candidate": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "crash"
+    ]
+  },
+  "workload_file": "workload.py",
+  "fixture_file": "fixtures.json",
+  "repeats": 3,
+  "timeout_seconds": 60,
+  "memory_mb": 512,
+  "cpus": 2,
+  "policy": {
+    "min_throughput_ratio": 0.8,
+    "max_latency_ratio": 1.3,
+    "max_abs_error": 1e-05,
+    "max_peak_memory_mb": 1024.0,
+    "max_failures": 0,
+    "min_samples": 100,
+    "max_timing_cv": 0.3
+  }
+}

touchstone/_assets/examples/cpu-drift.json

@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "name": "CPU / Injected output regression",
+  "description": "Deliberately changes one probability by 0.05. A fault demonstration, not a discovered framework defect.",
+  "demonstration": true,
+  "device": "cpu",
+  "baseline": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "correct"
+    ]
+  },
+  "candidate": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "drift"
+    ]
+  },
+  "workload_file": "workload.py",
+  "fixture_file": "fixtures.json",
+  "repeats": 3,
+  "timeout_seconds": 60,
+  "memory_mb": 512,
+  "cpus": 2,
+  "policy": {
+    "min_throughput_ratio": 0.8,
+    "max_latency_ratio": 1.3,
+    "max_abs_error": 1e-05,
+    "max_peak_memory_mb": 1024.0,
+    "max_failures": 0,
+    "min_samples": 100,
+    "max_timing_cv": 0.3
+  }
+}

touchstone/_assets/examples/cpu-missing.json

@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "name": "CPU / Incomplete output coverage",
+  "description": "Deliberately omits a fixture case. Incomplete evidence cannot qualify a release.",
+  "demonstration": true,
+  "device": "cpu",
+  "baseline": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "correct"
+    ]
+  },
+  "candidate": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "missing"
+    ]
+  },
+  "workload_file": "workload.py",
+  "fixture_file": "fixtures.json",
+  "repeats": 3,
+  "timeout_seconds": 60,
+  "memory_mb": 512,
+  "cpus": 2,
+  "policy": {
+    "min_throughput_ratio": 0.8,
+    "max_latency_ratio": 1.3,
+    "max_abs_error": 1e-05,
+    "max_peak_memory_mb": 1024.0,
+    "max_failures": 0,
+    "min_samples": 100,
+    "max_timing_cv": 0.3
+  }
+}

touchstone/_assets/examples/cpu-slow.json

@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "name": "CPU / Injected performance regression",
+  "description": "Deliberately sleeps during inference. Actual timing is measured; the slowdown is intentional.",
+  "demonstration": true,
+  "device": "cpu",
+  "baseline": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "correct"
+    ]
+  },
+  "candidate": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "slow"
+    ]
+  },
+  "workload_file": "workload.py",
+  "fixture_file": "fixtures.json",
+  "repeats": 3,
+  "timeout_seconds": 60,
+  "memory_mb": 512,
+  "cpus": 2,
+  "policy": {
+    "min_throughput_ratio": 0.8,
+    "max_latency_ratio": 1.3,
+    "max_abs_error": 1e-05,
+    "max_peak_memory_mb": 1024.0,
+    "max_failures": 0,
+    "min_samples": 100,
+    "max_timing_cv": 0.3
+  }
+}

touchstone/_assets/examples/cpu-timeout.json

@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "name": "CPU / Injected timeout",
+  "description": "Deliberately hangs. The controller must stop its own container and retain evidence.",
+  "demonstration": true,
+  "device": "cpu",
+  "baseline": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "correct"
+    ]
+  },
+  "candidate": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "timeout"
+    ]
+  },
+  "workload_file": "workload.py",
+  "fixture_file": "fixtures.json",
+  "repeats": 3,
+  "timeout_seconds": 10,
+  "memory_mb": 512,
+  "cpus": 2,
+  "policy": {
+    "min_throughput_ratio": 0.8,
+    "max_latency_ratio": 1.3,
+    "max_abs_error": 1e-05,
+    "max_peak_memory_mb": 1024.0,
+    "max_failures": 0,
+    "min_samples": 100,
+    "max_timing_cv": 0.3
+  }
+}

touchstone/_assets/examples/cpu-upgrade.json

@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "name": "CPU / Real runtime upgrade",
+  "description": "Actual image-version comparison with the same owned classifier fixture. No defect is intentionally injected; this is a small smoke qualification.",
+  "demonstration": true,
+  "device": "cpu",
+  "baseline": {
+    "image": "docker.io/library/python:3.12-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "correct"
+    ]
+  },
+  "candidate": {
+    "image": "docker.io/library/python:3.13-slim",
+    "command": [
+      "python",
+      "/workload/workload.py",
+      "--device",
+      "cpu",
+      "--variant",
+      "correct"
+    ]
+  },
+  "workload_file": "workload.py",
+  "fixture_file": "fixtures.json",
+  "repeats": 3,
+  "timeout_seconds": 60,
+  "memory_mb": 512,
+  "cpus": 2,
+  "policy": {
+    "min_throughput_ratio": 0.8,
+    "max_latency_ratio": 1.3,
+    "max_abs_error": 1e-05,
+    "max_peak_memory_mb": 1024.0,
+    "max_failures": 0,
+    "min_samples": 100,
+    "max_timing_cv": 0.3
+  }
+}