whetstone-envs 0.1.1__tar.gz

whetstone_envs-0.1.1/.gitignore

@@ -0,0 +1,25 @@
+.worktrees
+.DS_Store
+*.swp
+.env
+.env.*
+!.env.example
+.cache/
+.ruff_cache/
+.mypy_cache/
+.pytest_cache/
+.pyre/
+.pyright/
+.venv/
+build/
+dist/
+*.egg-info/
+htmlcov/
+.coverage
+.coverage.*
+.hypothesis/
+.nox/
+.tox/
+.docs/
+**/__pycache__/
+*.py[cod]

whetstone_envs-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 danielle rothermel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

whetstone_envs-0.1.1/PKG-INFO

@@ -0,0 +1,252 @@
+Metadata-Version: 2.4
+Name: whetstone-envs
+Version: 0.1.1
+Summary: Task-family-agnostic harness for reproducible quick-test environments.
+Project-URL: Repository, https://github.com/danielle-rothermel/whetstone-envs
+Project-URL: Issues, https://github.com/danielle-rothermel/whetstone-envs/issues
+Project-URL: Documentation, https://danielle-rothermel.github.io/whetstone-envs/
+Project-URL: Changelog, https://github.com/danielle-rothermel/whetstone-envs/blob/main/CHANGELOG.md
+Author-email: Danielle Rothermel <danielle.rothermel@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: dr-graph==0.1.2
+Requires-Dist: dr-serialize==0.1.2
+Requires-Dist: dr-store==0.1.4
+Requires-Dist: pydantic==2.13.4
+Description-Content-Type: text/markdown
+
+# whetstone-envs
+
+[![CI](https://github.com/danielle-rothermel/whetstone-envs/actions/workflows/ci.yml/badge.svg)](https://github.com/danielle-rothermel/whetstone-envs/actions/workflows/ci.yml)
+
+Task-family-agnostic contracts for reproducible quick-test environments.
+
+## Scope
+
+This repo owns the environment data and evaluation rules shared by Whetstone's
+quick-test task families, with no dependency on optimizer or execution-contract
+code:
+
+- [**Instances**][instances-source] define immutable task inputs,
+  private gold data, generation seeds, task strata, and public prompt identity.
+- [**Pools and splits**][pools-source] validate ordered instance
+  collections and allocate deterministic internal, official, and held-out
+  cohorts.
+- [**Probes**][probes-source] pair naive and ceiling templates,
+  render public prompt inputs, and normalize predictions for evaluation.
+- [**Scoring**][scoring-source] represents scored, failed, and
+  missing observations and aggregates complete repeat matrices through task,
+  stratum, and overall levels.
+- [**Manifests**][manifests-source] pin generated pools with
+  versioned identities and bounded canonical persistence.
+
+The task-family implementations and the adapter to Whetstone's optimizer live
+above this shared harness rather than inside its contracts.
+
+## Installation
+
+```bash
+uv add whetstone-envs
+```
+
+## Instances
+
+[`whetstone_envs.instances`][instances-source] owns the immutable
+unit passed through generation, prompting, scoring, splitting, and persistence.
+Prompt inputs are public; `gold` remains private evaluation data.
+
+```python
+@dataclass(frozen=True, slots=True)
+class Instance:
+    id: str
+    seed: int
+    strata: tuple[str, ...]
+    prompt_inputs: Mapping[str, str] = field(default_factory=lambda: ...)
+    gold: str = ""
+```
+
+```python
+def make_instance(
+    *,
+    id: str,
+    seed: int,
+    strata: tuple[str, ...] | str,
+    prompt_inputs: Mapping[str, str] | None = None,
+    gold: str = "",
+) -> Instance: ...
+
+def public_prompt_identity(
+    instance: Instance,
+) -> tuple[tuple[str, str], ...]: ...
+```
+
+## Pools and splits
+
+[`whetstone_envs.pools`][pools-source] owns validated ordered pools
+and the deterministic policy for selecting three disjoint evaluation cohorts.
+Split optimization is delegated to `dr-graph`; returned instances preserve pool
+order.
+
+```python
+@dataclass(frozen=True, slots=True)
+class PoolSplit:
+    internal_eval: tuple[Instance, ...]
+    official: tuple[Instance, ...]
+    held_out: tuple[Instance, ...]
+```
+
+```python
+@dataclass(frozen=True, slots=True)
+class TaskPool:
+    instances: tuple[Instance, ...]
+
+    @property
+    def strata(self) -> tuple[str, ...]: ...
+
+    def stratum_counts(self) -> dict[str, int]: ...
+    def in_stratum(self, label: str) -> tuple[Instance, ...]: ...
+    def split(
+        self,
+        internal_eval_n: int,
+        official_n: int,
+        held_out_n: int,
+    ) -> PoolSplit: ...
+```
+
+## Probes
+
+[`whetstone_envs.probes`][probes-source] owns the floor/ceiling
+prompt pair and the default renderer that can see only public prompt inputs.
+Normalization strips whitespace and complete outer triple-backtick fences.
+
+```python
+def render_with_prompt_inputs(template: str, instance: Instance) -> str: ...
+def normalize(prediction: str) -> str: ...
+```
+
+```python
+@dataclass(frozen=True, slots=True)
+class ProbePair:
+    naive_template: str
+    ceiling_template: str
+    render: Callable[[str, Instance], str] = render_with_prompt_inputs
+
+    def render_naive(self, instance: Instance) -> str: ...
+    def render_ceiling(self, instance: Instance) -> str: ...
+```
+
+## Scoring
+
+[`whetstone_envs.scoring`][scoring-source] keeps failures and absent
+results distinct from binary scores. Aggregation exposes a mean only when the
+complete planned task/repeat matrix is present and scored.
+
+```python
+@verify(UNIQUE)
+class Outcome(StrEnum):
+    SCORED = "scored"
+    FAILED = "failed"
+    MISSING = "missing"
+
+@dataclass(frozen=True, slots=True)
+class Observation:
+    task_id: str
+    repeat_id: int
+    outcome: Outcome = Outcome.SCORED
+    score: int | None = None
+```
+
+```python
+@dataclass(frozen=True, slots=True)
+class Aggregate:
+    mean: float | None
+    usable: int
+    failed_count: int
+    missing_count: int
+    label: str | None = None
+    children: tuple["Aggregate", ...] = field(default_factory=tuple)
+
+def aggregate(
+    observations: Iterable[Observation],
+    task_strata: Mapping[str, tuple[str, ...]],
+    *,
+    expected_repeat_ids: Iterable[int],
+) -> Aggregate: ...
+```
+
+`exact_match`, `scored`, `failed`, and `missing` provide the primary leaf-level
+constructors. `aggregate_task`, `aggregate_stratum`, and `aggregate_overall`
+expose the individual aggregation steps when callers already own the hierarchy.
+
+## Manifests
+
+[`whetstone_envs.manifests`][manifests-source] owns the serialized
+boundary for regenerated pool identity. Manifests use a closed Pydantic schema,
+`dr-serialize` identities, and `dr-store` canonical files.
+
+```python
+class Manifest(BaseModel):
+    generator_version: str
+    seed_range: tuple[int, int]
+    stratum_counts: Mapping[str, int]
+    content_hash: Sha256Digest
+    schema_version: int = MANIFEST_SCHEMA_VERSION
+
+    @classmethod
+    def from_pool(
+        cls,
+        pool: TaskPool,
+        *,
+        generator_version: str,
+        seed_range: tuple[int, int],
+    ) -> "Manifest": ...
+
+    def write(self, path: Path) -> None: ...
+    @classmethod
+    def read(cls, path: Path) -> "Manifest": ...
+    def matches_pool(self, pool: TaskPool) -> bool: ...
+
+def content_hash(pool: TaskPool) -> Sha256Digest: ...
+```
+
+## Terms and contracts
+
+The [published terms and contracts](https://danielle-rothermel.github.io/whetstone-envs/)
+render the authoritative
+[vocabulary](https://github.com/danielle-rothermel/whetstone-envs/blob/main/.defs/terms.toml)
+and
+[binding contracts](https://github.com/danielle-rothermel/whetstone-envs/blob/main/.defs/contracts.toml)
+directly from their TOML sources. The
+[changelog](https://github.com/danielle-rothermel/whetstone-envs/blob/main/CHANGELOG.md)
+records notable changes.
+
+## Development
+
+Install the locked development environment and commit hook once per clone:
+
+```bash
+uv sync --locked
+uv run pre-commit install
+```
+
+The hook runs the same formatting, lint, type, definitions, test, and package
+build gate used by CI. Run it directly at any time:
+
+```bash
+scripts/pre-check.sh
+```
+
+[instances-source]: https://github.com/danielle-rothermel/whetstone-envs/tree/main/src/whetstone_envs/instances
+[manifests-source]: https://github.com/danielle-rothermel/whetstone-envs/tree/main/src/whetstone_envs/manifests
+[pools-source]: https://github.com/danielle-rothermel/whetstone-envs/tree/main/src/whetstone_envs/pools
+[probes-source]: https://github.com/danielle-rothermel/whetstone-envs/tree/main/src/whetstone_envs/probes
+[scoring-source]: https://github.com/danielle-rothermel/whetstone-envs/tree/main/src/whetstone_envs/scoring

whetstone_envs-0.1.1/README.md

@@ -0,0 +1,226 @@
+# whetstone-envs
+
+[![CI](https://github.com/danielle-rothermel/whetstone-envs/actions/workflows/ci.yml/badge.svg)](https://github.com/danielle-rothermel/whetstone-envs/actions/workflows/ci.yml)
+
+Task-family-agnostic contracts for reproducible quick-test environments.
+
+## Scope
+
+This repo owns the environment data and evaluation rules shared by Whetstone's
+quick-test task families, with no dependency on optimizer or execution-contract
+code:
+
+- [**Instances**][instances-source] define immutable task inputs,
+  private gold data, generation seeds, task strata, and public prompt identity.
+- [**Pools and splits**][pools-source] validate ordered instance
+  collections and allocate deterministic internal, official, and held-out
+  cohorts.
+- [**Probes**][probes-source] pair naive and ceiling templates,
+  render public prompt inputs, and normalize predictions for evaluation.
+- [**Scoring**][scoring-source] represents scored, failed, and
+  missing observations and aggregates complete repeat matrices through task,
+  stratum, and overall levels.
+- [**Manifests**][manifests-source] pin generated pools with
+  versioned identities and bounded canonical persistence.
+
+The task-family implementations and the adapter to Whetstone's optimizer live
+above this shared harness rather than inside its contracts.
+
+## Installation
+
+```bash
+uv add whetstone-envs
+```
+
+## Instances
+
+[`whetstone_envs.instances`][instances-source] owns the immutable
+unit passed through generation, prompting, scoring, splitting, and persistence.
+Prompt inputs are public; `gold` remains private evaluation data.
+
+```python
+@dataclass(frozen=True, slots=True)
+class Instance:
+    id: str
+    seed: int
+    strata: tuple[str, ...]
+    prompt_inputs: Mapping[str, str] = field(default_factory=lambda: ...)
+    gold: str = ""
+```
+
+```python
+def make_instance(
+    *,
+    id: str,
+    seed: int,
+    strata: tuple[str, ...] | str,
+    prompt_inputs: Mapping[str, str] | None = None,
+    gold: str = "",
+) -> Instance: ...
+
+def public_prompt_identity(
+    instance: Instance,
+) -> tuple[tuple[str, str], ...]: ...
+```
+
+## Pools and splits
+
+[`whetstone_envs.pools`][pools-source] owns validated ordered pools
+and the deterministic policy for selecting three disjoint evaluation cohorts.
+Split optimization is delegated to `dr-graph`; returned instances preserve pool
+order.
+
+```python
+@dataclass(frozen=True, slots=True)
+class PoolSplit:
+    internal_eval: tuple[Instance, ...]
+    official: tuple[Instance, ...]
+    held_out: tuple[Instance, ...]
+```
+
+```python
+@dataclass(frozen=True, slots=True)
+class TaskPool:
+    instances: tuple[Instance, ...]
+
+    @property
+    def strata(self) -> tuple[str, ...]: ...
+
+    def stratum_counts(self) -> dict[str, int]: ...
+    def in_stratum(self, label: str) -> tuple[Instance, ...]: ...
+    def split(
+        self,
+        internal_eval_n: int,
+        official_n: int,
+        held_out_n: int,
+    ) -> PoolSplit: ...
+```
+
+## Probes
+
+[`whetstone_envs.probes`][probes-source] owns the floor/ceiling
+prompt pair and the default renderer that can see only public prompt inputs.
+Normalization strips whitespace and complete outer triple-backtick fences.
+
+```python
+def render_with_prompt_inputs(template: str, instance: Instance) -> str: ...
+def normalize(prediction: str) -> str: ...
+```
+
+```python
+@dataclass(frozen=True, slots=True)
+class ProbePair:
+    naive_template: str
+    ceiling_template: str
+    render: Callable[[str, Instance], str] = render_with_prompt_inputs
+
+    def render_naive(self, instance: Instance) -> str: ...
+    def render_ceiling(self, instance: Instance) -> str: ...
+```
+
+## Scoring
+
+[`whetstone_envs.scoring`][scoring-source] keeps failures and absent
+results distinct from binary scores. Aggregation exposes a mean only when the
+complete planned task/repeat matrix is present and scored.
+
+```python
+@verify(UNIQUE)
+class Outcome(StrEnum):
+    SCORED = "scored"
+    FAILED = "failed"
+    MISSING = "missing"
+
+@dataclass(frozen=True, slots=True)
+class Observation:
+    task_id: str
+    repeat_id: int
+    outcome: Outcome = Outcome.SCORED
+    score: int | None = None
+```
+
+```python
+@dataclass(frozen=True, slots=True)
+class Aggregate:
+    mean: float | None
+    usable: int
+    failed_count: int
+    missing_count: int
+    label: str | None = None
+    children: tuple["Aggregate", ...] = field(default_factory=tuple)
+
+def aggregate(
+    observations: Iterable[Observation],
+    task_strata: Mapping[str, tuple[str, ...]],
+    *,
+    expected_repeat_ids: Iterable[int],
+) -> Aggregate: ...
+```
+
+`exact_match`, `scored`, `failed`, and `missing` provide the primary leaf-level
+constructors. `aggregate_task`, `aggregate_stratum`, and `aggregate_overall`
+expose the individual aggregation steps when callers already own the hierarchy.
+
+## Manifests
+
+[`whetstone_envs.manifests`][manifests-source] owns the serialized
+boundary for regenerated pool identity. Manifests use a closed Pydantic schema,
+`dr-serialize` identities, and `dr-store` canonical files.
+
+```python
+class Manifest(BaseModel):
+    generator_version: str
+    seed_range: tuple[int, int]
+    stratum_counts: Mapping[str, int]
+    content_hash: Sha256Digest
+    schema_version: int = MANIFEST_SCHEMA_VERSION
+
+    @classmethod
+    def from_pool(
+        cls,
+        pool: TaskPool,
+        *,
+        generator_version: str,
+        seed_range: tuple[int, int],
+    ) -> "Manifest": ...
+
+    def write(self, path: Path) -> None: ...
+    @classmethod
+    def read(cls, path: Path) -> "Manifest": ...
+    def matches_pool(self, pool: TaskPool) -> bool: ...
+
+def content_hash(pool: TaskPool) -> Sha256Digest: ...
+```
+
+## Terms and contracts
+
+The [published terms and contracts](https://danielle-rothermel.github.io/whetstone-envs/)
+render the authoritative
+[vocabulary](https://github.com/danielle-rothermel/whetstone-envs/blob/main/.defs/terms.toml)
+and
+[binding contracts](https://github.com/danielle-rothermel/whetstone-envs/blob/main/.defs/contracts.toml)
+directly from their TOML sources. The
+[changelog](https://github.com/danielle-rothermel/whetstone-envs/blob/main/CHANGELOG.md)
+records notable changes.
+
+## Development
+
+Install the locked development environment and commit hook once per clone:
+
+```bash
+uv sync --locked
+uv run pre-commit install
+```
+
+The hook runs the same formatting, lint, type, definitions, test, and package
+build gate used by CI. Run it directly at any time:
+
+```bash
+scripts/pre-check.sh
+```
+
+[instances-source]: https://github.com/danielle-rothermel/whetstone-envs/tree/main/src/whetstone_envs/instances
+[manifests-source]: https://github.com/danielle-rothermel/whetstone-envs/tree/main/src/whetstone_envs/manifests
+[pools-source]: https://github.com/danielle-rothermel/whetstone-envs/tree/main/src/whetstone_envs/pools
+[probes-source]: https://github.com/danielle-rothermel/whetstone-envs/tree/main/src/whetstone_envs/probes
+[scoring-source]: https://github.com/danielle-rothermel/whetstone-envs/tree/main/src/whetstone_envs/scoring

whetstone_envs-0.1.1/pyproject.toml

@@ -0,0 +1,115 @@
+[project]
+name = "whetstone-envs"
+version = "0.1.1"
+description = "Task-family-agnostic harness for reproducible quick-test environments."
+readme = "README.md"
+license = "MIT"
+authors = [
+    { name = "Danielle Rothermel", email = "danielle.rothermel@gmail.com" }
+]
+requires-python = ">=3.12"
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Typing :: Typed",
+]
+dependencies = [
+    "dr-graph==0.1.2",
+    "dr-serialize==0.1.2",
+    "dr-store==0.1.4",
+    "pydantic==2.13.4",
+]
+
+[project.urls]
+Repository = "https://github.com/danielle-rothermel/whetstone-envs"
+Issues = "https://github.com/danielle-rothermel/whetstone-envs/issues"
+Documentation = "https://danielle-rothermel.github.io/whetstone-envs/"
+Changelog = "https://github.com/danielle-rothermel/whetstone-envs/blob/main/CHANGELOG.md"
+
+[build-system]
+requires = ["hatchling==1.31.0"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "actionlint-py==1.7.12.24",
+    "pre-commit>=4.6.0",
+    "pytest>=9.1.1",
+    "ruff>=0.15.18",
+    "ty>=0.0.51",
+]
+
+[tool.uv]
+required-version = "==0.11.25"
+
+[tool.hatch.build.targets.sdist]
+only-include = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.pyright]
+venvPath = "."
+venv = ".venv"
+
+[tool.ruff]
+include = ["src/**/*.py", "tests/**/*.py"]
+line-length = 79
+
+[tool.ruff.lint]
+select = [
+    "A",
+    "ARG",
+    "ASYNC",
+    "B",
+    "BLE",
+    "C4",
+    "DTZ",
+    "E",
+    "F",
+    "FA",
+    "FBT",
+    "FLY",
+    "FURB",
+    "G",
+    "I",
+    "ICN",
+    "ISC",
+    "LOG",
+    "N",
+    "NPY",
+    "PD",
+    "PERF",
+    "PIE",
+    "PL",
+    "PTH",
+    "PT",
+    "RET",
+    "RSE",
+    "RUF",
+    "S",
+    "SIM",
+    "SLOT",
+    "T10",
+    "TC",
+    "TID",
+    "TRY",
+    "UP",
+    "W",
+    "YTT",
+]
+ignore = ["PLR1711", "S101", "TRY003"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = ["PLC0415", "PLR2004", "PT011"]
+
+[tool.ruff.lint.flake8-type-checking]
+runtime-evaluated-decorators = ["dataclasses.dataclass"]
+
+[tool.ty.src]
+include = ["src", "tests"]

whetstone_envs-0.1.1/src/whetstone_envs/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

whetstone_envs-0.1.1/src/whetstone_envs/instances/__init__.py

@@ -0,0 +1,4 @@
+from whetstone_envs.instances.identity import public_prompt_identity
+from whetstone_envs.instances.instance import Instance, make_instance
+
+__all__ = ["Instance", "make_instance", "public_prompt_identity"]

whetstone_envs-0.1.1/src/whetstone_envs/instances/identity.py

@@ -0,0 +1,9 @@
+from whetstone_envs.instances.instance import Instance
+
+
+def public_prompt_identity(instance: Instance) -> tuple[tuple[str, str], ...]:
+    """Return the canonical identity of sorted ``prompt_inputs``.
+
+    This identity does not necessarily identify rendered text.
+    """
+    return tuple(sorted(instance.prompt_inputs.items()))