techtree 0.1.0__py3-none-any.whl

techtree/models/data_policy.py

@@ -0,0 +1,130 @@
+"""Rights and permitted future uses of Campaign artifacts. Spec section 11.4.
+
+A ``DataPolicy`` is required before any episode exists, because rights cannot
+be retrofitted honestly. Once a participant has run a comparison, asking them
+afterwards whether the transcripts may be used for training is asking a
+question whose answer was already assumed.
+
+The policy is a plain statement of permissions. It is immutable by digest:
+changing any permission produces a different policy, which produces a
+different ``CampaignSpec``, which is exactly the visibility the rule exists to
+create. Nothing here grants Techtree the ability to do anything: a policy
+that permits something is not a command that does it, and the only thing that
+sends a run anywhere is a person running ``techtree publish``.
+
+Contradiction checks between a public Climb and its policy live in
+:mod:`techtree.models.climb`, where both objects are in scope.
+"""
+
+from __future__ import annotations
+
+from typing import Literal, Self
+
+from pydantic import Field, model_validator
+
+from techtree.models.base import NonEmptyString, ProtocolModel
+
+__all__ = [
+    "CandidateSkillPolicy",
+    "DataOwner",
+    "DataPolicy",
+    "DerivedArtifactPolicy",
+    "RawEpisodePolicy",
+    "RevocationPolicy",
+]
+
+
+type Permission = Literal["allowed", "prohibited", "consent_required"]
+"""Whether a use is permitted outright, forbidden, or gated on fresh consent."""
+
+type Visibility = Literal["public", "private", "prohibited"]
+"""Whether a derived artifact may be published, kept, or not produced at all."""
+
+
+class DataOwner(ProtocolModel):
+    """Who owns the artifacts a Campaign produces."""
+
+    kind: Literal["participant", "account", "shared"]
+    account_ref: NonEmptyString | None = None
+
+    @model_validator(mode="after")
+    def _check_account_reference_matches_ownership(self) -> Self:
+        """Require an account reference exactly where ownership implies one."""
+        if self.kind == "account" and self.account_ref is None:
+            raise ValueError("account-owned data must name the owning account_ref")
+        if self.kind == "participant" and self.account_ref is not None:
+            raise ValueError(
+                "participant-owned data must not name an account_ref; use the "
+                "shared owner kind when an account also holds rights"
+            )
+        return self
+
+
+class RawEpisodePolicy(ProtocolModel):
+    """What may happen to raw episode transcripts."""
+
+    local_retention: Literal["allowed", "prohibited", "required"]
+    server_upload: Permission
+    public_release: Permission
+    reproduction_access: Permission
+    training_use: Permission
+
+
+class DerivedArtifactPolicy(ProtocolModel):
+    """What may happen to everything computed from the episodes."""
+
+    aggregate_scores: Visibility
+    uplift_report: Visibility
+    redacted_trace_projection: Visibility
+    anonymized_product_analytics: Permission
+
+
+class CandidateSkillPolicy(ProtocolModel):
+    """Who owns the submitted skill and whether it can be published."""
+
+    ownership: Literal["participant", "account", "shared"]
+    public_release: Literal[
+        "required_for_climb",
+        "allowed",
+        "prohibited",
+        "consent_required",
+    ]
+    training_use: Permission
+
+
+class RevocationPolicy(ProtocolModel):
+    """What a participant can withdraw later, and what stays published."""
+
+    future_use_revocable: bool
+    immutable_published_proofs_remain: bool
+
+
+class DataPolicy(ProtocolModel):
+    """The complete rights statement a Campaign runs under."""
+
+    schema_version: Literal["techtree.data-policy.v1alpha1"]
+    id: NonEmptyString
+    version: int = Field(ge=1)
+    owner: DataOwner
+    raw_episodes: RawEpisodePolicy
+    derived_artifacts: DerivedArtifactPolicy
+    candidate_skill: CandidateSkillPolicy
+    revocation: RevocationPolicy
+
+    @model_validator(mode="after")
+    def _check_internal_consistency(self) -> Self:
+        """Reject a policy that permits a use it also makes impossible."""
+        if self.raw_episodes.local_retention == "prohibited":
+            for name in (
+                "server_upload",
+                "public_release",
+                "reproduction_access",
+                "training_use",
+            ):
+                if getattr(self.raw_episodes, name) != "prohibited":
+                    raise ValueError(
+                        f"raw_episodes.{name} cannot be permitted while "
+                        "local_retention is prohibited; there would be nothing "
+                        "left to share"
+                    )
+        return self

techtree/models/engine.py

@@ -0,0 +1,156 @@
+"""The managed Verifiers engine. Spec 11.13, decisions 0003 A8/A9.
+
+The engine is a locked, content-addressed bundle: a Python version, a pinned
+Verifiers revision, and the reference packages, described by an
+``EngineDescriptor``. The bundle's digest covers the static files and is not
+stored inside the descriptor, because a document cannot contain its own hash.
+
+Host platforms use one vocabulary — Go/OCI-style ``<os>/<arch>`` — everywhere
+they appear. :func:`normalize_host_platform` is the only way a raw
+``sys.platform`` and ``platform.machine()`` pair becomes one of those strings,
+and it refuses rather than guesses on anything else. An unsupported host is a
+prerequisite failure with a name, not an install that fails later for reasons
+the user cannot read.
+
+The engine's host platform and the subject runtime's Docker platform share this
+vocabulary but remain separate concepts: one is where Techtree runs, the other
+is where the evaluated agent runs, and they are frequently not the same
+machine.
+"""
+
+from __future__ import annotations
+
+from typing import Final, Literal, Self
+
+from pydantic import model_validator
+
+from techtree.errors import PrerequisiteError
+from techtree.models.base import (
+    Digest,
+    NonEmptyString,
+    ProtocolModel,
+    StateModel,
+    UtcDateTime,
+)
+
+__all__ = [
+    "EngineDescriptor",
+    "EngineInstallation",
+    "EnginePackage",
+    "EngineStatus",
+    "HostPlatform",
+    "normalize_host_platform",
+]
+
+
+type HostPlatform = Literal[
+    "darwin/amd64",
+    "darwin/arm64",
+    "linux/amd64",
+    "linux/arm64",
+]
+"""The complete host-platform vocabulary. Decisions document 0003 A9."""
+
+#: Every machine string that means the same architecture. The keys are what
+#: ``platform.machine()`` reports across the platforms Techtree supports.
+_ARCHITECTURES: Final[dict[str, str]] = {
+    "aarch64": "arm64",
+    "amd64": "amd64",
+    "arm64": "arm64",
+    "x86_64": "amd64",
+}
+
+#: The closed vocabulary, keyed by the pair it is derived from. A table rather
+#: than string concatenation, so that the only values this function can return
+#: are the four the protocol defines.
+_HOST_PLATFORMS: Final[dict[tuple[str, str], HostPlatform]] = {
+    ("darwin", "amd64"): "darwin/amd64",
+    ("darwin", "arm64"): "darwin/arm64",
+    ("linux", "amd64"): "linux/amd64",
+    ("linux", "arm64"): "linux/arm64",
+}
+
+
+def normalize_host_platform(sys_platform: str, machine: str) -> HostPlatform:
+    """Return the ``<os>/<arch>`` name for a host, or refuse.
+
+    Raises :class:`~techtree.errors.PrerequisiteError` for any combination
+    outside the supported vocabulary. Guessing would produce an engine install
+    that resolves wheels for the wrong architecture and fails much later,
+    somewhere far less legible than here.
+    """
+    operating_system = sys_platform.strip().lower()
+    architecture = _ARCHITECTURES.get(machine.strip().lower(), "")
+    normalized = _HOST_PLATFORMS.get((operating_system, architecture))
+
+    if normalized is None:
+        raise PrerequisiteError(
+            f"unsupported host platform {sys_platform}/{machine}; Techtree "
+            "supports darwin and linux on arm64 and amd64",
+            code="unsupported_host_platform",
+            details={"sys_platform": sys_platform, "machine": machine},
+        )
+    return normalized
+
+
+class EnginePackage(ProtocolModel):
+    """One package shipped inside the engine bundle."""
+
+    name: NonEmptyString
+    version: NonEmptyString
+    source_digest: Digest
+
+
+class EngineDescriptor(ProtocolModel):
+    """What one engine bundle is, without saying what it hashes to."""
+
+    schema_version: Literal["techtree.engine.v1alpha1"]
+    name: NonEmptyString
+    python_version: NonEmptyString
+    verifiers_version: NonEmptyString
+    verifiers_revision: NonEmptyString
+    supported_hosts: list[HostPlatform]
+    packages: list[EnginePackage]
+
+    @model_validator(mode="after")
+    def _check_hosts_and_packages_are_listed_once(self) -> Self:
+        """Reject an empty or repeating host or package list."""
+        if not self.supported_hosts:
+            raise ValueError("an engine must support at least one host platform")
+        if len(set(self.supported_hosts)) != len(self.supported_hosts):
+            raise ValueError("supported_hosts must not repeat a platform")
+        names = [package.name for package in self.packages]
+        if len(set(names)) != len(names):
+            raise ValueError("an engine ships each package exactly once")
+        return self
+
+
+class EngineInstallation(StateModel):
+    """A locally installed engine, as the registry records it."""
+
+    digest: Digest
+    installed_at: UtcDateTime
+    python_executable: NonEmptyString
+    descriptor_digest: Digest
+    verified: bool
+
+
+class EngineStatus(ProtocolModel):
+    """What the CLI reports about one engine."""
+
+    digest: Digest
+    installed: bool
+    active: bool
+    verified: bool
+    path: NonEmptyString
+    python_executable: NonEmptyString | None
+    detail: NonEmptyString
+
+    @model_validator(mode="after")
+    def _check_status_is_coherent(self) -> Self:
+        """Reject a status that verifies or activates an engine that is absent."""
+        if not self.installed and (self.verified or self.active):
+            raise ValueError(
+                "an engine that is not installed cannot be active or verified"
+            )
+        return self

techtree/models/episode_receipt.py

@@ -0,0 +1,130 @@
+"""What one episode produced. Spec section 11.10.
+
+The shape is frozen now and fake-populated until WP6, which is the point: the
+statuses that say "this number is not evidence" have to exist before the first
+number does. ``development_only`` is a first-class score and evidence status
+rather than an absent field, so a fake receipt is unmistakably fake to a reader
+and to a service, and nothing has to infer it from context.
+
+A receipt points at ``campaign_spec_digest`` and carries the data policy that
+governed it. The public Climb is an optional context, never the anchor: a
+reproduction run has no Climb, and its receipts must still be complete.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Literal, Self
+
+from pydantic import model_validator
+
+from techtree.models.base import (
+    ArtifactRef,
+    Digest,
+    NonEmptyString,
+    ProtocolModel,
+)
+from techtree.models.campaign import ProgramRef, PublicContext
+from techtree.models.evaluation_backend import EvaluationBackendSpec
+from techtree.models.experiment import ExperimentVariant
+
+__all__ = [
+    "EpisodeReceipt",
+    "EvidenceStatus",
+    "NamedTraceReceipt",
+    "ScoreStatus",
+    "SubjectRuntimeReceipt",
+]
+
+
+class ScoreStatus(StrEnum):
+    """How much weight the recorded reward carries."""
+
+    PENDING = "pending"
+    VALID = "valid"
+    INVALID = "invalid"
+    ERRORED = "errored"
+    MISSING = "missing"
+    DEVELOPMENT_ONLY = "development_only"
+
+
+class EvidenceStatus(StrEnum):
+    """How complete the supporting evidence is."""
+
+    NOT_COLLECTED = "not_collected"
+    COMPLETE = "complete"
+    PARTIAL = "partial"
+    INVALID = "invalid"
+    DEVELOPMENT_ONLY = "development_only"
+
+
+class NamedTraceReceipt(ProtocolModel):
+    """One named trace within an episode."""
+
+    role: NonEmptyString
+    trace_id: NonEmptyString
+    trace_digest: Digest
+    task_hash: Digest
+    rewards: dict[str, float]
+    metrics: dict[str, float | None]
+    ok: bool
+
+
+class SubjectRuntimeReceipt(ProtocolModel):
+    """Where the subject agent actually executed, if it executed."""
+
+    kind: Literal["not_executed", "docker"]
+    resolved_image_digest: Digest | None = None
+    platform: NonEmptyString | None = None
+
+    @model_validator(mode="after")
+    def _check_runtime_evidence_matches_kind(self) -> Self:
+        """Reject runtime detail on an episode that never ran a runtime."""
+        if self.kind == "not_executed" and (
+            self.resolved_image_digest is not None or self.platform is not None
+        ):
+            raise ValueError(
+                "an episode that did not execute a runtime cannot report the "
+                "image or platform it executed on"
+            )
+        if self.kind == "docker" and self.resolved_image_digest is None:
+            raise ValueError("a docker episode records the image digest it ran")
+        return self
+
+
+class EpisodeReceipt(ProtocolModel):
+    """The complete record of one scored episode."""
+
+    schema_version: Literal["techtree.episode-receipt.v1alpha1"]
+    id: NonEmptyString
+    run_id: NonEmptyString
+    campaign_spec_digest: Digest
+    program_ref: ProgramRef | None
+    public_context: PublicContext | None
+    data_policy_digest: Digest
+    outcome_contract_digest: Digest | None
+    evaluation_backend: EvaluationBackendSpec
+    subject_runtime: SubjectRuntimeReceipt
+    variant: ExperimentVariant
+    experiment_manifest_digest: Digest
+    episode_id: NonEmptyString
+    episode_digest: Digest
+    task_hash: Digest
+    named_traces: dict[str, list[NamedTraceReceipt]]
+    score_status: ScoreStatus
+    evidence_status: EvidenceStatus
+    execution_backend: Literal["fake", "verifiers"]
+    artifacts: list[ArtifactRef]
+
+    @model_validator(mode="after")
+    def _check_fake_episodes_are_unmistakably_fake(self) -> Self:
+        """Refuse to let a fake episode wear a real score."""
+        if self.execution_backend == "fake" and (
+            self.score_status is not ScoreStatus.DEVELOPMENT_ONLY
+            or self.evidence_status is not EvidenceStatus.DEVELOPMENT_ONLY
+        ):
+            raise ValueError(
+                "a fake episode reports development_only score and evidence; "
+                "any other status would present invented numbers as results"
+            )
+        return self

techtree/models/evaluation_backend.py

@@ -0,0 +1,113 @@
+"""Who orchestrated and attested to an evaluation. Spec section 11.3.
+
+The evaluation backend and the subject runtime answer two different questions
+and are deliberately separate objects:
+
+``EvaluationBackendSpec``
+    Who ran the comparison and whose word the result rests on.
+
+``RuntimeSpec`` (in :mod:`techtree.models.campaign`)
+    Where the evaluated agent's process actually executed.
+
+Conflating them is how a self-reported local result quietly acquires the
+authority of a platform-attested one. Each backend kind therefore fixes the
+attestation it is allowed to claim, and the references that must or must not
+accompany it.
+
+The enum keeps the future backends so that stored documents and published
+schemas do not need a version bump when they arrive. Services enforce the
+narrower WP0–WP5 rule — only ``local_techtree`` — at the point of use, not
+here, because a document that merely mentions a future backend must still be
+parseable.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Literal, Self
+
+from pydantic import model_validator
+
+from techtree.models.base import NonEmptyString, ProtocolModel
+
+__all__ = [
+    "SUPPORTED_EVALUATION_BACKEND_KINDS",
+    "AttestationKind",
+    "EvaluationBackendKind",
+    "EvaluationBackendSpec",
+]
+
+
+class EvaluationBackendKind(StrEnum):
+    """Which system orchestrated the evaluation."""
+
+    LOCAL_TECHTREE = "local_techtree"
+    PRIME_LAB = "prime_lab"
+    INDEPENDENT_REPRODUCER = "independent_reproducer"
+
+
+class AttestationKind(StrEnum):
+    """Whose attestation the recorded result carries."""
+
+    PARTICIPANT = "participant"
+    PLATFORM = "platform"
+    INDEPENDENT = "independent"
+
+
+#: The only backend kinds WP0–WP5 services accept. The schema is wider than
+#: this on purpose; the runtime surface is not.
+SUPPORTED_EVALUATION_BACKEND_KINDS: frozenset[EvaluationBackendKind] = frozenset(
+    {EvaluationBackendKind.LOCAL_TECHTREE}
+)
+
+
+class EvaluationBackendSpec(ProtocolModel):
+    """The orchestrating backend and the attestation it carries."""
+
+    schema_version: Literal["techtree.evaluation-backend.v1alpha1"]
+    kind: EvaluationBackendKind
+    attestation: AttestationKind
+    workspace_ref: NonEmptyString | None = None
+    provider_run_ref: NonEmptyString | None = None
+    executor_identity: NonEmptyString | None = None
+
+    @model_validator(mode="after")
+    def _check_kind_agrees_with_its_evidence(self) -> Self:
+        """Reject attestation and reference combinations a kind cannot support."""
+        if self.kind is EvaluationBackendKind.LOCAL_TECHTREE:
+            if self.attestation is not AttestationKind.PARTICIPANT:
+                raise ValueError(
+                    "local_techtree evaluation is self-reported, so its "
+                    "attestation must be participant"
+                )
+            if self.workspace_ref is not None:
+                raise ValueError("local_techtree evaluation has no workspace_ref")
+            if self.provider_run_ref is not None:
+                raise ValueError("local_techtree evaluation has no provider_run_ref")
+            # executor_identity stays optional: WP0-WP5 record no identity.
+            return self
+
+        if self.kind is EvaluationBackendKind.PRIME_LAB:
+            if self.attestation is not AttestationKind.PLATFORM:
+                raise ValueError(
+                    "prime_lab evaluation is platform-attested, so its "
+                    "attestation must be platform"
+                )
+            if self.workspace_ref is None and self.provider_run_ref is None:
+                raise ValueError(
+                    "prime_lab evaluation must carry a workspace_ref or a "
+                    "provider_run_ref so the platform record can be found"
+                )
+            return self
+
+        if self.attestation is not AttestationKind.INDEPENDENT:
+            raise ValueError(
+                "independent_reproducer evaluation must carry an independent "
+                "attestation"
+            )
+        if self.executor_identity is None:
+            raise ValueError(
+                "independent_reproducer evaluation must name the executor_identity "
+                "that stands behind it"
+            )
+        return self

techtree/models/experiment.py

@@ -0,0 +1,154 @@
+"""One resolved variant of a Campaign, and the comparison. Spec section 11.8.
+
+An ``ExperimentManifest`` is a Campaign with every choice made: this taskset,
+this agent, this skill list. Two of them — baseline and candidate — are what a
+run actually executes.
+
+The comparison deliberately looks at ``configuration`` and nothing else. A
+manifest identifier, a variant name, a creation time, and a public context all
+differ between two correct manifests of the same experiment; comparing them
+would report differences that mean nothing and bury the one difference that
+means everything. ``ManifestComparison`` therefore compares the configurations
+and reports whether the only difference found is the one the mutation contract
+permits.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Literal, Self
+
+from pydantic import model_validator
+
+from techtree.models.base import (
+    Digest,
+    JsonValue,
+    NonEmptyString,
+    ProtocolModel,
+    UtcDateTime,
+)
+from techtree.models.campaign import (
+    AgentSpec,
+    BudgetSpec,
+    CampaignTaskset,
+    EnvironmentSpec,
+    EvidenceRequirements,
+    ExecutionSpec,
+    MutationContract,
+    MutationKind,
+    ProgramRef,
+    PublicContext,
+    ScoringSpec,
+)
+from techtree.models.evaluation_backend import EvaluationBackendSpec
+
+__all__ = [
+    "ExperimentConfiguration",
+    "ExperimentManifest",
+    "ExperimentVariant",
+    "JsonDifference",
+    "ManifestComparison",
+]
+
+
+class ExperimentVariant(StrEnum):
+    """Which side of the comparison a manifest describes."""
+
+    BASELINE = "baseline"
+    CANDIDATE = "candidate"
+
+
+class ExperimentConfiguration(ProtocolModel):
+    """The part of a manifest that is compared.
+
+    This mirrors the Campaign's scientific fields with the choices resolved. It
+    holds no identifier, no timestamp, and no public context, precisely so that
+    two manifests of the same experiment have byte-identical configurations
+    except where the mutation contract allows them to differ.
+    """
+
+    taskset: CampaignTaskset
+    environment: EnvironmentSpec
+    agents: dict[str, AgentSpec]
+    mutation_contract: MutationContract
+    evaluation_backend: EvaluationBackendSpec
+    execution: ExecutionSpec
+    scoring: ScoringSpec
+    evidence: EvidenceRequirements
+    budgets: BudgetSpec
+    data_policy_digest: Digest
+    outcome_contract_digest: Digest | None
+
+
+class ExperimentManifest(ProtocolModel):
+    """One fully resolved variant derived from a Campaign."""
+
+    schema_version: Literal["techtree.experiment.v1alpha1"]
+    id: NonEmptyString
+    campaign_spec_digest: Digest
+    program_ref: ProgramRef | None
+    public_context: PublicContext | None
+    variant: ExperimentVariant
+    configuration: ExperimentConfiguration
+    configuration_digest: Digest
+    created_at: UtcDateTime
+
+    @model_validator(mode="after")
+    def _check_variant_skill_count(self) -> Self:
+        """Hold each variant to the skill count its mutation kind requires."""
+        subject = self.configuration.agents.get("subject")
+        if subject is None:
+            raise ValueError("an experiment configuration defines a subject agent")
+        skills = len(subject.harness.skills)
+
+        if self.variant is ExperimentVariant.CANDIDATE:
+            if skills != 1:
+                raise ValueError("the candidate variant carries exactly one skill")
+            return self
+
+        # What the baseline carries is what the candidate is measured against,
+        # and the mutation kind is what says which that is (spec section 3.1):
+        # nothing for an insertion, the skill being revised for a replacement.
+        # The two sides' root digests are a property of the pair rather than of
+        # one manifest, so they are checked where the pair is — the builder when
+        # one is derived, ``manifests.compare`` when two are read back.
+        if self.configuration.mutation_contract.kind is MutationKind.SKILL_INSERTION:
+            if skills != 0:
+                raise ValueError("the baseline variant carries no candidate skill")
+        elif skills != 1:
+            raise ValueError(
+                "the baseline variant of a skill_replacement carries exactly one "
+                "skill to replace"
+            )
+        return self
+
+
+class JsonDifference(ProtocolModel):
+    """One JSON Pointer at which two configurations disagree."""
+
+    pointer: NonEmptyString
+    baseline: JsonValue | None
+    candidate: JsonValue | None
+
+
+class ManifestComparison(ProtocolModel):
+    """Whether the candidate differs from the baseline only where permitted."""
+
+    baseline_configuration_digest: Digest
+    candidate_configuration_digest: Digest
+    differences: list[JsonDifference]
+    allowed_differences: list[NonEmptyString]
+    controlled: bool
+    violations: list[NonEmptyString]
+
+    @model_validator(mode="after")
+    def _check_control_agrees_with_violations(self) -> Self:
+        """Reject a comparison that calls itself controlled while listing faults."""
+        if self.controlled and self.violations:
+            raise ValueError(
+                "a controlled comparison has no violations; listing both leaves "
+                "a reader unable to tell which one is true"
+            )
+        if not self.controlled and not self.violations:
+            raise ValueError("an uncontrolled comparison must say which rule it broke")
+        return self