techtree 0.1.0__py3-none-any.whl

techtree/models/run.py

@@ -0,0 +1,214 @@
+"""Run requests, phases, events, and local state. Spec 11.12, decisions 0003.
+
+A run has one phase at a time and moves forward through them. The phase names
+are part of the CLI contract, so they are an enum rather than free text, and
+the event stream records both the phase entered and the phase left — a reader
+reconstructing a run from events should never have to infer where it came from.
+
+``RunRequest`` is immutable: it is what was asked for. ``RunState`` is a
+``StateModel`` because it is what is currently true, rewritten as the worker
+makes progress. Keeping them in separate classes is what stops a heartbeat
+update from being able to alter the request it is executing.
+
+Decisions document 0003 A5 puts ``policy_acknowledgement`` here. A draft states
+which rights policy must be accepted; the run records that it *was* accepted,
+by which method, and when. Decisions document 0019 section 2 makes that one
+answer to one review rather than a handle that had to be presented: nothing is
+started without the review having been shown and explicitly accepted, and a
+caller that cannot be asked has to say so with a flag.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Literal, Self
+
+from pydantic import Field, model_validator
+
+from techtree.models.base import (
+    Digest,
+    JsonValue,
+    NonEmptyString,
+    ProtocolModel,
+    StateModel,
+    UtcDateTime,
+)
+from techtree.models.campaign import ProgramRef, PublicContext
+from techtree.models.cli import CliError
+from techtree.models.evaluation_backend import EvaluationBackendSpec
+
+__all__ = [
+    "ExecutorKind",
+    "PolicyAcknowledgement",
+    "RunEvent",
+    "RunPhase",
+    "RunProgress",
+    "RunRequest",
+    "RunState",
+    "RunStatus",
+    "VariantProgress",
+]
+
+type ExecutorKind = Literal["fake", "verifiers"]
+"""Which executor a run was created to be executed by.
+
+The two names are the two that exist, and they are the same two an
+:class:`~techtree.models.episode_receipt.EpisodeReceipt` records under
+``execution_backend`` — a run and the receipts it produces should not need a
+translation table to agree on what ran.
+
+The value is decided when the run is created, from the Campaign, because that
+is when a person is told what is about to happen and what it will cost. It is
+a record of the answer, not the place the answer is worked out: the worker
+re-reads the run's own staged Campaign and refuses a request that disagrees
+with it, so a hand-edited request buys nothing.
+"""
+
+
+class RunPhase(StrEnum):
+    """Where a run currently is."""
+
+    CREATED = "created"
+    VALIDATING_TASKSET = "validating_taskset"
+    RUNNING_BASELINE = "running_baseline"
+    RUNNING_CANDIDATE = "running_candidate"
+    #: Both variants in flight at once. The sequential pair above stays for the
+    #: fake executor; spec section 3.3 adds this one rather than replacing them.
+    RUNNING_VARIANTS = "running_variants"
+    BUILDING_RECEIPTS = "building_receipts"
+    VERIFYING_COMPARISON = "verifying_comparison"
+    BUILDING_REPORT = "building_report"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    CANCEL_REQUESTED = "cancel_requested"
+    CANCELLED = "cancelled"
+
+
+class PolicyAcknowledgement(ProtocolModel):
+    """That a specific rights policy was accepted, how, and when.
+
+    Decisions document 0003 A5, as amended by 0019 section 2.
+    ``explicit_cli_review`` says the review of what the run would do — its
+    size, the spending limit its Campaign declares, the one change being
+    measured, where model calls go, and what an upload would carry — was put in
+    front of whoever started it and
+    explicitly accepted. It covers both spellings of that answer at the command
+    line, the typed ``y`` and the flag an operator passes instead, because the
+    fact being recorded is the same one; who gave the answer is recorded
+    separately on the run's ``run.approved`` event.
+
+    ``host_agent_confirmation`` is the approval surface the plugin presents in
+    a conversation: the review is shown there, the person confirms there, and
+    the plugin then starts that exact draft. The process that writes this
+    record is not the one that asked the question, so which surface it was is
+    declared by whoever starts the run rather than guessed at from the fact
+    that nobody was prompted here.
+    """
+
+    data_policy_digest: Digest
+    method: Literal[
+        "explicit_cli_review",
+        "host_agent_confirmation",
+    ]
+    acknowledged_at: UtcDateTime
+
+
+class RunRequest(ProtocolModel):
+    """What was asked for, fixed at the moment the run was created."""
+
+    run_id: NonEmptyString
+    draft_id: NonEmptyString
+    draft_digest: Digest
+    campaign_spec_digest: Digest
+    program_ref: ProgramRef | None
+    public_context: PublicContext | None
+    data_policy_digest: Digest
+    outcome_contract_digest: Digest | None
+    evaluation_backend: EvaluationBackendSpec
+    taskset_lock_digest: Digest | None
+    baseline_manifest_digest: Digest
+    candidate_manifest_digest: Digest
+    policy_acknowledgement: PolicyAcknowledgement
+    executor_kind: ExecutorKind
+    created_at: UtcDateTime
+
+    @model_validator(mode="after")
+    def _check_acknowledged_policy_is_the_one_being_run(self) -> Self:
+        """Reject a run acknowledging a different policy than it executes under."""
+        if self.policy_acknowledgement.data_policy_digest != self.data_policy_digest:
+            raise ValueError(
+                "the acknowledged DataPolicy is not the one this run executes under"
+            )
+        if self.baseline_manifest_digest == self.candidate_manifest_digest:
+            raise ValueError("a run compares two different manifests")
+        return self
+
+
+class RunEvent(ProtocolModel):
+    """One appended record of something that happened to a run."""
+
+    sequence: int = Field(ge=0)
+    timestamp: UtcDateTime
+    run_id: NonEmptyString
+    previous_phase: RunPhase | None
+    phase: RunPhase
+    kind: NonEmptyString
+    details: dict[str, JsonValue]
+
+
+class RunProgress(StateModel):
+    """How far through the current phase the worker is."""
+
+    current: int = Field(ge=0)
+    total: int = Field(ge=0)
+    label: NonEmptyString
+
+    @model_validator(mode="after")
+    def _check_progress_is_within_its_total(self) -> Self:
+        """Reject progress that has passed the end of the work it describes."""
+        if self.current > self.total:
+            raise ValueError("progress cannot exceed its total")
+        return self
+
+
+class VariantProgress(StateModel):
+    """How far one side of a concurrent comparison has got. Spec section 3.3.
+
+    ``RunProgress`` measures one position in one phase, which is all a
+    sequential run has to report. When both variants are in flight there are two
+    positions at once, and each carries its own episode counts and its own
+    lifecycle, so they are projected side by side rather than flattened.
+    """
+
+    variant: Literal["baseline", "candidate"]
+    completed: int = Field(ge=0)
+    total: int = Field(ge=0)
+    running: int = Field(ge=0)
+    errored: int = Field(ge=0)
+    state: Literal["pending", "running", "completed", "failed", "cancelled"]
+
+
+class RunState(StateModel):
+    """What is currently true of a run, rewritten as it advances."""
+
+    run_id: NonEmptyString
+    phase: RunPhase
+    sequence: int = Field(ge=0)
+    updated_at: UtcDateTime
+    worker_pid: int | None
+    worker_started_at: UtcDateTime | None
+    heartbeat_at: UtcDateTime | None
+    cancel_requested_at: UtcDateTime | None
+    error: CliError | None
+    progress: RunProgress | None
+    variant_progress: dict[str, VariantProgress] = Field(default_factory=dict)
+    result_digest: Digest | None
+
+
+class RunStatus(ProtocolModel):
+    """A run's state plus the liveness facts only the host can determine."""
+
+    state: RunState
+    worker_alive: bool
+    heartbeat_stale: bool
+    result_available: bool

techtree/models/skill.py

@@ -0,0 +1,156 @@
+"""Candidate skills and the draft that submits them. Spec 11.7, decisions 0003.
+
+A ``SkillArtifact`` is the content-addressed description of what a participant
+wrote: the files, their digests, and the digest of the tree as a whole. No
+local path ever enters it, because a draft that recorded where a skill lived on
+one machine would not be reproducible on another and would leak the author's
+directory layout into a public object.
+
+A ``SubmissionDraft`` is the complete, reviewable statement of what is about to
+be run, including the rights the participant is being asked to accept.
+Decisions document 0003 A5 splits that from the moment of accepting: the draft
+carries a ``PolicyAcceptanceRequirement`` — the digest, whether acceptance is
+required, and a stable human-readable summary — while the acknowledgement
+itself is recorded on the ``RunRequest``. Stating what will have to be accepted
+and recording that it was accepted are two different facts, and the two objects
+keep them from being blurred.
+"""
+
+from __future__ import annotations
+
+from typing import Final, Literal, Self
+
+from pydantic import Field, model_validator
+
+from techtree.models.base import (
+    Digest,
+    NonEmptyString,
+    ProtocolModel,
+    UtcDateTime,
+)
+from techtree.models.campaign import ProgramRef, PublicContext
+
+__all__ = [
+    "SKILL_ENTRY_FILE",
+    "PolicyAcceptanceRequirement",
+    "SkillArtifact",
+    "SkillFile",
+    "SubmissionDraft",
+]
+
+#: The one file every instruction skill must contain. Its absence is not a
+#: warning: a skill with no instructions is not a skill.
+SKILL_ENTRY_FILE: Final = "SKILL.md"
+
+
+def _check_relative_posix_path(value: str) -> None:
+    """Raise when a path is absolute, escaping, or not POSIX-spelled."""
+    if value != value.strip():
+        raise ValueError(f"skill path has surrounding whitespace: {value!r}")
+    if value.startswith("/") or (len(value) > 1 and value[1] == ":"):
+        raise ValueError(f"skill paths are relative; got {value!r}")
+    if "\\" in value:
+        raise ValueError(f"skill paths use forward slashes; got {value!r}")
+    segments = value.split("/")
+    if any(segment in ("", ".", "..") for segment in segments):
+        raise ValueError(f"skill path is not a normalized relative path: {value!r}")
+
+
+class SkillFile(ProtocolModel):
+    """One file inside a skill, addressed by content."""
+
+    path: NonEmptyString
+    media_type: NonEmptyString
+    size: int = Field(ge=0)
+    digest: Digest
+
+    @model_validator(mode="after")
+    def _check_path(self) -> Self:
+        """Reject anything that is not a normalized relative POSIX path."""
+        _check_relative_posix_path(self.path)
+        return self
+
+
+class SkillArtifact(ProtocolModel):
+    """A complete candidate skill, addressed by content."""
+
+    schema_version: Literal["techtree.skill.v1alpha1"]
+    name: NonEmptyString
+    root_digest: Digest
+    archive_digest: Digest
+    files: list[SkillFile]
+    source_kind: Literal["manual"]
+    parent_skill_digest: Digest | None
+
+    @model_validator(mode="after")
+    def _check_file_list(self) -> Self:
+        """Require the entry file, a stable order, and no repeated path."""
+        paths = [file.path for file in self.files]
+        if SKILL_ENTRY_FILE not in paths:
+            raise ValueError(f"a skill must contain {SKILL_ENTRY_FILE}")
+        if len(set(paths)) != len(paths):
+            raise ValueError("a skill lists each path exactly once")
+        if paths != sorted(paths):
+            raise ValueError(
+                "skill files are sorted by path so that the same tree always "
+                "produces the same digest"
+            )
+        return self
+
+
+class PolicyAcceptanceRequirement(ProtocolModel):
+    """The rights policy a participant is being asked to accept.
+
+    Decisions document 0003 A5. This states what must be accepted; it does not
+    record that anyone accepted it.
+    """
+
+    data_policy_digest: Digest
+    required: bool
+    summary: NonEmptyString
+
+
+class SubmissionDraft(ProtocolModel):
+    """Everything a participant is about to commit to, in one object."""
+
+    schema_version: Literal["techtree.submission-draft.v1alpha1"]
+    id: NonEmptyString
+    campaign_spec_digest: Digest
+    program_ref: ProgramRef | None
+    public_context: PublicContext | None
+    data_policy_digest: Digest
+    outcome_contract_digest: Digest | None
+    skill_artifact: SkillArtifact
+    baseline_manifest_digest: Digest
+    candidate_manifest_digest: Digest
+    included_files: list[NonEmptyString]
+    estimated_episodes: int = Field(ge=1)
+    policy_acceptance: PolicyAcceptanceRequirement
+    warnings: list[NonEmptyString]
+    created_at: UtcDateTime
+
+    @model_validator(mode="after")
+    def _check_draft(self) -> Self:
+        """Keep the draft self-consistent about rights, files, and variants."""
+        if self.policy_acceptance.data_policy_digest != self.data_policy_digest:
+            raise ValueError(
+                "the draft asks for acceptance of a different DataPolicy than "
+                "the one it runs under"
+            )
+        if self.baseline_manifest_digest == self.candidate_manifest_digest:
+            raise ValueError(
+                "baseline and candidate manifests must differ; an identical "
+                "pair measures nothing"
+            )
+        for path in self.included_files:
+            _check_relative_posix_path(path)
+        if self.included_files != sorted(self.included_files):
+            raise ValueError("included_files is sorted")
+        if len(set(self.included_files)) != len(self.included_files):
+            raise ValueError("included_files lists each path exactly once")
+        artifact_paths = [file.path for file in self.skill_artifact.files]
+        if self.included_files != artifact_paths:
+            raise ValueError(
+                "included_files must be exactly the files in the skill artifact"
+            )
+        return self

techtree/models/uplift_report.py

@@ -0,0 +1,158 @@
+"""The result of one comparison. Spec section 11.11.
+
+An ``UpliftReport`` answers one question: did the candidate satisfy the
+scientific comparison contract? It does not answer whether anyone should deploy
+the result. That is a release decision, it depends on cost, risk, and product
+context that this object knows nothing about, and it belongs to a future
+``ReleaseDecision``.
+
+The five statuses are separate on purpose. A run can complete while its scores
+are development-only; scores can be valid while evidence is partial; a
+comparison can be controlled while publication is blocked by the data policy.
+Collapsing them into one "status" would force a caller to guess which of those
+five things a single word referred to.
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Literal, Self
+
+from pydantic import Field, model_validator
+
+from techtree.models.base import Digest, NonEmptyString, ProtocolModel, UtcDateTime
+from techtree.models.campaign import ProgramRef, PublicContext
+from techtree.models.episode_receipt import EvidenceStatus, ScoreStatus
+from techtree.models.evaluation_backend import EvaluationBackendSpec
+from techtree.models.experiment import ManifestComparison
+
+__all__ = [
+    "ComparisonStatus",
+    "ExecutionStatus",
+    "PrimaryUpliftResult",
+    "PublicationStatus",
+    "TaskDelta",
+    "UpliftDecision",
+    "UpliftReport",
+    "UpliftStatuses",
+]
+
+
+class ExecutionStatus(StrEnum):
+    """How the run itself ended."""
+
+    COMPLETED = "completed"
+    PARTIAL = "partial"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+
+class ComparisonStatus(StrEnum):
+    """Whether the two manifests differed only where permitted."""
+
+    PENDING = "pending"
+    CONTROLLED = "controlled"
+    CONTROLLED_WITH_WARNINGS = "controlled_with_warnings"
+    INVALID = "invalid"
+    DEVELOPMENT_ONLY = "development_only"
+
+
+class PublicationStatus(StrEnum):
+    """Where the report stands with respect to being published."""
+
+    NOT_REQUESTED = "not_requested"
+    BLOCKED = "blocked"
+    PENDING = "pending"
+    PUBLISHED = "published"
+    FAILED = "failed"
+
+
+class UpliftDecision(StrEnum):
+    """The verdict on the scientific comparison contract."""
+
+    ACCEPTED = "accepted"
+    REJECTED = "rejected"
+    INCONCLUSIVE = "inconclusive"
+    INVALID = "invalid"
+    DEVELOPMENT_ONLY = "development_only"
+
+
+class TaskDelta(ProtocolModel):
+    """What one task contributed to the comparison."""
+
+    task_hash: Digest
+    baseline_reward: float
+    candidate_reward: float
+    delta: float
+
+
+class PrimaryUpliftResult(ProtocolModel):
+    """The headline comparison on the primary reward."""
+
+    reward_name: NonEmptyString
+    baseline_mean: float
+    candidate_mean: float
+    absolute_delta: float
+    relative_delta: float | None
+    wins: int = Field(ge=0)
+    losses: int = Field(ge=0)
+    ties: int = Field(ge=0)
+
+
+class UpliftStatuses(ProtocolModel):
+    """The five independent statuses of a report."""
+
+    execution: ExecutionStatus
+    score: ScoreStatus
+    evidence: EvidenceStatus
+    comparison: ComparisonStatus
+    publication: PublicationStatus
+
+
+class UpliftReport(ProtocolModel):
+    """The complete result of one baseline-versus-candidate comparison."""
+
+    schema_version: Literal["techtree.uplift-report.v1alpha1"]
+    id: NonEmptyString
+    run_id: NonEmptyString
+    campaign_spec_digest: Digest
+    program_ref: ProgramRef | None
+    public_context: PublicContext | None
+    data_policy_digest: Digest
+    outcome_contract_digest: Digest | None
+    evaluation_backend: EvaluationBackendSpec
+    taskset_validation_receipt_digest: Digest
+    baseline_manifest_digest: Digest
+    candidate_manifest_digest: Digest
+    statuses: UpliftStatuses
+    manifest_comparison: ManifestComparison
+    primary_result: PrimaryUpliftResult
+    task_deltas: list[TaskDelta]
+    decision: UpliftDecision
+    proof_grade: Literal["development_only", "P1"]
+    publication_eligible: bool
+    created_at: UtcDateTime
+
+    @model_validator(mode="after")
+    def _check_report_cannot_overclaim(self) -> Self:
+        """Keep a development-only report from presenting itself as evidence."""
+        development_only = self.proof_grade == "development_only"
+        if development_only and self.decision is not UpliftDecision.DEVELOPMENT_ONLY:
+            raise ValueError(
+                "a development_only report reaches a development_only decision"
+            )
+        if development_only and self.publication_eligible:
+            raise ValueError("a development_only report is never publication eligible")
+        if self.publication_eligible and self.statuses.publication is (
+            PublicationStatus.BLOCKED
+        ):
+            raise ValueError(
+                "a report cannot be publication eligible while its publication "
+                "status is blocked"
+            )
+        if self.baseline_manifest_digest == self.candidate_manifest_digest:
+            raise ValueError(
+                "a report compares two different manifests; an identical pair "
+                "measures nothing"
+            )
+        return self