techtree 0.1.0__py3-none-any.whl

techtree/uplift/derive.py

@@ -0,0 +1,203 @@
+"""Turning a finished comparison into the next one. Spec section 7.19.
+
+A first run answers "does this Skill beat no Skill?". The question after it is
+"does this revision beat the Skill that already worked?", and spec section 3.1
+gives that its own mutation kind rather than its own protocol: it is the same
+Campaign with the mutation contract changed and the baseline given the Skill
+being revised.
+
+This module derives that Campaign, and it is written so that the derivation
+cannot quietly become a second experiment.
+
+*Every scientific field is copied, deeply.* Taskset, membership, validation
+receipt, environment, model, sampling, harness, runtime, tools, scoring,
+evidence, budgets, DataPolicy digest and OutcomeContract digest come across
+byte-identically. A value this module computed would be a value the source run
+never measured under, and the two reports would not be about the same taskset.
+
+*Exactly two things change, and both are named.* The mutation contract becomes
+``skill_replacement`` bounded at exactly one Skill, and the subject harness
+carries Skill v1. Those two are why the derived Campaign has a different
+digest, and they are the whole difference.
+
+*The public wrapper does not come across.* A ``ClimbManifest`` can only require
+``skill_insertion`` — :class:`~techtree.models.climb.CandidatePolicy` types it
+that way — so no public Climb wraps a replacement, and the derived Campaign
+carries no public context. Spec section 7.19 allows a separate Climb to
+authorize one later; none exists, so none is invented.
+
+The Skill the baseline carries is the one that was *evaluated*, identified by
+its content address. Nothing here reads a directory, so a Skill that has been
+edited since the first run cannot become the baseline the second run claims to
+have measured against.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from techtree.canonical import digest_object
+from techtree.errors import ValidationError
+from techtree.manifests.builder import (
+    build_baseline_manifest,
+    build_candidate_manifest,
+    build_skill_reference,
+)
+from techtree.manifests.compare import assert_controlled_comparison, compare_manifests
+from techtree.models.base import Digest
+from techtree.models.campaign import (
+    SKILL_MUTATION_POINTER,
+    SUBJECT_AGENT,
+    AgentSpec,
+    CampaignSpec,
+    HarnessSpec,
+    MutationContract,
+    MutationKind,
+)
+from techtree.models.experiment import ExperimentManifest, ManifestComparison
+from techtree.models.skill import SkillArtifact
+from techtree.models.uplift_report import UpliftReport
+
+__all__ = [
+    "REPLACEMENT_DERIVATION_FAILED",
+    "REPLACEMENT_PURPOSE",
+    "derive_replacement_manifests",
+    "derive_skill_replacement_campaign",
+]
+
+#: The one code every refusal in this module reports. Spec section 15 fixes the
+#: vocabulary; this is the derivation's own entry in it.
+REPLACEMENT_DERIVATION_FAILED = "replacement_derivation_failed"
+
+#: Spec section 7.19: the purpose remains what it was. A Campaign measuring
+#: something else is not one an improvement loop continues from, and changing
+#: its purpose on the way through would be answering a different question under
+#: the first question's name.
+REPLACEMENT_PURPOSE = "component_uplift"
+
+
+def derive_skill_replacement_campaign(
+    *,
+    source_campaign: CampaignSpec,
+    source_run: UpliftReport,
+    baseline_skill: SkillArtifact,
+    candidate_skill: SkillArtifact,
+) -> CampaignSpec:
+    """Derive the local Campaign that compares Skill v1 against Skill v2.
+
+    ``source_run`` is the signed report of the run being continued from. It is
+    required to be a report *of* ``source_campaign``, so a derivation cannot
+    pair one run's evidence with another run's science.
+    """
+    _require(
+        source_run.campaign_spec_digest == digest_object(source_campaign),
+        "this report is not a report of the Campaign it is being continued "
+        "from, so the run it describes measured something else",
+        expected=source_run.campaign_spec_digest,
+        computed=digest_object(source_campaign),
+    )
+    _require(
+        source_campaign.metadata.purpose == REPLACEMENT_PURPOSE,
+        "a Skill replacement continues a component uplift, and this Campaign's "
+        f"purpose is {source_campaign.metadata.purpose}",
+        purpose=source_campaign.metadata.purpose,
+    )
+    _require(
+        baseline_skill.root_digest != candidate_skill.root_digest,
+        "the proposed Skill has the same content tree as the Skill it would "
+        "replace, so the pair would measure nothing",
+        root_digest=baseline_skill.root_digest,
+    )
+
+    subject = source_campaign.agents[SUBJECT_AGENT]
+    replaced = AgentSpec(
+        model=subject.model.model_copy(deep=True),
+        sampling=subject.sampling.model_copy(deep=True),
+        harness=HarnessSpec(
+            id=subject.harness.id,
+            version=subject.harness.version,
+            use_bundled_skill=subject.harness.use_bundled_skill,
+            # The baseline of a replacement is the Skill being revised, named
+            # by the content address the first run actually evaluated.
+            skills=[build_skill_reference(baseline_skill)],
+        ),
+        runtime=subject.runtime.model_copy(deep=True),
+        trainable=subject.trainable,
+    )
+
+    return CampaignSpec(
+        schema_version=source_campaign.schema_version,
+        kind=source_campaign.kind,
+        metadata=source_campaign.metadata.model_copy(deep=True),
+        context=source_campaign.context.model_copy(deep=True),
+        taskset=source_campaign.taskset.model_copy(deep=True),
+        environment=source_campaign.environment.model_copy(deep=True),
+        agents={SUBJECT_AGENT: replaced},
+        mutation_contract=MutationContract(
+            kind=MutationKind.SKILL_REPLACEMENT,
+            target_agent="subject",
+            allowed_differences=[SKILL_MUTATION_POINTER],
+            # Both sides carry exactly one Skill: there is nothing to add and
+            # nothing to remove, only one tree to swap for another.
+            minimum_skills=1,
+            maximum_skills=1,
+        ),
+        evaluation_backend=source_campaign.evaluation_backend.model_copy(deep=True),
+        execution=source_campaign.execution.model_copy(deep=True),
+        scoring=source_campaign.scoring.model_copy(deep=True),
+        evidence=source_campaign.evidence.model_copy(deep=True),
+        budgets=source_campaign.budgets.model_copy(deep=True),
+        data_policy_digest=source_campaign.data_policy_digest,
+    )
+
+
+def derive_replacement_manifests(
+    *,
+    campaign: CampaignSpec,
+    candidate_skill: SkillArtifact,
+    campaign_digest: Digest | None = None,
+    created_at: datetime | None = None,
+) -> tuple[ExperimentManifest, ExperimentManifest, ManifestComparison]:
+    """Build both variants of a replacement and require the pair to be controlled.
+
+    The baseline is built from the Campaign alone, because a replacement
+    Campaign *is* its own baseline: it carries Skill v1 in the subject harness.
+    The candidate is the same configuration with that one list replaced. Both
+    carry no public context, and the comparison is required to be controlled
+    before either is returned.
+    """
+    _require(
+        campaign.mutation_contract.kind is MutationKind.SKILL_REPLACEMENT,
+        "these manifests would be a Skill replacement and this Campaign "
+        f"declares {campaign.mutation_contract.kind.value}",
+        mutation_kind=campaign.mutation_contract.kind.value,
+    )
+
+    digest = campaign_digest or digest_object(campaign)
+    baseline = build_baseline_manifest(
+        campaign=campaign,
+        campaign_digest=digest,
+        public_context=None,
+        created_at=created_at,
+    )
+    candidate = build_candidate_manifest(
+        campaign=campaign,
+        campaign_digest=digest,
+        skill=candidate_skill,
+        public_context=None,
+        created_at=created_at,
+    )
+    comparison = compare_manifests(baseline, candidate, campaign.mutation_contract)
+    assert_controlled_comparison(comparison)
+    return baseline, candidate, comparison
+
+
+def _require(condition: bool, message: str, **details: str) -> None:
+    """Raise a typed refusal unless the condition holds."""
+    if condition:
+        return
+    raise ValidationError(
+        message,
+        code=REPLACEMENT_DERIVATION_FAILED,
+        details=dict(details),
+    )

techtree/uplift/public_tasks.py

@@ -0,0 +1,151 @@
+"""What may be shown about a task, taskset by taskset. Decisions R1, 0014.
+
+Ratified decision R1 lets an improvement context carry a task's *public input*
+and forbids it carrying the subject's reply, the expected answer, grader
+source, or any hidden field. It also says the disclosure policy is
+taskset-specific and must never be inferred: "not secret" is not something to
+work out from a taskset nobody wrote a policy for.
+
+So this module is a lookup, not a rule. A taskset this build knows the policy
+for gets a projection that names its public input; every other taskset gets
+:func:`~techtree.uplift.context.hash_only_projection`, which names a task by
+its position and the head of its committed hash and shows nothing else. Adding
+a taskset here is a deliberate act, and the absence of one is a safe answer
+rather than a missing feature.
+
+Why it matters that the input is shown at all: the introductory Climb's whole
+subject is a Skill with one wrong rule in it, and which tasks it fails is the
+evidence for finding that rule. A reader given only "task 11 failed" and a hash
+cannot see a pattern in the inputs, because it has not been shown any. That is
+not privacy, it is an absence of evidence, and decision 0014 records what it
+cost: two rehearsal attempts in which the model diagnosed a defect the
+membership does not contain.
+
+The answer never travels, and it is worth being exact about how. What is read
+here is ``PROVING_INPUTS``, the frozen input list the reference taskset ships.
+The oracle that turns an input into an answer — ``branch_code`` and
+``branch_code_number`` — lives in ``algorithm.py`` in the same package, and
+that module *is* loaded into ``sys.modules``: ``dataset.py`` imports
+``normalize_input`` from it, so the input list cannot be read without it. What
+does not happen is the part that matters. Neither answer function is called,
+no answer is computed, nothing derived from one is stored, and nothing but the
+inputs leaves this module.
+
+Saying "the oracle is not imported" would have been the more comfortable
+sentence and it would have been false, which is the worse of the two
+properties a self-declaration can have.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from typing import Final
+
+from techtree.engines.bundle import embedded_engine_root
+from techtree.errors import ValidationError
+from techtree.models.base import Digest
+from techtree.models.campaign import CampaignSpec
+from techtree.uplift.context import (
+    TaskPublicProjection,
+    TaskPublicProjectionProvider,
+    hash_only_projection,
+)
+
+__all__ = [
+    "public_projection_for",
+]
+
+#: The one taskset this build has a disclosure policy for. Spec section 22:
+#: its inputs are single lowercase common tree names, its prompt template is
+#: published verbatim, and its answers are the only part that is withheld.
+_REFERENCE_TASKSET: Final = "procedure-transfer-v1"
+
+#: Where the frozen input list lives inside the packaged engine bundle. The
+#: module beside it computes answers and is deliberately not touched.
+_DATASET_MODULE: Final = "dataset.py"
+
+
+def public_projection_for(campaign: CampaignSpec) -> TaskPublicProjectionProvider:
+    """Return how much of each task this Campaign's taskset may show.
+
+    Args:
+        campaign: The Campaign whose committed membership is being projected.
+
+    Returns:
+        A provider naming each task's public input when the taskset has a
+        disclosure policy here, and the hash-only provider when it has none.
+    """
+    reference = campaign.taskset.ref
+    if reference.id != _REFERENCE_TASKSET or reference.package.name != (
+        _REFERENCE_TASKSET
+    ):
+        return hash_only_projection
+    return _branch_code_projection(campaign)
+
+
+def _branch_code_projection(campaign: CampaignSpec) -> TaskPublicProjectionProvider:
+    """Name each committed task by the public input the subject was given."""
+    committed = list(campaign.taskset.membership.ordered_task_hashes)
+    inputs = _proving_inputs()
+
+    def project(*, task_hash: Digest, position: int) -> TaskPublicProjection:
+        # The position and the hash have to agree before either is trusted to
+        # pick an input. They come from the same receipt, so a disagreement
+        # means the receipt and the Campaign are describing different runs, and
+        # labelling a task with another task's input would be worse than
+        # showing nothing.
+        if (
+            position >= len(committed)
+            or position >= len(inputs)
+            or committed[position] != task_hash
+        ):
+            return hash_only_projection(task_hash=task_hash, position=position)
+        named = hash_only_projection(task_hash=task_hash, position=position)
+        return TaskPublicProjection(
+            task_label=named.task_label,
+            public_prompt=inputs[position],
+        )
+
+    return project
+
+
+def _proving_inputs() -> tuple[str, ...]:
+    """Read the reference taskset's frozen public inputs, and nothing else.
+
+    The module is loaded from the packaged bundle by path rather than
+    imported by name: the package's ``__init__`` pulls in Verifiers, which
+    belongs to the managed engine environment and is not resolvable here. Only
+    the pure input list is wanted, and it has no dependency of its own beyond
+    the normalizer in the same package.
+    """
+    package = (
+        embedded_engine_root()
+        / "packages"
+        / _REFERENCE_TASKSET
+        / _REFERENCE_TASKSET.replace("-", "_")
+    )
+    module_name = f"{_REFERENCE_TASKSET.replace('-', '_')}.dataset"
+    cached = sys.modules.get(module_name)
+    if cached is not None:
+        return tuple(cached.PROVING_INPUTS)
+
+    algorithm_name = f"{_REFERENCE_TASKSET.replace('-', '_')}.algorithm"
+    for name, filename in (
+        (algorithm_name, "algorithm.py"),
+        (module_name, _DATASET_MODULE),
+    ):
+        if name in sys.modules:
+            continue
+        location = package / filename
+        specification = importlib.util.spec_from_file_location(name, str(location))
+        if specification is None or specification.loader is None:
+            raise ValidationError(
+                "this build cannot read the reference taskset's public inputs",
+                details={"module": name},
+            )
+        module = importlib.util.module_from_spec(specification)
+        sys.modules[name] = module
+        specification.loader.exec_module(module)
+
+    return tuple(sys.modules[module_name].PROVING_INPUTS)