meds-algorithm 0.4.0a4__py3-none-any.whl

meds_algorithm/configuration.py

@@ -0,0 +1,64 @@
+"""Algorithm-owned configuration types and shared per-invocation execution options."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+from .serialization import Serializable
+
+
+@dataclass(frozen=True)
+class AlgorithmConfig(Serializable):
+    """Parent for algorithm-owned typed configuration, defaults and validation."""
+
+    additional_predicates_file: Path | None = field(default=None, kw_only=True)
+
+    @classmethod
+    def from_file(cls, path: Path | str):
+        """Read a user JSON/YAML values file, or an exact serialized config.json envelope."""
+        path = Path(path).resolve()
+        values = yaml.safe_load(path.read_text(encoding="utf-8"))
+        if not isinstance(values, dict):
+            raise TypeError("Configuration file must contain an object")
+        if set(values) == {"schema_version", "type", "values"}:
+            return cls.load(path)
+        return cls.from_dict(values, base=path.parent)
+
+
+@dataclass(frozen=True, kw_only=True)
+class NNConfig(AlgorithmConfig):
+    """Framework-independent neural-network settings; concrete algorithms supply training defaults.
+
+    batch_size is examples per device per microbatch, before gradient accumulation.
+    Subclasses overriding validate() must call super().validate().
+    """
+
+    capacity: float = 1.0
+    learning_rate: float
+    batch_size: int
+    weight_decay: float = 0.0
+    dropout: float = 0.0
+
+    def validate(self):
+        super().validate()
+        if self.capacity <= 0:
+            raise ValueError("capacity must be positive")
+        if self.learning_rate <= 0:
+            raise ValueError("learning_rate must be positive")
+        if self.batch_size < 1:
+            raise ValueError("batch_size must be positive")
+        if self.weight_decay < 0:
+            raise ValueError("weight_decay must be nonnegative")
+        if not 0 <= self.dropout < 1:
+            raise ValueError("dropout must be in [0, 1)")
+
+
+@dataclass(frozen=True)
+class ExecutionOptions(Serializable):
+    seed: int = 0
+    overwrite: bool = False
+
+    def validate(self):
+        if self.seed < 0:
+            raise ValueError("seed must be nonnegative")

meds_algorithm/conformance/__init__.py

@@ -0,0 +1 @@
+"""Reusable assertions, synthetic fixtures and explicitly opted-in integration provisioning."""

meds_algorithm/conformance/assertions.py

@@ -0,0 +1,62 @@
+from pathlib import Path
+
+from ..data.index import read_index
+from ..evaluation.predictions import check_decoded
+
+
+def read_predictions(predictions_dir: Path, *, require_truth: bool = True):
+    """Read every prediction shard, validating each shard's schema and probability values."""
+    import polars as pl
+
+    paths = sorted(Path(predictions_dir).rglob("*.parquet"))
+    if not paths:
+        raise FileNotFoundError(f"No prediction parquet files under {predictions_dir}")
+    frames = []
+    for path in paths:
+        frame = pl.read_parquet(path)
+        if require_truth:
+            assert_prediction_schema(frame)
+        check_decoded(frame.select("subject_id", "prediction_time"), frame)
+        frames.append(frame)
+    combined = pl.concat(frames, how="diagonal_relaxed")
+    check_decoded(combined.select("subject_id", "prediction_time"), combined)
+    return combined
+
+
+def assert_prediction_keys(predictions_dir: Path, labels_dir: Path, splits) -> None:
+    """Compare actual prediction keys with requested keys, independent of row order."""
+    import polars as pl
+
+    key_columns = ["subject_id", "prediction_time"]
+
+    # MEDS-DEV evaluates every `{predictions_dir}/**/*.parquet` (configs/_evaluate_predictions.yaml), so
+    # every parquet under the predictions artifact must be a predictions file. Publish supporting tables
+    # (e.g. a zero-shot query bundle) in another format, such as Arrow IPC.
+    for path in sorted(Path(predictions_dir).rglob("*.parquet")):
+        schema = pl.read_parquet_schema(path)
+        if "predicted_boolean_probability" not in schema and "predicted_boolean_value" not in schema:
+            raise AssertionError(
+                f"{path} is not a predictions file; MEDS-DEV evaluates every parquet under the predictions "
+                "directory; store supporting tables in another format (e.g. Arrow IPC)"
+            )
+    try:
+        predictions = read_index(Path(predictions_dir))
+        expected = read_index(Path(labels_dir), splits)
+    except ValueError as error:
+        raise AssertionError(str(error)) from error
+    prediction_unique = predictions.unique()
+    expected_unique = expected.unique()
+    assert prediction_unique.height == predictions.height, "predictions contain duplicate evaluation keys"
+    assert expected_unique.height == expected.height, "requested evaluation index contains duplicate keys"
+    missing = expected_unique.join(prediction_unique, on=key_columns, how="anti")
+    extra = prediction_unique.join(expected_unique, on=key_columns, how="anti")
+    assert missing.is_empty() and extra.is_empty(), (
+        f"prediction key mismatch: {missing.height} missing and {extra.height} extra keys"
+    )
+
+
+def assert_prediction_schema(predictions) -> None:
+    """Validate predictions exactly as MEDS-DEV's pinned meds-evaluation (0.0.3) will."""
+    from meds_evaluation.schema import validate_binary_classification_schema
+
+    validate_binary_classification_schema(predictions)

meds_algorithm/conformance/end_to_end.py

@@ -0,0 +1,150 @@
+"""Reusable end-to-end conformance; execution and fitted-feature inspection are algorithm-owned."""
+
+from contextlib import suppress
+from pathlib import Path
+
+from meds_algorithm import (
+    ALGORITHM_PROFILES,
+    STEP_SPECS,
+    ZERO_SHOT_TASK_COMMANDS,
+    CommandName,
+    RelaxationLedger,
+    TaskSpec,
+    read_manifest,
+)
+
+from .assertions import assert_prediction_keys, read_predictions
+
+
+def _strip_labels(labels_dir: Path, destination: Path, splits) -> Path:
+    """Remove outcomes from selected splits while retaining permitted training/context outcomes."""
+    import polars as pl
+
+    from meds_algorithm import split_files
+
+    labels_dir = Path(labels_dir)
+    files = split_files(labels_dir)
+    stripped = set(files) if splits is None else set()
+    for split in splits or ():
+        # A separate training/context root need not contain any evaluated split.
+        with suppress(FileNotFoundError):
+            stripped.update(split_files(labels_dir, [split]))
+    for path in split_files(labels_dir):
+        target = destination / (path.relative_to(labels_dir) if Path(labels_dir).is_dir() else path.name)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if path in stripped:
+            pl.read_parquet(path, columns=["subject_id", "prediction_time"]).write_parquet(target)
+        else:
+            pl.read_parquet(path).write_parquet(target)
+    return destination
+
+
+def assert_profile_run(result, *, profile, run_e2e, fitted_codes, tmp_path):
+    """Check actual artifacts, task binding, key coverage, prediction reads and split canaries."""
+    artifacts = result.artifacts
+    assert list(artifacts) == [command.value for command in ALGORITHM_PROFILES[profile]]
+    for command, directory in artifacts.items():
+        manifest = read_manifest(directory)
+        assert manifest["command"]["name"] == command
+        assert manifest["artifact"]["type"] == STEP_SPECS[CommandName(command)].artifact_type.value
+        assert str(manifest["command"]["config_digest"]).startswith("sha256:"), (
+            f"{command} has no config digest"
+        )
+        resources = manifest["resources"]
+        assert resources["wall_seconds"] >= 0
+        memory = resources["memory"]
+        if memory["source"] == "unavailable":
+            assert memory["peak_process_tree_rss_bytes"] is None
+        else:
+            assert memory["peak_process_tree_rss_bytes"] > 0
+    prediction_step = ALGORITHM_PROFILES[profile][-1].value
+    read_predictions(artifacts[prediction_step])
+    prediction_manifest = read_manifest(artifacts[prediction_step])
+    assert prediction_manifest["artifact"]["type"] == "predictions"
+    assert_prediction_keys(artifacts[prediction_step], result.labels_dir, result.splits)
+
+    assert_label_blind_pipeline(result, profile=profile, run_e2e=run_e2e, tmp_path=tmp_path)
+
+    # Absence of transductive fitting (SPEC 2.1): codes that occur only in tuning or held-out subjects must
+    # not be part of anything the algorithm fitted.
+    assert set(result.canaries) == {"tuning", "held_out"}, (
+        "run_e2e must apply core conformance.signal.add_split_canaries and return the canaries"
+    )
+    leaked = set(result.canaries.values()) & set(fitted_codes(artifacts))
+    assert not leaked, f"statistics were fitted over non-training splits: {sorted(leaked)}"
+
+    # Zero-shot: every task-conditioned artifact names its task, and a ledger is published (SPEC 2.4, 3.1).
+    if not profile.startswith("zero_shot_"):
+        return
+    task = TaskSpec.from_arguments(result.inputs)
+    for command in ZERO_SHOT_TASK_COMMANDS.get(profile, ()):
+        manifest = read_manifest(artifacts[command.value])
+        assert manifest["semantics_digest"] == task.semantics_digest(), (
+            f"{command.value} is not bound to the task"
+        )
+        assert manifest["inputs"]["task"]["criteria_file"]["digest"].startswith("sha256:")
+    ledgers = [path.parent for d in artifacts.values() for path in Path(d).rglob("relaxations.yaml")]
+    assert ledgers, "every zero-shot run must publish relaxations.yaml"
+    for directory in ledgers:
+        RelaxationLedger.read(directory)  # raises unless query_language, decoded_quantity, and exact are set
+
+
+def assert_label_blind_pipeline(result, *, profile, run_e2e, tmp_path):
+    """Rerun every stage on the same histories with forbidden outcomes absent and fresh artifacts."""
+    assert result.inputs.get("external_meds_dir"), "E2EResult.inputs must record external_meds_dir"
+    assert type(result.inputs.get("seed")) is int, "E2EResult.inputs must record the run_e2e seed"
+    expected_steps = [step.value for step in ALGORITHM_PROFILES[profile]]
+    original = read_predictions(result.artifacts[expected_steps[-1]])
+    overrides = dict(result.inputs)
+    for argument in ("external_labels_dir", "external_index_dir", "external_evaluation_labels_dir"):
+        overrides[argument] = _strip_labels(
+            result.inputs.get(argument) or result.labels_dir,
+            tmp_path / "unlabelled-inputs" / argument,
+            None if profile.startswith("zero_shot_") else result.splits,
+        )
+    fresh_root = tmp_path / "label-blind-run"
+    blind = run_e2e(
+        fresh_root,
+        profile=profile,
+        **{
+            **overrides,
+            "splits": result.splits,
+        },
+    )
+    assert list(blind.artifacts) == expected_steps, "label-blind check must execute the full lifecycle"
+    old_paths = {Path(path).resolve() for path in result.artifacts.values()}
+    for step, path in blind.artifacts.items():
+        path = Path(path).resolve()
+        assert path not in old_paths and path.is_relative_to(fresh_root.resolve()), (
+            "label-blind check must publish fresh outputs inside its run directory"
+        )
+        manifest = read_manifest(path)
+        assert manifest["command"]["name"] == step
+        assert manifest["artifact"]["type"] == STEP_SPECS[CommandName(step)].artifact_type.value
+    prediction = blind.artifacts[expected_steps[-1]]
+    assert_prediction_keys(prediction, result.labels_dir, result.splits)
+    # Do not require the evaluation-only boolean_value on a deliberately unlabelled run.
+    import polars as pl
+    from polars.testing import assert_frame_equal
+
+    from ..data.index import read_index
+    from ..evaluation.predictions import check_decoded
+
+    blind_predictions = read_predictions(prediction, require_truth=False)
+    check_decoded(read_index(result.labels_dir, result.splits), blind_predictions)
+    columns = ["subject_id", "prediction_time", "predicted_boolean_probability"]
+    if "predicted_boolean_value" in original.columns:
+        columns.append("predicted_boolean_value")
+    try:
+        assert_frame_equal(
+            original.select(columns).sort(columns[:2]),
+            blind_predictions.select(columns).sort(columns[:2]),
+            check_exact=False,
+            rel_tol=1e-6,
+            abs_tol=1e-8,
+        )
+    except (AssertionError, pl.exceptions.ColumnNotFoundError) as error:
+        raise AssertionError(
+            "Predictions changed when forbidden labels were removed; fix leakage or make the "
+            "conformance fixture reproducible with the recorded seed"
+        ) from error

meds_algorithm/conformance/meds_dev.py

@@ -0,0 +1,127 @@
+"""Shared opt-in integration-test utilities."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+from pathlib import Path
+
+import pytest
+
+from ..runtime.process import run
+from .assertions import assert_prediction_keys as assert_prediction_keys
+from .assertions import assert_prediction_schema as assert_prediction_schema
+
+MEDS_DEV_URL = "https://github.com/Medical-Event-Data-Standard/MEDS-DEV.git"
+# Head of MEDS-DEV PR #330 (task_criteria_path), on top of dev, which contains PR #325 (predicates_path).
+# Repin to the merge commit once #330 merges: a branch commit can become unreachable after a squash merge.
+MEDS_DEV_PINNED_REF = "19a8aa3417fae3586b2e8f0f17aba19ddb0560d4"
+INSTALL_TIMEOUT = 1800
+RUN_TIMEOUT = 5400
+
+
+def venv_bin(checkout: Path) -> Path:
+    return checkout / ".venv" / ("Scripts" if sys.platform == "win32" else "bin")
+
+
+def assert_meds_dev_predicates_capability(checkout: Path) -> None:
+    """Require the explicit predicates_path passthrough proposed in MEDS-DEV PR #325."""
+    config = checkout / "src" / "MEDS_DEV" / "configs" / "_run_model.yaml"
+    implementation = checkout / "src" / "MEDS_DEV" / "models" / "__init__.py"
+    if not config.is_file() or not implementation.is_file():
+        raise AssertionError(f"{checkout} is not a MEDS-DEV source checkout")
+    if (
+        "predicates_path:" not in config.read_text()
+        or 'format_kwargs["predicates_path"]' not in implementation.read_text()
+    ):
+        raise AssertionError(
+            "MEDS-DEV lacks the predicates_path algorithm-command passthrough. Use MEDS_DEV_REF="
+            f"{MEDS_DEV_PINNED_REF} or a revision containing MEDS-DEV PR #325."
+        )
+
+
+def assert_meds_dev_task_criteria_capability(checkout: Path) -> None:
+    """Require the task_criteria_path passthrough proposed in MEDS-DEV PR #330 (zero-shot profiles)."""
+    implementation = checkout / "src" / "MEDS_DEV" / "models" / "__init__.py"
+    if 'format_kwargs["task_criteria_path"]' not in implementation.read_text():
+        raise AssertionError(
+            "MEDS-DEV lacks the task_criteria_path algorithm-command passthrough that zero-shot model.yaml "
+            f"commands use. Use MEDS_DEV_REF={MEDS_DEV_PINNED_REF} or a revision containing MEDS-DEV PR #330."
+        )
+
+
+def provision_meds_dev(request, tmp_path_factory, *, task_criteria: bool = False) -> Path:
+    source = os.environ.get("MEDS_DEV_DIR")
+    expression = request.config.getoption("markexpr", default="") or ""
+    from _pytest.mark.expression import Expression
+
+    selected = False
+    if expression.strip():
+        marker_expression = Expression.compile(expression)
+        selected = not marker_expression.evaluate(lambda name, **kwargs: name == "slow") and any(
+            marker_expression.evaluate(lambda name, marker=marker, **kwargs: name in {marker, "slow"})
+            for marker in ("meds_dev", "real_data")
+        )
+    if not selected:
+        pytest.skip("explicitly select -m meds_dev or -m real_data; MEDS_DEV_DIR alone does not opt in")
+    uv = shutil.which("uv")
+    git = shutil.which("git")
+    if uv is None or git is None:
+        pytest.skip("uv and git are required for MEDS-DEV integration")
+    checkout = tmp_path_factory.mktemp("meds-dev") / "MEDS-DEV"
+    run([git, "clone", source or MEDS_DEV_URL, checkout], timeout=INSTALL_TIMEOUT)
+    if source is None:
+        ref = os.environ.get("MEDS_DEV_REF", MEDS_DEV_PINNED_REF)
+        run([git, "-C", checkout, "checkout", "--detach", ref], timeout=INSTALL_TIMEOUT)
+    assert_meds_dev_predicates_capability(checkout)
+    if task_criteria:
+        assert_meds_dev_task_criteria_capability(checkout)
+    run([uv, "venv", checkout / ".venv"], timeout=INSTALL_TIMEOUT)
+    run(
+        [uv, "pip", "install", "-e", checkout, "--python", venv_bin(checkout) / "python"],
+        timeout=INSTALL_TIMEOUT,
+        env={"SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MEDS_DEV": "0.0.0"},
+    )
+    return checkout
+
+
+def prepare_mimic_demo(meds_dev, tmp_path):
+    """Return MEDS-DEV's ``(MIMIC demo, task labels, MIMIC predicates)`` paths.
+
+    Use ``MEDS_DEMO_DIR`` when set; otherwise invoke MEDS-DEV's own demo builder. Pin an existing
+    demo-compatible MEDS-DEV task rather than defining a task in this repository.
+    """
+    configured = os.environ.get("MEDS_DEMO_DIR")
+    dataset_dir = Path(configured).resolve() if configured else tmp_path / "mimic-iv-demo"
+    if not configured:
+        run(
+            [
+                venv_bin(meds_dev) / "meds-dev-dataset",
+                "dataset=MIMIC-IV",
+                f"output_dir={dataset_dir}",
+                "demo=true",
+            ],
+            timeout=RUN_TIMEOUT,
+        )
+    labels_dir = tmp_path / "mortality-in-icu-first-24h"
+    run(
+        [
+            venv_bin(meds_dev) / "meds-dev-task",
+            "task=mortality/in_icu/first_24h",
+            "dataset=MIMIC-IV",
+            f"dataset_dir={dataset_dir}",
+            f"output_dir={labels_dir}",
+        ],
+        # MEDS-DEV's task runner launches venv-installed helpers by bare name.
+        # Prepend the pinned checkout so an ambient MEDS-DEV install cannot win.
+        env={"PATH": f"{venv_bin(meds_dev)}{os.pathsep}{os.environ['PATH']}"},
+        timeout=RUN_TIMEOUT,
+    )
+    predicates_file = meds_dev / "src" / "MEDS_DEV" / "datasets" / "MIMIC-IV" / "predicates.yaml"
+    return dataset_dir, labels_dir, predicates_file
+
+
+def mimic_task_criteria(meds_dev) -> Path:
+    """MEDS-DEV's own ACES criteria for the demo task: what a zero-shot algorithm is asked to predict."""
+    return meds_dev / "src" / "MEDS_DEV" / "tasks" / "mortality" / "in_icu" / "first_24h.yaml"

meds_algorithm/conformance/reports.py

@@ -0,0 +1,49 @@
+"""Completion evidence shared by generated repositories."""
+
+import re
+
+
+def _sections(text, level):
+    parts = re.split(rf"^{'#' * level} (.+)\s*$", text, flags=re.MULTILINE)
+    return dict(zip(parts[1::2], parts[2::2], strict=True))
+
+
+def assert_implementation_report(report: str, profiles, stub_profiles) -> None:
+    """Validate shared evidence and each completed profile, allowing unfinished profile sections."""
+    assert set(stub_profiles) <= set(profiles), "Unknown profile in STUB_PROFILES"
+    complete = set(profiles) - set(stub_profiles)
+    if not complete:
+        return
+    sections = _sections(report, 2)
+    required = {
+        "Source",
+        "Source-element ledger",
+        "Inputs and predicates",
+        "Intermediate artifacts",
+        "Scientific deviations",
+        "Verification",
+        "Review gates",
+        "Resource measurements",
+        "Environment and provenance",
+    }
+    assert required <= sections.keys(), f"Missing report sections: {sorted(required - sections.keys())}"
+    choices = _sections(sections.get("Profile choices", ""), 3)
+    assert complete <= choices.keys(), f"Missing profile reports: {sorted(complete - choices.keys())}"
+    inactive_sections = {
+        section
+        for section, profile in {
+            "Zero-shot task translation": "zero_shot_direct",
+            "Zero-shot materialization": "zero_shot_materialized",
+        }.items()
+        if profile not in complete
+    }
+    checked = {
+        name: body
+        for name, body in sections.items()
+        if name != "Profile choices" and name not in inactive_sections
+    }
+    checked.update({f"Profile {profile}": choices[profile] for profile in complete})
+    for name, body in checked.items():
+        assert body.strip() and "TODO" not in body and "<paper/repository not yet supplied>" not in body, (
+            f"Incomplete implementation report: {name}"
+        )

meds_algorithm/conformance/signal.py

@@ -0,0 +1,141 @@
+"""Small example MEDS cohort for a algorithm-owned primary-behavior test."""
+
+from __future__ import annotations
+
+import json
+import random
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import polars as pl
+
+SIGNAL_CODE = "SIGNAL//POS"
+BACKGROUND_CODES = [f"BG//{index}" for index in range(6)]
+
+
+def build_signal_dataset(
+    root: Path,
+    *,
+    n_train: int = 200,
+    n_tuning: int = 40,
+    n_held_out: int = 40,
+    seed: int = 0,
+    shuffle_labels: bool = False,
+    signal_code: str = SIGNAL_CODE,
+    background_codes: list[str] | None = None,
+) -> Path:
+    """Write a tiny MEDS dataset where ``signal_code`` alone determines the binary label.
+
+    Positive and negative subjects have equal-length histories. The signal position and history length
+    are randomized independently of the label, so a algorithm must represent the code rather than exploit a
+    length or position leak. Shuffling labels supplies the paired negative control.
+
+    This is an example fixture, not an input representation imposed on the algorithm. Adapt or replace it
+    when the implementation needs codes from a fixed vocabulary, predicates, numeric values, or another
+    algorithm-specific binding.
+    """
+    root = Path(root)
+    rng = random.Random(seed)
+    background = list(background_codes or BACKGROUND_CODES)
+    if not background:
+        raise ValueError("background_codes must contain at least one code")
+
+    split_sizes = {"train": n_train, "tuning": n_tuning, "held_out": n_held_out}
+    subject_splits: list[dict] = []
+    subject_id = 1
+    base_time = datetime(2020, 1, 1)
+
+    for split, size in split_sizes.items():
+        events: list[dict] = []
+        labels: list[dict] = []
+        for _ in range(size):
+            positive = rng.random() < 0.5
+            start = base_time + timedelta(days=rng.randrange(100))
+            sequence = [rng.choice(background) for _ in range(rng.randint(4, 9))]
+            marker_position = rng.randrange(len(sequence) + 1)
+            marker = signal_code if positive else rng.choice(background)
+            sequence.insert(marker_position, marker)
+
+            for offset, code in enumerate(sequence):
+                events.append(
+                    {
+                        "subject_id": subject_id,
+                        "time": start + timedelta(hours=offset),
+                        "code": code,
+                        "numeric_value": None,
+                    }
+                )
+            labels.append(
+                {
+                    "subject_id": subject_id,
+                    "prediction_time": start + timedelta(hours=len(sequence) + 1),
+                    "boolean_value": positive,
+                }
+            )
+            subject_splits.append({"subject_id": subject_id, "split": split})
+            subject_id += 1
+
+        data_dir = root / "data" / split
+        labels_dir = root / "task_labels" / "signal_task"
+        data_dir.mkdir(parents=True, exist_ok=True)
+        labels_dir.mkdir(parents=True, exist_ok=True)
+        pl.DataFrame(events).with_columns(
+            pl.col("subject_id").cast(pl.Int64),
+            pl.col("time").cast(pl.Datetime("us")),
+            pl.col("code").cast(pl.String),
+            pl.col("numeric_value").cast(pl.Float32),
+        ).write_parquet(data_dir / "0.parquet")
+        label_frame = pl.DataFrame(labels).with_columns(
+            pl.col("subject_id").cast(pl.Int64),
+            pl.col("prediction_time").cast(pl.Datetime("us")),
+            pl.col("boolean_value").cast(pl.Boolean),
+        )
+        if shuffle_labels:
+            label_frame = label_frame.with_columns(
+                pl.col("boolean_value").shuffle(seed=seed + 1).alias("boolean_value")
+            )
+        label_frame.write_parquet(labels_dir / f"{split}.parquet")
+
+    metadata = root / "metadata"
+    metadata.mkdir(parents=True, exist_ok=True)
+    pl.DataFrame({"code": [*background, signal_code]}).write_parquet(metadata / "codes.parquet")
+    pl.DataFrame(subject_splits).with_columns(pl.col("subject_id").cast(pl.Int64)).write_parquet(
+        metadata / "subject_splits.parquet"
+    )
+    (metadata / "dataset.json").write_text(
+        json.dumps({"dataset_name": "synthetic_signal", "meds_version": "0.4"})
+    )
+    return root
+
+
+CANARY_CODES = {"tuning": "CANARY//tuning", "held_out": "CANARY//held_out"}
+
+
+def add_split_canaries(root: Path) -> dict[str, str]:
+    """Give every tuning and held-out subject one event with a code that occurs in no other split.
+
+    Statistics fitted on the training split alone (vocabularies, code frequencies, bins) can never contain
+    these codes; a algorithm that fits over every split will. The codes are not added to
+    ``metadata/codes.parquet``, so a vocabulary read from metadata does not pick them up either. Each canary
+    is placed at the subject's first event time, so it falls in history before any prediction time.
+    Returns the canary code of each split.
+    """
+    root = Path(root)
+    for split, code in CANARY_CODES.items():
+        for shard in sorted((root / "data" / split).glob("*.parquet")):
+            events = pl.read_parquet(shard)
+            first = (
+                events.filter(pl.col("time").is_not_null())
+                .group_by("subject_id")
+                .agg(pl.col("time").min())
+                .with_columns(pl.lit(code).alias("code"))
+            )
+            # Every other column (numeric_value, text_value, ...) is null on the canary events.
+            canaries = first.with_columns(
+                pl.lit(None, dtype=dtype).alias(column)
+                for column, dtype in events.schema.items()
+                if column not in first.columns
+            )
+            canaries = canaries.select(events.columns).cast(events.schema)
+            pl.concat([events, canaries]).sort("subject_id", "time", nulls_last=False).write_parquet(shard)
+    return dict(CANARY_CODES)

meds_algorithm/conformance/types.py

@@ -0,0 +1,29 @@
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class E2EResult:
+    """Artifacts plus the requested evaluation index used to verify prediction keys."""
+
+    artifacts: Mapping[str, Path]
+    labels_dir: Path
+    splits: Sequence[str]
+    # External inputs and the integer seed honored by run_e2e.
+    # Zero-shot profiles MUST include external_task_criteria_file
+    # (and external_predicates_file when used) so tests can check which task each artifact answers.
+    inputs: Mapping[str, Any] = field(default_factory=dict)
+    # Codes added by conformance.signal.add_split_canaries to tuning and held-out subjects.
+    canaries: Mapping[str, str] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class MEDSDevRun:
+    """MEDS-DEV invocation plus the evaluation index its predictions must cover."""
+
+    args: Sequence[str]
+    predictions_dir: Path
+    labels_dir: Path
+    splits: Sequence[str]

meds_algorithm/contracts/__init__.py

@@ -0,0 +1 @@
+"""Stable command, profile and artifact-type definitions."""