modelspec-dev 0.1.0__py3-none-any.whl

registry/templates.yaml

@@ -0,0 +1,166 @@
+# Decision templates (MODEL-178). These are partial decision specs: `where`
+# contains Musts, while `weights` contains Prefers. The loader validates every
+# condition, objective, class, and domain against the decision registries.
+
+schema_version: 1
+
+templates:
+  - id: budget-coding
+    name: Coding agent on a budget
+    purpose: Find an active coding model with enough context while keeping one task under $0.25.
+    where:
+      - condition: model.class = text-generator
+        reason: Coding work needs a model that generates text and code.
+      - condition: model.lifecycle = active
+        reason: An active model is still supported by its lab or provider.
+      - condition: model.context_window >= 200000
+        reason: A large repository needs room for code, tests, and instructions.
+      - condition: offering.cost_per_task <= 0.25
+        reason: The offering must stay within the per-task budget.
+    weights:
+      software_engineering:
+        weight: 0.6
+        reason: Prefer stronger evidence on real software-engineering tasks.
+      -offering.cost_per_task:
+        weight: 0.4
+        reason: Among qualifying offerings, prefer the cheaper task.
+    needs:
+      classes: [text-generator]
+      domains: [software_engineering]
+    teaches: Use Must for non-negotiable context and budget limits, then Prefer to trade coding strength against cost.
+
+  - id: private-self-host
+    name: Private assistant you host yourself
+    purpose: Find a commercially usable open-weights assistant that you can operate yourself.
+    where:
+      - condition: model.class = text-generator
+        reason: A conversational assistant needs a text-generating model.
+      - condition: model.weights_openness = open_weights
+        reason: Self-hosting requires downloadable weights; fits-your-hardware is coming in MODEL-174.
+      - condition: licence.commercial_use in {permitted, permitted_with_conditions}
+        reason: The licence must allow commercial use, including use with stated conditions.
+    weights:
+      chat_preference:
+        weight: 0.7
+        reason: Prefer assistants whose open-ended responses people rate more highly.
+      -offering.cost_per_task:
+        weight: 0.3
+        reason: Prefer lower task cost when the snapshot has an offering price.
+    needs:
+      classes: [text-generator]
+      domains: [chat_preference]
+    teaches: Use Must for deployment and licence requirements, then Prefer to balance assistant quality and cost.
+
+  - id: regulated-data
+    name: Regulated data
+    purpose: Find an offering with the data-handling commitments needed for regulated workloads.
+    where:
+      - condition: offering.data.trains_on_customer_data = false
+        reason: Customer prompts and outputs must not be available for model training.
+      - condition: offering.data.zero_retention = true
+        reason: The provider must offer a mode that stores neither prompts nor outputs.
+      - condition: offering.attestation.baa = true
+        reason: The provider must offer a HIPAA business associate agreement for the service.
+    weights:
+      chat_preference:
+        weight: 0.6
+        reason: Prefer the stronger general assistant among compliant offerings.
+      -offering.cost_per_task:
+        weight: 0.4
+        reason: Prefer the lower task cost after the compliance gates pass.
+    needs:
+      classes: []
+      domains: [chat_preference]
+    teaches: Governance promises are Musts because a high score cannot compensate for a missing data commitment.
+
+  - id: maths
+    name: Maths and proofs
+    purpose: Find a text model with the strongest evidence on mathematical problems and proofs.
+    where:
+      - condition: model.class = text-generator
+        reason: The answer must be generated as mathematical text or a proof.
+    weights:
+      maths:
+        weight: 1.0
+        reason: Rank only by the maths capability estimate.
+    needs:
+      classes: [text-generator]
+      domains: [maths]
+    teaches: Use one Must to select the right model class and one Prefer to state what best means.
+
+  - id: retrieval-embeddings
+    name: Retrieval embeddings
+    purpose: Find an embedding model that ranks relevant documents well for retrieval.
+    where:
+      - condition: model.class = vectoriser
+        reason: Retrieval embeddings require the model class that emits vectors.
+    weights:
+      retrieval:
+        weight: 1.0
+        reason: Rank only by the retrieval capability estimate.
+    needs:
+      classes: [vectoriser]
+      domains: [retrieval]
+    teaches: Model class is a Must, while measured retrieval quality is a Prefer.
+
+  - id: high-volume
+    name: High volume, good enough
+    purpose: Find a low-cost text model for many short assistant requests without ignoring response quality.
+    where:
+      - condition: model.class = text-generator
+        reason: The workload needs generated text responses.
+    weights:
+      -offering.cost_per_task:
+        weight: 0.7
+        reason: At high volume, small differences in task cost dominate total spend.
+      chat_preference:
+        weight: 0.3
+        reason: Keep a quality signal so cheap but poor responses do not win by cost alone.
+    task_tokens: {input: 2000, output: 500}
+    needs:
+      classes: [text-generator]
+      domains: [chat_preference]
+    teaches: Keep the model class as a Must, then make cost the stronger Prefer without turning quality into a gate.
+
+  - id: long-documents
+    name: Long documents
+    purpose: Find a text model that can accept million-token documents and produce strong written analysis.
+    where:
+      - condition: model.class = text-generator
+        reason: The result must be generated as text.
+      - condition: model.context_window >= 1000000
+        reason: The model must accept a million-token document in one request.
+    weights:
+      writing:
+        weight: 0.5
+        reason: Prefer clear, high-quality long-form writing.
+      reasoning:
+        weight: 0.3
+        reason: Prefer stronger multi-step analysis of the document.
+      -offering.cost_per_task:
+        weight: 0.2
+        reason: Use cost as a smaller tie-breaker after writing and reasoning quality.
+    needs:
+      classes: [text-generator]
+      domains: [writing, reasoning]
+    teaches: Context capacity is a Must; writing, reasoning, and cost remain trade-offs expressed as Prefers.
+
+  - id: eu-data
+    name: EU-only data handling
+    purpose: Find an offering that runs inference in an EU member state and does not train on customer data.
+    where:
+      - condition: offering.region in {AT, BE, BG, HR, CY, CZ, DE, DK, EE, ES, FI, FR, GR, HU, IE, IT, LT, LU, LV, MT, NL, PL, PT, RO, SE, SI, SK}
+        reason: Inference must run in one of the EU's 27 member-state ISO country codes (EU Publications Office Interinstitutional Style Guide, https://style-guide.europa.eu/en/content/-/isg/topic?identifier=annex-a5-list-countries-territories-currencies, checked 2026-09-27).
+      - condition: offering.data.trains_on_customer_data = false
+        reason: Customer prompts and outputs must not be available for model training.
+    weights:
+      chat_preference:
+        weight: 0.6
+        reason: Prefer the stronger general assistant among offerings that meet the data rules.
+      -offering.cost_per_task:
+        weight: 0.4
+        reason: Prefer lower task cost after the data-handling gates pass.
+    needs:
+      classes: []
+      domains: [chat_preference]
+    teaches: Residency and training policy are Musts; quality and cost only rank offerings that pass them.

schema/applicability.py

@@ -0,0 +1,147 @@
+"""Which questions a model's *class* can answer at all (MODEL-97).
+
+A `null` in this schema has always meant "not yet researched" — somebody should
+go and look. For a catalogue of one class that reading is nearly always right.
+For a catalogue of several it is sometimes a lie: a model that emits no text has
+no maximum output-token count to research, and publishing that `null` with the
+same meaning as an unresearched one asserts a gap that can never be closed.
+
+**Applicability is derived from `model_type`, never written on a card.** There
+is no per-card list to fall out of date, nothing to round-trip through
+`ModelCard.to_yaml()`, and no way for one card to disagree with its own class.
+The table here is reviewed once instead of 1,339 times, and
+`tests/test_class_and_null_semantics.py` fails if a path in it stops naming a
+real schema field.
+
+Two rules the table obeys, and the second is the important one:
+
+1. A rule names the classes for which a field **is** applicable. A card whose
+   class is not among them has nothing to research there.
+2. **A rule may depend only on the class.** Not on another card field. The
+   tempting rule — "hardware profiles are inapplicable when `open_weights` is
+   false" — is refused, because `licensing.open_weights` is `bool = False` with
+   no null state: an unresearched card is indistinguishable from a genuinely
+   closed one, so the rule would manufacture "there is nothing to know" across
+   the catalogue. That is MODEL-77's `data_residency: []` mistake repainted.
+
+What this file deliberately does **not** call inapplicable: architecture.
+A closed model still *has* layers and parameters; nobody has published them.
+That is unknown, possibly unobtainable, and much nearer `withheld` than
+"meaningless". Calling it inapplicable would assert there is nothing to know
+and quietly excuse the catalogue from ever asking.
+
+The section-level counterpart of this table is `__applicable_model_types__` on
+the nested models in `schema/card.py`, which predates it. `schema.card
+.inapplicable_paths` evaluates both and is the single entry point consumers use.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .enums import ModelType
+
+
+def model_types(*keys: str) -> frozenset[ModelType]:
+    """Resolve ModelType members by exact value or hyphen-prefix (``'llm-'``)."""
+    found: set[ModelType] = set()
+    for key in keys:
+        if key.endswith("-"):
+            matched = [t for t in ModelType if t.value.startswith(key)]
+            if not matched:
+                raise ValueError(f"no ModelType starts with {key!r}")
+            found.update(matched)
+        else:
+            found.add(ModelType(key))
+    return frozenset(found)
+
+
+#: Classes that emit text tokens, so an output-token limit, a token stream and
+#: an output tok/s are questions they can answer.
+#:
+#: Same membership as `pipeline.hardware.TOKEN_GENERATING_MODEL_TYPES`, which
+#: answers the neighbouring question "does it decode tokens, so is a tok/s
+#: prediction honest" (MODEL-53). Two names because they are two questions;
+#: a test holds them equal so neither drifts, and that test is where a future
+#: divergence gets argued rather than discovered.
+TEXT_WRITING_MODEL_TYPES: frozenset[ModelType] = model_types(
+    "llm-", "vlm", "medical", "legal", "financial", "agent-model", "router",
+    "adapter", "quantized-variant", "distilled", "merged",
+)
+
+#: Classes that take text *in*, so an input-token limit and a context window
+#: are questions they can answer. A decision model reads supplied state, so it
+#: is here; it writes no text, so it is not above.
+TEXT_READING_MODEL_TYPES: frozenset[ModelType] = TEXT_WRITING_MODEL_TYPES | model_types(
+    "text-encoder", "document-ocr", "reranker", "embedding-", "decision-model",
+    "reward-model", "safety-classifier",
+)
+
+
+@dataclass(frozen=True)
+class FieldRule:
+    """One field, the classes it applies to, and why it applies to no others."""
+
+    #: Dotted path from the `ModelCard` root. Held to a real field by a test.
+    path: str
+    #: What a reader is told the page cannot show.
+    label: str
+    #: The classes for which this field is a real question.
+    applies_to: frozenset[ModelType]
+    #: The sentence a page or a reviewer needs. Not published per field; it is
+    #: the reason the rule survives review.
+    because: str
+
+
+FIELD_RULES: tuple[FieldRule, ...] = (
+    FieldRule("modalities.text.max_output_tokens", "Max output tokens",
+              TEXT_WRITING_MODEL_TYPES, "this class emits no text"),
+    FieldRule("modalities.text.streaming", "Token streaming",
+              TEXT_WRITING_MODEL_TYPES, "this class emits no token stream"),
+    FieldRule("modalities.text.fill_in_middle", "Fill-in-the-middle",
+              TEXT_WRITING_MODEL_TYPES, "this class emits no text"),
+    # Added writing the first `decision-model` card (MODEL-101). "JSON mode" is
+    # a constraint on *generated text*: a toggle that makes a model's prose come
+    # back as valid JSON. A class that emits no prose has no such mode — and for
+    # a decision model the wrong reading is the dangerous one, since its answers
+    # are always typed, so a blank here invites "it cannot do structured output"
+    # exactly backwards. Not a claim that it lacks structure: a claim that the
+    # question is about text this class never writes.
+    FieldRule("modalities.text.json_mode", "JSON output mode",
+              TEXT_WRITING_MODEL_TYPES, "this class emits no text to constrain"),
+    FieldRule("inference_performance.api_tps_output", "API output tok/s",
+              TEXT_WRITING_MODEL_TYPES, "this class has no output token stream"),
+    FieldRule("modalities.text.max_input_tokens", "Max input tokens",
+              TEXT_READING_MODEL_TYPES, "this class takes no text input"),
+    FieldRule("modalities.text.context_window", "Context window",
+              TEXT_READING_MODEL_TYPES, "this class takes no text input"),
+)
+
+#: Human labels for the *sections* `__applicable_model_types__` gates. Used
+#: when a page or a report has to name a subtree rather than a field.
+SECTION_LABELS: dict[str, str] = {
+    "capabilities": "Generative capabilities",
+    "modalities.text": "Text modality",
+    "modalities.vision": "Vision",
+    "modalities.audio": "Audio",
+    "modalities.video": "Video",
+    "modalities.document": "Documents",
+    "modalities.image_generation": "Image generation",
+    "modalities.embeddings": "Embeddings",
+    "modalities.reranking": "Reranking",
+}
+
+_FIELD_LABELS: dict[str, str] = {rule.path: rule.label for rule in FIELD_RULES}
+
+
+def label_for(path: str) -> str:
+    """A reader's name for a dotted path; the path itself when none is set."""
+    return _FIELD_LABELS.get(path) or SECTION_LABELS.get(path) or path
+
+
+def reason_for(path: str) -> str | None:
+    """Why this path does not apply, for the rules that state one."""
+    for rule in FIELD_RULES:
+        if rule.path == path:
+            return rule.because
+    return None

schema/benchmark.py

@@ -0,0 +1,175 @@
+"""BenchmarkCard: the schema for one benchmark wiki page (benchmarks/<id>.md front matter).
+
+The template IS the schema, as with model cards: every field here is a front-matter key.
+Unknown = empty/None, never a guess. `models_covered` is derived from the model cards at build time
+and must not be authored.
+"""
+from __future__ import annotations
+
+import re
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from schema.enums import BenchmarkCategory
+
+ID_RE = re.compile(r"^[a-z0-9][a-z0-9_]{1,80}$")
+
+
+class Metric(BaseModel):
+    name: str = ""                       # e.g. "accuracy", "pass@1", "% resolved", "Elo"
+    direction: Literal["higher_is_better", "lower_is_better"] = "higher_is_better"
+    unit: str = ""                       # "%", "points", "Elo"
+    max_score: float | None = None
+    random_baseline: float | None = None
+    human_baseline: float | None = None
+    baseline_note: str = ""
+
+
+class Dataset(BaseModel):
+    size: int | None = None              # number of items/tasks/questions
+    size_note: str = ""                  # "500 tasks", "57 subjects x ~14k questions"
+    url: str = ""
+    license: str = ""                    # SPDX id or the licence name as published
+    languages: list[str] = Field(default_factory=list)
+    modalities: list[str] = Field(default_factory=list)   # text, code, image, video, audio
+    splits: str = ""
+    public_test_set: bool | None = None  # False when answers are held out
+
+
+class Publisher(BaseModel):
+    org: str = ""
+    authors: list[str] = Field(default_factory=list)
+    url: str = ""
+
+
+class Paper(BaseModel):
+    title: str = ""
+    arxiv: str = ""                      # e.g. "2310.06770"
+    url: str = ""
+    year: int | None = None
+
+
+class Lineage(BaseModel):
+    family: str = ""                     # id of the family page, e.g. "mmlu", "swe_bench"
+    predecessor: str = ""                # id
+    successors: list[str] = Field(default_factory=list)   # ids
+    variants: list[str] = Field(default_factory=list)     # ids of subsets / language / category variants
+
+
+class Saturation(BaseModel):
+    status: Literal["open", "watch", "saturated", "unknown"] = "unknown"
+    top_score: float | None = None
+    as_of: str = ""                      # YYYY-MM
+    note: str = ""
+
+
+class Contamination(BaseModel):
+    risk: Literal["low", "medium", "high", "unknown"] = "unknown"
+    note: str = ""
+
+
+class Harness(BaseModel):
+    lm_eval: str = ""                    # lm-evaluation-harness task name
+    inspect_evals: str = ""              # inspect_evals id
+    helm: str = ""                       # HELM scenario
+    opencompass: str = ""                # OpenCompass dataset
+    bigbench: str = ""                   # BIG-bench task
+    other: str = ""
+
+
+class Source(BaseModel):
+    url: str
+    title: str = ""
+    accessed: str = ""                   # YYYY-MM-DD
+
+    @field_validator("url")
+    @classmethod
+    def _http(cls, v: str) -> str:
+        if not v.startswith(("http://", "https://")):
+            raise ValueError("source url must be http(s)")
+        return v
+
+
+class DomainTag(BaseModel):
+    """One domain a benchmark measures, and how directly (MODEL-133).
+
+    `id` names a domain in `registry/domains.yaml`; `tests/test_decision_registry.py`
+    checks every tag against it. `direct`: the benchmark's tasks are instances of
+    the domain. `proxy`: correlated with it, a slice of it, or preference rather
+    than correctness. An absent tag means "not tagged", never "no domain".
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    id: str
+    directness: Literal["direct", "proxy"]
+
+
+class Freshness(BaseModel):
+    researched: str = ""                 # YYYY-MM-DD
+    researched_by: str = ""              # "sonnet-5 agent, batch 1"
+    reviewed: str = ""
+    reviewed_by: str = ""
+
+
+class BenchmarkCard(BaseModel):
+    id: str
+    name: str
+    aliases: list[str] = Field(default_factory=list)
+    page_kind: Literal["benchmark", "family", "subset"] = "benchmark"
+    category: BenchmarkCategory
+    subcategory: str = ""
+    status: Literal["active", "saturated", "deprecated", "superseded", "proposed", "unknown"] = "unknown"
+    summary: str = ""                    # one sentence, <= 200 chars
+    measures: str = ""                   # one paragraph
+    task_format: str = ""
+    metric: Metric = Metric()
+    dataset: Dataset = Dataset()
+    publisher: Publisher = Publisher()
+    paper: Paper = Paper()
+    leaderboard_url: str = ""
+    repo_url: str = ""
+    released: str = ""                   # YYYY or YYYY-MM
+    last_updated: str = ""
+    lineage: Lineage = Lineage()
+    saturation: Saturation = Saturation()
+    contamination: Contamination = Contamination()
+    harness: Harness = Harness()
+    tags: list[str] = Field(default_factory=list)
+    #: Optional and additive: pages without it still load (MODEL-133).
+    domains: list[DomainTag] = Field(default_factory=list)
+    sources: list[Source] = Field(default_factory=list)
+    freshness: Freshness = Freshness()
+
+    @field_validator("id")
+    @classmethod
+    def _id(cls, v: str) -> str:
+        if not ID_RE.match(v):
+            raise ValueError("id must be snake_case: lowercase letters, digits, underscores")
+        return v
+
+    @field_validator("domains")
+    @classmethod
+    def _one_tag_per_domain(cls, v: list[DomainTag]) -> list[DomainTag]:
+        ids = [tag.id for tag in v]
+        repeated = sorted({i for i in ids if ids.count(i) > 1})
+        if repeated:
+            raise ValueError(f"a domain may be tagged once per page: {', '.join(repeated)}")
+        return v
+
+    @field_validator("summary")
+    @classmethod
+    def _summary(cls, v: str) -> str:
+        if len(v) > 220:
+            raise ValueError("summary must be one sentence, at most 220 characters")
+        return v
+
+
+REQUIRED_SECTIONS = {
+    "benchmark": ["What it measures", "How it is scored", "Dataset and licence", "Who publishes it", "Lineage",
+                  "Saturation and contamination", "How to run it", "Reading the numbers"],
+    "family": ["What it measures", "How it is scored", "Dataset and licence", "Who publishes it", "Lineage",
+               "Saturation and contamination", "How to run it", "Reading the numbers"],
+    "subset": ["What it measures", "Reading the numbers"],
+}