modelspec-dev 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api/__init__.py +0 -0
- api/class_fit.py +334 -0
- api/classes.py +557 -0
- api/ranking/__init__.py +12 -0
- api/ranking/engine.py +1943 -0
- cli/__init__.py +0 -0
- cli/modelspec/__init__.py +0 -0
- cli/modelspec/cli.py +1819 -0
- cli/modelspec/commands/__init__.py +0 -0
- cli/modelspec/decide_cmd.py +333 -0
- cli/modelspec/offline.py +623 -0
- cli/modelspec/snapshot.py +698 -0
- cli/modelspec/snapshot_build_cmd.py +49 -0
- cli/modelspec/verify_cmd.py +125 -0
- cli/modelspec/vocab_cmd.py +204 -0
- cli/modelspec/vocabulary_cache.py +54 -0
- decision/__init__.py +13 -0
- decision/capability.py +872 -0
- decision/computed.py +125 -0
- decision/contract.py +1575 -0
- decision/engine.py +238 -0
- decision/excluded.py +34 -0
- decision/explain.py +908 -0
- decision/filter.py +796 -0
- decision/model.py +438 -0
- decision/normalise.py +604 -0
- decision/optimise.py +320 -0
- decision/registry.py +717 -0
- decision/relax.py +132 -0
- decision/resolve.py +111 -0
- decision/schema.py +21 -0
- decision/snapshot.py +1483 -0
- decision/sources.py +544 -0
- decision/templates.py +134 -0
- decision/verify.py +1745 -0
- decision/vocabulary.py +433 -0
- modelspec_dev-0.1.0.dist-info/METADATA +101 -0
- modelspec_dev-0.1.0.dist-info/RECORD +63 -0
- modelspec_dev-0.1.0.dist-info/WHEEL +4 -0
- modelspec_dev-0.1.0.dist-info/entry_points.txt +2 -0
- modelspec_dev-0.1.0.dist-info/licenses/LICENSE +43 -0
- modelspec_dev-0.1.0.dist-info/licenses/LICENSE-DATA +428 -0
- pipeline/__init__.py +0 -0
- pipeline/class_export.py +172 -0
- pipeline/hardware.py +434 -0
- pipeline/hosts.py +247 -0
- pipeline/load.py +224 -0
- pipeline/ranking.py +551 -0
- registry/domains.yaml +130 -0
- registry/facets.yaml +888 -0
- registry/harnesses.yaml +79 -0
- registry/providers.yaml +354 -0
- registry/sources.yaml +3059 -0
- registry/templates.yaml +166 -0
- schema/__init__.py +0 -0
- schema/applicability.py +147 -0
- schema/benchmark.py +175 -0
- schema/benchmark_eligibility.py +304 -0
- schema/card.py +1463 -0
- schema/enrichment.py +162 -0
- schema/enums.py +327 -0
- schema/graph.py +406 -0
- schema/suppliers.py +72 -0
decision/computed.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Facets the engine computes for one decision (MODEL-153).
|
|
2
|
+
|
|
3
|
+
A computed facet (``computed_by`` in ``registry/facets.yaml``) is never stored
|
|
4
|
+
in a snapshot: its value depends on the spec. ``offering.cost_per_task`` is the
|
|
5
|
+
offering's list price for one task at the spec's ``task_tokens``:
|
|
6
|
+
|
|
7
|
+
(offering.price.input × input + offering.price.output × output) / 1,000,000
|
|
8
|
+
|
|
9
|
+
It is unknown when either price is unknown. ``with_computed`` wraps a loaded
|
|
10
|
+
snapshot so the filter, the optimiser and the explanations read the computed
|
|
11
|
+
value exactly as they read a stored one, and can ask ``computed()`` for the
|
|
12
|
+
records and the formula behind it.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from decision.contract import TaskTokens
|
|
21
|
+
from decision.snapshot import UNKNOWN, Bitset3, FactValue, _FacetBitsets, _holds
|
|
22
|
+
|
|
23
|
+
COST_PER_TASK = "offering.cost_per_task"
|
|
24
|
+
COMPUTED_FACETS = (COST_PER_TASK,)
|
|
25
|
+
_PRICE_INPUT = "offering.price.input"
|
|
26
|
+
_PRICE_OUTPUT = "offering.price.output"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Computed:
|
|
31
|
+
"""A computed value, the snapshot records it came from, and how."""
|
|
32
|
+
|
|
33
|
+
value: float
|
|
34
|
+
records: tuple[str, ...]
|
|
35
|
+
sources: tuple[str, ...]
|
|
36
|
+
formula: str
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _price(fact: FactValue) -> float | None:
|
|
40
|
+
value = fact.value
|
|
41
|
+
if fact.state != "known" or isinstance(value, bool) or not isinstance(value, int | float):
|
|
42
|
+
return None
|
|
43
|
+
return float(value)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _number(value: float) -> str:
|
|
47
|
+
return f"{round(value, 10):,.10g}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class ComputedFacets:
|
|
51
|
+
"""A snapshot index that also answers the computed facets for one spec."""
|
|
52
|
+
|
|
53
|
+
def __init__(self, base: Any, task_tokens: TaskTokens) -> None:
|
|
54
|
+
self._base = base
|
|
55
|
+
self.task_tokens = task_tokens
|
|
56
|
+
self._values: dict[str, Computed | None] = {}
|
|
57
|
+
self._column: _FacetBitsets | None = None
|
|
58
|
+
|
|
59
|
+
def __getattr__(self, name: str) -> Any:
|
|
60
|
+
return getattr(self._base, name)
|
|
61
|
+
|
|
62
|
+
def computed(self, cid: str, facet_id: str) -> Computed | None:
|
|
63
|
+
"""The computed value of ``facet_id`` for ``cid``, or ``None`` when unknown
|
|
64
|
+
or when ``facet_id`` is not computed."""
|
|
65
|
+
if facet_id != COST_PER_TASK:
|
|
66
|
+
return None
|
|
67
|
+
if cid not in self._values:
|
|
68
|
+
self._values[cid] = self._cost_per_task(cid)
|
|
69
|
+
return self._values[cid]
|
|
70
|
+
|
|
71
|
+
def _cost_per_task(self, cid: str) -> Computed | None:
|
|
72
|
+
inputs = self._base.fact(cid, _PRICE_INPUT)
|
|
73
|
+
outputs = self._base.fact(cid, _PRICE_OUTPUT)
|
|
74
|
+
price_in, price_out = _price(inputs), _price(outputs)
|
|
75
|
+
if price_in is None or price_out is None:
|
|
76
|
+
return None
|
|
77
|
+
tokens = self.task_tokens
|
|
78
|
+
value = round((price_in * tokens.input + price_out * tokens.output) / 1_000_000, 12)
|
|
79
|
+
formula = (
|
|
80
|
+
f"({_number(price_in)} USD per 1M input tokens × {tokens.input:,} input tokens"
|
|
81
|
+
f" + {_number(price_out)} USD per 1M output tokens × {tokens.output:,} output tokens)"
|
|
82
|
+
f" ÷ 1,000,000 = {_number(value)} USD per task"
|
|
83
|
+
)
|
|
84
|
+
records = tuple(r for r in (inputs.record_id, outputs.record_id) if r)
|
|
85
|
+
sources = tuple(dict.fromkeys((*inputs.sources, *outputs.sources)))
|
|
86
|
+
return Computed(value, records, sources, formula)
|
|
87
|
+
|
|
88
|
+
# SnapshotIndex -----------------------------------------------------------
|
|
89
|
+
|
|
90
|
+
def fact(self, cid: str, facet_id: str) -> FactValue:
|
|
91
|
+
if facet_id not in COMPUTED_FACETS:
|
|
92
|
+
return self._base.fact(cid, facet_id)
|
|
93
|
+
found = self.computed(cid, facet_id)
|
|
94
|
+
return UNKNOWN if found is None else FactValue("known", found.value, found.sources)
|
|
95
|
+
|
|
96
|
+
def ids_where(self, facet_id: str, op: str, arg: Any) -> Bitset3:
|
|
97
|
+
if facet_id not in COMPUTED_FACETS:
|
|
98
|
+
return self._base.ids_where(facet_id, op, arg)
|
|
99
|
+
ids = self._base.candidates()
|
|
100
|
+
everyone = (1 << len(ids)) - 1
|
|
101
|
+
if self._column is None:
|
|
102
|
+
rows = []
|
|
103
|
+
for row, cid in enumerate(ids):
|
|
104
|
+
found = self.computed(cid, facet_id)
|
|
105
|
+
if found is not None:
|
|
106
|
+
rows.append((row, found.value))
|
|
107
|
+
self._column = _FacetBitsets(rows)
|
|
108
|
+
column = self._column
|
|
109
|
+
if op == "known":
|
|
110
|
+
return Bitset3(column.known, everyone & ~column.known, 0)
|
|
111
|
+
passing = column.passing(op, arg)
|
|
112
|
+
if passing is None:
|
|
113
|
+
passing = 0
|
|
114
|
+
for row, cid in enumerate(ids):
|
|
115
|
+
found = self.computed(cid, facet_id)
|
|
116
|
+
if found is not None and _holds(found.value, op, arg):
|
|
117
|
+
passing |= 1 << row
|
|
118
|
+
return Bitset3(passing, column.known & ~passing, everyone & ~column.known)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def with_computed(snapshot: Any, task_tokens: TaskTokens) -> ComputedFacets:
|
|
122
|
+
"""``snapshot``, answering the computed facets at ``task_tokens``."""
|
|
123
|
+
if isinstance(snapshot, ComputedFacets):
|
|
124
|
+
snapshot = snapshot._base
|
|
125
|
+
return ComputedFacets(snapshot, task_tokens)
|