modelroom 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modelroom/__init__.py +9 -0
- modelroom/answers.py +55 -0
- modelroom/binding.py +209 -0
- modelroom/catalog.py +185 -0
- modelroom/catalog.toml +569 -0
- modelroom/cli.py +724 -0
- modelroom/config.py +668 -0
- modelroom/contracts.py +798 -0
- modelroom/daemon.py +205 -0
- modelroom/dialog.py +463 -0
- modelroom/document.py +169 -0
- modelroom/examples.py +551 -0
- modelroom/fetch.py +190 -0
- modelroom/fetch_types.py +40 -0
- modelroom/fit.py +334 -0
- modelroom/guided.py +783 -0
- modelroom/guided_context.py +379 -0
- modelroom/guided_contracts.py +127 -0
- modelroom/guided_loadtest.py +295 -0
- modelroom/guided_models.py +556 -0
- modelroom/guided_search.py +266 -0
- modelroom/hf.py +489 -0
- modelroom/http.py +321 -0
- modelroom/importer.py +495 -0
- modelroom/intro.py +456 -0
- modelroom/llmfit.py +344 -0
- modelroom/loadtest.py +554 -0
- modelroom/measure.py +676 -0
- modelroom/measurements.py +494 -0
- modelroom/migrate.py +234 -0
- modelroom/ollama.py +388 -0
- modelroom/ollama_local.py +63 -0
- modelroom/profile.py +278 -0
- modelroom/protocol_v1.toml +31 -0
- modelroom/provenance.py +170 -0
- modelroom/quantization.py +308 -0
- modelroom/ranking.py +134 -0
- modelroom/relation.py +113 -0
- modelroom/render.py +454 -0
- modelroom/render_cmd.py +243 -0
- modelroom/screen.py +500 -0
- modelroom/search.py +794 -0
- modelroom/search_age.py +130 -0
- modelroom/search_apply.py +158 -0
- modelroom/search_pages.py +353 -0
- modelroom/search_word.py +238 -0
- modelroom/state.py +519 -0
- modelroom/toml_writer.py +71 -0
- modelroom/views.py +775 -0
- modelroom-0.1.0.dist-info/METADATA +390 -0
- modelroom-0.1.0.dist-info/RECORD +54 -0
- modelroom-0.1.0.dist-info/WHEEL +4 -0
- modelroom-0.1.0.dist-info/entry_points.txt +2 -0
- modelroom-0.1.0.dist-info/licenses/LICENSE +21 -0
modelroom/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""modelroom: which local model packages exist for your model families, and which fit your machine."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import version
|
|
4
|
+
|
|
5
|
+
# The only version source is [project] version in pyproject.toml; the package has to be
|
|
6
|
+
# installed (uv sync) for this to resolve, which is also how the tests run.
|
|
7
|
+
__version__ = version("modelroom")
|
|
8
|
+
|
|
9
|
+
__all__ = ["__version__"]
|
modelroom/answers.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""The answer file: `modelroom --answers <file>` takes the dialog's answers from TOML.
|
|
2
|
+
|
|
3
|
+
One key per question, as CONTRACTS.md, "Guided mode", lists them; `schema_version = 1` on top.
|
|
4
|
+
A question the file has no answer for ends the run with exit `2` and names the question, so a
|
|
5
|
+
self-test or a CI run never silently answers something it did not mean to
|
|
6
|
+
(`dialog.FileAsker`). Nothing else about the guided mode changes: the same steps run in the
|
|
7
|
+
same order and print the same lines.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import tomllib
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from .contracts import SchemaVersionError, check_schema_version
|
|
16
|
+
|
|
17
|
+
ANSWERS_SCHEMA_VERSION = 1
|
|
18
|
+
# Half-open, same convention as every other reader in this package.
|
|
19
|
+
ANSWERS_SCHEMA_RANGE: tuple[int, int] = (1, 2)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AnswerFileError(Exception):
|
|
23
|
+
"""The answer file does not read, or one of its answers is not a shape a question can take."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _is_answer(value: object) -> bool:
|
|
27
|
+
"""The four shapes a question's answer can have: text, a choice, a switch, a list of choices."""
|
|
28
|
+
if isinstance(value, bool) or isinstance(value, str):
|
|
29
|
+
return True
|
|
30
|
+
if isinstance(value, int):
|
|
31
|
+
return True
|
|
32
|
+
return isinstance(value, list) and all(isinstance(entry, str) for entry in value)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def read_answers(path: Path) -> dict[str, object]:
|
|
36
|
+
"""Every answer in the file at `path`, by question key; `schema_version` is not an answer.
|
|
37
|
+
|
|
38
|
+
Raises `AnswerFileError` when the file cannot be read or holds an answer of another shape,
|
|
39
|
+
and `SchemaVersionError` when its `schema_version` is missing or outside the accepted range.
|
|
40
|
+
"""
|
|
41
|
+
try:
|
|
42
|
+
raw = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
43
|
+
except (OSError, tomllib.TOMLDecodeError, UnicodeDecodeError) as exc:
|
|
44
|
+
raise AnswerFileError(f"{path}: cannot read the answer file: {exc}") from exc
|
|
45
|
+
version = raw.get("schema_version")
|
|
46
|
+
if not isinstance(version, int) or isinstance(version, bool):
|
|
47
|
+
raise SchemaVersionError(f"{path}: schema_version is missing or not an integer: {version!r}")
|
|
48
|
+
check_schema_version(version, ANSWERS_SCHEMA_RANGE, str(path))
|
|
49
|
+
answers = {key: value for key, value in raw.items() if key != "schema_version"}
|
|
50
|
+
for key, value in answers.items():
|
|
51
|
+
if not _is_answer(value):
|
|
52
|
+
raise AnswerFileError(
|
|
53
|
+
f"{path}: the answer {key!r} is not text, a whole number, true/false or a list of texts"
|
|
54
|
+
)
|
|
55
|
+
return answers
|
modelroom/binding.py
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""Which profile is "this machine": the pointer file in the home folder and the takeover rule.
|
|
2
|
+
|
|
3
|
+
The pointer file (`~/.modelroom/guided.json`) remembers the results folder the guided mode
|
|
4
|
+
last used and, per results folder, the `profile_id` of this machine's own profile there (the
|
|
5
|
+
local binding). `resolve_profile_target` decides, without any I/O, which profile a measurement
|
|
6
|
+
of this machine writes to (CONTRACTS.md, "Profile binding").
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Literal
|
|
15
|
+
|
|
16
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator
|
|
17
|
+
|
|
18
|
+
from .contracts import PROFILE_ID_RE, SchemaVersionError, check_schema_version
|
|
19
|
+
from .state import atomic_write_json
|
|
20
|
+
|
|
21
|
+
POINTER_SCHEMA_VERSION = 1
|
|
22
|
+
POINTER_SCHEMA_RANGE: tuple[int, int] = (1, 2)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class PointerFileError(Exception):
|
|
26
|
+
"""The pointer file exists but is not valid JSON or does not match `GuidedPointer`."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def default_pointer_path() -> Path:
|
|
30
|
+
"""`~/.modelroom/guided.json` of the current user."""
|
|
31
|
+
return Path.home() / ".modelroom" / "guided.json"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class GuidedPointer(BaseModel):
|
|
35
|
+
"""The pointer file: last results folder and, per results folder, this machine's profile.
|
|
36
|
+
|
|
37
|
+
Keys of `bindings` and `current` are absolute folder paths as the guided mode resolved them.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
model_config = ConfigDict(extra="forbid")
|
|
41
|
+
|
|
42
|
+
schema_version: int
|
|
43
|
+
current: str | None = None
|
|
44
|
+
bindings: dict[str, str] = Field(default_factory=dict)
|
|
45
|
+
|
|
46
|
+
@field_validator("bindings")
|
|
47
|
+
@classmethod
|
|
48
|
+
def _check_bindings(cls, value: dict[str, str]) -> dict[str, str]:
|
|
49
|
+
for folder, profile_id in value.items():
|
|
50
|
+
if not Path(folder).is_absolute():
|
|
51
|
+
raise ValueError(f"bindings keys are absolute results folders: {folder!r}")
|
|
52
|
+
if not PROFILE_ID_RE.fullmatch(profile_id):
|
|
53
|
+
raise ValueError(f"bindings values are profile_ids of 16 lowercase hex characters: {profile_id!r}")
|
|
54
|
+
return value
|
|
55
|
+
|
|
56
|
+
@field_validator("current")
|
|
57
|
+
@classmethod
|
|
58
|
+
def _check_current(cls, value: str | None) -> str | None:
|
|
59
|
+
if value is not None and not Path(value).is_absolute():
|
|
60
|
+
raise ValueError(f"current must be an absolute results folder: {value!r}")
|
|
61
|
+
return value
|
|
62
|
+
|
|
63
|
+
@model_validator(mode="after")
|
|
64
|
+
def _check_schema_version(self) -> "GuidedPointer":
|
|
65
|
+
if self.schema_version != POINTER_SCHEMA_VERSION:
|
|
66
|
+
raise ValueError(f"schema_version must be {POINTER_SCHEMA_VERSION}, got {self.schema_version}")
|
|
67
|
+
return self
|
|
68
|
+
|
|
69
|
+
def binding_for(self, results_dir: Path) -> str | None:
|
|
70
|
+
return self.bindings.get(str(results_dir))
|
|
71
|
+
|
|
72
|
+
def with_binding(self, results_dir: Path, profile_id: str) -> "GuidedPointer":
|
|
73
|
+
"""A copy that binds `profile_id` to `results_dir` and makes it the current folder."""
|
|
74
|
+
return GuidedPointer.model_validate(
|
|
75
|
+
{
|
|
76
|
+
"schema_version": POINTER_SCHEMA_VERSION,
|
|
77
|
+
"current": str(results_dir),
|
|
78
|
+
"bindings": {**self.bindings, str(results_dir): profile_id},
|
|
79
|
+
}
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
def with_current(self, results_dir: Path) -> "GuidedPointer":
|
|
83
|
+
"""A copy whose `current` is `results_dir`; the bindings stay as they are.
|
|
84
|
+
|
|
85
|
+
The guided mode remembers the folder it wrote a configuration in before anything has
|
|
86
|
+
been measured there, so the next run finds it without asking again; the binding follows
|
|
87
|
+
only when a measurement really produced a profile (`with_binding`).
|
|
88
|
+
"""
|
|
89
|
+
return GuidedPointer.model_validate(
|
|
90
|
+
{
|
|
91
|
+
"schema_version": POINTER_SCHEMA_VERSION,
|
|
92
|
+
"current": str(results_dir),
|
|
93
|
+
"bindings": dict(self.bindings),
|
|
94
|
+
}
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def read_pointer(path: Path) -> GuidedPointer:
|
|
99
|
+
"""The pointer file at `path`; an empty pointer when it does not exist yet.
|
|
100
|
+
|
|
101
|
+
A version outside the accepted range raises `SchemaVersionError`; broken content raises
|
|
102
|
+
`PointerFileError` naming the file.
|
|
103
|
+
"""
|
|
104
|
+
if not path.exists():
|
|
105
|
+
return GuidedPointer(schema_version=POINTER_SCHEMA_VERSION)
|
|
106
|
+
try:
|
|
107
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
108
|
+
except (OSError, ValueError) as exc: # ValueError: bad UTF-8/JSON and over-long integers
|
|
109
|
+
raise PointerFileError(f"{path}: cannot read pointer file: {exc}") from exc
|
|
110
|
+
if not isinstance(data, dict):
|
|
111
|
+
raise PointerFileError(f"{path}: expected a JSON object at the root")
|
|
112
|
+
version = data.get("schema_version")
|
|
113
|
+
if not isinstance(version, int) or isinstance(version, bool):
|
|
114
|
+
raise SchemaVersionError(f"{path}: schema_version is missing or not an integer: {version!r}")
|
|
115
|
+
check_schema_version(version, POINTER_SCHEMA_RANGE, str(path))
|
|
116
|
+
try:
|
|
117
|
+
return GuidedPointer.model_validate(data)
|
|
118
|
+
except ValidationError as exc:
|
|
119
|
+
raise PointerFileError(f"{path}: invalid pointer file: {exc}") from exc
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def write_pointer(path: Path, pointer: GuidedPointer) -> None:
|
|
123
|
+
"""Replace the pointer file atomically (it belongs to one user; no shared lock needed)."""
|
|
124
|
+
atomic_write_json(path, pointer.model_dump(mode="json"))
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@dataclass(frozen=True)
|
|
128
|
+
class KnownProfile:
|
|
129
|
+
"""What the takeover rule needs to know about a profile file in the results folder."""
|
|
130
|
+
|
|
131
|
+
profile_id: str
|
|
132
|
+
os_fingerprint: str
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@dataclass(frozen=True)
|
|
136
|
+
class ProfileTarget:
|
|
137
|
+
"""The decision: which profile a measurement of this machine goes to, and why.
|
|
138
|
+
|
|
139
|
+
`bound` -- the home binding holds; `adopt_config` -- `[machines.<name>].profile` becomes the
|
|
140
|
+
binding (no new profile); `new` -- a new profile, bound and written to the configuration;
|
|
141
|
+
`rewrite` -- that missing or foreign profile is measured again under its own id, because the
|
|
142
|
+
answer was "the same machine"; `ask_clone` -- the bound or configured profile is missing or
|
|
143
|
+
its fingerprint differs: the guided mode asks "same machine or a clone?", automation stops
|
|
144
|
+
and points to `hardware --same-machine` and `hardware --new-identity`.
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
action: Literal["bound", "adopt_config", "new", "rewrite", "ask_clone"]
|
|
148
|
+
profile_id: str
|
|
149
|
+
reason: str
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _fingerprints_agree(known: KnownProfile, local_fingerprint: str) -> bool:
|
|
153
|
+
# A profile without a fingerprint (migrated, or unreadable on its machine) cannot disagree.
|
|
154
|
+
return "none" in (known.os_fingerprint, local_fingerprint) or known.os_fingerprint == local_fingerprint
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def resolve_profile_target(
|
|
158
|
+
home_binding: str | None,
|
|
159
|
+
config_profile: str | None,
|
|
160
|
+
profiles: dict[str, KnownProfile],
|
|
161
|
+
local_fingerprint: str,
|
|
162
|
+
fresh_profile_id: str,
|
|
163
|
+
new_identity: bool = False,
|
|
164
|
+
same_machine: bool = False,
|
|
165
|
+
) -> ProfileTarget:
|
|
166
|
+
"""Pick the profile "this machine" writes to, in the order of the takeover rule.
|
|
167
|
+
|
|
168
|
+
(0) `new_identity` (`hardware --new-identity`, or "a clone" in the guided mode): a new
|
|
169
|
+
profile, replacing the binding. (1) The home binding for this results folder. (2) Else the
|
|
170
|
+
selected configuration machine's `profile`, adopted as the binding. (3) Else a new profile.
|
|
171
|
+
A binding or configured profile that is missing from `profiles` (the profile files found in
|
|
172
|
+
the results folder), or whose `os_fingerprint` differs from `local_fingerprint`, is
|
|
173
|
+
`ask_clone`. `local_fingerprint` is `none` when this machine's OS identifier is unreadable.
|
|
174
|
+
|
|
175
|
+
`same_machine` (`hardware --same-machine`, or "the same machine" in the guided mode) is the
|
|
176
|
+
answer to exactly that question: the `ask_clone` case becomes `rewrite`, a measurement under
|
|
177
|
+
the very id the binding names, so a results folder someone emptied is measured into again
|
|
178
|
+
(decided 2026-09-24). It changes nothing where the rule does not ask. The two switches are
|
|
179
|
+
two answers to one question and are never both given.
|
|
180
|
+
"""
|
|
181
|
+
if new_identity and same_machine:
|
|
182
|
+
raise ValueError("new_identity and same_machine are two answers to one question; pass one of them")
|
|
183
|
+
if not PROFILE_ID_RE.fullmatch(fresh_profile_id) or fresh_profile_id in profiles:
|
|
184
|
+
raise ValueError(f"fresh_profile_id must be a new profile_id: {fresh_profile_id!r}")
|
|
185
|
+
if new_identity:
|
|
186
|
+
return ProfileTarget("new", fresh_profile_id, "new identity requested")
|
|
187
|
+
for candidate, action, source in (
|
|
188
|
+
(home_binding, "bound", "home binding"),
|
|
189
|
+
(config_profile, "adopt_config", "configuration"),
|
|
190
|
+
):
|
|
191
|
+
if candidate is None:
|
|
192
|
+
continue
|
|
193
|
+
known = profiles.get(candidate)
|
|
194
|
+
if known is None:
|
|
195
|
+
missing = f"profile from {source} is missing in the results folder"
|
|
196
|
+
if same_machine:
|
|
197
|
+
return ProfileTarget("rewrite", candidate, f"{missing}; measured again under the same id")
|
|
198
|
+
return ProfileTarget("ask_clone", candidate, missing)
|
|
199
|
+
if not _fingerprints_agree(known, local_fingerprint):
|
|
200
|
+
differs = f"profile from {source} has another os_fingerprint"
|
|
201
|
+
if same_machine:
|
|
202
|
+
return ProfileTarget(
|
|
203
|
+
"rewrite",
|
|
204
|
+
candidate,
|
|
205
|
+
f"{differs}; measured again under the same id, the answer was that it is the same machine",
|
|
206
|
+
)
|
|
207
|
+
return ProfileTarget("ask_clone", candidate, differs)
|
|
208
|
+
return ProfileTarget(action, candidate, f"profile from {source}")
|
|
209
|
+
return ProfileTarget("new", fresh_profile_id, "no binding and no configured profile")
|
modelroom/catalog.py
ADDED
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
"""The shipped catalog (`catalog.toml`): publishers, current models, successors, Ollama names.
|
|
2
|
+
|
|
3
|
+
The search uses it to tell a publisher from a packager, to call a model `latest` or `legacy`
|
|
4
|
+
when the repo itself says nothing (`new_version`), and to know an Ollama name. Every model row
|
|
5
|
+
names its evidence; nothing in it is inferred (CONTRACTS.md, "Catalog rules").
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import tomllib
|
|
11
|
+
from datetime import date
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Literal
|
|
14
|
+
from urllib.parse import urlsplit
|
|
15
|
+
|
|
16
|
+
from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator
|
|
17
|
+
|
|
18
|
+
from .contracts import SchemaVersionError, check_schema_version, validate_hf_repo, validate_ollama_pair
|
|
19
|
+
|
|
20
|
+
CATALOG_FILE = Path(__file__).with_name("catalog.toml")
|
|
21
|
+
CATALOG_SCHEMA_VERSION = 1
|
|
22
|
+
CATALOG_SCHEMA_RANGE: tuple[int, int] = (1, 2)
|
|
23
|
+
|
|
24
|
+
Age = Literal["latest", "legacy", "unknown"]
|
|
25
|
+
_HUB = "https://huggingface.co/"
|
|
26
|
+
# `urlsplit(...).hostname` is lowercase and without the port.
|
|
27
|
+
_HUB_HOSTS = frozenset({"huggingface.co", "www.huggingface.co", "hf.co"})
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class CatalogError(Exception):
|
|
31
|
+
"""The catalog file cannot be read or does not validate."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class CatalogModel(BaseModel):
|
|
35
|
+
"""One publisher base model: current (`latest`), superseded (`successor`), or neither.
|
|
36
|
+
|
|
37
|
+
`latest` is a statement about this one model, never derived from a line or a number: it
|
|
38
|
+
carries the publisher page or collection that says so (`latest_source`) and the day the
|
|
39
|
+
maintainer checked it (`latest_checked`). Without them the model's age is `unknown`.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
model_config = ConfigDict(extra="forbid")
|
|
43
|
+
|
|
44
|
+
hf_repo: str
|
|
45
|
+
latest: bool = False
|
|
46
|
+
latest_source: str | None = None
|
|
47
|
+
latest_checked: date | None = None
|
|
48
|
+
successor: str | None = None
|
|
49
|
+
ollama_base: str | None = None
|
|
50
|
+
ollama_tag: str | None = None
|
|
51
|
+
source: str
|
|
52
|
+
|
|
53
|
+
@field_validator("hf_repo")
|
|
54
|
+
@classmethod
|
|
55
|
+
def _check_hf_repo(cls, value: str) -> str:
|
|
56
|
+
return validate_hf_repo(value)
|
|
57
|
+
|
|
58
|
+
@field_validator("source")
|
|
59
|
+
@classmethod
|
|
60
|
+
def _check_source(cls, value: str) -> str:
|
|
61
|
+
if not value.startswith(_HUB):
|
|
62
|
+
raise ValueError(f"source must be a https://huggingface.co/ page: {value!r}")
|
|
63
|
+
return value
|
|
64
|
+
|
|
65
|
+
@field_validator("latest_source")
|
|
66
|
+
@classmethod
|
|
67
|
+
def _check_latest_source(cls, value: str | None) -> str | None:
|
|
68
|
+
if value is not None:
|
|
69
|
+
parts = urlsplit(value)
|
|
70
|
+
if parts.scheme != "https" or not parts.hostname:
|
|
71
|
+
raise ValueError(f"latest_source must be an https page with a host: {value!r}")
|
|
72
|
+
return value
|
|
73
|
+
|
|
74
|
+
@model_validator(mode="after")
|
|
75
|
+
def _check_fields(self) -> "CatalogModel":
|
|
76
|
+
validate_ollama_pair(self.ollama_base, self.ollama_tag)
|
|
77
|
+
evidence = (self.latest_source, self.latest_checked)
|
|
78
|
+
if self.latest and None in evidence:
|
|
79
|
+
raise ValueError(f"{self.hf_repo}: latest needs latest_source and latest_checked")
|
|
80
|
+
if not self.latest and evidence != (None, None):
|
|
81
|
+
raise ValueError(f"{self.hf_repo}: latest_source and latest_checked belong to a latest model")
|
|
82
|
+
if self.successor is not None:
|
|
83
|
+
validate_hf_repo(self.successor)
|
|
84
|
+
if self.latest:
|
|
85
|
+
raise ValueError(f"{self.hf_repo}: a model with a successor is not latest")
|
|
86
|
+
if self.successor == self.hf_repo:
|
|
87
|
+
raise ValueError(f"{self.hf_repo}: successor must be another model")
|
|
88
|
+
return self
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class CatalogFamily(BaseModel):
|
|
92
|
+
model_config = ConfigDict(extra="forbid")
|
|
93
|
+
|
|
94
|
+
name: str = Field(min_length=1)
|
|
95
|
+
publisher: str = Field(min_length=1)
|
|
96
|
+
models: list[CatalogModel] = Field(min_length=1)
|
|
97
|
+
|
|
98
|
+
@model_validator(mode="after")
|
|
99
|
+
def _check_models(self) -> "CatalogFamily":
|
|
100
|
+
repos = [model.hf_repo for model in self.models]
|
|
101
|
+
if len(repos) != len(set(repos)):
|
|
102
|
+
raise ValueError(f"family {self.name}: hf_repo listed twice")
|
|
103
|
+
for model in self.models:
|
|
104
|
+
for repo in (model.hf_repo, model.successor):
|
|
105
|
+
if repo is not None and repo.split("/", 1)[0] != self.publisher:
|
|
106
|
+
raise ValueError(f"family {self.name}: {repo} is not under publisher {self.publisher}")
|
|
107
|
+
if model.latest_source is not None and not _is_publisher_evidence(model.latest_source, self.publisher):
|
|
108
|
+
raise ValueError(
|
|
109
|
+
f"family {self.name}: latest_source of {model.hf_repo} on the Hub must be the page or a "
|
|
110
|
+
f"collection of publisher {self.publisher}: {model.latest_source}"
|
|
111
|
+
)
|
|
112
|
+
_check_no_successor_cycle(self.models, f"family {self.name}")
|
|
113
|
+
return self
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _is_publisher_evidence(url: str, publisher: str) -> bool:
|
|
117
|
+
"""A Hub page proves `latest` only as the publisher's own page or one of its collections; a
|
|
118
|
+
page off the Hub (the publisher's site) is taken as the maintainer confirmed it."""
|
|
119
|
+
parts = urlsplit(url)
|
|
120
|
+
if parts.hostname not in _HUB_HOSTS:
|
|
121
|
+
return True
|
|
122
|
+
path = parts.path.strip("/")
|
|
123
|
+
return path == publisher or path.startswith(f"collections/{publisher}/")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _check_no_successor_cycle(models: list[CatalogModel], label: str) -> None:
|
|
127
|
+
successor_of = {model.hf_repo: model.successor for model in models}
|
|
128
|
+
for start in successor_of:
|
|
129
|
+
seen = {start}
|
|
130
|
+
current = successor_of.get(start)
|
|
131
|
+
while current is not None:
|
|
132
|
+
if current in seen:
|
|
133
|
+
raise ValueError(f"{label}: successor cycle through {current}")
|
|
134
|
+
seen.add(current)
|
|
135
|
+
current = successor_of.get(current)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class Catalog(BaseModel):
|
|
139
|
+
model_config = ConfigDict(extra="forbid")
|
|
140
|
+
|
|
141
|
+
schema_version: int
|
|
142
|
+
families: list[CatalogFamily] = Field(default_factory=list)
|
|
143
|
+
|
|
144
|
+
@model_validator(mode="after")
|
|
145
|
+
def _check(self) -> "Catalog":
|
|
146
|
+
if self.schema_version != CATALOG_SCHEMA_VERSION:
|
|
147
|
+
raise ValueError(f"schema_version must be {CATALOG_SCHEMA_VERSION}, got {self.schema_version}")
|
|
148
|
+
names = [family.name for family in self.families]
|
|
149
|
+
repos = [model.hf_repo for family in self.families for model in family.models]
|
|
150
|
+
if len(names) != len(set(names)) or len(repos) != len(set(repos)):
|
|
151
|
+
raise ValueError("family names and hf_repo entries must be unique across the catalog")
|
|
152
|
+
_check_no_successor_cycle([m for family in self.families for m in family.models], "catalog")
|
|
153
|
+
return self
|
|
154
|
+
|
|
155
|
+
def model_for(self, hf_repo: str) -> CatalogModel | None:
|
|
156
|
+
return next((m for f in self.families for m in f.models if m.hf_repo == hf_repo), None)
|
|
157
|
+
|
|
158
|
+
def is_publisher(self, owner: str) -> bool:
|
|
159
|
+
return any(family.publisher == owner for family in self.families)
|
|
160
|
+
|
|
161
|
+
def age_of(self, hf_repo: str) -> tuple[Age, str | None]:
|
|
162
|
+
"""`(age, successor)` from the catalog alone: `latest`, `legacy` with its successor, or
|
|
163
|
+
`unknown` when the model is not listed or listed without either statement."""
|
|
164
|
+
model = self.model_for(hf_repo)
|
|
165
|
+
if model is None:
|
|
166
|
+
return "unknown", None
|
|
167
|
+
if model.successor is not None:
|
|
168
|
+
return "legacy", model.successor
|
|
169
|
+
return ("latest", None) if model.latest else ("unknown", None)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def load_catalog(path: Path = CATALOG_FILE) -> Catalog:
|
|
173
|
+
"""Read and validate a catalog file; the default is the one shipped with the package."""
|
|
174
|
+
try:
|
|
175
|
+
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
176
|
+
except (OSError, tomllib.TOMLDecodeError) as exc:
|
|
177
|
+
raise CatalogError(f"{path}: cannot read catalog: {exc}") from exc
|
|
178
|
+
version = data.get("schema_version")
|
|
179
|
+
if not isinstance(version, int) or isinstance(version, bool):
|
|
180
|
+
raise SchemaVersionError(f"{path}: schema_version is missing or not an integer: {version!r}")
|
|
181
|
+
check_schema_version(version, CATALOG_SCHEMA_RANGE, str(path))
|
|
182
|
+
try:
|
|
183
|
+
return Catalog.model_validate(data)
|
|
184
|
+
except ValidationError as exc:
|
|
185
|
+
raise CatalogError(f"{path}: invalid catalog: {exc}") from exc
|