model-pin-check 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- model_pin_check/__init__.py +309 -0
- model_pin_check/__main__.py +212 -0
- model_pin_check/data.json +26522 -0
- model_pin_check-0.1.0.dist-info/METADATA +144 -0
- model_pin_check-0.1.0.dist-info/RECORD +9 -0
- model_pin_check-0.1.0.dist-info/WHEEL +5 -0
- model_pin_check-0.1.0.dist-info/entry_points.txt +2 -0
- model_pin_check-0.1.0.dist-info/licenses/LICENSE +27 -0
- model_pin_check-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
"""model-pin-check — find the AI model ids your repository pins and say, for
|
|
2
|
+
each one, whether the provider has already shut it down, with the provider's
|
|
3
|
+
own sentence as the evidence.
|
|
4
|
+
|
|
5
|
+
This is the Python twin of the npm package of the same name, shipped because
|
|
6
|
+
the audience for an AI model pin check writes Python: the pins live in `.py`
|
|
7
|
+
files, in Airflow DAGs, in `conftest.py`, in CI steps that have `pip` and no
|
|
8
|
+
Node at all, and the way a Python repository installs a check like this is
|
|
9
|
+
`pip install` or a `pre-commit` hook, neither of which can reach npm. The
|
|
10
|
+
records, the data file, the verdicts and the honesty rules are the same ones
|
|
11
|
+
the npm package uses, built by the same `build_data.py`, so the two agree by
|
|
12
|
+
construction rather than by hand.
|
|
13
|
+
|
|
14
|
+
Three rules do most of the work here, and all three exist because getting them
|
|
15
|
+
wrong would make the tool lie:
|
|
16
|
+
|
|
17
|
+
1. **A model id is not unique.** Records that share an id with another record
|
|
18
|
+
do so because the same id is served by several platforms on different
|
|
19
|
+
dates: `gpt-4o` is one date on Azure and another on OpenAI's own API. A
|
|
20
|
+
found id therefore resolves to a *set* of records, and "gone" is reported
|
|
21
|
+
only when every record for that id says gone. Gone on some platforms and
|
|
22
|
+
not others is reported as exactly that.
|
|
23
|
+
2. **A bare word is not a pin.** Some real ids are single words with no digit
|
|
24
|
+
and no hyphen — `command`, `whisper`, `tts`, `ada`, `davinci`. Matching
|
|
25
|
+
those on a token boundary would flag every "git command" in a comment, so
|
|
26
|
+
they are reported only when the whole string literal is the id, or the line
|
|
27
|
+
reads like `model = "command"`.
|
|
28
|
+
3. **Absence is not a clearance.** A model id this package does not hold is
|
|
29
|
+
not thereby supported; it is unknown to us, and every answer says so in
|
|
30
|
+
words. Nothing here guesses a date, a status or a successor: every value
|
|
31
|
+
printed came off a provider's own page with the sentence it was read from.
|
|
32
|
+
|
|
33
|
+
The library half. `__main__.py` is the command.
|
|
34
|
+
"""
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import json
|
|
38
|
+
import os
|
|
39
|
+
import pathlib
|
|
40
|
+
import re
|
|
41
|
+
|
|
42
|
+
__all__ = [
|
|
43
|
+
"DATA", "scan", "lookup", "verdict", "version_line", "STATE_WORD",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
__version__ = "0.1.0"
|
|
47
|
+
|
|
48
|
+
_HERE = pathlib.Path(__file__).resolve().parent
|
|
49
|
+
DATA = json.loads((_HERE / "data.json").read_text(encoding="utf-8"))
|
|
50
|
+
|
|
51
|
+
# Directories never worth walking: not your code, or build output. Every
|
|
52
|
+
# dotted directory that is NOT in here is walked, because a model id is
|
|
53
|
+
# routinely pinned in `.devcontainer/`, `.streamlit/`, `.cursor/`, `.claude/`,
|
|
54
|
+
# `.chainlit/` and `.gitlab/`. So this list names the dotted directories that
|
|
55
|
+
# hold a cache, a build, or somebody else's code.
|
|
56
|
+
SKIP_DIRS = frozenset({
|
|
57
|
+
".git", "node_modules", "dist", "build", "out", "target", "vendor",
|
|
58
|
+
".venv", "venv", "env", "__pycache__", ".mypy_cache", ".pytest_cache",
|
|
59
|
+
".next", ".nuxt", ".svelte-kit", "coverage", "site-packages", ".tox",
|
|
60
|
+
".gradle", ".idea", ".terraform", "bower_components", ".cache",
|
|
61
|
+
".svn", ".hg", ".direnv", ".ruff_cache", ".ipynb_checkpoints", ".yarn",
|
|
62
|
+
".pnpm-store", ".pnpm", ".bundle", ".m2", ".nuget", ".conda", ".turbo",
|
|
63
|
+
".parcel-cache", ".serverless", ".vercel", ".netlify", ".angular",
|
|
64
|
+
".dart_tool", ".pub-cache", ".stack-work", ".cargo", ".rustup", ".pyenv",
|
|
65
|
+
".nvm", ".npm", ".gem", ".sass-cache", ".eggs", ".nox", ".dvc",
|
|
66
|
+
".terragrunt-cache", ".history", ".cpanm", ".local",
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
# Where a hardcoded model id actually lives. Python first, because that is who
|
|
70
|
+
# installs this, but a Python repository is not only Python: the same service
|
|
71
|
+
# pins the id again in its Dockerfile, its Helm values and its front end.
|
|
72
|
+
CODE_EXT = frozenset({
|
|
73
|
+
".py", ".pyi", ".pyw", ".ipynb",
|
|
74
|
+
".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx", ".go", ".rb", ".java",
|
|
75
|
+
".kt", ".kts", ".rs", ".php", ".cs", ".swift", ".scala", ".sh", ".bash",
|
|
76
|
+
".zsh", ".pl", ".lua", ".r", ".dart", ".ex", ".exs", ".clj", ".c", ".cc",
|
|
77
|
+
".cpp", ".h", ".hpp",
|
|
78
|
+
".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf",
|
|
79
|
+
".properties", ".env", ".tf", ".tfvars", ".gradle", ".xml", ".plist",
|
|
80
|
+
".vue", ".svelte", ".astro", ".hcl", ".jsonc", ".json5", ".groovy",
|
|
81
|
+
})
|
|
82
|
+
# Extensionless files that are still config or code, matched without regard to
|
|
83
|
+
# case because a repository may hold `dockerfile` or `makefile`.
|
|
84
|
+
NAMED_LOWER = frozenset({
|
|
85
|
+
"dockerfile", "makefile", ".env", "procfile", "containerfile",
|
|
86
|
+
"jenkinsfile", "justfile", "rakefile", "vagrantfile", "brewfile",
|
|
87
|
+
})
|
|
88
|
+
# A file whose name begins with one of these and then a suffix:
|
|
89
|
+
# `Dockerfile.prod`, `Containerfile.dev`, `Jenkinsfile.release`. Its extension
|
|
90
|
+
# is `.prod`, so the extension list can never catch it.
|
|
91
|
+
NAMED_PREFIX = ("dockerfile.", "containerfile.", "jenkinsfile.", "makefile.")
|
|
92
|
+
|
|
93
|
+
# Added by include_docs: prose mentions a model id without pinning it.
|
|
94
|
+
DOC_EXT = frozenset({".md", ".mdx", ".markdown", ".rst", ".txt", ".adoc"})
|
|
95
|
+
|
|
96
|
+
MAX_BYTES = 1024 * 1024
|
|
97
|
+
MAX_LINE = 4000 # longer than this is a minified bundle, not source
|
|
98
|
+
|
|
99
|
+
STATE_WORD = {
|
|
100
|
+
"gone": "SHUT DOWN",
|
|
101
|
+
"gone_somewhere": "shut down on some platforms",
|
|
102
|
+
"on_notice": "on notice — the provider has deprecated it",
|
|
103
|
+
"dated": "still served, with a retirement date",
|
|
104
|
+
"live": "still listed",
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
_TOKEN_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]{1,90}")
|
|
108
|
+
_BARE_RE = re.compile(r"[-._/:\s]")
|
|
109
|
+
_KEY_RE = re.compile(
|
|
110
|
+
r"(model|model_id|model_name|modelid|deployment|engine)\s*[:=]\s*$",
|
|
111
|
+
re.I)
|
|
112
|
+
_FLAG_RE = re.compile(r"--model(?:-id|-name)?[= ]$", re.I)
|
|
113
|
+
_QUOTE_TAIL_RE = re.compile(r"['\"`]\s*$")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def named_file(name: str) -> bool:
|
|
117
|
+
"""Is this file worth opening, by its name alone?"""
|
|
118
|
+
low = name.lower()
|
|
119
|
+
if low in NAMED_LOWER or low.startswith(".env"):
|
|
120
|
+
return True
|
|
121
|
+
return any(low.startswith(p) for p in NAMED_PREFIX)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _index(models):
|
|
125
|
+
"""model_id -> records, keyed lowercase because providers vary the case."""
|
|
126
|
+
by_id: dict[str, list[dict]] = {}
|
|
127
|
+
for m in models:
|
|
128
|
+
by_id.setdefault(m["model_id"].lower(), []).append(m)
|
|
129
|
+
return by_id
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
INDEX = _index(DATA["models"])
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def is_bare_word(model_id: str) -> bool:
|
|
136
|
+
"""An id that needs the strict context test: one word, no digit, no dash."""
|
|
137
|
+
return not any(c.isdigit() for c in model_id) \
|
|
138
|
+
and not _BARE_RE.search(model_id)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def candidates(token: str):
|
|
142
|
+
"""The ways one token in a file might be carrying a model id.
|
|
143
|
+
|
|
144
|
+
`us.anthropic.claude-3-5-sonnet-20240620-v1:0` is a cross-region Bedrock
|
|
145
|
+
alias of a Bedrock id; `models/gemini-2.5-flash` and `openai/gpt-4o` are
|
|
146
|
+
the same id behind a namespace. Bedrock's own ids contain dots and a `:0`
|
|
147
|
+
tail, so the whole token is always tried first and never stripped away.
|
|
148
|
+
"""
|
|
149
|
+
out = [token]
|
|
150
|
+
if "/" in token:
|
|
151
|
+
parts = [p for p in token.split("/") if p]
|
|
152
|
+
if parts:
|
|
153
|
+
out.append(parts[-1])
|
|
154
|
+
out.extend(parts)
|
|
155
|
+
m = re.fullmatch(r"(?:us|eu|apac|us-gov)\.(.+)", token)
|
|
156
|
+
if m:
|
|
157
|
+
out.append(m.group(1))
|
|
158
|
+
return out
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def looks_pinned(line: str, matched: str, start: int) -> bool:
|
|
162
|
+
"""Is this match in a position where a pin lives, rather than in prose?"""
|
|
163
|
+
before = line[:start]
|
|
164
|
+
after = line[start + len(matched):]
|
|
165
|
+
if before[-1:] in ("'", '"', "`") and after[:1] in ("'", '"', "`"):
|
|
166
|
+
return True # the whole literal is the id
|
|
167
|
+
stripped = _QUOTE_TAIL_RE.sub("", before)
|
|
168
|
+
if _KEY_RE.search(stripped): # model: command / "model": command
|
|
169
|
+
return True
|
|
170
|
+
return bool(_FLAG_RE.search(before))
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def verdict(records) -> str:
|
|
174
|
+
"""What the records say about one id, across every platform holding it.
|
|
175
|
+
|
|
176
|
+
`gone` only when nowhere still serves it. "Deprecated" and "Legacy" are
|
|
177
|
+
the provider's own notice; a model still marked GA or Active with a
|
|
178
|
+
retirement date in the future is not on notice and must not be described
|
|
179
|
+
as if it were.
|
|
180
|
+
"""
|
|
181
|
+
states = [r.get("state") for r in records]
|
|
182
|
+
gone = [s for s in states if s == "gone"]
|
|
183
|
+
if records and len(gone) == len(records):
|
|
184
|
+
return "gone"
|
|
185
|
+
if gone:
|
|
186
|
+
return "gone_somewhere"
|
|
187
|
+
if "on_notice" in states:
|
|
188
|
+
return "on_notice"
|
|
189
|
+
if "dated" in states:
|
|
190
|
+
return "dated"
|
|
191
|
+
return "live"
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def scan_line(line: str):
|
|
195
|
+
"""One line of one file: every model id it carries, with its position."""
|
|
196
|
+
hits = []
|
|
197
|
+
for m in _TOKEN_RE.finditer(line):
|
|
198
|
+
token = m.group(0)
|
|
199
|
+
for cand in candidates(token):
|
|
200
|
+
recs = INDEX.get(cand.lower())
|
|
201
|
+
if not recs:
|
|
202
|
+
continue
|
|
203
|
+
at = line.lower().find(cand.lower(), m.start())
|
|
204
|
+
if at < 0:
|
|
205
|
+
continue
|
|
206
|
+
if is_bare_word(recs[0]["model_id"]) \
|
|
207
|
+
and not looks_pinned(line, cand, at):
|
|
208
|
+
continue
|
|
209
|
+
hits.append({
|
|
210
|
+
"matched": cand, "id": recs[0]["model_id"], "records": recs,
|
|
211
|
+
"column": at + 1, "token": token, "token_start": m.start(),
|
|
212
|
+
})
|
|
213
|
+
break # one id per token; the longest form wins
|
|
214
|
+
return hits
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def walk(root, include_docs: bool = False, max_bytes: int = MAX_BYTES):
|
|
218
|
+
"""Walk a directory, yielding the files worth reading."""
|
|
219
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
220
|
+
dirnames[:] = [d for d in dirnames
|
|
221
|
+
if d not in SKIP_DIRS and d.lower() not in SKIP_DIRS]
|
|
222
|
+
for name in sorted(filenames):
|
|
223
|
+
full = os.path.join(dirpath, name)
|
|
224
|
+
ext = os.path.splitext(name)[1].lower()
|
|
225
|
+
ok = ext in CODE_EXT or named_file(name) \
|
|
226
|
+
or (include_docs and ext in DOC_EXT)
|
|
227
|
+
if not ok:
|
|
228
|
+
continue
|
|
229
|
+
try:
|
|
230
|
+
if os.path.getsize(full) > max_bytes:
|
|
231
|
+
continue
|
|
232
|
+
except OSError:
|
|
233
|
+
continue
|
|
234
|
+
yield full
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def scan(paths=(".",), include_docs: bool = False,
|
|
238
|
+
max_bytes: int = MAX_BYTES) -> dict:
|
|
239
|
+
"""Scan one or more paths. Findings grouped by model id.
|
|
240
|
+
|
|
241
|
+
The return carries `files_scanned` as well as the findings, because the
|
|
242
|
+
denominator is part of the answer: "nothing found" means nothing was found
|
|
243
|
+
in *these* files, and a scan that opened no file at all must never read as
|
|
244
|
+
a clean bill of health.
|
|
245
|
+
"""
|
|
246
|
+
found: dict[str, dict] = {}
|
|
247
|
+
files: list[str] = []
|
|
248
|
+
for p in paths:
|
|
249
|
+
if os.path.isdir(p):
|
|
250
|
+
files.extend(walk(p, include_docs=include_docs,
|
|
251
|
+
max_bytes=max_bytes))
|
|
252
|
+
elif os.path.exists(p):
|
|
253
|
+
files.append(p)
|
|
254
|
+
else:
|
|
255
|
+
raise FileNotFoundError(f"cannot read {p}")
|
|
256
|
+
for f in files:
|
|
257
|
+
try:
|
|
258
|
+
text = pathlib.Path(f).read_text(encoding="utf-8", errors="replace")
|
|
259
|
+
except OSError:
|
|
260
|
+
continue
|
|
261
|
+
if "\x00" in text:
|
|
262
|
+
continue # binary
|
|
263
|
+
for i, line in enumerate(text.splitlines(), start=1):
|
|
264
|
+
if len(line) > MAX_LINE:
|
|
265
|
+
continue
|
|
266
|
+
for hit in scan_line(line):
|
|
267
|
+
entry = found.get(hit["id"])
|
|
268
|
+
if entry is None:
|
|
269
|
+
entry = found[hit["id"]] = {
|
|
270
|
+
"id": hit["id"],
|
|
271
|
+
"records": hit["records"],
|
|
272
|
+
"verdict": verdict(hit["records"]),
|
|
273
|
+
"hits": [],
|
|
274
|
+
}
|
|
275
|
+
entry["hits"].append({
|
|
276
|
+
"file": f, "line": i, "column": hit["column"],
|
|
277
|
+
"matched": hit["matched"], "token": hit["token"],
|
|
278
|
+
"text": line.strip()[:200],
|
|
279
|
+
})
|
|
280
|
+
order = {"gone": 0, "gone_somewhere": 1, "on_notice": 2, "dated": 3,
|
|
281
|
+
"live": 4}
|
|
282
|
+
findings = sorted(found.values(),
|
|
283
|
+
key=lambda f: (order.get(f["verdict"], 9), f["id"]))
|
|
284
|
+
return {"files_scanned": len(files), "findings": findings}
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def lookup(model_id) -> dict | None:
|
|
288
|
+
"""Look one id up without scanning anything. None when we hold no record.
|
|
289
|
+
|
|
290
|
+
None is "we do not hold this", never "this is fine": the caller is told so
|
|
291
|
+
in those words by the command, and a library caller should say the same.
|
|
292
|
+
"""
|
|
293
|
+
recs = INDEX.get(str(model_id or "").lower())
|
|
294
|
+
if not recs:
|
|
295
|
+
return None
|
|
296
|
+
return {"id": recs[0]["model_id"], "records": recs,
|
|
297
|
+
"verdict": verdict(recs)}
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def absence_note(model_id) -> str:
|
|
301
|
+
"""The sentence that goes with a miss. The denominator is in it on purpose."""
|
|
302
|
+
return (f"no record for {model_id!r} in {DATA['asset']} "
|
|
303
|
+
f"({len(DATA['models'])} records, built {DATA['built']}) — "
|
|
304
|
+
f"absence here is not evidence it is supported")
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def version_line() -> str:
|
|
308
|
+
return (f"model-pin-check {__version__}, data built {DATA['built']} "
|
|
309
|
+
f"({len(DATA['models'])} records)")
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""model-pin-check [paths...] — the command.
|
|
3
|
+
|
|
4
|
+
Exit 1 means at least one model id in your code has been shut down by every
|
|
5
|
+
provider we hold a record for: that call will not come back. Deprecated with a
|
|
6
|
+
date still in the future exits 0 and is listed, because a deprecated model
|
|
7
|
+
still answers and breaking a build over one would be wrong. --strict also
|
|
8
|
+
exits 1 when an id is gone on some platforms and not others.
|
|
9
|
+
|
|
10
|
+
This command never edits a file. The npm twin can rewrite a dead pin to the
|
|
11
|
+
successor the provider names (`--fix` / `--write`); that half is deliberately
|
|
12
|
+
not here yet, because a rewrite that is wrong edits somebody's code and the
|
|
13
|
+
rules that make it safe deserve their own tests rather than a hurried port.
|
|
14
|
+
The help text says so rather than leaving the reader to find out.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import sys
|
|
20
|
+
|
|
21
|
+
from . import (DATA, STATE_WORD, __version__, absence_note, lookup, scan,
|
|
22
|
+
version_line)
|
|
23
|
+
|
|
24
|
+
HELP = f"""model-pin-check — find the AI model ids your repo pins and say
|
|
25
|
+
whether the provider has already shut them down.
|
|
26
|
+
|
|
27
|
+
model-pin-check scan the current directory
|
|
28
|
+
model-pin-check src tests scan these paths
|
|
29
|
+
model-pin-check --model gpt-4o look one id up, scan nothing
|
|
30
|
+
|
|
31
|
+
--json machine-readable output (for CI and agents)
|
|
32
|
+
--model <id> look up one id and exit
|
|
33
|
+
--include-docs also scan .md/.rst/.txt (prose mentions, not pins)
|
|
34
|
+
--strict exit 1 when an id is gone on some platforms, not all
|
|
35
|
+
--quiet only print ids that are not plainly still listed
|
|
36
|
+
--full-quote print the whole quoted sentence, not the first 240 chars
|
|
37
|
+
-h, --help this
|
|
38
|
+
-V, --version version and the date the data was built
|
|
39
|
+
|
|
40
|
+
This command reads files and changes none. It also reports only the ids it
|
|
41
|
+
found in the files it opened: an id it did not name has not been cleared, and
|
|
42
|
+
an id we hold no record for is unknown to us, not supported.
|
|
43
|
+
|
|
44
|
+
Every answer carries the provider's own sentence and the page it came from.
|
|
45
|
+
Data: {len(DATA['models'])} records, {len(DATA['providers'])} providers, \
|
|
46
|
+
built {DATA['built']}.
|
|
47
|
+
{DATA['asset_page']}"""
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def date_line(r: dict) -> str:
|
|
51
|
+
if not r.get("shutdown_date"):
|
|
52
|
+
return "no shutdown date in the record"
|
|
53
|
+
s = r["shutdown_date"]
|
|
54
|
+
if r.get("shutdown_date_ambiguous"):
|
|
55
|
+
s += (f" (printed this way it reads as either {r['shutdown_date_iso']}"
|
|
56
|
+
f" or {r['shutdown_date_iso_latest']}; this tool uses the later"
|
|
57
|
+
f" one, so it never calls a model gone early)")
|
|
58
|
+
elif not r.get("shutdown_date_iso"):
|
|
59
|
+
s += " (not a fixed date)"
|
|
60
|
+
return s
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def print_record(r: dict, full_quote: bool) -> None:
|
|
64
|
+
where = " — ".join(x for x in (r.get("provider"), r.get("platform_scope"))
|
|
65
|
+
if x)
|
|
66
|
+
print(f" {where or 'provider not stated'}")
|
|
67
|
+
print(f" status: {r.get('lifecycle_status') or 'not stated'}"
|
|
68
|
+
f" shutdown: {date_line(r)}")
|
|
69
|
+
if r.get("replacement_model"):
|
|
70
|
+
print(" the provider names as replacement: "
|
|
71
|
+
f"{r['replacement_model']}")
|
|
72
|
+
q = " ".join((r.get("source_quote") or "").split())
|
|
73
|
+
shown = q if (full_quote or len(q) <= 240) else q[:240] + "…"
|
|
74
|
+
if shown:
|
|
75
|
+
print(f' "{shown}"')
|
|
76
|
+
if not r.get("date_in_quote") and r.get("shutdown_date"):
|
|
77
|
+
print(" (the date is not printed in this sentence — the "
|
|
78
|
+
"record page shows where it comes from)")
|
|
79
|
+
print(f" {r.get('source')}")
|
|
80
|
+
print(f" our record: {r.get('page')} "
|
|
81
|
+
f"last checked {r.get('last_checked')}")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def main(argv=None) -> int:
|
|
85
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
86
|
+
opts = {"json": False, "include_docs": False, "strict": False,
|
|
87
|
+
"quiet": False, "full_quote": False}
|
|
88
|
+
paths: list[str] = []
|
|
89
|
+
one = None
|
|
90
|
+
i = 0
|
|
91
|
+
while i < len(argv):
|
|
92
|
+
a = argv[i]
|
|
93
|
+
if a in ("-h", "--help"):
|
|
94
|
+
print(HELP)
|
|
95
|
+
return 0
|
|
96
|
+
if a in ("-V", "--version", "-v"):
|
|
97
|
+
print(version_line())
|
|
98
|
+
return 0
|
|
99
|
+
if a == "--json":
|
|
100
|
+
opts["json"] = True
|
|
101
|
+
elif a == "--include-docs":
|
|
102
|
+
opts["include_docs"] = True
|
|
103
|
+
elif a == "--strict":
|
|
104
|
+
opts["strict"] = True
|
|
105
|
+
elif a == "--quiet":
|
|
106
|
+
opts["quiet"] = True
|
|
107
|
+
elif a == "--full-quote":
|
|
108
|
+
opts["full_quote"] = True
|
|
109
|
+
elif a == "--model":
|
|
110
|
+
i += 1
|
|
111
|
+
one = argv[i] if i < len(argv) else None
|
|
112
|
+
if one is None:
|
|
113
|
+
print("--model needs an id", file=sys.stderr)
|
|
114
|
+
return 2
|
|
115
|
+
elif a.startswith("--model="):
|
|
116
|
+
one = a[len("--model="):]
|
|
117
|
+
elif a in ("--fix", "--write"):
|
|
118
|
+
print("this Python build reads files and rewrites none. The repair "
|
|
119
|
+
"half lives in the npm twin (npx model-pin-check --fix); it "
|
|
120
|
+
"is not ported yet because a wrong rewrite edits your code.",
|
|
121
|
+
file=sys.stderr)
|
|
122
|
+
return 2
|
|
123
|
+
elif a.startswith("-"):
|
|
124
|
+
print(f"unknown option {a} — try --help", file=sys.stderr)
|
|
125
|
+
return 2
|
|
126
|
+
else:
|
|
127
|
+
paths.append(a)
|
|
128
|
+
i += 1
|
|
129
|
+
|
|
130
|
+
if one is not None:
|
|
131
|
+
hit = lookup(one)
|
|
132
|
+
if hit is None:
|
|
133
|
+
note = absence_note(one)
|
|
134
|
+
print(json.dumps({"model": one, "found": False, "note": note},
|
|
135
|
+
indent=1) if opts["json"] else note)
|
|
136
|
+
return 0
|
|
137
|
+
if opts["json"]:
|
|
138
|
+
print(json.dumps(hit, indent=1, sort_keys=True))
|
|
139
|
+
else:
|
|
140
|
+
print(f"{hit['id']} — {STATE_WORD.get(hit['verdict'], hit['verdict'])}")
|
|
141
|
+
for r in hit["records"]:
|
|
142
|
+
print_record(r, opts["full_quote"])
|
|
143
|
+
if hit["verdict"] == "gone":
|
|
144
|
+
return 1
|
|
145
|
+
if opts["strict"] and hit["verdict"] == "gone_somewhere":
|
|
146
|
+
return 1
|
|
147
|
+
return 0
|
|
148
|
+
|
|
149
|
+
try:
|
|
150
|
+
result = scan(paths or ["."], include_docs=opts["include_docs"])
|
|
151
|
+
except FileNotFoundError as e:
|
|
152
|
+
print(str(e), file=sys.stderr)
|
|
153
|
+
return 2
|
|
154
|
+
|
|
155
|
+
def of(v):
|
|
156
|
+
return [f for f in result["findings"] if f["verdict"] == v]
|
|
157
|
+
|
|
158
|
+
gone, partial = of("gone"), of("gone_somewhere")
|
|
159
|
+
notice, dated = of("on_notice"), of("dated")
|
|
160
|
+
|
|
161
|
+
if opts["json"]:
|
|
162
|
+
print(json.dumps({
|
|
163
|
+
"tool": "model-pin-check", "language": "python",
|
|
164
|
+
"version": __version__, "data_built": DATA["built"],
|
|
165
|
+
"records_held": len(DATA["models"]),
|
|
166
|
+
"files_scanned": result["files_scanned"],
|
|
167
|
+
"counts": {"gone": len(gone),
|
|
168
|
+
"gone_on_some_platforms": len(partial),
|
|
169
|
+
"on_notice": len(notice), "dated": len(dated),
|
|
170
|
+
"total": len(result["findings"])},
|
|
171
|
+
"findings": result["findings"],
|
|
172
|
+
"note": ("only the ids found in the files scanned are reported; "
|
|
173
|
+
"an id not named here has not been cleared"),
|
|
174
|
+
"asset_page": DATA["asset_page"],
|
|
175
|
+
}, indent=1, sort_keys=True))
|
|
176
|
+
else:
|
|
177
|
+
show = [f for f in result["findings"] if f["verdict"] != "live"] \
|
|
178
|
+
if opts["quiet"] else result["findings"]
|
|
179
|
+
if not show:
|
|
180
|
+
print(f"No pinned model id in {result['files_scanned']} files "
|
|
181
|
+
f"matched a record in {DATA['asset']} "
|
|
182
|
+
f"({len(DATA['models'])} records, built {DATA['built']}).")
|
|
183
|
+
for f in show:
|
|
184
|
+
print("")
|
|
185
|
+
extra = (f" ({len(f['records'])} platforms hold this id)"
|
|
186
|
+
if len(f["records"]) > 1 else "")
|
|
187
|
+
print(f"{f['id']} — "
|
|
188
|
+
f"{STATE_WORD.get(f['verdict'], f['verdict'])}{extra}")
|
|
189
|
+
for h in f["hits"][:10]:
|
|
190
|
+
print(f" {h['file']}:{h['line']}:{h['column']} {h['text']}")
|
|
191
|
+
if len(f["hits"]) > 10:
|
|
192
|
+
print(f" … and {len(f['hits']) - 10} more places")
|
|
193
|
+
for r in f["records"]:
|
|
194
|
+
print_record(r, opts["full_quote"])
|
|
195
|
+
print("")
|
|
196
|
+
print(f"{len(result['findings'])} pinned model id(s) in "
|
|
197
|
+
f"{result['files_scanned']} files: {len(gone)} shut down, "
|
|
198
|
+
f"{len(partial)} shut down on some platforms, "
|
|
199
|
+
f"{len(notice)} deprecated, {len(dated)} dated but still "
|
|
200
|
+
f"served.")
|
|
201
|
+
print("Only the ids above were looked for; one this scan did not name "
|
|
202
|
+
"has not been cleared.")
|
|
203
|
+
|
|
204
|
+
if gone:
|
|
205
|
+
return 1
|
|
206
|
+
if opts["strict"] and partial:
|
|
207
|
+
return 1
|
|
208
|
+
return 0
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
if __name__ == "__main__":
|
|
212
|
+
raise SystemExit(main())
|