model-pin-check 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,27 @@
1
+ CC0 1.0 Universal — Public Domain Dedication
2
+
3
+ The person who associated a work with this deed has dedicated the work to the
4
+ public domain by waiving all of his or her rights to the work worldwide under
5
+ copyright law, including all related and neighboring rights, to the extent
6
+ allowed by law.
7
+
8
+ You can copy, modify, distribute and perform the work, even for commercial
9
+ purposes, all without asking permission.
10
+
11
+ Full text: https://creativecommons.org/publicdomain/zero/1.0/legalcode
12
+
13
+ ---
14
+
15
+ This dedication covers the compilation and the code in this package.
16
+
17
+ The underlying facts are taken from third-party sources, each of which is cited
18
+ by URL in the shipped data alongside the verbatim quote the value was read from:
19
+
20
+ * PyTorch's RELEASE.md and .ci/manywheel/build_cuda.sh — BSD-3-Clause
21
+ (pytorch/pytorch LICENSE)
22
+ * NVIDIA's CUDA Toolkit release notes and CUDA GPUs page — vendor technical
23
+ documentation
24
+
25
+ Facts are not copyrightable. What is reproduced from those sources is a fact
26
+ plus a short attributed quote linking back to it; neither document is
27
+ reproduced.
@@ -0,0 +1,144 @@
1
+ Metadata-Version: 2.4
2
+ Name: model-pin-check
3
+ Version: 0.1.0
4
+ Summary: Find the AI model ids your Python repo pins and say whether the provider has already shut each one down, with the provider's own sentence as evidence
5
+ Author: referencesource.org
6
+ License: CC0-1.0
7
+ Project-URL: Homepage, https://referencesource.org/ai-model-deprecation-and-retirement/
8
+ Project-URL: Source, https://referencesource.org/ai-model-deprecation-and-retirement/
9
+ Keywords: llm,openai,anthropic,gemini,bedrock,azure-openai,deprecation,model-retirement,eol,ci,pre-commit,lint
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Topic :: Software Development :: Quality Assurance
14
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
15
+ Requires-Python: >=3.8
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+ Dynamic: license-file
19
+
20
+ # model-pin-check
21
+
22
+ Find the AI model ids your repo pins and say, for each one, whether the
23
+ provider has already shut it down — with the provider's own sentence and the
24
+ page it came from, so you can check the answer rather than trust it.
25
+
26
+ ```
27
+ $ model-pin-check
28
+
29
+ claude-3-5-sonnet-20240620 — SHUT DOWN
30
+ .env:1:17 ANTHROPIC_MODEL=claude-3-5-sonnet-20240620
31
+ src/llm.py:4:13 MODEL = "claude-3-5-sonnet-20240620"
32
+ Anthropic — Anthropic API / Claude Platform on AWS / Microsoft Foundry
33
+ status: Retired shutdown: October 28, 2025
34
+ the provider names as replacement: claude-sonnet-5-5
35
+ "| October 28, 2025 | `claude-3-5-sonnet-20240620` | `claude-sonnet-5-5` |"
36
+ https://platform.claude.com/docs/en/about-claude/model-deprecations
37
+ our record: https://referencesource.org/ai-model-deprecation-and-retirement/claude-3-5-sonnet-20240620/ last checked 2026-11-11
38
+
39
+ 1 pinned model id(s) in 2 files: 1 shut down, 0 shut down on some platforms,
40
+ 0 deprecated, 0 dated but still served.
41
+ Only the ids above were looked for; one this scan did not name has not been
42
+ cleared.
43
+ $ echo $?
44
+ 1
45
+ ```
46
+
47
+ Exit 1 means at least one pinned id has been shut down by every provider we
48
+ hold a record for: that call will not come back. A model that is merely
49
+ deprecated with a date still in the future is listed and exits 0, because it
50
+ still answers and breaking your build over it would be wrong.
51
+
52
+ ## Install
53
+
54
+ ```
55
+ pip install model-pin-check # or: pipx install model-pin-check
56
+ model-pin-check
57
+ ```
58
+
59
+ As a `pre-commit` hook:
60
+
61
+ ```yaml
62
+ repos:
63
+ - repo: https://github.com/referencesource/model-pin-check
64
+ rev: v0.1.0
65
+ hooks:
66
+ - id: model-pin-check
67
+ ```
68
+
69
+ As a library:
70
+
71
+ ```python
72
+ from model_pin_check import scan, lookup
73
+
74
+ result = scan(["src"])
75
+ for f in result["findings"]:
76
+ print(f["id"], f["verdict"], len(f["hits"]), "places")
77
+
78
+ hit = lookup("gpt-4-32k") # None when we hold no record for the id
79
+ ```
80
+
81
+ No dependencies, and it makes no network calls: the records ship inside the
82
+ package, so it works in a sealed CI job and in an air-gapped build.
83
+
84
+ ## What it scans
85
+
86
+ Python first — `.py .pyi .ipynb` — then the rest of a Python service's repo:
87
+ `.json .jsonc .yaml .yml .toml .ini .cfg .env .tf .hcl`, `.js .ts .tsx .go
88
+ .rb .java .rs .php .cs .sh`, and extensionless or suffixed names like
89
+ `Dockerfile`, `Dockerfile.prod`, `Makefile`, `Procfile` and `Jenkinsfile`,
90
+ matched without regard to case.
91
+
92
+ Your own dotted directories are read too, because that is where a model id
93
+ often is: `.github/workflows/`, `.devcontainer/`, `.streamlit/`, `.cursor/`,
94
+ `.claude/`, `.chainlit/`, `.gitlab/`. Caches, build output and somebody
95
+ else's code are skipped — `.git`, `.venv`, `venv`, `__pycache__`,
96
+ `site-packages`, `.tox`, `node_modules`, `dist`, `build`, `.terraform` and
97
+ friends — because somebody else's pin is not your pin.
98
+
99
+ `--include-docs` adds `.md .rst .txt .adoc`. Those are prose mentions, not
100
+ pins, which is why they are off by default.
101
+
102
+ ## What it will not tell you
103
+
104
+ * **An id it did not name has not been cleared.** The scan reports what it
105
+ found in the files it opened, and prints how many files that was. A clean
106
+ run over nothing is not a clean bill of health.
107
+ * **An id we hold no record for is unknown to us, not supported.**
108
+ `--model some-new-id` says so in those words, with the number of records
109
+ held and the date they were built, so you can see the denominator.
110
+ * **Gone on one platform is not gone.** The same id is served by several
111
+ platforms on different dates: `gpt-4o` has one retirement date on Azure and
112
+ another on OpenAI's own API. "SHUT DOWN" is reported only when every record
113
+ for the id says so; otherwise you get "shut down on some platforms" and the
114
+ rows that disagree. `--strict` exits 1 for that case too.
115
+ * **A date that can be read two ways is read the later way.** Providers print
116
+ retirement dates every way there is, and `3/9/2026` is March 9 to an
117
+ American vendor and 3 September to a European one. Both readings are
118
+ printed and the later one decides, so a model is never called gone early.
119
+ * **It changes no files.** The npm twin can rewrite a dead pin to the
120
+ successor the provider names (`npx model-pin-check --fix`). That half is
121
+ not ported here yet: a wrong rewrite edits your code, and the rules that
122
+ make it safe deserve their own tests rather than a hurried port. `--fix`
123
+ here exits 2 and says this.
124
+
125
+ ## Where the data comes from
126
+
127
+ Every row was read off a provider's own deprecation or lifecycle page and
128
+ kept with the sentence it was read from, the page URL, and the date it was
129
+ last re-checked. The records are re-verified on a schedule and anything found
130
+ wrong is fixed or pulled.
131
+
132
+ The compilation: <https://referencesource.org/ai-model-deprecation-and-retirement/>
133
+
134
+ Facts and a short attributed quote from each provider's own documentation;
135
+ this compilation CC0-1.0. This is not a provider's own tool and is not
136
+ affiliated with any of them. Automated: built and maintained by a program.
137
+
138
+ ## The npm twin
139
+
140
+ The same records, the same verdicts, the same data file, for repos that have
141
+ Node: `npx model-pin-check`. It additionally rewrites dead pins. This Python
142
+ build exists because a `pip install` or a `pre-commit` hook is how a Python
143
+ repository installs a check, and plenty of Python CI jobs have no Node at
144
+ all.
@@ -0,0 +1,125 @@
1
+ # model-pin-check
2
+
3
+ Find the AI model ids your repo pins and say, for each one, whether the
4
+ provider has already shut it down — with the provider's own sentence and the
5
+ page it came from, so you can check the answer rather than trust it.
6
+
7
+ ```
8
+ $ model-pin-check
9
+
10
+ claude-3-5-sonnet-20240620 — SHUT DOWN
11
+ .env:1:17 ANTHROPIC_MODEL=claude-3-5-sonnet-20240620
12
+ src/llm.py:4:13 MODEL = "claude-3-5-sonnet-20240620"
13
+ Anthropic — Anthropic API / Claude Platform on AWS / Microsoft Foundry
14
+ status: Retired shutdown: October 28, 2025
15
+ the provider names as replacement: claude-sonnet-5-5
16
+ "| October 28, 2025 | `claude-3-5-sonnet-20240620` | `claude-sonnet-5-5` |"
17
+ https://platform.claude.com/docs/en/about-claude/model-deprecations
18
+ our record: https://referencesource.org/ai-model-deprecation-and-retirement/claude-3-5-sonnet-20240620/ last checked 2026-11-11
19
+
20
+ 1 pinned model id(s) in 2 files: 1 shut down, 0 shut down on some platforms,
21
+ 0 deprecated, 0 dated but still served.
22
+ Only the ids above were looked for; one this scan did not name has not been
23
+ cleared.
24
+ $ echo $?
25
+ 1
26
+ ```
27
+
28
+ Exit 1 means at least one pinned id has been shut down by every provider we
29
+ hold a record for: that call will not come back. A model that is merely
30
+ deprecated with a date still in the future is listed and exits 0, because it
31
+ still answers and breaking your build over it would be wrong.
32
+
33
+ ## Install
34
+
35
+ ```
36
+ pip install model-pin-check # or: pipx install model-pin-check
37
+ model-pin-check
38
+ ```
39
+
40
+ As a `pre-commit` hook:
41
+
42
+ ```yaml
43
+ repos:
44
+ - repo: https://github.com/referencesource/model-pin-check
45
+ rev: v0.1.0
46
+ hooks:
47
+ - id: model-pin-check
48
+ ```
49
+
50
+ As a library:
51
+
52
+ ```python
53
+ from model_pin_check import scan, lookup
54
+
55
+ result = scan(["src"])
56
+ for f in result["findings"]:
57
+ print(f["id"], f["verdict"], len(f["hits"]), "places")
58
+
59
+ hit = lookup("gpt-4-32k") # None when we hold no record for the id
60
+ ```
61
+
62
+ No dependencies, and it makes no network calls: the records ship inside the
63
+ package, so it works in a sealed CI job and in an air-gapped build.
64
+
65
+ ## What it scans
66
+
67
+ Python first — `.py .pyi .ipynb` — then the rest of a Python service's repo:
68
+ `.json .jsonc .yaml .yml .toml .ini .cfg .env .tf .hcl`, `.js .ts .tsx .go
69
+ .rb .java .rs .php .cs .sh`, and extensionless or suffixed names like
70
+ `Dockerfile`, `Dockerfile.prod`, `Makefile`, `Procfile` and `Jenkinsfile`,
71
+ matched without regard to case.
72
+
73
+ Your own dotted directories are read too, because that is where a model id
74
+ often is: `.github/workflows/`, `.devcontainer/`, `.streamlit/`, `.cursor/`,
75
+ `.claude/`, `.chainlit/`, `.gitlab/`. Caches, build output and somebody
76
+ else's code are skipped — `.git`, `.venv`, `venv`, `__pycache__`,
77
+ `site-packages`, `.tox`, `node_modules`, `dist`, `build`, `.terraform` and
78
+ friends — because somebody else's pin is not your pin.
79
+
80
+ `--include-docs` adds `.md .rst .txt .adoc`. Those are prose mentions, not
81
+ pins, which is why they are off by default.
82
+
83
+ ## What it will not tell you
84
+
85
+ * **An id it did not name has not been cleared.** The scan reports what it
86
+ found in the files it opened, and prints how many files that was. A clean
87
+ run over nothing is not a clean bill of health.
88
+ * **An id we hold no record for is unknown to us, not supported.**
89
+ `--model some-new-id` says so in those words, with the number of records
90
+ held and the date they were built, so you can see the denominator.
91
+ * **Gone on one platform is not gone.** The same id is served by several
92
+ platforms on different dates: `gpt-4o` has one retirement date on Azure and
93
+ another on OpenAI's own API. "SHUT DOWN" is reported only when every record
94
+ for the id says so; otherwise you get "shut down on some platforms" and the
95
+ rows that disagree. `--strict` exits 1 for that case too.
96
+ * **A date that can be read two ways is read the later way.** Providers print
97
+ retirement dates every way there is, and `3/9/2026` is March 9 to an
98
+ American vendor and 3 September to a European one. Both readings are
99
+ printed and the later one decides, so a model is never called gone early.
100
+ * **It changes no files.** The npm twin can rewrite a dead pin to the
101
+ successor the provider names (`npx model-pin-check --fix`). That half is
102
+ not ported here yet: a wrong rewrite edits your code, and the rules that
103
+ make it safe deserve their own tests rather than a hurried port. `--fix`
104
+ here exits 2 and says this.
105
+
106
+ ## Where the data comes from
107
+
108
+ Every row was read off a provider's own deprecation or lifecycle page and
109
+ kept with the sentence it was read from, the page URL, and the date it was
110
+ last re-checked. The records are re-verified on a schedule and anything found
111
+ wrong is fixed or pulled.
112
+
113
+ The compilation: <https://referencesource.org/ai-model-deprecation-and-retirement/>
114
+
115
+ Facts and a short attributed quote from each provider's own documentation;
116
+ this compilation CC0-1.0. This is not a provider's own tool and is not
117
+ affiliated with any of them. Automated: built and maintained by a program.
118
+
119
+ ## The npm twin
120
+
121
+ The same records, the same verdicts, the same data file, for repos that have
122
+ Node: `npx model-pin-check`. It additionally rewrites dead pins. This Python
123
+ build exists because a `pip install` or a `pre-commit` hook is how a Python
124
+ repository installs a check, and plenty of Python CI jobs have no Node at
125
+ all.
@@ -0,0 +1,36 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "model-pin-check"
7
+ version = "0.1.0"
8
+ description = "Find the AI model ids your Python repo pins and say whether the provider has already shut each one down, with the provider's own sentence as evidence"
9
+ readme = "README.md"
10
+ requires-python = ">=3.8"
11
+ license = { text = "CC0-1.0" }
12
+ authors = [{ name = "referencesource.org" }]
13
+ keywords = ["llm", "openai", "anthropic", "gemini", "bedrock", "azure-openai",
14
+ "deprecation", "model-retirement", "eol", "ci", "pre-commit",
15
+ "lint"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Intended Audience :: Developers",
19
+ "Programming Language :: Python :: 3",
20
+ "Topic :: Software Development :: Quality Assurance",
21
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
22
+ ]
23
+ dependencies = []
24
+
25
+ [project.urls]
26
+ Homepage = "https://referencesource.org/ai-model-deprecation-and-retirement/"
27
+ Source = "https://referencesource.org/ai-model-deprecation-and-retirement/"
28
+
29
+ [project.scripts]
30
+ model-pin-check = "model_pin_check.__main__:main"
31
+
32
+ [tool.setuptools.packages.find]
33
+ where = ["src"]
34
+
35
+ [tool.setuptools.package-data]
36
+ model_pin_check = ["data.json"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,309 @@
1
+ """model-pin-check — find the AI model ids your repository pins and say, for
2
+ each one, whether the provider has already shut it down, with the provider's
3
+ own sentence as the evidence.
4
+
5
+ This is the Python twin of the npm package of the same name, shipped because
6
+ the audience for an AI model pin check writes Python: the pins live in `.py`
7
+ files, in Airflow DAGs, in `conftest.py`, in CI steps that have `pip` and no
8
+ Node at all, and the way a Python repository installs a check like this is
9
+ `pip install` or a `pre-commit` hook, neither of which can reach npm. The
10
+ records, the data file, the verdicts and the honesty rules are the same ones
11
+ the npm package uses, built by the same `build_data.py`, so the two agree by
12
+ construction rather than by hand.
13
+
14
+ Three rules do most of the work here, and all three exist because getting them
15
+ wrong would make the tool lie:
16
+
17
+ 1. **A model id is not unique.** Records that share an id with another record
18
+ do so because the same id is served by several platforms on different
19
+ dates: `gpt-4o` is one date on Azure and another on OpenAI's own API. A
20
+ found id therefore resolves to a *set* of records, and "gone" is reported
21
+ only when every record for that id says gone. Gone on some platforms and
22
+ not others is reported as exactly that.
23
+ 2. **A bare word is not a pin.** Some real ids are single words with no digit
24
+ and no hyphen — `command`, `whisper`, `tts`, `ada`, `davinci`. Matching
25
+ those on a token boundary would flag every "git command" in a comment, so
26
+ they are reported only when the whole string literal is the id, or the line
27
+ reads like `model = "command"`.
28
+ 3. **Absence is not a clearance.** A model id this package does not hold is
29
+ not thereby supported; it is unknown to us, and every answer says so in
30
+ words. Nothing here guesses a date, a status or a successor: every value
31
+ printed came off a provider's own page with the sentence it was read from.
32
+
33
+ The library half. `__main__.py` is the command.
34
+ """
35
+ from __future__ import annotations
36
+
37
+ import json
38
+ import os
39
+ import pathlib
40
+ import re
41
+
42
+ __all__ = [
43
+ "DATA", "scan", "lookup", "verdict", "version_line", "STATE_WORD",
44
+ ]
45
+
46
+ __version__ = "0.1.0"
47
+
48
+ _HERE = pathlib.Path(__file__).resolve().parent
49
+ DATA = json.loads((_HERE / "data.json").read_text(encoding="utf-8"))
50
+
51
+ # Directories never worth walking: not your code, or build output. Every
52
+ # dotted directory that is NOT in here is walked, because a model id is
53
+ # routinely pinned in `.devcontainer/`, `.streamlit/`, `.cursor/`, `.claude/`,
54
+ # `.chainlit/` and `.gitlab/`. So this list names the dotted directories that
55
+ # hold a cache, a build, or somebody else's code.
56
+ SKIP_DIRS = frozenset({
57
+ ".git", "node_modules", "dist", "build", "out", "target", "vendor",
58
+ ".venv", "venv", "env", "__pycache__", ".mypy_cache", ".pytest_cache",
59
+ ".next", ".nuxt", ".svelte-kit", "coverage", "site-packages", ".tox",
60
+ ".gradle", ".idea", ".terraform", "bower_components", ".cache",
61
+ ".svn", ".hg", ".direnv", ".ruff_cache", ".ipynb_checkpoints", ".yarn",
62
+ ".pnpm-store", ".pnpm", ".bundle", ".m2", ".nuget", ".conda", ".turbo",
63
+ ".parcel-cache", ".serverless", ".vercel", ".netlify", ".angular",
64
+ ".dart_tool", ".pub-cache", ".stack-work", ".cargo", ".rustup", ".pyenv",
65
+ ".nvm", ".npm", ".gem", ".sass-cache", ".eggs", ".nox", ".dvc",
66
+ ".terragrunt-cache", ".history", ".cpanm", ".local",
67
+ })
68
+
69
+ # Where a hardcoded model id actually lives. Python first, because that is who
70
+ # installs this, but a Python repository is not only Python: the same service
71
+ # pins the id again in its Dockerfile, its Helm values and its front end.
72
+ CODE_EXT = frozenset({
73
+ ".py", ".pyi", ".pyw", ".ipynb",
74
+ ".js", ".mjs", ".cjs", ".jsx", ".ts", ".tsx", ".go", ".rb", ".java",
75
+ ".kt", ".kts", ".rs", ".php", ".cs", ".swift", ".scala", ".sh", ".bash",
76
+ ".zsh", ".pl", ".lua", ".r", ".dart", ".ex", ".exs", ".clj", ".c", ".cc",
77
+ ".cpp", ".h", ".hpp",
78
+ ".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf",
79
+ ".properties", ".env", ".tf", ".tfvars", ".gradle", ".xml", ".plist",
80
+ ".vue", ".svelte", ".astro", ".hcl", ".jsonc", ".json5", ".groovy",
81
+ })
82
+ # Extensionless files that are still config or code, matched without regard to
83
+ # case because a repository may hold `dockerfile` or `makefile`.
84
+ NAMED_LOWER = frozenset({
85
+ "dockerfile", "makefile", ".env", "procfile", "containerfile",
86
+ "jenkinsfile", "justfile", "rakefile", "vagrantfile", "brewfile",
87
+ })
88
+ # A file whose name begins with one of these and then a suffix:
89
+ # `Dockerfile.prod`, `Containerfile.dev`, `Jenkinsfile.release`. Its extension
90
+ # is `.prod`, so the extension list can never catch it.
91
+ NAMED_PREFIX = ("dockerfile.", "containerfile.", "jenkinsfile.", "makefile.")
92
+
93
+ # Added by include_docs: prose mentions a model id without pinning it.
94
+ DOC_EXT = frozenset({".md", ".mdx", ".markdown", ".rst", ".txt", ".adoc"})
95
+
96
+ MAX_BYTES = 1024 * 1024
97
+ MAX_LINE = 4000 # longer than this is a minified bundle, not source
98
+
99
+ STATE_WORD = {
100
+ "gone": "SHUT DOWN",
101
+ "gone_somewhere": "shut down on some platforms",
102
+ "on_notice": "on notice — the provider has deprecated it",
103
+ "dated": "still served, with a retirement date",
104
+ "live": "still listed",
105
+ }
106
+
107
+ _TOKEN_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]{1,90}")
108
+ _BARE_RE = re.compile(r"[-._/:\s]")
109
+ _KEY_RE = re.compile(
110
+ r"(model|model_id|model_name|modelid|deployment|engine)\s*[:=]\s*$",
111
+ re.I)
112
+ _FLAG_RE = re.compile(r"--model(?:-id|-name)?[= ]$", re.I)
113
+ _QUOTE_TAIL_RE = re.compile(r"['\"`]\s*$")
114
+
115
+
116
+ def named_file(name: str) -> bool:
117
+ """Is this file worth opening, by its name alone?"""
118
+ low = name.lower()
119
+ if low in NAMED_LOWER or low.startswith(".env"):
120
+ return True
121
+ return any(low.startswith(p) for p in NAMED_PREFIX)
122
+
123
+
124
+ def _index(models):
125
+ """model_id -> records, keyed lowercase because providers vary the case."""
126
+ by_id: dict[str, list[dict]] = {}
127
+ for m in models:
128
+ by_id.setdefault(m["model_id"].lower(), []).append(m)
129
+ return by_id
130
+
131
+
132
+ INDEX = _index(DATA["models"])
133
+
134
+
135
+ def is_bare_word(model_id: str) -> bool:
136
+ """An id that needs the strict context test: one word, no digit, no dash."""
137
+ return not any(c.isdigit() for c in model_id) \
138
+ and not _BARE_RE.search(model_id)
139
+
140
+
141
+ def candidates(token: str):
142
+ """The ways one token in a file might be carrying a model id.
143
+
144
+ `us.anthropic.claude-3-5-sonnet-20240620-v1:0` is a cross-region Bedrock
145
+ alias of a Bedrock id; `models/gemini-2.5-flash` and `openai/gpt-4o` are
146
+ the same id behind a namespace. Bedrock's own ids contain dots and a `:0`
147
+ tail, so the whole token is always tried first and never stripped away.
148
+ """
149
+ out = [token]
150
+ if "/" in token:
151
+ parts = [p for p in token.split("/") if p]
152
+ if parts:
153
+ out.append(parts[-1])
154
+ out.extend(parts)
155
+ m = re.fullmatch(r"(?:us|eu|apac|us-gov)\.(.+)", token)
156
+ if m:
157
+ out.append(m.group(1))
158
+ return out
159
+
160
+
161
+ def looks_pinned(line: str, matched: str, start: int) -> bool:
162
+ """Is this match in a position where a pin lives, rather than in prose?"""
163
+ before = line[:start]
164
+ after = line[start + len(matched):]
165
+ if before[-1:] in ("'", '"', "`") and after[:1] in ("'", '"', "`"):
166
+ return True # the whole literal is the id
167
+ stripped = _QUOTE_TAIL_RE.sub("", before)
168
+ if _KEY_RE.search(stripped): # model: command / "model": command
169
+ return True
170
+ return bool(_FLAG_RE.search(before))
171
+
172
+
173
+ def verdict(records) -> str:
174
+ """What the records say about one id, across every platform holding it.
175
+
176
+ `gone` only when nowhere still serves it. "Deprecated" and "Legacy" are
177
+ the provider's own notice; a model still marked GA or Active with a
178
+ retirement date in the future is not on notice and must not be described
179
+ as if it were.
180
+ """
181
+ states = [r.get("state") for r in records]
182
+ gone = [s for s in states if s == "gone"]
183
+ if records and len(gone) == len(records):
184
+ return "gone"
185
+ if gone:
186
+ return "gone_somewhere"
187
+ if "on_notice" in states:
188
+ return "on_notice"
189
+ if "dated" in states:
190
+ return "dated"
191
+ return "live"
192
+
193
+
194
+ def scan_line(line: str):
195
+ """One line of one file: every model id it carries, with its position."""
196
+ hits = []
197
+ for m in _TOKEN_RE.finditer(line):
198
+ token = m.group(0)
199
+ for cand in candidates(token):
200
+ recs = INDEX.get(cand.lower())
201
+ if not recs:
202
+ continue
203
+ at = line.lower().find(cand.lower(), m.start())
204
+ if at < 0:
205
+ continue
206
+ if is_bare_word(recs[0]["model_id"]) \
207
+ and not looks_pinned(line, cand, at):
208
+ continue
209
+ hits.append({
210
+ "matched": cand, "id": recs[0]["model_id"], "records": recs,
211
+ "column": at + 1, "token": token, "token_start": m.start(),
212
+ })
213
+ break # one id per token; the longest form wins
214
+ return hits
215
+
216
+
217
+ def walk(root, include_docs: bool = False, max_bytes: int = MAX_BYTES):
218
+ """Walk a directory, yielding the files worth reading."""
219
+ for dirpath, dirnames, filenames in os.walk(root):
220
+ dirnames[:] = [d for d in dirnames
221
+ if d not in SKIP_DIRS and d.lower() not in SKIP_DIRS]
222
+ for name in sorted(filenames):
223
+ full = os.path.join(dirpath, name)
224
+ ext = os.path.splitext(name)[1].lower()
225
+ ok = ext in CODE_EXT or named_file(name) \
226
+ or (include_docs and ext in DOC_EXT)
227
+ if not ok:
228
+ continue
229
+ try:
230
+ if os.path.getsize(full) > max_bytes:
231
+ continue
232
+ except OSError:
233
+ continue
234
+ yield full
235
+
236
+
237
+ def scan(paths=(".",), include_docs: bool = False,
238
+ max_bytes: int = MAX_BYTES) -> dict:
239
+ """Scan one or more paths. Findings grouped by model id.
240
+
241
+ The return carries `files_scanned` as well as the findings, because the
242
+ denominator is part of the answer: "nothing found" means nothing was found
243
+ in *these* files, and a scan that opened no file at all must never read as
244
+ a clean bill of health.
245
+ """
246
+ found: dict[str, dict] = {}
247
+ files: list[str] = []
248
+ for p in paths:
249
+ if os.path.isdir(p):
250
+ files.extend(walk(p, include_docs=include_docs,
251
+ max_bytes=max_bytes))
252
+ elif os.path.exists(p):
253
+ files.append(p)
254
+ else:
255
+ raise FileNotFoundError(f"cannot read {p}")
256
+ for f in files:
257
+ try:
258
+ text = pathlib.Path(f).read_text(encoding="utf-8", errors="replace")
259
+ except OSError:
260
+ continue
261
+ if "\x00" in text:
262
+ continue # binary
263
+ for i, line in enumerate(text.splitlines(), start=1):
264
+ if len(line) > MAX_LINE:
265
+ continue
266
+ for hit in scan_line(line):
267
+ entry = found.get(hit["id"])
268
+ if entry is None:
269
+ entry = found[hit["id"]] = {
270
+ "id": hit["id"],
271
+ "records": hit["records"],
272
+ "verdict": verdict(hit["records"]),
273
+ "hits": [],
274
+ }
275
+ entry["hits"].append({
276
+ "file": f, "line": i, "column": hit["column"],
277
+ "matched": hit["matched"], "token": hit["token"],
278
+ "text": line.strip()[:200],
279
+ })
280
+ order = {"gone": 0, "gone_somewhere": 1, "on_notice": 2, "dated": 3,
281
+ "live": 4}
282
+ findings = sorted(found.values(),
283
+ key=lambda f: (order.get(f["verdict"], 9), f["id"]))
284
+ return {"files_scanned": len(files), "findings": findings}
285
+
286
+
287
+ def lookup(model_id) -> dict | None:
288
+ """Look one id up without scanning anything. None when we hold no record.
289
+
290
+ None is "we do not hold this", never "this is fine": the caller is told so
291
+ in those words by the command, and a library caller should say the same.
292
+ """
293
+ recs = INDEX.get(str(model_id or "").lower())
294
+ if not recs:
295
+ return None
296
+ return {"id": recs[0]["model_id"], "records": recs,
297
+ "verdict": verdict(recs)}
298
+
299
+
300
+ def absence_note(model_id) -> str:
301
+ """The sentence that goes with a miss. The denominator is in it on purpose."""
302
+ return (f"no record for {model_id!r} in {DATA['asset']} "
303
+ f"({len(DATA['models'])} records, built {DATA['built']}) — "
304
+ f"absence here is not evidence it is supported")
305
+
306
+
307
+ def version_line() -> str:
308
+ return (f"model-pin-check {__version__}, data built {DATA['built']} "
309
+ f"({len(DATA['models'])} records)")