amagi-model-sunset 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3 @@
1
+ """amagi-model-sunset: find LLM model ids that a provider is retiring."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
@@ -0,0 +1,135 @@
1
+ """amagi-model-sunset command line."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import datetime as dt
6
+ import json
7
+ import os
8
+ import sys
9
+ from collections import OrderedDict
10
+ from typing import List, Optional
11
+
12
+ from . import __version__
13
+ from .core import Hit, failing, load_data, scan_path
14
+
15
+ ORDER = {"retired": 0, "retiring": 1, "unknown-date": 2, "active": 3}
16
+
17
+
18
+ def _label(h: Hit) -> str:
19
+ e = h.entry
20
+ if e.status == "retired":
21
+ return "RETIRED %s (%dd ago)" % (e.retires, -e.days_left)
22
+ if e.status == "retiring":
23
+ return "RETIRES %s (in %dd)" % (e.retires, e.days_left)
24
+ if e.status == "unknown-date":
25
+ return "DEPRECATED, no date"
26
+ return "ok"
27
+
28
+
29
+ def _sorted(hits: List[Hit]) -> List[Hit]:
30
+ return sorted(hits, key=lambda h: (ORDER[h.entry.status], h.entry.days_left if h.entry.days_left is not None else 0,
31
+ h.path, h.line))
32
+
33
+
34
+ def render_text(hits: List[Hit], show_all: bool, failed: List[Hit], within: int, scanned: str) -> str:
35
+ shown = [h for h in _sorted(hits) if show_all or h.entry.status != "active"]
36
+ out = []
37
+ for h in shown:
38
+ e = h.entry
39
+ flag = "" if e.verified else " [unverified date]"
40
+ out.append("%s:%d %s -> %s%s" % (h.path, h.line, h.text, _label(h), flag))
41
+ if e.status != "active" and e.replacement:
42
+ out.append(" suggest: %s" % e.replacement)
43
+ elif e.status == "active" and e.not_before:
44
+ out.append(" no retirement announced; provider says not sooner than %s" % e.not_before)
45
+ uniq = OrderedDict()
46
+ for h in hits:
47
+ uniq.setdefault((h.entry.status, h.model), 0)
48
+ uniq[(h.entry.status, h.model)] += 1
49
+ n = lambda s: sum(1 for (st, _m) in uniq if st == s) # noqa: E731
50
+ out.append("")
51
+ srcs = OrderedDict()
52
+ for h in _sorted(hits):
53
+ if h.entry.status != "active":
54
+ srcs.setdefault(h.entry.source, h.entry.checked)
55
+ for src, checked in srcs.items():
56
+ out.append("source: %s (checked %s)" % (src, checked))
57
+ out.append("scanned %s: %d model id(s) in %d place(s): %d retired, %d retiring, %d deprecated without a date, %d ok"
58
+ % (scanned, len(uniq), len(hits), n("retired"), n("retiring"), n("unknown-date"), n("active")))
59
+ if failed:
60
+ out.append("FAIL: %d place(s) use a model that is retired or retires within %d days" % (len(failed), within))
61
+ else:
62
+ out.append("PASS: nothing retired or retiring within %d days" % within)
63
+ return "\n".join(out)
64
+
65
+
66
+ def render_md(hits: List[Hit], show_all: bool, failed: List[Hit], within: int) -> str:
67
+ shown = [h for h in _sorted(hits) if show_all or h.entry.status != "active"]
68
+ out = ["| status | file | model | replacement | source |", "|---|---|---|---|---|"]
69
+ for h in shown:
70
+ e = h.entry
71
+ out.append("| %s | `%s:%d` | `%s` | %s | %s |" % (_label(h), h.path, h.line, h.text, e.replacement or "-", e.source))
72
+ out.append("")
73
+ out.append("%s: %d place(s) retired or retiring within %d days." % ("FAIL" if failed else "PASS", len(failed), within))
74
+ return "\n".join(out)
75
+
76
+
77
+ def build_parser() -> argparse.ArgumentParser:
78
+ p = argparse.ArgumentParser(
79
+ prog="amagi-model-sunset",
80
+ description="Find hard-coded LLM model ids in a repo and report each one's announced retirement date.",
81
+ epilog="Exit codes: 0 clean, 1 a model is retired or retires within --fail-within days, 2 usage error.",
82
+ )
83
+ p.add_argument("path", nargs="?", default=".", help="directory or file to scan (default: current directory)")
84
+ p.add_argument("--fail-within", type=int, default=30, metavar="DAYS",
85
+ help="exit 1 if a model is already retired or retires within DAYS days (default 30; 0 = only retired)")
86
+ p.add_argument("--format", choices=["text", "json", "md"], default="text", help="output format")
87
+ p.add_argument("--all", action="store_true", help="also list models with no retirement announced")
88
+ p.add_argument("--exclude", action="append", default=[], metavar="GLOB",
89
+ help="skip files matching this glob (repeatable)")
90
+ p.add_argument("--data", metavar="FILE", help="use this models.json instead of the bundled one")
91
+ p.add_argument("--today", metavar="YYYY-MM-DD", help="pretend today is this date (for CI reproducibility)")
92
+ p.add_argument("--list-models", action="store_true", help="print every model id in the data file and exit")
93
+ p.add_argument("--version", action="version", version="amagi-model-sunset " + __version__)
94
+ return p
95
+
96
+
97
+ def main(argv: Optional[List[str]] = None) -> int:
98
+ args = build_parser().parse_args(argv)
99
+ try:
100
+ today = dt.date.fromisoformat(args.today) if args.today else dt.date.today()
101
+ except ValueError:
102
+ print("error: --today must be YYYY-MM-DD", file=sys.stderr)
103
+ return 2
104
+ try:
105
+ data = load_data(args.data)
106
+ except (OSError, ValueError, KeyError) as exc:
107
+ print("error: cannot read data file: %s" % exc, file=sys.stderr)
108
+ return 2
109
+ if args.list_models:
110
+ for key in sorted(data.by_id):
111
+ rec = data.by_id[key]
112
+ print("%-45s %-10s %s" % (key, rec["provider"], rec.get("retires") or "-"))
113
+ return 0
114
+ if not os.path.exists(args.path):
115
+ print("error: %s does not exist" % args.path, file=sys.stderr)
116
+ return 2
117
+ if args.fail_within < 0:
118
+ print("error: --fail-within must be 0 or more", file=sys.stderr)
119
+ return 2
120
+ hits = scan_path(args.path, data, today, args.exclude)
121
+ failed = failing(hits, args.fail_within)
122
+ if args.format == "json":
123
+ print(json.dumps({"scanned": args.path, "today": today.isoformat(), "data_checked": data.checked,
124
+ "fail_within_days": args.fail_within, "failed": len(failed),
125
+ "hits": [h.to_dict() for h in _sorted(hits)]}, indent=2))
126
+ elif args.format == "md":
127
+ print(render_md(hits, args.all, failed, args.fail_within))
128
+ else:
129
+ print(render_text(hits, args.all, failed, args.fail_within, args.path))
130
+ print("data checked %s against the providers' own pages; run with --format json for sources" % data.checked)
131
+ return 1 if failed else 0
132
+
133
+
134
+ if __name__ == "__main__":
135
+ sys.exit(main())
@@ -0,0 +1,195 @@
1
+ """Find hard-coded LLM model ids in a source tree and look up their retirement dates."""
2
+ from __future__ import annotations
3
+
4
+ import datetime as dt
5
+ import fnmatch
6
+ import json
7
+ import os
8
+ import re
9
+ from dataclasses import dataclass, field
10
+ from importlib import resources
11
+ from typing import Dict, Iterable, Iterator, List, Optional
12
+
13
+ SKIP_DIRS = {
14
+ ".git", ".hg", ".svn", "node_modules", ".venv", "venv", "env", "__pycache__", ".tox", ".mypy_cache",
15
+ ".pytest_cache", ".ruff_cache", "dist", "build", ".next", ".idea", ".vscode", "site-packages", "target",
16
+ "vendor", ".terraform",
17
+ }
18
+ BINARY_EXT = {
19
+ ".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", ".zip", ".gz", ".tgz", ".bz2", ".xz", ".7z", ".whl",
20
+ ".jar", ".class", ".so", ".dll", ".exe", ".dylib", ".pyc", ".woff", ".woff2", ".ttf", ".otf", ".mp3", ".mp4",
21
+ ".mov", ".onnx", ".gguf", ".safetensors", ".bin", ".pt", ".parquet", ".sqlite", ".db", ".lock", ".svg",
22
+ }
23
+ MAX_BYTES = 1_000_000
24
+ IGNORE_MARK = "model-sunset: ignore"
25
+
26
+ # one candidate token: letters/digits plus . _ : @ - (so ids, Bedrock ids and ARN tails stay whole)
27
+ TOKEN_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@-]*[A-Za-z0-9]")
28
+ REGION_PREFIX = re.compile(r"^(?:us-gov|us|eu|apac|jp|au|global)\.")
29
+ VENDOR_PREFIX = re.compile(r"^(?:anthropic|openai|google|meta|amazon)\.")
30
+ BEDROCK_SUFFIX = re.compile(r"(?:-v\d+)?(?::\d+)?$")
31
+
32
+
33
+ @dataclass
34
+ class Entry:
35
+ provider: str
36
+ id: str
37
+ status: str # retired | retiring | active | unknown-date
38
+ retires: Optional[str]
39
+ days_left: Optional[int]
40
+ replacement: str
41
+ source: str
42
+ checked: str
43
+ verified: bool
44
+ note: str = ""
45
+ not_before: Optional[str] = None
46
+
47
+
48
+ @dataclass
49
+ class Hit:
50
+ path: str
51
+ line: int
52
+ text: str # the id as written in the file
53
+ model: str # normalized id that matched the data file
54
+ entry: Entry
55
+
56
+ def to_dict(self) -> dict:
57
+ e = self.entry
58
+ return {
59
+ "path": self.path, "line": self.line, "found": self.text, "model": self.model, "provider": e.provider,
60
+ "status": e.status, "retires": e.retires, "days_left": e.days_left, "replacement": e.replacement,
61
+ "source": e.source, "checked": e.checked, "verified": e.verified, "note": e.note,
62
+ }
63
+
64
+
65
+ @dataclass
66
+ class Data:
67
+ checked: str
68
+ by_id: Dict[str, dict] = field(default_factory=dict)
69
+
70
+
71
+ def load_data(path: Optional[str] = None) -> Data:
72
+ if path:
73
+ with open(path, encoding="utf-8") as fh:
74
+ doc = json.load(fh)
75
+ else:
76
+ doc = json.loads(resources.files("amagi_model_sunset").joinpath("data/models.json").read_text(encoding="utf-8"))
77
+ data = Data(checked=doc.get("checked", ""))
78
+ for m in doc["models"]:
79
+ for mid in m["ids"]:
80
+ data.by_id[mid.lower()] = m
81
+ return data
82
+
83
+
84
+ def parse_date(s: str) -> dt.date:
85
+ return dt.date.fromisoformat(s)
86
+
87
+
88
+ def make_entry(model_id: str, rec: dict, today: dt.date) -> Entry:
89
+ retires = rec.get("retires")
90
+ if retires:
91
+ days = (parse_date(retires) - today).days
92
+ status = "retired" if days <= 0 else "retiring"
93
+ else:
94
+ days = None
95
+ status = "unknown-date" if rec.get("note", "").lower().startswith(("deprecated", "not found")) or not rec.get(
96
+ "verified", True) else "active"
97
+ return Entry(
98
+ provider=rec["provider"], id=model_id, status=status, retires=retires, days_left=days,
99
+ replacement=rec.get("replacement", ""), source=rec.get("source", ""), checked=rec.get("checked", ""),
100
+ verified=bool(rec.get("verified", True)), note=rec.get("note", ""), not_before=rec.get("not_before"),
101
+ )
102
+
103
+
104
+ def normalize(token: str) -> List[str]:
105
+ """Return candidate lookup keys for a raw token (Bedrock, Vertex, fine-tune and ARN spellings)."""
106
+ t = token.lower()
107
+ if t.startswith("ft:"):
108
+ parts = t.split(":")
109
+ t = parts[1] if len(parts) > 1 else t
110
+ keys = [t]
111
+ t2 = REGION_PREFIX.sub("", t)
112
+ t2 = VENDOR_PREFIX.sub("", t2)
113
+ keys.append(t2)
114
+ t3 = BEDROCK_SUFFIX.sub("", t2)
115
+ keys.append(t3)
116
+ for k in list(keys):
117
+ if "@" in k:
118
+ keys.append(re.sub(r"@(\d{8})$", r"-\1", k))
119
+ keys.append(k.split("@")[0])
120
+ out: List[str] = []
121
+ for k in keys:
122
+ if k and k not in out:
123
+ out.append(k)
124
+ return out
125
+
126
+
127
+ def looks_like_model(token: str) -> bool:
128
+ return any(token.lower().startswith(p) or p in token.lower() for p in (
129
+ "gpt", "claude", "gemini", "o1", "o3", "o4", "tts-", "whisper", "text-", "chatgpt", "codex", "embedding",
130
+ "imagen", "veo", "deep-research", "gemini", "ft:", "dall-e", "sora", "babbage", "davinci",
131
+ "computer-use", "antigravity"))
132
+
133
+
134
+ def scan_text(text: str, path: str, data: Data, today: dt.date) -> Iterator[Hit]:
135
+ for n, line in enumerate(text.splitlines(), 1):
136
+ if IGNORE_MARK in line:
137
+ continue
138
+ for m in TOKEN_RE.finditer(line):
139
+ raw = m.group(0)
140
+ if not looks_like_model(raw):
141
+ continue
142
+ for key in normalize(raw):
143
+ rec = data.by_id.get(key)
144
+ if rec:
145
+ yield Hit(path, n, raw, key, make_entry(key, rec, today))
146
+ break
147
+
148
+
149
+ def iter_files(root: str, excludes: Iterable[str] = ()) -> Iterator[str]:
150
+ excludes = list(excludes)
151
+ if os.path.isfile(root):
152
+ yield root
153
+ return
154
+ for dirpath, dirnames, filenames in os.walk(root):
155
+ dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS)
156
+ for fn in sorted(filenames):
157
+ full = os.path.join(dirpath, fn)
158
+ rel = os.path.relpath(full, root).replace(os.sep, "/")
159
+ if os.path.splitext(fn)[1].lower() in BINARY_EXT:
160
+ continue
161
+ if any(fnmatch.fnmatch(rel, pat) or fnmatch.fnmatch(fn, pat) for pat in excludes):
162
+ continue
163
+ try:
164
+ if os.path.getsize(full) > MAX_BYTES:
165
+ continue
166
+ except OSError:
167
+ continue
168
+ yield full
169
+
170
+
171
+ def scan_path(root: str, data: Data, today: dt.date, excludes: Iterable[str] = ()) -> List[Hit]:
172
+ hits: List[Hit] = []
173
+ base = root if os.path.isdir(root) else os.path.dirname(root) or "."
174
+ for full in iter_files(root, excludes):
175
+ try:
176
+ with open(full, "rb") as fh:
177
+ raw = fh.read()
178
+ except OSError:
179
+ continue
180
+ if b"\x00" in raw[:4096]:
181
+ continue
182
+ text = raw.decode("utf-8", errors="replace")
183
+ rel = os.path.relpath(full, base).replace(os.sep, "/")
184
+ hits.extend(scan_text(text, rel, data, today))
185
+ return hits
186
+
187
+
188
+ def failing(hits: List[Hit], within_days: int) -> List[Hit]:
189
+ """Hits that should fail CI: already retired, or retiring within `within_days`."""
190
+ bad = []
191
+ for h in hits:
192
+ e = h.entry
193
+ if e.status == "retired" or (e.status == "retiring" and e.days_left is not None and e.days_left <= within_days):
194
+ bad.append(h)
195
+ return bad