sourcelock 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1310 @@
1
+ """OIG LEIE (List of Excluded Individuals/Entities) route adapter.
2
+
3
+ Source
4
+ ------
5
+ The authority is the monthly full database CSV at
6
+ ``https://oig.hhs.gov/exclusions/downloadables/UPDATED.csv`` (~15.5 MB,
7
+ ~84k records, replaced in place by the 10th of each month; the data month is
8
+ the month before the file's ``Last-Modified``). There is no API and OIG states
9
+ there are no plans for one; file downloads are the only machine interface.
10
+ The working table is built at call time from that file and cached in-process,
11
+ then revalidated on the ``SNAPSHOT_REVALIDATE_AFTER`` cadence against the
12
+ upstream ``Last-Modified``; nothing about the full file is vendored except a
13
+ 14-row sample (``_leie_sample.csv``, provenance in
14
+ ``tests/fixtures/leie/provenance.json``).
15
+
16
+ Fail-closed answers
17
+ -------------------
18
+ An exclusion screen has one dangerous direction: reporting a person clear.
19
+ Two conditions make a negative answer unearned, and in both the tools answer
20
+ ``screen_result: "indeterminate"`` rather than a miss, omit the field a caller
21
+ would read as a verdict (``excluded_in_snapshot``, ``count``), and add a
22
+ non-claim naming the gap:
23
+
24
+ * any row the CSV parser could not read (``MAX_UNPARSED_ROWS``) — that row was
25
+ never compared, so "not found" is not a finding;
26
+ * a matched record carrying a REINDATE. OIG removes reinstated parties from
27
+ the full file rather than dating them, so a populated one means the file no
28
+ longer means what this adapter was built against. The date is surfaced and
29
+ neither "excluded" nor "not excluded" is claimed; there are no reinstatement
30
+ semantics here to claim either with.
31
+
32
+ Verified file facts (empirical, 2026-08-01; dossier addendum §10)
33
+ -----------------------------------------------------------------
34
+ * Pure 7-bit ASCII, no BOM, CRLF line endings — across UPDATED.csv and the
35
+ monthly supplements. The parser decodes ``utf-8-sig`` and falls back to
36
+ ``latin-1`` with a receipt warning rather than crash on a future stray byte.
37
+ * Two CSV stylings exist: fully-quoted with ``00000000`` date-nulls
38
+ (UPDATED.csv, current supplements) and unquoted with bare ``0`` nulls for
39
+ dates AND NPIs (older supplements, e.g. 2501excl.csv). Null normalisation
40
+ treats ``""``/``"0"``/``"00000000"`` (dates) and ``""``/``"0"``/
41
+ ``"0000000000"`` (NPI) as null.
42
+ * Drift is keyed on a canonicalised content hash (parsed rows, nulls
43
+ normalised, order-independent), never on raw bytes, so an in-place repost
44
+ or a styling change alone never cries wolf. Receipts still carry the raw
45
+ ``sha256`` of the exact bytes fetched.
46
+
47
+ Matching contract (v1 decision)
48
+ -------------------------------
49
+ ``leie.check_npi`` is the deterministic primary tool: NPI-exact against the
50
+ snapshot. Only ~10.5% of LEIE records carry an NPI, so an NPI miss never
51
+ clears a provider. ``leie.candidate_search`` is name-based and explicitly
52
+ candidate-only: the bulk file has no SSN/EIN (Privacy Act), so identity is
53
+ verified only by a human at ``https://exclusions.oig.hhs.gov/`` (OIG's own
54
+ doctrine). Every tool also carries the state-exclusion-list non-claim: the
55
+ LEIE is federal; state Medicaid exclusion lists are separate and unchecked.
56
+
57
+ Zero-PHI posture
58
+ ----------------
59
+ Exclusion records are public, but DOB and street ADDRESS are excluded from
60
+ every tool result (identity confirmation happens at OIG's site anyway), and
61
+ the vendored sample/fixtures contain no real-person DOB.
62
+
63
+ Offline-harness floor
64
+ ---------------------
65
+ The shared test suite runs every adapter's canaries with sockets blocked
66
+ (the blocker raises a plain ``RuntimeError``, which is not a transport
67
+ failure). Canaries therefore catch that one case and observe the packaged
68
+ sample instead, with self-describing ``sample:*`` values, so an
69
+ offline ``lock init``/``doctor`` is green and honest. Real transport
70
+ failures surface as :class:`SourceUnreachable` and still report
71
+ ``unreachable``. Tools never take this fallback: a compliance answer must
72
+ never come from a 14-row sample.
73
+
74
+ Deviation from the dossier's canary plan: C3 (newest-supplement presence
75
+ probe) is not shipped as a canary because it has no offline-observable
76
+ equivalent under the shared no-network suite; its freshness question is
77
+ answered by ``leie.refresh_status`` and the ``leie.full_file`` staleness
78
+ check instead.
79
+ """
80
+
81
+ from __future__ import annotations
82
+
83
+ import csv
84
+ import hashlib
85
+ import json
86
+ import os
87
+ import time
88
+ from dataclasses import dataclass, replace
89
+ from datetime import date, datetime, timedelta, timezone
90
+ from email.utils import parsedate_to_datetime
91
+ from pathlib import Path
92
+ from typing import Mapping
93
+
94
+ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
95
+
96
+ from ..http import FetchResult, SourceUnreachable, as_cache_hit, fetch
97
+ from ..interfaces import Canary, CanaryObservation, SourceAdapter, ToolResult, ToolSpec
98
+ from ..npi import NPI_PATTERN, validate_npi
99
+ from ..receipts import build_receipt
100
+ from ..schemas import CanarySeverity, CanaryStatus, SourceContract
101
+
102
+ SOURCE_ID = "leie"
103
+ #: 2: answers became fail-closed. Tool results gained ``screen_result`` and
104
+ #: ``snapshot_complete``, records gained ``reinstatement_date``, and the two
105
+ #: cases that used to answer negatively without having screened — unparsed rows,
106
+ #: a populated REINDATE — now answer ``indeterminate``.
107
+ TRANSFORM_VERSION = "2"
108
+
109
+ AUTHORITY_URL = "https://oig.hhs.gov/exclusions/downloadables/UPDATED.csv"
110
+ HUB_URL = "https://oig.hhs.gov/exclusions/leie-database-supplement-downloads/"
111
+ WAIVERS_URL = "https://oig.hhs.gov/exclusions/waivers/"
112
+ VERIFICATION_URL = "https://exclusions.oig.hhs.gov/"
113
+ QUICK_TIPS_URL = "https://oig.hhs.gov/exclusions/leie-quick-tips-instructions/"
114
+ RECORD_LAYOUT_URL = "https://oig.hhs.gov/exclusions/files/leie_record_layout.pdf"
115
+
116
+ #: Override the full-file location (any URL, including file://) — tests, mirrors.
117
+ FULL_FILE_ENV_VAR = "HC_SOURCE_LEIE_FULL_FILE"
118
+ #: Override the waivers-page location — tests.
119
+ WAIVERS_ENV_VAR = "HC_SOURCE_LEIE_WAIVERS_URL"
120
+ #: Freeze "today" (ISO date) so freshness checks are testable.
121
+ TODAY_ENV_VAR = "HC_SOURCE_LEIE_TODAY"
122
+
123
+ MIN_EXPECTED_RECORDS = 70_000
124
+ MAX_EXPECTED_RECORDS = 110_000
125
+ MIN_EXPECTED_BYTES = 8_000_000
126
+ MAX_EXPECTED_BYTES = 40_000_000
127
+ STALE_AFTER_DAYS = 45
128
+ MAX_CANDIDATES = 25
129
+
130
+ #: How many rows the CSV parser may fail to read before an answer stops being a
131
+ #: screen. Zero on purpose. A row whose field count does not match the header is
132
+ #: a row nobody compared the subject against, and the old behaviour -- count it,
133
+ #: skip it, answer anyway -- turned an excluded party sitting in a malformed row
134
+ #: into a clean "not found". A miss is the dangerous direction on an exclusion
135
+ #: list, so any unparsed row makes a miss indeterminate rather than negative.
136
+ MAX_UNPARSED_ROWS = 0
137
+
138
+ EXPECTED_COLUMNS = (
139
+ "LASTNAME", "FIRSTNAME", "MIDNAME", "BUSNAME", "GENERAL", "SPECIALTY",
140
+ "UPIN", "NPI", "DOB", "ADDRESS", "CITY", "STATE", "ZIP",
141
+ "EXCLTYPE", "EXCLDATE", "REINDATE", "WAIVERDATE", "WVRSTATE",
142
+ )
143
+ #: The record-layout PDF spells the last column WAIVERSTATE; the CSV says
144
+ #: WVRSTATE. Parse by the CSV header, accept either spelling.
145
+ _WVRSTATE_ALIASES = {"WVRSTATE", "WAIVERSTATE"}
146
+
147
+ DATE_COLUMNS = ("DOB", "EXCLDATE", "REINDATE", "WAIVERDATE")
148
+ _NULL_DATES = {"", "0", "00000000"}
149
+ _NULL_NPIS = {"", "0", "0000000000"}
150
+
151
+ #: EXCLTYPE domain observed in the 2026-07-10 full file. Opaque tokens: mixed
152
+ #: case and embedded spaces are real — exact-match, never regex-parse.
153
+ KNOWN_EXCLTYPES = frozenset({
154
+ "1128a1", "1128a2", "1128a3", "1128a4", "1128Aa",
155
+ "1128b1", "1128b2", "1128b3", "1128b4", "1128b5", "1128b6", "1128b7",
156
+ "1128b8", "1128b11", "1128b12", "1128b14", "1128b15", "1128b16",
157
+ "1156", "1160", "BRCH SA", "BRCH CIA",
158
+ })
159
+
160
+ CONTRACT = SourceContract(
161
+ source_id=SOURCE_ID,
162
+ authority_url=AUTHORITY_URL,
163
+ fallback_url=None,
164
+ license_notes=(
165
+ "US federal public-domain data; keyless anonymous download, no license or "
166
+ "click-through. SSNs/EINs are deliberately excluded from the file (Privacy Act), "
167
+ "so a bulk-file hit is never identity-verified — verification happens at "
168
+ "https://exclusions.oig.hhs.gov/. Contains no AMA CPT content. Redistribution "
169
+ "obligation: preserve the candidate-only framing of name matches."
170
+ ),
171
+ cadence="monthly (updated by the 10th; contains exclusions through the end of the prior month)",
172
+ effective_date_semantics=(
173
+ "For record-level answers effective_from is the record's EXCLDATE — the "
174
+ "exclusion's legal effective date, which may postdate the file's data month. "
175
+ "For file-level answers effective_from is the snapshot's Last-Modified date. "
176
+ "effective_to is always open: an exclusion ends only by reinstatement, which "
177
+ "REMOVES the record from the next monthly full file."
178
+ ),
179
+ invariants=[
180
+ "the CSV header is exactly the locked 18 columns ending in WVRSTATE",
181
+ "full-file record count stays within 70,000–110,000",
182
+ "REINDATE is null for every record in the full file (reinstated people are removed)",
183
+ "NPI is a placeholder (0000000000/0/blank) on ~89.5% of records",
184
+ "the file is plain ASCII CSV; no SSN or EIN column exists",
185
+ ],
186
+ )
187
+
188
+ # ---------------------------------------------------------------------------
189
+ # Non-claims (stable keys; reviewers read these)
190
+ # ---------------------------------------------------------------------------
191
+
192
+ NC_STATE = (
193
+ "DOES_NOT_COVER_STATE_EXCLUSIONS: the LEIE is the federal OIG exclusion list only; "
194
+ "state Medicaid exclusion and termination lists are separate and are not checked "
195
+ "here — an LEIE miss does not mean the provider is excluded nowhere."
196
+ )
197
+ NC_CURRENCY = (
198
+ "DOES_NOT_PROVE_CURRENCY: this answer comes from the monthly LEIE full-file "
199
+ "snapshot (replaced by the 10th of each month, data through the end of the prior "
200
+ "month) and may lag OIG actions by up to a month."
201
+ )
202
+ NC_HISTORY = (
203
+ "DOES_NOT_PROVE_NEVER_EXCLUDED: reinstated individuals are removed from the "
204
+ "monthly full file; absence from the current snapshot says nothing about past "
205
+ "exclusions."
206
+ )
207
+ NC_NPI_COVERAGE = (
208
+ "DOES_NOT_CLEAR_ON_NPI_MISS: ~89.5% of LEIE records carry no NPI, so an NPI-exact "
209
+ "miss does not clear a provider — run leie.candidate_search and verify identity at "
210
+ "the official OIG online workflow: https://exclusions.oig.hhs.gov/."
211
+ )
212
+ NC_CANDIDATE = (
213
+ "NOT_IDENTITY_VERIFICATION: candidate match is NOT identity verification — the "
214
+ "bulk file lacks SSN/EIN (~10.5% of records have NPI); verify at the official OIG "
215
+ "online workflow: https://exclusions.oig.hhs.gov/."
216
+ )
217
+ NC_UNPARSED = (
218
+ "DID_NOT_SCREEN_THE_WHOLE_LIST: part of the snapshot could not be parsed, so rows "
219
+ "that may name this subject were never compared. Any absence of a match here is "
220
+ "indeterminate, not clearance — re-run once the file parses cleanly, and verify at "
221
+ "https://exclusions.oig.hhs.gov/."
222
+ )
223
+ NC_REINSTATEMENT = (
224
+ "DOES_NOT_IMPLEMENT_REINSTATEMENT: a matched record carries a reinstatement date. "
225
+ "OIG removes reinstated parties from the monthly full file, so a record that has "
226
+ "one means the file no longer behaves the way this adapter was built against. "
227
+ "SourceLock implements no reinstatement semantics and will not call such a record "
228
+ "excluded or not excluded — resolve it at https://exclusions.oig.hhs.gov/."
229
+ )
230
+
231
+ CHECK_NPI_NON_CLAIMS = [NC_STATE, NC_CURRENCY, NC_HISTORY, NC_NPI_COVERAGE]
232
+ CANDIDATE_NON_CLAIMS = [NC_CANDIDATE, NC_STATE, NC_CURRENCY, NC_HISTORY]
233
+ REFRESH_NON_CLAIMS = [NC_STATE, NC_CURRENCY, NC_HISTORY]
234
+
235
+ CADENCE_TEXT = (
236
+ "monthly; OIG updates all exclusion information by the 10th of every month, and "
237
+ "the data month is the month before the file's Last-Modified date"
238
+ )
239
+ RETENTION_TEXT = (
240
+ "OIG archives monthly supplement files for only the previous 12 months; older "
241
+ "supplements are not available upstream. Archive supplements yourself, or "
242
+ "re-baseline from UPDATED.csv when history is lost."
243
+ )
244
+
245
+ # ---------------------------------------------------------------------------
246
+ # Snapshot loading (the runtime working table)
247
+ # ---------------------------------------------------------------------------
248
+
249
+
250
+ def sample_path() -> Path:
251
+ """The packaged 14-row sample (DOB-scrubbed; never served by tools)."""
252
+ return Path(__file__).with_name("_leie_sample.csv")
253
+
254
+
255
+ def _sample_uri() -> str:
256
+ return sample_path().as_uri()
257
+
258
+
259
+ def _full_file_url() -> str:
260
+ return os.environ.get(FULL_FILE_ENV_VAR) or AUTHORITY_URL
261
+
262
+
263
+ def _waivers_url() -> str:
264
+ return os.environ.get(WAIVERS_ENV_VAR) or WAIVERS_URL
265
+
266
+
267
+ def _today() -> date:
268
+ override = os.environ.get(TODAY_ENV_VAR)
269
+ return date.fromisoformat(override) if override else date.today()
270
+
271
+
272
+ @dataclass(frozen=True)
273
+ class _Snapshot:
274
+ fetch_meta: FetchResult # content stripped; provenance for receipts
275
+ from_sample: bool
276
+ source_version: str
277
+ last_modified: datetime | None
278
+ data_month: str | None
279
+ columns: tuple[str, ...]
280
+ records: tuple[Mapping[str, str], ...]
281
+ npi_index: Mapping[str, tuple[int, ...]]
282
+ canonical_hash: str
283
+ schema_hash: str
284
+ reindate_populated: int
285
+ unknown_excltypes: tuple[str, ...]
286
+ parse_warnings: tuple[str, ...]
287
+ malformed_rows: int
288
+
289
+
290
+ @dataclass(frozen=True)
291
+ class _CacheEntry:
292
+ """One cached snapshot plus what we know about how current it is."""
293
+
294
+ snapshot: _Snapshot
295
+ #: Monotonic mark of the last revalidation ATTEMPT, successful or not. This
296
+ #: is what throttles retries, so a source that is down is probed once per
297
+ #: cadence instead of on every tool call.
298
+ checked_at: float
299
+ #: Monotonic mark of the last time upstream actually answered — either these
300
+ #: bytes were read, or a probe confirmed upstream had not moved.
301
+ verified_at: float
302
+ #: Warnings about the entry itself (a failed revalidation), as opposed to
303
+ #: the snapshot's own parse warnings.
304
+ warnings: tuple[str, ...] = ()
305
+ #: True when THIS hand-out came out of the in-process cache rather than off
306
+ #: the wire. Set on the copy that is returned, never on the stored entry:
307
+ #: the first hand-out of a freshly downloaded snapshot is not a cache hit,
308
+ #: and every one after it is.
309
+ reused: bool = False
310
+ #: Wall-clock moment upstream last confirmed these bytes are still current
311
+ #: -- a probe whose Last-Modified or hash matched. ``None`` means no probe
312
+ #: has ever confirmed them, so the fetch's own stamp stands. Never advanced
313
+ #: by a reuse that contacted nobody: "still current as of a moment ago" and
314
+ #: "still what we downloaded on Tuesday" are different claims.
315
+ confirmed_at: datetime | None = None
316
+ #: True when revalidation FAILED and this snapshot is being served anyway.
317
+ stale: bool = False
318
+
319
+ def fetch_meta(self) -> FetchResult:
320
+ """Provenance for a receipt built from this hand-out.
321
+
322
+ The receipt's ``cache_hit`` used to come straight off the original
323
+ FetchResult, so an answer computed from a snapshot downloaded twenty
324
+ minutes ago said ``cache_hit: false`` in its structured field while the
325
+ warning above it said it came from an in-process cache. A machine
326
+ reading the receipt believes the field.
327
+ """
328
+ meta = self.snapshot.fetch_meta
329
+ if not self.reused:
330
+ return meta
331
+ if self.confirmed_at is not None:
332
+ return as_cache_hit(meta, revalidated_at=self.confirmed_at)
333
+ return as_cache_hit(meta)
334
+
335
+
336
+ _SNAPSHOT_CACHE: dict[str, _CacheEntry] = {}
337
+
338
+
339
+ def clear_cache() -> None:
340
+ _SNAPSHOT_CACHE.clear()
341
+
342
+
343
+ def _schema_hash(columns: list[str] | tuple[str, ...]) -> str:
344
+ shape = json.dumps(list(columns), separators=(",", ":"))
345
+ return hashlib.sha256(shape.encode("utf-8")).hexdigest()
346
+
347
+
348
+ def _decode(content: bytes) -> tuple[str, list[str]]:
349
+ try:
350
+ return content.decode("utf-8-sig"), []
351
+ except UnicodeDecodeError:
352
+ return content.decode("latin-1"), [
353
+ "full file contained non-UTF-8 bytes and was decoded as latin-1; upstream "
354
+ "was pure ASCII when last verified (2026-08-01) — review the raw file."
355
+ ]
356
+
357
+
358
+ def _canonical_row(rec: Mapping[str, str]) -> str:
359
+ parts = []
360
+ for col in EXPECTED_COLUMNS:
361
+ v = rec.get(col, "").strip()
362
+ if col in DATE_COLUMNS and v in _NULL_DATES:
363
+ v = ""
364
+ elif col == "NPI" and v in _NULL_NPIS:
365
+ v = ""
366
+ parts.append(v)
367
+ return "\x1f".join(parts)
368
+
369
+
370
+ def _parse_last_modified(headers: Mapping[str, str]) -> datetime | None:
371
+ raw = headers.get("last-modified")
372
+ if not raw:
373
+ return None
374
+ try:
375
+ return parsedate_to_datetime(raw)
376
+ except (TypeError, ValueError):
377
+ return None
378
+
379
+
380
+ def _data_month(lm_date: date) -> str:
381
+ year, month = lm_date.year, lm_date.month - 1
382
+ if month == 0:
383
+ year, month = year - 1, 12
384
+ return f"{year}-{month:02d}"
385
+
386
+
387
+ def _parse_snapshot(result: FetchResult, *, expect_full: bool, from_sample: bool) -> _Snapshot:
388
+ text, warnings = _decode(result.content)
389
+ rows = list(csv.reader(text.splitlines()))
390
+ if not rows:
391
+ raise SourceUnreachable(result.url, "full file is empty")
392
+
393
+ header = [h.strip() for h in rows[0]]
394
+ lookup: dict[str, int] = {}
395
+ for idx, name in enumerate(header):
396
+ canonical = "WVRSTATE" if name in _WVRSTATE_ALIASES else name
397
+ lookup.setdefault(canonical, idx)
398
+ missing = [c for c in EXPECTED_COLUMNS if c not in lookup]
399
+ if missing:
400
+ raise SourceUnreachable(
401
+ result.url,
402
+ f"column layout changed: missing column(s) {missing}; fetch the current "
403
+ f"record layout from {RECORD_LAYOUT_URL} and update the adapter",
404
+ )
405
+ if tuple(header) != EXPECTED_COLUMNS:
406
+ warnings.append(f"header differs from the locked layout: {header}")
407
+
408
+ records: list[dict[str, str]] = []
409
+ malformed = 0
410
+ for row in rows[1:]:
411
+ if not row:
412
+ continue
413
+ if len(row) != len(header):
414
+ malformed += 1
415
+ continue
416
+ records.append({c: row[lookup[c]].strip() for c in EXPECTED_COLUMNS})
417
+ if malformed:
418
+ warnings.append(
419
+ f"{malformed} row(s) did not match the header's field count and could not "
420
+ "be parsed; the subjects named in them were never compared"
421
+ )
422
+
423
+ count = len(records)
424
+ if expect_full and not (MIN_EXPECTED_RECORDS <= count <= MAX_EXPECTED_RECORDS):
425
+ raise SourceUnreachable(
426
+ result.url,
427
+ f"record count {count} outside the sanity band "
428
+ f"[{MIN_EXPECTED_RECORDS}, {MAX_EXPECTED_RECORDS}] — treating the download "
429
+ "as truncated or corrupt; keep the prior snapshot and retry in 24h",
430
+ )
431
+
432
+ npi_index: dict[str, tuple[int, ...]] = {}
433
+ for j, rec in enumerate(records):
434
+ npi = rec["NPI"]
435
+ if npi not in _NULL_NPIS:
436
+ npi_index[npi] = npi_index.get(npi, ()) + (j,)
437
+
438
+ canonical = hashlib.sha256(
439
+ "\n".join(sorted(_canonical_row(r) for r in records)).encode("utf-8")
440
+ ).hexdigest()
441
+
442
+ lm = _parse_last_modified(result.headers)
443
+ if from_sample:
444
+ source_version = f"sample:{canonical[:8]}"
445
+ elif lm is not None:
446
+ source_version = lm.date().isoformat()
447
+ else:
448
+ source_version = f"canonical:{canonical[:12]}"
449
+
450
+ return _Snapshot(
451
+ fetch_meta=replace(result, content=b""),
452
+ from_sample=from_sample,
453
+ source_version=source_version,
454
+ last_modified=lm,
455
+ data_month=_data_month(lm.date()) if lm else None,
456
+ columns=tuple(header),
457
+ records=tuple(records),
458
+ npi_index=npi_index,
459
+ canonical_hash=canonical,
460
+ schema_hash=_schema_hash(header),
461
+ reindate_populated=sum(1 for r in records if r["REINDATE"] not in _NULL_DATES),
462
+ unknown_excltypes=tuple(sorted({r["EXCLTYPE"] for r in records} - KNOWN_EXCLTYPES)),
463
+ parse_warnings=tuple(warnings),
464
+ malformed_rows=malformed,
465
+ )
466
+
467
+
468
+ #: The LEIE full file measured 14.8 MB on 2026-08-01 and grows monotonically as
469
+ #: exclusions accumulate. 64 MiB is roughly 4x that: enough headroom for years
470
+ #: of growth, tight enough that a hostile or broken mirror cannot hand this
471
+ #: process an unbounded body. Raising it is a deliberate act, not a default.
472
+ LEIE_MAX_BYTES = 64 * 1024 * 1024
473
+
474
+
475
+ #: How long a cached snapshot may be served before upstream is asked whether it
476
+ #: moved. OIG replaces UPDATED.csv by the 10th of every month, but not on a
477
+ #: fixed day, and NC_CURRENCY promises callers a lag of at most a month. The
478
+ #: cache used to have no expiry at all, so a long-lived process (the MCP server)
479
+ #: pinned the first snapshot it ever read and kept printing that promise —
480
+ #: after two publication cycles the promise was simply false. One day is the
481
+ #: interval that keeps it true with margin: a month-long interval would let a
482
+ #: server answer from a file two releases old and still be "within cadence",
483
+ #: and the check costs one ~2 KB ranged GET per day per process.
484
+ SNAPSHOT_REVALIDATE_AFTER = timedelta(days=1)
485
+
486
+
487
+ def _monotonic() -> float:
488
+ """Elapsed-time clock for the cache. Monotonic, so a wall-clock jump or an
489
+ NTP correction can neither expire a fresh snapshot nor freeze a stale one."""
490
+ return time.monotonic()
491
+
492
+
493
+ def _load_snapshot(url: str, *, from_sample: bool) -> _Snapshot:
494
+ """Read and parse the full file. Always goes upstream; caching is _cached_entry's job."""
495
+ result = fetch(url, max_bytes=LEIE_MAX_BYTES)
496
+ return _parse_snapshot(result, expect_full=not from_sample, from_sample=from_sample)
497
+
498
+
499
+ def _store(url: str, snapshot: _Snapshot) -> _CacheEntry:
500
+ mark = _monotonic()
501
+ entry = _CacheEntry(snapshot=snapshot, checked_at=mark, verified_at=mark)
502
+ # One dict assignment, after the new snapshot is fully parsed. A reader
503
+ # concurrent with a revalidation sees either the whole old entry or the
504
+ # whole new one, never a half-swapped table.
505
+ _SNAPSHOT_CACHE[url] = entry
506
+ return entry
507
+
508
+
509
+ def _confirm(url: str, entry: _CacheEntry) -> _CacheEntry:
510
+ """Upstream had not moved. Keep the bytes — and their retrieval stamp.
511
+
512
+ Nothing was re-read here, so re-stamping ``retrieved_at`` would put a read
513
+ that never happened on every receipt these bytes go on to support.
514
+ ``confirmed_at`` is the field that DOES move: upstream really did just say
515
+ these bytes are current, and that is the one thing this branch establishes.
516
+ """
517
+ mark = _monotonic()
518
+ confirmed = replace(
519
+ entry,
520
+ checked_at=mark,
521
+ verified_at=mark,
522
+ warnings=(),
523
+ reused=True,
524
+ stale=False,
525
+ confirmed_at=datetime.now(timezone.utc),
526
+ )
527
+ _SNAPSHOT_CACHE[url] = confirmed
528
+ return confirmed
529
+
530
+
531
+ def _keep_stale(url: str, entry: _CacheEntry, reason: str) -> _CacheEntry:
532
+ """Revalidation failed. Serve the old snapshot and say how old it is.
533
+
534
+ Discarding a good snapshot because the network blinked would turn a
535
+ transient outage into no answer at all; pretending it is current would be
536
+ worse. The warning names the retrieval stamp so the caller can price it.
537
+ """
538
+ confirmed = entry.confirmed_at or entry.snapshot.fetch_meta.revalidated_at
539
+ warning = (
540
+ f"STALE: LEIE snapshot could not be revalidated ({reason}); still serving the "
541
+ f"snapshot retrieved {entry.snapshot.fetch_meta.retrieved_at.isoformat()} "
542
+ f"(source_version {entry.snapshot.source_version}), last confirmed current by "
543
+ f"upstream {confirmed.isoformat() if confirmed else 'never'}. The receipt's "
544
+ f"cache_hit and revalidated_at say the same thing in machine-readable form. "
545
+ f"This answer is that old — OIG republishes monthly, so re-check {HUB_URL} if "
546
+ "the outage persists."
547
+ )
548
+ stale = replace(
549
+ entry, checked_at=_monotonic(), warnings=(warning,), reused=True, stale=True
550
+ )
551
+ _SNAPSHOT_CACHE[url] = stale
552
+ return stale
553
+
554
+
555
+ def _revalidate(url: str, entry: _CacheEntry) -> _CacheEntry:
556
+ try:
557
+ probe = _probe(url)
558
+ cached_lm = entry.snapshot.last_modified
559
+ probe_lm = _parse_last_modified(probe.headers)
560
+ if cached_lm is not None and probe_lm is not None and probe_lm == cached_lm:
561
+ return _confirm(url, entry)
562
+ if _content_range_total(probe.headers) is None and probe.sha256 == entry.snapshot.fetch_meta.sha256:
563
+ # No Content-Range means the probe body IS the whole body (a file://
564
+ # mirror, or a server that ignored the Range header), so its hash is
565
+ # comparable to the full-file hash on the receipt. Same bytes, no
566
+ # re-parse. Never compare a 206's hash: that is 2 KB, not the file.
567
+ return _confirm(url, entry)
568
+ snapshot = _load_snapshot(url, from_sample=False)
569
+ except SourceUnreachable as exc:
570
+ return _keep_stale(url, entry, exc.reason)
571
+ except RuntimeError as exc:
572
+ # The offline harness's socket blocker raises a plain RuntimeError. It
573
+ # is not a transport failure, but for a cached snapshot the handling is
574
+ # the same: keep what we have and say it was not rechecked.
575
+ return _keep_stale(url, entry, type(exc).__name__)
576
+ return _store(url, snapshot)
577
+
578
+
579
+ def _cached_entry(*, from_sample: bool = False) -> _CacheEntry:
580
+ url = _sample_uri() if from_sample else _full_file_url()
581
+ entry = _SNAPSHOT_CACHE.get(url)
582
+ if entry is None:
583
+ return _store(url, _load_snapshot(url, from_sample=from_sample))
584
+ if from_sample:
585
+ # The packaged sample ships inside this process. It has no upstream to
586
+ # have moved, so revalidating it would only spend a fetch.
587
+ return replace(entry, reused=True)
588
+ if _monotonic() - entry.checked_at < SNAPSHOT_REVALIDATE_AFTER.total_seconds():
589
+ # Inside the revalidation window: reuse, and say so. Nobody was
590
+ # contacted, so `confirmed_at` stays where it was.
591
+ return replace(entry, reused=True)
592
+ return _revalidate(url, entry)
593
+
594
+
595
+ def _cached_snapshot(*, from_sample: bool = False) -> _Snapshot:
596
+ return _cached_entry(from_sample=from_sample).snapshot
597
+
598
+
599
+ def _probe(url: str | None = None) -> FetchResult:
600
+ """One cheap ranged GET of the full-file URL (headers + first ~2 KB)."""
601
+ return fetch(url or _full_file_url(), headers={"Range": "bytes=0-2047"})
602
+
603
+
604
+ # ---------------------------------------------------------------------------
605
+ # Typed public parameters
606
+ # ---------------------------------------------------------------------------
607
+
608
+
609
+ class NoParams(BaseModel):
610
+ model_config = ConfigDict(extra="forbid")
611
+
612
+
613
+ class CheckNpiParams(BaseModel):
614
+ model_config = ConfigDict(extra="forbid")
615
+
616
+ npi: str = Field(
617
+ pattern=NPI_PATTERN,
618
+ description="10-digit NPI to check against the LEIE full-file snapshot.",
619
+ )
620
+
621
+ @field_validator("npi")
622
+ @classmethod
623
+ def _real_npi(cls, v: str) -> str:
624
+ # The placeholder check runs first because it fails the Luhn too, and
625
+ # "you passed the LEIE's no-NPI sentinel" is the diagnosis the caller
626
+ # needs; "check digit failed" would send them looking for a typo.
627
+ if v == "0000000000":
628
+ raise ValueError(
629
+ "0000000000 is the LEIE placeholder for 'no NPI on record', not a real NPI"
630
+ )
631
+ # Shape is not validity. Ten digits that cannot be an NPI used to reach
632
+ # the snapshot and come back as a confident miss on a number that does
633
+ # not exist, which is exactly the false clearance this adapter must not
634
+ # produce. hc_source.npi is the one validator; provider uses it too.
635
+ return validate_npi(v)
636
+
637
+
638
+ _NAME_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9 .,'&#\-]*$"
639
+
640
+
641
+ class CandidateSearchParams(BaseModel):
642
+ model_config = ConfigDict(extra="forbid")
643
+
644
+ last_name: str | None = Field(
645
+ default=None, min_length=2, max_length=30, pattern=_NAME_PATTERN,
646
+ description="Individual's last name, matched exactly (case-insensitive).",
647
+ )
648
+ first_name: str | None = Field(
649
+ default=None, min_length=1, max_length=20, pattern=_NAME_PATTERN,
650
+ description="Optional first-name prefix filter; requires last_name.",
651
+ )
652
+ business_name: str | None = Field(
653
+ default=None, min_length=2, max_length=40, pattern=_NAME_PATTERN,
654
+ description="Entity name fragment, matched as a case-insensitive substring.",
655
+ )
656
+ state: str | None = Field(
657
+ default=None, pattern=r"^[A-Za-z]{2}$",
658
+ description="Two-letter address-state filter as it appears in the file.",
659
+ )
660
+
661
+ @field_validator("state")
662
+ @classmethod
663
+ def _upper(cls, v: str | None) -> str | None:
664
+ return v.upper() if v else v
665
+
666
+ @model_validator(mode="after")
667
+ def _exactly_one_subject(self) -> "CandidateSearchParams":
668
+ if bool(self.last_name) == bool(self.business_name):
669
+ raise ValueError(
670
+ "provide exactly one of last_name (individuals) or business_name (entities)"
671
+ )
672
+ if self.first_name and not self.last_name:
673
+ raise ValueError("first_name only narrows a last_name search")
674
+ return self
675
+
676
+
677
+ # ---------------------------------------------------------------------------
678
+ # Record projection (public fields only: no DOB, no street address)
679
+ # ---------------------------------------------------------------------------
680
+
681
+
682
+ def _iso_date(v: str) -> str | None:
683
+ if v in _NULL_DATES:
684
+ return None
685
+ if len(v) == 8 and v.isdigit():
686
+ try:
687
+ return date(int(v[:4]), int(v[4:6]), int(v[6:8])).isoformat()
688
+ except ValueError:
689
+ return v
690
+ return v
691
+
692
+
693
+ def _record_effective(rec: Mapping[str, str]) -> date | None:
694
+ iso = _iso_date(rec["EXCLDATE"])
695
+ try:
696
+ return date.fromisoformat(iso) if iso else None
697
+ except ValueError:
698
+ return None
699
+
700
+
701
+ def _public_record(rec: Mapping[str, str]) -> dict[str, object]:
702
+ waiver = None
703
+ if rec["WAIVERDATE"] not in _NULL_DATES:
704
+ waiver = {"date": _iso_date(rec["WAIVERDATE"]), "state": rec["WVRSTATE"] or None}
705
+ return {
706
+ "record_type": "entity" if rec["BUSNAME"] else "individual",
707
+ "last_name": rec["LASTNAME"] or None,
708
+ "first_name": rec["FIRSTNAME"] or None,
709
+ "mid_name": rec["MIDNAME"] or None,
710
+ "business_name": rec["BUSNAME"] or None,
711
+ "general": rec["GENERAL"] or None,
712
+ "specialty": rec["SPECIALTY"] or None,
713
+ "upin": rec["UPIN"] or None,
714
+ "npi": rec["NPI"] if rec["NPI"] not in _NULL_NPIS else None,
715
+ "city": rec["CITY"] or None,
716
+ "state": rec["STATE"] or None,
717
+ "zip": rec["ZIP"] or None,
718
+ "excltype": rec["EXCLTYPE"],
719
+ "exclusion_date": _iso_date(rec["EXCLDATE"]),
720
+ # Null on every record in a well-formed full file: OIG removes
721
+ # reinstated parties rather than dating them. Surfaced anyway, because
722
+ # the day it is not null the caller needs to see the date, not a verdict
723
+ # this adapter has no semantics for.
724
+ "reinstatement_date": _iso_date(rec["REINDATE"]),
725
+ "waiver": waiver,
726
+ }
727
+
728
+
729
+ def _norm(value: str) -> str:
730
+ return " ".join(value.split()).casefold()
731
+
732
+
733
+ # ---------------------------------------------------------------------------
734
+ # Handlers
735
+ # ---------------------------------------------------------------------------
736
+
737
+
738
+ def _snapshot_date(snapshot: _Snapshot) -> date | None:
739
+ return snapshot.last_modified.date() if snapshot.last_modified else None
740
+
741
+
742
+ def _check_npi(params: CheckNpiParams) -> ToolResult:
743
+ entry = _cached_entry()
744
+ snapshot = entry.snapshot
745
+ warnings = list(snapshot.parse_warnings) + list(entry.warnings)
746
+ non_claims = list(CHECK_NPI_NON_CLAIMS)
747
+
748
+ unparsed = snapshot.malformed_rows > MAX_UNPARSED_ROWS
749
+ if unparsed:
750
+ non_claims.append(NC_UNPARSED)
751
+
752
+ indices = snapshot.npi_index.get(params.npi, ())
753
+ records = [_public_record(snapshot.records[j]) for j in indices]
754
+ reinstated = [r for r in records if r["reinstatement_date"]]
755
+
756
+ # Everything a caller may read is keyed on screen_result. It is "excluded"
757
+ # or "indeterminate" and never a clear: a clean no-match returns data=None
758
+ # instead, so no shape of this payload can be read as clearance.
759
+ common: dict[str, object] = {
760
+ "npi": params.npi,
761
+ "data_month": snapshot.data_month,
762
+ "snapshot_complete": not unparsed,
763
+ "verification_url": VERIFICATION_URL,
764
+ }
765
+
766
+ if reinstated:
767
+ # A record with a reinstatement date contradicts the full file's own
768
+ # invariant, so the honest answer is neither "excluded" (the party may
769
+ # have been reinstated) nor "not excluded" (the record is still here).
770
+ # excluded_in_snapshot is OMITTED rather than set false or null: a
771
+ # caller that reads it gets a KeyError, not a falsy value.
772
+ data: dict[str, object] | None = {
773
+ **common,
774
+ "screen_result": "indeterminate",
775
+ "indeterminate_reason": "reinstatement_date_present",
776
+ "records": records,
777
+ }
778
+ non_claims.append(NC_REINSTATEMENT)
779
+ warnings.append(
780
+ f"{len(reinstated)} matched record(s) carry a reinstatement date, which the "
781
+ "LEIE full file is not supposed to contain. SourceLock implements no "
782
+ f"reinstatement semantics: this is not a verdict either way — resolve it at "
783
+ f"{VERIFICATION_URL}."
784
+ )
785
+ effective = _snapshot_date(snapshot)
786
+ elif indices:
787
+ data = {
788
+ **common,
789
+ "screen_result": "excluded",
790
+ "excluded_in_snapshot": True,
791
+ "match_basis": "npi-exact",
792
+ "records": records,
793
+ }
794
+ effective = (
795
+ _record_effective(snapshot.records[indices[0]])
796
+ if len(indices) == 1
797
+ else _snapshot_date(snapshot)
798
+ )
799
+ if unparsed:
800
+ warnings.append(
801
+ f"{snapshot.malformed_rows} row(s) of the snapshot could not be parsed, so "
802
+ "this record list may be incomplete; the match itself stands."
803
+ )
804
+ elif unparsed:
805
+ # No match, but the screen did not read the whole list. Fail closed.
806
+ data = {
807
+ **common,
808
+ "screen_result": "indeterminate",
809
+ "indeterminate_reason": "unparsed_rows",
810
+ "unparsed_rows": snapshot.malformed_rows,
811
+ "records": records,
812
+ }
813
+ warnings.append(
814
+ f"{snapshot.malformed_rows} row(s) of the LEIE snapshot "
815
+ f"({snapshot.source_version}) could not be parsed, so this NPI was not "
816
+ "compared against the whole list. Reporting indeterminate rather than "
817
+ f"not-found — verify at {VERIFICATION_URL}."
818
+ )
819
+ effective = _snapshot_date(snapshot)
820
+ else:
821
+ data = None
822
+ effective = _snapshot_date(snapshot)
823
+ warnings.append(
824
+ "The requested NPI was not found in the LEIE full-file snapshot "
825
+ f"({snapshot.source_version}). ~89.5% of LEIE records carry no NPI — run "
826
+ f"leie.candidate_search and verify identity at {VERIFICATION_URL} before "
827
+ "treating this as clearance."
828
+ )
829
+
830
+ return ToolResult(
831
+ data=data,
832
+ receipt=build_receipt(
833
+ contract=CONTRACT,
834
+ route="leie.check_npi",
835
+ fetch=entry.fetch_meta(),
836
+ source_version=snapshot.source_version,
837
+ transform_version=TRANSFORM_VERSION,
838
+ effective_from=effective,
839
+ warnings=warnings,
840
+ non_claims=non_claims,
841
+ ),
842
+ )
843
+
844
+
845
+ def _candidate_search(params: CandidateSearchParams) -> ToolResult:
846
+ entry = _cached_entry()
847
+ snapshot = entry.snapshot
848
+ warnings = list(snapshot.parse_warnings) + list(entry.warnings)
849
+ non_claims = list(CANDIDATE_NON_CLAIMS)
850
+ unparsed = snapshot.malformed_rows > MAX_UNPARSED_ROWS
851
+ if unparsed:
852
+ non_claims.append(NC_UNPARSED)
853
+
854
+ matches: list[int] = []
855
+ if params.last_name:
856
+ last = _norm(params.last_name)
857
+ first = _norm(params.first_name) if params.first_name else None
858
+ for j, rec in enumerate(snapshot.records):
859
+ if not rec["LASTNAME"] or _norm(rec["LASTNAME"]) != last:
860
+ continue
861
+ if first and not _norm(rec["FIRSTNAME"]).startswith(first):
862
+ continue
863
+ matches.append(j)
864
+ else:
865
+ fragment = _norm(params.business_name or "")
866
+ for j, rec in enumerate(snapshot.records):
867
+ if rec["BUSNAME"] and fragment in _norm(rec["BUSNAME"]):
868
+ matches.append(j)
869
+
870
+ if params.state:
871
+ matches = [j for j in matches if snapshot.records[j]["STATE"] == params.state]
872
+
873
+ matches.sort(
874
+ key=lambda j: (
875
+ snapshot.records[j]["LASTNAME"],
876
+ snapshot.records[j]["FIRSTNAME"],
877
+ snapshot.records[j]["BUSNAME"],
878
+ snapshot.records[j]["EXCLDATE"],
879
+ j,
880
+ )
881
+ )
882
+ truncated = len(matches) > MAX_CANDIDATES
883
+ picked = matches[:MAX_CANDIDATES]
884
+ if truncated:
885
+ warnings.append(
886
+ f"more than {MAX_CANDIDATES} candidate rows matched; returning the first "
887
+ f"{MAX_CANDIDATES} — narrow with first_name or state"
888
+ )
889
+
890
+ if picked:
891
+ candidates = [
892
+ dict(_public_record(snapshot.records[j]), match_basis="name-candidate")
893
+ for j in picked
894
+ ]
895
+ data: dict[str, object] | None = {
896
+ "screen_result": "candidates",
897
+ "candidates": candidates,
898
+ "count": len(candidates),
899
+ "truncated": truncated,
900
+ "snapshot_complete": not unparsed,
901
+ "data_month": snapshot.data_month,
902
+ "verification_url": VERIFICATION_URL,
903
+ }
904
+ reinstated = [c for c in candidates if c["reinstatement_date"]]
905
+ if reinstated:
906
+ non_claims.append(NC_REINSTATEMENT)
907
+ warnings.append(
908
+ f"{len(reinstated)} returned candidate(s) carry a reinstatement date, "
909
+ "which the LEIE full file is not supposed to contain. SourceLock "
910
+ "implements no reinstatement semantics: their exclusion status is "
911
+ f"undetermined here — resolve it at {VERIFICATION_URL}."
912
+ )
913
+ if unparsed:
914
+ warnings.append(
915
+ f"{snapshot.malformed_rows} row(s) of the snapshot could not be parsed, "
916
+ "so this candidate list may be missing matches."
917
+ )
918
+ elif unparsed:
919
+ # Nothing matched, but the search did not read the whole list. The
920
+ # candidate/count keys are OMITTED so an empty list cannot be read as
921
+ # "searched, found nobody".
922
+ data = {
923
+ "screen_result": "indeterminate",
924
+ "indeterminate_reason": "unparsed_rows",
925
+ "unparsed_rows": snapshot.malformed_rows,
926
+ "snapshot_complete": False,
927
+ "data_month": snapshot.data_month,
928
+ "verification_url": VERIFICATION_URL,
929
+ }
930
+ warnings.append(
931
+ f"{snapshot.malformed_rows} row(s) of the LEIE snapshot "
932
+ f"({snapshot.source_version}) could not be parsed, so this name was not "
933
+ "searched against the whole list. Reporting indeterminate rather than "
934
+ f"no-candidates — verify at {VERIFICATION_URL}."
935
+ )
936
+ else:
937
+ data = None
938
+ warnings.append(
939
+ "No candidate rows matched. Per OIG search guidance, also try maiden or "
940
+ "former names, each half of a hyphenated name separately, and spelling "
941
+ "variants — apostrophes, hyphens, ampersands and commas are significant."
942
+ )
943
+
944
+ return ToolResult(
945
+ data=data,
946
+ receipt=build_receipt(
947
+ contract=CONTRACT,
948
+ route="leie.candidate_search",
949
+ fetch=entry.fetch_meta(),
950
+ source_version=snapshot.source_version,
951
+ transform_version=TRANSFORM_VERSION,
952
+ effective_from=_snapshot_date(snapshot),
953
+ warnings=warnings,
954
+ non_claims=non_claims,
955
+ ),
956
+ )
957
+
958
+
959
+ def _refresh_status(_: NoParams) -> ToolResult:
960
+ result = _probe()
961
+ warnings: list[str] = []
962
+ lm = _parse_last_modified(result.headers)
963
+ today = _today()
964
+
965
+ if lm is not None:
966
+ lm_date = lm.date()
967
+ age_days: int | None = (today - lm_date).days
968
+ stale: bool | None = age_days > STALE_AFTER_DAYS
969
+ data_month = _data_month(lm_date)
970
+ next_year, next_month = lm_date.year, lm_date.month + 1
971
+ if next_month == 13:
972
+ next_year, next_month = next_year + 1, 1
973
+ next_expected: str | None = f"{next_year}-{next_month:02d}-10"
974
+ source_version = lm_date.isoformat()
975
+ effective: date | None = lm_date
976
+ if stale:
977
+ warnings.append(
978
+ f"snapshot is stale: last modified {lm_date.isoformat()} ({age_days} "
979
+ f"days ago) against OIG's monthly by-the-10th cadence — check {HUB_URL}"
980
+ )
981
+ else:
982
+ age_days = stale = None
983
+ data_month = next_expected = None
984
+ source_version = f"sha256:{result.sha256[:12]}"
985
+ effective = None
986
+ warnings.append(
987
+ "no Last-Modified header (local mirror or override); snapshot age unknown"
988
+ )
989
+
990
+ data = {
991
+ "source_url": result.url,
992
+ "last_modified": lm.isoformat() if lm else None,
993
+ "data_month": data_month,
994
+ "age_days": age_days,
995
+ "stale": stale,
996
+ "cadence": CADENCE_TEXT,
997
+ "supplement_retention": RETENTION_TEXT,
998
+ "next_update_expected": next_expected,
999
+ "verification_url": VERIFICATION_URL,
1000
+ }
1001
+ return ToolResult(
1002
+ data=data,
1003
+ receipt=build_receipt(
1004
+ contract=CONTRACT,
1005
+ route="leie.refresh_status",
1006
+ fetch=result,
1007
+ source_version=source_version,
1008
+ transform_version=TRANSFORM_VERSION,
1009
+ effective_from=effective,
1010
+ warnings=warnings,
1011
+ non_claims=REFRESH_NON_CLAIMS,
1012
+ ),
1013
+ )
1014
+
1015
+
1016
+ # ---------------------------------------------------------------------------
1017
+ # Canaries
1018
+ # ---------------------------------------------------------------------------
1019
+
1020
+
1021
+ def _content_range_total(headers: Mapping[str, str]) -> int | None:
1022
+ raw = headers.get("content-range", "")
1023
+ if "/" not in raw:
1024
+ return None
1025
+ total = raw.rsplit("/", 1)[1].strip()
1026
+ return int(total) if total.isdigit() else None
1027
+
1028
+
1029
+ def _observe_full_file() -> CanaryObservation:
1030
+ try:
1031
+ result = _probe()
1032
+ except SourceUnreachable:
1033
+ raise
1034
+ except RuntimeError:
1035
+ # Offline harness (socket blocker): observe the packaged sample with a
1036
+ # self-describing token. Real transport failures raise above.
1037
+ snapshot = _cached_snapshot(from_sample=True)
1038
+ return CanaryObservation(
1039
+ value=f"sample:{snapshot.canonical_hash[:8]}",
1040
+ schema_hash=snapshot.schema_hash,
1041
+ note="offline: observed the packaged sample, not the live source",
1042
+ )
1043
+
1044
+ content_type = result.headers.get("content-type", "")
1045
+ if content_type and not content_type.startswith("text/csv"):
1046
+ return CanaryObservation(
1047
+ value=f"unexpected-content-type:{content_type.split(';')[0]}",
1048
+ upstream_status=result.status,
1049
+ )
1050
+ total = _content_range_total(result.headers)
1051
+ if total is not None and not (MIN_EXPECTED_BYTES <= total <= MAX_EXPECTED_BYTES):
1052
+ return CanaryObservation(value=f"size-out-of-band:{total}", upstream_status=result.status)
1053
+
1054
+ lm = _parse_last_modified(result.headers)
1055
+ if lm is None:
1056
+ # Local mirror / file override: key on canonical content, never raw bytes.
1057
+ snapshot = _cached_snapshot()
1058
+ return CanaryObservation(
1059
+ value=f"canonical:{snapshot.canonical_hash[:12]}",
1060
+ schema_hash=snapshot.schema_hash,
1061
+ note="no Last-Modified header; keyed on canonical content",
1062
+ )
1063
+ lm_date = lm.date()
1064
+ if (_today() - lm_date).days > STALE_AFTER_DAYS:
1065
+ return CanaryObservation(value=f"stale:{lm_date.isoformat()}", upstream_status=result.status)
1066
+ return CanaryObservation(
1067
+ value=lm_date.isoformat(),
1068
+ upstream_status=result.status,
1069
+ note="Last-Modified of UPDATED.csv; a new monthly release moves this forward",
1070
+ )
1071
+
1072
+
1073
+ def _observe_header() -> CanaryObservation:
1074
+ try:
1075
+ result = _probe()
1076
+ except SourceUnreachable:
1077
+ raise
1078
+ except RuntimeError:
1079
+ result = fetch(_sample_uri())
1080
+
1081
+ text = result.content.decode("utf-8-sig", errors="replace")
1082
+ if text.lstrip().startswith("<"):
1083
+ return CanaryObservation(value="html-body", upstream_status=result.status)
1084
+ lines = text.splitlines()
1085
+ columns = [c.strip() for c in next(csv.reader(lines[:1]), [])] if lines else []
1086
+ if not columns:
1087
+ return CanaryObservation(value="empty-header", upstream_status=result.status)
1088
+ return CanaryObservation(
1089
+ value=str(len(columns)),
1090
+ schema_hash=_schema_hash(columns),
1091
+ upstream_status=result.status,
1092
+ note="column count of the UPDATED.csv header; the hash locks the exact names",
1093
+ )
1094
+
1095
+
1096
+ def _observe_content() -> CanaryObservation:
1097
+ try:
1098
+ snapshot = _cached_snapshot()
1099
+ except SourceUnreachable:
1100
+ raise
1101
+ except RuntimeError:
1102
+ snapshot = _cached_snapshot(from_sample=True)
1103
+
1104
+ if snapshot.reindate_populated:
1105
+ return CanaryObservation(
1106
+ value=f"reindate-populated:{snapshot.reindate_populated}",
1107
+ schema_hash=snapshot.schema_hash,
1108
+ note="REINDATE must be null in the full file; OIG may have changed semantics",
1109
+ )
1110
+ note = f"records={len(snapshot.records)}"
1111
+ if snapshot.unknown_excltypes:
1112
+ note += "; new EXCLTYPE tokens: " + ", ".join(snapshot.unknown_excltypes)
1113
+ value = (
1114
+ f"sample:{snapshot.canonical_hash[:8]}"
1115
+ if snapshot.from_sample
1116
+ else f"canonical:{snapshot.canonical_hash[:16]}"
1117
+ )
1118
+ return CanaryObservation(value=value, schema_hash=snapshot.schema_hash, note=note)
1119
+
1120
+
1121
+ def _observe_waivers() -> CanaryObservation:
1122
+ """Advisory cross-check: observes, never drifts, never fails.
1123
+
1124
+ Its ``value`` is the constant ``"advisory"``, so it cannot drift by
1125
+ construction -- which is the point, and also the trap. When the cross-check
1126
+ MISMATCHES, an unchanged constant would be graded ``ok`` while the invariant
1127
+ the canary exists to test is broken. So a mismatch sets ``stale``: doctor
1128
+ downgrades the match to STALE (exit 4), because a comparison that succeeded
1129
+ against something the canary says is wrong is not assurance.
1130
+
1131
+ A SKIP is different and deliberately not stale. The advisory being
1132
+ unavailable -- offline, or the waivers page returning a non-2xx -- says
1133
+ nothing about whether the exclusion data is right, and the note already
1134
+ carries the reason.
1135
+ """
1136
+ stale = False
1137
+ try:
1138
+ page = fetch(_waivers_url(), raise_for_status=False)
1139
+ try:
1140
+ snapshot = _cached_snapshot()
1141
+ except Exception: # noqa: BLE001 - advisory must not fail on the data path either
1142
+ snapshot = _cached_snapshot(from_sample=True)
1143
+ names = sorted({
1144
+ r["LASTNAME"]
1145
+ for r in snapshot.records
1146
+ if r["WAIVERDATE"] not in _NULL_DATES and r["LASTNAME"]
1147
+ })
1148
+ if page.status is not None and not (200 <= page.status < 300):
1149
+ note = f"skipped: waivers page returned HTTP {page.status}"
1150
+ else:
1151
+ text = page.content.decode("utf-8", errors="replace").casefold()
1152
+ missing = [n for n in names if n.casefold() not in text]
1153
+ if missing:
1154
+ stale = True
1155
+ note = (
1156
+ f"mismatch: {len(missing)} of {len(names)} CSV waiver surnames not on "
1157
+ "the waivers page — waiver scope is defined by the letters there"
1158
+ )
1159
+ else:
1160
+ note = f"ok: {len(names)}/{len(names)} CSV waiver surnames found on the waivers page"
1161
+ except Exception as exc: # noqa: BLE001 - advisory by design
1162
+ note = f"skipped: {type(exc).__name__}"
1163
+ return CanaryObservation(value="advisory", note=note, stale=stale)
1164
+
1165
+
1166
+ def _full_file_remediation(status: CanaryStatus, observed: str | None, expected: str | None) -> str:
1167
+ if status is CanaryStatus.UNREACHABLE:
1168
+ return (
1169
+ f"UPDATED.csv could not be probed at {AUTHORITY_URL}. Check network and the "
1170
+ f"hub page {HUB_URL} for a moved link; if the page is gone, email "
1171
+ "exclusions@oig.hhs.gov. Keep serving the last verified snapshot, labelled "
1172
+ "with its data month, until resolved."
1173
+ )
1174
+ if observed and observed.startswith("sample:"):
1175
+ return (
1176
+ "The canary observed the packaged sample because the live source was not "
1177
+ "reachable at observation time. Re-run `hc-source lock init` with network "
1178
+ "access so the lockfile pins the real upstream."
1179
+ )
1180
+ if observed and (observed.startswith("stale:") or observed.startswith("size-out-of-band:")
1181
+ or observed.startswith("unexpected-content-type:")):
1182
+ return (
1183
+ f"UPDATED.csv looks wrong upstream ({observed}). Open {HUB_URL}, confirm the "
1184
+ "'Updated LEIE Database (CSV)' link, and update the adapter URL if it moved. "
1185
+ "Keep the prior snapshot until a clean file is published."
1186
+ )
1187
+ return (
1188
+ f"OIG published a new monthly LEIE file (pinned {expected}, observed {observed}). "
1189
+ "Review the change, then re-pin with `hc-source lock init`."
1190
+ )
1191
+
1192
+
1193
+ ADAPTER: SourceAdapter
1194
+
1195
+
1196
+ class LeieAdapter(SourceAdapter):
1197
+ source_id = SOURCE_ID
1198
+ contract = CONTRACT
1199
+
1200
+ # These canaries read a live CMS service, so a single 502 is not news --
1201
+ # it is Tuesday. Two retries with exponential backoff, applied only to
1202
+ # transient failures (see hc_source.doctor._observe). A gate that goes red
1203
+ # on somebody else's bad afternoon gets uninstalled.
1204
+ canary_retries = 2
1205
+
1206
+ def canaries(self) -> list[Canary]:
1207
+ return [
1208
+ Canary(
1209
+ canary_id="leie.full_file",
1210
+ source_id=SOURCE_ID,
1211
+ description="Presence, type, size band, and monthly freshness of UPDATED.csv.",
1212
+ observe=_observe_full_file,
1213
+ remediation=(
1214
+ f"UPDATED.csv moved, went stale, or changed shape. Open {HUB_URL}, "
1215
+ "locate the 'Updated LEIE Database (CSV)' link, update the adapter "
1216
+ "URL if needed, then re-pin with `hc-source lock init`."
1217
+ ),
1218
+ remediation_for=_full_file_remediation,
1219
+ ),
1220
+ Canary(
1221
+ canary_id="leie.header",
1222
+ source_id=SOURCE_ID,
1223
+ description="The locked 18-column CSV header of UPDATED.csv.",
1224
+ observe=_observe_header,
1225
+ remediation=(
1226
+ "LEIE column layout changed. Diff the live header against the locked "
1227
+ f"18 columns, fetch the current record layout from {RECORD_LAYOUT_URL} "
1228
+ "(follows a 301), regenerate the field map, bump TRANSFORM_VERSION in "
1229
+ "hc_source/adapters/leie.py, then re-pin with `hc-source lock init`. "
1230
+ "Do not ingest until the new header is locked."
1231
+ ),
1232
+ ),
1233
+ Canary(
1234
+ canary_id="leie.content",
1235
+ source_id=SOURCE_ID,
1236
+ description=(
1237
+ "Canonicalised content hash of the full file (styling- and "
1238
+ "order-independent), plus the REINDATE-null invariant."
1239
+ ),
1240
+ observe=_observe_content,
1241
+ remediation=(
1242
+ "LEIE full-file content moved — a new monthly release or an in-place "
1243
+ "repost. Review the change, then re-pin with `hc-source lock init`. "
1244
+ "If the observed value starts with 'reindate-populated', OIG changed "
1245
+ f"full-file semantics: re-read {QUICK_TIPS_URL} before ingesting. If "
1246
+ "it starts with 'sample:', the live source was unreachable at "
1247
+ "observation time — re-pin with network access."
1248
+ ),
1249
+ ),
1250
+ Canary(
1251
+ canary_id="leie.waivers",
1252
+ source_id=SOURCE_ID,
1253
+ description=(
1254
+ "Advisory cross-check of CSV waiver rows against the OIG waivers "
1255
+ "page (WARN-only: HTML scrape, never drifts or fails a run)."
1256
+ ),
1257
+ observe=_observe_waivers,
1258
+ # Declared, not merely intended. This canary scrapes an HTML
1259
+ # page OIG restyles at will and reports a cross-check whose
1260
+ # disagreement means "a human should read the waiver letters",
1261
+ # never "your data is wrong". It was already described as
1262
+ # WARN-only; now the exit code agrees with the description.
1263
+ severity=CanarySeverity.ADVISORY,
1264
+ remediation=(
1265
+ f"Advisory only: waiver rows in UPDATED.csv disagree with {WAIVERS_URL}. "
1266
+ "Waiver scope is defined by the letters on that page, not the CSV; "
1267
+ "flag affected records as 'waiver status uncertain — see the OIG "
1268
+ "waiver letter' until the next monthly file."
1269
+ ),
1270
+ ),
1271
+ ]
1272
+
1273
+ def tools(self) -> list[ToolSpec]:
1274
+ return [
1275
+ ToolSpec(
1276
+ name="leie.check_npi",
1277
+ description=(
1278
+ "Deterministic NPI-exact check against the current monthly LEIE "
1279
+ "full-file snapshot; a miss never clears a provider (~89.5% of "
1280
+ "records carry no NPI)."
1281
+ ),
1282
+ params_model=CheckNpiParams,
1283
+ handler=_check_npi,
1284
+ tags=("lookup", "exclusions"),
1285
+ ),
1286
+ ToolSpec(
1287
+ name="leie.candidate_search",
1288
+ description=(
1289
+ "Name-based CANDIDATE matching in the LEIE snapshot — candidates "
1290
+ "only, never identity verification; verify at "
1291
+ "https://exclusions.oig.hhs.gov/."
1292
+ ),
1293
+ params_model=CandidateSearchParams,
1294
+ handler=_candidate_search,
1295
+ tags=("search", "exclusions"),
1296
+ ),
1297
+ ToolSpec(
1298
+ name="leie.refresh_status",
1299
+ description=(
1300
+ "Freshness of the LEIE snapshot versus OIG's monthly by-the-10th "
1301
+ "cadence, plus the 12-month supplement retention limit."
1302
+ ),
1303
+ params_model=NoParams,
1304
+ handler=_refresh_status,
1305
+ tags=("metadata",),
1306
+ ),
1307
+ ]
1308
+
1309
+
1310
+ ADAPTER = LeieAdapter()