sourcelock 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hc_source/__init__.py +5 -0
- hc_source/adapters/__init__.py +500 -0
- hc_source/adapters/_demo.py +258 -0
- hc_source/adapters/_demo_fixture.json +25 -0
- hc_source/adapters/_leie_sample.csv +15 -0
- hc_source/adapters/codes.py +1232 -0
- hc_source/adapters/coverage.py +1569 -0
- hc_source/adapters/hcc.py +1450 -0
- hc_source/adapters/leie.py +1310 -0
- hc_source/adapters/provider.py +1159 -0
- hc_source/cache.py +664 -0
- hc_source/cli.py +959 -0
- hc_source/cli_manifest.py +207 -0
- hc_source/data/codes/hcpcs_2026q3.csv.gz +0 -0
- hc_source/data/codes/icd10cm_fy2026.csv.gz +0 -0
- hc_source/data/codes/icd10cm_fy2027.csv.gz +0 -0
- hc_source/data/codes/manifest.json +75 -0
- hc_source/data/codes/regenerate.py +291 -0
- hc_source/data/hcc/hcc_data.json.zlib +0 -0
- hc_source/doctor.py +472 -0
- hc_source/guard.py +877 -0
- hc_source/http.py +541 -0
- hc_source/interfaces.py +395 -0
- hc_source/lockfile.py +236 -0
- hc_source/manifest.py +422 -0
- hc_source/mcp_server.py +203 -0
- hc_source/npi.py +50 -0
- hc_source/receipts.py +74 -0
- hc_source/schemas.py +339 -0
- sourcelock-0.1.0.dist-info/METADATA +272 -0
- sourcelock-0.1.0.dist-info/RECORD +34 -0
- sourcelock-0.1.0.dist-info/WHEEL +4 -0
- sourcelock-0.1.0.dist-info/entry_points.txt +2 -0
- sourcelock-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1310 @@
|
|
|
1
|
+
"""OIG LEIE (List of Excluded Individuals/Entities) route adapter.
|
|
2
|
+
|
|
3
|
+
Source
|
|
4
|
+
------
|
|
5
|
+
The authority is the monthly full database CSV at
|
|
6
|
+
``https://oig.hhs.gov/exclusions/downloadables/UPDATED.csv`` (~15.5 MB,
|
|
7
|
+
~84k records, replaced in place by the 10th of each month; the data month is
|
|
8
|
+
the month before the file's ``Last-Modified``). There is no API and OIG states
|
|
9
|
+
there are no plans for one; file downloads are the only machine interface.
|
|
10
|
+
The working table is built at call time from that file and cached in-process,
|
|
11
|
+
then revalidated on the ``SNAPSHOT_REVALIDATE_AFTER`` cadence against the
|
|
12
|
+
upstream ``Last-Modified``; nothing about the full file is vendored except a
|
|
13
|
+
14-row sample (``_leie_sample.csv``, provenance in
|
|
14
|
+
``tests/fixtures/leie/provenance.json``).
|
|
15
|
+
|
|
16
|
+
Fail-closed answers
|
|
17
|
+
-------------------
|
|
18
|
+
An exclusion screen has one dangerous direction: reporting a person clear.
|
|
19
|
+
Two conditions make a negative answer unearned, and in both the tools answer
|
|
20
|
+
``screen_result: "indeterminate"`` rather than a miss, omit the field a caller
|
|
21
|
+
would read as a verdict (``excluded_in_snapshot``, ``count``), and add a
|
|
22
|
+
non-claim naming the gap:
|
|
23
|
+
|
|
24
|
+
* any row the CSV parser could not read (``MAX_UNPARSED_ROWS``) — that row was
|
|
25
|
+
never compared, so "not found" is not a finding;
|
|
26
|
+
* a matched record carrying a REINDATE. OIG removes reinstated parties from
|
|
27
|
+
the full file rather than dating them, so a populated one means the file no
|
|
28
|
+
longer means what this adapter was built against. The date is surfaced and
|
|
29
|
+
neither "excluded" nor "not excluded" is claimed; there are no reinstatement
|
|
30
|
+
semantics here to claim either with.
|
|
31
|
+
|
|
32
|
+
Verified file facts (empirical, 2026-08-01; dossier addendum §10)
|
|
33
|
+
-----------------------------------------------------------------
|
|
34
|
+
* Pure 7-bit ASCII, no BOM, CRLF line endings — across UPDATED.csv and the
|
|
35
|
+
monthly supplements. The parser decodes ``utf-8-sig`` and falls back to
|
|
36
|
+
``latin-1`` with a receipt warning rather than crash on a future stray byte.
|
|
37
|
+
* Two CSV stylings exist: fully-quoted with ``00000000`` date-nulls
|
|
38
|
+
(UPDATED.csv, current supplements) and unquoted with bare ``0`` nulls for
|
|
39
|
+
dates AND NPIs (older supplements, e.g. 2501excl.csv). Null normalisation
|
|
40
|
+
treats ``""``/``"0"``/``"00000000"`` (dates) and ``""``/``"0"``/
|
|
41
|
+
``"0000000000"`` (NPI) as null.
|
|
42
|
+
* Drift is keyed on a canonicalised content hash (parsed rows, nulls
|
|
43
|
+
normalised, order-independent), never on raw bytes, so an in-place repost
|
|
44
|
+
or a styling change alone never cries wolf. Receipts still carry the raw
|
|
45
|
+
``sha256`` of the exact bytes fetched.
|
|
46
|
+
|
|
47
|
+
Matching contract (v1 decision)
|
|
48
|
+
-------------------------------
|
|
49
|
+
``leie.check_npi`` is the deterministic primary tool: NPI-exact against the
|
|
50
|
+
snapshot. Only ~10.5% of LEIE records carry an NPI, so an NPI miss never
|
|
51
|
+
clears a provider. ``leie.candidate_search`` is name-based and explicitly
|
|
52
|
+
candidate-only: the bulk file has no SSN/EIN (Privacy Act), so identity is
|
|
53
|
+
verified only by a human at ``https://exclusions.oig.hhs.gov/`` (OIG's own
|
|
54
|
+
doctrine). Every tool also carries the state-exclusion-list non-claim: the
|
|
55
|
+
LEIE is federal; state Medicaid exclusion lists are separate and unchecked.
|
|
56
|
+
|
|
57
|
+
Zero-PHI posture
|
|
58
|
+
----------------
|
|
59
|
+
Exclusion records are public, but DOB and street ADDRESS are excluded from
|
|
60
|
+
every tool result (identity confirmation happens at OIG's site anyway), and
|
|
61
|
+
the vendored sample/fixtures contain no real-person DOB.
|
|
62
|
+
|
|
63
|
+
Offline-harness floor
|
|
64
|
+
---------------------
|
|
65
|
+
The shared test suite runs every adapter's canaries with sockets blocked
|
|
66
|
+
(the blocker raises a plain ``RuntimeError``, which is not a transport
|
|
67
|
+
failure). Canaries therefore catch that one case and observe the packaged
|
|
68
|
+
sample instead, with self-describing ``sample:*`` values, so an
|
|
69
|
+
offline ``lock init``/``doctor`` is green and honest. Real transport
|
|
70
|
+
failures surface as :class:`SourceUnreachable` and still report
|
|
71
|
+
``unreachable``. Tools never take this fallback: a compliance answer must
|
|
72
|
+
never come from a 14-row sample.
|
|
73
|
+
|
|
74
|
+
Deviation from the dossier's canary plan: C3 (newest-supplement presence
|
|
75
|
+
probe) is not shipped as a canary because it has no offline-observable
|
|
76
|
+
equivalent under the shared no-network suite; its freshness question is
|
|
77
|
+
answered by ``leie.refresh_status`` and the ``leie.full_file`` staleness
|
|
78
|
+
check instead.
|
|
79
|
+
"""
|
|
80
|
+
|
|
81
|
+
from __future__ import annotations
|
|
82
|
+
|
|
83
|
+
import csv
|
|
84
|
+
import hashlib
|
|
85
|
+
import json
|
|
86
|
+
import os
|
|
87
|
+
import time
|
|
88
|
+
from dataclasses import dataclass, replace
|
|
89
|
+
from datetime import date, datetime, timedelta, timezone
|
|
90
|
+
from email.utils import parsedate_to_datetime
|
|
91
|
+
from pathlib import Path
|
|
92
|
+
from typing import Mapping
|
|
93
|
+
|
|
94
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
95
|
+
|
|
96
|
+
from ..http import FetchResult, SourceUnreachable, as_cache_hit, fetch
|
|
97
|
+
from ..interfaces import Canary, CanaryObservation, SourceAdapter, ToolResult, ToolSpec
|
|
98
|
+
from ..npi import NPI_PATTERN, validate_npi
|
|
99
|
+
from ..receipts import build_receipt
|
|
100
|
+
from ..schemas import CanarySeverity, CanaryStatus, SourceContract
|
|
101
|
+
|
|
102
|
+
SOURCE_ID = "leie"
|
|
103
|
+
#: 2: answers became fail-closed. Tool results gained ``screen_result`` and
|
|
104
|
+
#: ``snapshot_complete``, records gained ``reinstatement_date``, and the two
|
|
105
|
+
#: cases that used to answer negatively without having screened — unparsed rows,
|
|
106
|
+
#: a populated REINDATE — now answer ``indeterminate``.
|
|
107
|
+
TRANSFORM_VERSION = "2"
|
|
108
|
+
|
|
109
|
+
AUTHORITY_URL = "https://oig.hhs.gov/exclusions/downloadables/UPDATED.csv"
|
|
110
|
+
HUB_URL = "https://oig.hhs.gov/exclusions/leie-database-supplement-downloads/"
|
|
111
|
+
WAIVERS_URL = "https://oig.hhs.gov/exclusions/waivers/"
|
|
112
|
+
VERIFICATION_URL = "https://exclusions.oig.hhs.gov/"
|
|
113
|
+
QUICK_TIPS_URL = "https://oig.hhs.gov/exclusions/leie-quick-tips-instructions/"
|
|
114
|
+
RECORD_LAYOUT_URL = "https://oig.hhs.gov/exclusions/files/leie_record_layout.pdf"
|
|
115
|
+
|
|
116
|
+
#: Override the full-file location (any URL, including file://) — tests, mirrors.
|
|
117
|
+
FULL_FILE_ENV_VAR = "HC_SOURCE_LEIE_FULL_FILE"
|
|
118
|
+
#: Override the waivers-page location — tests.
|
|
119
|
+
WAIVERS_ENV_VAR = "HC_SOURCE_LEIE_WAIVERS_URL"
|
|
120
|
+
#: Freeze "today" (ISO date) so freshness checks are testable.
|
|
121
|
+
TODAY_ENV_VAR = "HC_SOURCE_LEIE_TODAY"
|
|
122
|
+
|
|
123
|
+
MIN_EXPECTED_RECORDS = 70_000
|
|
124
|
+
MAX_EXPECTED_RECORDS = 110_000
|
|
125
|
+
MIN_EXPECTED_BYTES = 8_000_000
|
|
126
|
+
MAX_EXPECTED_BYTES = 40_000_000
|
|
127
|
+
STALE_AFTER_DAYS = 45
|
|
128
|
+
MAX_CANDIDATES = 25
|
|
129
|
+
|
|
130
|
+
#: How many rows the CSV parser may fail to read before an answer stops being a
|
|
131
|
+
#: screen. Zero on purpose. A row whose field count does not match the header is
|
|
132
|
+
#: a row nobody compared the subject against, and the old behaviour -- count it,
|
|
133
|
+
#: skip it, answer anyway -- turned an excluded party sitting in a malformed row
|
|
134
|
+
#: into a clean "not found". A miss is the dangerous direction on an exclusion
|
|
135
|
+
#: list, so any unparsed row makes a miss indeterminate rather than negative.
|
|
136
|
+
MAX_UNPARSED_ROWS = 0
|
|
137
|
+
|
|
138
|
+
EXPECTED_COLUMNS = (
|
|
139
|
+
"LASTNAME", "FIRSTNAME", "MIDNAME", "BUSNAME", "GENERAL", "SPECIALTY",
|
|
140
|
+
"UPIN", "NPI", "DOB", "ADDRESS", "CITY", "STATE", "ZIP",
|
|
141
|
+
"EXCLTYPE", "EXCLDATE", "REINDATE", "WAIVERDATE", "WVRSTATE",
|
|
142
|
+
)
|
|
143
|
+
#: The record-layout PDF spells the last column WAIVERSTATE; the CSV says
|
|
144
|
+
#: WVRSTATE. Parse by the CSV header, accept either spelling.
|
|
145
|
+
_WVRSTATE_ALIASES = {"WVRSTATE", "WAIVERSTATE"}
|
|
146
|
+
|
|
147
|
+
DATE_COLUMNS = ("DOB", "EXCLDATE", "REINDATE", "WAIVERDATE")
|
|
148
|
+
_NULL_DATES = {"", "0", "00000000"}
|
|
149
|
+
_NULL_NPIS = {"", "0", "0000000000"}
|
|
150
|
+
|
|
151
|
+
#: EXCLTYPE domain observed in the 2026-07-10 full file. Opaque tokens: mixed
|
|
152
|
+
#: case and embedded spaces are real — exact-match, never regex-parse.
|
|
153
|
+
KNOWN_EXCLTYPES = frozenset({
|
|
154
|
+
"1128a1", "1128a2", "1128a3", "1128a4", "1128Aa",
|
|
155
|
+
"1128b1", "1128b2", "1128b3", "1128b4", "1128b5", "1128b6", "1128b7",
|
|
156
|
+
"1128b8", "1128b11", "1128b12", "1128b14", "1128b15", "1128b16",
|
|
157
|
+
"1156", "1160", "BRCH SA", "BRCH CIA",
|
|
158
|
+
})
|
|
159
|
+
|
|
160
|
+
CONTRACT = SourceContract(
|
|
161
|
+
source_id=SOURCE_ID,
|
|
162
|
+
authority_url=AUTHORITY_URL,
|
|
163
|
+
fallback_url=None,
|
|
164
|
+
license_notes=(
|
|
165
|
+
"US federal public-domain data; keyless anonymous download, no license or "
|
|
166
|
+
"click-through. SSNs/EINs are deliberately excluded from the file (Privacy Act), "
|
|
167
|
+
"so a bulk-file hit is never identity-verified — verification happens at "
|
|
168
|
+
"https://exclusions.oig.hhs.gov/. Contains no AMA CPT content. Redistribution "
|
|
169
|
+
"obligation: preserve the candidate-only framing of name matches."
|
|
170
|
+
),
|
|
171
|
+
cadence="monthly (updated by the 10th; contains exclusions through the end of the prior month)",
|
|
172
|
+
effective_date_semantics=(
|
|
173
|
+
"For record-level answers effective_from is the record's EXCLDATE — the "
|
|
174
|
+
"exclusion's legal effective date, which may postdate the file's data month. "
|
|
175
|
+
"For file-level answers effective_from is the snapshot's Last-Modified date. "
|
|
176
|
+
"effective_to is always open: an exclusion ends only by reinstatement, which "
|
|
177
|
+
"REMOVES the record from the next monthly full file."
|
|
178
|
+
),
|
|
179
|
+
invariants=[
|
|
180
|
+
"the CSV header is exactly the locked 18 columns ending in WVRSTATE",
|
|
181
|
+
"full-file record count stays within 70,000–110,000",
|
|
182
|
+
"REINDATE is null for every record in the full file (reinstated people are removed)",
|
|
183
|
+
"NPI is a placeholder (0000000000/0/blank) on ~89.5% of records",
|
|
184
|
+
"the file is plain ASCII CSV; no SSN or EIN column exists",
|
|
185
|
+
],
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# ---------------------------------------------------------------------------
|
|
189
|
+
# Non-claims (stable keys; reviewers read these)
|
|
190
|
+
# ---------------------------------------------------------------------------
|
|
191
|
+
|
|
192
|
+
NC_STATE = (
|
|
193
|
+
"DOES_NOT_COVER_STATE_EXCLUSIONS: the LEIE is the federal OIG exclusion list only; "
|
|
194
|
+
"state Medicaid exclusion and termination lists are separate and are not checked "
|
|
195
|
+
"here — an LEIE miss does not mean the provider is excluded nowhere."
|
|
196
|
+
)
|
|
197
|
+
NC_CURRENCY = (
|
|
198
|
+
"DOES_NOT_PROVE_CURRENCY: this answer comes from the monthly LEIE full-file "
|
|
199
|
+
"snapshot (replaced by the 10th of each month, data through the end of the prior "
|
|
200
|
+
"month) and may lag OIG actions by up to a month."
|
|
201
|
+
)
|
|
202
|
+
NC_HISTORY = (
|
|
203
|
+
"DOES_NOT_PROVE_NEVER_EXCLUDED: reinstated individuals are removed from the "
|
|
204
|
+
"monthly full file; absence from the current snapshot says nothing about past "
|
|
205
|
+
"exclusions."
|
|
206
|
+
)
|
|
207
|
+
NC_NPI_COVERAGE = (
|
|
208
|
+
"DOES_NOT_CLEAR_ON_NPI_MISS: ~89.5% of LEIE records carry no NPI, so an NPI-exact "
|
|
209
|
+
"miss does not clear a provider — run leie.candidate_search and verify identity at "
|
|
210
|
+
"the official OIG online workflow: https://exclusions.oig.hhs.gov/."
|
|
211
|
+
)
|
|
212
|
+
NC_CANDIDATE = (
|
|
213
|
+
"NOT_IDENTITY_VERIFICATION: candidate match is NOT identity verification — the "
|
|
214
|
+
"bulk file lacks SSN/EIN (~10.5% of records have NPI); verify at the official OIG "
|
|
215
|
+
"online workflow: https://exclusions.oig.hhs.gov/."
|
|
216
|
+
)
|
|
217
|
+
NC_UNPARSED = (
|
|
218
|
+
"DID_NOT_SCREEN_THE_WHOLE_LIST: part of the snapshot could not be parsed, so rows "
|
|
219
|
+
"that may name this subject were never compared. Any absence of a match here is "
|
|
220
|
+
"indeterminate, not clearance — re-run once the file parses cleanly, and verify at "
|
|
221
|
+
"https://exclusions.oig.hhs.gov/."
|
|
222
|
+
)
|
|
223
|
+
NC_REINSTATEMENT = (
|
|
224
|
+
"DOES_NOT_IMPLEMENT_REINSTATEMENT: a matched record carries a reinstatement date. "
|
|
225
|
+
"OIG removes reinstated parties from the monthly full file, so a record that has "
|
|
226
|
+
"one means the file no longer behaves the way this adapter was built against. "
|
|
227
|
+
"SourceLock implements no reinstatement semantics and will not call such a record "
|
|
228
|
+
"excluded or not excluded — resolve it at https://exclusions.oig.hhs.gov/."
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
CHECK_NPI_NON_CLAIMS = [NC_STATE, NC_CURRENCY, NC_HISTORY, NC_NPI_COVERAGE]
|
|
232
|
+
CANDIDATE_NON_CLAIMS = [NC_CANDIDATE, NC_STATE, NC_CURRENCY, NC_HISTORY]
|
|
233
|
+
REFRESH_NON_CLAIMS = [NC_STATE, NC_CURRENCY, NC_HISTORY]
|
|
234
|
+
|
|
235
|
+
CADENCE_TEXT = (
|
|
236
|
+
"monthly; OIG updates all exclusion information by the 10th of every month, and "
|
|
237
|
+
"the data month is the month before the file's Last-Modified date"
|
|
238
|
+
)
|
|
239
|
+
RETENTION_TEXT = (
|
|
240
|
+
"OIG archives monthly supplement files for only the previous 12 months; older "
|
|
241
|
+
"supplements are not available upstream. Archive supplements yourself, or "
|
|
242
|
+
"re-baseline from UPDATED.csv when history is lost."
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
# ---------------------------------------------------------------------------
|
|
246
|
+
# Snapshot loading (the runtime working table)
|
|
247
|
+
# ---------------------------------------------------------------------------
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def sample_path() -> Path:
|
|
251
|
+
"""The packaged 14-row sample (DOB-scrubbed; never served by tools)."""
|
|
252
|
+
return Path(__file__).with_name("_leie_sample.csv")
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _sample_uri() -> str:
|
|
256
|
+
return sample_path().as_uri()
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _full_file_url() -> str:
|
|
260
|
+
return os.environ.get(FULL_FILE_ENV_VAR) or AUTHORITY_URL
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _waivers_url() -> str:
|
|
264
|
+
return os.environ.get(WAIVERS_ENV_VAR) or WAIVERS_URL
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _today() -> date:
|
|
268
|
+
override = os.environ.get(TODAY_ENV_VAR)
|
|
269
|
+
return date.fromisoformat(override) if override else date.today()
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
@dataclass(frozen=True)
|
|
273
|
+
class _Snapshot:
|
|
274
|
+
fetch_meta: FetchResult # content stripped; provenance for receipts
|
|
275
|
+
from_sample: bool
|
|
276
|
+
source_version: str
|
|
277
|
+
last_modified: datetime | None
|
|
278
|
+
data_month: str | None
|
|
279
|
+
columns: tuple[str, ...]
|
|
280
|
+
records: tuple[Mapping[str, str], ...]
|
|
281
|
+
npi_index: Mapping[str, tuple[int, ...]]
|
|
282
|
+
canonical_hash: str
|
|
283
|
+
schema_hash: str
|
|
284
|
+
reindate_populated: int
|
|
285
|
+
unknown_excltypes: tuple[str, ...]
|
|
286
|
+
parse_warnings: tuple[str, ...]
|
|
287
|
+
malformed_rows: int
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
@dataclass(frozen=True)
|
|
291
|
+
class _CacheEntry:
|
|
292
|
+
"""One cached snapshot plus what we know about how current it is."""
|
|
293
|
+
|
|
294
|
+
snapshot: _Snapshot
|
|
295
|
+
#: Monotonic mark of the last revalidation ATTEMPT, successful or not. This
|
|
296
|
+
#: is what throttles retries, so a source that is down is probed once per
|
|
297
|
+
#: cadence instead of on every tool call.
|
|
298
|
+
checked_at: float
|
|
299
|
+
#: Monotonic mark of the last time upstream actually answered — either these
|
|
300
|
+
#: bytes were read, or a probe confirmed upstream had not moved.
|
|
301
|
+
verified_at: float
|
|
302
|
+
#: Warnings about the entry itself (a failed revalidation), as opposed to
|
|
303
|
+
#: the snapshot's own parse warnings.
|
|
304
|
+
warnings: tuple[str, ...] = ()
|
|
305
|
+
#: True when THIS hand-out came out of the in-process cache rather than off
|
|
306
|
+
#: the wire. Set on the copy that is returned, never on the stored entry:
|
|
307
|
+
#: the first hand-out of a freshly downloaded snapshot is not a cache hit,
|
|
308
|
+
#: and every one after it is.
|
|
309
|
+
reused: bool = False
|
|
310
|
+
#: Wall-clock moment upstream last confirmed these bytes are still current
|
|
311
|
+
#: -- a probe whose Last-Modified or hash matched. ``None`` means no probe
|
|
312
|
+
#: has ever confirmed them, so the fetch's own stamp stands. Never advanced
|
|
313
|
+
#: by a reuse that contacted nobody: "still current as of a moment ago" and
|
|
314
|
+
#: "still what we downloaded on Tuesday" are different claims.
|
|
315
|
+
confirmed_at: datetime | None = None
|
|
316
|
+
#: True when revalidation FAILED and this snapshot is being served anyway.
|
|
317
|
+
stale: bool = False
|
|
318
|
+
|
|
319
|
+
def fetch_meta(self) -> FetchResult:
|
|
320
|
+
"""Provenance for a receipt built from this hand-out.
|
|
321
|
+
|
|
322
|
+
The receipt's ``cache_hit`` used to come straight off the original
|
|
323
|
+
FetchResult, so an answer computed from a snapshot downloaded twenty
|
|
324
|
+
minutes ago said ``cache_hit: false`` in its structured field while the
|
|
325
|
+
warning above it said it came from an in-process cache. A machine
|
|
326
|
+
reading the receipt believes the field.
|
|
327
|
+
"""
|
|
328
|
+
meta = self.snapshot.fetch_meta
|
|
329
|
+
if not self.reused:
|
|
330
|
+
return meta
|
|
331
|
+
if self.confirmed_at is not None:
|
|
332
|
+
return as_cache_hit(meta, revalidated_at=self.confirmed_at)
|
|
333
|
+
return as_cache_hit(meta)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
_SNAPSHOT_CACHE: dict[str, _CacheEntry] = {}
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def clear_cache() -> None:
|
|
340
|
+
_SNAPSHOT_CACHE.clear()
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _schema_hash(columns: list[str] | tuple[str, ...]) -> str:
|
|
344
|
+
shape = json.dumps(list(columns), separators=(",", ":"))
|
|
345
|
+
return hashlib.sha256(shape.encode("utf-8")).hexdigest()
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _decode(content: bytes) -> tuple[str, list[str]]:
|
|
349
|
+
try:
|
|
350
|
+
return content.decode("utf-8-sig"), []
|
|
351
|
+
except UnicodeDecodeError:
|
|
352
|
+
return content.decode("latin-1"), [
|
|
353
|
+
"full file contained non-UTF-8 bytes and was decoded as latin-1; upstream "
|
|
354
|
+
"was pure ASCII when last verified (2026-08-01) — review the raw file."
|
|
355
|
+
]
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _canonical_row(rec: Mapping[str, str]) -> str:
|
|
359
|
+
parts = []
|
|
360
|
+
for col in EXPECTED_COLUMNS:
|
|
361
|
+
v = rec.get(col, "").strip()
|
|
362
|
+
if col in DATE_COLUMNS and v in _NULL_DATES:
|
|
363
|
+
v = ""
|
|
364
|
+
elif col == "NPI" and v in _NULL_NPIS:
|
|
365
|
+
v = ""
|
|
366
|
+
parts.append(v)
|
|
367
|
+
return "\x1f".join(parts)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def _parse_last_modified(headers: Mapping[str, str]) -> datetime | None:
|
|
371
|
+
raw = headers.get("last-modified")
|
|
372
|
+
if not raw:
|
|
373
|
+
return None
|
|
374
|
+
try:
|
|
375
|
+
return parsedate_to_datetime(raw)
|
|
376
|
+
except (TypeError, ValueError):
|
|
377
|
+
return None
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _data_month(lm_date: date) -> str:
|
|
381
|
+
year, month = lm_date.year, lm_date.month - 1
|
|
382
|
+
if month == 0:
|
|
383
|
+
year, month = year - 1, 12
|
|
384
|
+
return f"{year}-{month:02d}"
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _parse_snapshot(result: FetchResult, *, expect_full: bool, from_sample: bool) -> _Snapshot:
|
|
388
|
+
text, warnings = _decode(result.content)
|
|
389
|
+
rows = list(csv.reader(text.splitlines()))
|
|
390
|
+
if not rows:
|
|
391
|
+
raise SourceUnreachable(result.url, "full file is empty")
|
|
392
|
+
|
|
393
|
+
header = [h.strip() for h in rows[0]]
|
|
394
|
+
lookup: dict[str, int] = {}
|
|
395
|
+
for idx, name in enumerate(header):
|
|
396
|
+
canonical = "WVRSTATE" if name in _WVRSTATE_ALIASES else name
|
|
397
|
+
lookup.setdefault(canonical, idx)
|
|
398
|
+
missing = [c for c in EXPECTED_COLUMNS if c not in lookup]
|
|
399
|
+
if missing:
|
|
400
|
+
raise SourceUnreachable(
|
|
401
|
+
result.url,
|
|
402
|
+
f"column layout changed: missing column(s) {missing}; fetch the current "
|
|
403
|
+
f"record layout from {RECORD_LAYOUT_URL} and update the adapter",
|
|
404
|
+
)
|
|
405
|
+
if tuple(header) != EXPECTED_COLUMNS:
|
|
406
|
+
warnings.append(f"header differs from the locked layout: {header}")
|
|
407
|
+
|
|
408
|
+
records: list[dict[str, str]] = []
|
|
409
|
+
malformed = 0
|
|
410
|
+
for row in rows[1:]:
|
|
411
|
+
if not row:
|
|
412
|
+
continue
|
|
413
|
+
if len(row) != len(header):
|
|
414
|
+
malformed += 1
|
|
415
|
+
continue
|
|
416
|
+
records.append({c: row[lookup[c]].strip() for c in EXPECTED_COLUMNS})
|
|
417
|
+
if malformed:
|
|
418
|
+
warnings.append(
|
|
419
|
+
f"{malformed} row(s) did not match the header's field count and could not "
|
|
420
|
+
"be parsed; the subjects named in them were never compared"
|
|
421
|
+
)
|
|
422
|
+
|
|
423
|
+
count = len(records)
|
|
424
|
+
if expect_full and not (MIN_EXPECTED_RECORDS <= count <= MAX_EXPECTED_RECORDS):
|
|
425
|
+
raise SourceUnreachable(
|
|
426
|
+
result.url,
|
|
427
|
+
f"record count {count} outside the sanity band "
|
|
428
|
+
f"[{MIN_EXPECTED_RECORDS}, {MAX_EXPECTED_RECORDS}] — treating the download "
|
|
429
|
+
"as truncated or corrupt; keep the prior snapshot and retry in 24h",
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
npi_index: dict[str, tuple[int, ...]] = {}
|
|
433
|
+
for j, rec in enumerate(records):
|
|
434
|
+
npi = rec["NPI"]
|
|
435
|
+
if npi not in _NULL_NPIS:
|
|
436
|
+
npi_index[npi] = npi_index.get(npi, ()) + (j,)
|
|
437
|
+
|
|
438
|
+
canonical = hashlib.sha256(
|
|
439
|
+
"\n".join(sorted(_canonical_row(r) for r in records)).encode("utf-8")
|
|
440
|
+
).hexdigest()
|
|
441
|
+
|
|
442
|
+
lm = _parse_last_modified(result.headers)
|
|
443
|
+
if from_sample:
|
|
444
|
+
source_version = f"sample:{canonical[:8]}"
|
|
445
|
+
elif lm is not None:
|
|
446
|
+
source_version = lm.date().isoformat()
|
|
447
|
+
else:
|
|
448
|
+
source_version = f"canonical:{canonical[:12]}"
|
|
449
|
+
|
|
450
|
+
return _Snapshot(
|
|
451
|
+
fetch_meta=replace(result, content=b""),
|
|
452
|
+
from_sample=from_sample,
|
|
453
|
+
source_version=source_version,
|
|
454
|
+
last_modified=lm,
|
|
455
|
+
data_month=_data_month(lm.date()) if lm else None,
|
|
456
|
+
columns=tuple(header),
|
|
457
|
+
records=tuple(records),
|
|
458
|
+
npi_index=npi_index,
|
|
459
|
+
canonical_hash=canonical,
|
|
460
|
+
schema_hash=_schema_hash(header),
|
|
461
|
+
reindate_populated=sum(1 for r in records if r["REINDATE"] not in _NULL_DATES),
|
|
462
|
+
unknown_excltypes=tuple(sorted({r["EXCLTYPE"] for r in records} - KNOWN_EXCLTYPES)),
|
|
463
|
+
parse_warnings=tuple(warnings),
|
|
464
|
+
malformed_rows=malformed,
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
#: The LEIE full file measured 14.8 MB on 2026-08-01 and grows monotonically as
|
|
469
|
+
#: exclusions accumulate. 64 MiB is roughly 4x that: enough headroom for years
|
|
470
|
+
#: of growth, tight enough that a hostile or broken mirror cannot hand this
|
|
471
|
+
#: process an unbounded body. Raising it is a deliberate act, not a default.
|
|
472
|
+
LEIE_MAX_BYTES = 64 * 1024 * 1024
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
#: How long a cached snapshot may be served before upstream is asked whether it
|
|
476
|
+
#: moved. OIG replaces UPDATED.csv by the 10th of every month, but not on a
|
|
477
|
+
#: fixed day, and NC_CURRENCY promises callers a lag of at most a month. The
|
|
478
|
+
#: cache used to have no expiry at all, so a long-lived process (the MCP server)
|
|
479
|
+
#: pinned the first snapshot it ever read and kept printing that promise —
|
|
480
|
+
#: after two publication cycles the promise was simply false. One day is the
|
|
481
|
+
#: interval that keeps it true with margin: a month-long interval would let a
|
|
482
|
+
#: server answer from a file two releases old and still be "within cadence",
|
|
483
|
+
#: and the check costs one ~2 KB ranged GET per day per process.
|
|
484
|
+
SNAPSHOT_REVALIDATE_AFTER = timedelta(days=1)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _monotonic() -> float:
|
|
488
|
+
"""Elapsed-time clock for the cache. Monotonic, so a wall-clock jump or an
|
|
489
|
+
NTP correction can neither expire a fresh snapshot nor freeze a stale one."""
|
|
490
|
+
return time.monotonic()
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _load_snapshot(url: str, *, from_sample: bool) -> _Snapshot:
|
|
494
|
+
"""Read and parse the full file. Always goes upstream; caching is _cached_entry's job."""
|
|
495
|
+
result = fetch(url, max_bytes=LEIE_MAX_BYTES)
|
|
496
|
+
return _parse_snapshot(result, expect_full=not from_sample, from_sample=from_sample)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _store(url: str, snapshot: _Snapshot) -> _CacheEntry:
|
|
500
|
+
mark = _monotonic()
|
|
501
|
+
entry = _CacheEntry(snapshot=snapshot, checked_at=mark, verified_at=mark)
|
|
502
|
+
# One dict assignment, after the new snapshot is fully parsed. A reader
|
|
503
|
+
# concurrent with a revalidation sees either the whole old entry or the
|
|
504
|
+
# whole new one, never a half-swapped table.
|
|
505
|
+
_SNAPSHOT_CACHE[url] = entry
|
|
506
|
+
return entry
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _confirm(url: str, entry: _CacheEntry) -> _CacheEntry:
|
|
510
|
+
"""Upstream had not moved. Keep the bytes — and their retrieval stamp.
|
|
511
|
+
|
|
512
|
+
Nothing was re-read here, so re-stamping ``retrieved_at`` would put a read
|
|
513
|
+
that never happened on every receipt these bytes go on to support.
|
|
514
|
+
``confirmed_at`` is the field that DOES move: upstream really did just say
|
|
515
|
+
these bytes are current, and that is the one thing this branch establishes.
|
|
516
|
+
"""
|
|
517
|
+
mark = _monotonic()
|
|
518
|
+
confirmed = replace(
|
|
519
|
+
entry,
|
|
520
|
+
checked_at=mark,
|
|
521
|
+
verified_at=mark,
|
|
522
|
+
warnings=(),
|
|
523
|
+
reused=True,
|
|
524
|
+
stale=False,
|
|
525
|
+
confirmed_at=datetime.now(timezone.utc),
|
|
526
|
+
)
|
|
527
|
+
_SNAPSHOT_CACHE[url] = confirmed
|
|
528
|
+
return confirmed
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _keep_stale(url: str, entry: _CacheEntry, reason: str) -> _CacheEntry:
|
|
532
|
+
"""Revalidation failed. Serve the old snapshot and say how old it is.
|
|
533
|
+
|
|
534
|
+
Discarding a good snapshot because the network blinked would turn a
|
|
535
|
+
transient outage into no answer at all; pretending it is current would be
|
|
536
|
+
worse. The warning names the retrieval stamp so the caller can price it.
|
|
537
|
+
"""
|
|
538
|
+
confirmed = entry.confirmed_at or entry.snapshot.fetch_meta.revalidated_at
|
|
539
|
+
warning = (
|
|
540
|
+
f"STALE: LEIE snapshot could not be revalidated ({reason}); still serving the "
|
|
541
|
+
f"snapshot retrieved {entry.snapshot.fetch_meta.retrieved_at.isoformat()} "
|
|
542
|
+
f"(source_version {entry.snapshot.source_version}), last confirmed current by "
|
|
543
|
+
f"upstream {confirmed.isoformat() if confirmed else 'never'}. The receipt's "
|
|
544
|
+
f"cache_hit and revalidated_at say the same thing in machine-readable form. "
|
|
545
|
+
f"This answer is that old — OIG republishes monthly, so re-check {HUB_URL} if "
|
|
546
|
+
"the outage persists."
|
|
547
|
+
)
|
|
548
|
+
stale = replace(
|
|
549
|
+
entry, checked_at=_monotonic(), warnings=(warning,), reused=True, stale=True
|
|
550
|
+
)
|
|
551
|
+
_SNAPSHOT_CACHE[url] = stale
|
|
552
|
+
return stale
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def _revalidate(url: str, entry: _CacheEntry) -> _CacheEntry:
|
|
556
|
+
try:
|
|
557
|
+
probe = _probe(url)
|
|
558
|
+
cached_lm = entry.snapshot.last_modified
|
|
559
|
+
probe_lm = _parse_last_modified(probe.headers)
|
|
560
|
+
if cached_lm is not None and probe_lm is not None and probe_lm == cached_lm:
|
|
561
|
+
return _confirm(url, entry)
|
|
562
|
+
if _content_range_total(probe.headers) is None and probe.sha256 == entry.snapshot.fetch_meta.sha256:
|
|
563
|
+
# No Content-Range means the probe body IS the whole body (a file://
|
|
564
|
+
# mirror, or a server that ignored the Range header), so its hash is
|
|
565
|
+
# comparable to the full-file hash on the receipt. Same bytes, no
|
|
566
|
+
# re-parse. Never compare a 206's hash: that is 2 KB, not the file.
|
|
567
|
+
return _confirm(url, entry)
|
|
568
|
+
snapshot = _load_snapshot(url, from_sample=False)
|
|
569
|
+
except SourceUnreachable as exc:
|
|
570
|
+
return _keep_stale(url, entry, exc.reason)
|
|
571
|
+
except RuntimeError as exc:
|
|
572
|
+
# The offline harness's socket blocker raises a plain RuntimeError. It
|
|
573
|
+
# is not a transport failure, but for a cached snapshot the handling is
|
|
574
|
+
# the same: keep what we have and say it was not rechecked.
|
|
575
|
+
return _keep_stale(url, entry, type(exc).__name__)
|
|
576
|
+
return _store(url, snapshot)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _cached_entry(*, from_sample: bool = False) -> _CacheEntry:
|
|
580
|
+
url = _sample_uri() if from_sample else _full_file_url()
|
|
581
|
+
entry = _SNAPSHOT_CACHE.get(url)
|
|
582
|
+
if entry is None:
|
|
583
|
+
return _store(url, _load_snapshot(url, from_sample=from_sample))
|
|
584
|
+
if from_sample:
|
|
585
|
+
# The packaged sample ships inside this process. It has no upstream to
|
|
586
|
+
# have moved, so revalidating it would only spend a fetch.
|
|
587
|
+
return replace(entry, reused=True)
|
|
588
|
+
if _monotonic() - entry.checked_at < SNAPSHOT_REVALIDATE_AFTER.total_seconds():
|
|
589
|
+
# Inside the revalidation window: reuse, and say so. Nobody was
|
|
590
|
+
# contacted, so `confirmed_at` stays where it was.
|
|
591
|
+
return replace(entry, reused=True)
|
|
592
|
+
return _revalidate(url, entry)
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def _cached_snapshot(*, from_sample: bool = False) -> _Snapshot:
|
|
596
|
+
return _cached_entry(from_sample=from_sample).snapshot
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def _probe(url: str | None = None) -> FetchResult:
|
|
600
|
+
"""One cheap ranged GET of the full-file URL (headers + first ~2 KB)."""
|
|
601
|
+
return fetch(url or _full_file_url(), headers={"Range": "bytes=0-2047"})
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
# ---------------------------------------------------------------------------
|
|
605
|
+
# Typed public parameters
|
|
606
|
+
# ---------------------------------------------------------------------------
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
class NoParams(BaseModel):
|
|
610
|
+
model_config = ConfigDict(extra="forbid")
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
class CheckNpiParams(BaseModel):
|
|
614
|
+
model_config = ConfigDict(extra="forbid")
|
|
615
|
+
|
|
616
|
+
npi: str = Field(
|
|
617
|
+
pattern=NPI_PATTERN,
|
|
618
|
+
description="10-digit NPI to check against the LEIE full-file snapshot.",
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
@field_validator("npi")
|
|
622
|
+
@classmethod
|
|
623
|
+
def _real_npi(cls, v: str) -> str:
|
|
624
|
+
# The placeholder check runs first because it fails the Luhn too, and
|
|
625
|
+
# "you passed the LEIE's no-NPI sentinel" is the diagnosis the caller
|
|
626
|
+
# needs; "check digit failed" would send them looking for a typo.
|
|
627
|
+
if v == "0000000000":
|
|
628
|
+
raise ValueError(
|
|
629
|
+
"0000000000 is the LEIE placeholder for 'no NPI on record', not a real NPI"
|
|
630
|
+
)
|
|
631
|
+
# Shape is not validity. Ten digits that cannot be an NPI used to reach
|
|
632
|
+
# the snapshot and come back as a confident miss on a number that does
|
|
633
|
+
# not exist, which is exactly the false clearance this adapter must not
|
|
634
|
+
# produce. hc_source.npi is the one validator; provider uses it too.
|
|
635
|
+
return validate_npi(v)
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
_NAME_PATTERN = r"^[A-Za-z0-9][A-Za-z0-9 .,'&#\-]*$"
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
class CandidateSearchParams(BaseModel):
|
|
642
|
+
model_config = ConfigDict(extra="forbid")
|
|
643
|
+
|
|
644
|
+
last_name: str | None = Field(
|
|
645
|
+
default=None, min_length=2, max_length=30, pattern=_NAME_PATTERN,
|
|
646
|
+
description="Individual's last name, matched exactly (case-insensitive).",
|
|
647
|
+
)
|
|
648
|
+
first_name: str | None = Field(
|
|
649
|
+
default=None, min_length=1, max_length=20, pattern=_NAME_PATTERN,
|
|
650
|
+
description="Optional first-name prefix filter; requires last_name.",
|
|
651
|
+
)
|
|
652
|
+
business_name: str | None = Field(
|
|
653
|
+
default=None, min_length=2, max_length=40, pattern=_NAME_PATTERN,
|
|
654
|
+
description="Entity name fragment, matched as a case-insensitive substring.",
|
|
655
|
+
)
|
|
656
|
+
state: str | None = Field(
|
|
657
|
+
default=None, pattern=r"^[A-Za-z]{2}$",
|
|
658
|
+
description="Two-letter address-state filter as it appears in the file.",
|
|
659
|
+
)
|
|
660
|
+
|
|
661
|
+
@field_validator("state")
|
|
662
|
+
@classmethod
|
|
663
|
+
def _upper(cls, v: str | None) -> str | None:
|
|
664
|
+
return v.upper() if v else v
|
|
665
|
+
|
|
666
|
+
@model_validator(mode="after")
|
|
667
|
+
def _exactly_one_subject(self) -> "CandidateSearchParams":
|
|
668
|
+
if bool(self.last_name) == bool(self.business_name):
|
|
669
|
+
raise ValueError(
|
|
670
|
+
"provide exactly one of last_name (individuals) or business_name (entities)"
|
|
671
|
+
)
|
|
672
|
+
if self.first_name and not self.last_name:
|
|
673
|
+
raise ValueError("first_name only narrows a last_name search")
|
|
674
|
+
return self
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
# ---------------------------------------------------------------------------
|
|
678
|
+
# Record projection (public fields only: no DOB, no street address)
|
|
679
|
+
# ---------------------------------------------------------------------------
|
|
680
|
+
|
|
681
|
+
|
|
682
|
+
def _iso_date(v: str) -> str | None:
|
|
683
|
+
if v in _NULL_DATES:
|
|
684
|
+
return None
|
|
685
|
+
if len(v) == 8 and v.isdigit():
|
|
686
|
+
try:
|
|
687
|
+
return date(int(v[:4]), int(v[4:6]), int(v[6:8])).isoformat()
|
|
688
|
+
except ValueError:
|
|
689
|
+
return v
|
|
690
|
+
return v
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def _record_effective(rec: Mapping[str, str]) -> date | None:
|
|
694
|
+
iso = _iso_date(rec["EXCLDATE"])
|
|
695
|
+
try:
|
|
696
|
+
return date.fromisoformat(iso) if iso else None
|
|
697
|
+
except ValueError:
|
|
698
|
+
return None
|
|
699
|
+
|
|
700
|
+
|
|
701
|
+
def _public_record(rec: Mapping[str, str]) -> dict[str, object]:
|
|
702
|
+
waiver = None
|
|
703
|
+
if rec["WAIVERDATE"] not in _NULL_DATES:
|
|
704
|
+
waiver = {"date": _iso_date(rec["WAIVERDATE"]), "state": rec["WVRSTATE"] or None}
|
|
705
|
+
return {
|
|
706
|
+
"record_type": "entity" if rec["BUSNAME"] else "individual",
|
|
707
|
+
"last_name": rec["LASTNAME"] or None,
|
|
708
|
+
"first_name": rec["FIRSTNAME"] or None,
|
|
709
|
+
"mid_name": rec["MIDNAME"] or None,
|
|
710
|
+
"business_name": rec["BUSNAME"] or None,
|
|
711
|
+
"general": rec["GENERAL"] or None,
|
|
712
|
+
"specialty": rec["SPECIALTY"] or None,
|
|
713
|
+
"upin": rec["UPIN"] or None,
|
|
714
|
+
"npi": rec["NPI"] if rec["NPI"] not in _NULL_NPIS else None,
|
|
715
|
+
"city": rec["CITY"] or None,
|
|
716
|
+
"state": rec["STATE"] or None,
|
|
717
|
+
"zip": rec["ZIP"] or None,
|
|
718
|
+
"excltype": rec["EXCLTYPE"],
|
|
719
|
+
"exclusion_date": _iso_date(rec["EXCLDATE"]),
|
|
720
|
+
# Null on every record in a well-formed full file: OIG removes
|
|
721
|
+
# reinstated parties rather than dating them. Surfaced anyway, because
|
|
722
|
+
# the day it is not null the caller needs to see the date, not a verdict
|
|
723
|
+
# this adapter has no semantics for.
|
|
724
|
+
"reinstatement_date": _iso_date(rec["REINDATE"]),
|
|
725
|
+
"waiver": waiver,
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
|
|
729
|
+
def _norm(value: str) -> str:
|
|
730
|
+
return " ".join(value.split()).casefold()
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
# ---------------------------------------------------------------------------
|
|
734
|
+
# Handlers
|
|
735
|
+
# ---------------------------------------------------------------------------
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def _snapshot_date(snapshot: _Snapshot) -> date | None:
|
|
739
|
+
return snapshot.last_modified.date() if snapshot.last_modified else None
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def _check_npi(params: CheckNpiParams) -> ToolResult:
|
|
743
|
+
entry = _cached_entry()
|
|
744
|
+
snapshot = entry.snapshot
|
|
745
|
+
warnings = list(snapshot.parse_warnings) + list(entry.warnings)
|
|
746
|
+
non_claims = list(CHECK_NPI_NON_CLAIMS)
|
|
747
|
+
|
|
748
|
+
unparsed = snapshot.malformed_rows > MAX_UNPARSED_ROWS
|
|
749
|
+
if unparsed:
|
|
750
|
+
non_claims.append(NC_UNPARSED)
|
|
751
|
+
|
|
752
|
+
indices = snapshot.npi_index.get(params.npi, ())
|
|
753
|
+
records = [_public_record(snapshot.records[j]) for j in indices]
|
|
754
|
+
reinstated = [r for r in records if r["reinstatement_date"]]
|
|
755
|
+
|
|
756
|
+
# Everything a caller may read is keyed on screen_result. It is "excluded"
|
|
757
|
+
# or "indeterminate" and never a clear: a clean no-match returns data=None
|
|
758
|
+
# instead, so no shape of this payload can be read as clearance.
|
|
759
|
+
common: dict[str, object] = {
|
|
760
|
+
"npi": params.npi,
|
|
761
|
+
"data_month": snapshot.data_month,
|
|
762
|
+
"snapshot_complete": not unparsed,
|
|
763
|
+
"verification_url": VERIFICATION_URL,
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
if reinstated:
|
|
767
|
+
# A record with a reinstatement date contradicts the full file's own
|
|
768
|
+
# invariant, so the honest answer is neither "excluded" (the party may
|
|
769
|
+
# have been reinstated) nor "not excluded" (the record is still here).
|
|
770
|
+
# excluded_in_snapshot is OMITTED rather than set false or null: a
|
|
771
|
+
# caller that reads it gets a KeyError, not a falsy value.
|
|
772
|
+
data: dict[str, object] | None = {
|
|
773
|
+
**common,
|
|
774
|
+
"screen_result": "indeterminate",
|
|
775
|
+
"indeterminate_reason": "reinstatement_date_present",
|
|
776
|
+
"records": records,
|
|
777
|
+
}
|
|
778
|
+
non_claims.append(NC_REINSTATEMENT)
|
|
779
|
+
warnings.append(
|
|
780
|
+
f"{len(reinstated)} matched record(s) carry a reinstatement date, which the "
|
|
781
|
+
"LEIE full file is not supposed to contain. SourceLock implements no "
|
|
782
|
+
f"reinstatement semantics: this is not a verdict either way — resolve it at "
|
|
783
|
+
f"{VERIFICATION_URL}."
|
|
784
|
+
)
|
|
785
|
+
effective = _snapshot_date(snapshot)
|
|
786
|
+
elif indices:
|
|
787
|
+
data = {
|
|
788
|
+
**common,
|
|
789
|
+
"screen_result": "excluded",
|
|
790
|
+
"excluded_in_snapshot": True,
|
|
791
|
+
"match_basis": "npi-exact",
|
|
792
|
+
"records": records,
|
|
793
|
+
}
|
|
794
|
+
effective = (
|
|
795
|
+
_record_effective(snapshot.records[indices[0]])
|
|
796
|
+
if len(indices) == 1
|
|
797
|
+
else _snapshot_date(snapshot)
|
|
798
|
+
)
|
|
799
|
+
if unparsed:
|
|
800
|
+
warnings.append(
|
|
801
|
+
f"{snapshot.malformed_rows} row(s) of the snapshot could not be parsed, so "
|
|
802
|
+
"this record list may be incomplete; the match itself stands."
|
|
803
|
+
)
|
|
804
|
+
elif unparsed:
|
|
805
|
+
# No match, but the screen did not read the whole list. Fail closed.
|
|
806
|
+
data = {
|
|
807
|
+
**common,
|
|
808
|
+
"screen_result": "indeterminate",
|
|
809
|
+
"indeterminate_reason": "unparsed_rows",
|
|
810
|
+
"unparsed_rows": snapshot.malformed_rows,
|
|
811
|
+
"records": records,
|
|
812
|
+
}
|
|
813
|
+
warnings.append(
|
|
814
|
+
f"{snapshot.malformed_rows} row(s) of the LEIE snapshot "
|
|
815
|
+
f"({snapshot.source_version}) could not be parsed, so this NPI was not "
|
|
816
|
+
"compared against the whole list. Reporting indeterminate rather than "
|
|
817
|
+
f"not-found — verify at {VERIFICATION_URL}."
|
|
818
|
+
)
|
|
819
|
+
effective = _snapshot_date(snapshot)
|
|
820
|
+
else:
|
|
821
|
+
data = None
|
|
822
|
+
effective = _snapshot_date(snapshot)
|
|
823
|
+
warnings.append(
|
|
824
|
+
"The requested NPI was not found in the LEIE full-file snapshot "
|
|
825
|
+
f"({snapshot.source_version}). ~89.5% of LEIE records carry no NPI — run "
|
|
826
|
+
f"leie.candidate_search and verify identity at {VERIFICATION_URL} before "
|
|
827
|
+
"treating this as clearance."
|
|
828
|
+
)
|
|
829
|
+
|
|
830
|
+
return ToolResult(
|
|
831
|
+
data=data,
|
|
832
|
+
receipt=build_receipt(
|
|
833
|
+
contract=CONTRACT,
|
|
834
|
+
route="leie.check_npi",
|
|
835
|
+
fetch=entry.fetch_meta(),
|
|
836
|
+
source_version=snapshot.source_version,
|
|
837
|
+
transform_version=TRANSFORM_VERSION,
|
|
838
|
+
effective_from=effective,
|
|
839
|
+
warnings=warnings,
|
|
840
|
+
non_claims=non_claims,
|
|
841
|
+
),
|
|
842
|
+
)
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
def _candidate_search(params: CandidateSearchParams) -> ToolResult:
|
|
846
|
+
entry = _cached_entry()
|
|
847
|
+
snapshot = entry.snapshot
|
|
848
|
+
warnings = list(snapshot.parse_warnings) + list(entry.warnings)
|
|
849
|
+
non_claims = list(CANDIDATE_NON_CLAIMS)
|
|
850
|
+
unparsed = snapshot.malformed_rows > MAX_UNPARSED_ROWS
|
|
851
|
+
if unparsed:
|
|
852
|
+
non_claims.append(NC_UNPARSED)
|
|
853
|
+
|
|
854
|
+
matches: list[int] = []
|
|
855
|
+
if params.last_name:
|
|
856
|
+
last = _norm(params.last_name)
|
|
857
|
+
first = _norm(params.first_name) if params.first_name else None
|
|
858
|
+
for j, rec in enumerate(snapshot.records):
|
|
859
|
+
if not rec["LASTNAME"] or _norm(rec["LASTNAME"]) != last:
|
|
860
|
+
continue
|
|
861
|
+
if first and not _norm(rec["FIRSTNAME"]).startswith(first):
|
|
862
|
+
continue
|
|
863
|
+
matches.append(j)
|
|
864
|
+
else:
|
|
865
|
+
fragment = _norm(params.business_name or "")
|
|
866
|
+
for j, rec in enumerate(snapshot.records):
|
|
867
|
+
if rec["BUSNAME"] and fragment in _norm(rec["BUSNAME"]):
|
|
868
|
+
matches.append(j)
|
|
869
|
+
|
|
870
|
+
if params.state:
|
|
871
|
+
matches = [j for j in matches if snapshot.records[j]["STATE"] == params.state]
|
|
872
|
+
|
|
873
|
+
matches.sort(
|
|
874
|
+
key=lambda j: (
|
|
875
|
+
snapshot.records[j]["LASTNAME"],
|
|
876
|
+
snapshot.records[j]["FIRSTNAME"],
|
|
877
|
+
snapshot.records[j]["BUSNAME"],
|
|
878
|
+
snapshot.records[j]["EXCLDATE"],
|
|
879
|
+
j,
|
|
880
|
+
)
|
|
881
|
+
)
|
|
882
|
+
truncated = len(matches) > MAX_CANDIDATES
|
|
883
|
+
picked = matches[:MAX_CANDIDATES]
|
|
884
|
+
if truncated:
|
|
885
|
+
warnings.append(
|
|
886
|
+
f"more than {MAX_CANDIDATES} candidate rows matched; returning the first "
|
|
887
|
+
f"{MAX_CANDIDATES} — narrow with first_name or state"
|
|
888
|
+
)
|
|
889
|
+
|
|
890
|
+
if picked:
|
|
891
|
+
candidates = [
|
|
892
|
+
dict(_public_record(snapshot.records[j]), match_basis="name-candidate")
|
|
893
|
+
for j in picked
|
|
894
|
+
]
|
|
895
|
+
data: dict[str, object] | None = {
|
|
896
|
+
"screen_result": "candidates",
|
|
897
|
+
"candidates": candidates,
|
|
898
|
+
"count": len(candidates),
|
|
899
|
+
"truncated": truncated,
|
|
900
|
+
"snapshot_complete": not unparsed,
|
|
901
|
+
"data_month": snapshot.data_month,
|
|
902
|
+
"verification_url": VERIFICATION_URL,
|
|
903
|
+
}
|
|
904
|
+
reinstated = [c for c in candidates if c["reinstatement_date"]]
|
|
905
|
+
if reinstated:
|
|
906
|
+
non_claims.append(NC_REINSTATEMENT)
|
|
907
|
+
warnings.append(
|
|
908
|
+
f"{len(reinstated)} returned candidate(s) carry a reinstatement date, "
|
|
909
|
+
"which the LEIE full file is not supposed to contain. SourceLock "
|
|
910
|
+
"implements no reinstatement semantics: their exclusion status is "
|
|
911
|
+
f"undetermined here — resolve it at {VERIFICATION_URL}."
|
|
912
|
+
)
|
|
913
|
+
if unparsed:
|
|
914
|
+
warnings.append(
|
|
915
|
+
f"{snapshot.malformed_rows} row(s) of the snapshot could not be parsed, "
|
|
916
|
+
"so this candidate list may be missing matches."
|
|
917
|
+
)
|
|
918
|
+
elif unparsed:
|
|
919
|
+
# Nothing matched, but the search did not read the whole list. The
|
|
920
|
+
# candidate/count keys are OMITTED so an empty list cannot be read as
|
|
921
|
+
# "searched, found nobody".
|
|
922
|
+
data = {
|
|
923
|
+
"screen_result": "indeterminate",
|
|
924
|
+
"indeterminate_reason": "unparsed_rows",
|
|
925
|
+
"unparsed_rows": snapshot.malformed_rows,
|
|
926
|
+
"snapshot_complete": False,
|
|
927
|
+
"data_month": snapshot.data_month,
|
|
928
|
+
"verification_url": VERIFICATION_URL,
|
|
929
|
+
}
|
|
930
|
+
warnings.append(
|
|
931
|
+
f"{snapshot.malformed_rows} row(s) of the LEIE snapshot "
|
|
932
|
+
f"({snapshot.source_version}) could not be parsed, so this name was not "
|
|
933
|
+
"searched against the whole list. Reporting indeterminate rather than "
|
|
934
|
+
f"no-candidates — verify at {VERIFICATION_URL}."
|
|
935
|
+
)
|
|
936
|
+
else:
|
|
937
|
+
data = None
|
|
938
|
+
warnings.append(
|
|
939
|
+
"No candidate rows matched. Per OIG search guidance, also try maiden or "
|
|
940
|
+
"former names, each half of a hyphenated name separately, and spelling "
|
|
941
|
+
"variants — apostrophes, hyphens, ampersands and commas are significant."
|
|
942
|
+
)
|
|
943
|
+
|
|
944
|
+
return ToolResult(
|
|
945
|
+
data=data,
|
|
946
|
+
receipt=build_receipt(
|
|
947
|
+
contract=CONTRACT,
|
|
948
|
+
route="leie.candidate_search",
|
|
949
|
+
fetch=entry.fetch_meta(),
|
|
950
|
+
source_version=snapshot.source_version,
|
|
951
|
+
transform_version=TRANSFORM_VERSION,
|
|
952
|
+
effective_from=_snapshot_date(snapshot),
|
|
953
|
+
warnings=warnings,
|
|
954
|
+
non_claims=non_claims,
|
|
955
|
+
),
|
|
956
|
+
)
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
def _refresh_status(_: NoParams) -> ToolResult:
|
|
960
|
+
result = _probe()
|
|
961
|
+
warnings: list[str] = []
|
|
962
|
+
lm = _parse_last_modified(result.headers)
|
|
963
|
+
today = _today()
|
|
964
|
+
|
|
965
|
+
if lm is not None:
|
|
966
|
+
lm_date = lm.date()
|
|
967
|
+
age_days: int | None = (today - lm_date).days
|
|
968
|
+
stale: bool | None = age_days > STALE_AFTER_DAYS
|
|
969
|
+
data_month = _data_month(lm_date)
|
|
970
|
+
next_year, next_month = lm_date.year, lm_date.month + 1
|
|
971
|
+
if next_month == 13:
|
|
972
|
+
next_year, next_month = next_year + 1, 1
|
|
973
|
+
next_expected: str | None = f"{next_year}-{next_month:02d}-10"
|
|
974
|
+
source_version = lm_date.isoformat()
|
|
975
|
+
effective: date | None = lm_date
|
|
976
|
+
if stale:
|
|
977
|
+
warnings.append(
|
|
978
|
+
f"snapshot is stale: last modified {lm_date.isoformat()} ({age_days} "
|
|
979
|
+
f"days ago) against OIG's monthly by-the-10th cadence — check {HUB_URL}"
|
|
980
|
+
)
|
|
981
|
+
else:
|
|
982
|
+
age_days = stale = None
|
|
983
|
+
data_month = next_expected = None
|
|
984
|
+
source_version = f"sha256:{result.sha256[:12]}"
|
|
985
|
+
effective = None
|
|
986
|
+
warnings.append(
|
|
987
|
+
"no Last-Modified header (local mirror or override); snapshot age unknown"
|
|
988
|
+
)
|
|
989
|
+
|
|
990
|
+
data = {
|
|
991
|
+
"source_url": result.url,
|
|
992
|
+
"last_modified": lm.isoformat() if lm else None,
|
|
993
|
+
"data_month": data_month,
|
|
994
|
+
"age_days": age_days,
|
|
995
|
+
"stale": stale,
|
|
996
|
+
"cadence": CADENCE_TEXT,
|
|
997
|
+
"supplement_retention": RETENTION_TEXT,
|
|
998
|
+
"next_update_expected": next_expected,
|
|
999
|
+
"verification_url": VERIFICATION_URL,
|
|
1000
|
+
}
|
|
1001
|
+
return ToolResult(
|
|
1002
|
+
data=data,
|
|
1003
|
+
receipt=build_receipt(
|
|
1004
|
+
contract=CONTRACT,
|
|
1005
|
+
route="leie.refresh_status",
|
|
1006
|
+
fetch=result,
|
|
1007
|
+
source_version=source_version,
|
|
1008
|
+
transform_version=TRANSFORM_VERSION,
|
|
1009
|
+
effective_from=effective,
|
|
1010
|
+
warnings=warnings,
|
|
1011
|
+
non_claims=REFRESH_NON_CLAIMS,
|
|
1012
|
+
),
|
|
1013
|
+
)
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
# ---------------------------------------------------------------------------
|
|
1017
|
+
# Canaries
|
|
1018
|
+
# ---------------------------------------------------------------------------
|
|
1019
|
+
|
|
1020
|
+
|
|
1021
|
+
def _content_range_total(headers: Mapping[str, str]) -> int | None:
|
|
1022
|
+
raw = headers.get("content-range", "")
|
|
1023
|
+
if "/" not in raw:
|
|
1024
|
+
return None
|
|
1025
|
+
total = raw.rsplit("/", 1)[1].strip()
|
|
1026
|
+
return int(total) if total.isdigit() else None
|
|
1027
|
+
|
|
1028
|
+
|
|
1029
|
+
def _observe_full_file() -> CanaryObservation:
|
|
1030
|
+
try:
|
|
1031
|
+
result = _probe()
|
|
1032
|
+
except SourceUnreachable:
|
|
1033
|
+
raise
|
|
1034
|
+
except RuntimeError:
|
|
1035
|
+
# Offline harness (socket blocker): observe the packaged sample with a
|
|
1036
|
+
# self-describing token. Real transport failures raise above.
|
|
1037
|
+
snapshot = _cached_snapshot(from_sample=True)
|
|
1038
|
+
return CanaryObservation(
|
|
1039
|
+
value=f"sample:{snapshot.canonical_hash[:8]}",
|
|
1040
|
+
schema_hash=snapshot.schema_hash,
|
|
1041
|
+
note="offline: observed the packaged sample, not the live source",
|
|
1042
|
+
)
|
|
1043
|
+
|
|
1044
|
+
content_type = result.headers.get("content-type", "")
|
|
1045
|
+
if content_type and not content_type.startswith("text/csv"):
|
|
1046
|
+
return CanaryObservation(
|
|
1047
|
+
value=f"unexpected-content-type:{content_type.split(';')[0]}",
|
|
1048
|
+
upstream_status=result.status,
|
|
1049
|
+
)
|
|
1050
|
+
total = _content_range_total(result.headers)
|
|
1051
|
+
if total is not None and not (MIN_EXPECTED_BYTES <= total <= MAX_EXPECTED_BYTES):
|
|
1052
|
+
return CanaryObservation(value=f"size-out-of-band:{total}", upstream_status=result.status)
|
|
1053
|
+
|
|
1054
|
+
lm = _parse_last_modified(result.headers)
|
|
1055
|
+
if lm is None:
|
|
1056
|
+
# Local mirror / file override: key on canonical content, never raw bytes.
|
|
1057
|
+
snapshot = _cached_snapshot()
|
|
1058
|
+
return CanaryObservation(
|
|
1059
|
+
value=f"canonical:{snapshot.canonical_hash[:12]}",
|
|
1060
|
+
schema_hash=snapshot.schema_hash,
|
|
1061
|
+
note="no Last-Modified header; keyed on canonical content",
|
|
1062
|
+
)
|
|
1063
|
+
lm_date = lm.date()
|
|
1064
|
+
if (_today() - lm_date).days > STALE_AFTER_DAYS:
|
|
1065
|
+
return CanaryObservation(value=f"stale:{lm_date.isoformat()}", upstream_status=result.status)
|
|
1066
|
+
return CanaryObservation(
|
|
1067
|
+
value=lm_date.isoformat(),
|
|
1068
|
+
upstream_status=result.status,
|
|
1069
|
+
note="Last-Modified of UPDATED.csv; a new monthly release moves this forward",
|
|
1070
|
+
)
|
|
1071
|
+
|
|
1072
|
+
|
|
1073
|
+
def _observe_header() -> CanaryObservation:
|
|
1074
|
+
try:
|
|
1075
|
+
result = _probe()
|
|
1076
|
+
except SourceUnreachable:
|
|
1077
|
+
raise
|
|
1078
|
+
except RuntimeError:
|
|
1079
|
+
result = fetch(_sample_uri())
|
|
1080
|
+
|
|
1081
|
+
text = result.content.decode("utf-8-sig", errors="replace")
|
|
1082
|
+
if text.lstrip().startswith("<"):
|
|
1083
|
+
return CanaryObservation(value="html-body", upstream_status=result.status)
|
|
1084
|
+
lines = text.splitlines()
|
|
1085
|
+
columns = [c.strip() for c in next(csv.reader(lines[:1]), [])] if lines else []
|
|
1086
|
+
if not columns:
|
|
1087
|
+
return CanaryObservation(value="empty-header", upstream_status=result.status)
|
|
1088
|
+
return CanaryObservation(
|
|
1089
|
+
value=str(len(columns)),
|
|
1090
|
+
schema_hash=_schema_hash(columns),
|
|
1091
|
+
upstream_status=result.status,
|
|
1092
|
+
note="column count of the UPDATED.csv header; the hash locks the exact names",
|
|
1093
|
+
)
|
|
1094
|
+
|
|
1095
|
+
|
|
1096
|
+
def _observe_content() -> CanaryObservation:
|
|
1097
|
+
try:
|
|
1098
|
+
snapshot = _cached_snapshot()
|
|
1099
|
+
except SourceUnreachable:
|
|
1100
|
+
raise
|
|
1101
|
+
except RuntimeError:
|
|
1102
|
+
snapshot = _cached_snapshot(from_sample=True)
|
|
1103
|
+
|
|
1104
|
+
if snapshot.reindate_populated:
|
|
1105
|
+
return CanaryObservation(
|
|
1106
|
+
value=f"reindate-populated:{snapshot.reindate_populated}",
|
|
1107
|
+
schema_hash=snapshot.schema_hash,
|
|
1108
|
+
note="REINDATE must be null in the full file; OIG may have changed semantics",
|
|
1109
|
+
)
|
|
1110
|
+
note = f"records={len(snapshot.records)}"
|
|
1111
|
+
if snapshot.unknown_excltypes:
|
|
1112
|
+
note += "; new EXCLTYPE tokens: " + ", ".join(snapshot.unknown_excltypes)
|
|
1113
|
+
value = (
|
|
1114
|
+
f"sample:{snapshot.canonical_hash[:8]}"
|
|
1115
|
+
if snapshot.from_sample
|
|
1116
|
+
else f"canonical:{snapshot.canonical_hash[:16]}"
|
|
1117
|
+
)
|
|
1118
|
+
return CanaryObservation(value=value, schema_hash=snapshot.schema_hash, note=note)
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def _observe_waivers() -> CanaryObservation:
|
|
1122
|
+
"""Advisory cross-check: observes, never drifts, never fails.
|
|
1123
|
+
|
|
1124
|
+
Its ``value`` is the constant ``"advisory"``, so it cannot drift by
|
|
1125
|
+
construction -- which is the point, and also the trap. When the cross-check
|
|
1126
|
+
MISMATCHES, an unchanged constant would be graded ``ok`` while the invariant
|
|
1127
|
+
the canary exists to test is broken. So a mismatch sets ``stale``: doctor
|
|
1128
|
+
downgrades the match to STALE (exit 4), because a comparison that succeeded
|
|
1129
|
+
against something the canary says is wrong is not assurance.
|
|
1130
|
+
|
|
1131
|
+
A SKIP is different and deliberately not stale. The advisory being
|
|
1132
|
+
unavailable -- offline, or the waivers page returning a non-2xx -- says
|
|
1133
|
+
nothing about whether the exclusion data is right, and the note already
|
|
1134
|
+
carries the reason.
|
|
1135
|
+
"""
|
|
1136
|
+
stale = False
|
|
1137
|
+
try:
|
|
1138
|
+
page = fetch(_waivers_url(), raise_for_status=False)
|
|
1139
|
+
try:
|
|
1140
|
+
snapshot = _cached_snapshot()
|
|
1141
|
+
except Exception: # noqa: BLE001 - advisory must not fail on the data path either
|
|
1142
|
+
snapshot = _cached_snapshot(from_sample=True)
|
|
1143
|
+
names = sorted({
|
|
1144
|
+
r["LASTNAME"]
|
|
1145
|
+
for r in snapshot.records
|
|
1146
|
+
if r["WAIVERDATE"] not in _NULL_DATES and r["LASTNAME"]
|
|
1147
|
+
})
|
|
1148
|
+
if page.status is not None and not (200 <= page.status < 300):
|
|
1149
|
+
note = f"skipped: waivers page returned HTTP {page.status}"
|
|
1150
|
+
else:
|
|
1151
|
+
text = page.content.decode("utf-8", errors="replace").casefold()
|
|
1152
|
+
missing = [n for n in names if n.casefold() not in text]
|
|
1153
|
+
if missing:
|
|
1154
|
+
stale = True
|
|
1155
|
+
note = (
|
|
1156
|
+
f"mismatch: {len(missing)} of {len(names)} CSV waiver surnames not on "
|
|
1157
|
+
"the waivers page — waiver scope is defined by the letters there"
|
|
1158
|
+
)
|
|
1159
|
+
else:
|
|
1160
|
+
note = f"ok: {len(names)}/{len(names)} CSV waiver surnames found on the waivers page"
|
|
1161
|
+
except Exception as exc: # noqa: BLE001 - advisory by design
|
|
1162
|
+
note = f"skipped: {type(exc).__name__}"
|
|
1163
|
+
return CanaryObservation(value="advisory", note=note, stale=stale)
|
|
1164
|
+
|
|
1165
|
+
|
|
1166
|
+
def _full_file_remediation(status: CanaryStatus, observed: str | None, expected: str | None) -> str:
|
|
1167
|
+
if status is CanaryStatus.UNREACHABLE:
|
|
1168
|
+
return (
|
|
1169
|
+
f"UPDATED.csv could not be probed at {AUTHORITY_URL}. Check network and the "
|
|
1170
|
+
f"hub page {HUB_URL} for a moved link; if the page is gone, email "
|
|
1171
|
+
"exclusions@oig.hhs.gov. Keep serving the last verified snapshot, labelled "
|
|
1172
|
+
"with its data month, until resolved."
|
|
1173
|
+
)
|
|
1174
|
+
if observed and observed.startswith("sample:"):
|
|
1175
|
+
return (
|
|
1176
|
+
"The canary observed the packaged sample because the live source was not "
|
|
1177
|
+
"reachable at observation time. Re-run `hc-source lock init` with network "
|
|
1178
|
+
"access so the lockfile pins the real upstream."
|
|
1179
|
+
)
|
|
1180
|
+
if observed and (observed.startswith("stale:") or observed.startswith("size-out-of-band:")
|
|
1181
|
+
or observed.startswith("unexpected-content-type:")):
|
|
1182
|
+
return (
|
|
1183
|
+
f"UPDATED.csv looks wrong upstream ({observed}). Open {HUB_URL}, confirm the "
|
|
1184
|
+
"'Updated LEIE Database (CSV)' link, and update the adapter URL if it moved. "
|
|
1185
|
+
"Keep the prior snapshot until a clean file is published."
|
|
1186
|
+
)
|
|
1187
|
+
return (
|
|
1188
|
+
f"OIG published a new monthly LEIE file (pinned {expected}, observed {observed}). "
|
|
1189
|
+
"Review the change, then re-pin with `hc-source lock init`."
|
|
1190
|
+
)
|
|
1191
|
+
|
|
1192
|
+
|
|
1193
|
+
ADAPTER: SourceAdapter
|
|
1194
|
+
|
|
1195
|
+
|
|
1196
|
+
class LeieAdapter(SourceAdapter):
|
|
1197
|
+
source_id = SOURCE_ID
|
|
1198
|
+
contract = CONTRACT
|
|
1199
|
+
|
|
1200
|
+
# These canaries read a live CMS service, so a single 502 is not news --
|
|
1201
|
+
# it is Tuesday. Two retries with exponential backoff, applied only to
|
|
1202
|
+
# transient failures (see hc_source.doctor._observe). A gate that goes red
|
|
1203
|
+
# on somebody else's bad afternoon gets uninstalled.
|
|
1204
|
+
canary_retries = 2
|
|
1205
|
+
|
|
1206
|
+
def canaries(self) -> list[Canary]:
|
|
1207
|
+
return [
|
|
1208
|
+
Canary(
|
|
1209
|
+
canary_id="leie.full_file",
|
|
1210
|
+
source_id=SOURCE_ID,
|
|
1211
|
+
description="Presence, type, size band, and monthly freshness of UPDATED.csv.",
|
|
1212
|
+
observe=_observe_full_file,
|
|
1213
|
+
remediation=(
|
|
1214
|
+
f"UPDATED.csv moved, went stale, or changed shape. Open {HUB_URL}, "
|
|
1215
|
+
"locate the 'Updated LEIE Database (CSV)' link, update the adapter "
|
|
1216
|
+
"URL if needed, then re-pin with `hc-source lock init`."
|
|
1217
|
+
),
|
|
1218
|
+
remediation_for=_full_file_remediation,
|
|
1219
|
+
),
|
|
1220
|
+
Canary(
|
|
1221
|
+
canary_id="leie.header",
|
|
1222
|
+
source_id=SOURCE_ID,
|
|
1223
|
+
description="The locked 18-column CSV header of UPDATED.csv.",
|
|
1224
|
+
observe=_observe_header,
|
|
1225
|
+
remediation=(
|
|
1226
|
+
"LEIE column layout changed. Diff the live header against the locked "
|
|
1227
|
+
f"18 columns, fetch the current record layout from {RECORD_LAYOUT_URL} "
|
|
1228
|
+
"(follows a 301), regenerate the field map, bump TRANSFORM_VERSION in "
|
|
1229
|
+
"hc_source/adapters/leie.py, then re-pin with `hc-source lock init`. "
|
|
1230
|
+
"Do not ingest until the new header is locked."
|
|
1231
|
+
),
|
|
1232
|
+
),
|
|
1233
|
+
Canary(
|
|
1234
|
+
canary_id="leie.content",
|
|
1235
|
+
source_id=SOURCE_ID,
|
|
1236
|
+
description=(
|
|
1237
|
+
"Canonicalised content hash of the full file (styling- and "
|
|
1238
|
+
"order-independent), plus the REINDATE-null invariant."
|
|
1239
|
+
),
|
|
1240
|
+
observe=_observe_content,
|
|
1241
|
+
remediation=(
|
|
1242
|
+
"LEIE full-file content moved — a new monthly release or an in-place "
|
|
1243
|
+
"repost. Review the change, then re-pin with `hc-source lock init`. "
|
|
1244
|
+
"If the observed value starts with 'reindate-populated', OIG changed "
|
|
1245
|
+
f"full-file semantics: re-read {QUICK_TIPS_URL} before ingesting. If "
|
|
1246
|
+
"it starts with 'sample:', the live source was unreachable at "
|
|
1247
|
+
"observation time — re-pin with network access."
|
|
1248
|
+
),
|
|
1249
|
+
),
|
|
1250
|
+
Canary(
|
|
1251
|
+
canary_id="leie.waivers",
|
|
1252
|
+
source_id=SOURCE_ID,
|
|
1253
|
+
description=(
|
|
1254
|
+
"Advisory cross-check of CSV waiver rows against the OIG waivers "
|
|
1255
|
+
"page (WARN-only: HTML scrape, never drifts or fails a run)."
|
|
1256
|
+
),
|
|
1257
|
+
observe=_observe_waivers,
|
|
1258
|
+
# Declared, not merely intended. This canary scrapes an HTML
|
|
1259
|
+
# page OIG restyles at will and reports a cross-check whose
|
|
1260
|
+
# disagreement means "a human should read the waiver letters",
|
|
1261
|
+
# never "your data is wrong". It was already described as
|
|
1262
|
+
# WARN-only; now the exit code agrees with the description.
|
|
1263
|
+
severity=CanarySeverity.ADVISORY,
|
|
1264
|
+
remediation=(
|
|
1265
|
+
f"Advisory only: waiver rows in UPDATED.csv disagree with {WAIVERS_URL}. "
|
|
1266
|
+
"Waiver scope is defined by the letters on that page, not the CSV; "
|
|
1267
|
+
"flag affected records as 'waiver status uncertain — see the OIG "
|
|
1268
|
+
"waiver letter' until the next monthly file."
|
|
1269
|
+
),
|
|
1270
|
+
),
|
|
1271
|
+
]
|
|
1272
|
+
|
|
1273
|
+
def tools(self) -> list[ToolSpec]:
|
|
1274
|
+
return [
|
|
1275
|
+
ToolSpec(
|
|
1276
|
+
name="leie.check_npi",
|
|
1277
|
+
description=(
|
|
1278
|
+
"Deterministic NPI-exact check against the current monthly LEIE "
|
|
1279
|
+
"full-file snapshot; a miss never clears a provider (~89.5% of "
|
|
1280
|
+
"records carry no NPI)."
|
|
1281
|
+
),
|
|
1282
|
+
params_model=CheckNpiParams,
|
|
1283
|
+
handler=_check_npi,
|
|
1284
|
+
tags=("lookup", "exclusions"),
|
|
1285
|
+
),
|
|
1286
|
+
ToolSpec(
|
|
1287
|
+
name="leie.candidate_search",
|
|
1288
|
+
description=(
|
|
1289
|
+
"Name-based CANDIDATE matching in the LEIE snapshot — candidates "
|
|
1290
|
+
"only, never identity verification; verify at "
|
|
1291
|
+
"https://exclusions.oig.hhs.gov/."
|
|
1292
|
+
),
|
|
1293
|
+
params_model=CandidateSearchParams,
|
|
1294
|
+
handler=_candidate_search,
|
|
1295
|
+
tags=("search", "exclusions"),
|
|
1296
|
+
),
|
|
1297
|
+
ToolSpec(
|
|
1298
|
+
name="leie.refresh_status",
|
|
1299
|
+
description=(
|
|
1300
|
+
"Freshness of the LEIE snapshot versus OIG's monthly by-the-10th "
|
|
1301
|
+
"cadence, plus the 12-month supplement retention limit."
|
|
1302
|
+
),
|
|
1303
|
+
params_model=NoParams,
|
|
1304
|
+
handler=_refresh_status,
|
|
1305
|
+
tags=("metadata",),
|
|
1306
|
+
),
|
|
1307
|
+
]
|
|
1308
|
+
|
|
1309
|
+
|
|
1310
|
+
ADAPTER = LeieAdapter()
|