agentdata 0.3.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentdata/__init__.py +33 -0
- agentdata/__main__.py +4 -0
- agentdata/cache.py +115 -0
- agentdata/cli.py +173 -0
- agentdata/client.py +158 -0
- agentdata/company/__init__.py +3 -0
- agentdata/company/cf.py +145 -0
- agentdata/company/ch_zefix.py +100 -0
- agentdata/company/cli.py +243 -0
- agentdata/company/co_rues.py +72 -0
- agentdata/company/edgar_cik.py +84 -0
- agentdata/company/edgar_fts.py +101 -0
- agentdata/company/faces.py +746 -0
- agentdata/company/fdic_bankfind.py +107 -0
- agentdata/company/fi_ytj.py +89 -0
- agentdata/company/fr_recherche.py +124 -0
- agentdata/company/fts.py +121 -0
- agentdata/company/gleif.py +178 -0
- agentdata/company/kr_hometax.py +112 -0
- agentdata/company/no_brreg.py +124 -0
- agentdata/company/ror.py +105 -0
- agentdata/company/ted.py +117 -0
- agentdata/company/usaspending.py +124 -0
- agentdata/company/vies.py +66 -0
- agentdata/daterange.py +35 -0
- agentdata/economy/__init__.py +10 -0
- agentdata/economy/bis.py +132 -0
- agentdata/economy/cli.py +199 -0
- agentdata/economy/ecb.py +117 -0
- agentdata/economy/eurostat.py +163 -0
- agentdata/economy/faces.py +573 -0
- agentdata/economy/fred.py +92 -0
- agentdata/economy/ilo_stat.py +99 -0
- agentdata/economy/imf_sdmx.py +133 -0
- agentdata/economy/un_comtrade.py +199 -0
- agentdata/economy/unsd_sdg.py +154 -0
- agentdata/economy/wb.py +208 -0
- agentdata/economy/who_xmart.py +177 -0
- agentdata/entries.py +30 -0
- agentdata/errors.py +38 -0
- agentdata/esg/__init__.py +5 -0
- agentdata/esg/cli.py +180 -0
- agentdata/esg/climatetrace.py +260 -0
- agentdata/esg/eiti.py +187 -0
- agentdata/esg/epa_frs.py +138 -0
- agentdata/esg/euasl.py +75 -0
- agentdata/esg/eutl.py +66 -0
- agentdata/esg/faces.py +498 -0
- agentdata/esg/paris_mou.py +240 -0
- agentdata/esg/thetis_mrv.py +67 -0
- agentdata/healthcare/__init__.py +7 -0
- agentdata/healthcare/au_tga.py +142 -0
- agentdata/healthcare/ca_dpd.py +123 -0
- agentdata/healthcare/ca_mdall.py +155 -0
- agentdata/healthcare/cli.py +375 -0
- agentdata/healthcare/ctgov.py +101 -0
- agentdata/healthcare/dailymed.py +102 -0
- agentdata/healthcare/epmc.py +108 -0
- agentdata/healthcare/eudamed.py +124 -0
- agentdata/healthcare/faces.py +811 -0
- agentdata/healthcare/fda_crl.py +86 -0
- agentdata/healthcare/fda_devices.py +199 -0
- agentdata/healthcare/fda_drugs.py +156 -0
- agentdata/healthcare/fda_recalls.py +137 -0
- agentdata/healthcare/gb_mhra.py +137 -0
- agentdata/healthcare/gudid.py +84 -0
- agentdata/healthcare/mesh.py +89 -0
- agentdata/healthcare/rasff.py +164 -0
- agentdata/healthcare/rxnav.py +67 -0
- agentdata/healthcare/safety_gate.py +175 -0
- agentdata/healthcare/shortages.py +107 -0
- agentdata/legal/__init__.py +4 -0
- agentdata/legal/afdb.py +93 -0
- agentdata/legal/ca_sema.py +143 -0
- agentdata/legal/cli.py +312 -0
- agentdata/legal/co_sic.py +70 -0
- agentdata/legal/courtlistener.py +77 -0
- agentdata/legal/csl.py +139 -0
- agentdata/legal/dgcomp.py +125 -0
- agentdata/legal/ecfr.py +129 -0
- agentdata/legal/faces.py +700 -0
- agentdata/legal/federalregister.py +188 -0
- agentdata/legal/gb_cma.py +117 -0
- agentdata/legal/gb_tna_caselaw.py +245 -0
- agentdata/legal/idb.py +85 -0
- agentdata/legal/lda.py +147 -0
- agentdata/legal/sipri.py +178 -0
- agentdata/legal/ukec.py +148 -0
- agentdata/legal/wb.py +92 -0
- agentdata/procurement/__init__.py +5 -0
- agentdata/procurement/br_pncp.py +126 -0
- agentdata/procurement/cli.py +155 -0
- agentdata/procurement/co_secop.py +88 -0
- agentdata/procurement/faces.py +239 -0
- agentdata/procurement/ny_open_data.py +82 -0
- agentdata/procurement/ungm.py +137 -0
- agentdata/procurement/wb_proc.py +84 -0
- agentdata/research/__init__.py +4 -0
- agentdata/research/biorxiv.py +66 -0
- agentdata/research/cli.py +129 -0
- agentdata/research/crossref.py +121 -0
- agentdata/research/faces.py +349 -0
- agentdata/research/gb_gtr.py +58 -0
- agentdata/research/icite.py +52 -0
- agentdata/research/nih_reporter.py +143 -0
- agentdata/research/nsf.py +68 -0
- agentdata/research/openalex.py +145 -0
- agentdata/research/opencitations.py +50 -0
- agentdata/security/__init__.py +3 -0
- agentdata/security/cisa_kev.py +42 -0
- agentdata/security/cli.py +37 -0
- agentdata/security/faces.py +91 -0
- agentdata/security/osv.py +72 -0
- agentdata/transport.py +541 -0
- agentdata-0.3.26.dist-info/METADATA +266 -0
- agentdata-0.3.26.dist-info/RECORD +120 -0
- agentdata-0.3.26.dist-info/WHEEL +5 -0
- agentdata-0.3.26.dist-info/entry_points.txt +10 -0
- agentdata-0.3.26.dist-info/licenses/LICENSE +201 -0
- agentdata-0.3.26.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""safety_gate — EU Safety Gate (the ex-RAPEX rapid alert system for dangerous non-food products)
|
|
2
|
+
weekly-report XML face provider (ec.europa.eu/safety-gate-alerts/api/download/; keyless).
|
|
3
|
+
|
|
4
|
+
healthcare domain (0.3.19; admitted 2026-11-02, probed). Shapes verified by first-hand live probing
|
|
5
|
+
2026-11-02 (the probe corpus lives in tmp/probe-ca-eu-faces/):
|
|
6
|
+
GET /safety-gate-alerts/api/download/weeklyReport/list/xml/en
|
|
7
|
+
→ the whole weekly-report index in ONE XML document (1,122 reports measured 2026-11-02;
|
|
8
|
+
Report-2005-1 → Report-2026-43). Each <weeklyReport> entry: reference (Report-YYYY-WW),
|
|
9
|
+
publicationDate (dd/mm/yyyy), year/month/day/week, and URL — the URL path embeds the numeric
|
|
10
|
+
reportId (…/detail/xml/10000329), which is the join key into the detail face.
|
|
11
|
+
GET /safety-gate-alerts/api/download/weeklyReport/detail/xml/<reportId>?language=en&search=WEB_REPORT|:|<reportId>,
|
|
12
|
+
→ the single weekly report as XML: a <Safety-Gate> root (report_language/report_date/report_year/
|
|
13
|
+
report_week) wrapping N <notifications> entries (67 measured for report 10000329 = 2026-W43).
|
|
14
|
+
Alert fields: caseNumber (SR/03023/26), reference (the alert-detail URL → the numeric alert id),
|
|
15
|
+
category, product, brand, name, type_numberOfModel, batchNumber, barcode, riskType, danger
|
|
16
|
+
(the risk narrative), measures (the concatenated operator/measure/effective-date blocks),
|
|
17
|
+
URLrecall, description, notifyingCountry, countryOfOrigin, type (Consumer…), level
|
|
18
|
+
(Serious risk…), onlineTrader, companyRecallCode, productionDates.
|
|
19
|
+
language is a query param but search is the SPA's own echo parameter — both are FIXED for this
|
|
20
|
+
face (constructed from the report id; the en channel is the one probed).
|
|
21
|
+
The list/detail pair is the same shape family as gb_tna_caselaw (an XML face among JSON APIs).
|
|
22
|
+
Local row slicing (limit/skip) and the year post-filter slice the single cached full documents.
|
|
23
|
+
TTL: list 24h (new reports land weekly) / detail 7 days (a published weekly report is immutable);
|
|
24
|
+
polite rate 1 rps (the download family publishes no quota).
|
|
25
|
+
"""
|
|
26
|
+
import re
|
|
27
|
+
import xml.etree.ElementTree as ET
|
|
28
|
+
|
|
29
|
+
from ..errors import SdkError
|
|
30
|
+
|
|
31
|
+
provider = {
|
|
32
|
+
"id": "safety_gate",
|
|
33
|
+
"baseUrl": "https://ec.europa.eu",
|
|
34
|
+
"rate": {"rps": 1, "burst": 2},
|
|
35
|
+
"ops": {
|
|
36
|
+
"weekly_list": {"path": "/safety-gate-alerts/api/download/weeklyReport/list/xml/en", "accept": "text"},
|
|
37
|
+
# the detail query string is baked into the path byte-exact as probed (the ukec shape: urlencode
|
|
38
|
+
# would re-encode the SPA's echo parameter WEB_REPORT|:|<id>, — a silent near-miss risk not worth taking)
|
|
39
|
+
"weekly_detail": {"path": "/safety-gate-alerts/api/download/weeklyReport/detail/xml/{report_id}"
|
|
40
|
+
"?language=en&search=WEB_REPORT%7C:%7C{report_id},",
|
|
41
|
+
"accept": "text"},
|
|
42
|
+
},
|
|
43
|
+
"ttl": {"weekly_list": 24 * 3600, "weekly_detail": 7 * 86400},
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
_REPORT_ID = re.compile(r"/detail/xml/(\d+)") # the list face's download URL → the report id
|
|
47
|
+
_ALERT_ID = re.compile(r"alertDetail/(\d+)") # the detail rows' reference URL → the alert id (a different path shape)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _txt(el):
|
|
51
|
+
return (el.text or "").strip() or None if el is not None else None
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def alerts_from_upstream(xml_text):
|
|
55
|
+
"""The list XML → weekly-report index rows (pure function, offline-testable).
|
|
56
|
+
|
|
57
|
+
The reportId lives only inside the URL element — it is carved out here as the join key."""
|
|
58
|
+
root = ET.fromstring(xml_text)
|
|
59
|
+
rows = []
|
|
60
|
+
for wr in root.iter("weeklyReport"):
|
|
61
|
+
url = _txt(wr.find("URL")) or ""
|
|
62
|
+
rid = _REPORT_ID.search(url)
|
|
63
|
+
rows.append({
|
|
64
|
+
"report_id": rid.group(1) if rid else None, # the join key into the detail face
|
|
65
|
+
"reference": _txt(wr.find("reference")), # Report-YYYY-WW
|
|
66
|
+
"publication_date": _txt(wr.find("publicationDate")),
|
|
67
|
+
"year": _txt(wr.find("year")),
|
|
68
|
+
"week": _txt(wr.find("week")),
|
|
69
|
+
"url": url or None,
|
|
70
|
+
})
|
|
71
|
+
return {"rows": rows, "count": len(rows)}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def report_from_upstream(xml_text, report_id):
|
|
75
|
+
"""The detail XML → {report header, rows[]} (pure function, offline-testable)."""
|
|
76
|
+
root = ET.fromstring(xml_text)
|
|
77
|
+
rows = []
|
|
78
|
+
for n in root.iter("notifications"):
|
|
79
|
+
ref_url = _txt(n.find("reference")) or ""
|
|
80
|
+
aid = _ALERT_ID.search(ref_url)
|
|
81
|
+
rows.append({
|
|
82
|
+
"order": _txt(n.find("order")),
|
|
83
|
+
"case_number": _txt(n.find("caseNumber")),
|
|
84
|
+
"alert_id": aid.group(1) if aid else None, # the numeric id from the alert-detail URL
|
|
85
|
+
"alert_url": ref_url or None,
|
|
86
|
+
"category": _txt(n.find("category")),
|
|
87
|
+
"product": _txt(n.find("product")),
|
|
88
|
+
"brand": _txt(n.find("brand")),
|
|
89
|
+
"name": _txt(n.find("name")),
|
|
90
|
+
"type_number_of_model": _txt(n.find("type_numberOfModel")),
|
|
91
|
+
"batch_number": _txt(n.find("batchNumber")),
|
|
92
|
+
"barcode": _txt(n.find("barcode")),
|
|
93
|
+
"risk_type": _txt(n.find("riskType")), # e.g. "Electric shock, Fire"
|
|
94
|
+
"danger": _txt(n.find("danger")), # the risk narrative
|
|
95
|
+
"measures": _txt(n.find("measures")), # operator/measure/effective-date blocks
|
|
96
|
+
"url_recall": _txt(n.find("URLrecall")),
|
|
97
|
+
"description": _txt(n.find("description")),
|
|
98
|
+
"notifying_country": _txt(n.find("notifyingCountry")),
|
|
99
|
+
"country_of_origin": _txt(n.find("countryOfOrigin")),
|
|
100
|
+
"type": _txt(n.find("type")), # e.g. Consumer
|
|
101
|
+
"level": _txt(n.find("level")), # e.g. Serious risk
|
|
102
|
+
"online_trader": _txt(n.find("onlineTrader")),
|
|
103
|
+
"company_recall_code": _txt(n.find("companyRecallCode")),
|
|
104
|
+
"production_dates": _txt(n.find("productionDates")),
|
|
105
|
+
})
|
|
106
|
+
return {
|
|
107
|
+
"report_id": str(report_id),
|
|
108
|
+
# the detail XML carries no Report-YYYY-WW reference field — the caller stitches it from the
|
|
109
|
+
# list face; the report_year/report_week pair below is the upstream's own framing
|
|
110
|
+
"report_date": _txt(root.find("report_date")),
|
|
111
|
+
"report_year": _txt(root.find("report_year")),
|
|
112
|
+
"report_week": _txt(root.find("report_week")),
|
|
113
|
+
"report_language": _txt(root.find("report_language")),
|
|
114
|
+
"rows": rows, "count": len(rows),
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _slice(rows, limit, skip):
|
|
119
|
+
off = int(skip) if skip is not None else 0
|
|
120
|
+
if off < 0:
|
|
121
|
+
raise SdkError("usage_error", provider="safety_gate", detail=f"skip must be >=0, got \"{skip}\"")
|
|
122
|
+
size = int(limit) if limit is not None else 200
|
|
123
|
+
if size < 1:
|
|
124
|
+
raise SdkError("usage_error", provider="safety_gate", detail=f"limit must be >=1 (local post-slice, no upstream cap), got \"{limit}\"")
|
|
125
|
+
return rows[off:off + size]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def weekly_list_raw(*, year=None, limit=200, skip=0, cache=None, no_cache=False):
|
|
129
|
+
"""Flat-panel low-level face: the weekly-report index (1,122 reports 2005→now, one cached XML).
|
|
130
|
+
|
|
131
|
+
year = an optional local post-filter on the report year (the upstream index has no filter —
|
|
132
|
+
it always serves the whole index); limit/skip is a local post-slice.
|
|
133
|
+
→ {rows[], count, total, params, _meta} — total = the whole index size (post-year-filter)."""
|
|
134
|
+
from ..transport import call_provider
|
|
135
|
+
|
|
136
|
+
params = {}
|
|
137
|
+
if year is not None:
|
|
138
|
+
y = str(year).strip()
|
|
139
|
+
if not y.isdigit():
|
|
140
|
+
raise SdkError("usage_error", provider="safety_gate", detail=f"year must be a 4-digit year (a local post-filter on the report index), got \"{year}\"")
|
|
141
|
+
params["year"] = y
|
|
142
|
+
r = call_provider(provider, "weekly_list", {}, cache=cache, no_cache=no_cache)
|
|
143
|
+
out = alerts_from_upstream(r["data"])
|
|
144
|
+
rows = [x for x in out["rows"] if (not params.get("year") or x["year"] == params["year"])]
|
|
145
|
+
rows = _slice(rows, limit, skip)
|
|
146
|
+
return {"rows": rows, "count": len(rows), "total": len(out["rows"]) if not params.get("year")
|
|
147
|
+
else sum(1 for x in out["rows"] if x["year"] == params["year"]),
|
|
148
|
+
"params": params or None, "_meta": r}
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _clean_report_id(report_id):
|
|
152
|
+
"""Numeric reportId; a pasted detail URL gets its id carved out (the join key echoed by the list rows)."""
|
|
153
|
+
s = str(report_id or "").strip()
|
|
154
|
+
m = _REPORT_ID.search(s)
|
|
155
|
+
if m:
|
|
156
|
+
s = m.group(1)
|
|
157
|
+
if not s.isdigit():
|
|
158
|
+
raise SdkError("usage_error", provider="safety_gate",
|
|
159
|
+
detail=f"report_id must be the numeric report id (e.g. 10000329 — the report_id field echoed by safety_gate_alerts; a pasted detail URL also works), got \"{report_id}\"")
|
|
160
|
+
return s
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def weekly_detail_raw(report_id, *, limit=200, skip=0, cache=None, no_cache=False):
|
|
164
|
+
"""Flat-panel low-level face: one weekly report's alerts (the detail XML — 67 alerts for 2026-W43).
|
|
165
|
+
|
|
166
|
+
→ {report header…, rows[], count, params, _meta} — the alert rows carry caseNumber/risk/level/
|
|
167
|
+
notifyingCountry/countryOfOrigin/onlineTrader (the company-relevant edges) verbatim."""
|
|
168
|
+
from ..transport import call_provider
|
|
169
|
+
|
|
170
|
+
rid = _clean_report_id(report_id)
|
|
171
|
+
params = {"report_id": rid}
|
|
172
|
+
r = call_provider(provider, "weekly_detail", params, cache=cache, no_cache=no_cache)
|
|
173
|
+
out = report_from_upstream(r["data"] or "", rid)
|
|
174
|
+
rows = _slice(out["rows"], limit, skip)
|
|
175
|
+
return {**out, "rows": rows, "count": len(rows), "params": params, "_meta": r}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""shortages — FDA Drug Shortages provider (openFDA drug/shortages.json; keyless).
|
|
2
|
+
|
|
3
|
+
Wave 5 of the healthcare domain: related
|
|
4
|
+
kind=drug_shortages — manufacturer/drug name → drug shortage records, the supply-risk event stream: the last piece of the pipeline × safety × supply triad.
|
|
5
|
+
Row shapes measured 2026-10-23:
|
|
6
|
+
GET /drug/shortages.json?search=<q>&limit=N&skip=M&sort=update_date:desc
|
|
7
|
+
search segments (three, AND-combinable, measured q1=179/q2=82/q3=AND 18):
|
|
8
|
+
manufacturer → openfda.manufacturer_name.exact:"Fresenius Kabi USA, LLC" (exact phrase; bare tokens scatter)
|
|
9
|
+
drug name → generic_name:"bupivacaine" (phrase search, multi-word ingredient names match whole)
|
|
10
|
+
status → status:Current (the enum Current/Resolved/To Be Discontinued — measured on a 100-row Fresenius sample as 83/1/16)
|
|
11
|
+
**zero hits = 404 {"error":{"code":"NOT_FOUND","message":"No matches found!"}}** — a legal answer, not an error
|
|
12
|
+
(the openFDA convention); the op declares notFound:[404], and upstream_not_found turns into an empty set at this layer.
|
|
13
|
+
Row fields: generic_name/company_name/availability/status/update_type/update_date/initial_posting_date/
|
|
14
|
+
package_ndc/presentation/dosage_form/shortage_reason/therapeutic_category/contact_info
|
|
15
|
+
+ the openfda enrichment block (manufacturer_name/application_number/product_ndc/rxcui/substance_name)
|
|
16
|
+
— company_name = the manufacturer registered on the shortage record (may differ from openfda.manufacturer_name: that is the normalization layer, both columns are kept).
|
|
17
|
+
update_date can be sorted desc (measured: 10/15/2026 comes back first); the date form is MM/DD/YYYY.
|
|
18
|
+
TTL: a daily cadence (meta.last_updated measured 2026-10-22) → 24h; the keyless quota is 240/min → polite tier 2 rps.
|
|
19
|
+
"""
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
|
|
22
|
+
from ..errors import SdkError
|
|
23
|
+
|
|
24
|
+
provider = {
|
|
25
|
+
"id": "shortages",
|
|
26
|
+
"baseUrl": "https://api.fda.gov",
|
|
27
|
+
"rate": {"rps": 2, "burst": 5, "group": "fda"},
|
|
28
|
+
"ops": {
|
|
29
|
+
"search": {"path": "/drug/shortages.json", "notFound": [404]}, # params: search, limit, skip, sort
|
|
30
|
+
},
|
|
31
|
+
"ttl": {"search": 24 * 3600},
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def rows_from_upstream(results):
|
|
36
|
+
"""Upstream rows → shortage record rows (pure function, testable offline): both company columns are kept (the registered manufacturer + the openFDA normalized manufacturer)."""
|
|
37
|
+
rows = results if isinstance(results, list) else []
|
|
38
|
+
out = []
|
|
39
|
+
for r in rows:
|
|
40
|
+
of = r.get("openfda") or {}
|
|
41
|
+
out.append({
|
|
42
|
+
"generic_name": r.get("generic_name"),
|
|
43
|
+
"company_name": r.get("company_name"), # the manufacturer registered on the shortage record (company edge, primary column)
|
|
44
|
+
"manufacturers": of.get("manufacturer_name"), # the openFDA normalized manufacturer (company edge, comparison column)
|
|
45
|
+
"availability": r.get("availability"),
|
|
46
|
+
"status": r.get("status"),
|
|
47
|
+
"update_type": r.get("update_type"),
|
|
48
|
+
"update_date": r.get("update_date"),
|
|
49
|
+
"first_posted": r.get("initial_posting_date"),
|
|
50
|
+
"package_ndc": r.get("package_ndc"),
|
|
51
|
+
"presentation": r.get("presentation"),
|
|
52
|
+
"dosage_form": r.get("dosage_form"),
|
|
53
|
+
"shortage_reason": r.get("shortage_reason"),
|
|
54
|
+
"therapeutic_category": r.get("therapeutic_category"),
|
|
55
|
+
"applications": of.get("application_number"),
|
|
56
|
+
"product_ndcs": of.get("product_ndc"),
|
|
57
|
+
"rxcuis": of.get("rxcui"),
|
|
58
|
+
"substances": of.get("substance_name"),
|
|
59
|
+
"contact": r.get("contact_info"),
|
|
60
|
+
})
|
|
61
|
+
return out
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def search_raw(search, *, limit=20, skip=0, sort="update_date:desc", cache=None, no_cache=False):
|
|
65
|
+
"""0.2.0 mirror face: the search expression is passed through verbatim (openFDA query syntax) + the upstream param names limit/skip/sort.
|
|
66
|
+
|
|
67
|
+
Hard limit (protocol, confirmed by probing on 2026-10-23): limit 1-1000 — 1001 → 400 BAD_REQUEST
|
|
68
|
+
«Limit cannot exceed 1000 results for search requests. Use the skip or search_after param»;
|
|
69
|
+
deeper paging goes through skip — search_after is not accepted (probed 2026-10-28): with sort → 400 «search_after has 1 value(s)
|
|
70
|
+
but sort has 2» (a semicolon double value → an ES parse error; the default sort → 200 but the rows are identical to the baseline = silently ignored); and the whole
|
|
71
|
+
database holds only ~1604 rows, so limit 1000 + skip reaches the bottom in two pages — the deep-paging gap does not exist, skip is complete.
|
|
72
|
+
search is required (local safety gate: under the keyless 240/min quota a bare full-table query is served by the bulk harvest line, not by this single-lookup face).
|
|
73
|
+
A free sort string passes through (measured update_date:desc works; the default is ours, not the upstream one).
|
|
74
|
+
→ {rows[], total, count, skip, limit_applied, has_more, search, sort, _meta}; zero hits = an empty set that is not cached (same as above).
|
|
75
|
+
"""
|
|
76
|
+
from ..transport import call_provider
|
|
77
|
+
|
|
78
|
+
q = str(search or "").strip()
|
|
79
|
+
if not q:
|
|
80
|
+
raise SdkError("usage_error", provider="shortages",
|
|
81
|
+
detail="search requires an expression (e.g. generic_name:'bupivacaine' — the phrase segment needs double quotes; a bare full-table query is served by the bulk line)")
|
|
82
|
+
size = int(limit or 20)
|
|
83
|
+
if not 1 <= size <= 1000:
|
|
84
|
+
raise SdkError("usage_error", provider="shortages",
|
|
85
|
+
detail=f"limit must be 1-1000 (upstream 400: Limit cannot exceed 1000 results for search "
|
|
86
|
+
f"requests — measured 2026-10-23; to page deeper use skip, the whole database of ~1604 rows is reached in two pages), got {size}")
|
|
87
|
+
sk = max(0, int(skip or 0))
|
|
88
|
+
so = str(sort or "").strip()
|
|
89
|
+
params = {"search": q, "limit": size}
|
|
90
|
+
if sk:
|
|
91
|
+
params["skip"] = sk
|
|
92
|
+
if so:
|
|
93
|
+
params["sort"] = so
|
|
94
|
+
try:
|
|
95
|
+
r = call_provider(provider, "search", params, cache=cache, no_cache=no_cache)
|
|
96
|
+
except SdkError as e:
|
|
97
|
+
if e.code == "upstream_not_found":
|
|
98
|
+
return {"rows": [], "total": 0, "count": 0, "skip": sk, "limit_applied": size,
|
|
99
|
+
"has_more": False, "search": q, "sort": so or None,
|
|
100
|
+
"_meta": {"from_cache": False,
|
|
101
|
+
"fetched_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")}}
|
|
102
|
+
raise
|
|
103
|
+
data = r.get("data") or {}
|
|
104
|
+
rows = rows_from_upstream(data.get("results"))
|
|
105
|
+
total = (data.get("meta") or {}).get("results", {}).get("total", len(rows))
|
|
106
|
+
return {"rows": rows, "total": total, "count": len(rows), "skip": sk, "limit_applied": size,
|
|
107
|
+
"has_more": total > sk + len(rows), "search": q, "sort": so or None, "_meta": r}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
"""legal domain package — provider adapters laid flat (one source per file) + faces/cli (0.2.6 per-domain split).
|
|
2
|
+
"""
|
|
3
|
+
from . import (afdb, ca_sema, csl, courtlistener, co_sic, dgcomp, ecfr, # noqa: F401
|
|
4
|
+
federalregister, idb, lda, sipri, gb_tna_caselaw, ukec, gb_cma, wb)
|
agentdata/legal/afdb.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""afdb — African Development Bank debarred-entities roster provider (afdb.org json-feed; keyless).
|
|
2
|
+
|
|
3
|
+
legal debarment-roster face (0.3.21; admitted 2026-11-02, probed tmp/probe-debar-0321). Shape measured 2026-11-02:
|
|
4
|
+
GET /en/debarred-entities-json-feed → a bare JSON array, 1,440 rows × 7 columns (Type FIRM 1,196 / INDIVIDUAL 244,
|
|
5
|
+
Name, Nationality, From, To, Basis, Notes) — a single-shot full dump (the -csv-export sibling serves byte-identical
|
|
6
|
+
content; the -xml-export sibling returns a 2.4KB stub). To = empty means ongoing (169 rows); Notes ∈ {ONGOING,
|
|
7
|
+
INDEFINITE, empty}.
|
|
8
|
+
**Basis is the cross-debarment edge, self-evidenced**: "CROSS DEBARMENT : WBG" 653 / "CROSS-DEBARMENT: ADB" 377 /
|
|
9
|
+
"CROSS-DEBARMENT: IDB" 161+2 / "CROSS DEBARMENT : EBRD" 127 / "DEBARMENT:AFDB" 120 — note the upstream spacing is
|
|
10
|
+
**inconsistent** ("CROSS DEBARMENT : WBG" with spaces vs "CROSS-DEBARMENT: ADB" hyphenated): the basis filter is a
|
|
11
|
+
contains-match precisely so "WBG"/"ADB"/"IDB"/"EBRD" hit regardless of punctuation.
|
|
12
|
+
**Cloudflare UA-fingerprint gate** (pinned 2026-11-02): the default agentdata UA passes then 403s flakily on repeat;
|
|
13
|
+
a bare "Mozilla/5.0" UA is refused outright; the **full browser document suite passes consistently** — per-request
|
|
14
|
+
header override (ext 8), same family as UNGM (0.3.12).
|
|
15
|
+
Screening = local full-register scan (upstream has no query surface), cache 24h.
|
|
16
|
+
TTL: 24h; polite tier 0.2 rps (single-endpoint face; also keeps the Cloudflare gate calm).
|
|
17
|
+
"""
|
|
18
|
+
from ..errors import SdkError
|
|
19
|
+
|
|
20
|
+
provider = {
|
|
21
|
+
"id": "afdb",
|
|
22
|
+
"baseUrl": "https://www.afdb.org",
|
|
23
|
+
"rate": {"rps": 0.2, "burst": 1},
|
|
24
|
+
"ops": {
|
|
25
|
+
"debarred": {"path": "/en/debarred-entities-json-feed", "timeoutMs": 90000},
|
|
26
|
+
},
|
|
27
|
+
"ttl": {"debarred": 24 * 3600},
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
# Cloudflare UA-fingerprint gate (pinned 2026-11-02, tmp/probe-debar-0321 p3): default agentdata UA → flaky
|
|
31
|
+
# 403 on repeat; bare Mozilla/5.0 → 403; this full document suite → 200 twice in a row — ext-8 override
|
|
32
|
+
_CF_BROWSER_HEADERS = {
|
|
33
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
34
|
+
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
|
35
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.7",
|
|
36
|
+
"Accept-Language": "en-US,en;q=0.9",
|
|
37
|
+
"Referer": "https://www.afdb.org/en/projects-operations/debarment-and-sanctions-procedures",
|
|
38
|
+
"Upgrade-Insecure-Requests": "1",
|
|
39
|
+
"Sec-Fetch-Dest": "document",
|
|
40
|
+
"Sec-Fetch-Mode": "navigate",
|
|
41
|
+
"Sec-Fetch-Site": "none",
|
|
42
|
+
"Sec-Fetch-User": "?1",
|
|
43
|
+
"sec-ch-ua": '"Chromium";v="131", "Not_A Brand";v="24"',
|
|
44
|
+
"sec-ch-ua-mobile": "?0",
|
|
45
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def rows_from_upstream(data):
|
|
50
|
+
"""Upstream JSON array → debarred-entity rows (pure function; the seven upstream columns verbatim —
|
|
51
|
+
the register is already narrow, and To="" / Notes carry the ongoing/indefinite semantics)."""
|
|
52
|
+
return [{k: r.get(k) for k in ("Type", "Name", "Nationality", "From", "To", "Basis", "Notes")}
|
|
53
|
+
for r in (data or [])]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def filter_rows(rows, name=None, type=None, basis=None):
|
|
57
|
+
"""Local register scan (pure function). name: contains (case-insensitive) over Name; type: exact over
|
|
58
|
+
Type (FIRM/INDIVIDUAL); basis: **contains** over Basis — upstream spacing is inconsistent
|
|
59
|
+
("CROSS DEBARMENT : WBG" vs "CROSS-DEBARMENT: ADB"), so filter on the bank token (WBG/ADB/IDB/EBRD)."""
|
|
60
|
+
needle = str(name or "").strip().lower() or None
|
|
61
|
+
ty_needle = str(type or "").strip().upper() or None
|
|
62
|
+
bs_needle = str(basis or "").strip().lower() or None
|
|
63
|
+
hits = []
|
|
64
|
+
for r in rows:
|
|
65
|
+
if ty_needle and str(r.get("Type") or "").strip().upper() != ty_needle:
|
|
66
|
+
continue
|
|
67
|
+
if bs_needle and bs_needle not in str(r.get("Basis") or "").lower():
|
|
68
|
+
continue
|
|
69
|
+
if needle and needle not in str(r.get("Name") or "").lower():
|
|
70
|
+
continue
|
|
71
|
+
hits.append(r)
|
|
72
|
+
return hits
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def debarred_entities_raw(name=None, *, type=None, basis=None, limit=20, cache=None, no_cache=False):
|
|
76
|
+
"""Full-register scan (pulls the whole feed, caches 24h, then scans locally)
|
|
77
|
+
→ {rows, total_matched, scanned_rows, limit_applied, params, _meta}.
|
|
78
|
+
|
|
79
|
+
name = contains over Name; type = FIRM/INDIVIDUAL exact; basis = contains over Basis (bank token:
|
|
80
|
+
WBG/ADB/IDB/EBRD — punctuation upstream is inconsistent). No filter → enumerate from the register head."""
|
|
81
|
+
from ..transport import call_provider
|
|
82
|
+
|
|
83
|
+
size = int(limit) if limit is not None else 20
|
|
84
|
+
if not 1 <= size <= 100:
|
|
85
|
+
raise SdkError("usage_error", provider="afdb",
|
|
86
|
+
detail=f"limit must be 1-100 (local payload wall), got '{limit}'")
|
|
87
|
+
r = call_provider(provider, "debarred", {}, cache=cache, no_cache=no_cache,
|
|
88
|
+
extra_headers=_CF_BROWSER_HEADERS)
|
|
89
|
+
rows = rows_from_upstream(r["data"])
|
|
90
|
+
hits = filter_rows(rows, name=name, type=type, basis=basis)
|
|
91
|
+
return {"rows": hits[:size], "total_matched": len(hits), "scanned_rows": len(rows),
|
|
92
|
+
"limit_applied": size,
|
|
93
|
+
"params": {"name": name, "type": type, "basis": basis}, "_meta": r}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""ca_sema — Canada SEMA sanctions listings provider (Global Affairs Canada sema-lmes.xml; keyless).
|
|
2
|
+
|
|
3
|
+
legal sanctions-roster face (0.3.21; admitted 2026-11-02, probed tmp/probe-debar-0321). Shape measured 2026-11-02:
|
|
4
|
+
GET /world-monde/assets/office_docs/international_relations-relations_internationales/sanctions/sema-lmes.xml
|
|
5
|
+
→ <data-set><record>… — the Consolidated Canadian Autonomous Sanctions List under the Special Economic
|
|
6
|
+
Measures Act, **5,708 live records** (persons 3,526 with GivenName / entities 2,181 via EntityOrShip — of which
|
|
7
|
+
**731 ships carrying an IMO number**; the census-era 11,328 counted cumulative history). Tags are **bilingual
|
|
8
|
+
paired names** (Country-Pays, LastName-NomDeFamille, GivenName-Prenom, Aliases-Alias, DateOfBirthOrShipBuildDate-…,
|
|
9
|
+
EntityOrShip-EntiteOuNavire, TitleOrShipType-…, ShipIMONumber-…, Schedule-Annexe, Item-NumeroDarticle,
|
|
10
|
+
DateOfListing-…); values are bilingual too ("Syria / Syrie", ship types "General Cargo/Multi Purpose | Cargo
|
|
11
|
+
classique/navire polyvalent" — the **| separates the language halves**). Aliases are semicolon-delimited where
|
|
12
|
+
multiple (222 records). Schedule references the SEMA schedule ("1, Part 1" 1,945 / "1, Part 2" 1,062 / "Part 1"
|
|
13
|
+
742 / "1.1" 731…; 329 records carry none). Listing dates run 2008→2026 (2026 already at 518).
|
|
14
|
+
**Accept content-negotiation gate** (pinned 2026-11-02): the asset answers 406 to Accept: application/json —
|
|
15
|
+
the op overrides Accept to text/xml (ext 8); the plain agentdata UA passes.
|
|
16
|
+
Screening = local full-register scan (upstream has no query surface), cache 24h.
|
|
17
|
+
TTL: 24h; polite tier 0.2 rps (single-file face, 2.9MB).
|
|
18
|
+
"""
|
|
19
|
+
import xml.etree.ElementTree as ET
|
|
20
|
+
|
|
21
|
+
from ..errors import SdkError
|
|
22
|
+
|
|
23
|
+
provider = {
|
|
24
|
+
"id": "ca_sema",
|
|
25
|
+
"baseUrl": "https://www.international.gc.ca",
|
|
26
|
+
"rate": {"rps": 0.2, "burst": 1},
|
|
27
|
+
"ops": {
|
|
28
|
+
"listings": {
|
|
29
|
+
"path": "/world-monde/assets/office_docs/international_relations-relations_internationales/sanctions/sema-lmes.xml",
|
|
30
|
+
"accept": "text", # XML surface — the raw text comes back and this module parses it
|
|
31
|
+
"timeoutMs": 120000, # 2.9MB XML
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
"ttl": {"listings": 24 * 3600},
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
# Accept content-negotiation gate (pinned 2026-11-02): application/json → 406 Not Acceptable;
|
|
38
|
+
# text/xml (and */*) → 206/200 — ext-8 override over the transport's default JSON Accept
|
|
39
|
+
_XML_ACCEPT = {"Accept": "text/xml, */*"}
|
|
40
|
+
|
|
41
|
+
# bilingual paired tag names (upstream verbatim — the EN/FR halves live in one element name)
|
|
42
|
+
_T = {
|
|
43
|
+
"country": "Country-Pays",
|
|
44
|
+
"last_name": "LastName-NomDeFamille",
|
|
45
|
+
"given_name": "GivenName-Prenom",
|
|
46
|
+
"aliases": "Aliases-Alias",
|
|
47
|
+
"dob_or_ship_build": "DateOfBirthOrShipBuildDate-DateDeNaissanceOuDateDeConstructionDuNavire",
|
|
48
|
+
"entity_or_ship": "EntityOrShip-EntiteOuNavire",
|
|
49
|
+
"title_or_ship_type": "TitleOrShipType-TitreOuTypeDeNavire",
|
|
50
|
+
"ship_imo": "ShipIMONumber-NumeroOMIDuNavire",
|
|
51
|
+
"schedule": "Schedule-Annexe",
|
|
52
|
+
"item": "Item-NumeroDarticle",
|
|
53
|
+
"date_of_listing": "DateOfListing-DateDinscription",
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _txt(rec, key):
|
|
58
|
+
el = rec.find(_T[key])
|
|
59
|
+
return (el.text or "").strip() if el is not None and el.text else None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def rows_from_xml(text):
|
|
63
|
+
"""XML raw text → sanction-listing rows (pure function, offline-testable). Each record becomes
|
|
64
|
+
{kind, name, matched faces…}: kind = ship (IMO present) / entity (EntityOrShip) / person (GivenName×LastName);
|
|
65
|
+
name = the assembled display form ("Khazalbek Bakhtibekovich Atabekov" for persons, the EntityOrShip value
|
|
66
|
+
otherwise); aliases = semicolon-split; every other field verbatim bilingual."""
|
|
67
|
+
if not text:
|
|
68
|
+
return []
|
|
69
|
+
root = ET.fromstring(text)
|
|
70
|
+
rows = []
|
|
71
|
+
for rec in root.findall("record"):
|
|
72
|
+
imo = _txt(rec, "ship_imo")
|
|
73
|
+
entity = _txt(rec, "entity_or_ship")
|
|
74
|
+
given, last = _txt(rec, "given_name"), _txt(rec, "last_name")
|
|
75
|
+
if imo:
|
|
76
|
+
kind = "ship"
|
|
77
|
+
elif entity:
|
|
78
|
+
kind = "entity"
|
|
79
|
+
else:
|
|
80
|
+
kind = "person"
|
|
81
|
+
if kind == "person":
|
|
82
|
+
name = " ".join(x for x in (given, last) if x) or None
|
|
83
|
+
else:
|
|
84
|
+
name = entity
|
|
85
|
+
aliases = []
|
|
86
|
+
for el in rec.findall(_T["aliases"]):
|
|
87
|
+
aliases.extend(a.strip() for a in (el.text or "").split(";") if a.strip())
|
|
88
|
+
rows.append({
|
|
89
|
+
"kind": kind,
|
|
90
|
+
"name": name,
|
|
91
|
+
"aliases": aliases,
|
|
92
|
+
"country": _txt(rec, "country"), # bilingual verbatim ("Syria / Syrie")
|
|
93
|
+
"dob_or_ship_build": _txt(rec, "dob_or_ship_build"),
|
|
94
|
+
"title_or_ship_type": _txt(rec, "title_or_ship_type"), # person title OR ship type (bilingual)
|
|
95
|
+
"ship_imo": imo, # the join key onto shipping rosters (THETIS-MRV family)
|
|
96
|
+
"schedule": _txt(rec, "schedule"),
|
|
97
|
+
"item": _txt(rec, "item"),
|
|
98
|
+
"date_of_listing": _txt(rec, "date_of_listing"),
|
|
99
|
+
})
|
|
100
|
+
return rows
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def filter_rows(rows, name=None, country=None):
|
|
104
|
+
"""Local register scan (pure function). name: contains (case-insensitive) over name **and** aliases
|
|
105
|
+
(matched_on echoes the hit face); country: contains over the bilingual country cell ("Syria" and "Syrie"
|
|
106
|
+
both hit the "Syria / Syrie" row)."""
|
|
107
|
+
needle = str(name or "").strip().lower() or None
|
|
108
|
+
cn_needle = str(country or "").strip().lower() or None
|
|
109
|
+
hits = []
|
|
110
|
+
for r in rows:
|
|
111
|
+
if cn_needle and cn_needle not in str(r.get("country") or "").lower():
|
|
112
|
+
continue
|
|
113
|
+
if not needle:
|
|
114
|
+
hits.append({**r, "matched_on": None})
|
|
115
|
+
continue
|
|
116
|
+
if needle in str(r.get("name") or "").lower():
|
|
117
|
+
hits.append({**r, "matched_on": "name"})
|
|
118
|
+
continue
|
|
119
|
+
alias = next((a for a in r.get("aliases") or [] if needle in a.lower()), None)
|
|
120
|
+
if alias:
|
|
121
|
+
hits.append({**r, "matched_on": "alias"})
|
|
122
|
+
return hits
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def sanctions_raw(name=None, *, country=None, limit=20, cache=None, no_cache=False):
|
|
126
|
+
"""Full-register scan (pulls the whole XML, caches 24h, then scans locally)
|
|
127
|
+
→ {rows, total_matched, scanned_rows, limit_applied, params, _meta}.
|
|
128
|
+
|
|
129
|
+
name = contains over name + aliases; country = contains over the bilingual country cell.
|
|
130
|
+
No filter → enumerate from the register head."""
|
|
131
|
+
from ..transport import call_provider
|
|
132
|
+
|
|
133
|
+
size = int(limit) if limit is not None else 20
|
|
134
|
+
if not 1 <= size <= 100:
|
|
135
|
+
raise SdkError("usage_error", provider="ca_sema",
|
|
136
|
+
detail=f"limit must be 1-100 (local payload wall), got '{limit}'")
|
|
137
|
+
r = call_provider(provider, "listings", {}, cache=cache, no_cache=no_cache,
|
|
138
|
+
extra_headers=_XML_ACCEPT)
|
|
139
|
+
rows = rows_from_xml(r["data"])
|
|
140
|
+
hits = filter_rows(rows, name=name, country=country)
|
|
141
|
+
return {"rows": hits[:size], "total_matched": len(hits), "scanned_rows": len(rows),
|
|
142
|
+
"limit_applied": size,
|
|
143
|
+
"params": {"name": name, "country": country}, "_meta": r}
|