agentdata 0.3.26__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. agentdata/__init__.py +33 -0
  2. agentdata/__main__.py +4 -0
  3. agentdata/cache.py +115 -0
  4. agentdata/cli.py +173 -0
  5. agentdata/client.py +158 -0
  6. agentdata/company/__init__.py +3 -0
  7. agentdata/company/cf.py +145 -0
  8. agentdata/company/ch_zefix.py +100 -0
  9. agentdata/company/cli.py +243 -0
  10. agentdata/company/co_rues.py +72 -0
  11. agentdata/company/edgar_cik.py +84 -0
  12. agentdata/company/edgar_fts.py +101 -0
  13. agentdata/company/faces.py +746 -0
  14. agentdata/company/fdic_bankfind.py +107 -0
  15. agentdata/company/fi_ytj.py +89 -0
  16. agentdata/company/fr_recherche.py +124 -0
  17. agentdata/company/fts.py +121 -0
  18. agentdata/company/gleif.py +178 -0
  19. agentdata/company/kr_hometax.py +112 -0
  20. agentdata/company/no_brreg.py +124 -0
  21. agentdata/company/ror.py +105 -0
  22. agentdata/company/ted.py +117 -0
  23. agentdata/company/usaspending.py +124 -0
  24. agentdata/company/vies.py +66 -0
  25. agentdata/daterange.py +35 -0
  26. agentdata/economy/__init__.py +10 -0
  27. agentdata/economy/bis.py +132 -0
  28. agentdata/economy/cli.py +199 -0
  29. agentdata/economy/ecb.py +117 -0
  30. agentdata/economy/eurostat.py +163 -0
  31. agentdata/economy/faces.py +573 -0
  32. agentdata/economy/fred.py +92 -0
  33. agentdata/economy/ilo_stat.py +99 -0
  34. agentdata/economy/imf_sdmx.py +133 -0
  35. agentdata/economy/un_comtrade.py +199 -0
  36. agentdata/economy/unsd_sdg.py +154 -0
  37. agentdata/economy/wb.py +208 -0
  38. agentdata/economy/who_xmart.py +177 -0
  39. agentdata/entries.py +30 -0
  40. agentdata/errors.py +38 -0
  41. agentdata/esg/__init__.py +5 -0
  42. agentdata/esg/cli.py +180 -0
  43. agentdata/esg/climatetrace.py +260 -0
  44. agentdata/esg/eiti.py +187 -0
  45. agentdata/esg/epa_frs.py +138 -0
  46. agentdata/esg/euasl.py +75 -0
  47. agentdata/esg/eutl.py +66 -0
  48. agentdata/esg/faces.py +498 -0
  49. agentdata/esg/paris_mou.py +240 -0
  50. agentdata/esg/thetis_mrv.py +67 -0
  51. agentdata/healthcare/__init__.py +7 -0
  52. agentdata/healthcare/au_tga.py +142 -0
  53. agentdata/healthcare/ca_dpd.py +123 -0
  54. agentdata/healthcare/ca_mdall.py +155 -0
  55. agentdata/healthcare/cli.py +375 -0
  56. agentdata/healthcare/ctgov.py +101 -0
  57. agentdata/healthcare/dailymed.py +102 -0
  58. agentdata/healthcare/epmc.py +108 -0
  59. agentdata/healthcare/eudamed.py +124 -0
  60. agentdata/healthcare/faces.py +811 -0
  61. agentdata/healthcare/fda_crl.py +86 -0
  62. agentdata/healthcare/fda_devices.py +199 -0
  63. agentdata/healthcare/fda_drugs.py +156 -0
  64. agentdata/healthcare/fda_recalls.py +137 -0
  65. agentdata/healthcare/gb_mhra.py +137 -0
  66. agentdata/healthcare/gudid.py +84 -0
  67. agentdata/healthcare/mesh.py +89 -0
  68. agentdata/healthcare/rasff.py +164 -0
  69. agentdata/healthcare/rxnav.py +67 -0
  70. agentdata/healthcare/safety_gate.py +175 -0
  71. agentdata/healthcare/shortages.py +107 -0
  72. agentdata/legal/__init__.py +4 -0
  73. agentdata/legal/afdb.py +93 -0
  74. agentdata/legal/ca_sema.py +143 -0
  75. agentdata/legal/cli.py +312 -0
  76. agentdata/legal/co_sic.py +70 -0
  77. agentdata/legal/courtlistener.py +77 -0
  78. agentdata/legal/csl.py +139 -0
  79. agentdata/legal/dgcomp.py +125 -0
  80. agentdata/legal/ecfr.py +129 -0
  81. agentdata/legal/faces.py +700 -0
  82. agentdata/legal/federalregister.py +188 -0
  83. agentdata/legal/gb_cma.py +117 -0
  84. agentdata/legal/gb_tna_caselaw.py +245 -0
  85. agentdata/legal/idb.py +85 -0
  86. agentdata/legal/lda.py +147 -0
  87. agentdata/legal/sipri.py +178 -0
  88. agentdata/legal/ukec.py +148 -0
  89. agentdata/legal/wb.py +92 -0
  90. agentdata/procurement/__init__.py +5 -0
  91. agentdata/procurement/br_pncp.py +126 -0
  92. agentdata/procurement/cli.py +155 -0
  93. agentdata/procurement/co_secop.py +88 -0
  94. agentdata/procurement/faces.py +239 -0
  95. agentdata/procurement/ny_open_data.py +82 -0
  96. agentdata/procurement/ungm.py +137 -0
  97. agentdata/procurement/wb_proc.py +84 -0
  98. agentdata/research/__init__.py +4 -0
  99. agentdata/research/biorxiv.py +66 -0
  100. agentdata/research/cli.py +129 -0
  101. agentdata/research/crossref.py +121 -0
  102. agentdata/research/faces.py +349 -0
  103. agentdata/research/gb_gtr.py +58 -0
  104. agentdata/research/icite.py +52 -0
  105. agentdata/research/nih_reporter.py +143 -0
  106. agentdata/research/nsf.py +68 -0
  107. agentdata/research/openalex.py +145 -0
  108. agentdata/research/opencitations.py +50 -0
  109. agentdata/security/__init__.py +3 -0
  110. agentdata/security/cisa_kev.py +42 -0
  111. agentdata/security/cli.py +37 -0
  112. agentdata/security/faces.py +91 -0
  113. agentdata/security/osv.py +72 -0
  114. agentdata/transport.py +541 -0
  115. agentdata-0.3.26.dist-info/METADATA +266 -0
  116. agentdata-0.3.26.dist-info/RECORD +120 -0
  117. agentdata-0.3.26.dist-info/WHEEL +5 -0
  118. agentdata-0.3.26.dist-info/entry_points.txt +10 -0
  119. agentdata-0.3.26.dist-info/licenses/LICENSE +201 -0
  120. agentdata-0.3.26.dist-info/top_level.txt +1 -0
@@ -0,0 +1,175 @@
1
+ """safety_gate — EU Safety Gate (the ex-RAPEX rapid alert system for dangerous non-food products)
2
+ weekly-report XML face provider (ec.europa.eu/safety-gate-alerts/api/download/; keyless).
3
+
4
+ healthcare domain (0.3.19; admitted 2026-11-02, probed). Shapes verified by first-hand live probing
5
+ 2026-11-02 (the probe corpus lives in tmp/probe-ca-eu-faces/):
6
+ GET /safety-gate-alerts/api/download/weeklyReport/list/xml/en
7
+ → the whole weekly-report index in ONE XML document (1,122 reports measured 2026-11-02;
8
+ Report-2005-1 → Report-2026-43). Each <weeklyReport> entry: reference (Report-YYYY-WW),
9
+ publicationDate (dd/mm/yyyy), year/month/day/week, and URL — the URL path embeds the numeric
10
+ reportId (…/detail/xml/10000329), which is the join key into the detail face.
11
+ GET /safety-gate-alerts/api/download/weeklyReport/detail/xml/<reportId>?language=en&search=WEB_REPORT|:|<reportId>,
12
+ → the single weekly report as XML: a <Safety-Gate> root (report_language/report_date/report_year/
13
+ report_week) wrapping N <notifications> entries (67 measured for report 10000329 = 2026-W43).
14
+ Alert fields: caseNumber (SR/03023/26), reference (the alert-detail URL → the numeric alert id),
15
+ category, product, brand, name, type_numberOfModel, batchNumber, barcode, riskType, danger
16
+ (the risk narrative), measures (the concatenated operator/measure/effective-date blocks),
17
+ URLrecall, description, notifyingCountry, countryOfOrigin, type (Consumer…), level
18
+ (Serious risk…), onlineTrader, companyRecallCode, productionDates.
19
+ language is a query param but search is the SPA's own echo parameter — both are FIXED for this
20
+ face (constructed from the report id; the en channel is the one probed).
21
+ The list/detail pair is the same shape family as gb_tna_caselaw (an XML face among JSON APIs).
22
+ Local row slicing (limit/skip) and the year post-filter slice the single cached full documents.
23
+ TTL: list 24h (new reports land weekly) / detail 7 days (a published weekly report is immutable);
24
+ polite rate 1 rps (the download family publishes no quota).
25
+ """
26
+ import re
27
+ import xml.etree.ElementTree as ET
28
+
29
+ from ..errors import SdkError
30
+
31
+ provider = {
32
+ "id": "safety_gate",
33
+ "baseUrl": "https://ec.europa.eu",
34
+ "rate": {"rps": 1, "burst": 2},
35
+ "ops": {
36
+ "weekly_list": {"path": "/safety-gate-alerts/api/download/weeklyReport/list/xml/en", "accept": "text"},
37
+ # the detail query string is baked into the path byte-exact as probed (the ukec shape: urlencode
38
+ # would re-encode the SPA's echo parameter WEB_REPORT|:|<id>, — a silent near-miss risk not worth taking)
39
+ "weekly_detail": {"path": "/safety-gate-alerts/api/download/weeklyReport/detail/xml/{report_id}"
40
+ "?language=en&search=WEB_REPORT%7C:%7C{report_id},",
41
+ "accept": "text"},
42
+ },
43
+ "ttl": {"weekly_list": 24 * 3600, "weekly_detail": 7 * 86400},
44
+ }
45
+
46
+ _REPORT_ID = re.compile(r"/detail/xml/(\d+)") # the list face's download URL → the report id
47
+ _ALERT_ID = re.compile(r"alertDetail/(\d+)") # the detail rows' reference URL → the alert id (a different path shape)
48
+
49
+
50
+ def _txt(el):
51
+ return (el.text or "").strip() or None if el is not None else None
52
+
53
+
54
+ def alerts_from_upstream(xml_text):
55
+ """The list XML → weekly-report index rows (pure function, offline-testable).
56
+
57
+ The reportId lives only inside the URL element — it is carved out here as the join key."""
58
+ root = ET.fromstring(xml_text)
59
+ rows = []
60
+ for wr in root.iter("weeklyReport"):
61
+ url = _txt(wr.find("URL")) or ""
62
+ rid = _REPORT_ID.search(url)
63
+ rows.append({
64
+ "report_id": rid.group(1) if rid else None, # the join key into the detail face
65
+ "reference": _txt(wr.find("reference")), # Report-YYYY-WW
66
+ "publication_date": _txt(wr.find("publicationDate")),
67
+ "year": _txt(wr.find("year")),
68
+ "week": _txt(wr.find("week")),
69
+ "url": url or None,
70
+ })
71
+ return {"rows": rows, "count": len(rows)}
72
+
73
+
74
+ def report_from_upstream(xml_text, report_id):
75
+ """The detail XML → {report header, rows[]} (pure function, offline-testable)."""
76
+ root = ET.fromstring(xml_text)
77
+ rows = []
78
+ for n in root.iter("notifications"):
79
+ ref_url = _txt(n.find("reference")) or ""
80
+ aid = _ALERT_ID.search(ref_url)
81
+ rows.append({
82
+ "order": _txt(n.find("order")),
83
+ "case_number": _txt(n.find("caseNumber")),
84
+ "alert_id": aid.group(1) if aid else None, # the numeric id from the alert-detail URL
85
+ "alert_url": ref_url or None,
86
+ "category": _txt(n.find("category")),
87
+ "product": _txt(n.find("product")),
88
+ "brand": _txt(n.find("brand")),
89
+ "name": _txt(n.find("name")),
90
+ "type_number_of_model": _txt(n.find("type_numberOfModel")),
91
+ "batch_number": _txt(n.find("batchNumber")),
92
+ "barcode": _txt(n.find("barcode")),
93
+ "risk_type": _txt(n.find("riskType")), # e.g. "Electric shock, Fire"
94
+ "danger": _txt(n.find("danger")), # the risk narrative
95
+ "measures": _txt(n.find("measures")), # operator/measure/effective-date blocks
96
+ "url_recall": _txt(n.find("URLrecall")),
97
+ "description": _txt(n.find("description")),
98
+ "notifying_country": _txt(n.find("notifyingCountry")),
99
+ "country_of_origin": _txt(n.find("countryOfOrigin")),
100
+ "type": _txt(n.find("type")), # e.g. Consumer
101
+ "level": _txt(n.find("level")), # e.g. Serious risk
102
+ "online_trader": _txt(n.find("onlineTrader")),
103
+ "company_recall_code": _txt(n.find("companyRecallCode")),
104
+ "production_dates": _txt(n.find("productionDates")),
105
+ })
106
+ return {
107
+ "report_id": str(report_id),
108
+ # the detail XML carries no Report-YYYY-WW reference field — the caller stitches it from the
109
+ # list face; the report_year/report_week pair below is the upstream's own framing
110
+ "report_date": _txt(root.find("report_date")),
111
+ "report_year": _txt(root.find("report_year")),
112
+ "report_week": _txt(root.find("report_week")),
113
+ "report_language": _txt(root.find("report_language")),
114
+ "rows": rows, "count": len(rows),
115
+ }
116
+
117
+
118
+ def _slice(rows, limit, skip):
119
+ off = int(skip) if skip is not None else 0
120
+ if off < 0:
121
+ raise SdkError("usage_error", provider="safety_gate", detail=f"skip must be >=0, got \"{skip}\"")
122
+ size = int(limit) if limit is not None else 200
123
+ if size < 1:
124
+ raise SdkError("usage_error", provider="safety_gate", detail=f"limit must be >=1 (local post-slice, no upstream cap), got \"{limit}\"")
125
+ return rows[off:off + size]
126
+
127
+
128
+ def weekly_list_raw(*, year=None, limit=200, skip=0, cache=None, no_cache=False):
129
+ """Flat-panel low-level face: the weekly-report index (1,122 reports 2005→now, one cached XML).
130
+
131
+ year = an optional local post-filter on the report year (the upstream index has no filter —
132
+ it always serves the whole index); limit/skip is a local post-slice.
133
+ → {rows[], count, total, params, _meta} — total = the whole index size (post-year-filter)."""
134
+ from ..transport import call_provider
135
+
136
+ params = {}
137
+ if year is not None:
138
+ y = str(year).strip()
139
+ if not y.isdigit():
140
+ raise SdkError("usage_error", provider="safety_gate", detail=f"year must be a 4-digit year (a local post-filter on the report index), got \"{year}\"")
141
+ params["year"] = y
142
+ r = call_provider(provider, "weekly_list", {}, cache=cache, no_cache=no_cache)
143
+ out = alerts_from_upstream(r["data"])
144
+ rows = [x for x in out["rows"] if (not params.get("year") or x["year"] == params["year"])]
145
+ rows = _slice(rows, limit, skip)
146
+ return {"rows": rows, "count": len(rows), "total": len(out["rows"]) if not params.get("year")
147
+ else sum(1 for x in out["rows"] if x["year"] == params["year"]),
148
+ "params": params or None, "_meta": r}
149
+
150
+
151
+ def _clean_report_id(report_id):
152
+ """Numeric reportId; a pasted detail URL gets its id carved out (the join key echoed by the list rows)."""
153
+ s = str(report_id or "").strip()
154
+ m = _REPORT_ID.search(s)
155
+ if m:
156
+ s = m.group(1)
157
+ if not s.isdigit():
158
+ raise SdkError("usage_error", provider="safety_gate",
159
+ detail=f"report_id must be the numeric report id (e.g. 10000329 — the report_id field echoed by safety_gate_alerts; a pasted detail URL also works), got \"{report_id}\"")
160
+ return s
161
+
162
+
163
+ def weekly_detail_raw(report_id, *, limit=200, skip=0, cache=None, no_cache=False):
164
+ """Flat-panel low-level face: one weekly report's alerts (the detail XML — 67 alerts for 2026-W43).
165
+
166
+ → {report header…, rows[], count, params, _meta} — the alert rows carry caseNumber/risk/level/
167
+ notifyingCountry/countryOfOrigin/onlineTrader (the company-relevant edges) verbatim."""
168
+ from ..transport import call_provider
169
+
170
+ rid = _clean_report_id(report_id)
171
+ params = {"report_id": rid}
172
+ r = call_provider(provider, "weekly_detail", params, cache=cache, no_cache=no_cache)
173
+ out = report_from_upstream(r["data"] or "", rid)
174
+ rows = _slice(out["rows"], limit, skip)
175
+ return {**out, "rows": rows, "count": len(rows), "params": params, "_meta": r}
@@ -0,0 +1,107 @@
1
+ """shortages — FDA Drug Shortages provider (openFDA drug/shortages.json; keyless).
2
+
3
+ Wave 5 of the healthcare domain: related
4
+ kind=drug_shortages — manufacturer/drug name → drug shortage records, the supply-risk event stream: the last piece of the pipeline × safety × supply triad.
5
+ Row shapes measured 2026-10-23:
6
+ GET /drug/shortages.json?search=<q>&limit=N&skip=M&sort=update_date:desc
7
+ search segments (three, AND-combinable, measured q1=179/q2=82/q3=AND 18):
8
+ manufacturer → openfda.manufacturer_name.exact:"Fresenius Kabi USA, LLC" (exact phrase; bare tokens scatter)
9
+ drug name → generic_name:"bupivacaine" (phrase search, multi-word ingredient names match whole)
10
+ status → status:Current (the enum Current/Resolved/To Be Discontinued — measured on a 100-row Fresenius sample as 83/1/16)
11
+ **zero hits = 404 {"error":{"code":"NOT_FOUND","message":"No matches found!"}}** — a legal answer, not an error
12
+ (the openFDA convention); the op declares notFound:[404], and upstream_not_found turns into an empty set at this layer.
13
+ Row fields: generic_name/company_name/availability/status/update_type/update_date/initial_posting_date/
14
+ package_ndc/presentation/dosage_form/shortage_reason/therapeutic_category/contact_info
15
+ + the openfda enrichment block (manufacturer_name/application_number/product_ndc/rxcui/substance_name)
16
+ — company_name = the manufacturer registered on the shortage record (may differ from openfda.manufacturer_name: that is the normalization layer, both columns are kept).
17
+ update_date can be sorted desc (measured: 10/15/2026 comes back first); the date form is MM/DD/YYYY.
18
+ TTL: a daily cadence (meta.last_updated measured 2026-10-22) → 24h; the keyless quota is 240/min → polite tier 2 rps.
19
+ """
20
+ from datetime import datetime, timezone
21
+
22
+ from ..errors import SdkError
23
+
24
+ provider = {
25
+ "id": "shortages",
26
+ "baseUrl": "https://api.fda.gov",
27
+ "rate": {"rps": 2, "burst": 5, "group": "fda"},
28
+ "ops": {
29
+ "search": {"path": "/drug/shortages.json", "notFound": [404]}, # params: search, limit, skip, sort
30
+ },
31
+ "ttl": {"search": 24 * 3600},
32
+ }
33
+
34
+
35
+ def rows_from_upstream(results):
36
+ """Upstream rows → shortage record rows (pure function, testable offline): both company columns are kept (the registered manufacturer + the openFDA normalized manufacturer)."""
37
+ rows = results if isinstance(results, list) else []
38
+ out = []
39
+ for r in rows:
40
+ of = r.get("openfda") or {}
41
+ out.append({
42
+ "generic_name": r.get("generic_name"),
43
+ "company_name": r.get("company_name"), # the manufacturer registered on the shortage record (company edge, primary column)
44
+ "manufacturers": of.get("manufacturer_name"), # the openFDA normalized manufacturer (company edge, comparison column)
45
+ "availability": r.get("availability"),
46
+ "status": r.get("status"),
47
+ "update_type": r.get("update_type"),
48
+ "update_date": r.get("update_date"),
49
+ "first_posted": r.get("initial_posting_date"),
50
+ "package_ndc": r.get("package_ndc"),
51
+ "presentation": r.get("presentation"),
52
+ "dosage_form": r.get("dosage_form"),
53
+ "shortage_reason": r.get("shortage_reason"),
54
+ "therapeutic_category": r.get("therapeutic_category"),
55
+ "applications": of.get("application_number"),
56
+ "product_ndcs": of.get("product_ndc"),
57
+ "rxcuis": of.get("rxcui"),
58
+ "substances": of.get("substance_name"),
59
+ "contact": r.get("contact_info"),
60
+ })
61
+ return out
62
+
63
+
64
+ def search_raw(search, *, limit=20, skip=0, sort="update_date:desc", cache=None, no_cache=False):
65
+ """0.2.0 mirror face: the search expression is passed through verbatim (openFDA query syntax) + the upstream param names limit/skip/sort.
66
+
67
+ Hard limit (protocol, confirmed by probing on 2026-10-23): limit 1-1000 — 1001 → 400 BAD_REQUEST
68
+ «Limit cannot exceed 1000 results for search requests. Use the skip or search_after param»;
69
+ deeper paging goes through skip — search_after is not accepted (probed 2026-10-28): with sort → 400 «search_after has 1 value(s)
70
+ but sort has 2» (a semicolon double value → an ES parse error; the default sort → 200 but the rows are identical to the baseline = silently ignored); and the whole
71
+ database holds only ~1604 rows, so limit 1000 + skip reaches the bottom in two pages — the deep-paging gap does not exist, skip is complete.
72
+ search is required (local safety gate: under the keyless 240/min quota a bare full-table query is served by the bulk harvest line, not by this single-lookup face).
73
+ A free sort string passes through (measured update_date:desc works; the default is ours, not the upstream one).
74
+ → {rows[], total, count, skip, limit_applied, has_more, search, sort, _meta}; zero hits = an empty set that is not cached (same as above).
75
+ """
76
+ from ..transport import call_provider
77
+
78
+ q = str(search or "").strip()
79
+ if not q:
80
+ raise SdkError("usage_error", provider="shortages",
81
+ detail="search requires an expression (e.g. generic_name:'bupivacaine' — the phrase segment needs double quotes; a bare full-table query is served by the bulk line)")
82
+ size = int(limit or 20)
83
+ if not 1 <= size <= 1000:
84
+ raise SdkError("usage_error", provider="shortages",
85
+ detail=f"limit must be 1-1000 (upstream 400: Limit cannot exceed 1000 results for search "
86
+ f"requests — measured 2026-10-23; to page deeper use skip, the whole database of ~1604 rows is reached in two pages), got {size}")
87
+ sk = max(0, int(skip or 0))
88
+ so = str(sort or "").strip()
89
+ params = {"search": q, "limit": size}
90
+ if sk:
91
+ params["skip"] = sk
92
+ if so:
93
+ params["sort"] = so
94
+ try:
95
+ r = call_provider(provider, "search", params, cache=cache, no_cache=no_cache)
96
+ except SdkError as e:
97
+ if e.code == "upstream_not_found":
98
+ return {"rows": [], "total": 0, "count": 0, "skip": sk, "limit_applied": size,
99
+ "has_more": False, "search": q, "sort": so or None,
100
+ "_meta": {"from_cache": False,
101
+ "fetched_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")}}
102
+ raise
103
+ data = r.get("data") or {}
104
+ rows = rows_from_upstream(data.get("results"))
105
+ total = (data.get("meta") or {}).get("results", {}).get("total", len(rows))
106
+ return {"rows": rows, "total": total, "count": len(rows), "skip": sk, "limit_applied": size,
107
+ "has_more": total > sk + len(rows), "search": q, "sort": so or None, "_meta": r}
@@ -0,0 +1,4 @@
1
+ """legal domain package — provider adapters laid flat (one source per file) + faces/cli (0.2.6 per-domain split).
2
+ """
3
+ from . import (afdb, ca_sema, csl, courtlistener, co_sic, dgcomp, ecfr, # noqa: F401
4
+ federalregister, idb, lda, sipri, gb_tna_caselaw, ukec, gb_cma, wb)
@@ -0,0 +1,93 @@
1
+ """afdb — African Development Bank debarred-entities roster provider (afdb.org json-feed; keyless).
2
+
3
+ legal debarment-roster face (0.3.21; admitted 2026-11-02, probed tmp/probe-debar-0321). Shape measured 2026-11-02:
4
+ GET /en/debarred-entities-json-feed → a bare JSON array, 1,440 rows × 7 columns (Type FIRM 1,196 / INDIVIDUAL 244,
5
+ Name, Nationality, From, To, Basis, Notes) — a single-shot full dump (the -csv-export sibling serves byte-identical
6
+ content; the -xml-export sibling returns a 2.4KB stub). To = empty means ongoing (169 rows); Notes ∈ {ONGOING,
7
+ INDEFINITE, empty}.
8
+ **Basis is the cross-debarment edge, self-evidenced**: "CROSS DEBARMENT : WBG" 653 / "CROSS-DEBARMENT: ADB" 377 /
9
+ "CROSS-DEBARMENT: IDB" 161+2 / "CROSS DEBARMENT : EBRD" 127 / "DEBARMENT:AFDB" 120 — note the upstream spacing is
10
+ **inconsistent** ("CROSS DEBARMENT : WBG" with spaces vs "CROSS-DEBARMENT: ADB" hyphenated): the basis filter is a
11
+ contains-match precisely so "WBG"/"ADB"/"IDB"/"EBRD" hit regardless of punctuation.
12
+ **Cloudflare UA-fingerprint gate** (pinned 2026-11-02): the default agentdata UA passes then 403s flakily on repeat;
13
+ a bare "Mozilla/5.0" UA is refused outright; the **full browser document suite passes consistently** — per-request
14
+ header override (ext 8), same family as UNGM (0.3.12).
15
+ Screening = local full-register scan (upstream has no query surface), cache 24h.
16
+ TTL: 24h; polite tier 0.2 rps (single-endpoint face; also keeps the Cloudflare gate calm).
17
+ """
18
+ from ..errors import SdkError
19
+
20
+ provider = {
21
+ "id": "afdb",
22
+ "baseUrl": "https://www.afdb.org",
23
+ "rate": {"rps": 0.2, "burst": 1},
24
+ "ops": {
25
+ "debarred": {"path": "/en/debarred-entities-json-feed", "timeoutMs": 90000},
26
+ },
27
+ "ttl": {"debarred": 24 * 3600},
28
+ }
29
+
30
+ # Cloudflare UA-fingerprint gate (pinned 2026-11-02, tmp/probe-debar-0321 p3): default agentdata UA → flaky
31
+ # 403 on repeat; bare Mozilla/5.0 → 403; this full document suite → 200 twice in a row — ext-8 override
32
+ _CF_BROWSER_HEADERS = {
33
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
34
+ "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
35
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.7",
36
+ "Accept-Language": "en-US,en;q=0.9",
37
+ "Referer": "https://www.afdb.org/en/projects-operations/debarment-and-sanctions-procedures",
38
+ "Upgrade-Insecure-Requests": "1",
39
+ "Sec-Fetch-Dest": "document",
40
+ "Sec-Fetch-Mode": "navigate",
41
+ "Sec-Fetch-Site": "none",
42
+ "Sec-Fetch-User": "?1",
43
+ "sec-ch-ua": '"Chromium";v="131", "Not_A Brand";v="24"',
44
+ "sec-ch-ua-mobile": "?0",
45
+ "sec-ch-ua-platform": '"Windows"',
46
+ }
47
+
48
+
49
+ def rows_from_upstream(data):
50
+ """Upstream JSON array → debarred-entity rows (pure function; the seven upstream columns verbatim —
51
+ the register is already narrow, and To="" / Notes carry the ongoing/indefinite semantics)."""
52
+ return [{k: r.get(k) for k in ("Type", "Name", "Nationality", "From", "To", "Basis", "Notes")}
53
+ for r in (data or [])]
54
+
55
+
56
+ def filter_rows(rows, name=None, type=None, basis=None):
57
+ """Local register scan (pure function). name: contains (case-insensitive) over Name; type: exact over
58
+ Type (FIRM/INDIVIDUAL); basis: **contains** over Basis — upstream spacing is inconsistent
59
+ ("CROSS DEBARMENT : WBG" vs "CROSS-DEBARMENT: ADB"), so filter on the bank token (WBG/ADB/IDB/EBRD)."""
60
+ needle = str(name or "").strip().lower() or None
61
+ ty_needle = str(type or "").strip().upper() or None
62
+ bs_needle = str(basis or "").strip().lower() or None
63
+ hits = []
64
+ for r in rows:
65
+ if ty_needle and str(r.get("Type") or "").strip().upper() != ty_needle:
66
+ continue
67
+ if bs_needle and bs_needle not in str(r.get("Basis") or "").lower():
68
+ continue
69
+ if needle and needle not in str(r.get("Name") or "").lower():
70
+ continue
71
+ hits.append(r)
72
+ return hits
73
+
74
+
75
+ def debarred_entities_raw(name=None, *, type=None, basis=None, limit=20, cache=None, no_cache=False):
76
+ """Full-register scan (pulls the whole feed, caches 24h, then scans locally)
77
+ → {rows, total_matched, scanned_rows, limit_applied, params, _meta}.
78
+
79
+ name = contains over Name; type = FIRM/INDIVIDUAL exact; basis = contains over Basis (bank token:
80
+ WBG/ADB/IDB/EBRD — punctuation upstream is inconsistent). No filter → enumerate from the register head."""
81
+ from ..transport import call_provider
82
+
83
+ size = int(limit) if limit is not None else 20
84
+ if not 1 <= size <= 100:
85
+ raise SdkError("usage_error", provider="afdb",
86
+ detail=f"limit must be 1-100 (local payload wall), got '{limit}'")
87
+ r = call_provider(provider, "debarred", {}, cache=cache, no_cache=no_cache,
88
+ extra_headers=_CF_BROWSER_HEADERS)
89
+ rows = rows_from_upstream(r["data"])
90
+ hits = filter_rows(rows, name=name, type=type, basis=basis)
91
+ return {"rows": hits[:size], "total_matched": len(hits), "scanned_rows": len(rows),
92
+ "limit_applied": size,
93
+ "params": {"name": name, "type": type, "basis": basis}, "_meta": r}
@@ -0,0 +1,143 @@
1
+ """ca_sema — Canada SEMA sanctions listings provider (Global Affairs Canada sema-lmes.xml; keyless).
2
+
3
+ legal sanctions-roster face (0.3.21; admitted 2026-11-02, probed tmp/probe-debar-0321). Shape measured 2026-11-02:
4
+ GET /world-monde/assets/office_docs/international_relations-relations_internationales/sanctions/sema-lmes.xml
5
+ → <data-set><record>… — the Consolidated Canadian Autonomous Sanctions List under the Special Economic
6
+ Measures Act, **5,708 live records** (persons 3,526 with GivenName / entities 2,181 via EntityOrShip — of which
7
+ **731 ships carrying an IMO number**; the census-era 11,328 counted cumulative history). Tags are **bilingual
8
+ paired names** (Country-Pays, LastName-NomDeFamille, GivenName-Prenom, Aliases-Alias, DateOfBirthOrShipBuildDate-…,
9
+ EntityOrShip-EntiteOuNavire, TitleOrShipType-…, ShipIMONumber-…, Schedule-Annexe, Item-NumeroDarticle,
10
+ DateOfListing-…); values are bilingual too ("Syria / Syrie", ship types "General Cargo/Multi Purpose | Cargo
11
+ classique/navire polyvalent" — the **| separates the language halves**). Aliases are semicolon-delimited where
12
+ multiple (222 records). Schedule references the SEMA schedule ("1, Part 1" 1,945 / "1, Part 2" 1,062 / "Part 1"
13
+ 742 / "1.1" 731…; 329 records carry none). Listing dates run 2008→2026 (2026 already at 518).
14
+ **Accept content-negotiation gate** (pinned 2026-11-02): the asset answers 406 to Accept: application/json —
15
+ the op overrides Accept to text/xml (ext 8); the plain agentdata UA passes.
16
+ Screening = local full-register scan (upstream has no query surface), cache 24h.
17
+ TTL: 24h; polite tier 0.2 rps (single-file face, 2.9MB).
18
+ """
19
+ import xml.etree.ElementTree as ET
20
+
21
+ from ..errors import SdkError
22
+
23
+ provider = {
24
+ "id": "ca_sema",
25
+ "baseUrl": "https://www.international.gc.ca",
26
+ "rate": {"rps": 0.2, "burst": 1},
27
+ "ops": {
28
+ "listings": {
29
+ "path": "/world-monde/assets/office_docs/international_relations-relations_internationales/sanctions/sema-lmes.xml",
30
+ "accept": "text", # XML surface — the raw text comes back and this module parses it
31
+ "timeoutMs": 120000, # 2.9MB XML
32
+ },
33
+ },
34
+ "ttl": {"listings": 24 * 3600},
35
+ }
36
+
37
+ # Accept content-negotiation gate (pinned 2026-11-02): application/json → 406 Not Acceptable;
38
+ # text/xml (and */*) → 206/200 — ext-8 override over the transport's default JSON Accept
39
+ _XML_ACCEPT = {"Accept": "text/xml, */*"}
40
+
41
+ # bilingual paired tag names (upstream verbatim — the EN/FR halves live in one element name)
42
+ _T = {
43
+ "country": "Country-Pays",
44
+ "last_name": "LastName-NomDeFamille",
45
+ "given_name": "GivenName-Prenom",
46
+ "aliases": "Aliases-Alias",
47
+ "dob_or_ship_build": "DateOfBirthOrShipBuildDate-DateDeNaissanceOuDateDeConstructionDuNavire",
48
+ "entity_or_ship": "EntityOrShip-EntiteOuNavire",
49
+ "title_or_ship_type": "TitleOrShipType-TitreOuTypeDeNavire",
50
+ "ship_imo": "ShipIMONumber-NumeroOMIDuNavire",
51
+ "schedule": "Schedule-Annexe",
52
+ "item": "Item-NumeroDarticle",
53
+ "date_of_listing": "DateOfListing-DateDinscription",
54
+ }
55
+
56
+
57
+ def _txt(rec, key):
58
+ el = rec.find(_T[key])
59
+ return (el.text or "").strip() if el is not None and el.text else None
60
+
61
+
62
+ def rows_from_xml(text):
63
+ """XML raw text → sanction-listing rows (pure function, offline-testable). Each record becomes
64
+ {kind, name, matched faces…}: kind = ship (IMO present) / entity (EntityOrShip) / person (GivenName×LastName);
65
+ name = the assembled display form ("Khazalbek Bakhtibekovich Atabekov" for persons, the EntityOrShip value
66
+ otherwise); aliases = semicolon-split; every other field verbatim bilingual."""
67
+ if not text:
68
+ return []
69
+ root = ET.fromstring(text)
70
+ rows = []
71
+ for rec in root.findall("record"):
72
+ imo = _txt(rec, "ship_imo")
73
+ entity = _txt(rec, "entity_or_ship")
74
+ given, last = _txt(rec, "given_name"), _txt(rec, "last_name")
75
+ if imo:
76
+ kind = "ship"
77
+ elif entity:
78
+ kind = "entity"
79
+ else:
80
+ kind = "person"
81
+ if kind == "person":
82
+ name = " ".join(x for x in (given, last) if x) or None
83
+ else:
84
+ name = entity
85
+ aliases = []
86
+ for el in rec.findall(_T["aliases"]):
87
+ aliases.extend(a.strip() for a in (el.text or "").split(";") if a.strip())
88
+ rows.append({
89
+ "kind": kind,
90
+ "name": name,
91
+ "aliases": aliases,
92
+ "country": _txt(rec, "country"), # bilingual verbatim ("Syria / Syrie")
93
+ "dob_or_ship_build": _txt(rec, "dob_or_ship_build"),
94
+ "title_or_ship_type": _txt(rec, "title_or_ship_type"), # person title OR ship type (bilingual)
95
+ "ship_imo": imo, # the join key onto shipping rosters (THETIS-MRV family)
96
+ "schedule": _txt(rec, "schedule"),
97
+ "item": _txt(rec, "item"),
98
+ "date_of_listing": _txt(rec, "date_of_listing"),
99
+ })
100
+ return rows
101
+
102
+
103
+ def filter_rows(rows, name=None, country=None):
104
+ """Local register scan (pure function). name: contains (case-insensitive) over name **and** aliases
105
+ (matched_on echoes the hit face); country: contains over the bilingual country cell ("Syria" and "Syrie"
106
+ both hit the "Syria / Syrie" row)."""
107
+ needle = str(name or "").strip().lower() or None
108
+ cn_needle = str(country or "").strip().lower() or None
109
+ hits = []
110
+ for r in rows:
111
+ if cn_needle and cn_needle not in str(r.get("country") or "").lower():
112
+ continue
113
+ if not needle:
114
+ hits.append({**r, "matched_on": None})
115
+ continue
116
+ if needle in str(r.get("name") or "").lower():
117
+ hits.append({**r, "matched_on": "name"})
118
+ continue
119
+ alias = next((a for a in r.get("aliases") or [] if needle in a.lower()), None)
120
+ if alias:
121
+ hits.append({**r, "matched_on": "alias"})
122
+ return hits
123
+
124
+
125
+ def sanctions_raw(name=None, *, country=None, limit=20, cache=None, no_cache=False):
126
+ """Full-register scan (pulls the whole XML, caches 24h, then scans locally)
127
+ → {rows, total_matched, scanned_rows, limit_applied, params, _meta}.
128
+
129
+ name = contains over name + aliases; country = contains over the bilingual country cell.
130
+ No filter → enumerate from the register head."""
131
+ from ..transport import call_provider
132
+
133
+ size = int(limit) if limit is not None else 20
134
+ if not 1 <= size <= 100:
135
+ raise SdkError("usage_error", provider="ca_sema",
136
+ detail=f"limit must be 1-100 (local payload wall), got '{limit}'")
137
+ r = call_provider(provider, "listings", {}, cache=cache, no_cache=no_cache,
138
+ extra_headers=_XML_ACCEPT)
139
+ rows = rows_from_xml(r["data"])
140
+ hits = filter_rows(rows, name=name, country=country)
141
+ return {"rows": hits[:size], "total_matched": len(hits), "scanned_rows": len(rows),
142
+ "limit_applied": size,
143
+ "params": {"name": name, "country": country}, "_meta": r}