agentdata 0.3.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentdata/__init__.py +33 -0
- agentdata/__main__.py +4 -0
- agentdata/cache.py +115 -0
- agentdata/cli.py +173 -0
- agentdata/client.py +158 -0
- agentdata/company/__init__.py +3 -0
- agentdata/company/cf.py +145 -0
- agentdata/company/ch_zefix.py +100 -0
- agentdata/company/cli.py +243 -0
- agentdata/company/co_rues.py +72 -0
- agentdata/company/edgar_cik.py +84 -0
- agentdata/company/edgar_fts.py +101 -0
- agentdata/company/faces.py +746 -0
- agentdata/company/fdic_bankfind.py +107 -0
- agentdata/company/fi_ytj.py +89 -0
- agentdata/company/fr_recherche.py +124 -0
- agentdata/company/fts.py +121 -0
- agentdata/company/gleif.py +178 -0
- agentdata/company/kr_hometax.py +112 -0
- agentdata/company/no_brreg.py +124 -0
- agentdata/company/ror.py +105 -0
- agentdata/company/ted.py +117 -0
- agentdata/company/usaspending.py +124 -0
- agentdata/company/vies.py +66 -0
- agentdata/daterange.py +35 -0
- agentdata/economy/__init__.py +10 -0
- agentdata/economy/bis.py +132 -0
- agentdata/economy/cli.py +199 -0
- agentdata/economy/ecb.py +117 -0
- agentdata/economy/eurostat.py +163 -0
- agentdata/economy/faces.py +573 -0
- agentdata/economy/fred.py +92 -0
- agentdata/economy/ilo_stat.py +99 -0
- agentdata/economy/imf_sdmx.py +133 -0
- agentdata/economy/un_comtrade.py +199 -0
- agentdata/economy/unsd_sdg.py +154 -0
- agentdata/economy/wb.py +208 -0
- agentdata/economy/who_xmart.py +177 -0
- agentdata/entries.py +30 -0
- agentdata/errors.py +38 -0
- agentdata/esg/__init__.py +5 -0
- agentdata/esg/cli.py +180 -0
- agentdata/esg/climatetrace.py +260 -0
- agentdata/esg/eiti.py +187 -0
- agentdata/esg/epa_frs.py +138 -0
- agentdata/esg/euasl.py +75 -0
- agentdata/esg/eutl.py +66 -0
- agentdata/esg/faces.py +498 -0
- agentdata/esg/paris_mou.py +240 -0
- agentdata/esg/thetis_mrv.py +67 -0
- agentdata/healthcare/__init__.py +7 -0
- agentdata/healthcare/au_tga.py +142 -0
- agentdata/healthcare/ca_dpd.py +123 -0
- agentdata/healthcare/ca_mdall.py +155 -0
- agentdata/healthcare/cli.py +375 -0
- agentdata/healthcare/ctgov.py +101 -0
- agentdata/healthcare/dailymed.py +102 -0
- agentdata/healthcare/epmc.py +108 -0
- agentdata/healthcare/eudamed.py +124 -0
- agentdata/healthcare/faces.py +811 -0
- agentdata/healthcare/fda_crl.py +86 -0
- agentdata/healthcare/fda_devices.py +199 -0
- agentdata/healthcare/fda_drugs.py +156 -0
- agentdata/healthcare/fda_recalls.py +137 -0
- agentdata/healthcare/gb_mhra.py +137 -0
- agentdata/healthcare/gudid.py +84 -0
- agentdata/healthcare/mesh.py +89 -0
- agentdata/healthcare/rasff.py +164 -0
- agentdata/healthcare/rxnav.py +67 -0
- agentdata/healthcare/safety_gate.py +175 -0
- agentdata/healthcare/shortages.py +107 -0
- agentdata/legal/__init__.py +4 -0
- agentdata/legal/afdb.py +93 -0
- agentdata/legal/ca_sema.py +143 -0
- agentdata/legal/cli.py +312 -0
- agentdata/legal/co_sic.py +70 -0
- agentdata/legal/courtlistener.py +77 -0
- agentdata/legal/csl.py +139 -0
- agentdata/legal/dgcomp.py +125 -0
- agentdata/legal/ecfr.py +129 -0
- agentdata/legal/faces.py +700 -0
- agentdata/legal/federalregister.py +188 -0
- agentdata/legal/gb_cma.py +117 -0
- agentdata/legal/gb_tna_caselaw.py +245 -0
- agentdata/legal/idb.py +85 -0
- agentdata/legal/lda.py +147 -0
- agentdata/legal/sipri.py +178 -0
- agentdata/legal/ukec.py +148 -0
- agentdata/legal/wb.py +92 -0
- agentdata/procurement/__init__.py +5 -0
- agentdata/procurement/br_pncp.py +126 -0
- agentdata/procurement/cli.py +155 -0
- agentdata/procurement/co_secop.py +88 -0
- agentdata/procurement/faces.py +239 -0
- agentdata/procurement/ny_open_data.py +82 -0
- agentdata/procurement/ungm.py +137 -0
- agentdata/procurement/wb_proc.py +84 -0
- agentdata/research/__init__.py +4 -0
- agentdata/research/biorxiv.py +66 -0
- agentdata/research/cli.py +129 -0
- agentdata/research/crossref.py +121 -0
- agentdata/research/faces.py +349 -0
- agentdata/research/gb_gtr.py +58 -0
- agentdata/research/icite.py +52 -0
- agentdata/research/nih_reporter.py +143 -0
- agentdata/research/nsf.py +68 -0
- agentdata/research/openalex.py +145 -0
- agentdata/research/opencitations.py +50 -0
- agentdata/security/__init__.py +3 -0
- agentdata/security/cisa_kev.py +42 -0
- agentdata/security/cli.py +37 -0
- agentdata/security/faces.py +91 -0
- agentdata/security/osv.py +72 -0
- agentdata/transport.py +541 -0
- agentdata-0.3.26.dist-info/METADATA +266 -0
- agentdata-0.3.26.dist-info/RECORD +120 -0
- agentdata-0.3.26.dist-info/WHEEL +5 -0
- agentdata-0.3.26.dist-info/entry_points.txt +10 -0
- agentdata-0.3.26.dist-info/licenses/LICENSE +201 -0
- agentdata-0.3.26.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""fred — US Federal Reserve Economic Data (fred.stlouisfed.org/graph/fredgraph.csv; keyless).
|
|
2
|
+
|
|
3
|
+
One face: series (one FRED series over a date window).
|
|
4
|
+
|
|
5
|
+
Channel decisions (verified 2026-11-02, tmp/probe-economy/):
|
|
6
|
+
* GET /fredgraph.csv?id={SERIES}[&cosd=&coed=] answers text/csv:
|
|
7
|
+
"observation_date,SERIES" then one row per observation (1947-01-01,21.480 for CPIAUCSL).
|
|
8
|
+
* SEVERAL series ids in one call answer application/zip (a ZIP with README.txt + one CSV
|
|
9
|
+
per series) — a binary payload, destroyed by the JSON-only transport (the 0.3.22
|
|
10
|
+
binary-wall ruling) -> one series per call, enforced locally.
|
|
11
|
+
* An unknown series id answers an HTML ERROR PAGE ("Error - St. Louis Fed", 29 KB) instead
|
|
12
|
+
of a 4xx -> the magic-number check is mandatory: the answer must start with
|
|
13
|
+
"observation_date", otherwise it is an upstream_not_found (an unknown series id), never
|
|
14
|
+
data. (A 2xx non-CSV answer is never passed through as if it were a series.)
|
|
15
|
+
* A missing observation is a lone "." in the CSV -> null (a gap is a fact).
|
|
16
|
+
* cosd / coed are the upstream observation-window bounds (verified: GDP 2020-01-01..2021-01-01).
|
|
17
|
+
Rate: FRED publishes no public limit for the CSV endpoint -> polite tier 1 rps / burst 2.
|
|
18
|
+
TTL: 12h (the cadence differs per series — daily, monthly, quarterly).
|
|
19
|
+
"""
|
|
20
|
+
import csv
|
|
21
|
+
import io
|
|
22
|
+
|
|
23
|
+
from ..errors import SdkError
|
|
24
|
+
|
|
25
|
+
provider = {
|
|
26
|
+
"id": "fred",
|
|
27
|
+
"baseUrl": "https://fred.stlouisfed.org/graph",
|
|
28
|
+
"rate": {"rps": 1, "burst": 2},
|
|
29
|
+
"ops": {
|
|
30
|
+
"series": {"path": "/fredgraph.csv", "accept": "text", "notFound": [404]},
|
|
31
|
+
},
|
|
32
|
+
"ttl": {"series": 12 * 3600},
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
_MAGIC = "observation_date"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _norm_series(series):
|
|
39
|
+
"""One series id per call (several ids answer a ZIP = binary wall)."""
|
|
40
|
+
s = str(series or "").strip().upper()
|
|
41
|
+
if not s:
|
|
42
|
+
raise SdkError("usage_error", provider="fred",
|
|
43
|
+
detail="fred_series requires --series (a FRED series id, e.g. GDP, CPIAUCSL, "
|
|
44
|
+
"DGS10) — several ids in one call answer a ZIP archive, which the "
|
|
45
|
+
"JSON transport cannot carry")
|
|
46
|
+
if "," in s or ";" in s:
|
|
47
|
+
raise SdkError("usage_error", provider="fred",
|
|
48
|
+
detail="fred_series takes ONE series id per call — the upstream answers a "
|
|
49
|
+
"ZIP archive (binary) when several ids are requested")
|
|
50
|
+
return s
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def series_raw(series, *, start=None, end=None, cache=None, no_cache=False):
|
|
54
|
+
"""-> {total, count, rows, params, _meta}; /fredgraph.csv?id={series}
|
|
55
|
+
series = the FRED series id; start / end = the upstream cosd / coed observation window;
|
|
56
|
+
the rows are long-form {date, series, value} and a missing observation is null."""
|
|
57
|
+
from ..transport import call_provider
|
|
58
|
+
|
|
59
|
+
sid = _norm_series(series)
|
|
60
|
+
params = {"id": sid}
|
|
61
|
+
s = str(start).strip() if start else None
|
|
62
|
+
e = str(end).strip() if end else None
|
|
63
|
+
if s:
|
|
64
|
+
params["cosd"] = s
|
|
65
|
+
if e:
|
|
66
|
+
params["coed"] = e
|
|
67
|
+
r = call_provider(provider, "series", params, cache=cache, no_cache=no_cache)
|
|
68
|
+
text = r["data"]
|
|
69
|
+
if not isinstance(text, str) or not text.lstrip().startswith(_MAGIC):
|
|
70
|
+
# the upstream answers an HTML error page for an unknown series id
|
|
71
|
+
raise SdkError("upstream_not_found", provider="fred",
|
|
72
|
+
detail=f"FRED answered a non-CSV page for series '{sid}' (the upstream "
|
|
73
|
+
"returns an HTML error page for an unknown series id)")
|
|
74
|
+
recs = list(csv.DictReader(io.StringIO(text)))
|
|
75
|
+
rows = []
|
|
76
|
+
for rec in recs:
|
|
77
|
+
date = rec.get("observation_date") or rec.get("DATE")
|
|
78
|
+
for col, val in rec.items():
|
|
79
|
+
if col in (None, "observation_date", "DATE"):
|
|
80
|
+
continue
|
|
81
|
+
v = (val or "").strip()
|
|
82
|
+
if v in (".", "", "NA", "NaN"):
|
|
83
|
+
v = None
|
|
84
|
+
else:
|
|
85
|
+
try:
|
|
86
|
+
v = float(v)
|
|
87
|
+
except ValueError:
|
|
88
|
+
pass
|
|
89
|
+
rows.append({"date": date, "series": col, "value": v})
|
|
90
|
+
rows.sort(key=lambda x: ((x.get("date") or ""), (x.get("series") or "")))
|
|
91
|
+
return {"total": len(rows), "count": len(rows), "rows": rows,
|
|
92
|
+
"params": {"series": sid, "start": s, "end": e}, "_meta": r}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""ilo_stat — ILOSTAT bulk API via the rplumber endpoint (rplumber.ilo.org; keyless).
|
|
2
|
+
|
|
3
|
+
One face: indicator (one indicator x one country x a year window).
|
|
4
|
+
|
|
5
|
+
Shapes verified by first-hand live probing 2026-11-02 (tmp/probe-economy/):
|
|
6
|
+
GET /data/indicator/?id=EMP_TEMP_SEX_ECO_NB_A&ref_area=USA&timefrom=2020&format=json
|
|
7
|
+
-> [ {ref_area, source, indicator, sex, classif1, time, obs_value, obs_status,
|
|
8
|
+
note_indicator, note_source}, ... ] (612 rows for USA 2020-2025: 6 years x 3 sexes x
|
|
9
|
+
34 classif1 buckets — the economic-sector breakdown of employment)
|
|
10
|
+
Traps found by probing:
|
|
11
|
+
* the format flag is the QUERY parameter `format=json` — the path suffix ".jsondata" is a
|
|
12
|
+
400 ("Unsupported format") and ".json" as a path suffix is a 404 ("wrong query");
|
|
13
|
+
* the trailing slash of /data/indicator/ is part of the route;
|
|
14
|
+
* the CLIENT STACK decides: the curl subprocess returns an EMPTY 200 (0 bytes) and a
|
|
15
|
+
schannel TLS handshake failure on this host, while the urllib channel (the SDK default)
|
|
16
|
+
answers 612 rows / 118,127 bytes every time — the mirrored twin of the AU ABS lesson
|
|
17
|
+
(curl-schannel family). This provider therefore keeps the default urllib channel and
|
|
18
|
+
must NOT be pinned to transport='curl';
|
|
19
|
+
* the upper year bound is spelled `timeto` upstream — `timeuntil` is accepted and SILENTLY
|
|
20
|
+
IGNORED, so the answer runs from `timefrom` to the newest year (verified 2026-11-02:
|
|
21
|
+
timefrom=2020&timeuntil=2022 -> 612 rows 2020-2025, the same set as no upper bound;
|
|
22
|
+
timefrom=2020&timeto=2022 -> 306 rows 2020-2022). A 200 that looks healthy and drops
|
|
23
|
+
the filter — the "silent clamp" family, caught by the row-count cross-check;
|
|
24
|
+
* the answer carries Content-Type: application/octet-stream while the body IS JSON — the
|
|
25
|
+
body is parsed, not the content type.
|
|
26
|
+
Rate: the public rplumber endpoint publishes no limit -> polite tier 0.5 rps / burst 2.
|
|
27
|
+
TTL: 24h (ILOSTAT refreshes monthly/quarterly per series).
|
|
28
|
+
"""
|
|
29
|
+
from ..errors import SdkError
|
|
30
|
+
|
|
31
|
+
provider = {
|
|
32
|
+
"id": "ilo_stat",
|
|
33
|
+
"baseUrl": "https://rplumber.ilo.org/data",
|
|
34
|
+
"rate": {"rps": 0.5, "burst": 2},
|
|
35
|
+
"ops": {
|
|
36
|
+
"indicator": {"path": "/indicator/", "notFound": [404]},
|
|
37
|
+
},
|
|
38
|
+
"ttl": {"indicator": 24 * 3600},
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def indicator_raw(indicator, *, ref_area=None, time_from=None, time_until=None, sex=None,
|
|
43
|
+
classif=None, source=None, cache=None, no_cache=False):
|
|
44
|
+
"""-> {total, count, rows, params, _meta}; /data/indicator/
|
|
45
|
+
indicator = the ILOSTAT indicator code (e.g. EMP_TEMP_SEX_ECO_NB_A); ref_area = the
|
|
46
|
+
country (ISO-3, e.g. USA); time_from / time_until = the year bounds (the upstream
|
|
47
|
+
timefrom / timeto — `timeuntil` is ignored upstream, see the module docstring);
|
|
48
|
+
sex / classif / source = the upstream dimension filters.
|
|
49
|
+
Local wall: --ref-area is required — the world-wide answer of one indicator is orders of
|
|
50
|
+
magnitude bigger than the country answer."""
|
|
51
|
+
from ..transport import call_provider
|
|
52
|
+
|
|
53
|
+
ind = str(indicator or "").strip().upper()
|
|
54
|
+
if not ind:
|
|
55
|
+
raise SdkError("usage_error", provider="ilo_stat",
|
|
56
|
+
detail="ilo_stat_indicator requires --indicator (the ILOSTAT code, "
|
|
57
|
+
"e.g. EMP_TEMP_SEX_ECO_NB_A)")
|
|
58
|
+
area = str(ref_area or "").strip().upper()
|
|
59
|
+
if not area:
|
|
60
|
+
raise SdkError("usage_error", provider="ilo_stat",
|
|
61
|
+
detail="ilo_stat_indicator requires --ref-area (the ISO-3 country code, "
|
|
62
|
+
"e.g. USA) — the world-wide answer of one indicator is orders of "
|
|
63
|
+
"magnitude bigger than the country answer")
|
|
64
|
+
params = {"id": ind, "ref_area": area, "format": "json"}
|
|
65
|
+
if time_from:
|
|
66
|
+
params["timefrom"] = str(time_from).strip()
|
|
67
|
+
if time_until:
|
|
68
|
+
params["timeto"] = str(time_until).strip() # NOT `timeuntil` — silently ignored upstream
|
|
69
|
+
if sex:
|
|
70
|
+
params["sex"] = str(sex).strip().upper()
|
|
71
|
+
if classif:
|
|
72
|
+
params["classif1"] = str(classif).strip().upper()
|
|
73
|
+
if source:
|
|
74
|
+
params["source"] = str(source).strip()
|
|
75
|
+
r = call_provider(provider, "indicator", params, cache=cache, no_cache=no_cache)
|
|
76
|
+
data = r["data"]
|
|
77
|
+
if not isinstance(data, list):
|
|
78
|
+
raise SdkError("upstream_unavailable", provider="ilo_stat",
|
|
79
|
+
detail=f"Unexpected ILOSTAT payload shape: {str(data)[:160]!r}")
|
|
80
|
+
rows = [{
|
|
81
|
+
"ref_area": x.get("ref_area"),
|
|
82
|
+
"indicator": x.get("indicator"),
|
|
83
|
+
"source": x.get("source"),
|
|
84
|
+
"sex": x.get("sex"),
|
|
85
|
+
"classif1": x.get("classif1"),
|
|
86
|
+
"time": x.get("time"),
|
|
87
|
+
"obs_value": x.get("obs_value"),
|
|
88
|
+
"obs_status": x.get("obs_status"),
|
|
89
|
+
"note_indicator": x.get("note_indicator"),
|
|
90
|
+
"note_source": x.get("note_source"),
|
|
91
|
+
} for x in data]
|
|
92
|
+
# Newest year first, deterministically (the upstream order is not pinned): the reason
|
|
93
|
+
# line names rows[0] as the latest observation, so the order has to be ours.
|
|
94
|
+
rows.sort(key=lambda x: (x.get("time") or ""), reverse=True)
|
|
95
|
+
return {"total": len(rows), "count": len(rows), "rows": rows,
|
|
96
|
+
"params": {"indicator": ind, "ref_area": area, "time_from": time_from,
|
|
97
|
+
"time_until": time_until, "sex": sex, "classif1": classif,
|
|
98
|
+
"source": source},
|
|
99
|
+
"_meta": r}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""imf_sdmx — IMF SDMX 2.1 data service (api.imf.org/external/sdmx/2.1; keyless).
|
|
2
|
+
|
|
3
|
+
One face: sdmx_series — one dataflow × one country × one indicator over an optional period
|
|
4
|
+
window (the trailing key dimensions — FREQUENCY and beyond — are omittable upstream; the face
|
|
5
|
+
keeps the optional trailing FREQUENCY and nothing else).
|
|
6
|
+
|
|
7
|
+
Channel decision (the ecb csvdata idiom, 0.3.23): the SDMX 2.1 endpoint answers the Accept
|
|
8
|
+
header FIRST — with the transport default `Accept: application/json` the answer comes back as
|
|
9
|
+
SDMX-XML (mangled by the JSON-only transport); asking explicitly for
|
|
10
|
+
`application/vnd.sdmx.data+csv;version=1.0.0` yields the SDMX 2.1 generic CSV (DATAFLOW, one
|
|
11
|
+
column per key dimension, TIME_PERIOD, OBS_VALUE, then attribute columns — verified
|
|
12
|
+
2026-11-02: WEO/USA.NGDP_RPCH -> 200, 111,674 B, 52 data rows 1980-2031 at full precision).
|
|
13
|
+
|
|
14
|
+
Walls and traps (research chapter 87, 2026-11-02):
|
|
15
|
+
* the country-only key (ONE key segment) is the multi-megabyte wall — the whole dataset for
|
|
16
|
+
one country. The face requires BOTH --country and --indicator, so a one-segment key is
|
|
17
|
+
never formed here.
|
|
18
|
+
* the DSD codelists come back partial/empty (the indicator dictionary is not fetchable that
|
|
19
|
+
way) — the CSV columns ARE the dimension list, and they vary per flow: WEO/GDD spell them
|
|
20
|
+
COUNTRY.INDICATOR.FREQUENCY; UNFCCC inserts INDUSTRY/GAS_TYPE before the indicator, so the
|
|
21
|
+
country.indicator key shape fits the WEO/GDD/CPI family and NOT UNFCCC (documented limit,
|
|
22
|
+
wrong-order keys upstream 404).
|
|
23
|
+
* SCALE is a display hint — OBS_VALUE already carries the true value (DataMapper's rounded
|
|
24
|
+
1-decimal numbers are the reason that surface is a background board, not a source).
|
|
25
|
+
|
|
26
|
+
Rate: no published limit -> polite 0.5 rps / burst 2. TTL 24h (the WEO family refreshes on its
|
|
27
|
+
own vintage cadence; 101 of the 229 dataflows are standing, the rest are monthly vintages).
|
|
28
|
+
"""
|
|
29
|
+
import csv
|
|
30
|
+
import io
|
|
31
|
+
import re
|
|
32
|
+
|
|
33
|
+
from ..errors import SdkError
|
|
34
|
+
|
|
35
|
+
provider = {
|
|
36
|
+
"id": "imf_sdmx",
|
|
37
|
+
"baseUrl": "https://api.imf.org/external/sdmx/2.1",
|
|
38
|
+
"rate": {"rps": 0.5, "burst": 2},
|
|
39
|
+
"ops": {
|
|
40
|
+
"sdmx_series": {"path": "/data/{flow}/{key}", "accept": "text", "notFound": [404]},
|
|
41
|
+
},
|
|
42
|
+
"ttl": {"sdmx_series": 24 * 3600},
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
_TOKEN_RE = re.compile(r"^[A-Z0-9_]+$")
|
|
46
|
+
|
|
47
|
+
# The SDMX 2.1 generic CSV: ask for the CSV flavour explicitly (see the module docstring —
|
|
48
|
+
# the JSON Accept is ignored upstream and the answer degrades to XML).
|
|
49
|
+
_CSV_ACCEPT = {"Accept": "application/vnd.sdmx.data+csv;version=1.0.0"}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _norm_token(value, what, provider_id="imf_sdmx"):
|
|
53
|
+
"""The key parts are upper-case tokens (USA, NGDP_RPCH, WEO, A)."""
|
|
54
|
+
t = str(value or "").strip().upper()
|
|
55
|
+
if not t or not _TOKEN_RE.match(t):
|
|
56
|
+
raise SdkError("usage_error", provider=provider_id,
|
|
57
|
+
detail=f"{what} must be an upper-case code token (letters/digits/'_'), "
|
|
58
|
+
f"got {value!r}")
|
|
59
|
+
return t
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _dim_names(head):
|
|
63
|
+
"""The key-dimension columns of the SDMX 2.1 generic CSV: everything between DATAFLOW and
|
|
64
|
+
TIME_PERIOD (they vary per flow — WEO/GDD say COUNTRY,INDICATOR,FREQUENCY; UNFCCC inserts
|
|
65
|
+
INDUSTRY/GAS_TYPE — so the boundary, not a fixed list, is the reliable rule)."""
|
|
66
|
+
names = []
|
|
67
|
+
for col in head:
|
|
68
|
+
if col in ("DATAFLOW",):
|
|
69
|
+
continue
|
|
70
|
+
if col == "TIME_PERIOD":
|
|
71
|
+
break
|
|
72
|
+
names.append(col)
|
|
73
|
+
return names
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _parse_csv(text):
|
|
77
|
+
lines = [ln for ln in (text or "").splitlines() if ln.strip()]
|
|
78
|
+
if not lines:
|
|
79
|
+
raise SdkError("upstream_unavailable", provider="imf_sdmx",
|
|
80
|
+
detail="The IMF SDMX CSV answer was empty")
|
|
81
|
+
head = lines[0].split(",")
|
|
82
|
+
if "TIME_PERIOD" not in head or "OBS_VALUE" not in head:
|
|
83
|
+
# not the generic CSV (an SDMX XML error page, a gateway notice)
|
|
84
|
+
raise SdkError("upstream_unavailable", provider="imf_sdmx",
|
|
85
|
+
detail=f"The IMF SDMX answer is not the generic CSV table "
|
|
86
|
+
f"(head {lines[0][:120]!r})")
|
|
87
|
+
dims = _dim_names(head)
|
|
88
|
+
rows = []
|
|
89
|
+
for rec in csv.DictReader(io.StringIO("\n".join(lines))):
|
|
90
|
+
try:
|
|
91
|
+
value = float(rec.get("OBS_VALUE"))
|
|
92
|
+
except (TypeError, ValueError):
|
|
93
|
+
value = None
|
|
94
|
+
out = {"flow": (rec.get("DATAFLOW") or "").split(":")[-1].split("(")[0] or None,
|
|
95
|
+
"dims": {d: rec.get(d) for d in dims if rec.get(d) not in (None, "")},
|
|
96
|
+
"period": rec.get("TIME_PERIOD"),
|
|
97
|
+
"value": value,
|
|
98
|
+
"scale": rec.get("SCALE") or None,
|
|
99
|
+
"series_name": rec.get("SERIES_NAME") or None,
|
|
100
|
+
"description": rec.get("FULL_DESCRIPTION") or None,
|
|
101
|
+
"doi": rec.get("DOI") or None}
|
|
102
|
+
rows.append(out)
|
|
103
|
+
return rows
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def sdmx_series_raw(flow, country, indicator, frequency=None, *, start=None, end=None,
|
|
107
|
+
cache=None, no_cache=False):
|
|
108
|
+
"""-> {total, count, rows, params, _meta}; /data/{flow}/{COUNTRY.INDICATOR[.FREQUENCY]}
|
|
109
|
+
flow = the dataflow id (WEO, GDD, CPI …); country + indicator are BOTH required — the
|
|
110
|
+
country-only key is the multi-megabyte wall; frequency = the optional trailing key
|
|
111
|
+
dimension; start / end = the upstream startPeriod / endPeriod bounds."""
|
|
112
|
+
from ..transport import call_provider
|
|
113
|
+
|
|
114
|
+
fl = _norm_token(flow, "--flow")
|
|
115
|
+
cc = _norm_token(country, "--country")
|
|
116
|
+
ind = _norm_token(indicator, "--indicator")
|
|
117
|
+
freq = _norm_token(frequency, "--frequency") if frequency else None
|
|
118
|
+
key = ".".join([cc, ind] + ([freq] if freq else []))
|
|
119
|
+
s = str(start).strip() if start else None
|
|
120
|
+
e = str(end).strip() if end else None
|
|
121
|
+
params = {"flow": fl, "key": key}
|
|
122
|
+
if s:
|
|
123
|
+
params["startPeriod"] = s
|
|
124
|
+
if e:
|
|
125
|
+
params["endPeriod"] = e
|
|
126
|
+
r = call_provider(provider, "sdmx_series", params, cache=cache, no_cache=no_cache,
|
|
127
|
+
extra_headers=_CSV_ACCEPT)
|
|
128
|
+
rows = _parse_csv(r["data"])
|
|
129
|
+
rows.sort(key=lambda x: (x.get("period") or ""))
|
|
130
|
+
return {"total": len(rows), "count": len(rows), "rows": rows, "key": key,
|
|
131
|
+
"params": {"flow": fl, "country": cc, "indicator": ind, "frequency": freq,
|
|
132
|
+
"start": s, "end": e},
|
|
133
|
+
"_meta": r}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""un_comtrade — UN Comtrade public preview API (comtradeapi.un.org; keyless, no registration).
|
|
2
|
+
|
|
3
|
+
One face: trade (one reporter x partner x period x commodity trade flow).
|
|
4
|
+
|
|
5
|
+
Channels (verified 2026-11-02, tmp/probe-economy/):
|
|
6
|
+
* data: GET /public/v1/preview/{type}/{freq}/{cl}?reporterCode=&period=&partnerCode=&cmdCode=
|
|
7
|
+
&flowCode=&maxRecords=&includeDesc=
|
|
8
|
+
-> {"elapsedTime":"0.95 secs","count":1,"data":[{typeCode, freqCode, refPeriodId, period,
|
|
9
|
+
reporterCode, reporterISO, reporterDesc, flowCode, flowDesc, partnerCode, partnerISO,
|
|
10
|
+
partnerDesc, partner2Code/ISO/Desc, classificationCode, cmdCode, cmdDesc, customsCode,
|
|
11
|
+
mosCode, motCode, qtyUnitCode, qty, netWgt, grossWgt, cifvalue, fobvalue, primaryValue,
|
|
12
|
+
isAggregate}], "error":""}
|
|
13
|
+
The preview answer is capped at 500 records; the full subscription API is keyed (out of scope).
|
|
14
|
+
* areas: GET /files/v1/app/reference/partnerAreas.json -> 310 area rows {id, text,
|
|
15
|
+
PartnerCode, PartnerDesc, PartnerCodeIsoAlpha2, PartnerCodeIsoAlpha3, isGroup}.
|
|
16
|
+
|
|
17
|
+
The area-code trap (the reference is NOT a clean ISO3 map):
|
|
18
|
+
* 16 ISO3 codes carry SEVERAL rows — the historical ones carry a "(...YYYY)" suffix in the
|
|
19
|
+
text (Germany: "Fed. Rep. of Germany (...1990)" = 280 vs the current "Germany" = 276);
|
|
20
|
+
* the USA carries THREE rows: 840 "United States of America", 842 "USA", 841 "USA and Puerto
|
|
21
|
+
Rico (...1980)" — and the live preview answers reporterCode=842 for the USA (840 is the
|
|
22
|
+
ISO 3166 numeric, 842 is the Comtrade trade code). So a naive "ISO3 -> code" dict built by
|
|
23
|
+
insertion order picks the WRONG code (841) — the resolution rule below drops the historical
|
|
24
|
+
rows and then prefers the shortest current name, which reproduces every current code
|
|
25
|
+
(USA 842, DEU 276, FRA 251, IND 699, PAK 586, VNM 704, NOR 579 …);
|
|
26
|
+
* reporterCode=USA (the ISO3 string) is a 400 "The field reporterCode is invalid" — the
|
|
27
|
+
numeric code is mandatory, hence the translation step;
|
|
28
|
+
* partner 0 = the World aggregate (partnerISO "W00").
|
|
29
|
+
Rate: the preview answers with a rate-limit header in the paid API; the public preview is
|
|
30
|
+
polite-tier 0.5 rps / burst 2. TTL: 12h (the preview data is refreshed hourly).
|
|
31
|
+
"""
|
|
32
|
+
from ..errors import SdkError
|
|
33
|
+
|
|
34
|
+
provider = {
|
|
35
|
+
"id": "un_comtrade",
|
|
36
|
+
"baseUrl": "https://comtradeapi.un.org",
|
|
37
|
+
"rate": {"rps": 0.5, "burst": 2},
|
|
38
|
+
"ops": {
|
|
39
|
+
"trade": {"path": "/public/v1/preview/{type_code}/{freq_code}/{cl_code}", "notFound": [404]},
|
|
40
|
+
"areas": {"path": "/files/v1/app/reference/partnerAreas.json", "notFound": [404]},
|
|
41
|
+
},
|
|
42
|
+
"ttl": {"trade": 12 * 3600, "areas": 7 * 86400},
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
_PREVIEW_CAP = 500 # the public preview answers at most 500 records per call
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _norm_code(value, what):
|
|
49
|
+
"""A Comtrade area code: digits pass through, an ISO3 code is translated."""
|
|
50
|
+
v = str(value or "").strip()
|
|
51
|
+
if not v:
|
|
52
|
+
return None
|
|
53
|
+
if v.isdigit():
|
|
54
|
+
return int(v)
|
|
55
|
+
return v.upper()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _pick_area(rows, iso3):
|
|
59
|
+
"""The Comtrade code of one ISO3 area, resolved against the reference rows.
|
|
60
|
+
Rule (verified against the live preview): drop the historical rows (their text carries a
|
|
61
|
+
"(...YYYY)" suffix), then prefer the shortest current name — USA -> 842 (not 840, not 841),
|
|
62
|
+
DEU -> 276, FRA -> 251, IND -> 699, PAK -> 586."""
|
|
63
|
+
code = str(iso3 or "").strip().upper()
|
|
64
|
+
if not code:
|
|
65
|
+
return None
|
|
66
|
+
cands = [r for r in rows if (r.get("PartnerCodeIsoAlpha3") or "").strip().upper() == code]
|
|
67
|
+
if not cands:
|
|
68
|
+
return None
|
|
69
|
+
live = [r for r in cands if "(..." not in (r.get("text") or "")]
|
|
70
|
+
pool = live or cands
|
|
71
|
+
best = sorted(pool, key=lambda r: (len((r.get("text") or "").strip()), r.get("PartnerCode") or 0))[0]
|
|
72
|
+
return best.get("PartnerCode")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _areas(cache=None, no_cache=False):
|
|
76
|
+
"""-> (rows, meta): the area reference table (310 rows)."""
|
|
77
|
+
from ..transport import call_provider
|
|
78
|
+
r = call_provider(provider, "areas", {}, cache=cache, no_cache=no_cache)
|
|
79
|
+
data = r["data"]
|
|
80
|
+
rows = data.get("results") if isinstance(data, dict) else None
|
|
81
|
+
if rows is None:
|
|
82
|
+
rows = data if isinstance(data, list) else []
|
|
83
|
+
return rows, r
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _resolve_area(value, what, *, cache=None, no_cache=False):
|
|
87
|
+
"""An area argument -> the numeric Comtrade code (the ISO3 is translated)."""
|
|
88
|
+
v = str(value or "").strip()
|
|
89
|
+
if not v:
|
|
90
|
+
raise SdkError("usage_error", provider="un_comtrade",
|
|
91
|
+
detail=f"un_comtrade_trade requires {what}")
|
|
92
|
+
if v.isdigit():
|
|
93
|
+
return int(v)
|
|
94
|
+
up = v.upper()
|
|
95
|
+
if up in ("W00", "WORLD", "WLD"):
|
|
96
|
+
return 0
|
|
97
|
+
rows, meta = _areas(cache=cache, no_cache=no_cache)
|
|
98
|
+
code = _pick_area(rows, up)
|
|
99
|
+
if code is None:
|
|
100
|
+
raise SdkError("upstream_not_found", provider="un_comtrade",
|
|
101
|
+
detail=f"No Comtrade area code for '{v}' (the area reference has no such "
|
|
102
|
+
"ISO3 code — check the spelling)")
|
|
103
|
+
return int(code)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _row(rec):
|
|
107
|
+
return {
|
|
108
|
+
"type_code": rec.get("typeCode"),
|
|
109
|
+
"freq_code": rec.get("freqCode"),
|
|
110
|
+
"period": rec.get("period"),
|
|
111
|
+
"ref_period_id": rec.get("refPeriodId"),
|
|
112
|
+
"ref_year": rec.get("refYear"),
|
|
113
|
+
"reporter_code": rec.get("reporterCode"),
|
|
114
|
+
"reporter_iso3": rec.get("reporterISO"),
|
|
115
|
+
"reporter_name": rec.get("reporterDesc"),
|
|
116
|
+
"flow_code": rec.get("flowCode"),
|
|
117
|
+
"flow_name": rec.get("flowDesc"),
|
|
118
|
+
"partner_code": rec.get("partnerCode"),
|
|
119
|
+
"partner_iso3": rec.get("partnerISO"),
|
|
120
|
+
"partner_name": rec.get("partnerDesc"),
|
|
121
|
+
"partner2_code": rec.get("partner2Code"),
|
|
122
|
+
"partner2_iso3": rec.get("partner2ISO"),
|
|
123
|
+
"classification_code": rec.get("classificationCode"),
|
|
124
|
+
"cmd_code": rec.get("cmdCode"),
|
|
125
|
+
"cmd_name": rec.get("cmdDesc"),
|
|
126
|
+
"customs_code": rec.get("customsCode"),
|
|
127
|
+
"customs_name": rec.get("customsDesc"),
|
|
128
|
+
"mot_code": rec.get("motCode"),
|
|
129
|
+
"mot_name": rec.get("motDesc"),
|
|
130
|
+
"qty_unit_code": rec.get("qtyUnitCode"),
|
|
131
|
+
"qty_unit": rec.get("qtyUnitAbbr"),
|
|
132
|
+
"qty": rec.get("qty"),
|
|
133
|
+
"net_weight": rec.get("netWgt"),
|
|
134
|
+
"gross_weight": rec.get("grossWgt"),
|
|
135
|
+
"cif_value": rec.get("cifvalue"),
|
|
136
|
+
"fob_value": rec.get("fobvalue"),
|
|
137
|
+
"primary_value": rec.get("primaryValue"),
|
|
138
|
+
"is_aggregate": rec.get("isAggregate"),
|
|
139
|
+
"is_reported": rec.get("isReported"),
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def trade_raw(reporter, *, period=None, partner=None, cmd=None, flow=None, type_code="C",
|
|
144
|
+
freq_code="A", cl_code="HS", max_records=500, partner2=None, customs=None,
|
|
145
|
+
mot=None, cache=None, no_cache=False):
|
|
146
|
+
"""-> {total, count, rows, params, _meta}; /public/v1/preview/{type}/{freq}/{cl}
|
|
147
|
+
reporter = the reporting country (ISO3 or the numeric code — the ISO3 is translated through
|
|
148
|
+
the area reference); period = the upstream period (2023 / 202301 / a comma list);
|
|
149
|
+
partner = the partner area (ISO3 or code; 0 / "W00" = the World aggregate);
|
|
150
|
+
cmd = the commodity code (a comma list; TOTAL = all goods); flow = the trade flow
|
|
151
|
+
(X export / M import / "X,M"); type_code / freq_code / cl_code = the path segments
|
|
152
|
+
(C commodities / A annual / HS classification); max_records 1-500 (the preview cap)."""
|
|
153
|
+
from ..transport import call_provider
|
|
154
|
+
|
|
155
|
+
rep = _resolve_area(reporter, "--reporter (the reporting country)", cache=cache, no_cache=no_cache)
|
|
156
|
+
per = str(period or "").strip()
|
|
157
|
+
if not per:
|
|
158
|
+
raise SdkError("usage_error", provider="un_comtrade",
|
|
159
|
+
detail="un_comtrade_trade requires --period (e.g. 2023 or 202301)")
|
|
160
|
+
par = _resolve_area(partner if partner not in (None, "") else "W00",
|
|
161
|
+
"--partner (the partner area; '0' = the World aggregate)",
|
|
162
|
+
cache=cache, no_cache=no_cache)
|
|
163
|
+
mr = int(max_records or 500)
|
|
164
|
+
if not 1 <= mr <= _PREVIEW_CAP:
|
|
165
|
+
raise SdkError("usage_error", provider="un_comtrade",
|
|
166
|
+
detail=f"maxRecords must be 1-{_PREVIEW_CAP} (the public preview cap)")
|
|
167
|
+
params = {
|
|
168
|
+
"type_code": str(type_code or "C").strip().upper(),
|
|
169
|
+
"freq_code": str(freq_code or "A").strip().upper(),
|
|
170
|
+
"cl_code": str(cl_code or "HS").strip().upper(),
|
|
171
|
+
"reporterCode": rep,
|
|
172
|
+
"period": per,
|
|
173
|
+
"partnerCode": par,
|
|
174
|
+
"cmdCode": str(cmd or "TOTAL").strip(),
|
|
175
|
+
"maxRecords": mr,
|
|
176
|
+
"includeDesc": "true",
|
|
177
|
+
}
|
|
178
|
+
if flow:
|
|
179
|
+
params["flowCode"] = str(flow).strip().upper()
|
|
180
|
+
if partner2 not in (None, ""):
|
|
181
|
+
params["partner2Code"] = _resolve_area(partner2, "--partner2", cache=cache, no_cache=no_cache)
|
|
182
|
+
if customs not in (None, ""):
|
|
183
|
+
params["customsCode"] = str(customs).strip()
|
|
184
|
+
if mot not in (None, ""):
|
|
185
|
+
params["motCode"] = str(mot).strip()
|
|
186
|
+
r = call_provider(provider, "trade", params, cache=cache, no_cache=no_cache)
|
|
187
|
+
data = r["data"]
|
|
188
|
+
if not isinstance(data, dict):
|
|
189
|
+
raise SdkError("upstream_unavailable", provider="un_comtrade",
|
|
190
|
+
detail=f"Unexpected Comtrade payload shape: {str(data)[:160]!r}")
|
|
191
|
+
err = (data.get("error") or "").strip()
|
|
192
|
+
if err:
|
|
193
|
+
raise SdkError("upstream_unavailable", provider="un_comtrade", detail=f"Comtrade: {err}")
|
|
194
|
+
recs = data.get("data") or []
|
|
195
|
+
rows = [_row(x) for x in recs]
|
|
196
|
+
return {"total": data.get("count"), "count": len(rows), "rows": rows,
|
|
197
|
+
"elapsed": data.get("elapsedTime"),
|
|
198
|
+
"params": {k: v for k, v in params.items() if k not in ("type_code", "freq_code", "cl_code")},
|
|
199
|
+
"_meta": r}
|