bedrok 0.3.2__tar.gz → 0.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bedrok-0.3.2 → bedrok-0.3.4}/PKG-INFO +1 -1
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/aliases.py +2 -1
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/config.py +5 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/geography.py +10 -1
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/releases.py +44 -2
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/units.py +4 -1
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/warnings.py +17 -2
- bedrok-0.3.4/bedrok/ingest/adapters/bns.py +128 -0
- bedrok-0.3.4/bedrok/ingest/adapters/jodi.py +103 -0
- bedrok-0.3.4/bedrok/ingest/adapters/nbk.py +194 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/sdmx.py +17 -0
- bedrok-0.3.4/bedrok/ingest/calendars.py +362 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/pipeline.py +26 -10
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ops/docs.py +6 -1
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ops/doctor.py +3 -1
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ops/freshness.py +7 -8
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/seeds/macro_catalog.csv +36 -1
- bedrok-0.3.4/bedrok/seeds/release_calendar.csv +46 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/seeds/release_rules.csv +37 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/seeds/sources.csv +3 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/seeds/warnings.csv +2 -2
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/surfaces/mcp_server.py +2 -1
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok.egg-info/PKG-INFO +1 -1
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok.egg-info/SOURCES.txt +6 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/pyproject.toml +1 -1
- bedrok-0.3.4/tests/test_bns_adapter.py +89 -0
- bedrok-0.3.4/tests/test_calendars.py +437 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_fx_series.py +16 -1
- bedrok-0.3.4/tests/test_jodi_adapter.py +73 -0
- bedrok-0.3.4/tests/test_nbk_adapter.py +146 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_profile.py +1 -1
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_releases.py +19 -1
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_sdmx_adapter.py +15 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_warnings.py +10 -0
- bedrok-0.3.2/bedrok/ingest/calendars.py +0 -125
- bedrok-0.3.2/bedrok/seeds/release_calendar.csv +0 -9
- bedrok-0.3.2/tests/test_calendars.py +0 -165
- {bedrok-0.3.2 → bedrok-0.3.4}/LICENSE +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/README.md +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/__init__.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/__main__.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/__init__.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/charting.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/dates.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/licensing.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/query.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/schema.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/sources.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/core/store.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/engines/__init__.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/engines/pctile.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/engines/rates_infl.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/__init__.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/__init__.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/_util.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/bea.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/bls.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/boc.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/boe.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/census.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/cftc.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/destatis.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/eia.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/estat.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/eurostat.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/fedboard.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/fedfiles.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/fred.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/ine.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/mof_jp.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/nber.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/nyfed.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/ons.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/pxweb.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/statcan.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/adapters/worldbank.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/backfill.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/derived.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/live.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/refresh.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ingest/transforms.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ops/__init__.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ops/smoke.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ops/tieout.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/ops/validate.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/seeds/bulk_catalog.csv +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/seeds/charting.md +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/seeds/gaps.md +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok/surfaces/__init__.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok.egg-info/dependency_links.txt +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok.egg-info/entry_points.txt +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok.egg-info/requires.txt +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/bedrok.egg-info/top_level.txt +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/setup.cfg +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_boe_adapter.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_cache_rule.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_cftc_adapter.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_charting.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_dates.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_derived.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_doctor.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_eia_adapter.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_eurostat_bulk.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_fedboard_adapter.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_fedfiles_adapter.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_freshness.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_geography.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_install.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_layering.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_licensing.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_macro_seed.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_mcp_server.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_mof_jp_adapter.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_ons_adapter.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_pctile.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_publication.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_refresh.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_repoint.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_restricted_sync.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_second_wave_adapters.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_smoke.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_sources.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_store.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_tieout.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_transforms.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_us_agency_adapters.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_validate.py +0 -0
- {bedrok-0.3.2 → bedrok-0.3.4}/tests/test_worldbank_adapter.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bedrok
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.4
|
|
4
4
|
Summary: Public macroeconomic data from central banks, statistics offices and international organisations, in your AI assistant: a local MCP server and store. No keys needed to start. Not financial advice.
|
|
5
5
|
Author: Andreu Paddack
|
|
6
6
|
License-Expression: MIT
|
|
@@ -31,7 +31,8 @@ ALIASES: dict[str, str] = {
|
|
|
31
31
|
"gdp_annual_uk": "uk_gdp_annual",
|
|
32
32
|
# The FX pairs are spelled in capitals in fx_rates, the all_series view and
|
|
33
33
|
# the CLI; as series they are lower case (session D2). Both answer.
|
|
34
|
-
**{p: p.lower() for p in ("EURUSD", "EURGBP", "GBPUSD", "USDEUR", "GBPEUR", "USDGBP"
|
|
34
|
+
**{p: p.lower() for p in ("EURUSD", "EURGBP", "GBPUSD", "USDEUR", "GBPEUR", "USDGBP",
|
|
35
|
+
"USDKZT", "EURKZT")},
|
|
35
36
|
}
|
|
36
37
|
|
|
37
38
|
|
|
@@ -112,6 +112,11 @@ def fx_pairs() -> list[tuple[str, str, str]]:
|
|
|
112
112
|
("EURUSD", "ecb", "EXR/D.USD.EUR.SP00.A"),
|
|
113
113
|
("EURGBP", "ecb", "EXR/D.GBP.EUR.SP00.A"),
|
|
114
114
|
("GBPUSD", "fedboard", "H10:RXI$US_N.B.UK"), # FRED's DEXUSUK is this H.10 series
|
|
115
|
+
# The tenge (session K1): the National Bank of Kazakhstan's own official
|
|
116
|
+
# rates, both of them, so EURKZT needs no cross through the dollar.
|
|
117
|
+
# No reciprocals: nothing asks for tenge-based rates.
|
|
118
|
+
("USDKZT", "nbk", "fx:5"),
|
|
119
|
+
("EURKZT", "nbk", "fx:6"),
|
|
115
120
|
]
|
|
116
121
|
derived = [
|
|
117
122
|
("USDEUR", "derived", "1/EURUSD"),
|
|
@@ -23,7 +23,7 @@ CODES = {
|
|
|
23
23
|
"de": "Germany", "fr": "France", "it": "Italy", "es": "Spain",
|
|
24
24
|
"nl": "Netherlands", "be": "Belgium", "at": "Austria", "pt": "Portugal",
|
|
25
25
|
"ie": "Ireland", "el": "Greece", "fi": "Finland", "se": "Sweden",
|
|
26
|
-
"no": "Norway", "dk": "Denmark",
|
|
26
|
+
"no": "Norway", "dk": "Denmark", "kz": "Kazakhstan",
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
# Words a person types for a place → its code. Two-letter country codes that
|
|
@@ -44,6 +44,7 @@ NAMES = {
|
|
|
44
44
|
"greece": "el", "greek": "el", "finland": "fi", "finnish": "fi",
|
|
45
45
|
"sweden": "se", "swedish": "se", "norway": "no", "norwegian": "no",
|
|
46
46
|
"denmark": "dk", "danish": "dk",
|
|
47
|
+
"kazakhstan": "kz", "kazakh": "kz", "kazakhstani": "kz",
|
|
47
48
|
}
|
|
48
49
|
|
|
49
50
|
# Places Bedrok carries only as the last token of a broad family (Eurostat's
|
|
@@ -88,6 +89,12 @@ _PREFIX = {"ecb": "ea"}
|
|
|
88
89
|
ISO2 = {**{c: c.upper() for c in CODES}, **{c: c.upper() for c in FAMILY_CODES},
|
|
89
90
|
"uk": "GB", "el": "GR", "ea": "", "eu": ""}
|
|
90
91
|
|
|
92
|
+
# An exchange rate is about the country of its quote currency when that
|
|
93
|
+
# currency has one country: usdkzt is Kazakhstan's. The euro, the dollar and
|
|
94
|
+
# sterling are left out on purpose: "euro area exchange rate" must not bring
|
|
95
|
+
# every pair into a search.
|
|
96
|
+
_QUOTE_CURRENCY = {"kzt": "kz"}
|
|
97
|
+
|
|
91
98
|
# Sources whose series are not about a country.
|
|
92
99
|
_PLACELESS_SOURCES = {"worldbank", "cftc"}
|
|
93
100
|
|
|
@@ -97,6 +104,8 @@ def of_series(series_id: str, source: str | None = None) -> str | None:
|
|
|
97
104
|
if source in _PLACELESS_SOURCES:
|
|
98
105
|
return None
|
|
99
106
|
sid = series_id.lower()
|
|
107
|
+
if re.fullmatch(r"[a-z]{6}", sid) and sid[3:] in _QUOTE_CURRENCY:
|
|
108
|
+
return _QUOTE_CURRENCY[sid[3:]]
|
|
100
109
|
first = sid.split("_", 1)[0]
|
|
101
110
|
if first in CODES:
|
|
102
111
|
return first
|
|
@@ -54,6 +54,28 @@ SPAN_DAYS = {"daily": 3, "weekly": 7, "monthly": 61, "quarterly": 183, "annual":
|
|
|
54
54
|
|
|
55
55
|
_MONTHS = ("January", "February", "March", "April", "May", "June", "July", "August",
|
|
56
56
|
"September", "October", "November", "December")
|
|
57
|
+
_ORDINALS = ("first", "second", "third", "fourth")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def parse_period(text: str | None) -> date | None:
|
|
61
|
+
"""A calendar's reference period as the label Bedrok dates it by (the
|
|
62
|
+
period's first day): "October 2026", "Aug-2026", "third quarter 2026",
|
|
63
|
+
"Quarter 3/2026". Anything else is None: the date is still kept, with no
|
|
64
|
+
period, and matched by its window instead."""
|
|
65
|
+
t = " ".join(str(text or "").split()).lower()
|
|
66
|
+
months = [m.lower() for m in _MONTHS]
|
|
67
|
+
m = re.fullmatch(r"([a-z]+)[ -](\d{4})", t)
|
|
68
|
+
if m:
|
|
69
|
+
word, year = m.group(1), int(m.group(2))
|
|
70
|
+
hit = [i for i, name in enumerate(months) if name == word or name[:3] == word]
|
|
71
|
+
if hit:
|
|
72
|
+
return date(year, hit[0] + 1, 1)
|
|
73
|
+
m = re.fullmatch(r"(first|second|third|fourth) quarter (\d{4})", t) \
|
|
74
|
+
or re.fullmatch(r"quarter ([1-4])/(\d{4})", t)
|
|
75
|
+
if m:
|
|
76
|
+
q = _ORDINALS.index(m.group(1)) + 1 if m.group(1) in _ORDINALS else int(m.group(1))
|
|
77
|
+
return date(int(m.group(2)), 3 * q - 2, 1)
|
|
78
|
+
return None
|
|
57
79
|
|
|
58
80
|
|
|
59
81
|
def parse_row(raw: dict) -> dict:
|
|
@@ -70,8 +92,10 @@ def parse_row(raw: dict) -> dict:
|
|
|
70
92
|
raise ValueError(f"{key}: time {r['time']!r} is not HH:MM")
|
|
71
93
|
lo, hi = int(r["lag_min"]), int(r["lag_max"])
|
|
72
94
|
# A flash can come before its month is over (Eurostat's HICP on the last
|
|
73
|
-
# working day, Destatis's CPI around the 28th): a small negative lag.
|
|
74
|
-
|
|
95
|
+
# working day, Destatis's CPI around the 28th): a small negative lag. A
|
|
96
|
+
# quarterly survey dated to its own quarter is published inside it (the
|
|
97
|
+
# Fed's SLOOS, the ECB's bank lending survey): up to a quarter before.
|
|
98
|
+
if not -90 <= lo <= hi:
|
|
75
99
|
raise ValueError(f"{key}: lags {lo}..{hi}")
|
|
76
100
|
ZoneInfo(r["tz"]) # raises on an unknown zone
|
|
77
101
|
hh, mm = map(int, r["time"].split(":"))
|
|
@@ -92,6 +116,24 @@ FEEDS = {
|
|
|
92
116
|
"label": "BEA's release schedule"},
|
|
93
117
|
"ons_next_release": {"publisher": "ONS", "tz": "Europe/London", "fetched": False,
|
|
94
118
|
"label": "the ONS's next-release date"},
|
|
119
|
+
# The National Bank of Kazakhstan prints "Next: dd.mm.yyyy" on each
|
|
120
|
+
# statistics page; it is read after the page's series is fetched, one
|
|
121
|
+
# extra request per page. Asia/Almaty is UTC+5 since 2024-03-01 (tzdata
|
|
122
|
+
# 2024a); an older zone database says +6.
|
|
123
|
+
"nbk_next_release": {"publisher": "National Bank of Kazakhstan", "tz": "Asia/Almaty",
|
|
124
|
+
"fetched": False, "extra_request": True,
|
|
125
|
+
"label": "the National Bank of Kazakhstan's next-release date"},
|
|
126
|
+
"statcan_json": {"publisher": "Statistics Canada", "tz": "America/Toronto",
|
|
127
|
+
"fetched": True, "label": "Statistics Canada's release schedule"},
|
|
128
|
+
"ine_ics": {"publisher": "INE", "tz": "Europe/Madrid", "fetched": True,
|
|
129
|
+
"label": "INE's release calendar"},
|
|
130
|
+
"ecb_statscal": {"publisher": "ECB", "tz": "Europe/Berlin", "fetched": True,
|
|
131
|
+
"label": "the ECB's statistical calendar"},
|
|
132
|
+
"fedboard_pages": {"publisher": "Federal Reserve Board", "tz": "America/New_York",
|
|
133
|
+
"fetched": True, "label": "the Fed Board's release calendar"},
|
|
134
|
+
"bns_calendar": {"publisher": "Bureau of National Statistics of Kazakhstan",
|
|
135
|
+
"tz": "Asia/Almaty", "fetched": True,
|
|
136
|
+
"label": "the Bureau of National Statistics' release calendar"},
|
|
95
137
|
}
|
|
96
138
|
# An exact date has no window to wait through: the period is late a day after
|
|
97
139
|
# it, which is long enough for a publisher's API to catch up with its website.
|
|
@@ -12,13 +12,14 @@ from __future__ import annotations
|
|
|
12
12
|
|
|
13
13
|
import re
|
|
14
14
|
|
|
15
|
-
CURRENCIES = r"(USD|EUR|GBP|JPY|CAD|AUD|CHF|CNY|SEK|NOK|DKK)"
|
|
15
|
+
CURRENCIES = r"(USD|EUR|GBP|JPY|CAD|AUD|CHF|CNY|SEK|NOK|DKK|KZT)"
|
|
16
16
|
SCALES = r"(thousands|millions|billions)"
|
|
17
17
|
|
|
18
18
|
GRAMMAR = [
|
|
19
19
|
r"percent",
|
|
20
20
|
r"percent annualised",
|
|
21
21
|
r"percent of GDP",
|
|
22
|
+
r"percent of non-oil GDP", # an oil producer's fiscal balance
|
|
22
23
|
r"percent change on (previous|a year earlier) (month|quarter)",
|
|
23
24
|
r"percentage points( of GDP)?",
|
|
24
25
|
r"net percent",
|
|
@@ -65,6 +66,8 @@ def dimension(units: str) -> str:
|
|
|
65
66
|
return "rate"
|
|
66
67
|
if u in ("percent of GDP", "percentage points of GDP"):
|
|
67
68
|
return "share of GDP"
|
|
69
|
+
if u == "percent of non-oil GDP":
|
|
70
|
+
return "share of non-oil GDP"
|
|
68
71
|
m = _MONEY.fullmatch(u)
|
|
69
72
|
if m:
|
|
70
73
|
return f"money:{m.group(2)}"
|
|
@@ -106,7 +106,11 @@ def _tokens(text: str) -> set[str]:
|
|
|
106
106
|
# cover. ISO codes are not used for matching: "in", "it" and "no" are words
|
|
107
107
|
# before they are India, Italy and Norway.
|
|
108
108
|
_EXTRA = {"china": {"prc"}, "russia": {"russian"}, "turkey": {"turkish", "turkiye"},
|
|
109
|
-
"argentina": {"argentine", "argentinian"}, "india": {"indian"}, "opec": {"opec"}
|
|
109
|
+
"argentina": {"argentine", "argentinian"}, "india": {"indian"}, "opec": {"opec"},
|
|
110
|
+
# A region's warning is about its countries: "kazakhstan exports"
|
|
111
|
+
# must find the mirror-trade case (session K3).
|
|
112
|
+
"caucasus and central asia": {"kazakhstan", "kazakh", "armenia", "armenian",
|
|
113
|
+
"kyrgyz", "kyrgyzstan"}}
|
|
110
114
|
|
|
111
115
|
|
|
112
116
|
def _words(text: str) -> str:
|
|
@@ -142,6 +146,17 @@ _NOT_PLACES = {"global"}
|
|
|
142
146
|
_COVERED = {iso for iso in geography.ISO2.values() if iso}
|
|
143
147
|
|
|
144
148
|
|
|
149
|
+
def _covered(row: dict, hits_place: set[str]) -> bool:
|
|
150
|
+
"""Whether Bedrok carries the place a search reached this warning by: the
|
|
151
|
+
warning's own country, or a covered country named for a region's warning
|
|
152
|
+
("kazakhstan" for the Caucasus and Central Asia), so that a regional case
|
|
153
|
+
also needs the words to be about it."""
|
|
154
|
+
if (row.get("iso2") or "") in _COVERED:
|
|
155
|
+
return True
|
|
156
|
+
return any(geography.ISO2.get(geography.NAMES.get(n, ""), "") in _COVERED
|
|
157
|
+
for n in hits_place)
|
|
158
|
+
|
|
159
|
+
|
|
145
160
|
def search(text: str, limit: int = 5) -> list[dict]:
|
|
146
161
|
"""Warnings whose country and concept match the words asked for. A place
|
|
147
162
|
must be named: "inflation" alone is not a question about Argentina."""
|
|
@@ -159,7 +174,7 @@ def search(text: str, limit: int = 5) -> list[dict]:
|
|
|
159
174
|
# also be about what was asked: "us 10 year yield" drew all five BLS
|
|
160
175
|
# warnings (the D2 self-test). Where it does not, the warning is the
|
|
161
176
|
# whole answer and the place alone is enough.
|
|
162
|
-
if (r
|
|
177
|
+
if _covered(r, hits_place) and \
|
|
163
178
|
not (words - set().union(*map(_tokens, hits_place))) & about:
|
|
164
179
|
continue
|
|
165
180
|
scored.append((3 * len(hits_place) + len(words & about), r["warning_id"], r))
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Bureau of National Statistics of Kazakhstan adapter: the Taldau API (no key).
|
|
2
|
+
|
|
3
|
+
`source_code` is an indicator id and the parameters that pin one series:
|
|
4
|
+
|
|
5
|
+
703076?period=4&measure=1&terms=741880,2695743,4772381&dics=67,848,2753
|
|
6
|
+
|
|
7
|
+
period Taldau's period type: 4 month, 5 quarter, 9 quarter to date
|
|
8
|
+
(year-to-date figures at each quarter's end), 7 year.
|
|
9
|
+
measure the indicator's `measureId` from GetIndexAttributes. A wrong one
|
|
10
|
+
answers an empty body, not an error.
|
|
11
|
+
terms one term per dimension (GetSegmentList, or the data call itself
|
|
12
|
+
with idx=<dimension> lists a dimension's elements); 741880 is the
|
|
13
|
+
Republic of Kazakhstan, 2695730 / 2695732 / 2695743 compare with
|
|
14
|
+
the previous period / the same period a year earlier / December
|
|
15
|
+
2010.
|
|
16
|
+
dics the dimensions' directory ids, in the same order.
|
|
17
|
+
|
|
18
|
+
and, optionally, Bedrok's own transformations, each stated in the series'
|
|
19
|
+
notes:
|
|
20
|
+
|
|
21
|
+
as=change the bureau publishes changes as a relative (109.3 for a
|
|
22
|
+
9.3% rise): subtract 100.
|
|
23
|
+
ytd=decumulate quarterly figures published year to date: Q1 as published,
|
|
24
|
+
each later quarter minus the one before it in the same
|
|
25
|
+
year. A quarter whose predecessor is missing is dropped.
|
|
26
|
+
Only for levels: a growth rate cannot be de-cumulated.
|
|
27
|
+
scale=1e-9 multiply (tenge to billions of tenge).
|
|
28
|
+
|
|
29
|
+
GET https://taldau.stat.gov.kz/en/Api/GetIndexTreeData?p_measure_id=…
|
|
30
|
+
&p_index_id=…&p_period_id=…&p_terms=…&p_term_id=741880&p_dicIds=…
|
|
31
|
+
&idx=0&p_parent_id=
|
|
32
|
+
|
|
33
|
+
answers a list of rows whose first is the Republic of Kazakhstan; values sit
|
|
34
|
+
in keys `yMMYYYY` (MM is the period's last month), as strings, and "x" marks
|
|
35
|
+
a suppressed figure.
|
|
36
|
+
|
|
37
|
+
Licence: official statistical information may be used for any purpose,
|
|
38
|
+
freely and free of charge, with a reference to the source
|
|
39
|
+
(https://stat.gov.kz/en/description/). Attribution.
|
|
40
|
+
"""
|
|
41
|
+
from __future__ import annotations
|
|
42
|
+
|
|
43
|
+
import functools
|
|
44
|
+
import re
|
|
45
|
+
from datetime import date
|
|
46
|
+
from urllib.parse import parse_qsl
|
|
47
|
+
|
|
48
|
+
import pandas as pd
|
|
49
|
+
import requests
|
|
50
|
+
|
|
51
|
+
API = "https://taldau.stat.gov.kz/en/Api/GetIndexTreeData"
|
|
52
|
+
KAZAKHSTAN = "741880"
|
|
53
|
+
QUARTERLY = {"5", "9"}
|
|
54
|
+
TIMEOUT = 60
|
|
55
|
+
_KEY = re.compile(r"y(\d{2})(\d{4})")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def fetch(source_code: str, start: str | None = None) -> pd.DataFrame:
|
|
59
|
+
index, _, query = source_code.partition("?")
|
|
60
|
+
opts = dict(parse_qsl(query))
|
|
61
|
+
for need in ("period", "measure", "terms", "dics"):
|
|
62
|
+
if need not in opts:
|
|
63
|
+
raise ValueError(f"BNS: source_code needs {need}=")
|
|
64
|
+
rows = _download(int(index), opts["period"], opts["measure"], opts["terms"], opts["dics"])
|
|
65
|
+
df = parse(rows, opts["period"], opts.get("as"), opts.get("ytd"), float(opts.get("scale", 1)))
|
|
66
|
+
if start and not df.empty:
|
|
67
|
+
df = df[df["date"] >= pd.Timestamp(start).date()].reset_index(drop=True)
|
|
68
|
+
return df
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@functools.lru_cache(maxsize=64)
|
|
72
|
+
def _download(index: int, period: str, measure: str, terms: str, dics: str):
|
|
73
|
+
"""One GET per series per run. Cleared by pipeline.clear_download_caches()."""
|
|
74
|
+
r = requests.get(API, timeout=TIMEOUT, params={
|
|
75
|
+
"p_measure_id": measure, "p_index_id": index, "p_period_id": period,
|
|
76
|
+
"p_terms": terms, "p_term_id": KAZAKHSTAN, "p_dicIds": dics, "idx": 0,
|
|
77
|
+
"p_parent_id": ""})
|
|
78
|
+
r.raise_for_status()
|
|
79
|
+
if not r.text.strip():
|
|
80
|
+
raise ValueError("BNS: empty answer (check the measure id against GetIndexAttributes)")
|
|
81
|
+
return r.json()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _period_start(key: str, period: str) -> date | None:
|
|
85
|
+
m = _KEY.fullmatch(key)
|
|
86
|
+
if not m:
|
|
87
|
+
return None
|
|
88
|
+
month, year = int(m.group(1)), int(m.group(2))
|
|
89
|
+
if period in QUARTERLY:
|
|
90
|
+
month -= 2 # the key names the quarter's last month
|
|
91
|
+
return date(year, month, 1) if 1 <= month <= 12 else None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def parse(rows, period: str, as_: str | None = None, ytd: str | None = None,
|
|
95
|
+
scale: float = 1.0) -> pd.DataFrame:
|
|
96
|
+
"""Taldau rows → the Republic of Kazakhstan's series as [date, value]."""
|
|
97
|
+
row = next((r for r in rows or [] if str(r.get("id")) == KAZAKHSTAN), None)
|
|
98
|
+
if row is None:
|
|
99
|
+
raise ValueError("BNS: no row for the Republic of Kazakhstan in the answer")
|
|
100
|
+
pairs = []
|
|
101
|
+
for key, raw in row.items():
|
|
102
|
+
d = _period_start(key, period)
|
|
103
|
+
if d is None or raw in (None, "", "x"):
|
|
104
|
+
continue
|
|
105
|
+
pairs.append((d, float(str(raw).replace(",", "."))))
|
|
106
|
+
df = pd.DataFrame(pairs, columns=["date", "value"]).sort_values("date").reset_index(drop=True)
|
|
107
|
+
if ytd == "decumulate":
|
|
108
|
+
df = _decumulate(df)
|
|
109
|
+
if as_ == "change":
|
|
110
|
+
df["value"] = df["value"] - 100
|
|
111
|
+
# Rounded to drop float residue (109.3 - 100 is 9.299999...), far below
|
|
112
|
+
# the one or two decimals the bureau publishes.
|
|
113
|
+
df["value"] = (df["value"] * scale).round(9)
|
|
114
|
+
return df
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _decumulate(df: pd.DataFrame) -> pd.DataFrame:
|
|
118
|
+
"""Year-to-date quarterly levels → discrete quarters."""
|
|
119
|
+
held = dict(zip(df["date"], df["value"]))
|
|
120
|
+
out = []
|
|
121
|
+
for d, v in held.items():
|
|
122
|
+
if d.month == 1:
|
|
123
|
+
out.append((d, v))
|
|
124
|
+
continue
|
|
125
|
+
before = date(d.year, d.month - 3, 1)
|
|
126
|
+
if before in held:
|
|
127
|
+
out.append((d, v - held[before]))
|
|
128
|
+
return pd.DataFrame(out, columns=["date", "value"]).sort_values("date").reset_index(drop=True)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""JODI oil adapter: the Joint Organisations Data Initiative's World Database
|
|
2
|
+
(no key).
|
|
3
|
+
|
|
4
|
+
`source_code` is `<REF_AREA>/<ENERGY_PRODUCT>/<FLOW_BREAKDOWN>/<UNIT_MEASURE>`
|
|
5
|
+
with an optional first month, e.g. `KZ/CRUDEOIL/INDPROD/KBD?from=2016-01`:
|
|
6
|
+
Kazakhstan's crude oil production in thousand barrels a day from January
|
|
7
|
+
2016.
|
|
8
|
+
|
|
9
|
+
JODI publishes the primary-products database as one CSV per year,
|
|
10
|
+
|
|
11
|
+
https://www.jodidata.org/_resources/files/downloads/oil-data/annual-csv/primary/<year>.csv
|
|
12
|
+
|
|
13
|
+
with the current year as `primaryyear<year>.csv` (the name may change once
|
|
14
|
+
the year is over, so a 404 tries the other one). Each file is 8 to 12 MB and
|
|
15
|
+
holds every country; columns are REF_AREA, TIME_PERIOD (YYYY-MM),
|
|
16
|
+
ENERGY_PRODUCT, FLOW_BREAKDOWN, UNIT_MEASURE, OBS_VALUE, ASSESSMENT_CODE.
|
|
17
|
+
A month not yet reported is "-", a figure never reported "N/A"; both are
|
|
18
|
+
dropped. The single zip on the downloads page is a Beyond 20/20 binary, not
|
|
19
|
+
CSV. The database is updated on or around the 20th of each month.
|
|
20
|
+
|
|
21
|
+
Licence: "freely available"; cite as International Energy Forum (IEF), Joint
|
|
22
|
+
Organizations Data Initiative (JODI) Database
|
|
23
|
+
(https://www.jodidata.org/oil/support/user-guide.aspx). Attribution.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import csv
|
|
28
|
+
import functools
|
|
29
|
+
import io
|
|
30
|
+
from datetime import date
|
|
31
|
+
from urllib.parse import parse_qsl
|
|
32
|
+
|
|
33
|
+
import pandas as pd
|
|
34
|
+
import requests
|
|
35
|
+
|
|
36
|
+
from bedrok.ingest.adapters._util import BROWSER_UA
|
|
37
|
+
|
|
38
|
+
BASE = "https://www.jodidata.org/_resources/files/downloads/oil-data/annual-csv/primary"
|
|
39
|
+
FIRST_YEAR = 2002
|
|
40
|
+
TIMEOUT = 180
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def fetch(source_code: str, start: str | None = None, today: date | None = None) -> pd.DataFrame:
|
|
44
|
+
key, _, query = source_code.partition("?")
|
|
45
|
+
parts = key.split("/")
|
|
46
|
+
if len(parts) != 4:
|
|
47
|
+
raise ValueError(f"JODI: source_code needs AREA/PRODUCT/FLOW/UNIT, got {key!r}")
|
|
48
|
+
first = dict(parse_qsl(query)).get("from")
|
|
49
|
+
lower = max([d for d in (first, str(start)[:7] if start else None) if d] or [f"{FIRST_YEAR}-01"])
|
|
50
|
+
this_year = (today or date.today()).year
|
|
51
|
+
rows = []
|
|
52
|
+
for year in range(max(int(lower[:4]), FIRST_YEAR), this_year + 1):
|
|
53
|
+
rows += parse(_year(year, year == this_year), *parts)
|
|
54
|
+
df = pd.DataFrame(rows, columns=["date", "value"])
|
|
55
|
+
if df.empty:
|
|
56
|
+
raise ValueError(f"JODI: no rows for {key}")
|
|
57
|
+
df = (df[df["date"] >= date(int(lower[:4]), int(lower[5:7]), 1)]
|
|
58
|
+
.drop_duplicates(subset="date", keep="last").sort_values("date").reset_index(drop=True))
|
|
59
|
+
return df
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _year(year: int, current: bool) -> str:
|
|
63
|
+
names = [f"primaryyear{year}.csv", f"{year}.csv"]
|
|
64
|
+
if not current:
|
|
65
|
+
names.reverse()
|
|
66
|
+
for i, name in enumerate(names):
|
|
67
|
+
try:
|
|
68
|
+
return _download(f"{BASE}/{name}")
|
|
69
|
+
except requests.HTTPError as e:
|
|
70
|
+
missing = getattr(e.response, "status_code", None) == 404
|
|
71
|
+
if missing and i == 0:
|
|
72
|
+
continue
|
|
73
|
+
# A new year's file exists only after JODI's first update of the
|
|
74
|
+
# year (about the 20th); until then that year simply has no rows.
|
|
75
|
+
# A past year missing under both names is an error.
|
|
76
|
+
if missing and current:
|
|
77
|
+
return ""
|
|
78
|
+
raise
|
|
79
|
+
raise AssertionError("unreachable")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@functools.lru_cache(maxsize=32)
|
|
83
|
+
def _download(url: str) -> str:
|
|
84
|
+
"""One GET per yearly file per run: crude and NGL share them. Cleared by
|
|
85
|
+
pipeline.clear_download_caches()."""
|
|
86
|
+
r = requests.get(url, headers=BROWSER_UA, timeout=TIMEOUT)
|
|
87
|
+
r.raise_for_status()
|
|
88
|
+
return r.text
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def parse(text: str, area: str, product: str, flow: str, unit: str) -> list[tuple]:
|
|
92
|
+
"""One yearly CSV → [(month start, value)] for one series."""
|
|
93
|
+
out = []
|
|
94
|
+
for row in csv.reader(io.StringIO(text)):
|
|
95
|
+
if len(row) < 6 or row[0] != area or row[2] != product or row[3] != flow or row[4] != unit:
|
|
96
|
+
continue
|
|
97
|
+
try:
|
|
98
|
+
value = float(row[5])
|
|
99
|
+
month = date(int(row[1][:4]), int(row[1][5:7]), 1)
|
|
100
|
+
except ValueError:
|
|
101
|
+
continue # "-" not yet reported, "N/A" never reported
|
|
102
|
+
out.append((month, value))
|
|
103
|
+
return out
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""National Bank of Kazakhstan adapter (no API key).
|
|
2
|
+
|
|
3
|
+
The bank serves its statistics three ways, and the history lives in different
|
|
4
|
+
ones, so `source_code` names which:
|
|
5
|
+
|
|
6
|
+
records:<section>/<slug>#<field>
|
|
7
|
+
A statistics page's own JSON. GET https://nationalbank.kz/en/<section>/
|
|
8
|
+
<slug>/records answers the page's whole table, every month since the
|
|
9
|
+
1990s, as a list of rows; <field> is one of its columns
|
|
10
|
+
(`m_3`, `gross_international_reserve_volume_million_dollar`). Each row
|
|
11
|
+
is dated by the month of its `reporting_date`: recent rows are stamped
|
|
12
|
+
at the month's end, older ones on an arbitrary day inside the month.
|
|
13
|
+
Checked against the IMF on 2026-10-28: M3 equals the IMF's broad money
|
|
14
|
+
to the million in every month compared, and reserves match within the
|
|
15
|
+
gold-valuation differences, with no month shift.
|
|
16
|
+
api:<formId>?<column>=<value>&...
|
|
17
|
+
The open-data API (data.nationalbank.kz/api/v1, a beta): long-format
|
|
18
|
+
rows filtered on the named columns, which must leave one row per date.
|
|
19
|
+
Most of its tables start only in 2018 to 2023; the effective exchange
|
|
20
|
+
rates (form 299, from 1995) and inflation expectations (form 305, from
|
|
21
|
+
2016) are the ones worth taking from it.
|
|
22
|
+
base-rate
|
|
23
|
+
The base rate's decisions, one row per decision, dated by the day it
|
|
24
|
+
took effect (from 2015-09-02, when the base rate replaced the
|
|
25
|
+
refinancing rate as the policy rate).
|
|
26
|
+
fx:<rate id>
|
|
27
|
+
Official exchange rates from the rates report page (5 = USD, 6 = EUR),
|
|
28
|
+
KZT per unit, daily from 1999-10-19. A rate is set the day before the
|
|
29
|
+
date it applies to, so the newest row is dated tomorrow.
|
|
30
|
+
|
|
31
|
+
`next_release(source_code)` reads a statistics page's "Next:" date for the
|
|
32
|
+
release hook (ingest/pipeline.py); it is an extra request, made only after a
|
|
33
|
+
`records:` series has been fetched.
|
|
34
|
+
|
|
35
|
+
Licence: free reuse with reference to the source, software included
|
|
36
|
+
(https://nationalbank.kz/en/page/data-usage-terms). Attribution.
|
|
37
|
+
"""
|
|
38
|
+
from __future__ import annotations
|
|
39
|
+
|
|
40
|
+
import functools
|
|
41
|
+
import html
|
|
42
|
+
import re
|
|
43
|
+
from datetime import date, datetime, timedelta
|
|
44
|
+
from urllib.parse import parse_qsl
|
|
45
|
+
|
|
46
|
+
import pandas as pd
|
|
47
|
+
import requests
|
|
48
|
+
|
|
49
|
+
from bedrok.ingest.adapters._util import UnpinnedKey
|
|
50
|
+
|
|
51
|
+
SITE = "https://nationalbank.kz/en"
|
|
52
|
+
API = "https://data.nationalbank.kz/api/v1/data"
|
|
53
|
+
FX_REPORT = f"{SITE}/exchangerates/ezhednevnye-oficialnye-rynochnye-kursy-valyut/report"
|
|
54
|
+
FX_FIRST = date(1999, 10, 1)
|
|
55
|
+
PAGE_SIZE = 10000
|
|
56
|
+
TIMEOUT = 60
|
|
57
|
+
HEADERS = {"Accept-Language": "en"}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def fetch(source_code: str, start: str | None = None) -> pd.DataFrame:
|
|
61
|
+
kind, _, rest = source_code.partition(":")
|
|
62
|
+
if kind == "records":
|
|
63
|
+
page, _, field = rest.partition("#")
|
|
64
|
+
df = parse_records(_download(f"{SITE}/{page}/records"), field)
|
|
65
|
+
elif kind == "api":
|
|
66
|
+
form, _, query = rest.partition("?")
|
|
67
|
+
df = parse_api(_download(f"{API}?formId={int(form)}&pageSize={PAGE_SIZE}"),
|
|
68
|
+
dict(parse_qsl(query)))
|
|
69
|
+
elif kind == "base-rate":
|
|
70
|
+
today = date.today()
|
|
71
|
+
df = parse_base_rate(_download(f"{API}/base-rate?from=1990-01-01&to={today:%Y-%m-%d}"))
|
|
72
|
+
elif kind == "fx":
|
|
73
|
+
first = max(pd.Timestamp(start).date(), FX_FIRST) if start else FX_FIRST
|
|
74
|
+
last = date.today() + timedelta(days=1)
|
|
75
|
+
url = (f"{FX_REPORT}?beginDate={first:%d.%m.%Y}&endDate={last:%d.%m.%Y}"
|
|
76
|
+
f"&rates[]={int(rest)}")
|
|
77
|
+
df = parse_fx(_download(url, as_text=True))
|
|
78
|
+
else:
|
|
79
|
+
raise ValueError(f"NBK: unknown source_code form {source_code!r}")
|
|
80
|
+
if start and not df.empty:
|
|
81
|
+
df = df[df["date"] >= pd.Timestamp(start).date()].reset_index(drop=True)
|
|
82
|
+
return df
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@functools.lru_cache(maxsize=64)
|
|
86
|
+
def _download(url: str, as_text: bool = False):
|
|
87
|
+
"""One GET per URL per run: the three reserves series share one page.
|
|
88
|
+
Cleared by pipeline.clear_download_caches()."""
|
|
89
|
+
r = requests.get(url, headers=HEADERS, timeout=TIMEOUT)
|
|
90
|
+
r.raise_for_status()
|
|
91
|
+
return r.text if as_text else r.json()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _frame(pairs) -> pd.DataFrame:
|
|
95
|
+
df = pd.DataFrame(pairs, columns=["date", "value"])
|
|
96
|
+
df["value"] = pd.to_numeric(df["value"], errors="coerce")
|
|
97
|
+
df = df.dropna(subset=["date", "value"]).sort_values("date").reset_index(drop=True)
|
|
98
|
+
return df
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _rows(js) -> list:
|
|
102
|
+
"""A `/records` answer is a list; tolerate it wrapped in an object."""
|
|
103
|
+
if isinstance(js, list):
|
|
104
|
+
return js
|
|
105
|
+
if isinstance(js, dict):
|
|
106
|
+
for v in js.values():
|
|
107
|
+
if isinstance(v, list):
|
|
108
|
+
return v
|
|
109
|
+
raise ValueError("NBK: no rows in the answer")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def parse_records(js, field: str) -> pd.DataFrame:
|
|
113
|
+
"""A statistics page's rows → one column, dated by its reporting month.
|
|
114
|
+
Two rows in one month would be two answers for one period: refuse."""
|
|
115
|
+
rows = _rows(js)
|
|
116
|
+
if rows and field not in rows[-1]:
|
|
117
|
+
raise ValueError(f"NBK: no column {field!r} in the table")
|
|
118
|
+
seen, pairs = set(), []
|
|
119
|
+
for row in rows:
|
|
120
|
+
stamp = str(row.get("reporting_date") or "")[:10]
|
|
121
|
+
if not stamp:
|
|
122
|
+
continue
|
|
123
|
+
d = date.fromisoformat(stamp).replace(day=1)
|
|
124
|
+
if d in seen:
|
|
125
|
+
raise ValueError(f"NBK: two rows for {d:%Y-%m}")
|
|
126
|
+
seen.add(d)
|
|
127
|
+
pairs.append((d, row.get(field)))
|
|
128
|
+
return _frame(pairs)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def parse_api(js: dict, filters: dict) -> pd.DataFrame:
|
|
132
|
+
"""Open-data API rows → the one series the filters pin."""
|
|
133
|
+
rows = js.get("rows") or []
|
|
134
|
+
total = js.get("totalRows")
|
|
135
|
+
if total is not None and int(total) > len(rows):
|
|
136
|
+
raise ValueError(f"NBK: the answer was paged ({len(rows)} of {total} rows)")
|
|
137
|
+
keep = [r for r in rows if all(str(r.get(k)) == v for k, v in filters.items())]
|
|
138
|
+
dates = [r.get("report_date") for r in keep]
|
|
139
|
+
if len(dates) != len(set(dates)):
|
|
140
|
+
others = sorted({k for r in keep for k in r} - {"report_date", "amount", *filters})
|
|
141
|
+
raise UnpinnedKey(f"NBK: several rows per date; pin one of {others}")
|
|
142
|
+
return _frame([(date.fromisoformat(str(r["report_date"])[:10]), r.get("amount"))
|
|
143
|
+
for r in keep])
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def parse_base_rate(js) -> pd.DataFrame:
|
|
147
|
+
"""The base rate's decisions → one row per decision, dated when it took effect."""
|
|
148
|
+
if not isinstance(js, list):
|
|
149
|
+
raise ValueError("NBK: the base-rate answer is not a list of decisions")
|
|
150
|
+
return _frame([(date.fromisoformat(str(r["effectiveDate"])[:10]), r.get("rate"))
|
|
151
|
+
for r in js if r.get("effectiveDate")])
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
_TR = re.compile(r"<tr[^>]*>(.*?)</tr>", re.S)
|
|
155
|
+
_TD = re.compile(r"<t[dh][^>]*>(.*?)</t[dh]>", re.S)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _cells(row: str) -> list[str]:
|
|
159
|
+
return [html.unescape(re.sub(r"<[^>]+>", "", c)).strip() for c in _TD.findall(row)]
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def parse_fx(text: str) -> pd.DataFrame:
|
|
163
|
+
"""The rates report's table → KZT per one unit: each row is a date, the
|
|
164
|
+
quantity the rate is quoted for and the rate, so divide."""
|
|
165
|
+
pairs = []
|
|
166
|
+
for row in _TR.findall(text):
|
|
167
|
+
cells = _cells(row)
|
|
168
|
+
if len(cells) < 3 or not re.fullmatch(r"\d{4}-\d{2}-\d{2}", cells[0]):
|
|
169
|
+
continue
|
|
170
|
+
qty = float(cells[1].replace(",", "."))
|
|
171
|
+
rate = float(cells[2].replace(" ", "").replace(",", "."))
|
|
172
|
+
pairs.append((date.fromisoformat(cells[0]), rate / qty))
|
|
173
|
+
if not pairs:
|
|
174
|
+
raise ValueError("NBK: no rates in the report (the page changed?)")
|
|
175
|
+
return _frame(pairs).drop_duplicates(subset="date", keep="last").reset_index(drop=True)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
_NEXT = re.compile(r"Next:\s*(\d{2}\.\d{2}\.\d{4})")
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def parse_next(text: str) -> date | None:
|
|
182
|
+
"""A statistics page's "Next: 30.10.2026", or None."""
|
|
183
|
+
m = _NEXT.search(re.sub(r"<[^>]+>", " ", text or ""))
|
|
184
|
+
return datetime.strptime(m.group(1), "%d.%m.%Y").date() if m else None
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def next_release(source_code: str) -> date | None:
|
|
188
|
+
"""The date the page behind a `records:` series names for its next
|
|
189
|
+
publication. Other forms have no such page."""
|
|
190
|
+
kind, _, rest = source_code.partition(":")
|
|
191
|
+
if kind != "records":
|
|
192
|
+
return None
|
|
193
|
+
page = rest.partition("#")[0]
|
|
194
|
+
return parse_next(_download(f"{SITE}/{page}", as_text=True))
|
|
@@ -119,8 +119,23 @@ def parse_xml(stream) -> pd.DataFrame:
|
|
|
119
119
|
return _finish(pd.DataFrame(rows, columns=["TIME_PERIOD", "OBS_VALUE"]))
|
|
120
120
|
|
|
121
121
|
|
|
122
|
+
def _split_scale(code: str) -> tuple[str, float]:
|
|
123
|
+
"""`FLOW/KEY#scale=1e-6` → ("FLOW/KEY", 1e-6). The IMF answers balance of
|
|
124
|
+
payments in whole US dollars and states the display scale only as a
|
|
125
|
+
`SCALE` attribute, so a row asks for millions with this suffix; it never
|
|
126
|
+
reaches the publisher."""
|
|
127
|
+
key, _, frag = code.partition("#")
|
|
128
|
+
if not frag:
|
|
129
|
+
return code, 1.0
|
|
130
|
+
name, _, value = frag.partition("=")
|
|
131
|
+
if name != "scale":
|
|
132
|
+
raise ValueError(f"unknown SDMX code option {frag!r}")
|
|
133
|
+
return key, float(value)
|
|
134
|
+
|
|
135
|
+
|
|
122
136
|
def fetch(spec: dict, code: str, start: str | None = None) -> pd.DataFrame:
|
|
123
137
|
"""Fetch one series. `spec` is the publisher's row from the source list."""
|
|
138
|
+
code, scale = _split_scale(code)
|
|
124
139
|
fmt = spec.get("format") or "accept"
|
|
125
140
|
headers = dict(BROWSER_UA) if spec.get("user_agent") == "browser" else {}
|
|
126
141
|
params = {}
|
|
@@ -146,4 +161,6 @@ def fetch(spec: dict, code: str, start: str | None = None) -> pd.DataFrame:
|
|
|
146
161
|
else parse_csv(r.text, latest_of=spec.get("latest_of") or None))
|
|
147
162
|
if start and not df.empty:
|
|
148
163
|
df = df[df["date"] >= pd.Timestamp(start).date()].reset_index(drop=True)
|
|
164
|
+
if scale != 1.0 and not df.empty:
|
|
165
|
+
df["value"] = df["value"] * scale
|
|
149
166
|
return df
|