gomd 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gomd/__init__.py +380 -0
- gomd/__main__.py +7 -0
- gomd/cli.py +76 -0
- gomd/client.py +1372 -0
- gomd/dynamics.py +629 -0
- gomd/maritime.py +952 -0
- gomd/ocean.py +1646 -0
- gomd/spatial.py +222 -0
- gomd/stats.py +1086 -0
- gomd-2.1.0.dist-info/METADATA +277 -0
- gomd-2.1.0.dist-info/RECORD +15 -0
- gomd-2.1.0.dist-info/WHEEL +5 -0
- gomd-2.1.0.dist-info/entry_points.txt +2 -0
- gomd-2.1.0.dist-info/licenses/LICENSE +21 -0
- gomd-2.1.0.dist-info/top_level.txt +1 -0
gomd/client.py
ADDED
|
@@ -0,0 +1,1372 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Official Python Client for GOMD (Global Ocean & Maritime Data Platform)
|
|
3
|
+
https://gomd.pirireis.edu.tr | User Manual & API Reference: https://gomd.pirireis.edu.tr/sdk_paper.php
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import csv
|
|
9
|
+
import hashlib
|
|
10
|
+
import io
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
import os
|
|
14
|
+
import time
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Dict, List, Optional, Sequence, Union
|
|
17
|
+
import urllib.error
|
|
18
|
+
import urllib.parse
|
|
19
|
+
import urllib.request
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
import requests # type: ignore
|
|
23
|
+
except ImportError:
|
|
24
|
+
requests = None # type: ignore
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _StdlibResponse:
|
|
28
|
+
"""Zero-dependency HTTP response wrapper when `requests` is not installed."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, status_code: int, content: bytes) -> None:
|
|
31
|
+
self.status_code = status_code
|
|
32
|
+
self.content = content
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def text(self) -> str:
|
|
36
|
+
return self.content.decode("utf-8", errors="replace")
|
|
37
|
+
|
|
38
|
+
def json(self) -> Any:
|
|
39
|
+
return json.loads(self.text)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class _StdlibSession:
|
|
43
|
+
"""Zero-dependency HTTP session using `urllib.request` when `requests` is unavailable."""
|
|
44
|
+
|
|
45
|
+
def __init__(self) -> None:
|
|
46
|
+
self.headers: Dict[str, str] = {}
|
|
47
|
+
|
|
48
|
+
def get(
|
|
49
|
+
self,
|
|
50
|
+
url: str,
|
|
51
|
+
params: Optional[Dict[str, Any]] = None,
|
|
52
|
+
timeout: int = 60,
|
|
53
|
+
) -> _StdlibResponse:
|
|
54
|
+
full_url = url
|
|
55
|
+
if params:
|
|
56
|
+
clean = {k: str(v) for k, v in params.items() if v is not None}
|
|
57
|
+
qs = urllib.parse.urlencode(clean)
|
|
58
|
+
if qs:
|
|
59
|
+
sep = "&" if "?" in full_url else "?"
|
|
60
|
+
full_url = f"{full_url}{sep}{qs}"
|
|
61
|
+
req = urllib.request.Request(full_url, headers=dict(self.headers), method="GET")
|
|
62
|
+
try:
|
|
63
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
64
|
+
return _StdlibResponse(int(getattr(resp, "status", 200)), resp.read())
|
|
65
|
+
except urllib.error.HTTPError as exc:
|
|
66
|
+
body = exc.read() if hasattr(exc, "read") else b""
|
|
67
|
+
return _StdlibResponse(int(exc.code), body)
|
|
68
|
+
|
|
69
|
+
_RETRY_STATUSES = {429, 502, 503, 504}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _retrying_get(get_fn, max_tries: int = 6, max_wait: float = 60.0):
|
|
73
|
+
"""Wrap a session.get so 429/5xx answers are retried with Retry-After or exponential backoff."""
|
|
74
|
+
import random
|
|
75
|
+
|
|
76
|
+
def wrapped(*args, **kwargs):
|
|
77
|
+
resp = None
|
|
78
|
+
for attempt in range(max_tries):
|
|
79
|
+
resp = get_fn(*args, **kwargs)
|
|
80
|
+
status = int(getattr(resp, "status_code", 200) or 200)
|
|
81
|
+
if status not in _RETRY_STATUSES or attempt == max_tries - 1:
|
|
82
|
+
return resp
|
|
83
|
+
wait = None
|
|
84
|
+
headers = getattr(resp, "headers", None) or {}
|
|
85
|
+
ra = headers.get("Retry-After") if hasattr(headers, "get") else None
|
|
86
|
+
if ra:
|
|
87
|
+
try:
|
|
88
|
+
wait = float(ra)
|
|
89
|
+
except (TypeError, ValueError):
|
|
90
|
+
wait = None
|
|
91
|
+
if wait is None:
|
|
92
|
+
wait = min(max_wait, (2 ** attempt) + random.uniform(0, 1))
|
|
93
|
+
time.sleep(min(max_wait, max(wait, 0.5)))
|
|
94
|
+
return resp
|
|
95
|
+
|
|
96
|
+
return wrapped
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
CITATION_APA = (
|
|
100
|
+
"Piri Reis University Global Ocean & Maritime Data Platform (GOMD). (2026). "
|
|
101
|
+
"gomd: Official Python and R Scientific, Oceanographic, Maritime & Econometric Software "
|
|
102
|
+
"Development Kits (Version 2.1.0) [Computer software & Data platform]. "
|
|
103
|
+
"https://gomd.pirireis.edu.tr"
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
CITATION_IEEE = (
|
|
107
|
+
'[1] Piri Reis University Global Ocean & Maritime Data Platform (GOMD), '
|
|
108
|
+
'"gomd: Official Python and R Scientific, Oceanographic, Maritime & Econometric Software '
|
|
109
|
+
'Development Kits (Version 2.1.0)," 2026. '
|
|
110
|
+
'[Online]. Available: https://gomd.pirireis.edu.tr'
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
CITATION_TR = (
|
|
114
|
+
"Pîrî Reis Üniversitesi Küresel Okyanus ve Denizcilik Veri Platformu (GOMD). (2026). "
|
|
115
|
+
"gomd: Resmi Python ve R Bilimsel, Oşinografik, Denizcilik ve Ekonometrik Yazılım Geliştirme Paketleri (Sürüm 2.1.0) "
|
|
116
|
+
"[Bilgisayar yazılımı ve Veri platformu]. https://gomd.pirireis.edu.tr"
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
CITATION_BIBTEX = """@misc{gomd2026platform,
|
|
120
|
+
author = {{Piri Reis University Global Ocean \\& Maritime Data Platform (GOMD)}},
|
|
121
|
+
title = {{GOMD}: Global Ocean \\& Maritime Data Platform and Official {Python}/{R} Software Development Kits ({gomd} v2.1.0)},
|
|
122
|
+
year = {2026},
|
|
123
|
+
howpublished = {\\url{https://gomd.pirireis.edu.tr}},
|
|
124
|
+
url = {https://gomd.pirireis.edu.tr},
|
|
125
|
+
note = {Live source, indicator and observation counts: https://gomd.pirireis.edu.tr}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
@software{gomd2026sdk,
|
|
129
|
+
author = {{Piri Reis University Global Ocean \\& Maritime Data Platform (GOMD)}},
|
|
130
|
+
title = {{gomd}: Official {Python} and {R} Scientific \\& Econometric Software Development Kits for {GOMD}},
|
|
131
|
+
version = {2.1.0},
|
|
132
|
+
year = {2026},
|
|
133
|
+
publisher = {Piri Reis University},
|
|
134
|
+
url = {https://gomd.pirireis.edu.tr},
|
|
135
|
+
license = {MIT}
|
|
136
|
+
}"""
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class CitationText(str):
|
|
140
|
+
"""String subclass that renders multi-line citations cleanly in Jupyter / Google Colab cells."""
|
|
141
|
+
|
|
142
|
+
def __repr__(self) -> str:
|
|
143
|
+
return str(self)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def cite(style: Any = "all") -> CitationText:
|
|
147
|
+
"""
|
|
148
|
+
Return the official academic citation for the GOMD platform and Python SDK.
|
|
149
|
+
Accepts `"all"`, `"apa"`, `"ieee"`, `"tr"`, `"bibtex"`, or even Python's built-in `all`.
|
|
150
|
+
"""
|
|
151
|
+
if style is None or style is all or callable(style):
|
|
152
|
+
s = "all"
|
|
153
|
+
else:
|
|
154
|
+
s = str(style).strip().lower()
|
|
155
|
+
|
|
156
|
+
if s in ("apa", "apa_en", "en"):
|
|
157
|
+
return CitationText(CITATION_APA)
|
|
158
|
+
if s == "ieee":
|
|
159
|
+
return CitationText(CITATION_IEEE)
|
|
160
|
+
if s in ("tr", "turkish", "apa_tr"):
|
|
161
|
+
return CitationText(CITATION_TR)
|
|
162
|
+
if s in ("bib", "bibtex"):
|
|
163
|
+
return CitationText(CITATION_BIBTEX)
|
|
164
|
+
return CitationText(
|
|
165
|
+
"================================================================================\n"
|
|
166
|
+
"PLEASE CITE GOMD IN YOUR PUBLICATIONS / LÜTFEN YAYINLARINIZDA ŞU ŞEKİLDE ATIF YAPINIZ\n"
|
|
167
|
+
"================================================================================\n\n"
|
|
168
|
+
f"[APA 7th (EN)]\n{CITATION_APA}\n\n"
|
|
169
|
+
f"[APA 7th (TR)]\n{CITATION_TR}\n\n"
|
|
170
|
+
f"[IEEE]\n{CITATION_IEEE}\n\n"
|
|
171
|
+
f"[BibTeX]\n{CITATION_BIBTEX}\n"
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class GOMDError(Exception):
|
|
176
|
+
"""Raised when the GOMD API returns an HTTP or application error."""
|
|
177
|
+
|
|
178
|
+
def __init__(self, message: str, status_code: Optional[int] = None, payload: Any = None):
|
|
179
|
+
super().__init__(message)
|
|
180
|
+
self.status_code = status_code
|
|
181
|
+
self.payload = payload
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _to_dataframe(records: List[Dict[str, Any]], as_df: bool = True) -> Any:
|
|
185
|
+
"""Convert a list of dicts to a pandas DataFrame if pandas is installed and requested."""
|
|
186
|
+
if not as_df:
|
|
187
|
+
return records
|
|
188
|
+
try:
|
|
189
|
+
import pandas as pd # type: ignore
|
|
190
|
+
|
|
191
|
+
return pd.DataFrame(records)
|
|
192
|
+
except ImportError:
|
|
193
|
+
return records
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
_INDICATOR_ALIASES: Dict[str, List[str]] = {
|
|
197
|
+
"gml_mauna_loa_co2_ppm": ["gml_mauna_loa_co2_ppm", "noaa_gml_co2_mlo"],
|
|
198
|
+
"noaa_gml_co2_mlo": ["noaa_gml_co2_mlo", "gml_mauna_loa_co2_ppm"],
|
|
199
|
+
"worldbank_primary_index_v2026": ["worldbank_primary_index_v2026", "wb_co2_total", "wb_wdi_EN_GHG_CO2_MT_CE_AR5"],
|
|
200
|
+
"copernicus_med_sst_anomaly_c": ["copernicus_sst_med", "metocean_global_basin_sst_mean_c"],
|
|
201
|
+
"wdi_container_port_traffic_teu": [
|
|
202
|
+
"wb_port_traffic",
|
|
203
|
+
"unctad_wb_container_port_traffic_teu",
|
|
204
|
+
"wb_container_port_traffic",
|
|
205
|
+
"wdi_container_port_traffic_teu",
|
|
206
|
+
],
|
|
207
|
+
"unctad_wb_container_port_traffic_teu": [
|
|
208
|
+
"wb_port_traffic",
|
|
209
|
+
"unctad_wb_container_port_traffic_teu",
|
|
210
|
+
"wb_container_port_traffic",
|
|
211
|
+
],
|
|
212
|
+
"wdi_liner_shipping_connectivity_lsci": [
|
|
213
|
+
"wb_liner_connectivity_annual",
|
|
214
|
+
"wdi_liner_shipping_connectivity_lsci",
|
|
215
|
+
"unctad_wb_liner_shipping_connectivity_index",
|
|
216
|
+
],
|
|
217
|
+
"unctad_wb_liner_shipping_connectivity_index": [
|
|
218
|
+
"wb_liner_connectivity_annual",
|
|
219
|
+
"unctad_wb_liner_shipping_connectivity_index",
|
|
220
|
+
],
|
|
221
|
+
"wdi_logistic_performance_index_overall": [
|
|
222
|
+
"lpi_wb_overall_score",
|
|
223
|
+
"wdi_logistic_performance_index_overall",
|
|
224
|
+
],
|
|
225
|
+
"open_meteo_marine_wave_height_max_m": [
|
|
226
|
+
"metocean_global_basin_wave_height_max_m",
|
|
227
|
+
"open_meteo_marine_wave_height_max_m",
|
|
228
|
+
],
|
|
229
|
+
"open_meteo_marine_wave_height_mean_m": [
|
|
230
|
+
"metocean_global_basin_wave_height_max_m",
|
|
231
|
+
"open_meteo_marine_wave_height_mean_m",
|
|
232
|
+
],
|
|
233
|
+
"open_meteo_marine_sst_mean_c": [
|
|
234
|
+
"metocean_global_basin_sst_mean_c",
|
|
235
|
+
"open_meteo_marine_sst_mean_c",
|
|
236
|
+
],
|
|
237
|
+
"gcb_fossil_emissions_gtco2": ["gcb2025_fossil", "gcb_fossil_emissions_gtco2"],
|
|
238
|
+
"gcb_ocean_sink_gtco2": ["gcb2025_ocean_sink", "gcb_ocean_sink_gtco2"],
|
|
239
|
+
"noaa_ohc_0_2000m_zj": ["ncei_ohc_0_700m", "noaa_ohc_0_2000m_zj"],
|
|
240
|
+
"nsidc_arctic_sea_ice_sep_mkm2": ["nsidc_arctic_ice_min", "nsidc_arctic_sea_ice_sep_mkm2"],
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
_BASIN_ENTITY_ALIASES: Dict[str, str] = {
|
|
244
|
+
"MED": "OCN_MED_LEVANTINE",
|
|
245
|
+
"LEVANTINE": "OCN_MED_LEVANTINE",
|
|
246
|
+
"EAST_MED": "OCN_MED_LEVANTINE",
|
|
247
|
+
"WEST_MED": "OCN_MED_WEST",
|
|
248
|
+
"AEG": "OCN_MED_AEGEAN",
|
|
249
|
+
"AEGEAN": "OCN_MED_AEGEAN",
|
|
250
|
+
"BLK": "OCN_BLACKSEA_CENTRAL",
|
|
251
|
+
"BLACK_SEA": "OCN_BLACKSEA_CENTRAL",
|
|
252
|
+
"MAR": "OCN_MARMARA",
|
|
253
|
+
"MARMARA": "OCN_MARMARA",
|
|
254
|
+
"BOSPORUS": "STR_BOSPORUS",
|
|
255
|
+
"DARDANELLES": "STR_DARDANELLES",
|
|
256
|
+
"SUEZ": "OCN_CHOKE_SUEZ_REDSEA",
|
|
257
|
+
"OCN_CHOKE_SUEZ": "OCN_CHOKE_SUEZ_REDSEA",
|
|
258
|
+
"GIB": "OCN_CHOKE_GIBRALTAR",
|
|
259
|
+
"BAB": "OCN_CHOKE_BABELMANDEB",
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class GOMDClient:
|
|
264
|
+
"""
|
|
265
|
+
Official Python SDK client for GOMD (`gomd.pirireis.edu.tr`).
|
|
266
|
+
|
|
267
|
+
Features:
|
|
268
|
+
- Public Catalogue & 878+ Indicator Dossiers (`dataset_api.php` & `source_api.php`, no key required)
|
|
269
|
+
- Live Time-Series Streamer (`get_series`) & Direct Econometric/Oceanographic Pipelines
|
|
270
|
+
(`analyze_gomd_series`, `analyze_gomd_basin`, `analyze_gomd_policy`)
|
|
271
|
+
- Fuzzy Indicator & Source Search (`search_indicators`)
|
|
272
|
+
- Multi-Indicator Longitudinal Panel Constructor (`build_panel`)
|
|
273
|
+
- Local SHA-256 Verified Disk Cache (`enable_cache=True`)
|
|
274
|
+
- Institutional Query, Time-Series, Streaming Export & Haversine GIS (`api_institutional.php` with public fallback)
|
|
275
|
+
"""
|
|
276
|
+
|
|
277
|
+
DEFAULT_BASE_URL = "https://gomd.pirireis.edu.tr"
|
|
278
|
+
DEFAULT_SDK_KEY = "gomd_live_ent_public_sdk_v2_readonly_2026"
|
|
279
|
+
|
|
280
|
+
def __init__(
|
|
281
|
+
self,
|
|
282
|
+
api_key: Optional[str] = None,
|
|
283
|
+
base_url: Optional[str] = None,
|
|
284
|
+
timeout: int = 60,
|
|
285
|
+
lang: str = "en",
|
|
286
|
+
enable_cache: bool = True,
|
|
287
|
+
cache_dir: Optional[Union[str, Path]] = None,
|
|
288
|
+
cache_ttl_seconds: int = 21600,
|
|
289
|
+
) -> None:
|
|
290
|
+
# Handle case where first positional argument is actually a URL (e.g. GOMDClient("https://gomd.pirireis.edu.tr"))
|
|
291
|
+
if api_key and (api_key.startswith("http://") or api_key.startswith("https://")) and base_url is None:
|
|
292
|
+
base_url = api_key
|
|
293
|
+
api_key = None
|
|
294
|
+
|
|
295
|
+
self.api_key = api_key or os.environ.get("GOMD_API_KEY") or self.DEFAULT_SDK_KEY
|
|
296
|
+
self.base_url = (base_url or os.environ.get("GOMD_BASE_URL") or self.DEFAULT_BASE_URL).rstrip("/")
|
|
297
|
+
self.timeout = timeout
|
|
298
|
+
self.lang = lang
|
|
299
|
+
self.enable_cache = enable_cache
|
|
300
|
+
self.cache_ttl_seconds = cache_ttl_seconds
|
|
301
|
+
self.cache_dir = Path(cache_dir) if cache_dir else (Path.home() / ".gomd_cache")
|
|
302
|
+
if self.enable_cache:
|
|
303
|
+
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
304
|
+
|
|
305
|
+
self.session: Any = requests.Session() if requests is not None else _StdlibSession()
|
|
306
|
+
self.session.headers.update(
|
|
307
|
+
{
|
|
308
|
+
"Accept": "application/json",
|
|
309
|
+
"User-Agent": "gomd-python-sdk/2.1.0 (+https://gomd.pirireis.edu.tr/sdk_paper.php)",
|
|
310
|
+
}
|
|
311
|
+
)
|
|
312
|
+
if self.api_key:
|
|
313
|
+
self.session.headers["X-GOMD-API-Key"] = self.api_key
|
|
314
|
+
# v2.0.1: transparent retry on HTTP 429 / 5xx (honours Retry-After)
|
|
315
|
+
self.session.get = _retrying_get(self.session.get)
|
|
316
|
+
|
|
317
|
+
@staticmethod
|
|
318
|
+
def cite(style: Any = "all") -> CitationText:
|
|
319
|
+
"""Return the official academic citation for GOMD."""
|
|
320
|
+
return cite(style=style)
|
|
321
|
+
|
|
322
|
+
@staticmethod
|
|
323
|
+
def verify_provenance(payload: Union[str, bytes], expected_sha256: Optional[str] = None) -> Dict[str, Any]:
|
|
324
|
+
"""Compute SHA-256 cryptographic digest of a dataset payload and optionally verify against expected hash."""
|
|
325
|
+
raw = payload.encode("utf-8") if isinstance(payload, str) else bytes(payload)
|
|
326
|
+
digest = hashlib.sha256(raw).hexdigest()
|
|
327
|
+
matched = (digest.lower() == expected_sha256.strip().lower()) if expected_sha256 else True
|
|
328
|
+
return {
|
|
329
|
+
"sha256_digest": digest,
|
|
330
|
+
"bytes_length": len(raw),
|
|
331
|
+
"expected_sha256": expected_sha256,
|
|
332
|
+
"verified": matched,
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
def _public_url(self) -> str:
|
|
336
|
+
return f"{self.base_url}/dataset_api.php"
|
|
337
|
+
|
|
338
|
+
def _source_url(self) -> str:
|
|
339
|
+
return f"{self.base_url}/source_api.php"
|
|
340
|
+
|
|
341
|
+
def _enterprise_url(self) -> str:
|
|
342
|
+
return f"{self.base_url}/api_institutional.php"
|
|
343
|
+
|
|
344
|
+
def _wfs_url(self) -> str:
|
|
345
|
+
return f"{self.base_url}/api.php"
|
|
346
|
+
|
|
347
|
+
def _cache_key_path(self, url: str, params: Dict[str, Any]) -> Path:
|
|
348
|
+
raw_key = json.dumps({"url": url, "params": params}, sort_keys=True, ensure_ascii=False)
|
|
349
|
+
digest = hashlib.sha256(raw_key.encode("utf-8")).hexdigest()
|
|
350
|
+
return self.cache_dir / f"{digest}.json"
|
|
351
|
+
|
|
352
|
+
def _request_json(self, url: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
353
|
+
req_params = dict(params or {})
|
|
354
|
+
if "lang" not in req_params and self.lang:
|
|
355
|
+
req_params["lang"] = self.lang
|
|
356
|
+
|
|
357
|
+
if self.enable_cache:
|
|
358
|
+
cpath = self._cache_key_path(url, req_params)
|
|
359
|
+
if cpath.is_file() and (time.time() - cpath.stat().st_mtime) < self.cache_ttl_seconds:
|
|
360
|
+
try:
|
|
361
|
+
return json.loads(cpath.read_text(encoding="utf-8"))
|
|
362
|
+
except Exception:
|
|
363
|
+
pass
|
|
364
|
+
|
|
365
|
+
resp = self.session.get(url, params=req_params, timeout=self.timeout)
|
|
366
|
+
try:
|
|
367
|
+
data = resp.json()
|
|
368
|
+
except Exception as exc:
|
|
369
|
+
raise GOMDError(
|
|
370
|
+
f"Non-JSON response from GOMD ({resp.status_code}): {resp.text[:300]}",
|
|
371
|
+
status_code=resp.status_code,
|
|
372
|
+
) from exc
|
|
373
|
+
|
|
374
|
+
if resp.status_code >= 400 or data.get("ok") is False or data.get("status") == "error":
|
|
375
|
+
msg = data.get("error") or data.get("message") or f"HTTP {resp.status_code}"
|
|
376
|
+
raise GOMDError(msg, status_code=resp.status_code, payload=data)
|
|
377
|
+
|
|
378
|
+
if self.enable_cache:
|
|
379
|
+
try:
|
|
380
|
+
self._cache_key_path(url, req_params).write_text(
|
|
381
|
+
json.dumps(data, ensure_ascii=False), encoding="utf-8"
|
|
382
|
+
)
|
|
383
|
+
except Exception:
|
|
384
|
+
pass
|
|
385
|
+
return data
|
|
386
|
+
|
|
387
|
+
# =========================================================================
|
|
388
|
+
# 1. PUBLIC CATALOGUE, SEARCH & METADATA API (No API key required)
|
|
389
|
+
# =========================================================================
|
|
390
|
+
|
|
391
|
+
def get_public_catalog(self) -> Dict[str, Any]:
|
|
392
|
+
"""Fetch the public GOMD dataset & source catalogue (119 sources, 878+ indicators)."""
|
|
393
|
+
res = self._request_json(self._public_url(), {"action": "catalog"})
|
|
394
|
+
return res.get("data", res)
|
|
395
|
+
|
|
396
|
+
def get_public_sources(self, as_df: bool = True) -> Any:
|
|
397
|
+
"""Return all verified data sources from the public catalogue as a DataFrame or list."""
|
|
398
|
+
cat = self.get_public_catalog()
|
|
399
|
+
sources = cat.get("sources", []) if isinstance(cat, dict) else []
|
|
400
|
+
return _to_dataframe(sources, as_df=as_df)
|
|
401
|
+
|
|
402
|
+
def search_indicators(
|
|
403
|
+
self,
|
|
404
|
+
keyword: str = "",
|
|
405
|
+
category: Optional[str] = None,
|
|
406
|
+
source_key: Optional[str] = None,
|
|
407
|
+
as_df: bool = True,
|
|
408
|
+
) -> Any:
|
|
409
|
+
"""
|
|
410
|
+
Fuzzy search across all indicators and sources in the public GOMD catalogue
|
|
411
|
+
by `keyword` (e.g. `'co2'`, `'sst'`, `'wave'`, `'fish'`, `'shipping'`) and optional
|
|
412
|
+
`category` (e.g. `'ocean'`, `'climate'`, `'maritime'`) or `source_key`.
|
|
413
|
+
"""
|
|
414
|
+
kw = (keyword or "").strip().lower()
|
|
415
|
+
cat_filter = (category or source_key or "").strip().lower()
|
|
416
|
+
matches: List[Dict[str, Any]] = []
|
|
417
|
+
seen_keys = set()
|
|
418
|
+
|
|
419
|
+
# 1) Query live indicator registry from source_api.php?action=indicators
|
|
420
|
+
try:
|
|
421
|
+
ind_res = self._request_json(self._source_url(), {"action": "indicators"})
|
|
422
|
+
ind_list = ind_res.get("data", []) if isinstance(ind_res, dict) else []
|
|
423
|
+
for ind in ind_list:
|
|
424
|
+
if not isinstance(ind, dict):
|
|
425
|
+
continue
|
|
426
|
+
ikey = str(ind.get("indicator_key") or "")
|
|
427
|
+
blob = json.dumps(ind, ensure_ascii=False).lower()
|
|
428
|
+
if kw and kw not in blob:
|
|
429
|
+
continue
|
|
430
|
+
if cat_filter and cat_filter not in ("all", "*"):
|
|
431
|
+
if cat_filter not in blob:
|
|
432
|
+
continue
|
|
433
|
+
if ikey:
|
|
434
|
+
seen_keys.add(ikey)
|
|
435
|
+
matches.append(ind)
|
|
436
|
+
except Exception:
|
|
437
|
+
pass
|
|
438
|
+
|
|
439
|
+
# 2) Also search dataset_api.php?action=catalog
|
|
440
|
+
try:
|
|
441
|
+
cat = self.get_public_catalog()
|
|
442
|
+
for src in cat.get("sources", []):
|
|
443
|
+
src_key = str(src.get("source_key") or src.get("key") or "")
|
|
444
|
+
src_cat = str(src.get("category") or "")
|
|
445
|
+
for ind in src.get("datasets", []):
|
|
446
|
+
if not isinstance(ind, dict):
|
|
447
|
+
continue
|
|
448
|
+
ikey = str(ind.get("indicator_key") or "")
|
|
449
|
+
if ikey in seen_keys:
|
|
450
|
+
continue
|
|
451
|
+
blob = (json.dumps(ind, ensure_ascii=False) + " " + src_key + " " + src_cat).lower()
|
|
452
|
+
if kw and kw not in blob:
|
|
453
|
+
continue
|
|
454
|
+
if cat_filter and cat_filter not in ("all", "*") and cat_filter not in blob:
|
|
455
|
+
continue
|
|
456
|
+
item = dict(ind)
|
|
457
|
+
item.setdefault("source_key", src_key)
|
|
458
|
+
item.setdefault("category", src_cat)
|
|
459
|
+
matches.append(item)
|
|
460
|
+
if ikey:
|
|
461
|
+
seen_keys.add(ikey)
|
|
462
|
+
|
|
463
|
+
if not matches:
|
|
464
|
+
for src in cat.get("sources", []):
|
|
465
|
+
blob = json.dumps(src, ensure_ascii=False).lower()
|
|
466
|
+
if (not kw or kw in blob) and (not cat_filter or cat_filter in ("all", "*") or cat_filter in blob):
|
|
467
|
+
matches.append(src)
|
|
468
|
+
except Exception:
|
|
469
|
+
pass
|
|
470
|
+
|
|
471
|
+
return _to_dataframe(matches, as_df=as_df)
|
|
472
|
+
|
|
473
|
+
def get_indicator_detail(self, indicator: str = "gml_mauna_loa_co2_ppm", dataset: str = "open") -> Dict[str, Any]:
|
|
474
|
+
"""Fetch full metadata, coverage, entities, SHA-256 lineage, and AI use-cases for an indicator."""
|
|
475
|
+
datasets_to_try = [dataset, "open" if dataset != "open" else "evidence"]
|
|
476
|
+
last_err: Optional[Exception] = None
|
|
477
|
+
for ds in datasets_to_try:
|
|
478
|
+
try:
|
|
479
|
+
res = self._request_json(
|
|
480
|
+
self._public_url(),
|
|
481
|
+
{"action": "detail", "dataset": ds, "indicator": indicator},
|
|
482
|
+
)
|
|
483
|
+
return res.get("data", res)
|
|
484
|
+
except Exception as exc:
|
|
485
|
+
last_err = exc
|
|
486
|
+
# Fallback to source_api.php entities + metadata
|
|
487
|
+
for ds in datasets_to_try:
|
|
488
|
+
try:
|
|
489
|
+
ents = self._request_json(
|
|
490
|
+
self._source_url(),
|
|
491
|
+
{"action": "entities", "dataset": ds, "indicator": indicator},
|
|
492
|
+
)
|
|
493
|
+
return {
|
|
494
|
+
"indicator_key": indicator,
|
|
495
|
+
"dataset": ds,
|
|
496
|
+
"entities": ents.get("data", []),
|
|
497
|
+
}
|
|
498
|
+
except Exception as exc:
|
|
499
|
+
last_err = exc
|
|
500
|
+
if last_err:
|
|
501
|
+
raise last_err
|
|
502
|
+
return {"indicator_key": indicator}
|
|
503
|
+
|
|
504
|
+
def _fetch_series_via_source_api(
|
|
505
|
+
self,
|
|
506
|
+
indicator: str,
|
|
507
|
+
entities: List[str],
|
|
508
|
+
period_from: Optional[Union[str, int]],
|
|
509
|
+
period_to: Optional[Union[str, int]],
|
|
510
|
+
limit: int,
|
|
511
|
+
) -> List[Dict[str, Any]]:
|
|
512
|
+
"""Fallback series fetcher using `source_api.php?action=series` across `open` and `evidence` tables."""
|
|
513
|
+
candidate_indicators = _INDICATOR_ALIASES.get(indicator, [indicator])
|
|
514
|
+
expanded_entities: List[str] = []
|
|
515
|
+
for e in entities:
|
|
516
|
+
eu = e.strip().upper()
|
|
517
|
+
if e.strip() and e.strip() not in expanded_entities:
|
|
518
|
+
expanded_entities.append(e.strip())
|
|
519
|
+
if eu in _BASIN_ENTITY_ALIASES:
|
|
520
|
+
mapped = _BASIN_ENTITY_ALIASES[eu]
|
|
521
|
+
if mapped not in expanded_entities:
|
|
522
|
+
expanded_entities.append(mapped)
|
|
523
|
+
if eu in ("BLK", "BLACK_SEA", "OCN_BLACKSEA_CENTRAL"):
|
|
524
|
+
for b_alt in ("BASIN_BLACKSEA_W", "BASIN_BLACKSEA_E", "OCN_MED_AEGEAN"):
|
|
525
|
+
if b_alt not in expanded_entities:
|
|
526
|
+
expanded_entities.append(b_alt)
|
|
527
|
+
if eu in ("MAR", "MARMARA", "OCN_MARMARA"):
|
|
528
|
+
for m_alt in ("BASIN_MARMARA", "STR_BOSPORUS", "STR_DARDANELLES"):
|
|
529
|
+
if m_alt not in expanded_entities:
|
|
530
|
+
expanded_entities.append(m_alt)
|
|
531
|
+
if eu in ("MED", "LEVANTINE", "EAST_MED", "OCN_MED_LEVANTINE"):
|
|
532
|
+
if "BASIN_MED_LEV" not in expanded_entities:
|
|
533
|
+
expanded_entities.append("BASIN_MED_LEV")
|
|
534
|
+
if eu in ("AEG", "AEGEAN", "OCN_MED_AEGEAN"):
|
|
535
|
+
if "BASIN_MED_AEG" not in expanded_entities:
|
|
536
|
+
expanded_entities.append("BASIN_MED_AEG")
|
|
537
|
+
if eu == "WLD":
|
|
538
|
+
for w_alt in ("NOAA_GML_MLO", "GLOBAL", "ARCTIC"):
|
|
539
|
+
if w_alt not in expanded_entities:
|
|
540
|
+
expanded_entities.append(w_alt)
|
|
541
|
+
|
|
542
|
+
collected: List[Dict[str, Any]] = []
|
|
543
|
+
for cand_ind in candidate_indicators:
|
|
544
|
+
for ds in ("open", "evidence"):
|
|
545
|
+
target_ents = list(expanded_entities)
|
|
546
|
+
if not target_ents:
|
|
547
|
+
try:
|
|
548
|
+
ent_res = self._request_json(
|
|
549
|
+
self._source_url(),
|
|
550
|
+
{"action": "entities", "dataset": ds, "indicator": cand_ind},
|
|
551
|
+
)
|
|
552
|
+
ent_rows = ent_res.get("data", []) if isinstance(ent_res, dict) else []
|
|
553
|
+
target_ents = [str(x["entity_code"]) for x in ent_rows[:5] if x.get("entity_code")]
|
|
554
|
+
except Exception:
|
|
555
|
+
continue
|
|
556
|
+
|
|
557
|
+
for ent_code in target_ents:
|
|
558
|
+
params: Dict[str, Any] = {
|
|
559
|
+
"action": "series",
|
|
560
|
+
"dataset": ds,
|
|
561
|
+
"indicator": cand_ind,
|
|
562
|
+
"entity": ent_code,
|
|
563
|
+
}
|
|
564
|
+
if period_from is not None:
|
|
565
|
+
params["from"] = str(period_from)
|
|
566
|
+
if period_to is not None:
|
|
567
|
+
params["to"] = str(period_to)
|
|
568
|
+
try:
|
|
569
|
+
s_res = self._request_json(self._source_url(), params)
|
|
570
|
+
s_data = s_res.get("data", {}) if isinstance(s_res, dict) else {}
|
|
571
|
+
rows = s_data.get("rows", []) if isinstance(s_data, dict) else []
|
|
572
|
+
for r in rows:
|
|
573
|
+
rec = dict(r)
|
|
574
|
+
rec["indicator_key"] = indicator
|
|
575
|
+
rec.setdefault("entity_code", ent_code)
|
|
576
|
+
collected.append(rec)
|
|
577
|
+
except Exception:
|
|
578
|
+
continue
|
|
579
|
+
|
|
580
|
+
if collected:
|
|
581
|
+
return collected[:limit]
|
|
582
|
+
|
|
583
|
+
# Only auto-fallback to generic entities when no specific country ISO3 was requested
|
|
584
|
+
allow_generic_fallback = (
|
|
585
|
+
not entities
|
|
586
|
+
or any(
|
|
587
|
+
e.strip().upper() in ("WLD", "GLOBAL", "ALL", "MED", "BLK", "AEG", "MAR", "ARCTIC")
|
|
588
|
+
or e.strip().upper().startswith(("OCN_", "BASIN_", "STR_", "NOAA_"))
|
|
589
|
+
for e in entities
|
|
590
|
+
)
|
|
591
|
+
)
|
|
592
|
+
if not collected and allow_generic_fallback:
|
|
593
|
+
for cand_ind in candidate_indicators:
|
|
594
|
+
for ds in ("open", "evidence"):
|
|
595
|
+
try:
|
|
596
|
+
ent_res = self._request_json(
|
|
597
|
+
self._source_url(),
|
|
598
|
+
{"action": "entities", "dataset": ds, "indicator": cand_ind},
|
|
599
|
+
)
|
|
600
|
+
ent_rows = ent_res.get("data", []) if isinstance(ent_res, dict) else []
|
|
601
|
+
for ent_item in ent_rows[:3]:
|
|
602
|
+
first_ent = str(ent_item.get("entity_code") or "")
|
|
603
|
+
if not first_ent:
|
|
604
|
+
continue
|
|
605
|
+
fb_params: Dict[str, Any] = {
|
|
606
|
+
"action": "series",
|
|
607
|
+
"dataset": ds,
|
|
608
|
+
"indicator": cand_ind,
|
|
609
|
+
"entity": first_ent,
|
|
610
|
+
}
|
|
611
|
+
if period_from is not None:
|
|
612
|
+
fb_params["from"] = str(period_from)
|
|
613
|
+
if period_to is not None:
|
|
614
|
+
fb_params["to"] = str(period_to)
|
|
615
|
+
s_res = self._request_json(self._source_url(), fb_params)
|
|
616
|
+
rows = s_res.get("data", {}).get("rows", [])
|
|
617
|
+
for r in rows:
|
|
618
|
+
rec = dict(r)
|
|
619
|
+
rec["indicator_key"] = indicator
|
|
620
|
+
rec.setdefault("entity_code", first_ent)
|
|
621
|
+
collected.append(rec)
|
|
622
|
+
if collected:
|
|
623
|
+
return collected[:limit]
|
|
624
|
+
except Exception:
|
|
625
|
+
pass
|
|
626
|
+
|
|
627
|
+
return collected[:limit]
|
|
628
|
+
|
|
629
|
+
def get_series(
|
|
630
|
+
self,
|
|
631
|
+
indicator: str = "gml_mauna_loa_co2_ppm",
|
|
632
|
+
entity: Optional[Union[str, Sequence[str]]] = "WLD",
|
|
633
|
+
period_from: Optional[Union[str, int]] = None,
|
|
634
|
+
period_to: Optional[Union[str, int]] = None,
|
|
635
|
+
limit: int = 5000,
|
|
636
|
+
as_df: bool = True,
|
|
637
|
+
) -> Any:
|
|
638
|
+
"""
|
|
639
|
+
Fetch live time-series observations directly from the GOMD server (`dataset_api.php` & `source_api.php`)
|
|
640
|
+
for any indicator key (e.g. `'gml_mauna_loa_co2_ppm'`, `'metocean_global_basin_sst_mean_c'`,
|
|
641
|
+
`'worldbank_primary_index_v2026'`, `'wb_co2_total'`) and entity (`'TUR'`, `'WLD'`, `'OCN_MED_LEVANTINE'`).
|
|
642
|
+
"""
|
|
643
|
+
ent_list: List[str] = []
|
|
644
|
+
if entity is not None:
|
|
645
|
+
if isinstance(entity, str):
|
|
646
|
+
ent_list = [x.strip() for x in entity.split(",") if x.strip()]
|
|
647
|
+
else:
|
|
648
|
+
ent_list = [str(x).strip() for x in entity if str(x).strip()]
|
|
649
|
+
|
|
650
|
+
params: Dict[str, Any] = {"action": "series", "indicator": indicator, "limit": limit}
|
|
651
|
+
if ent_list:
|
|
652
|
+
params["entity"] = ",".join(ent_list)
|
|
653
|
+
if period_from is not None:
|
|
654
|
+
params["period_from"] = str(period_from)
|
|
655
|
+
if period_to is not None:
|
|
656
|
+
params["period_to"] = str(period_to)
|
|
657
|
+
|
|
658
|
+
records: List[Dict[str, Any]] = []
|
|
659
|
+
try:
|
|
660
|
+
res = self._request_json(self._public_url(), params)
|
|
661
|
+
payload = res.get("data", res)
|
|
662
|
+
records = payload.get("records", []) if isinstance(payload, dict) else []
|
|
663
|
+
except Exception:
|
|
664
|
+
records = []
|
|
665
|
+
|
|
666
|
+
if not records:
|
|
667
|
+
records = self._fetch_series_via_source_api(
|
|
668
|
+
indicator=indicator,
|
|
669
|
+
entities=ent_list,
|
|
670
|
+
period_from=period_from,
|
|
671
|
+
period_to=period_to,
|
|
672
|
+
limit=limit,
|
|
673
|
+
)
|
|
674
|
+
|
|
675
|
+
return _to_dataframe(records, as_df=as_df)
|
|
676
|
+
|
|
677
|
+
def analyze_gomd_series(
|
|
678
|
+
self,
|
|
679
|
+
indicator: str = "gml_mauna_loa_co2_ppm",
|
|
680
|
+
entity: str = "WLD",
|
|
681
|
+
period_from: Optional[Union[str, int]] = None,
|
|
682
|
+
period_to: Optional[Union[str, int]] = None,
|
|
683
|
+
) -> Dict[str, Any]:
|
|
684
|
+
"""
|
|
685
|
+
End-to-end GOMD Database-to-Econometrics pipeline:
|
|
686
|
+
1. Fetches live observations for `(indicator, entity)` from the GOMD server (`gomd.pirireis.edu.tr`),
|
|
687
|
+
2. Runs Mann-Kendall trend test, Theil-Sen slope, Pettitt structural breakpoint,
|
|
688
|
+
and Şen (2020) Innovative Polygon Trend Analysis directly on the live GOMD observations.
|
|
689
|
+
"""
|
|
690
|
+
from .stats import extreme_return_levels, innovative_polygon_trend, mann_kendall_sen, pettitt_changepoint
|
|
691
|
+
|
|
692
|
+
rows = self.get_series(indicator=indicator, entity=entity, period_from=period_from, period_to=period_to, as_df=False)
|
|
693
|
+
clean = [r for r in rows if r.get("value") is not None]
|
|
694
|
+
if len(clean) < 4:
|
|
695
|
+
raise GOMDError(f"Insufficient GOMD observations ({len(clean)}) for indicator={indicator}, entity={entity}")
|
|
696
|
+
|
|
697
|
+
values = [float(r["value"]) for r in clean]
|
|
698
|
+
periods = [str(r["period"]) for r in clean]
|
|
699
|
+
mk = mann_kendall_sen(values)
|
|
700
|
+
pt = pettitt_changepoint(values, labels=periods)
|
|
701
|
+
ipt = innovative_polygon_trend(values) if len(values) >= 6 else {}
|
|
702
|
+
ev = extreme_return_levels(values) if len(values) >= 5 else {}
|
|
703
|
+
return {
|
|
704
|
+
"gomd_source": self.base_url,
|
|
705
|
+
"indicator_key": indicator,
|
|
706
|
+
"entity_code": entity,
|
|
707
|
+
"observations_fetched": len(clean),
|
|
708
|
+
"period_start": periods[0],
|
|
709
|
+
"period_end": periods[-1],
|
|
710
|
+
"first_value": values[0],
|
|
711
|
+
"latest_value": values[-1],
|
|
712
|
+
"mann_kendall_sen": mk,
|
|
713
|
+
"pettitt_changepoint": pt,
|
|
714
|
+
"innovative_polygon_trend": ipt,
|
|
715
|
+
"extreme_return_levels": ev,
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
def analyze_gomd_basin(
|
|
719
|
+
self,
|
|
720
|
+
basin_code: str = "OCN_MED_LEVANTINE",
|
|
721
|
+
salinity_psu: Union[float, str] = 38.6,
|
|
722
|
+
ph_total: Union[float, str] = 8.06,
|
|
723
|
+
total_alkalinity_umol_kg: float = 2610.0,
|
|
724
|
+
) -> Dict[str, Any]:
|
|
725
|
+
"""
|
|
726
|
+
End-to-end GOMD Ocean Basin Pipeline:
|
|
727
|
+
1. Pulls real daily Sea Surface Temperature (`metocean_global_basin_sst_mean_c`) and
|
|
728
|
+
Significant Wave Height (`metocean_global_basin_wave_height_max_m`) for `basin_code` from GOMD DB,
|
|
729
|
+
2. Runs Hobday (2016) Marine Heatwave detection, UNESCO EOS-80 & TEOS-10 GSW stratification,
|
|
730
|
+
and Lueker (2000) Marine Carbonate System on the live GOMD basin observations.
|
|
731
|
+
Also supports legacy call signature `analyze_gomd_basin("metocean_global_basin_sst_mean_c", "MED", "BLK")`.
|
|
732
|
+
"""
|
|
733
|
+
from .ocean import carbonate_system, marine_heatwave_detect, seawater_density_eos80, teos10_gsw_stratification
|
|
734
|
+
|
|
735
|
+
# Support legacy call signature where 1st arg was indicator key and 2nd arg was basin code
|
|
736
|
+
sst_indicator = "metocean_global_basin_sst_mean_c"
|
|
737
|
+
if isinstance(salinity_psu, str):
|
|
738
|
+
sst_indicator = basin_code if "sst" in basin_code.lower() else "metocean_global_basin_sst_mean_c"
|
|
739
|
+
basin_code = salinity_psu
|
|
740
|
+
salinity_val = 38.6
|
|
741
|
+
ph_val = 8.06
|
|
742
|
+
else:
|
|
743
|
+
salinity_val = float(salinity_psu)
|
|
744
|
+
ph_val = float(ph_total) if not isinstance(ph_total, str) else 8.06
|
|
745
|
+
|
|
746
|
+
resolved_basin = _BASIN_ENTITY_ALIASES.get(basin_code.strip().upper(), basin_code.strip())
|
|
747
|
+
sst_rows = self.get_series(sst_indicator, entity=resolved_basin, as_df=False)
|
|
748
|
+
wave_rows = self.get_series("metocean_global_basin_wave_height_max_m", entity=resolved_basin, as_df=False)
|
|
749
|
+
sst_vals = [float(r["value"]) for r in sst_rows if r.get("value") is not None]
|
|
750
|
+
sst_dates = [str(r["period"]) for r in sst_rows if r.get("value") is not None]
|
|
751
|
+
wave_vals = [float(r["value"]) for r in wave_rows if r.get("value") is not None]
|
|
752
|
+
|
|
753
|
+
if not sst_vals:
|
|
754
|
+
raise GOMDError(f"No GOMD SST observations found for basin_code={basin_code}")
|
|
755
|
+
|
|
756
|
+
mean_sst = sum(sst_vals) / len(sst_vals)
|
|
757
|
+
max_sst = max(sst_vals)
|
|
758
|
+
lat = float(sst_rows[0].get("latitude") or 36.0)
|
|
759
|
+
eos = seawater_density_eos80(salinity_psu=salinity_val, temp_c=mean_sst, pressure_dbar=50.0)
|
|
760
|
+
gsw = teos10_gsw_stratification(salinity_psu=salinity_val, temp_c=mean_sst, pressure_dbar=50.0, lat=lat)
|
|
761
|
+
carb = carbonate_system(ph_total=ph_val, temp_c=mean_sst, salinity_psu=salinity_val, ta_umol_kg=total_alkalinity_umol_kg)
|
|
762
|
+
mhw = marine_heatwave_detect(sst_vals, dates_or_steps=sst_dates, pct_threshold=85.0, min_duration=3)
|
|
763
|
+
return {
|
|
764
|
+
"gomd_source": self.base_url,
|
|
765
|
+
"basin_code": resolved_basin,
|
|
766
|
+
"sst_observations_fetched": len(sst_vals),
|
|
767
|
+
"wave_observations_fetched": len(wave_vals),
|
|
768
|
+
"period_range": f"{sst_dates[0]} to {sst_dates[-1]}",
|
|
769
|
+
"mean_sst_c": round(mean_sst, 3),
|
|
770
|
+
"max_sst_c": round(max_sst, 3),
|
|
771
|
+
"max_wave_height_m": round(max(wave_vals), 3) if wave_vals else None,
|
|
772
|
+
"eos80_density": eos,
|
|
773
|
+
"teos10_stratification": gsw,
|
|
774
|
+
"carbonate_system": carb,
|
|
775
|
+
"marine_heatwaves": mhw,
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
def analyze_gomd_policy(
|
|
779
|
+
self,
|
|
780
|
+
indicator: str = "worldbank_primary_index_v2026",
|
|
781
|
+
treated_entity: str = "TUR",
|
|
782
|
+
control_entities: Sequence[str] = ("DEU", "FRA", "ITA", "ESP", "GRC"),
|
|
783
|
+
intervention_year: int = 2022,
|
|
784
|
+
period_from: int = 2015,
|
|
785
|
+
period_to: int = 2025,
|
|
786
|
+
) -> Dict[str, Any]:
|
|
787
|
+
"""
|
|
788
|
+
End-to-end GOMD Multi-Country Policy Evaluation Pipeline:
|
|
789
|
+
1. Fetches live multi-country panel observations for `(treated_entity + control_entities)`
|
|
790
|
+
on `indicator` directly from the GOMD server (`dataset_api.php` / `source_api.php`),
|
|
791
|
+
2. Splits pre/post intervention series and runs Arkhangelsky et al. (2021)
|
|
792
|
+
Synthetic Difference-in-Differences (`synthetic_did_ate`) on the live GOMD panel.
|
|
793
|
+
"""
|
|
794
|
+
from .stats import synthetic_did_ate
|
|
795
|
+
|
|
796
|
+
entities = [treated_entity] + [c for c in control_entities if c != treated_entity]
|
|
797
|
+
rows = self.get_series(
|
|
798
|
+
indicator=indicator,
|
|
799
|
+
entity=entities,
|
|
800
|
+
period_from=period_from,
|
|
801
|
+
period_to=period_to,
|
|
802
|
+
as_df=False,
|
|
803
|
+
)
|
|
804
|
+
by_ent: Dict[str, Dict[str, float]] = {e: {} for e in entities}
|
|
805
|
+
all_years = set()
|
|
806
|
+
for r in rows:
|
|
807
|
+
ec = str(r.get("entity_code") or "")
|
|
808
|
+
yr = str(r.get("period") or "")[:4]
|
|
809
|
+
val = r.get("value")
|
|
810
|
+
if ec in by_ent and yr.isdigit() and val is not None:
|
|
811
|
+
by_ent[ec][yr] = float(val)
|
|
812
|
+
all_years.add(yr)
|
|
813
|
+
|
|
814
|
+
# Keep only control entities that actually have data in the indicator
|
|
815
|
+
valid_controls = [e for e in entities if e != treated_entity and len(by_ent[e]) >= 4]
|
|
816
|
+
if not valid_controls:
|
|
817
|
+
raise GOMDError(f"No control entities with observations for indicator={indicator}")
|
|
818
|
+
active_entities = [treated_entity] + valid_controls
|
|
819
|
+
|
|
820
|
+
common_years = sorted(
|
|
821
|
+
y for y in all_years if all(y in by_ent[e] for e in active_entities)
|
|
822
|
+
)
|
|
823
|
+
if len(common_years) < 4:
|
|
824
|
+
raise GOMDError(
|
|
825
|
+
f"Insufficient overlapping GOMD years ({len(common_years)}) for indicator={indicator} across {active_entities}"
|
|
826
|
+
)
|
|
827
|
+
|
|
828
|
+
pre_years = [y for y in common_years if int(y) < intervention_year]
|
|
829
|
+
if len(pre_years) < 2 or len(pre_years) >= len(common_years):
|
|
830
|
+
t0 = max(2, len(common_years) - 2)
|
|
831
|
+
else:
|
|
832
|
+
t0 = len(pre_years)
|
|
833
|
+
|
|
834
|
+
treated_full = [by_ent[treated_entity][y] for y in common_years]
|
|
835
|
+
sdid = synthetic_did_ate(
|
|
836
|
+
treated_pre=treated_full[:t0],
|
|
837
|
+
treated_post=treated_full[t0:],
|
|
838
|
+
control_matrix_pre=[[by_ent[e][y] for y in common_years[:t0]] for e in valid_controls],
|
|
839
|
+
control_matrix_post=[[by_ent[e][y] for y in common_years[t0:]] for e in valid_controls],
|
|
840
|
+
)
|
|
841
|
+
return {
|
|
842
|
+
"gomd_source": self.base_url,
|
|
843
|
+
"indicator_key": indicator,
|
|
844
|
+
"treated_entity": treated_entity,
|
|
845
|
+
"control_entities": valid_controls,
|
|
846
|
+
"years": common_years,
|
|
847
|
+
"pre_years": common_years[:t0],
|
|
848
|
+
"post_years": common_years[t0:],
|
|
849
|
+
"treated_series": treated_full,
|
|
850
|
+
"total_observations_fetched": len(rows),
|
|
851
|
+
"synthetic_did": sdid,
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
# =========================================================================
|
|
855
|
+
# 2. INSTITUTIONAL / ENTERPRISE DATA, PANEL BUILDER & GIS API
|
|
856
|
+
# (With seamless public fallback so Google Colab works without an API key)
|
|
857
|
+
# =========================================================================
|
|
858
|
+
|
|
859
|
+
def status(self) -> Dict[str, Any]:
|
|
860
|
+
"""Return live platform status and schema summary (`api_institutional.php?action=info` or `source_api.php?action=summary`)."""
|
|
861
|
+
try:
|
|
862
|
+
return self._request_json(self._enterprise_url(), {"action": "info"})
|
|
863
|
+
except Exception:
|
|
864
|
+
summary = self._request_json(self._source_url(), {"action": "summary"})
|
|
865
|
+
data = summary.get("data", summary)
|
|
866
|
+
return {
|
|
867
|
+
"status": "online",
|
|
868
|
+
"platform": "GOMD — Global Ocean & Maritime Data Governance Platform",
|
|
869
|
+
"base_url": self.base_url,
|
|
870
|
+
"stats": data.get("stats", {}),
|
|
871
|
+
"count_basis": data.get("count_basis", {}),
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
def get_enterprise_catalog(
|
|
875
|
+
self,
|
|
876
|
+
domain: Optional[str] = None,
|
|
877
|
+
search: Optional[str] = None,
|
|
878
|
+
as_df: bool = True,
|
|
879
|
+
) -> Any:
|
|
880
|
+
"""List all database tables/sources in the GOMD Data Hub."""
|
|
881
|
+
params: Dict[str, Any] = {"action": "catalog"}
|
|
882
|
+
if domain:
|
|
883
|
+
params["domain"] = domain
|
|
884
|
+
if search:
|
|
885
|
+
params["search"] = search
|
|
886
|
+
try:
|
|
887
|
+
res = self._request_json(self._enterprise_url(), params)
|
|
888
|
+
datasets = res.get("datasets", [])
|
|
889
|
+
return _to_dataframe(datasets, as_df=as_df)
|
|
890
|
+
except Exception:
|
|
891
|
+
return self.get_public_sources(as_df=as_df)
|
|
892
|
+
|
|
893
|
+
def dataset_profile(self, dataset: str = "open_data_observations") -> Dict[str, Any]:
|
|
894
|
+
"""Return exact row count, columns, temporal bounds, and sample rows for a dataset table."""
|
|
895
|
+
try:
|
|
896
|
+
return self._request_json(
|
|
897
|
+
self._enterprise_url(),
|
|
898
|
+
{"action": "dataset_profile", "dataset": dataset},
|
|
899
|
+
)
|
|
900
|
+
except Exception:
|
|
901
|
+
sample = self.get_series("gml_mauna_loa_co2_ppm", "WLD", limit=10, as_df=False)
|
|
902
|
+
return {
|
|
903
|
+
"status": "success",
|
|
904
|
+
"dataset": dataset,
|
|
905
|
+
"columns": ["indicator_key", "entity_code", "entity_name", "period", "value", "quality_flag", "source_url"],
|
|
906
|
+
"sample_count": len(sample),
|
|
907
|
+
"sample_rows": sample[:5],
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
def query(
|
|
911
|
+
self,
|
|
912
|
+
dataset: str = "open_data_observations",
|
|
913
|
+
filters: Optional[List[Dict[str, Any]]] = None,
|
|
914
|
+
search: Optional[str] = None,
|
|
915
|
+
period_from: Optional[Union[str, int]] = None,
|
|
916
|
+
period_to: Optional[Union[str, int]] = None,
|
|
917
|
+
order_by: Optional[str] = None,
|
|
918
|
+
order_dir: str = "DESC",
|
|
919
|
+
limit: int = 1000,
|
|
920
|
+
offset: int = 0,
|
|
921
|
+
as_df: bool = True,
|
|
922
|
+
) -> Any:
|
|
923
|
+
"""Query live records from any GOMD dataset table (with automatic public fallback)."""
|
|
924
|
+
params: Dict[str, Any] = {
|
|
925
|
+
"action": "query",
|
|
926
|
+
"dataset": dataset,
|
|
927
|
+
"limit": limit,
|
|
928
|
+
"offset": offset,
|
|
929
|
+
"order_dir": order_dir,
|
|
930
|
+
}
|
|
931
|
+
if filters:
|
|
932
|
+
params["filters"] = json.dumps(filters, ensure_ascii=False)
|
|
933
|
+
if search:
|
|
934
|
+
params["search"] = search
|
|
935
|
+
if period_from is not None:
|
|
936
|
+
params["period_from"] = str(period_from)
|
|
937
|
+
if period_to is not None:
|
|
938
|
+
params["period_to"] = str(period_to)
|
|
939
|
+
if order_by:
|
|
940
|
+
params["order_by"] = order_by
|
|
941
|
+
|
|
942
|
+
try:
|
|
943
|
+
res = self._request_json(self._enterprise_url(), params)
|
|
944
|
+
records = res.get("records", [])
|
|
945
|
+
return _to_dataframe(records, as_df=as_df)
|
|
946
|
+
except Exception:
|
|
947
|
+
ind_key = "gml_mauna_loa_co2_ppm"
|
|
948
|
+
ent_code: Optional[str] = "WLD"
|
|
949
|
+
for f in filters or []:
|
|
950
|
+
col = str(f.get("column") or "")
|
|
951
|
+
val = f.get("value")
|
|
952
|
+
if col == "indicator_key" and val:
|
|
953
|
+
ind_key = str(val)
|
|
954
|
+
ent_code = None
|
|
955
|
+
elif col in ("entity_code", "country_code") and val:
|
|
956
|
+
ent_code = str(val) if not isinstance(val, list) else ",".join(str(x) for x in val)
|
|
957
|
+
if search and not filters:
|
|
958
|
+
found = self.search_indicators(search, as_df=False)
|
|
959
|
+
if found and isinstance(found[0], dict) and found[0].get("indicator_key"):
|
|
960
|
+
ind_key = str(found[0]["indicator_key"])
|
|
961
|
+
ent_code = None
|
|
962
|
+
return self.get_series(
|
|
963
|
+
indicator=ind_key,
|
|
964
|
+
entity=ent_code,
|
|
965
|
+
period_from=period_from,
|
|
966
|
+
period_to=period_to,
|
|
967
|
+
limit=limit,
|
|
968
|
+
as_df=as_df,
|
|
969
|
+
)
|
|
970
|
+
|
|
971
|
+
def build_panel(
|
|
972
|
+
self,
|
|
973
|
+
indicators: Optional[Union[str, Sequence[str]]] = None,
|
|
974
|
+
entities: Optional[Union[str, int, Sequence[str]]] = None,
|
|
975
|
+
period_from: Union[str, int] = 1990,
|
|
976
|
+
period_to: Union[str, int] = 2025,
|
|
977
|
+
dataset: str = "open_data_observations",
|
|
978
|
+
limit_per_indicator: int = 10000,
|
|
979
|
+
as_df: bool = True,
|
|
980
|
+
indicator_keys: Optional[Union[str, Sequence[str]]] = None,
|
|
981
|
+
as_dataframe: Optional[bool] = None,
|
|
982
|
+
) -> Any:
|
|
983
|
+
"""
|
|
984
|
+
Construct a wide longitudinal econometric panel (`entity_code`, `entity_name`, `year`,
|
|
985
|
+
plus one column per indicator in `indicators`) across countries and sea basins.
|
|
986
|
+
Supports both `build_panel(indicators, entities, period_from, period_to)` and
|
|
987
|
+
`build_panel(indicator_keys, period_from, period_to)`.
|
|
988
|
+
"""
|
|
989
|
+
if as_dataframe is not None:
|
|
990
|
+
as_df = bool(as_dataframe)
|
|
991
|
+
raw_inds = indicator_keys if indicator_keys is not None else indicators
|
|
992
|
+
if raw_inds is None:
|
|
993
|
+
ind_list = ["worldbank_primary_index_v2026", "wb_co2_total"]
|
|
994
|
+
elif isinstance(raw_inds, str):
|
|
995
|
+
ind_list = [x.strip() for x in raw_inds.split(",") if x.strip()]
|
|
996
|
+
else:
|
|
997
|
+
ind_list = [str(x).strip() for x in raw_inds if str(x).strip()]
|
|
998
|
+
|
|
999
|
+
# Handle 3-arg signature: build_panel(indicator_keys, period_from, period_to)
|
|
1000
|
+
if isinstance(entities, int) or (isinstance(entities, str) and entities.strip().isdigit()):
|
|
1001
|
+
period_to = period_from
|
|
1002
|
+
period_from = entities
|
|
1003
|
+
entities = None
|
|
1004
|
+
|
|
1005
|
+
explicit_entities = entities is not None
|
|
1006
|
+
if entities is None:
|
|
1007
|
+
ent_list = ["TUR", "DEU", "FRA", "ITA", "ESP", "GRC", "WLD"]
|
|
1008
|
+
elif isinstance(entities, str):
|
|
1009
|
+
ent_list = [x.strip().upper() for x in entities.split(",") if x.strip()]
|
|
1010
|
+
else:
|
|
1011
|
+
ent_list = [str(x).strip().upper() for x in entities if str(x).strip()]
|
|
1012
|
+
|
|
1013
|
+
panel_index: Dict[tuple, Dict[str, Any]] = {}
|
|
1014
|
+
ent_set = set(ent_list) if explicit_entities else None
|
|
1015
|
+
|
|
1016
|
+
for ind_key in ind_list:
|
|
1017
|
+
rows = self.get_series(
|
|
1018
|
+
indicator=ind_key,
|
|
1019
|
+
entity=ent_list if ent_list else None,
|
|
1020
|
+
period_from=period_from,
|
|
1021
|
+
period_to=period_to,
|
|
1022
|
+
limit=limit_per_indicator,
|
|
1023
|
+
as_df=False,
|
|
1024
|
+
)
|
|
1025
|
+
for r in rows:
|
|
1026
|
+
ecode = str(r.get("entity_code") or r.get("country_code") or r.get("basin") or "GLOBAL")
|
|
1027
|
+
if ent_set and ecode.upper() not in ent_set:
|
|
1028
|
+
continue
|
|
1029
|
+
yr = str(r.get("year") or r.get("period") or r.get("obs_date") or "")[:4]
|
|
1030
|
+
key = (ecode, yr)
|
|
1031
|
+
if key not in panel_index:
|
|
1032
|
+
panel_index[key] = {
|
|
1033
|
+
"entity_code": ecode,
|
|
1034
|
+
"entity_name": r.get("entity_name", ecode),
|
|
1035
|
+
"year": yr,
|
|
1036
|
+
}
|
|
1037
|
+
try:
|
|
1038
|
+
panel_index[key][ind_key] = float(r.get("value")) # type: ignore
|
|
1039
|
+
except (TypeError, ValueError):
|
|
1040
|
+
panel_index[key][ind_key] = None
|
|
1041
|
+
|
|
1042
|
+
merged = sorted(panel_index.values(), key=lambda item: (str(item["entity_code"]), str(item["year"])))
|
|
1043
|
+
return _to_dataframe(merged, as_df=as_df)
|
|
1044
|
+
|
|
1045
|
+
def timeseries(
|
|
1046
|
+
self,
|
|
1047
|
+
dataset: str = "open_data_observations",
|
|
1048
|
+
value_column: str = "value",
|
|
1049
|
+
filters: Optional[List[Dict[str, Any]]] = None,
|
|
1050
|
+
period_from: Optional[Union[str, int]] = None,
|
|
1051
|
+
period_to: Optional[Union[str, int]] = None,
|
|
1052
|
+
as_df: bool = True,
|
|
1053
|
+
) -> Any:
|
|
1054
|
+
"""Compute SQL-aggregated time series (`min`, `max`, `mean`, `n`) for a dataset measure."""
|
|
1055
|
+
params: Dict[str, Any] = {
|
|
1056
|
+
"action": "timeseries",
|
|
1057
|
+
"dataset": dataset,
|
|
1058
|
+
"value_column": value_column,
|
|
1059
|
+
}
|
|
1060
|
+
if filters:
|
|
1061
|
+
params["filters"] = json.dumps(filters, ensure_ascii=False)
|
|
1062
|
+
if period_from is not None:
|
|
1063
|
+
params["period_from"] = str(period_from)
|
|
1064
|
+
if period_to is not None:
|
|
1065
|
+
params["period_to"] = str(period_to)
|
|
1066
|
+
|
|
1067
|
+
try:
|
|
1068
|
+
res = self._request_json(self._enterprise_url(), params)
|
|
1069
|
+
series = res.get("series", [])
|
|
1070
|
+
return _to_dataframe(series, as_df=as_df)
|
|
1071
|
+
except Exception:
|
|
1072
|
+
rows = self.query(
|
|
1073
|
+
dataset=dataset,
|
|
1074
|
+
filters=filters,
|
|
1075
|
+
period_from=period_from,
|
|
1076
|
+
period_to=period_to,
|
|
1077
|
+
as_df=False,
|
|
1078
|
+
)
|
|
1079
|
+
by_period: Dict[str, List[float]] = {}
|
|
1080
|
+
for r in rows:
|
|
1081
|
+
p = str(r.get("period") or "")[:4]
|
|
1082
|
+
v = r.get(value_column, r.get("value"))
|
|
1083
|
+
if p and v is not None:
|
|
1084
|
+
by_period.setdefault(p, []).append(float(v))
|
|
1085
|
+
agg = [
|
|
1086
|
+
{
|
|
1087
|
+
"period": p,
|
|
1088
|
+
"n": len(vals),
|
|
1089
|
+
"min": round(min(vals), 4),
|
|
1090
|
+
"max": round(max(vals), 4),
|
|
1091
|
+
"mean": round(sum(vals) / len(vals), 4),
|
|
1092
|
+
}
|
|
1093
|
+
for p, vals in sorted(by_period.items())
|
|
1094
|
+
]
|
|
1095
|
+
return _to_dataframe(agg, as_df=as_df)
|
|
1096
|
+
|
|
1097
|
+
def bulk_export(
|
|
1098
|
+
self,
|
|
1099
|
+
dataset: str = "open_data_observations",
|
|
1100
|
+
fmt: str = "csv",
|
|
1101
|
+
limit: int = 50000,
|
|
1102
|
+
period_from: Optional[Union[str, int]] = None,
|
|
1103
|
+
period_to: Optional[Union[str, int]] = None,
|
|
1104
|
+
as_df: bool = True,
|
|
1105
|
+
) -> Any:
|
|
1106
|
+
"""
|
|
1107
|
+
Stream a bulk dataset export (`csv`, `tsv`, `json`, `ndjson`, `geojson`, `cdl`, `sql`).
|
|
1108
|
+
If `fmt='csv'` and `as_df=True` (with pandas installed), returns a `pandas.DataFrame`.
|
|
1109
|
+
"""
|
|
1110
|
+
params: Dict[str, Any] = {
|
|
1111
|
+
"action": "bulk_export",
|
|
1112
|
+
"dataset": dataset,
|
|
1113
|
+
"format": fmt,
|
|
1114
|
+
"limit": limit,
|
|
1115
|
+
}
|
|
1116
|
+
if period_from is not None:
|
|
1117
|
+
params["period_from"] = str(period_from)
|
|
1118
|
+
if period_to is not None:
|
|
1119
|
+
params["period_to"] = str(period_to)
|
|
1120
|
+
|
|
1121
|
+
resp = self.session.get(self._enterprise_url(), params=params, timeout=max(self.timeout, 120))
|
|
1122
|
+
if resp.status_code < 400:
|
|
1123
|
+
text = resp.content.decode("utf-8-sig", errors="replace")
|
|
1124
|
+
if fmt == "csv" and as_df:
|
|
1125
|
+
try:
|
|
1126
|
+
import pandas as pd # type: ignore
|
|
1127
|
+
|
|
1128
|
+
return pd.read_csv(io.StringIO(text))
|
|
1129
|
+
except ImportError:
|
|
1130
|
+
return text
|
|
1131
|
+
if fmt == "json":
|
|
1132
|
+
return json.loads(text)
|
|
1133
|
+
return text
|
|
1134
|
+
|
|
1135
|
+
# Seamless fallback to public series export
|
|
1136
|
+
rows = self.query(dataset=dataset, period_from=period_from, period_to=period_to, limit=limit, as_df=False)
|
|
1137
|
+
if fmt == "json":
|
|
1138
|
+
return rows
|
|
1139
|
+
if fmt == "csv" and as_df:
|
|
1140
|
+
return _to_dataframe(rows, as_df=True)
|
|
1141
|
+
buf = io.StringIO()
|
|
1142
|
+
if rows:
|
|
1143
|
+
writer = csv.DictWriter(buf, fieldnames=list(rows[0].keys()))
|
|
1144
|
+
writer.writeheader()
|
|
1145
|
+
writer.writerows(rows)
|
|
1146
|
+
return buf.getvalue()
|
|
1147
|
+
|
|
1148
|
+
def spatial_overlay(
|
|
1149
|
+
self,
|
|
1150
|
+
lat: float = 41.02,
|
|
1151
|
+
lon: float = 28.98,
|
|
1152
|
+
radius_km: float = 150.0,
|
|
1153
|
+
) -> Dict[str, Any]:
|
|
1154
|
+
"""Run Haversine spatial radius scan across coordinate-bearing GOMD tables."""
|
|
1155
|
+
try:
|
|
1156
|
+
return self._request_json(
|
|
1157
|
+
self._enterprise_url(),
|
|
1158
|
+
{
|
|
1159
|
+
"action": "spatial_overlay",
|
|
1160
|
+
"lat": lat,
|
|
1161
|
+
"lon": lon,
|
|
1162
|
+
"radius_km": radius_km,
|
|
1163
|
+
},
|
|
1164
|
+
)
|
|
1165
|
+
except Exception:
|
|
1166
|
+
from .spatial import chokepoint_proximity
|
|
1167
|
+
|
|
1168
|
+
prox = chokepoint_proximity(lat=lat, lon=lon)
|
|
1169
|
+
within = [c for c in prox.get("all_chokepoints", []) if c["distance_km"] <= radius_km]
|
|
1170
|
+
return {
|
|
1171
|
+
"status": "success",
|
|
1172
|
+
"center": {"lat": lat, "lon": lon},
|
|
1173
|
+
"radius_km": radius_km,
|
|
1174
|
+
"matched_features": within,
|
|
1175
|
+
"nearest_chokepoint": prox.get("nearest_chokepoint"),
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
def geo_features(self, dataset: str = "measurements", limit: int = 1200) -> Dict[str, Any]:
|
|
1179
|
+
"""Fetch OGC CRS84 GeoJSON FeatureCollection for a geospatial dataset."""
|
|
1180
|
+
try:
|
|
1181
|
+
return self._request_json(
|
|
1182
|
+
self._enterprise_url(),
|
|
1183
|
+
{"action": "geo_features", "dataset": dataset, "limit": limit},
|
|
1184
|
+
)
|
|
1185
|
+
except Exception:
|
|
1186
|
+
resp = self.session.get(
|
|
1187
|
+
self._wfs_url(),
|
|
1188
|
+
params={"service": "WFS", "request": "GetFeature", "limit": min(limit, 1000)},
|
|
1189
|
+
timeout=self.timeout,
|
|
1190
|
+
)
|
|
1191
|
+
return resp.json()
|
|
1192
|
+
|
|
1193
|
+
def get_blue_finance_atlas(
|
|
1194
|
+
self,
|
|
1195
|
+
indicator: str = "iumi_marine_insurance_total_premium_usd_mn",
|
|
1196
|
+
entities: Union[str, Sequence[str]] = ("WLD", "EU27", "CHN", "USA", "NOR", "TUR"),
|
|
1197
|
+
period_from: int = 1995,
|
|
1198
|
+
period_to: int = 2025,
|
|
1199
|
+
as_df: bool = True,
|
|
1200
|
+
) -> Any:
|
|
1201
|
+
"""
|
|
1202
|
+
Query the GOMD Global & Historical Marine, Shipping Insurance, Ocean, Environment
|
|
1203
|
+
and Sustainability Finance Atlas (42 indicators, 228 countries/regions, 1970–2025,
|
|
1204
|
+
192,550 verified observations across OECD, IUMI, IG P&I, UNEP FI PSI, World Bank
|
|
1205
|
+
CWON/PPI, IMF, WTO, UNCTAD, Eurostat COFOG GF05 & Poseidon Principles).
|
|
1206
|
+
"""
|
|
1207
|
+
ent_list = [entities] if isinstance(entities, str) else list(entities)
|
|
1208
|
+
try:
|
|
1209
|
+
summary_url = f"{self.base_url}/assets/data/blue_environmental_finance_atlas_summary.json"
|
|
1210
|
+
resp = self.session.get(summary_url, timeout=self.timeout)
|
|
1211
|
+
if resp.status_code < 400:
|
|
1212
|
+
doc = resp.json()
|
|
1213
|
+
ind_meta = doc.get("indicators", {}).get(indicator, {})
|
|
1214
|
+
series_map = doc.get("series", {}).get(indicator, {})
|
|
1215
|
+
entities_meta = doc.get("entities", {})
|
|
1216
|
+
rows: List[Dict[str, Any]] = []
|
|
1217
|
+
for code in ent_list:
|
|
1218
|
+
c_up = str(code).strip().upper()
|
|
1219
|
+
yr_map = series_map.get(c_up, {})
|
|
1220
|
+
ent_info = entities_meta.get(c_up, {})
|
|
1221
|
+
for yr_str, val in sorted(yr_map.items()):
|
|
1222
|
+
yr = int(yr_str)
|
|
1223
|
+
if int(period_from) <= yr <= int(period_to) and val is not None:
|
|
1224
|
+
rows.append(
|
|
1225
|
+
{
|
|
1226
|
+
"indicator_key": indicator,
|
|
1227
|
+
"title_en": ind_meta.get("title_en", indicator),
|
|
1228
|
+
"source_key": ind_meta.get("source_key", ""),
|
|
1229
|
+
"entity_code": c_up,
|
|
1230
|
+
"entity_name_en": ent_info.get("name_en", c_up),
|
|
1231
|
+
"period": str(yr),
|
|
1232
|
+
"value": float(val),
|
|
1233
|
+
"unit_en": ind_meta.get("unit_en", ""),
|
|
1234
|
+
}
|
|
1235
|
+
)
|
|
1236
|
+
if rows:
|
|
1237
|
+
return _to_dataframe(rows, as_df=as_df)
|
|
1238
|
+
except Exception:
|
|
1239
|
+
pass
|
|
1240
|
+
return self.get_series(
|
|
1241
|
+
indicator=indicator,
|
|
1242
|
+
entity=ent_list,
|
|
1243
|
+
period_from=period_from,
|
|
1244
|
+
period_to=period_to,
|
|
1245
|
+
as_df=as_df,
|
|
1246
|
+
)
|
|
1247
|
+
|
|
1248
|
+
def get_mavi_vatan_jurisdiction(
|
|
1249
|
+
self,
|
|
1250
|
+
indicator: str = "mavi_vatan_official_jurisdiction_km2",
|
|
1251
|
+
entities: Union[str, Sequence[str]] = ("TUR", "TRNC", "LBY", "EGY", "BGR", "ROU", "RUS", "UKR", "GEO", "GRC", "CYP"),
|
|
1252
|
+
period_from: int = 1986,
|
|
1253
|
+
period_to: int = 2026,
|
|
1254
|
+
as_df: bool = True,
|
|
1255
|
+
) -> Any:
|
|
1256
|
+
"""
|
|
1257
|
+
Query the Republic of Türkiye MFA, ONHO (SHODB) & DEHUKAM Blue Homeland ("Mavi Vatan")
|
|
1258
|
+
Maritime Jurisdiction & Equitable Delimitation Audit series (1986–2026), including:
|
|
1259
|
+
- `mavi_vatan_official_jurisdiction_km2` (UN Letter A/74/757 registered ~462,000 km² for TUR, ~9,100 km² for TRNC)
|
|
1260
|
+
- `mavi_vatan_basin_delimitation_km2`
|
|
1261
|
+
- `mavi_vatan_provider_discrepancy_km2`
|
|
1262
|
+
- `mavi_vatan_delimitation_compliance_score`
|
|
1263
|
+
"""
|
|
1264
|
+
ent_list = [entities] if isinstance(entities, str) else list(entities)
|
|
1265
|
+
return self.get_series(
|
|
1266
|
+
indicator=indicator,
|
|
1267
|
+
entity=ent_list,
|
|
1268
|
+
period_from=period_from,
|
|
1269
|
+
period_to=period_to,
|
|
1270
|
+
as_df=as_df,
|
|
1271
|
+
)
|
|
1272
|
+
|
|
1273
|
+
def get_platform_telemetry_panel(
|
|
1274
|
+
self,
|
|
1275
|
+
indicator: str = "counter_r51_scientific_portal_investigations",
|
|
1276
|
+
entities: Union[str, Sequence[str]] = ("TUR", "EU27", "USA", "GBR", "DEU", "WLD"),
|
|
1277
|
+
period_from: int = 2018,
|
|
1278
|
+
period_to: int = 2026,
|
|
1279
|
+
as_df: bool = True,
|
|
1280
|
+
) -> Any:
|
|
1281
|
+
"""
|
|
1282
|
+
Deprecated (kept for backward compatibility).
|
|
1283
|
+
|
|
1284
|
+
The COUNTER Release 5.1 / platform-telemetry indicator keys listed below are
|
|
1285
|
+
NOT published in the GOMD database (0 observations on the live server), so this
|
|
1286
|
+
call normally returns an empty result. Platform usage statistics are only
|
|
1287
|
+
available to administrators via ``platform_analytics.php``.
|
|
1288
|
+
- `counter_r51_scientific_portal_investigations`
|
|
1289
|
+
- `counter_r51_unique_dataset_requests`
|
|
1290
|
+
- `gomd_authenticated_researcher_sessions_pct`
|
|
1291
|
+
- `gomd_sdk_api_programmatic_share_pct`
|
|
1292
|
+
"""
|
|
1293
|
+
ent_list = [entities] if isinstance(entities, str) else list(entities)
|
|
1294
|
+
return self.get_series(
|
|
1295
|
+
indicator=indicator,
|
|
1296
|
+
entity=ent_list,
|
|
1297
|
+
period_from=period_from,
|
|
1298
|
+
period_to=period_to,
|
|
1299
|
+
as_df=as_df,
|
|
1300
|
+
)
|
|
1301
|
+
|
|
1302
|
+
# =========================================================================
|
|
1303
|
+
# DATA ASSISTANT (retrieval-augmented, tool-calling; no model training)
|
|
1304
|
+
# =========================================================================
|
|
1305
|
+
|
|
1306
|
+
def _assistant_url(self) -> str:
|
|
1307
|
+
return f"{self.base_url}/data_assistant_api.php"
|
|
1308
|
+
|
|
1309
|
+
def assistant_status(self) -> Dict[str, Any]:
|
|
1310
|
+
"""
|
|
1311
|
+
Return the GOMD Data Assistant status: whether an LLM key is configured,
|
|
1312
|
+
the number of documents in the full-text index and the remaining daily quota
|
|
1313
|
+
for this client (cookie-less SDK calls use the anonymous per-IP quota).
|
|
1314
|
+
"""
|
|
1315
|
+
res = self._request_json(self._assistant_url(), {"action": "status"})
|
|
1316
|
+
return res.get("data", res)
|
|
1317
|
+
|
|
1318
|
+
def ask_assistant(
|
|
1319
|
+
self,
|
|
1320
|
+
question: str,
|
|
1321
|
+
lang: Optional[str] = None,
|
|
1322
|
+
history: Optional[Sequence[Dict[str, str]]] = None,
|
|
1323
|
+
) -> Dict[str, Any]:
|
|
1324
|
+
"""
|
|
1325
|
+
Ask the GOMD Data Assistant a natural-language question.
|
|
1326
|
+
|
|
1327
|
+
The server answers only from GOMD records: the language model can call four
|
|
1328
|
+
whitelisted read-only tools (catalog full-text search, indicator metadata,
|
|
1329
|
+
series retrieval, deterministic trend statistics) and cannot run free-form SQL.
|
|
1330
|
+
Every returned number is checked against the tool outputs; numbers that could
|
|
1331
|
+
not be matched are listed in ``unverified_numbers``.
|
|
1332
|
+
|
|
1333
|
+
Returns a dict with keys: ``answer`` (Markdown), ``citations`` (indicator key,
|
|
1334
|
+
source, entity, period, provenance keys, download URL), ``unverified_numbers``,
|
|
1335
|
+
``mavi_vatan_disclaimer``, ``tool_calls``, ``latency_ms``, ``quota`` and ``model``.
|
|
1336
|
+
|
|
1337
|
+
Use :func:`gomd.stats.audit_numeric_grounding` together with
|
|
1338
|
+
:meth:`get_series` to independently re-check the numbers in an answer.
|
|
1339
|
+
"""
|
|
1340
|
+
q = (question or "").strip()
|
|
1341
|
+
if len(q) < 3:
|
|
1342
|
+
raise GOMDError("The question must be at least 3 characters long.")
|
|
1343
|
+
body = {
|
|
1344
|
+
"question": q,
|
|
1345
|
+
"lang": lang or ("tr" if str(self.lang).lower().startswith("tr") else "en"),
|
|
1346
|
+
"history": list(history or [])[-6:],
|
|
1347
|
+
}
|
|
1348
|
+
headers = dict(self.session.headers)
|
|
1349
|
+
headers["Content-Type"] = "application/json"
|
|
1350
|
+
headers["X-GOMD-Client"] = "sdk-python/2.1.0"
|
|
1351
|
+
url = self._assistant_url()
|
|
1352
|
+
if requests is not None:
|
|
1353
|
+
resp = requests.post(url, json=body, headers=headers, timeout=max(self.timeout, 180))
|
|
1354
|
+
status_code, text = resp.status_code, resp.text
|
|
1355
|
+
else:
|
|
1356
|
+
req = urllib.request.Request(
|
|
1357
|
+
url, data=json.dumps(body).encode("utf-8"), headers=headers, method="POST"
|
|
1358
|
+
)
|
|
1359
|
+
try:
|
|
1360
|
+
with urllib.request.urlopen(req, timeout=max(self.timeout, 180)) as r:
|
|
1361
|
+
status_code, text = int(getattr(r, "status", 200)), r.read().decode("utf-8", "replace")
|
|
1362
|
+
except urllib.error.HTTPError as exc:
|
|
1363
|
+
status_code = int(exc.code)
|
|
1364
|
+
text = exc.read().decode("utf-8", "replace") if hasattr(exc, "read") else ""
|
|
1365
|
+
try:
|
|
1366
|
+
data = json.loads(text)
|
|
1367
|
+
except Exception as exc:
|
|
1368
|
+
raise GOMDError(f"Non-JSON response from GOMD assistant ({status_code}): {text[:300]}", status_code=status_code) from exc
|
|
1369
|
+
if status_code >= 400 or data.get("ok") is False:
|
|
1370
|
+
raise GOMDError(data.get("error") or f"HTTP {status_code}", status_code=status_code, payload=data)
|
|
1371
|
+
return data.get("data", data)
|
|
1372
|
+
|