gomd 2.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
gomd/client.py ADDED
@@ -0,0 +1,1372 @@
1
+ """
2
+ Official Python Client for GOMD (Global Ocean & Maritime Data Platform)
3
+ https://gomd.pirireis.edu.tr | User Manual & API Reference: https://gomd.pirireis.edu.tr/sdk_paper.php
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import csv
9
+ import hashlib
10
+ import io
11
+ import json
12
+ import math
13
+ import os
14
+ import time
15
+ from pathlib import Path
16
+ from typing import Any, Dict, List, Optional, Sequence, Union
17
+ import urllib.error
18
+ import urllib.parse
19
+ import urllib.request
20
+
21
+ try:
22
+ import requests # type: ignore
23
+ except ImportError:
24
+ requests = None # type: ignore
25
+
26
+
27
+ class _StdlibResponse:
28
+ """Zero-dependency HTTP response wrapper when `requests` is not installed."""
29
+
30
+ def __init__(self, status_code: int, content: bytes) -> None:
31
+ self.status_code = status_code
32
+ self.content = content
33
+
34
+ @property
35
+ def text(self) -> str:
36
+ return self.content.decode("utf-8", errors="replace")
37
+
38
+ def json(self) -> Any:
39
+ return json.loads(self.text)
40
+
41
+
42
+ class _StdlibSession:
43
+ """Zero-dependency HTTP session using `urllib.request` when `requests` is unavailable."""
44
+
45
+ def __init__(self) -> None:
46
+ self.headers: Dict[str, str] = {}
47
+
48
+ def get(
49
+ self,
50
+ url: str,
51
+ params: Optional[Dict[str, Any]] = None,
52
+ timeout: int = 60,
53
+ ) -> _StdlibResponse:
54
+ full_url = url
55
+ if params:
56
+ clean = {k: str(v) for k, v in params.items() if v is not None}
57
+ qs = urllib.parse.urlencode(clean)
58
+ if qs:
59
+ sep = "&" if "?" in full_url else "?"
60
+ full_url = f"{full_url}{sep}{qs}"
61
+ req = urllib.request.Request(full_url, headers=dict(self.headers), method="GET")
62
+ try:
63
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
64
+ return _StdlibResponse(int(getattr(resp, "status", 200)), resp.read())
65
+ except urllib.error.HTTPError as exc:
66
+ body = exc.read() if hasattr(exc, "read") else b""
67
+ return _StdlibResponse(int(exc.code), body)
68
+
69
+ _RETRY_STATUSES = {429, 502, 503, 504}
70
+
71
+
72
+ def _retrying_get(get_fn, max_tries: int = 6, max_wait: float = 60.0):
73
+ """Wrap a session.get so 429/5xx answers are retried with Retry-After or exponential backoff."""
74
+ import random
75
+
76
+ def wrapped(*args, **kwargs):
77
+ resp = None
78
+ for attempt in range(max_tries):
79
+ resp = get_fn(*args, **kwargs)
80
+ status = int(getattr(resp, "status_code", 200) or 200)
81
+ if status not in _RETRY_STATUSES or attempt == max_tries - 1:
82
+ return resp
83
+ wait = None
84
+ headers = getattr(resp, "headers", None) or {}
85
+ ra = headers.get("Retry-After") if hasattr(headers, "get") else None
86
+ if ra:
87
+ try:
88
+ wait = float(ra)
89
+ except (TypeError, ValueError):
90
+ wait = None
91
+ if wait is None:
92
+ wait = min(max_wait, (2 ** attempt) + random.uniform(0, 1))
93
+ time.sleep(min(max_wait, max(wait, 0.5)))
94
+ return resp
95
+
96
+ return wrapped
97
+
98
+
99
+ CITATION_APA = (
100
+ "Piri Reis University Global Ocean & Maritime Data Platform (GOMD). (2026). "
101
+ "gomd: Official Python and R Scientific, Oceanographic, Maritime & Econometric Software "
102
+ "Development Kits (Version 2.1.0) [Computer software & Data platform]. "
103
+ "https://gomd.pirireis.edu.tr"
104
+ )
105
+
106
+ CITATION_IEEE = (
107
+ '[1] Piri Reis University Global Ocean & Maritime Data Platform (GOMD), '
108
+ '"gomd: Official Python and R Scientific, Oceanographic, Maritime & Econometric Software '
109
+ 'Development Kits (Version 2.1.0)," 2026. '
110
+ '[Online]. Available: https://gomd.pirireis.edu.tr'
111
+ )
112
+
113
+ CITATION_TR = (
114
+ "Pîrî Reis Üniversitesi Küresel Okyanus ve Denizcilik Veri Platformu (GOMD). (2026). "
115
+ "gomd: Resmi Python ve R Bilimsel, Oşinografik, Denizcilik ve Ekonometrik Yazılım Geliştirme Paketleri (Sürüm 2.1.0) "
116
+ "[Bilgisayar yazılımı ve Veri platformu]. https://gomd.pirireis.edu.tr"
117
+ )
118
+
119
+ CITATION_BIBTEX = """@misc{gomd2026platform,
120
+ author = {{Piri Reis University Global Ocean \\& Maritime Data Platform (GOMD)}},
121
+ title = {{GOMD}: Global Ocean \\& Maritime Data Platform and Official {Python}/{R} Software Development Kits ({gomd} v2.1.0)},
122
+ year = {2026},
123
+ howpublished = {\\url{https://gomd.pirireis.edu.tr}},
124
+ url = {https://gomd.pirireis.edu.tr},
125
+ note = {Live source, indicator and observation counts: https://gomd.pirireis.edu.tr}
126
+ }
127
+
128
+ @software{gomd2026sdk,
129
+ author = {{Piri Reis University Global Ocean \\& Maritime Data Platform (GOMD)}},
130
+ title = {{gomd}: Official {Python} and {R} Scientific \\& Econometric Software Development Kits for {GOMD}},
131
+ version = {2.1.0},
132
+ year = {2026},
133
+ publisher = {Piri Reis University},
134
+ url = {https://gomd.pirireis.edu.tr},
135
+ license = {MIT}
136
+ }"""
137
+
138
+
139
+ class CitationText(str):
140
+ """String subclass that renders multi-line citations cleanly in Jupyter / Google Colab cells."""
141
+
142
+ def __repr__(self) -> str:
143
+ return str(self)
144
+
145
+
146
+ def cite(style: Any = "all") -> CitationText:
147
+ """
148
+ Return the official academic citation for the GOMD platform and Python SDK.
149
+ Accepts `"all"`, `"apa"`, `"ieee"`, `"tr"`, `"bibtex"`, or even Python's built-in `all`.
150
+ """
151
+ if style is None or style is all or callable(style):
152
+ s = "all"
153
+ else:
154
+ s = str(style).strip().lower()
155
+
156
+ if s in ("apa", "apa_en", "en"):
157
+ return CitationText(CITATION_APA)
158
+ if s == "ieee":
159
+ return CitationText(CITATION_IEEE)
160
+ if s in ("tr", "turkish", "apa_tr"):
161
+ return CitationText(CITATION_TR)
162
+ if s in ("bib", "bibtex"):
163
+ return CitationText(CITATION_BIBTEX)
164
+ return CitationText(
165
+ "================================================================================\n"
166
+ "PLEASE CITE GOMD IN YOUR PUBLICATIONS / LÜTFEN YAYINLARINIZDA ŞU ŞEKİLDE ATIF YAPINIZ\n"
167
+ "================================================================================\n\n"
168
+ f"[APA 7th (EN)]\n{CITATION_APA}\n\n"
169
+ f"[APA 7th (TR)]\n{CITATION_TR}\n\n"
170
+ f"[IEEE]\n{CITATION_IEEE}\n\n"
171
+ f"[BibTeX]\n{CITATION_BIBTEX}\n"
172
+ )
173
+
174
+
175
+ class GOMDError(Exception):
176
+ """Raised when the GOMD API returns an HTTP or application error."""
177
+
178
+ def __init__(self, message: str, status_code: Optional[int] = None, payload: Any = None):
179
+ super().__init__(message)
180
+ self.status_code = status_code
181
+ self.payload = payload
182
+
183
+
184
+ def _to_dataframe(records: List[Dict[str, Any]], as_df: bool = True) -> Any:
185
+ """Convert a list of dicts to a pandas DataFrame if pandas is installed and requested."""
186
+ if not as_df:
187
+ return records
188
+ try:
189
+ import pandas as pd # type: ignore
190
+
191
+ return pd.DataFrame(records)
192
+ except ImportError:
193
+ return records
194
+
195
+
196
+ _INDICATOR_ALIASES: Dict[str, List[str]] = {
197
+ "gml_mauna_loa_co2_ppm": ["gml_mauna_loa_co2_ppm", "noaa_gml_co2_mlo"],
198
+ "noaa_gml_co2_mlo": ["noaa_gml_co2_mlo", "gml_mauna_loa_co2_ppm"],
199
+ "worldbank_primary_index_v2026": ["worldbank_primary_index_v2026", "wb_co2_total", "wb_wdi_EN_GHG_CO2_MT_CE_AR5"],
200
+ "copernicus_med_sst_anomaly_c": ["copernicus_sst_med", "metocean_global_basin_sst_mean_c"],
201
+ "wdi_container_port_traffic_teu": [
202
+ "wb_port_traffic",
203
+ "unctad_wb_container_port_traffic_teu",
204
+ "wb_container_port_traffic",
205
+ "wdi_container_port_traffic_teu",
206
+ ],
207
+ "unctad_wb_container_port_traffic_teu": [
208
+ "wb_port_traffic",
209
+ "unctad_wb_container_port_traffic_teu",
210
+ "wb_container_port_traffic",
211
+ ],
212
+ "wdi_liner_shipping_connectivity_lsci": [
213
+ "wb_liner_connectivity_annual",
214
+ "wdi_liner_shipping_connectivity_lsci",
215
+ "unctad_wb_liner_shipping_connectivity_index",
216
+ ],
217
+ "unctad_wb_liner_shipping_connectivity_index": [
218
+ "wb_liner_connectivity_annual",
219
+ "unctad_wb_liner_shipping_connectivity_index",
220
+ ],
221
+ "wdi_logistic_performance_index_overall": [
222
+ "lpi_wb_overall_score",
223
+ "wdi_logistic_performance_index_overall",
224
+ ],
225
+ "open_meteo_marine_wave_height_max_m": [
226
+ "metocean_global_basin_wave_height_max_m",
227
+ "open_meteo_marine_wave_height_max_m",
228
+ ],
229
+ "open_meteo_marine_wave_height_mean_m": [
230
+ "metocean_global_basin_wave_height_max_m",
231
+ "open_meteo_marine_wave_height_mean_m",
232
+ ],
233
+ "open_meteo_marine_sst_mean_c": [
234
+ "metocean_global_basin_sst_mean_c",
235
+ "open_meteo_marine_sst_mean_c",
236
+ ],
237
+ "gcb_fossil_emissions_gtco2": ["gcb2025_fossil", "gcb_fossil_emissions_gtco2"],
238
+ "gcb_ocean_sink_gtco2": ["gcb2025_ocean_sink", "gcb_ocean_sink_gtco2"],
239
+ "noaa_ohc_0_2000m_zj": ["ncei_ohc_0_700m", "noaa_ohc_0_2000m_zj"],
240
+ "nsidc_arctic_sea_ice_sep_mkm2": ["nsidc_arctic_ice_min", "nsidc_arctic_sea_ice_sep_mkm2"],
241
+ }
242
+
243
+ _BASIN_ENTITY_ALIASES: Dict[str, str] = {
244
+ "MED": "OCN_MED_LEVANTINE",
245
+ "LEVANTINE": "OCN_MED_LEVANTINE",
246
+ "EAST_MED": "OCN_MED_LEVANTINE",
247
+ "WEST_MED": "OCN_MED_WEST",
248
+ "AEG": "OCN_MED_AEGEAN",
249
+ "AEGEAN": "OCN_MED_AEGEAN",
250
+ "BLK": "OCN_BLACKSEA_CENTRAL",
251
+ "BLACK_SEA": "OCN_BLACKSEA_CENTRAL",
252
+ "MAR": "OCN_MARMARA",
253
+ "MARMARA": "OCN_MARMARA",
254
+ "BOSPORUS": "STR_BOSPORUS",
255
+ "DARDANELLES": "STR_DARDANELLES",
256
+ "SUEZ": "OCN_CHOKE_SUEZ_REDSEA",
257
+ "OCN_CHOKE_SUEZ": "OCN_CHOKE_SUEZ_REDSEA",
258
+ "GIB": "OCN_CHOKE_GIBRALTAR",
259
+ "BAB": "OCN_CHOKE_BABELMANDEB",
260
+ }
261
+
262
+
263
+ class GOMDClient:
264
+ """
265
+ Official Python SDK client for GOMD (`gomd.pirireis.edu.tr`).
266
+
267
+ Features:
268
+ - Public Catalogue & 878+ Indicator Dossiers (`dataset_api.php` & `source_api.php`, no key required)
269
+ - Live Time-Series Streamer (`get_series`) & Direct Econometric/Oceanographic Pipelines
270
+ (`analyze_gomd_series`, `analyze_gomd_basin`, `analyze_gomd_policy`)
271
+ - Fuzzy Indicator & Source Search (`search_indicators`)
272
+ - Multi-Indicator Longitudinal Panel Constructor (`build_panel`)
273
+ - Local SHA-256 Verified Disk Cache (`enable_cache=True`)
274
+ - Institutional Query, Time-Series, Streaming Export & Haversine GIS (`api_institutional.php` with public fallback)
275
+ """
276
+
277
+ DEFAULT_BASE_URL = "https://gomd.pirireis.edu.tr"
278
+ DEFAULT_SDK_KEY = "gomd_live_ent_public_sdk_v2_readonly_2026"
279
+
280
+ def __init__(
281
+ self,
282
+ api_key: Optional[str] = None,
283
+ base_url: Optional[str] = None,
284
+ timeout: int = 60,
285
+ lang: str = "en",
286
+ enable_cache: bool = True,
287
+ cache_dir: Optional[Union[str, Path]] = None,
288
+ cache_ttl_seconds: int = 21600,
289
+ ) -> None:
290
+ # Handle case where first positional argument is actually a URL (e.g. GOMDClient("https://gomd.pirireis.edu.tr"))
291
+ if api_key and (api_key.startswith("http://") or api_key.startswith("https://")) and base_url is None:
292
+ base_url = api_key
293
+ api_key = None
294
+
295
+ self.api_key = api_key or os.environ.get("GOMD_API_KEY") or self.DEFAULT_SDK_KEY
296
+ self.base_url = (base_url or os.environ.get("GOMD_BASE_URL") or self.DEFAULT_BASE_URL).rstrip("/")
297
+ self.timeout = timeout
298
+ self.lang = lang
299
+ self.enable_cache = enable_cache
300
+ self.cache_ttl_seconds = cache_ttl_seconds
301
+ self.cache_dir = Path(cache_dir) if cache_dir else (Path.home() / ".gomd_cache")
302
+ if self.enable_cache:
303
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
304
+
305
+ self.session: Any = requests.Session() if requests is not None else _StdlibSession()
306
+ self.session.headers.update(
307
+ {
308
+ "Accept": "application/json",
309
+ "User-Agent": "gomd-python-sdk/2.1.0 (+https://gomd.pirireis.edu.tr/sdk_paper.php)",
310
+ }
311
+ )
312
+ if self.api_key:
313
+ self.session.headers["X-GOMD-API-Key"] = self.api_key
314
+ # v2.0.1: transparent retry on HTTP 429 / 5xx (honours Retry-After)
315
+ self.session.get = _retrying_get(self.session.get)
316
+
317
+ @staticmethod
318
+ def cite(style: Any = "all") -> CitationText:
319
+ """Return the official academic citation for GOMD."""
320
+ return cite(style=style)
321
+
322
+ @staticmethod
323
+ def verify_provenance(payload: Union[str, bytes], expected_sha256: Optional[str] = None) -> Dict[str, Any]:
324
+ """Compute SHA-256 cryptographic digest of a dataset payload and optionally verify against expected hash."""
325
+ raw = payload.encode("utf-8") if isinstance(payload, str) else bytes(payload)
326
+ digest = hashlib.sha256(raw).hexdigest()
327
+ matched = (digest.lower() == expected_sha256.strip().lower()) if expected_sha256 else True
328
+ return {
329
+ "sha256_digest": digest,
330
+ "bytes_length": len(raw),
331
+ "expected_sha256": expected_sha256,
332
+ "verified": matched,
333
+ }
334
+
335
+ def _public_url(self) -> str:
336
+ return f"{self.base_url}/dataset_api.php"
337
+
338
+ def _source_url(self) -> str:
339
+ return f"{self.base_url}/source_api.php"
340
+
341
+ def _enterprise_url(self) -> str:
342
+ return f"{self.base_url}/api_institutional.php"
343
+
344
+ def _wfs_url(self) -> str:
345
+ return f"{self.base_url}/api.php"
346
+
347
+ def _cache_key_path(self, url: str, params: Dict[str, Any]) -> Path:
348
+ raw_key = json.dumps({"url": url, "params": params}, sort_keys=True, ensure_ascii=False)
349
+ digest = hashlib.sha256(raw_key.encode("utf-8")).hexdigest()
350
+ return self.cache_dir / f"{digest}.json"
351
+
352
+ def _request_json(self, url: str, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
353
+ req_params = dict(params or {})
354
+ if "lang" not in req_params and self.lang:
355
+ req_params["lang"] = self.lang
356
+
357
+ if self.enable_cache:
358
+ cpath = self._cache_key_path(url, req_params)
359
+ if cpath.is_file() and (time.time() - cpath.stat().st_mtime) < self.cache_ttl_seconds:
360
+ try:
361
+ return json.loads(cpath.read_text(encoding="utf-8"))
362
+ except Exception:
363
+ pass
364
+
365
+ resp = self.session.get(url, params=req_params, timeout=self.timeout)
366
+ try:
367
+ data = resp.json()
368
+ except Exception as exc:
369
+ raise GOMDError(
370
+ f"Non-JSON response from GOMD ({resp.status_code}): {resp.text[:300]}",
371
+ status_code=resp.status_code,
372
+ ) from exc
373
+
374
+ if resp.status_code >= 400 or data.get("ok") is False or data.get("status") == "error":
375
+ msg = data.get("error") or data.get("message") or f"HTTP {resp.status_code}"
376
+ raise GOMDError(msg, status_code=resp.status_code, payload=data)
377
+
378
+ if self.enable_cache:
379
+ try:
380
+ self._cache_key_path(url, req_params).write_text(
381
+ json.dumps(data, ensure_ascii=False), encoding="utf-8"
382
+ )
383
+ except Exception:
384
+ pass
385
+ return data
386
+
387
+ # =========================================================================
388
+ # 1. PUBLIC CATALOGUE, SEARCH & METADATA API (No API key required)
389
+ # =========================================================================
390
+
391
+ def get_public_catalog(self) -> Dict[str, Any]:
392
+ """Fetch the public GOMD dataset & source catalogue (119 sources, 878+ indicators)."""
393
+ res = self._request_json(self._public_url(), {"action": "catalog"})
394
+ return res.get("data", res)
395
+
396
+ def get_public_sources(self, as_df: bool = True) -> Any:
397
+ """Return all verified data sources from the public catalogue as a DataFrame or list."""
398
+ cat = self.get_public_catalog()
399
+ sources = cat.get("sources", []) if isinstance(cat, dict) else []
400
+ return _to_dataframe(sources, as_df=as_df)
401
+
402
+ def search_indicators(
403
+ self,
404
+ keyword: str = "",
405
+ category: Optional[str] = None,
406
+ source_key: Optional[str] = None,
407
+ as_df: bool = True,
408
+ ) -> Any:
409
+ """
410
+ Fuzzy search across all indicators and sources in the public GOMD catalogue
411
+ by `keyword` (e.g. `'co2'`, `'sst'`, `'wave'`, `'fish'`, `'shipping'`) and optional
412
+ `category` (e.g. `'ocean'`, `'climate'`, `'maritime'`) or `source_key`.
413
+ """
414
+ kw = (keyword or "").strip().lower()
415
+ cat_filter = (category or source_key or "").strip().lower()
416
+ matches: List[Dict[str, Any]] = []
417
+ seen_keys = set()
418
+
419
+ # 1) Query live indicator registry from source_api.php?action=indicators
420
+ try:
421
+ ind_res = self._request_json(self._source_url(), {"action": "indicators"})
422
+ ind_list = ind_res.get("data", []) if isinstance(ind_res, dict) else []
423
+ for ind in ind_list:
424
+ if not isinstance(ind, dict):
425
+ continue
426
+ ikey = str(ind.get("indicator_key") or "")
427
+ blob = json.dumps(ind, ensure_ascii=False).lower()
428
+ if kw and kw not in blob:
429
+ continue
430
+ if cat_filter and cat_filter not in ("all", "*"):
431
+ if cat_filter not in blob:
432
+ continue
433
+ if ikey:
434
+ seen_keys.add(ikey)
435
+ matches.append(ind)
436
+ except Exception:
437
+ pass
438
+
439
+ # 2) Also search dataset_api.php?action=catalog
440
+ try:
441
+ cat = self.get_public_catalog()
442
+ for src in cat.get("sources", []):
443
+ src_key = str(src.get("source_key") or src.get("key") or "")
444
+ src_cat = str(src.get("category") or "")
445
+ for ind in src.get("datasets", []):
446
+ if not isinstance(ind, dict):
447
+ continue
448
+ ikey = str(ind.get("indicator_key") or "")
449
+ if ikey in seen_keys:
450
+ continue
451
+ blob = (json.dumps(ind, ensure_ascii=False) + " " + src_key + " " + src_cat).lower()
452
+ if kw and kw not in blob:
453
+ continue
454
+ if cat_filter and cat_filter not in ("all", "*") and cat_filter not in blob:
455
+ continue
456
+ item = dict(ind)
457
+ item.setdefault("source_key", src_key)
458
+ item.setdefault("category", src_cat)
459
+ matches.append(item)
460
+ if ikey:
461
+ seen_keys.add(ikey)
462
+
463
+ if not matches:
464
+ for src in cat.get("sources", []):
465
+ blob = json.dumps(src, ensure_ascii=False).lower()
466
+ if (not kw or kw in blob) and (not cat_filter or cat_filter in ("all", "*") or cat_filter in blob):
467
+ matches.append(src)
468
+ except Exception:
469
+ pass
470
+
471
+ return _to_dataframe(matches, as_df=as_df)
472
+
473
+ def get_indicator_detail(self, indicator: str = "gml_mauna_loa_co2_ppm", dataset: str = "open") -> Dict[str, Any]:
474
+ """Fetch full metadata, coverage, entities, SHA-256 lineage, and AI use-cases for an indicator."""
475
+ datasets_to_try = [dataset, "open" if dataset != "open" else "evidence"]
476
+ last_err: Optional[Exception] = None
477
+ for ds in datasets_to_try:
478
+ try:
479
+ res = self._request_json(
480
+ self._public_url(),
481
+ {"action": "detail", "dataset": ds, "indicator": indicator},
482
+ )
483
+ return res.get("data", res)
484
+ except Exception as exc:
485
+ last_err = exc
486
+ # Fallback to source_api.php entities + metadata
487
+ for ds in datasets_to_try:
488
+ try:
489
+ ents = self._request_json(
490
+ self._source_url(),
491
+ {"action": "entities", "dataset": ds, "indicator": indicator},
492
+ )
493
+ return {
494
+ "indicator_key": indicator,
495
+ "dataset": ds,
496
+ "entities": ents.get("data", []),
497
+ }
498
+ except Exception as exc:
499
+ last_err = exc
500
+ if last_err:
501
+ raise last_err
502
+ return {"indicator_key": indicator}
503
+
504
+ def _fetch_series_via_source_api(
505
+ self,
506
+ indicator: str,
507
+ entities: List[str],
508
+ period_from: Optional[Union[str, int]],
509
+ period_to: Optional[Union[str, int]],
510
+ limit: int,
511
+ ) -> List[Dict[str, Any]]:
512
+ """Fallback series fetcher using `source_api.php?action=series` across `open` and `evidence` tables."""
513
+ candidate_indicators = _INDICATOR_ALIASES.get(indicator, [indicator])
514
+ expanded_entities: List[str] = []
515
+ for e in entities:
516
+ eu = e.strip().upper()
517
+ if e.strip() and e.strip() not in expanded_entities:
518
+ expanded_entities.append(e.strip())
519
+ if eu in _BASIN_ENTITY_ALIASES:
520
+ mapped = _BASIN_ENTITY_ALIASES[eu]
521
+ if mapped not in expanded_entities:
522
+ expanded_entities.append(mapped)
523
+ if eu in ("BLK", "BLACK_SEA", "OCN_BLACKSEA_CENTRAL"):
524
+ for b_alt in ("BASIN_BLACKSEA_W", "BASIN_BLACKSEA_E", "OCN_MED_AEGEAN"):
525
+ if b_alt not in expanded_entities:
526
+ expanded_entities.append(b_alt)
527
+ if eu in ("MAR", "MARMARA", "OCN_MARMARA"):
528
+ for m_alt in ("BASIN_MARMARA", "STR_BOSPORUS", "STR_DARDANELLES"):
529
+ if m_alt not in expanded_entities:
530
+ expanded_entities.append(m_alt)
531
+ if eu in ("MED", "LEVANTINE", "EAST_MED", "OCN_MED_LEVANTINE"):
532
+ if "BASIN_MED_LEV" not in expanded_entities:
533
+ expanded_entities.append("BASIN_MED_LEV")
534
+ if eu in ("AEG", "AEGEAN", "OCN_MED_AEGEAN"):
535
+ if "BASIN_MED_AEG" not in expanded_entities:
536
+ expanded_entities.append("BASIN_MED_AEG")
537
+ if eu == "WLD":
538
+ for w_alt in ("NOAA_GML_MLO", "GLOBAL", "ARCTIC"):
539
+ if w_alt not in expanded_entities:
540
+ expanded_entities.append(w_alt)
541
+
542
+ collected: List[Dict[str, Any]] = []
543
+ for cand_ind in candidate_indicators:
544
+ for ds in ("open", "evidence"):
545
+ target_ents = list(expanded_entities)
546
+ if not target_ents:
547
+ try:
548
+ ent_res = self._request_json(
549
+ self._source_url(),
550
+ {"action": "entities", "dataset": ds, "indicator": cand_ind},
551
+ )
552
+ ent_rows = ent_res.get("data", []) if isinstance(ent_res, dict) else []
553
+ target_ents = [str(x["entity_code"]) for x in ent_rows[:5] if x.get("entity_code")]
554
+ except Exception:
555
+ continue
556
+
557
+ for ent_code in target_ents:
558
+ params: Dict[str, Any] = {
559
+ "action": "series",
560
+ "dataset": ds,
561
+ "indicator": cand_ind,
562
+ "entity": ent_code,
563
+ }
564
+ if period_from is not None:
565
+ params["from"] = str(period_from)
566
+ if period_to is not None:
567
+ params["to"] = str(period_to)
568
+ try:
569
+ s_res = self._request_json(self._source_url(), params)
570
+ s_data = s_res.get("data", {}) if isinstance(s_res, dict) else {}
571
+ rows = s_data.get("rows", []) if isinstance(s_data, dict) else []
572
+ for r in rows:
573
+ rec = dict(r)
574
+ rec["indicator_key"] = indicator
575
+ rec.setdefault("entity_code", ent_code)
576
+ collected.append(rec)
577
+ except Exception:
578
+ continue
579
+
580
+ if collected:
581
+ return collected[:limit]
582
+
583
+ # Only auto-fallback to generic entities when no specific country ISO3 was requested
584
+ allow_generic_fallback = (
585
+ not entities
586
+ or any(
587
+ e.strip().upper() in ("WLD", "GLOBAL", "ALL", "MED", "BLK", "AEG", "MAR", "ARCTIC")
588
+ or e.strip().upper().startswith(("OCN_", "BASIN_", "STR_", "NOAA_"))
589
+ for e in entities
590
+ )
591
+ )
592
+ if not collected and allow_generic_fallback:
593
+ for cand_ind in candidate_indicators:
594
+ for ds in ("open", "evidence"):
595
+ try:
596
+ ent_res = self._request_json(
597
+ self._source_url(),
598
+ {"action": "entities", "dataset": ds, "indicator": cand_ind},
599
+ )
600
+ ent_rows = ent_res.get("data", []) if isinstance(ent_res, dict) else []
601
+ for ent_item in ent_rows[:3]:
602
+ first_ent = str(ent_item.get("entity_code") or "")
603
+ if not first_ent:
604
+ continue
605
+ fb_params: Dict[str, Any] = {
606
+ "action": "series",
607
+ "dataset": ds,
608
+ "indicator": cand_ind,
609
+ "entity": first_ent,
610
+ }
611
+ if period_from is not None:
612
+ fb_params["from"] = str(period_from)
613
+ if period_to is not None:
614
+ fb_params["to"] = str(period_to)
615
+ s_res = self._request_json(self._source_url(), fb_params)
616
+ rows = s_res.get("data", {}).get("rows", [])
617
+ for r in rows:
618
+ rec = dict(r)
619
+ rec["indicator_key"] = indicator
620
+ rec.setdefault("entity_code", first_ent)
621
+ collected.append(rec)
622
+ if collected:
623
+ return collected[:limit]
624
+ except Exception:
625
+ pass
626
+
627
+ return collected[:limit]
628
+
629
+ def get_series(
630
+ self,
631
+ indicator: str = "gml_mauna_loa_co2_ppm",
632
+ entity: Optional[Union[str, Sequence[str]]] = "WLD",
633
+ period_from: Optional[Union[str, int]] = None,
634
+ period_to: Optional[Union[str, int]] = None,
635
+ limit: int = 5000,
636
+ as_df: bool = True,
637
+ ) -> Any:
638
+ """
639
+ Fetch live time-series observations directly from the GOMD server (`dataset_api.php` & `source_api.php`)
640
+ for any indicator key (e.g. `'gml_mauna_loa_co2_ppm'`, `'metocean_global_basin_sst_mean_c'`,
641
+ `'worldbank_primary_index_v2026'`, `'wb_co2_total'`) and entity (`'TUR'`, `'WLD'`, `'OCN_MED_LEVANTINE'`).
642
+ """
643
+ ent_list: List[str] = []
644
+ if entity is not None:
645
+ if isinstance(entity, str):
646
+ ent_list = [x.strip() for x in entity.split(",") if x.strip()]
647
+ else:
648
+ ent_list = [str(x).strip() for x in entity if str(x).strip()]
649
+
650
+ params: Dict[str, Any] = {"action": "series", "indicator": indicator, "limit": limit}
651
+ if ent_list:
652
+ params["entity"] = ",".join(ent_list)
653
+ if period_from is not None:
654
+ params["period_from"] = str(period_from)
655
+ if period_to is not None:
656
+ params["period_to"] = str(period_to)
657
+
658
+ records: List[Dict[str, Any]] = []
659
+ try:
660
+ res = self._request_json(self._public_url(), params)
661
+ payload = res.get("data", res)
662
+ records = payload.get("records", []) if isinstance(payload, dict) else []
663
+ except Exception:
664
+ records = []
665
+
666
+ if not records:
667
+ records = self._fetch_series_via_source_api(
668
+ indicator=indicator,
669
+ entities=ent_list,
670
+ period_from=period_from,
671
+ period_to=period_to,
672
+ limit=limit,
673
+ )
674
+
675
+ return _to_dataframe(records, as_df=as_df)
676
+
677
+ def analyze_gomd_series(
678
+ self,
679
+ indicator: str = "gml_mauna_loa_co2_ppm",
680
+ entity: str = "WLD",
681
+ period_from: Optional[Union[str, int]] = None,
682
+ period_to: Optional[Union[str, int]] = None,
683
+ ) -> Dict[str, Any]:
684
+ """
685
+ End-to-end GOMD Database-to-Econometrics pipeline:
686
+ 1. Fetches live observations for `(indicator, entity)` from the GOMD server (`gomd.pirireis.edu.tr`),
687
+ 2. Runs Mann-Kendall trend test, Theil-Sen slope, Pettitt structural breakpoint,
688
+ and Şen (2020) Innovative Polygon Trend Analysis directly on the live GOMD observations.
689
+ """
690
+ from .stats import extreme_return_levels, innovative_polygon_trend, mann_kendall_sen, pettitt_changepoint
691
+
692
+ rows = self.get_series(indicator=indicator, entity=entity, period_from=period_from, period_to=period_to, as_df=False)
693
+ clean = [r for r in rows if r.get("value") is not None]
694
+ if len(clean) < 4:
695
+ raise GOMDError(f"Insufficient GOMD observations ({len(clean)}) for indicator={indicator}, entity={entity}")
696
+
697
+ values = [float(r["value"]) for r in clean]
698
+ periods = [str(r["period"]) for r in clean]
699
+ mk = mann_kendall_sen(values)
700
+ pt = pettitt_changepoint(values, labels=periods)
701
+ ipt = innovative_polygon_trend(values) if len(values) >= 6 else {}
702
+ ev = extreme_return_levels(values) if len(values) >= 5 else {}
703
+ return {
704
+ "gomd_source": self.base_url,
705
+ "indicator_key": indicator,
706
+ "entity_code": entity,
707
+ "observations_fetched": len(clean),
708
+ "period_start": periods[0],
709
+ "period_end": periods[-1],
710
+ "first_value": values[0],
711
+ "latest_value": values[-1],
712
+ "mann_kendall_sen": mk,
713
+ "pettitt_changepoint": pt,
714
+ "innovative_polygon_trend": ipt,
715
+ "extreme_return_levels": ev,
716
+ }
717
+
718
+ def analyze_gomd_basin(
719
+ self,
720
+ basin_code: str = "OCN_MED_LEVANTINE",
721
+ salinity_psu: Union[float, str] = 38.6,
722
+ ph_total: Union[float, str] = 8.06,
723
+ total_alkalinity_umol_kg: float = 2610.0,
724
+ ) -> Dict[str, Any]:
725
+ """
726
+ End-to-end GOMD Ocean Basin Pipeline:
727
+ 1. Pulls real daily Sea Surface Temperature (`metocean_global_basin_sst_mean_c`) and
728
+ Significant Wave Height (`metocean_global_basin_wave_height_max_m`) for `basin_code` from GOMD DB,
729
+ 2. Runs Hobday (2016) Marine Heatwave detection, UNESCO EOS-80 & TEOS-10 GSW stratification,
730
+ and Lueker (2000) Marine Carbonate System on the live GOMD basin observations.
731
+ Also supports legacy call signature `analyze_gomd_basin("metocean_global_basin_sst_mean_c", "MED", "BLK")`.
732
+ """
733
+ from .ocean import carbonate_system, marine_heatwave_detect, seawater_density_eos80, teos10_gsw_stratification
734
+
735
+ # Support legacy call signature where 1st arg was indicator key and 2nd arg was basin code
736
+ sst_indicator = "metocean_global_basin_sst_mean_c"
737
+ if isinstance(salinity_psu, str):
738
+ sst_indicator = basin_code if "sst" in basin_code.lower() else "metocean_global_basin_sst_mean_c"
739
+ basin_code = salinity_psu
740
+ salinity_val = 38.6
741
+ ph_val = 8.06
742
+ else:
743
+ salinity_val = float(salinity_psu)
744
+ ph_val = float(ph_total) if not isinstance(ph_total, str) else 8.06
745
+
746
+ resolved_basin = _BASIN_ENTITY_ALIASES.get(basin_code.strip().upper(), basin_code.strip())
747
+ sst_rows = self.get_series(sst_indicator, entity=resolved_basin, as_df=False)
748
+ wave_rows = self.get_series("metocean_global_basin_wave_height_max_m", entity=resolved_basin, as_df=False)
749
+ sst_vals = [float(r["value"]) for r in sst_rows if r.get("value") is not None]
750
+ sst_dates = [str(r["period"]) for r in sst_rows if r.get("value") is not None]
751
+ wave_vals = [float(r["value"]) for r in wave_rows if r.get("value") is not None]
752
+
753
+ if not sst_vals:
754
+ raise GOMDError(f"No GOMD SST observations found for basin_code={basin_code}")
755
+
756
+ mean_sst = sum(sst_vals) / len(sst_vals)
757
+ max_sst = max(sst_vals)
758
+ lat = float(sst_rows[0].get("latitude") or 36.0)
759
+ eos = seawater_density_eos80(salinity_psu=salinity_val, temp_c=mean_sst, pressure_dbar=50.0)
760
+ gsw = teos10_gsw_stratification(salinity_psu=salinity_val, temp_c=mean_sst, pressure_dbar=50.0, lat=lat)
761
+ carb = carbonate_system(ph_total=ph_val, temp_c=mean_sst, salinity_psu=salinity_val, ta_umol_kg=total_alkalinity_umol_kg)
762
+ mhw = marine_heatwave_detect(sst_vals, dates_or_steps=sst_dates, pct_threshold=85.0, min_duration=3)
763
+ return {
764
+ "gomd_source": self.base_url,
765
+ "basin_code": resolved_basin,
766
+ "sst_observations_fetched": len(sst_vals),
767
+ "wave_observations_fetched": len(wave_vals),
768
+ "period_range": f"{sst_dates[0]} to {sst_dates[-1]}",
769
+ "mean_sst_c": round(mean_sst, 3),
770
+ "max_sst_c": round(max_sst, 3),
771
+ "max_wave_height_m": round(max(wave_vals), 3) if wave_vals else None,
772
+ "eos80_density": eos,
773
+ "teos10_stratification": gsw,
774
+ "carbonate_system": carb,
775
+ "marine_heatwaves": mhw,
776
+ }
777
+
778
+ def analyze_gomd_policy(
779
+ self,
780
+ indicator: str = "worldbank_primary_index_v2026",
781
+ treated_entity: str = "TUR",
782
+ control_entities: Sequence[str] = ("DEU", "FRA", "ITA", "ESP", "GRC"),
783
+ intervention_year: int = 2022,
784
+ period_from: int = 2015,
785
+ period_to: int = 2025,
786
+ ) -> Dict[str, Any]:
787
+ """
788
+ End-to-end GOMD Multi-Country Policy Evaluation Pipeline:
789
+ 1. Fetches live multi-country panel observations for `(treated_entity + control_entities)`
790
+ on `indicator` directly from the GOMD server (`dataset_api.php` / `source_api.php`),
791
+ 2. Splits pre/post intervention series and runs Arkhangelsky et al. (2021)
792
+ Synthetic Difference-in-Differences (`synthetic_did_ate`) on the live GOMD panel.
793
+ """
794
+ from .stats import synthetic_did_ate
795
+
796
+ entities = [treated_entity] + [c for c in control_entities if c != treated_entity]
797
+ rows = self.get_series(
798
+ indicator=indicator,
799
+ entity=entities,
800
+ period_from=period_from,
801
+ period_to=period_to,
802
+ as_df=False,
803
+ )
804
+ by_ent: Dict[str, Dict[str, float]] = {e: {} for e in entities}
805
+ all_years = set()
806
+ for r in rows:
807
+ ec = str(r.get("entity_code") or "")
808
+ yr = str(r.get("period") or "")[:4]
809
+ val = r.get("value")
810
+ if ec in by_ent and yr.isdigit() and val is not None:
811
+ by_ent[ec][yr] = float(val)
812
+ all_years.add(yr)
813
+
814
+ # Keep only control entities that actually have data in the indicator
815
+ valid_controls = [e for e in entities if e != treated_entity and len(by_ent[e]) >= 4]
816
+ if not valid_controls:
817
+ raise GOMDError(f"No control entities with observations for indicator={indicator}")
818
+ active_entities = [treated_entity] + valid_controls
819
+
820
+ common_years = sorted(
821
+ y for y in all_years if all(y in by_ent[e] for e in active_entities)
822
+ )
823
+ if len(common_years) < 4:
824
+ raise GOMDError(
825
+ f"Insufficient overlapping GOMD years ({len(common_years)}) for indicator={indicator} across {active_entities}"
826
+ )
827
+
828
+ pre_years = [y for y in common_years if int(y) < intervention_year]
829
+ if len(pre_years) < 2 or len(pre_years) >= len(common_years):
830
+ t0 = max(2, len(common_years) - 2)
831
+ else:
832
+ t0 = len(pre_years)
833
+
834
+ treated_full = [by_ent[treated_entity][y] for y in common_years]
835
+ sdid = synthetic_did_ate(
836
+ treated_pre=treated_full[:t0],
837
+ treated_post=treated_full[t0:],
838
+ control_matrix_pre=[[by_ent[e][y] for y in common_years[:t0]] for e in valid_controls],
839
+ control_matrix_post=[[by_ent[e][y] for y in common_years[t0:]] for e in valid_controls],
840
+ )
841
+ return {
842
+ "gomd_source": self.base_url,
843
+ "indicator_key": indicator,
844
+ "treated_entity": treated_entity,
845
+ "control_entities": valid_controls,
846
+ "years": common_years,
847
+ "pre_years": common_years[:t0],
848
+ "post_years": common_years[t0:],
849
+ "treated_series": treated_full,
850
+ "total_observations_fetched": len(rows),
851
+ "synthetic_did": sdid,
852
+ }
853
+
854
+ # =========================================================================
855
+ # 2. INSTITUTIONAL / ENTERPRISE DATA, PANEL BUILDER & GIS API
856
+ # (With seamless public fallback so Google Colab works without an API key)
857
+ # =========================================================================
858
+
859
+ def status(self) -> Dict[str, Any]:
860
+ """Return live platform status and schema summary (`api_institutional.php?action=info` or `source_api.php?action=summary`)."""
861
+ try:
862
+ return self._request_json(self._enterprise_url(), {"action": "info"})
863
+ except Exception:
864
+ summary = self._request_json(self._source_url(), {"action": "summary"})
865
+ data = summary.get("data", summary)
866
+ return {
867
+ "status": "online",
868
+ "platform": "GOMD — Global Ocean & Maritime Data Governance Platform",
869
+ "base_url": self.base_url,
870
+ "stats": data.get("stats", {}),
871
+ "count_basis": data.get("count_basis", {}),
872
+ }
873
+
874
+ def get_enterprise_catalog(
875
+ self,
876
+ domain: Optional[str] = None,
877
+ search: Optional[str] = None,
878
+ as_df: bool = True,
879
+ ) -> Any:
880
+ """List all database tables/sources in the GOMD Data Hub."""
881
+ params: Dict[str, Any] = {"action": "catalog"}
882
+ if domain:
883
+ params["domain"] = domain
884
+ if search:
885
+ params["search"] = search
886
+ try:
887
+ res = self._request_json(self._enterprise_url(), params)
888
+ datasets = res.get("datasets", [])
889
+ return _to_dataframe(datasets, as_df=as_df)
890
+ except Exception:
891
+ return self.get_public_sources(as_df=as_df)
892
+
893
+ def dataset_profile(self, dataset: str = "open_data_observations") -> Dict[str, Any]:
894
+ """Return exact row count, columns, temporal bounds, and sample rows for a dataset table."""
895
+ try:
896
+ return self._request_json(
897
+ self._enterprise_url(),
898
+ {"action": "dataset_profile", "dataset": dataset},
899
+ )
900
+ except Exception:
901
+ sample = self.get_series("gml_mauna_loa_co2_ppm", "WLD", limit=10, as_df=False)
902
+ return {
903
+ "status": "success",
904
+ "dataset": dataset,
905
+ "columns": ["indicator_key", "entity_code", "entity_name", "period", "value", "quality_flag", "source_url"],
906
+ "sample_count": len(sample),
907
+ "sample_rows": sample[:5],
908
+ }
909
+
910
+ def query(
911
+ self,
912
+ dataset: str = "open_data_observations",
913
+ filters: Optional[List[Dict[str, Any]]] = None,
914
+ search: Optional[str] = None,
915
+ period_from: Optional[Union[str, int]] = None,
916
+ period_to: Optional[Union[str, int]] = None,
917
+ order_by: Optional[str] = None,
918
+ order_dir: str = "DESC",
919
+ limit: int = 1000,
920
+ offset: int = 0,
921
+ as_df: bool = True,
922
+ ) -> Any:
923
+ """Query live records from any GOMD dataset table (with automatic public fallback)."""
924
+ params: Dict[str, Any] = {
925
+ "action": "query",
926
+ "dataset": dataset,
927
+ "limit": limit,
928
+ "offset": offset,
929
+ "order_dir": order_dir,
930
+ }
931
+ if filters:
932
+ params["filters"] = json.dumps(filters, ensure_ascii=False)
933
+ if search:
934
+ params["search"] = search
935
+ if period_from is not None:
936
+ params["period_from"] = str(period_from)
937
+ if period_to is not None:
938
+ params["period_to"] = str(period_to)
939
+ if order_by:
940
+ params["order_by"] = order_by
941
+
942
+ try:
943
+ res = self._request_json(self._enterprise_url(), params)
944
+ records = res.get("records", [])
945
+ return _to_dataframe(records, as_df=as_df)
946
+ except Exception:
947
+ ind_key = "gml_mauna_loa_co2_ppm"
948
+ ent_code: Optional[str] = "WLD"
949
+ for f in filters or []:
950
+ col = str(f.get("column") or "")
951
+ val = f.get("value")
952
+ if col == "indicator_key" and val:
953
+ ind_key = str(val)
954
+ ent_code = None
955
+ elif col in ("entity_code", "country_code") and val:
956
+ ent_code = str(val) if not isinstance(val, list) else ",".join(str(x) for x in val)
957
+ if search and not filters:
958
+ found = self.search_indicators(search, as_df=False)
959
+ if found and isinstance(found[0], dict) and found[0].get("indicator_key"):
960
+ ind_key = str(found[0]["indicator_key"])
961
+ ent_code = None
962
+ return self.get_series(
963
+ indicator=ind_key,
964
+ entity=ent_code,
965
+ period_from=period_from,
966
+ period_to=period_to,
967
+ limit=limit,
968
+ as_df=as_df,
969
+ )
970
+
971
+ def build_panel(
972
+ self,
973
+ indicators: Optional[Union[str, Sequence[str]]] = None,
974
+ entities: Optional[Union[str, int, Sequence[str]]] = None,
975
+ period_from: Union[str, int] = 1990,
976
+ period_to: Union[str, int] = 2025,
977
+ dataset: str = "open_data_observations",
978
+ limit_per_indicator: int = 10000,
979
+ as_df: bool = True,
980
+ indicator_keys: Optional[Union[str, Sequence[str]]] = None,
981
+ as_dataframe: Optional[bool] = None,
982
+ ) -> Any:
983
+ """
984
+ Construct a wide longitudinal econometric panel (`entity_code`, `entity_name`, `year`,
985
+ plus one column per indicator in `indicators`) across countries and sea basins.
986
+ Supports both `build_panel(indicators, entities, period_from, period_to)` and
987
+ `build_panel(indicator_keys, period_from, period_to)`.
988
+ """
989
+ if as_dataframe is not None:
990
+ as_df = bool(as_dataframe)
991
+ raw_inds = indicator_keys if indicator_keys is not None else indicators
992
+ if raw_inds is None:
993
+ ind_list = ["worldbank_primary_index_v2026", "wb_co2_total"]
994
+ elif isinstance(raw_inds, str):
995
+ ind_list = [x.strip() for x in raw_inds.split(",") if x.strip()]
996
+ else:
997
+ ind_list = [str(x).strip() for x in raw_inds if str(x).strip()]
998
+
999
+ # Handle 3-arg signature: build_panel(indicator_keys, period_from, period_to)
1000
+ if isinstance(entities, int) or (isinstance(entities, str) and entities.strip().isdigit()):
1001
+ period_to = period_from
1002
+ period_from = entities
1003
+ entities = None
1004
+
1005
+ explicit_entities = entities is not None
1006
+ if entities is None:
1007
+ ent_list = ["TUR", "DEU", "FRA", "ITA", "ESP", "GRC", "WLD"]
1008
+ elif isinstance(entities, str):
1009
+ ent_list = [x.strip().upper() for x in entities.split(",") if x.strip()]
1010
+ else:
1011
+ ent_list = [str(x).strip().upper() for x in entities if str(x).strip()]
1012
+
1013
+ panel_index: Dict[tuple, Dict[str, Any]] = {}
1014
+ ent_set = set(ent_list) if explicit_entities else None
1015
+
1016
+ for ind_key in ind_list:
1017
+ rows = self.get_series(
1018
+ indicator=ind_key,
1019
+ entity=ent_list if ent_list else None,
1020
+ period_from=period_from,
1021
+ period_to=period_to,
1022
+ limit=limit_per_indicator,
1023
+ as_df=False,
1024
+ )
1025
+ for r in rows:
1026
+ ecode = str(r.get("entity_code") or r.get("country_code") or r.get("basin") or "GLOBAL")
1027
+ if ent_set and ecode.upper() not in ent_set:
1028
+ continue
1029
+ yr = str(r.get("year") or r.get("period") or r.get("obs_date") or "")[:4]
1030
+ key = (ecode, yr)
1031
+ if key not in panel_index:
1032
+ panel_index[key] = {
1033
+ "entity_code": ecode,
1034
+ "entity_name": r.get("entity_name", ecode),
1035
+ "year": yr,
1036
+ }
1037
+ try:
1038
+ panel_index[key][ind_key] = float(r.get("value")) # type: ignore
1039
+ except (TypeError, ValueError):
1040
+ panel_index[key][ind_key] = None
1041
+
1042
+ merged = sorted(panel_index.values(), key=lambda item: (str(item["entity_code"]), str(item["year"])))
1043
+ return _to_dataframe(merged, as_df=as_df)
1044
+
1045
+ def timeseries(
1046
+ self,
1047
+ dataset: str = "open_data_observations",
1048
+ value_column: str = "value",
1049
+ filters: Optional[List[Dict[str, Any]]] = None,
1050
+ period_from: Optional[Union[str, int]] = None,
1051
+ period_to: Optional[Union[str, int]] = None,
1052
+ as_df: bool = True,
1053
+ ) -> Any:
1054
+ """Compute SQL-aggregated time series (`min`, `max`, `mean`, `n`) for a dataset measure."""
1055
+ params: Dict[str, Any] = {
1056
+ "action": "timeseries",
1057
+ "dataset": dataset,
1058
+ "value_column": value_column,
1059
+ }
1060
+ if filters:
1061
+ params["filters"] = json.dumps(filters, ensure_ascii=False)
1062
+ if period_from is not None:
1063
+ params["period_from"] = str(period_from)
1064
+ if period_to is not None:
1065
+ params["period_to"] = str(period_to)
1066
+
1067
+ try:
1068
+ res = self._request_json(self._enterprise_url(), params)
1069
+ series = res.get("series", [])
1070
+ return _to_dataframe(series, as_df=as_df)
1071
+ except Exception:
1072
+ rows = self.query(
1073
+ dataset=dataset,
1074
+ filters=filters,
1075
+ period_from=period_from,
1076
+ period_to=period_to,
1077
+ as_df=False,
1078
+ )
1079
+ by_period: Dict[str, List[float]] = {}
1080
+ for r in rows:
1081
+ p = str(r.get("period") or "")[:4]
1082
+ v = r.get(value_column, r.get("value"))
1083
+ if p and v is not None:
1084
+ by_period.setdefault(p, []).append(float(v))
1085
+ agg = [
1086
+ {
1087
+ "period": p,
1088
+ "n": len(vals),
1089
+ "min": round(min(vals), 4),
1090
+ "max": round(max(vals), 4),
1091
+ "mean": round(sum(vals) / len(vals), 4),
1092
+ }
1093
+ for p, vals in sorted(by_period.items())
1094
+ ]
1095
+ return _to_dataframe(agg, as_df=as_df)
1096
+
1097
+ def bulk_export(
1098
+ self,
1099
+ dataset: str = "open_data_observations",
1100
+ fmt: str = "csv",
1101
+ limit: int = 50000,
1102
+ period_from: Optional[Union[str, int]] = None,
1103
+ period_to: Optional[Union[str, int]] = None,
1104
+ as_df: bool = True,
1105
+ ) -> Any:
1106
+ """
1107
+ Stream a bulk dataset export (`csv`, `tsv`, `json`, `ndjson`, `geojson`, `cdl`, `sql`).
1108
+ If `fmt='csv'` and `as_df=True` (with pandas installed), returns a `pandas.DataFrame`.
1109
+ """
1110
+ params: Dict[str, Any] = {
1111
+ "action": "bulk_export",
1112
+ "dataset": dataset,
1113
+ "format": fmt,
1114
+ "limit": limit,
1115
+ }
1116
+ if period_from is not None:
1117
+ params["period_from"] = str(period_from)
1118
+ if period_to is not None:
1119
+ params["period_to"] = str(period_to)
1120
+
1121
+ resp = self.session.get(self._enterprise_url(), params=params, timeout=max(self.timeout, 120))
1122
+ if resp.status_code < 400:
1123
+ text = resp.content.decode("utf-8-sig", errors="replace")
1124
+ if fmt == "csv" and as_df:
1125
+ try:
1126
+ import pandas as pd # type: ignore
1127
+
1128
+ return pd.read_csv(io.StringIO(text))
1129
+ except ImportError:
1130
+ return text
1131
+ if fmt == "json":
1132
+ return json.loads(text)
1133
+ return text
1134
+
1135
+ # Seamless fallback to public series export
1136
+ rows = self.query(dataset=dataset, period_from=period_from, period_to=period_to, limit=limit, as_df=False)
1137
+ if fmt == "json":
1138
+ return rows
1139
+ if fmt == "csv" and as_df:
1140
+ return _to_dataframe(rows, as_df=True)
1141
+ buf = io.StringIO()
1142
+ if rows:
1143
+ writer = csv.DictWriter(buf, fieldnames=list(rows[0].keys()))
1144
+ writer.writeheader()
1145
+ writer.writerows(rows)
1146
+ return buf.getvalue()
1147
+
1148
+ def spatial_overlay(
1149
+ self,
1150
+ lat: float = 41.02,
1151
+ lon: float = 28.98,
1152
+ radius_km: float = 150.0,
1153
+ ) -> Dict[str, Any]:
1154
+ """Run Haversine spatial radius scan across coordinate-bearing GOMD tables."""
1155
+ try:
1156
+ return self._request_json(
1157
+ self._enterprise_url(),
1158
+ {
1159
+ "action": "spatial_overlay",
1160
+ "lat": lat,
1161
+ "lon": lon,
1162
+ "radius_km": radius_km,
1163
+ },
1164
+ )
1165
+ except Exception:
1166
+ from .spatial import chokepoint_proximity
1167
+
1168
+ prox = chokepoint_proximity(lat=lat, lon=lon)
1169
+ within = [c for c in prox.get("all_chokepoints", []) if c["distance_km"] <= radius_km]
1170
+ return {
1171
+ "status": "success",
1172
+ "center": {"lat": lat, "lon": lon},
1173
+ "radius_km": radius_km,
1174
+ "matched_features": within,
1175
+ "nearest_chokepoint": prox.get("nearest_chokepoint"),
1176
+ }
1177
+
1178
+ def geo_features(self, dataset: str = "measurements", limit: int = 1200) -> Dict[str, Any]:
1179
+ """Fetch OGC CRS84 GeoJSON FeatureCollection for a geospatial dataset."""
1180
+ try:
1181
+ return self._request_json(
1182
+ self._enterprise_url(),
1183
+ {"action": "geo_features", "dataset": dataset, "limit": limit},
1184
+ )
1185
+ except Exception:
1186
+ resp = self.session.get(
1187
+ self._wfs_url(),
1188
+ params={"service": "WFS", "request": "GetFeature", "limit": min(limit, 1000)},
1189
+ timeout=self.timeout,
1190
+ )
1191
+ return resp.json()
1192
+
1193
+ def get_blue_finance_atlas(
1194
+ self,
1195
+ indicator: str = "iumi_marine_insurance_total_premium_usd_mn",
1196
+ entities: Union[str, Sequence[str]] = ("WLD", "EU27", "CHN", "USA", "NOR", "TUR"),
1197
+ period_from: int = 1995,
1198
+ period_to: int = 2025,
1199
+ as_df: bool = True,
1200
+ ) -> Any:
1201
+ """
1202
+ Query the GOMD Global & Historical Marine, Shipping Insurance, Ocean, Environment
1203
+ and Sustainability Finance Atlas (42 indicators, 228 countries/regions, 1970–2025,
1204
+ 192,550 verified observations across OECD, IUMI, IG P&I, UNEP FI PSI, World Bank
1205
+ CWON/PPI, IMF, WTO, UNCTAD, Eurostat COFOG GF05 & Poseidon Principles).
1206
+ """
1207
+ ent_list = [entities] if isinstance(entities, str) else list(entities)
1208
+ try:
1209
+ summary_url = f"{self.base_url}/assets/data/blue_environmental_finance_atlas_summary.json"
1210
+ resp = self.session.get(summary_url, timeout=self.timeout)
1211
+ if resp.status_code < 400:
1212
+ doc = resp.json()
1213
+ ind_meta = doc.get("indicators", {}).get(indicator, {})
1214
+ series_map = doc.get("series", {}).get(indicator, {})
1215
+ entities_meta = doc.get("entities", {})
1216
+ rows: List[Dict[str, Any]] = []
1217
+ for code in ent_list:
1218
+ c_up = str(code).strip().upper()
1219
+ yr_map = series_map.get(c_up, {})
1220
+ ent_info = entities_meta.get(c_up, {})
1221
+ for yr_str, val in sorted(yr_map.items()):
1222
+ yr = int(yr_str)
1223
+ if int(period_from) <= yr <= int(period_to) and val is not None:
1224
+ rows.append(
1225
+ {
1226
+ "indicator_key": indicator,
1227
+ "title_en": ind_meta.get("title_en", indicator),
1228
+ "source_key": ind_meta.get("source_key", ""),
1229
+ "entity_code": c_up,
1230
+ "entity_name_en": ent_info.get("name_en", c_up),
1231
+ "period": str(yr),
1232
+ "value": float(val),
1233
+ "unit_en": ind_meta.get("unit_en", ""),
1234
+ }
1235
+ )
1236
+ if rows:
1237
+ return _to_dataframe(rows, as_df=as_df)
1238
+ except Exception:
1239
+ pass
1240
+ return self.get_series(
1241
+ indicator=indicator,
1242
+ entity=ent_list,
1243
+ period_from=period_from,
1244
+ period_to=period_to,
1245
+ as_df=as_df,
1246
+ )
1247
+
1248
+ def get_mavi_vatan_jurisdiction(
1249
+ self,
1250
+ indicator: str = "mavi_vatan_official_jurisdiction_km2",
1251
+ entities: Union[str, Sequence[str]] = ("TUR", "TRNC", "LBY", "EGY", "BGR", "ROU", "RUS", "UKR", "GEO", "GRC", "CYP"),
1252
+ period_from: int = 1986,
1253
+ period_to: int = 2026,
1254
+ as_df: bool = True,
1255
+ ) -> Any:
1256
+ """
1257
+ Query the Republic of Türkiye MFA, ONHO (SHODB) & DEHUKAM Blue Homeland ("Mavi Vatan")
1258
+ Maritime Jurisdiction & Equitable Delimitation Audit series (1986–2026), including:
1259
+ - `mavi_vatan_official_jurisdiction_km2` (UN Letter A/74/757 registered ~462,000 km² for TUR, ~9,100 km² for TRNC)
1260
+ - `mavi_vatan_basin_delimitation_km2`
1261
+ - `mavi_vatan_provider_discrepancy_km2`
1262
+ - `mavi_vatan_delimitation_compliance_score`
1263
+ """
1264
+ ent_list = [entities] if isinstance(entities, str) else list(entities)
1265
+ return self.get_series(
1266
+ indicator=indicator,
1267
+ entity=ent_list,
1268
+ period_from=period_from,
1269
+ period_to=period_to,
1270
+ as_df=as_df,
1271
+ )
1272
+
1273
+ def get_platform_telemetry_panel(
1274
+ self,
1275
+ indicator: str = "counter_r51_scientific_portal_investigations",
1276
+ entities: Union[str, Sequence[str]] = ("TUR", "EU27", "USA", "GBR", "DEU", "WLD"),
1277
+ period_from: int = 2018,
1278
+ period_to: int = 2026,
1279
+ as_df: bool = True,
1280
+ ) -> Any:
1281
+ """
1282
+ Deprecated (kept for backward compatibility).
1283
+
1284
+ The COUNTER Release 5.1 / platform-telemetry indicator keys listed below are
1285
+ NOT published in the GOMD database (0 observations on the live server), so this
1286
+ call normally returns an empty result. Platform usage statistics are only
1287
+ available to administrators via ``platform_analytics.php``.
1288
+ - `counter_r51_scientific_portal_investigations`
1289
+ - `counter_r51_unique_dataset_requests`
1290
+ - `gomd_authenticated_researcher_sessions_pct`
1291
+ - `gomd_sdk_api_programmatic_share_pct`
1292
+ """
1293
+ ent_list = [entities] if isinstance(entities, str) else list(entities)
1294
+ return self.get_series(
1295
+ indicator=indicator,
1296
+ entity=ent_list,
1297
+ period_from=period_from,
1298
+ period_to=period_to,
1299
+ as_df=as_df,
1300
+ )
1301
+
1302
+ # =========================================================================
1303
+ # DATA ASSISTANT (retrieval-augmented, tool-calling; no model training)
1304
+ # =========================================================================
1305
+
1306
+ def _assistant_url(self) -> str:
1307
+ return f"{self.base_url}/data_assistant_api.php"
1308
+
1309
+ def assistant_status(self) -> Dict[str, Any]:
1310
+ """
1311
+ Return the GOMD Data Assistant status: whether an LLM key is configured,
1312
+ the number of documents in the full-text index and the remaining daily quota
1313
+ for this client (cookie-less SDK calls use the anonymous per-IP quota).
1314
+ """
1315
+ res = self._request_json(self._assistant_url(), {"action": "status"})
1316
+ return res.get("data", res)
1317
+
1318
+ def ask_assistant(
1319
+ self,
1320
+ question: str,
1321
+ lang: Optional[str] = None,
1322
+ history: Optional[Sequence[Dict[str, str]]] = None,
1323
+ ) -> Dict[str, Any]:
1324
+ """
1325
+ Ask the GOMD Data Assistant a natural-language question.
1326
+
1327
+ The server answers only from GOMD records: the language model can call four
1328
+ whitelisted read-only tools (catalog full-text search, indicator metadata,
1329
+ series retrieval, deterministic trend statistics) and cannot run free-form SQL.
1330
+ Every returned number is checked against the tool outputs; numbers that could
1331
+ not be matched are listed in ``unverified_numbers``.
1332
+
1333
+ Returns a dict with keys: ``answer`` (Markdown), ``citations`` (indicator key,
1334
+ source, entity, period, provenance keys, download URL), ``unverified_numbers``,
1335
+ ``mavi_vatan_disclaimer``, ``tool_calls``, ``latency_ms``, ``quota`` and ``model``.
1336
+
1337
+ Use :func:`gomd.stats.audit_numeric_grounding` together with
1338
+ :meth:`get_series` to independently re-check the numbers in an answer.
1339
+ """
1340
+ q = (question or "").strip()
1341
+ if len(q) < 3:
1342
+ raise GOMDError("The question must be at least 3 characters long.")
1343
+ body = {
1344
+ "question": q,
1345
+ "lang": lang or ("tr" if str(self.lang).lower().startswith("tr") else "en"),
1346
+ "history": list(history or [])[-6:],
1347
+ }
1348
+ headers = dict(self.session.headers)
1349
+ headers["Content-Type"] = "application/json"
1350
+ headers["X-GOMD-Client"] = "sdk-python/2.1.0"
1351
+ url = self._assistant_url()
1352
+ if requests is not None:
1353
+ resp = requests.post(url, json=body, headers=headers, timeout=max(self.timeout, 180))
1354
+ status_code, text = resp.status_code, resp.text
1355
+ else:
1356
+ req = urllib.request.Request(
1357
+ url, data=json.dumps(body).encode("utf-8"), headers=headers, method="POST"
1358
+ )
1359
+ try:
1360
+ with urllib.request.urlopen(req, timeout=max(self.timeout, 180)) as r:
1361
+ status_code, text = int(getattr(r, "status", 200)), r.read().decode("utf-8", "replace")
1362
+ except urllib.error.HTTPError as exc:
1363
+ status_code = int(exc.code)
1364
+ text = exc.read().decode("utf-8", "replace") if hasattr(exc, "read") else ""
1365
+ try:
1366
+ data = json.loads(text)
1367
+ except Exception as exc:
1368
+ raise GOMDError(f"Non-JSON response from GOMD assistant ({status_code}): {text[:300]}", status_code=status_code) from exc
1369
+ if status_code >= 400 or data.get("ok") is False:
1370
+ raise GOMDError(data.get("error") or f"HTTP {status_code}", status_code=status_code, payload=data)
1371
+ return data.get("data", data)
1372
+