nielytics 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nielytics/__init__.py +56 -0
- nielytics/climate/__init__.py +1 -0
- nielytics/climate/providers/__init__.py +47 -0
- nielytics/climate/providers/_http.py +182 -0
- nielytics/climate/providers/nasa_landslides.py +144 -0
- nielytics/climate/providers/nasa_power.py +108 -0
- nielytics/climate/providers/noaa_ibtracs.py +189 -0
- nielytics/climate/providers/open_meteo.py +108 -0
- nielytics/climate/providers/open_meteo_flood.py +70 -0
- nielytics/core/__init__.py +1 -0
- nielytics/core/cleaner.py +163 -0
- nielytics/core/geographic.py +200 -0
- nielytics/core/loader.py +117 -0
- nielytics/core/outliers.py +70 -0
- nielytics/core/profiler.py +164 -0
- nielytics/core/statistics.py +238 -0
- nielytics/core/timeseries.py +97 -0
- nielytics/core/visualization.py +337 -0
- nielytics/dataset.py +95 -0
- nielytics/exceptions.py +16 -0
- nielytics/geospatial/__init__.py +16 -0
- nielytics/geospatial/coordinates.py +40 -0
- nielytics/geospatial/geojson.py +326 -0
- nielytics/geospatial/map_layers.py +71 -0
- nielytics/hazards/__init__.py +9 -0
- nielytics/hazards/_common.py +193 -0
- nielytics/hazards/_coordinates.py +19 -0
- nielytics/hazards/cyclone.py +123 -0
- nielytics/hazards/drought.py +81 -0
- nielytics/hazards/flood.py +89 -0
- nielytics/hazards/heatwave.py +82 -0
- nielytics/hazards/landslide.py +108 -0
- nielytics/models/__init__.py +5 -0
- nielytics/models/schemas.py +36 -0
- nielytics-0.1.0.dist-info/METADATA +97 -0
- nielytics-0.1.0.dist-info/RECORD +38 -0
- nielytics-0.1.0.dist-info/WHEEL +4 -0
- nielytics-0.1.0.dist-info/licenses/LICENSE +21 -0
nielytics/__init__.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""NIElytics: one backend-safe public package for analytics and indicators."""
|
|
2
|
+
|
|
3
|
+
from .core.visualization import (
|
|
4
|
+
plot_bar,
|
|
5
|
+
plot_box,
|
|
6
|
+
plot_correlation,
|
|
7
|
+
plot_distribution,
|
|
8
|
+
plot_geographic,
|
|
9
|
+
plot_histogram,
|
|
10
|
+
plot_line,
|
|
11
|
+
plot_scatter,
|
|
12
|
+
plot_seasonal,
|
|
13
|
+
plot_timeseries,
|
|
14
|
+
)
|
|
15
|
+
from .dataset import Dataset
|
|
16
|
+
from .exceptions import (
|
|
17
|
+
DataLoadError,
|
|
18
|
+
NIElyticsError,
|
|
19
|
+
OptionalDependencyError,
|
|
20
|
+
UnsupportedFormatError,
|
|
21
|
+
ValidationError,
|
|
22
|
+
)
|
|
23
|
+
from .hazards import (
|
|
24
|
+
CycloneAnalyzer,
|
|
25
|
+
DroughtAnalyzer,
|
|
26
|
+
FloodAnalyzer,
|
|
27
|
+
HeatwaveAnalyzer,
|
|
28
|
+
LandslideAnalyzer,
|
|
29
|
+
)
|
|
30
|
+
from .models import AnalysisResult
|
|
31
|
+
|
|
32
|
+
__version__ = "0.1.0"
|
|
33
|
+
__all__ = [
|
|
34
|
+
"AnalysisResult",
|
|
35
|
+
"CycloneAnalyzer",
|
|
36
|
+
"DataLoadError",
|
|
37
|
+
"Dataset",
|
|
38
|
+
"DroughtAnalyzer",
|
|
39
|
+
"FloodAnalyzer",
|
|
40
|
+
"HeatwaveAnalyzer",
|
|
41
|
+
"LandslideAnalyzer",
|
|
42
|
+
"NIElyticsError",
|
|
43
|
+
"OptionalDependencyError",
|
|
44
|
+
"UnsupportedFormatError",
|
|
45
|
+
"ValidationError",
|
|
46
|
+
"plot_bar",
|
|
47
|
+
"plot_box",
|
|
48
|
+
"plot_correlation",
|
|
49
|
+
"plot_distribution",
|
|
50
|
+
"plot_geographic",
|
|
51
|
+
"plot_histogram",
|
|
52
|
+
"plot_line",
|
|
53
|
+
"plot_scatter",
|
|
54
|
+
"plot_seasonal",
|
|
55
|
+
"plot_timeseries",
|
|
56
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Verified provider connectors for climate inputs; no network calls on import."""
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Verified public climate data connectors. Network is used only when explicitly called.
|
|
2
|
+
|
|
3
|
+
All online functions accept ``client=HTTPClient(...)`` for injectable transport,
|
|
4
|
+
offline mode, explicit cache TTL and bounded retries; results are AnalysisResult.
|
|
5
|
+
No credentials or forecast APIs are supplied.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from ._http import HTTPClient, OfflineCacheError, ProviderError, ResponseError
|
|
9
|
+
from .nasa_landslides import (
|
|
10
|
+
fetch_landslide_events,
|
|
11
|
+
filter_landslides_by_region,
|
|
12
|
+
landslide_events_to_geojson,
|
|
13
|
+
load_landslide_catalog,
|
|
14
|
+
)
|
|
15
|
+
from .nasa_power import fetch_nasa_power_data, fetch_nasa_precipitation
|
|
16
|
+
from .noaa_ibtracs import (
|
|
17
|
+
cyclone_frequency,
|
|
18
|
+
cyclone_tracks_to_geojson,
|
|
19
|
+
get_cyclone_intensity,
|
|
20
|
+
get_cyclone_tracks,
|
|
21
|
+
load_ibtracs,
|
|
22
|
+
)
|
|
23
|
+
from .open_meteo import fetch_historical_weather, fetch_rainfall_data, fetch_temperature_data
|
|
24
|
+
from .open_meteo_flood import analyze_discharge_anomalies, fetch_river_discharge
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"HTTPClient",
|
|
28
|
+
"OfflineCacheError",
|
|
29
|
+
"ProviderError",
|
|
30
|
+
"ResponseError",
|
|
31
|
+
"analyze_discharge_anomalies",
|
|
32
|
+
"cyclone_frequency",
|
|
33
|
+
"cyclone_tracks_to_geojson",
|
|
34
|
+
"fetch_historical_weather",
|
|
35
|
+
"fetch_landslide_events",
|
|
36
|
+
"fetch_nasa_power_data",
|
|
37
|
+
"fetch_nasa_precipitation",
|
|
38
|
+
"fetch_rainfall_data",
|
|
39
|
+
"fetch_river_discharge",
|
|
40
|
+
"fetch_temperature_data",
|
|
41
|
+
"filter_landslides_by_region",
|
|
42
|
+
"get_cyclone_intensity",
|
|
43
|
+
"get_cyclone_tracks",
|
|
44
|
+
"landslide_events_to_geojson",
|
|
45
|
+
"load_ibtracs",
|
|
46
|
+
"load_landslide_catalog",
|
|
47
|
+
]
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""Explicit, offline-capable provider HTTP transport. No requests occur on import."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
import time
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from datetime import UTC, datetime
|
|
12
|
+
from email.utils import parsedate_to_datetime
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from urllib.error import HTTPError, URLError
|
|
15
|
+
from urllib.parse import urlencode
|
|
16
|
+
from urllib.request import Request, urlopen
|
|
17
|
+
|
|
18
|
+
from nielytics.exceptions import NIElyticsError, ValidationError
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class ProviderError(NIElyticsError):
|
|
22
|
+
"""Provider transport or response failed; inspect the message for context."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class OfflineCacheError(ProviderError):
|
|
26
|
+
"""No acceptable cached response is available in offline mode."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class ResponseError(ProviderError):
|
|
30
|
+
"""Response does not match the documented provider schema."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class HTTPClient:
|
|
34
|
+
"""GET transport with injected ``opener(request, timeout=...)`` and optional disk cache.
|
|
35
|
+
|
|
36
|
+
``cache_dir=None`` disables caching. ``ttl_seconds`` determines when cached
|
|
37
|
+
data expires, including in offline mode unless ``allow_expired_offline`` is
|
|
38
|
+
explicitly enabled. ``offline=True`` never contacts the network. Successful
|
|
39
|
+
responses alone are cached, keyed by full URL and query string. No secrets
|
|
40
|
+
are stored by these public, credential-free connectors. Retries apply only
|
|
41
|
+
to 429, 5xx, and connection failures; sleep and clock are injectable.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
*,
|
|
47
|
+
opener: Callable = urlopen,
|
|
48
|
+
timeout: float = 20,
|
|
49
|
+
retries: int = 2,
|
|
50
|
+
backoff: float = 1,
|
|
51
|
+
cache_dir: str | Path | None = None,
|
|
52
|
+
ttl_seconds: float = 86400,
|
|
53
|
+
offline: bool = False,
|
|
54
|
+
allow_expired_offline: bool = False,
|
|
55
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
56
|
+
clock: Callable[[], float] = time.time,
|
|
57
|
+
) -> None:
|
|
58
|
+
if not math.isfinite(timeout) or timeout <= 0:
|
|
59
|
+
raise ValidationError("timeout must be finite and positive")
|
|
60
|
+
if isinstance(retries, bool) or not isinstance(retries, int) or not 0 <= retries <= 10:
|
|
61
|
+
raise ValidationError("retries must be an integer from 0 to 10")
|
|
62
|
+
if not math.isfinite(backoff) or backoff < 0:
|
|
63
|
+
raise ValidationError("backoff must be finite and nonnegative")
|
|
64
|
+
if not math.isfinite(ttl_seconds) or ttl_seconds < 0:
|
|
65
|
+
raise ValidationError("ttl_seconds must be finite and nonnegative")
|
|
66
|
+
if offline and cache_dir is None:
|
|
67
|
+
raise ValidationError("offline mode requires cache_dir")
|
|
68
|
+
self.opener = opener
|
|
69
|
+
self.timeout = timeout
|
|
70
|
+
self.retries = retries
|
|
71
|
+
self.backoff = backoff
|
|
72
|
+
self.cache_dir = Path(cache_dir) if cache_dir is not None else None
|
|
73
|
+
self.ttl_seconds = ttl_seconds
|
|
74
|
+
self.offline = offline
|
|
75
|
+
self.allow_expired_offline = allow_expired_offline
|
|
76
|
+
self.sleep = sleep
|
|
77
|
+
self.clock = clock
|
|
78
|
+
|
|
79
|
+
def get(self, endpoint: str, params: dict[str, str] | None = None) -> tuple[bytes, dict]:
|
|
80
|
+
"""Retrieve bytes and provenance; respect offline/expiry before network."""
|
|
81
|
+
url = endpoint + ("?" + urlencode(params) if params else "")
|
|
82
|
+
cache_path = (
|
|
83
|
+
self.cache_dir / (hashlib.sha256(url.encode("utf-8")).hexdigest() + ".json")
|
|
84
|
+
if self.cache_dir is not None else None
|
|
85
|
+
)
|
|
86
|
+
if cache_path is not None and cache_path.exists():
|
|
87
|
+
try:
|
|
88
|
+
cached = json.loads(cache_path.read_text(encoding="utf-8"))
|
|
89
|
+
fetched = datetime.fromisoformat(cached["retrieved_at"])
|
|
90
|
+
age = self.clock() - fetched.timestamp()
|
|
91
|
+
if fetched.tzinfo is None or age < 0 or not isinstance(cached["body"], str):
|
|
92
|
+
raise ValueError("invalid cached payload")
|
|
93
|
+
if age < self.ttl_seconds or (self.offline and self.allow_expired_offline):
|
|
94
|
+
return cached["body"].encode("utf-8"), self._provenance(
|
|
95
|
+
url, cached["retrieved_at"], True, age >= self.ttl_seconds
|
|
96
|
+
)
|
|
97
|
+
except (OSError, ValueError, TypeError, KeyError) as exc:
|
|
98
|
+
raise OfflineCacheError(f"Invalid cache entry for {url}") from exc
|
|
99
|
+
if self.offline:
|
|
100
|
+
raise OfflineCacheError(f"No unexpired cached response for {url}")
|
|
101
|
+
|
|
102
|
+
for attempt in range(self.retries + 1):
|
|
103
|
+
try:
|
|
104
|
+
with self.opener(Request(url, headers={"User-Agent": "nielytics/0 (research data client)"}), timeout=self.timeout) as response:
|
|
105
|
+
body = response.read()
|
|
106
|
+
status = response.status
|
|
107
|
+
headers = getattr(response, "headers", {})
|
|
108
|
+
if status in (429, 500, 502, 503, 504):
|
|
109
|
+
raise HTTPError(url, status, "Provider temporarily unavailable", headers, None)
|
|
110
|
+
if status != 200:
|
|
111
|
+
raise ProviderError(f"Provider returned HTTP {status}: {url}")
|
|
112
|
+
if not body:
|
|
113
|
+
raise ResponseError(f"Provider returned an empty response: {url}")
|
|
114
|
+
retrieved_at = datetime.fromtimestamp(self.clock(), tz=UTC).isoformat()
|
|
115
|
+
if cache_path is not None:
|
|
116
|
+
try:
|
|
117
|
+
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
|
118
|
+
temporary = cache_path.with_name(cache_path.name + f".{os.getpid()}.tmp")
|
|
119
|
+
temporary.write_text(json.dumps({"retrieved_at": retrieved_at, "body": body.decode("utf-8")}), encoding="utf-8")
|
|
120
|
+
temporary.replace(cache_path)
|
|
121
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
122
|
+
raise ProviderError(f"Cannot save provider cache at {cache_path}") from exc
|
|
123
|
+
return body, self._provenance(url, retrieved_at, False, False)
|
|
124
|
+
except HTTPError as exc:
|
|
125
|
+
if exc.code not in (429, 500, 502, 503, 504) or attempt == self.retries:
|
|
126
|
+
raise ProviderError(f"Provider returned HTTP {exc.code}: {url}") from exc
|
|
127
|
+
retry_after = exc.headers.get("Retry-After") if exc.headers else None
|
|
128
|
+
delay = self._retry_after(retry_after)
|
|
129
|
+
except (URLError, TimeoutError, OSError) as exc:
|
|
130
|
+
if attempt == self.retries:
|
|
131
|
+
raise ProviderError(f"Could not retrieve {url}: {exc}") from exc
|
|
132
|
+
delay = min(60, self.backoff * 2**attempt)
|
|
133
|
+
self.sleep(delay)
|
|
134
|
+
raise AssertionError("unreachable")
|
|
135
|
+
|
|
136
|
+
def _retry_after(self, value: str | None) -> float:
|
|
137
|
+
if value:
|
|
138
|
+
try:
|
|
139
|
+
return max(0, min(60, float(value)))
|
|
140
|
+
except ValueError:
|
|
141
|
+
try:
|
|
142
|
+
return max(0, min(60, (parsedate_to_datetime(value) - datetime.fromtimestamp(self.clock(), UTC)).total_seconds()))
|
|
143
|
+
except (ValueError, TypeError, OverflowError):
|
|
144
|
+
pass
|
|
145
|
+
return min(60, self.backoff)
|
|
146
|
+
|
|
147
|
+
@staticmethod
|
|
148
|
+
def _provenance(url: str, retrieved_at: str, cached: bool, expired: bool) -> dict:
|
|
149
|
+
return {"source_url": url, "retrieved_at": retrieved_at, "cached": cached, "cache_expired": expired}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def parse_json(body: bytes) -> dict:
|
|
153
|
+
"""Decode a provider JSON object and surface provider-specific errors."""
|
|
154
|
+
try:
|
|
155
|
+
value = json.loads(body)
|
|
156
|
+
except (ValueError, UnicodeDecodeError) as exc:
|
|
157
|
+
raise ResponseError("Provider returned invalid JSON") from exc
|
|
158
|
+
if not isinstance(value, dict):
|
|
159
|
+
raise ResponseError("Provider response must be a JSON object")
|
|
160
|
+
if value.get("error"):
|
|
161
|
+
raise ResponseError(f"Provider rejected request: {value.get('reason', value['error'])}")
|
|
162
|
+
return value
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def coordinates(latitude: float, longitude: float) -> None:
|
|
166
|
+
"""Require finite WGS84 scalar coordinates."""
|
|
167
|
+
for name, value, limit in (("latitude", latitude, 90), ("longitude", longitude, 180)):
|
|
168
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or abs(value) > limit:
|
|
169
|
+
raise ValidationError(f"{name} must be a finite WGS84 coordinate within ±{limit}")
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def date_range(start_date: str, end_date: str) -> None:
|
|
173
|
+
"""Validate strict ISO calendar dates in chronological order."""
|
|
174
|
+
from datetime import date
|
|
175
|
+
|
|
176
|
+
try:
|
|
177
|
+
start = date.fromisoformat(start_date)
|
|
178
|
+
end = date.fromisoformat(end_date)
|
|
179
|
+
if start.isoformat() != start_date or end.isoformat() != end_date or start > end:
|
|
180
|
+
raise ValueError
|
|
181
|
+
except (ValueError, TypeError) as exc:
|
|
182
|
+
raise ValidationError("start_date and end_date must be YYYY-MM-DD and ordered") from exc
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""NASA's one-time Global Landslide Catalog CSV export (not live warnings)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import io
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from nielytics.exceptions import ValidationError
|
|
11
|
+
from nielytics.models.schemas import AnalysisResult
|
|
12
|
+
|
|
13
|
+
from ._http import HTTPClient, ResponseError, coordinates
|
|
14
|
+
|
|
15
|
+
CATALOG_URL = "https://data.nasa.gov/docs/legacy/Global_Landslide_Catalog_Export/Global_Landslide_Catalog_Export_rows.csv"
|
|
16
|
+
CATALOG_PAGE = "https://data.nasa.gov/dataset/global-landslide-catalog-export"
|
|
17
|
+
_REQUIRED = {"event_id", "event_date", "longitude", "latitude", "country_code", "landslide_trigger"}
|
|
18
|
+
_LIMITATIONS = [
|
|
19
|
+
"NASA labels this a one-time export current as of March 7, 2016, although the published CSV contains later-dated rows; not a live warning feed.",
|
|
20
|
+
"Media and other reports are incomplete and location/date accuracy varies by event.",
|
|
21
|
+
"NASA requests agreement to catalog terms and citation of Kirschbaum et al. (2010, 2015) for exports.",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _parse_catalog(body: bytes, provenance: dict) -> AnalysisResult:
|
|
26
|
+
try:
|
|
27
|
+
reader = csv.DictReader(io.StringIO(body.decode("utf-8-sig"), newline=""))
|
|
28
|
+
if not reader.fieldnames or not _REQUIRED.issubset(reader.fieldnames) or len(set(reader.fieldnames)) != len(reader.fieldnames):
|
|
29
|
+
raise ResponseError("NASA GLC CSV is missing required columns or has duplicate headers")
|
|
30
|
+
records = []
|
|
31
|
+
for row in reader:
|
|
32
|
+
if None in row or any(value is None for value in row.values()):
|
|
33
|
+
raise ResponseError("NASA GLC CSV contains a row with an inconsistent number of columns")
|
|
34
|
+
if all(not value for value in row.values()):
|
|
35
|
+
continue
|
|
36
|
+
event_id = row["event_id"].strip()
|
|
37
|
+
if not event_id:
|
|
38
|
+
raise ResponseError("NASA GLC CSV contains an event without event_id")
|
|
39
|
+
raw_date = row["event_date"].strip()
|
|
40
|
+
try:
|
|
41
|
+
event_date = datetime.strptime(raw_date, "%m/%d/%Y %I:%M:%S %p").date().isoformat() if raw_date else None
|
|
42
|
+
except ValueError as exc:
|
|
43
|
+
raise ResponseError(f"NASA GLC event {event_id} has invalid event_date") from exc
|
|
44
|
+
lon_text, lat_text = row["longitude"].strip(), row["latitude"].strip()
|
|
45
|
+
longitude = latitude = None
|
|
46
|
+
if lon_text and lat_text:
|
|
47
|
+
try:
|
|
48
|
+
longitude, latitude = float(lon_text), float(lat_text)
|
|
49
|
+
coordinates(latitude, longitude)
|
|
50
|
+
except (ValueError, ValidationError) as exc:
|
|
51
|
+
raise ResponseError(f"NASA GLC event {event_id} has invalid coordinates") from exc
|
|
52
|
+
records.append({"event_id": event_id, "date": event_date, "event_date_raw": raw_date or None,
|
|
53
|
+
"longitude": longitude, "latitude": latitude,
|
|
54
|
+
"country_code": row["country_code"].strip() or None,
|
|
55
|
+
"landslide_trigger": row["landslide_trigger"].strip() or None,
|
|
56
|
+
"location_accuracy": row.get("location_accuracy", "").strip() or None})
|
|
57
|
+
except (UnicodeDecodeError, csv.Error, AttributeError) as exc:
|
|
58
|
+
raise ResponseError("NASA GLC catalog is not a valid UTF-8 CSV export") from exc
|
|
59
|
+
if not records:
|
|
60
|
+
raise ResponseError("NASA GLC catalog contains no events")
|
|
61
|
+
dated = [record["date"] for record in records if record["date"] is not None]
|
|
62
|
+
return AnalysisResult(feature="landslide_events", data={"records": records}, metadata={
|
|
63
|
+
"source": "NASA Global Landslide Catalog (GLC) one-time export",
|
|
64
|
+
"source_dataset": CATALOG_PAGE, "data_type": "historical_catalog",
|
|
65
|
+
"coverage": {"start": min(dated, default=None), "end": max(dated, default=None)},
|
|
66
|
+
"units": {"longitude": "degrees_east", "latitude": "degrees_north"},
|
|
67
|
+
"missing_value": "Blank cells normalized to null; a missing coordinate suppresses both coordinates",
|
|
68
|
+
"event_time_standard": "Catalog times are preserved as supplied; UTC offset is not specified",
|
|
69
|
+
"attribution": "NASA GLC; cite Kirschbaum et al. (2010) doi:10.1007/s11069-009-9401-4 and Kirschbaum et al. (2015) doi:10.1016/j.geomorph.2015.03.016",
|
|
70
|
+
"license": "Not specified on NASA Open Data resource page; review catalog terms before use",
|
|
71
|
+
"limitations": _LIMITATIONS.copy(), **provenance,
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def fetch_landslide_events(*, accept_terms: bool = False, client: HTTPClient | None = None) -> AnalysisResult:
|
|
76
|
+
"""Download the verified historic CSV only after user accepts NASA catalog terms."""
|
|
77
|
+
if accept_terms is not True:
|
|
78
|
+
raise ValidationError(f"Review {CATALOG_PAGE} and explicitly accept its export terms before downloading")
|
|
79
|
+
body, provenance = (client or HTTPClient()).get(CATALOG_URL)
|
|
80
|
+
return _parse_catalog(body, provenance)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def load_landslide_catalog(path: str | Path) -> AnalysisResult:
|
|
84
|
+
"""Load a previously downloaded GLC CSV locally without contacting NASA."""
|
|
85
|
+
try:
|
|
86
|
+
data = Path(path).read_bytes()
|
|
87
|
+
except (OSError, TypeError, ValueError) as exc:
|
|
88
|
+
raise ValidationError(f"Cannot read local GLC catalog: {path}") from exc
|
|
89
|
+
return _parse_catalog(data, {"source_url": CATALOG_URL, "retrieved_at": None,
|
|
90
|
+
"cached": False, "cache_expired": None, "local_file": str(path)})
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def filter_landslides_by_region(
|
|
94
|
+
result: AnalysisResult, *, bbox: tuple[float, float, float, float] | None = None,
|
|
95
|
+
country_code: str | None = None,
|
|
96
|
+
) -> AnalysisResult:
|
|
97
|
+
"""Filter geolocated catalog events by WGS84 bbox and/or recorded country code."""
|
|
98
|
+
if result.feature != "landslide_events":
|
|
99
|
+
raise ValidationError("A landslide_events result is required")
|
|
100
|
+
if bbox is None and country_code is None:
|
|
101
|
+
raise ValidationError("Specify bbox and/or country_code")
|
|
102
|
+
if country_code is not None and (not isinstance(country_code, str) or len(country_code) != 2 or not country_code.isalpha()):
|
|
103
|
+
raise ValidationError("country_code must be a two-letter ISO code")
|
|
104
|
+
if bbox is not None:
|
|
105
|
+
if not isinstance(bbox, (tuple, list)) or len(bbox) != 4:
|
|
106
|
+
raise ValidationError("bbox must be (west, south, east, north)")
|
|
107
|
+
west, south, east, north = bbox
|
|
108
|
+
coordinates(south, west)
|
|
109
|
+
coordinates(north, east)
|
|
110
|
+
if west > east or south > north:
|
|
111
|
+
raise ValidationError("bbox boundaries must be ordered")
|
|
112
|
+
records = [record for record in result.data["records"] if
|
|
113
|
+
(country_code is None or record["country_code"] == country_code.upper()) and
|
|
114
|
+
(bbox is None or (record["longitude"] is not None and west <= record["longitude"] <= east
|
|
115
|
+
and south <= record["latitude"] <= north))]
|
|
116
|
+
dated = [record["date"] for record in records if record["date"] is not None]
|
|
117
|
+
return AnalysisResult(feature="landslide_events", data={"records": records}, metadata={
|
|
118
|
+
**result.metadata, "coverage": {"start": min(dated, default=None), "end": max(dated, default=None)},
|
|
119
|
+
"region_filter": {"bbox": list(bbox) if bbox is not None else None,
|
|
120
|
+
"country_code": country_code.upper() if country_code else None},
|
|
121
|
+
})
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def landslide_events_to_geojson(result: AnalysisResult) -> dict:
|
|
125
|
+
"""Export only valid catalog point positions; missing positions are omitted."""
|
|
126
|
+
from nielytics.geospatial.geojson import points_to_geojson
|
|
127
|
+
|
|
128
|
+
if result.feature != "landslide_events":
|
|
129
|
+
raise ValidationError("A landslide_events result is required")
|
|
130
|
+
records = result.data["records"]
|
|
131
|
+
points = ({"id": row["event_id"], "longitude": row["longitude"], "latitude": row["latitude"],
|
|
132
|
+
"properties": {"event_date": row["date"], "event_date_raw": row["event_date_raw"],
|
|
133
|
+
"country_code": row["country_code"],
|
|
134
|
+
"landslide_trigger": row["landslide_trigger"],
|
|
135
|
+
"location_accuracy": row["location_accuracy"]},
|
|
136
|
+
"provenance": {"source_url": result.metadata["source_url"],
|
|
137
|
+
"retrieved_at": result.metadata["retrieved_at"]}}
|
|
138
|
+
for row in records if row["longitude"] is not None and row["latitude"] is not None)
|
|
139
|
+
coverage = result.metadata["coverage"]
|
|
140
|
+
return points_to_geojson(points, crs="EPSG:4326", source=result.metadata["source"],
|
|
141
|
+
period=f"{coverage['start']} to {coverage['end']}" if coverage["start"] is not None else None,
|
|
142
|
+
units="WGS84 decimal degrees", method="direct catalog event coordinate export",
|
|
143
|
+
limitations=[*result.metadata["limitations"],
|
|
144
|
+
f"{sum(row['longitude'] is None or row['latitude'] is None for row in records)} records without coordinates omitted."])
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""NASA POWER modeled daily point precipitation and air temperature."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from datetime import date, datetime
|
|
7
|
+
|
|
8
|
+
from nielytics.exceptions import ValidationError
|
|
9
|
+
from nielytics.models.schemas import AnalysisResult
|
|
10
|
+
|
|
11
|
+
from ._http import HTTPClient, ResponseError, coordinates, date_range, parse_json
|
|
12
|
+
|
|
13
|
+
POWER_URL = "https://power.larc.nasa.gov/api/temporal/daily/point"
|
|
14
|
+
_PARAMETERS = {"PRECTOTCORR": "mm/day", "T2M_MAX": "C"}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def fetch_nasa_power_data(
|
|
18
|
+
latitude: float,
|
|
19
|
+
longitude: float,
|
|
20
|
+
start_date: str,
|
|
21
|
+
end_date: str,
|
|
22
|
+
*,
|
|
23
|
+
parameters: tuple[str, ...] = ("PRECTOTCORR", "T2M_MAX"),
|
|
24
|
+
client: HTTPClient | None = None,
|
|
25
|
+
) -> AnalysisResult:
|
|
26
|
+
"""Fetch verified AG daily parameters in Local Solar Time (LST).
|
|
27
|
+
|
|
28
|
+
POWER ``header.fill_value`` is converted to None; returned parameter
|
|
29
|
+
units must match the documented values. No arbitrary parameter schema is
|
|
30
|
+
assumed. Historic data are modeled/gridded, not gauge declarations.
|
|
31
|
+
"""
|
|
32
|
+
coordinates(latitude, longitude)
|
|
33
|
+
date_range(start_date, end_date)
|
|
34
|
+
if start_date < "1981-01-01" or date.fromisoformat(end_date) >= date.today():
|
|
35
|
+
raise ValidationError("POWER daily historical dates must be from 1981-01-01 through yesterday")
|
|
36
|
+
if (not isinstance(parameters, (tuple, list)) or not parameters or
|
|
37
|
+
any(not isinstance(parameter, str) for parameter in parameters) or
|
|
38
|
+
len(set(parameters)) != len(parameters) or any(parameter not in _PARAMETERS for parameter in parameters)):
|
|
39
|
+
raise ValidationError(f"parameters must be unique values from {tuple(_PARAMETERS)}")
|
|
40
|
+
params = {"parameters": ",".join(parameters), "community": "AG", "longitude": str(longitude),
|
|
41
|
+
"latitude": str(latitude), "start": start_date.replace("-", ""), "end": end_date.replace("-", ""),
|
|
42
|
+
"format": "JSON", "time-standard": "LST"}
|
|
43
|
+
body, provenance = (client or HTTPClient()).get(POWER_URL, params)
|
|
44
|
+
payload = parse_json(body)
|
|
45
|
+
header, props, units_meta = payload.get("header"), payload.get("properties"), payload.get("parameters")
|
|
46
|
+
if not isinstance(header, dict) or not isinstance(props, dict) or not isinstance(units_meta, dict) or not isinstance(props.get("parameter"), dict):
|
|
47
|
+
raise ResponseError("POWER response missing header, properties.parameter, or parameters units")
|
|
48
|
+
if header.get("time_standard") != "LST":
|
|
49
|
+
raise ResponseError("POWER returned an unexpected time standard")
|
|
50
|
+
fill = header.get("fill_value")
|
|
51
|
+
if isinstance(fill, bool) or not isinstance(fill, (int, float)) or not math.isfinite(fill):
|
|
52
|
+
raise ResponseError("POWER header.fill_value is missing or invalid")
|
|
53
|
+
values = props["parameter"]
|
|
54
|
+
dates: set[str] | None = None
|
|
55
|
+
for parameter in parameters:
|
|
56
|
+
if not isinstance(values.get(parameter), dict) or not isinstance(units_meta.get(parameter), dict) or units_meta[parameter].get("units") != _PARAMETERS[parameter]:
|
|
57
|
+
raise ResponseError(f"POWER missing/invalid parameter {parameter} or units")
|
|
58
|
+
these_dates = set(values[parameter])
|
|
59
|
+
if dates is not None and dates != these_dates:
|
|
60
|
+
raise ResponseError("POWER parameter date coverage is inconsistent")
|
|
61
|
+
dates = these_dates
|
|
62
|
+
if not dates:
|
|
63
|
+
raise ResponseError("POWER response contains no daily observations")
|
|
64
|
+
records = []
|
|
65
|
+
for raw_date in sorted(dates):
|
|
66
|
+
try:
|
|
67
|
+
parsed = datetime.strptime(raw_date, "%Y%m%d").date()
|
|
68
|
+
if parsed.strftime("%Y%m%d") != raw_date or not start_date <= parsed.isoformat() <= end_date:
|
|
69
|
+
raise ValueError
|
|
70
|
+
except (ValueError, TypeError) as exc:
|
|
71
|
+
raise ResponseError("POWER returned an invalid or out-of-range observation date") from exc
|
|
72
|
+
row = {"date": parsed.isoformat()}
|
|
73
|
+
for parameter in parameters:
|
|
74
|
+
value = values[parameter][raw_date]
|
|
75
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
|
76
|
+
raise ResponseError(f"POWER returned invalid value for {parameter} on {raw_date}")
|
|
77
|
+
if parameter == "PRECTOTCORR" and value < 0 and value != fill:
|
|
78
|
+
raise ResponseError("POWER precipitation cannot be negative except declared fill")
|
|
79
|
+
row[parameter] = None if value == fill else value
|
|
80
|
+
records.append(row)
|
|
81
|
+
if records[0]["date"] != start_date or records[-1]["date"] != end_date or len(records) != (date.fromisoformat(end_date) - date.fromisoformat(start_date)).days + 1:
|
|
82
|
+
raise ResponseError("POWER returned incomplete daily date coverage")
|
|
83
|
+
geometry = payload.get("geometry")
|
|
84
|
+
if not isinstance(geometry, dict) or geometry.get("type") != "Point" or not isinstance(geometry.get("coordinates"), list) or len(geometry["coordinates"]) < 2:
|
|
85
|
+
raise ResponseError("POWER response missing Point geometry")
|
|
86
|
+
lon, lat = geometry["coordinates"][:2]
|
|
87
|
+
try:
|
|
88
|
+
coordinates(lat, lon)
|
|
89
|
+
except ValidationError as exc:
|
|
90
|
+
raise ResponseError("POWER response has invalid point coordinates") from exc
|
|
91
|
+
return AnalysisResult(feature="nasa_power_daily", data={"records": records}, metadata={
|
|
92
|
+
"source": "NASA POWER Daily Point API", "source_dataset": "POWER daily meteorology",
|
|
93
|
+
"data_type": "historical_modeled", "requested_location": {"latitude": latitude, "longitude": longitude},
|
|
94
|
+
"grid_cell": {"latitude": lat, "longitude": lon}, "time_standard": "LST",
|
|
95
|
+
"units": {parameter: _PARAMETERS[parameter] for parameter in parameters},
|
|
96
|
+
"coverage": {"start": records[0]["date"], "end": records[-1]["date"]},
|
|
97
|
+
"missing_value": f"header.fill_value ({fill}) normalized to null", "provider_sources": header.get("sources", []),
|
|
98
|
+
"service_version": header.get("api", {}).get("version") if isinstance(header.get("api"), dict) else None,
|
|
99
|
+
"limitations": ["Gridded modeled daily values, not station records or disaster declarations.",
|
|
100
|
+
"POWER LST days differ from UTC and civil timezone boundaries.",
|
|
101
|
+
"Meteorological parameters generally have 2–3 days of publication latency."],
|
|
102
|
+
"attribution": "NASA POWER; cite POWER project and Daily service version and accessed date", **provenance,
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def fetch_nasa_precipitation(latitude: float, longitude: float, start_date: str, end_date: str, *, client: HTTPClient | None = None) -> AnalysisResult:
|
|
107
|
+
"""Fetch POWER corrected daily precipitation PRECTOTCORR in mm/day."""
|
|
108
|
+
return fetch_nasa_power_data(latitude, longitude, start_date, end_date, parameters=("PRECTOTCORR",), client=client)
|