mostlyrightmd-weather 0.1.2__py3-none-any.whl

mostlyright/weather/__init__.py

@@ -0,0 +1,36 @@
+"""mostlyright.weather — direct public-API access for AWC, IEM, GHCNh, NWS CLI.
+
+Local-first; no hosted backend; no API keys. Parsers are byte-faithful lifts
+from ``monorepo-v0.14.1``; HTTP fetchers and the parquet cache are net-new
+Sprint 0 code so the SDK can run without the v0.14.1 ingest service.
+
+Lift inventory (provenance for parity-critical code). Source SHA refers to
+the v0.14.1 release tag of ``Tarabcak/monorepo`` (commit
+``514fcdab227e845145ca32b989355647466231d9``).
+
+| Module                  | Source path                                            | Source SHA | Lift date  | Modifications                                                       |
+|-------------------------|--------------------------------------------------------|------------|------------|---------------------------------------------------------------------|
+| _awc.py                 | monorepo-v0.14.1/src/mostlyright/weather/_awc.py       | 514fcda    | 2026-05-21 | namespace rename only (imports point at ``mostlyright._internal``)   |
+| _iem.py                 | monorepo-v0.14.1/src/mostlyright/weather/_iem.py       | 514fcda    | 2026-05-21 | namespace rename only                                               |
+| _climate.py             | monorepo-v0.14.1/src/mostlyright/weather/_climate.py   | 514fcda    | 2026-05-21 | namespace rename only                                               |
+| _ghcnh.py               | monorepo-v0.14.1/src/mostlyright/weather/_ghcnh.py     | 514fcda    | 2026-05-21 | namespace rename only                                               |
+| _fetchers/__init__.py   | n/a (NEW)                                              | n/a        | 2026-05-21 | NEW (Sprint 0 Wave 1 Lane F) — fetcher package marker               |
+| _fetchers/awc.py        | n/a (NEW)                                              | n/a        | 2026-05-21 | NEW (Sprint 0 Wave 1 Lane F) — historical AWC range fetcher         |
+| _fetchers/iem_asos.py   | n/a (NEW)                                              | n/a        | 2026-05-21 | NEW (Sprint 0 Wave 1 Lane F) — monthly-chunked IEM ASOS METAR fetcher |
+| _fetchers/iem_cli.py    | n/a (NEW)                                              | n/a        | 2026-05-21 | NEW (Sprint 0 Wave 1 Lane F) — IEM CLI settlement-grade fetcher     |
+| _fetchers/ghcnh.py      | n/a (NEW)                                              | n/a        | 2026-05-21 | NEW (Sprint 0 Wave 1 Lane F) — per-year NCEI GHCNh PSV fetcher      |
+| cache.py                | n/a (NEW)                                              | n/a        | 2026-05-21 | NEW (Sprint 0 Wave 1 Lane F) — local parquet cache, filelock-guarded |
+
+``_bounds`` is imported from ``mostlyright._internal`` (lifted there from
+``monorepo-v0.14.1/src/mostlyright/_bounds.py``) — see the parallel lift
+inventory in ``mostlyright._internal.__init__``.
+
+Public surface kept stable for Vojtech's existing ``mostlyright==0.14.1``
+workflow: ``raw_metar`` is preserved on observation rows so MetPy re-parse
+keeps working without preprocessing in v0.1.0.
+"""
+
+from mostlyright.weather.obs import obs as obs  # re-export Phase 7 public surface
+
+__version__ = "0.1.0rc1"
+__all__ = ["__version__", "obs"]

mostlyright/weather/_awc.py

@@ -0,0 +1,347 @@
+"""AWC METAR transform — maps AWC JSON response to observation schema dict.
+
+This is THE shared transform. Both the SDK and ingest worker import it.
+Output dicts validate against specs/observation.json (additionalProperties: false).
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from datetime import UTC, datetime
+from typing import Any
+
+from mostlyright._internal._bounds import (
+    MAX_RAW_METAR_LEN,
+    MAX_VISIBILITY_MILES,
+    MAX_WX_CODES_LEN,
+    SKY_BASE_MAX_FT,
+    SLP_MAX_MB,
+    SLP_MIN_MB,
+    STATION_CODE_RE,
+    TEMP_MAX_C,
+    TEMP_MIN_C,
+    WIND_DIR_BOUNDS,
+    WIND_GUST_MAX,
+    WIND_SPEED_MAX,
+    bounded_float,
+    bounded_float_min,
+    bounded_int,
+)
+from mostlyright._internal._convert import celsius_to_fahrenheit, hpa_to_inhg
+
+
+def icao_to_station_code(icao: str) -> str:
+    """Strip leading K for 4-letter CONUS ICAO codes."""
+    upper = icao.strip().upper()
+    if upper.startswith("K") and len(upper) == 4:
+        return upper[1:]
+    return upper
+
+
+def parse_awc_visibility(vis: Any) -> float | None:
+    """Parse AWC visibility: '10+', '1/2', '2 1/4', '3/4', plain numbers.
+
+    Returns miles or None. Caps at 99.99.
+    """
+    if vis is None:
+        return None
+
+    s = str(vis)
+    if s == "" or s == "null":
+        return None
+
+    # "10+" -> 10
+    if s.endswith("+"):
+        try:
+            n = float(s[:-1])
+        except (ValueError, OverflowError):
+            return None
+        if not math.isfinite(n):
+            return None
+        return min(n, MAX_VISIBILITY_MILES)
+
+    # Mixed number: "1 1/2", "2 1/4"
+    if " " in s and "/" in s:
+        parts = s.split(" ", 1)
+        if len(parts) != 2:
+            return None
+        frac_parts = parts[1].split("/")
+        if len(frac_parts) != 2:
+            return None
+        try:
+            w = float(parts[0])
+            n = float(frac_parts[0])
+            d = float(frac_parts[1])
+        except (ValueError, OverflowError):
+            return None
+        if not (math.isfinite(w) and math.isfinite(n) and math.isfinite(d) and d != 0):
+            return None
+        return min(w + n / d, MAX_VISIBILITY_MILES)
+
+    # Simple fraction: "1/2", "1/4", "3/4", or "M1/4" (below-quarter-mile
+    # AWC/METAR convention — codex review W3A P2). The leading 'M' means
+    # "less than", which we represent as the same fractional value (the
+    # observation schema treats this as the visibility value, not a flag).
+    if "/" in s:
+        if s.startswith("M") or s.startswith("m"):
+            s = s[1:]
+        frac_parts = s.split("/")
+        if len(frac_parts) != 2:
+            return None
+        try:
+            n = float(frac_parts[0])
+            d = float(frac_parts[1])
+        except (ValueError, OverflowError):
+            return None
+        if not (math.isfinite(n) and math.isfinite(d) and d != 0):
+            return None
+        return min(n / d, MAX_VISIBILITY_MILES)
+
+    # Plain number
+    try:
+        n = float(s)
+    except (ValueError, OverflowError):
+        return None
+    if not math.isfinite(n):
+        return None
+    return min(n, MAX_VISIBILITY_MILES)
+
+
+def map_cloud_cover(cover: str | None) -> str | None:
+    """Map AWC cloud cover code to standard abbreviation."""
+    if cover is None:
+        return None
+    upper = cover.upper()
+    if upper in ("CLR", "SKC", "FEW", "SCT", "BKN", "OVC", "VV"):
+        return upper
+    if upper == "CAVOK":
+        return "CLR"
+    return None
+
+
+def _cloud_layer(layer: Any) -> tuple[str | None, int | None]:
+    """Extract cover and base from a cloud layer dict. Safe against non-dict entries."""
+    if not isinstance(layer, dict):
+        return None, None
+    base = bounded_int(_safe_int(layer.get("base")), 0, SKY_BASE_MAX_FT)
+    return map_cloud_cover(layer.get("cover")), base
+
+
+def _safe_int(v: Any) -> int | None:
+    """Convert to int. Returns None on bad input."""
+    if v is None:
+        return None
+    try:
+        f = float(v)
+        if not math.isfinite(f):
+            return None
+        return round(f)
+    except (ValueError, TypeError, OverflowError):
+        return None
+
+
+def _safe_float(v: Any) -> float | None:
+    """Convert to float. Returns None on bad input."""
+    if v is None:
+        return None
+    try:
+        f = float(v)
+        return f if math.isfinite(f) else None
+    except (ValueError, TypeError, OverflowError):
+        return None
+
+
+def _safe_precip(v: Any) -> float | None:
+    """Parse precipitation. Trace 'T' → 0.0, numeric passthrough, else None."""
+    if v is None:
+        return None
+    if isinstance(v, str) and v.strip().upper() == "T":
+        return 0.0
+    return _safe_float(v)
+
+
+_PK_WND_RE = re.compile(r"PK WND (\d{3})(\d{2,3})/(\d{4})")
+
+# T-group in METAR remarks: T{s}{SSS}{s}{DDD}
+# s=0 positive, s=1 negative. SSS/DDD = tenths of °C.
+# Example: T02560167 → 25.6°C / 16.7°C. T10390061 → -3.9°C / 6.1°C.
+_TGROUP_RE = re.compile(r"\bT([01])(\d{3})([01])(\d{3})\b")
+
+
+def _parse_peak_wind(
+    raw_metar: str | None,
+) -> tuple[int | None, int | None, str | None]:
+    """Parse PK WND from METAR remarks. Returns (dir, speed_kt, time_hhmm)."""
+    if not raw_metar:
+        return None, None, None
+    match = _PK_WND_RE.search(raw_metar)
+    if not match:
+        return None, None, None
+    direction = int(match.group(1))
+    speed = int(match.group(2))
+    time_hhmm = match.group(3)
+    if not (0 <= direction <= 360) or speed < 0:
+        return None, None, None
+    return direction, speed, time_hhmm
+
+
+def _parse_tgroup(raw_metar: str | None) -> tuple[float | None, float | None]:
+    """Parse T-group from METAR remarks for tenths-precision temperature.
+
+    ASOS stations always include T-group in remarks. Format: T{s}{SSS}{s}{DDD}
+    where s=0 positive, s=1 negative, SSS=temp tenths °C, DDD=dewpoint tenths °C.
+    Searches only the remarks section (after RMK) to avoid false positives.
+    Returns (temp_c, dewpoint_c) or (None, None) if not found.
+    """
+    if not raw_metar:
+        return None, None
+    # T-group is a remarks-only element — search only after RMK.
+    # No RMK section = no T-group. Do NOT fallback to full string
+    # to avoid false positives on body group patterns.
+    rmk_idx = raw_metar.find("RMK")
+    if rmk_idx < 0:
+        return None, None
+    match = _TGROUP_RE.search(raw_metar[rmk_idx:])
+    if not match:
+        return None, None
+    t_sign = -1 if match.group(1) == "1" else 1
+    t_val = int(match.group(2)) / 10.0 * t_sign
+    d_sign = -1 if match.group(3) == "1" else 1
+    d_val = int(match.group(4)) / 10.0 * d_sign
+    return t_val, d_val
+
+
+def awc_to_observation(m: dict[str, Any]) -> dict[str, Any] | None:
+    """Convert a parsed AWC METAR dict to an observation schema dict.
+
+    Returns None if icaoId or obsTime is invalid.
+    Output matches specs/observation.json (no extra fields).
+    """
+    icao_id = m.get("icaoId")
+    if not isinstance(icao_id, str) or not icao_id:
+        return None
+
+    obs_time = m.get("obsTime")
+    if not isinstance(obs_time, int | float):
+        return None
+
+    station_code = icao_to_station_code(icao_id)
+    if not STATION_CODE_RE.match(station_code):
+        return None
+
+    try:
+        dt = datetime.fromtimestamp(obs_time, tz=UTC)
+    except (OSError, OverflowError, ValueError):
+        return None
+    if not (1970 <= dt.year <= 2100):
+        return None
+    observed_at = dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    metar_type = (m.get("metarType") or "METAR").upper()
+    observation_type = "SPECI" if metar_type == "SPECI" else "METAR"
+
+    # Wind direction: handle "VRB" -> None, bounded [0, 360]
+    wdir: int | None = None
+    raw_wdir = m.get("wdir")
+    if raw_wdir is not None:
+        if isinstance(raw_wdir, int | float):
+            wdir = bounded_int(int(raw_wdir), *WIND_DIR_BOUNDS)
+        elif raw_wdir != "VRB":
+            try:
+                parsed = float(raw_wdir)
+                if math.isfinite(parsed):
+                    wdir = bounded_int(int(parsed), *WIND_DIR_BOUNDS)
+            except (ValueError, TypeError):
+                pass
+
+    wspd = bounded_int(_safe_int(m.get("wspd")), 0, WIND_SPEED_MAX)
+    wgst = bounded_int(_safe_int(m.get("wgst")), 0, WIND_GUST_MAX)
+
+    # Altimeter: AWC altim is in hPa, convert to inHg (no rounding)
+    altim = hpa_to_inhg(_safe_float(m.get("altim")))
+
+    # Sea-level pressure (already in mb/hPa)
+    slp = _safe_float(m.get("slp"))
+    if slp is not None and not (SLP_MIN_MB <= slp <= SLP_MAX_MB):
+        slp = None
+
+    # Cloud layers (safe against non-dict entries)
+    clouds = m.get("clouds") or []
+    cov1, base1 = _cloud_layer(clouds[0]) if len(clouds) > 0 else (None, None)
+    cov2, base2 = _cloud_layer(clouds[1]) if len(clouds) > 1 else (None, None)
+    cov3, base3 = _cloud_layer(clouds[2]) if len(clouds) > 2 else (None, None)
+    cov4, base4 = _cloud_layer(clouds[3]) if len(clouds) > 3 else (None, None)
+
+    # Raw METAR (truncate to 2048)
+    raw_ob = m.get("rawOb")
+    raw_metar: str | None = None
+    if isinstance(raw_ob, str):
+        raw_metar = raw_ob[:MAX_RAW_METAR_LEN]
+
+    # Weather codes
+    raw_wx = m.get("wxString")
+    weather_codes: str | None = None
+    if isinstance(raw_wx, str):
+        weather_codes = raw_wx[:MAX_WX_CODES_LEN]
+
+    # Temperature: T-group (tenths precision) overrides body group (whole degree).
+    # ASOS always includes T-group in remarks. If present, use it.
+    # Note: KNYC (Central Park) is NOT an ASOS station — may lack T-group,
+    # falling back to whole-degree body group temps from AWC.
+    temp_c = _safe_float(m.get("temp"))
+    dewp_c = _safe_float(m.get("dewp"))
+    tgroup_temp, tgroup_dewp = _parse_tgroup(raw_metar)
+    if tgroup_temp is not None:
+        temp_c = tgroup_temp
+    if tgroup_dewp is not None:
+        dewp_c = tgroup_dewp
+    temp_c = bounded_float(temp_c, TEMP_MIN_C, TEMP_MAX_C, field="temp_c")
+    dewp_c = bounded_float(dewp_c, TEMP_MIN_C, TEMP_MAX_C, field="dewpoint_c")
+    temp_f = celsius_to_fahrenheit(temp_c)
+    dewpoint_f = celsius_to_fahrenheit(dewp_c)
+
+    # Peak wind from METAR remarks (PK WND dddss/hhmm), bounded
+    pk_dir, pk_spd, pk_time = _parse_peak_wind(raw_metar)
+    pk_dir = bounded_int(pk_dir, *WIND_DIR_BOUNDS)
+    pk_spd = bounded_int(pk_spd, 0, WIND_GUST_MAX)
+
+    # Precipitation (AWC provides 'precip' field; 'T' = trace → 0.0, non-negative)
+    precip = bounded_float_min(_safe_precip(m.get("precip")), 0.0)
+
+    # QC field bitmask
+    qc_raw = m.get("qcField")
+    qc_field = _safe_int(qc_raw)
+
+    return {
+        "station_code": station_code,
+        "observed_at": observed_at,
+        "observation_type": observation_type,
+        "source": "awc",
+        "temp_c": temp_c,
+        "dewpoint_c": dewp_c,
+        "temp_f": temp_f,
+        "dewpoint_f": dewpoint_f,
+        "wind_dir_degrees": wdir,
+        "wind_speed_kt": wspd,
+        "wind_gust_kt": wgst,
+        "altimeter_inhg": altim,
+        "sea_level_pressure_mb": slp,
+        "sky_cover_1": cov1,
+        "sky_base_1_ft": base1,
+        "sky_cover_2": cov2,
+        "sky_base_2_ft": base2,
+        "sky_cover_3": cov3,
+        "sky_base_3_ft": base3,
+        "sky_cover_4": cov4,
+        "sky_base_4_ft": base4,
+        "visibility_miles": parse_awc_visibility(m.get("visib")),
+        "weather_codes": weather_codes,
+        "precip_1hr_inches": precip,
+        "peak_wind_gust_kt": pk_spd,
+        "peak_wind_dir": pk_dir,
+        "peak_wind_time": pk_time,
+        "snow_depth_inches": None,  # not available from AWC
+        "qc_field": qc_field,
+        "raw_metar": raw_metar,
+    }

mostlyright/weather/_climate.py

@@ -0,0 +1,186 @@
+"""IEM CLI daily climate report parser.
+
+Parses IEM's cli.py JSON endpoint into climate records matching
+specs/climate.json. THE Kalshi settlement source.
+
+Report type priority determines dedup: final (3.0) overwrites
+preliminary (1.0), but a second final never overwrites the first
+(strict >, first-seen wins at equal priority). The overnight final
+IS the Kalshi settlement value.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from datetime import date, datetime, timezone
+from typing import Any
+
+log = logging.getLogger(__name__)
+
+REPORT_TYPE_PRIORITY: dict[str, float] = {
+    "final": 3.0,
+    "ncei_final": 2.5,
+    "correction": 2.0,
+    "preliminary": 1.0,
+    "estimated": 0.0,
+}
+
+# Climate temp bounds from specs/climate.json
+HIGH_TEMP_MIN_F = -60
+HIGH_TEMP_MAX_F = 150
+LOW_TEMP_MIN_F = -80
+LOW_TEMP_MAX_F = 130
+
+_PRODUCT_TS_RE = re.compile(r"^(\d{12})")
+_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+
+
+def _parse_product_timestamp(product: str) -> datetime | None:
+    """Extract issued timestamp from product field.
+
+    Format: "202501160620-KFFC-CDUS42-CLIATL" -> first 12 chars = YYYYMMDDHHmm.
+    """
+    if not product:
+        return None
+    m = _PRODUCT_TS_RE.match(product)
+    if not m:
+        return None
+    try:
+        return datetime.strptime(m.group(1), "%Y%m%d%H%M").replace(tzinfo=timezone.utc)  # noqa: UP017
+    except ValueError:
+        return None
+
+
+def infer_report_type(product: str | None, observation_date: str) -> str:
+    """Infer report type from product timestamp vs observation date.
+
+    - Same day as observation -> preliminary
+    - Next day, 04:00-10:00 UTC -> final (overnight report)
+    - Next day, outside window -> correction
+    - >1 day later -> correction
+    - Unparseable -> preliminary (safe default)
+    """
+    if not product:
+        return "preliminary"
+
+    issued = _parse_product_timestamp(product)
+    if issued is None:
+        return "preliminary"
+
+    try:
+        obs_date = date.fromisoformat(observation_date)
+    except (ValueError, TypeError):
+        return "preliminary"
+
+    issued_date = issued.date()
+    delta_days = (issued_date - obs_date).days
+
+    if delta_days <= 0:
+        return "preliminary"
+    if delta_days == 1:
+        # 04:00-10:00 UTC window: empirically derived for CONUS ASOS stations.
+        # Eastern WFOs issue overnight CLI ~midnight local (04-05 UTC).
+        # Western WFOs (PHX UTC-7) issue ~midnight local (07 UTC).
+        # The window covers all CONUS timezones.
+        if 4 <= issued.hour <= 10:
+            return "final"
+        return "correction"
+    # >1 day later
+    return "correction"
+
+
+def _parse_temp(val: Any) -> int | None:
+    """Parse temperature value. 'M', null, empty -> None. Otherwise int."""
+    if val is None or val == "M" or val == "":
+        return None
+    try:
+        return round(float(val))
+    except (ValueError, TypeError):
+        return None
+
+
+def parse_cli_record(record: dict[str, Any], station_code: str) -> dict[str, Any] | None:
+    """Parse a single IEM CLI record into a climate schema dict.
+
+    Returns dict with 9 fields (8 schema + report_type_priority), or None
+    if both high and low are missing or observation_date is invalid.
+
+    observation_date: Uses IEM's `valid` field directly. IEM CLI data
+    reports by local calendar day (NWS convention). This is THE correct
+    behavior: "March 31 high" means the station's local March 31.
+    """
+    observation_date = record.get("valid")
+    if not observation_date or not isinstance(observation_date, str):
+        return None
+    if not _DATE_RE.match(observation_date):
+        return None
+    # Codex W3A P2: regex matches "2025-02-31" but it's not a real date.
+    # Validate via date.fromisoformat() so invalid dates drop here (the
+    # alternative is infer_report_type() catching the ValueError later and
+    # silently classifying as "preliminary" — would corrupt settlement data).
+    try:
+        date.fromisoformat(observation_date)
+    except ValueError:
+        return None
+
+    high = _parse_temp(record.get("high"))
+    low = _parse_temp(record.get("low"))
+
+    # Bounds check: reject physically impossible settlement values
+    if high is not None and not (HIGH_TEMP_MIN_F <= high <= HIGH_TEMP_MAX_F):
+        log.warning(
+            "%s %s: high_temp_f=%d out of bounds [%d, %d], setting None",
+            station_code,
+            observation_date,
+            high,
+            HIGH_TEMP_MIN_F,
+            HIGH_TEMP_MAX_F,
+        )
+        high = None
+    if low is not None and not (LOW_TEMP_MIN_F <= low <= LOW_TEMP_MAX_F):
+        log.warning(
+            "%s %s: low_temp_f=%d out of bounds [%d, %d], setting None",
+            station_code,
+            observation_date,
+            low,
+            LOW_TEMP_MIN_F,
+            LOW_TEMP_MAX_F,
+        )
+        low = None
+
+    if high is None and low is None:
+        return None
+
+    product = record.get("product") or None
+    report_type = infer_report_type(product, observation_date)
+    priority = REPORT_TYPE_PRIORITY[report_type]
+
+    # Extract issued_at from product timestamp
+    issued_at: str | None = None
+    if product:
+        issued_dt = _parse_product_timestamp(product)
+        if issued_dt:
+            issued_at = issued_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    return {
+        "station_code": station_code,
+        "observation_date": observation_date,
+        "high_temp_f": high,
+        "low_temp_f": low,
+        "report_type": report_type,
+        "report_type_priority": priority,
+        "source": "iem",
+        "product_id": product if isinstance(product, str) else None,
+        "issued_at": issued_at,
+    }
+
+
+def parse_cli_response(data: list[dict[str, Any]], station_code: str) -> list[dict[str, Any]]:
+    """Parse all CLI records, filter None (both temps missing)."""
+    results: list[dict[str, Any]] = []
+    for record in data:
+        parsed = parse_cli_record(record, station_code)
+        if parsed is not None:
+            results.append(parsed)
+    return results

mostlyright/weather/_fetchers/__init__.py

@@ -0,0 +1,20 @@
+"""HTTP fetcher modules for mostlyright.weather data sources.
+
+Sprint 0 Wave 3B (Lane F, net-new code — NOT lifted from mostlyright==0.14.1):
+
+- ``awc`` — AWC live METAR fetcher (last ~168 hours, no historical depth).
+- ``iem_asos`` — IEM ASOS historical METAR (CSV) with monthly chunking + cache.
+- ``iem_cli`` — NWS CLI settlement fetcher via IEM ``json/cli.py``. Year granularity.
+- ``ghcnh`` — NOAA GHCNh historical PSV fetcher with annual cache.
+
+Each fetcher returns either RAW response dicts (AWC) or local file paths
+(IEM ASOS / IEM CLI / GHCNh after ``download_with_retry`` atomic write).
+Callers compose with parsers in :mod:`mostlyright.weather` (``_awc``, ``_iem``,
+``_climate``, ``_ghcnh``) to produce schema-valid observation / climate dicts.
+
+Fetchers do NOT call the parsers directly — parser composition lives in
+``mostlyright.weather.observations.fetch()`` (Wave 4) so the cache layer can
+sit between raw bytes and parsed observations.
+"""
+
+from __future__ import annotations