swpc-kit 0.3.0__py3-none-any.whl

swpc_kit/parse/_common.py

@@ -0,0 +1,353 @@
+"""Helpers shared by the parsers: decoding, error classification, fields.
+
+The JSON products come in two shapes (see ``docs/FONTI.md``): an array of
+objects (since 2026), or an array of arrays whose first row holds the column
+names and whose numbers are quoted (before 2026). :func:`json_records` turns
+both into :class:`Record` objects, whose accessors convert and check one
+field at a time and raise :class:`ParseError` naming the field and record.
+"""
+
+import json
+import math
+import re
+from collections.abc import Callable, Iterable, Mapping
+from datetime import UTC, datetime
+from typing import Any
+
+from swpc_kit.errors import ParseError, SourceNotReady, UnexpectedContent
+from swpc_kit.models.records import SkippedRecord
+
+_TIME = re.compile(
+    r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d{1,6})?)?(?:Z|[+-]\d{2}:\d{2})?"
+)
+
+
+_DECIMAL = re.compile(r"-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?")
+_INTEGER = re.compile(r"-?\d+")
+_PARTIAL_NUMBER = re.compile(r"-?(?:\d+(?:\.\d*)?(?:[eE][+-]?\d*)?)?")
+_LITERALS = ("true", "false", "null")
+
+
+def _cut_inside_a_token(text: str, pos: int) -> bool:
+    """Whether ``text`` ends inside the literal or number that starts near ``pos``.
+
+    A file cut in ``"bz": -1.`` or ``"active": fals`` fails to decode before
+    its end, but it is still a truncated file, not invalid JSON.
+    """
+    start = pos
+    while start > 0 and (text[start - 1].isalnum() or text[start - 1] in "+-."):
+        start -= 1
+    token = text[start:]
+    if not token:
+        return False
+    if any(len(token) < len(word) and word.startswith(token) for word in _LITERALS):
+        return True
+    return _PARTIAL_NUMBER.fullmatch(token) is not None
+
+
+def decode(raw: str | bytes) -> str:
+    """Return ``raw`` as text, without a leading byte order mark.
+
+    Raises:
+        UnexpectedContent: If the bytes are not UTF-8.
+    """
+    if isinstance(raw, bytes):
+        try:
+            return raw.decode("utf-8-sig")
+        except UnicodeDecodeError as exc:
+            raise UnexpectedContent(f"content is not UTF-8 text: {exc}") from None
+    return raw.removeprefix("")
+
+
+def load_json(raw: str | bytes) -> Any:
+    """Decode a JSON product, classifying what is wrong with it.
+
+    Returns:
+        The decoded JSON value.
+
+    Raises:
+        SourceNotReady: If the content is empty or the JSON stops early (SWPC
+            serving a file while rewriting it).
+        UnexpectedContent: If the content is HTML or XML, or not JSON.
+    """
+    text = decode(raw)
+    stripped = text.strip()
+    if not stripped:
+        raise SourceNotReady("empty content")
+    if stripped.startswith("<"):
+        raise UnexpectedContent("got HTML or XML instead of JSON")
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError as exc:
+        end = text.rstrip()
+        if (
+            exc.pos >= len(end)
+            or exc.msg.startswith("Unterminated string")
+            or _cut_inside_a_token(end, exc.pos)
+        ):
+            raise SourceNotReady(f"truncated JSON: {exc}") from None
+        raise UnexpectedContent(f"not valid JSON: {exc}") from None
+
+
+def parse_time(text: str) -> datetime:
+    """Parse an ISO-like time from SWPC as an aware UTC datetime.
+
+    Accepts ``T`` or a space between date and time, optional seconds and
+    fractions, and an optional ``Z`` or offset. A time without a zone is UTC.
+
+    Raises:
+        ValueError: If the text is not such a time.
+    """
+    if not _TIME.fullmatch(text):
+        raise ValueError(f"not a time: {text!r}")
+    try:
+        value = datetime.fromisoformat(text)
+    except ValueError:
+        raise ValueError(f"not a valid time: {text!r}") from None
+    if value.tzinfo is None:
+        return value.replace(tzinfo=UTC)
+    return value.astimezone(UTC)
+
+
+class Record:
+    """One record of a JSON product, with typed accessors.
+
+    Args:
+        index: 0-based position of the record in the JSON array.
+        data: The record's fields.
+    """
+
+    __slots__ = ("data", "index")
+
+    def __init__(self, index: int, data: Mapping[str, Any]) -> None:
+        self.index = index
+        self.data = data
+
+    def error(self, message: str, field: str) -> ParseError:
+        """Return a :class:`ParseError` located at ``field`` of this record."""
+        return ParseError(message, field=field, index=self.index)
+
+    def missing(self, field: str) -> ParseError:
+        """Return the error for ``field`` missing from this record.
+
+        :func:`read_records` tells a field missing from every record (a
+        format no parser knows) from one missing from some records only.
+        """
+        return MissingField(field, self.index)
+
+    def _get(self, field: str) -> Any:
+        try:
+            return self.data[field]
+        except KeyError:
+            raise self.missing(field) from None
+
+    def get_str(self, field: str) -> str:
+        """Return a string field."""
+        value = self._get(field)
+        if not isinstance(value, str):
+            raise self.error(f"expected a string, got {value!r}", field)
+        return value
+
+    def get_optional_str(self, field: str) -> str | None:
+        """Return a string field that may be ``null``."""
+        if self._get(field) is None:
+            return None
+        return self.get_str(field)
+
+    def get_time(self, field: str) -> datetime:
+        """Return a time field as an aware UTC datetime."""
+        try:
+            return parse_time(self.get_str(field))
+        except ValueError as exc:
+            raise self.error(str(exc), field) from None
+
+    def get_float(
+        self,
+        field: str,
+        *,
+        minimum: float | None = None,
+        maximum: float | None = None,
+    ) -> float:
+        """Return a number field (a JSON number or a quoted one)."""
+        value = self._get(field)
+        number: float | None = None
+        if (isinstance(value, int | float) and not isinstance(value, bool)) or (
+            isinstance(value, str) and _DECIMAL.fullmatch(value.strip())
+        ):
+            number = float(value)
+        if number is None or not math.isfinite(number):
+            raise self.error(f"expected a number, got {value!r}", field)
+        _check_range(self, field, number, minimum, maximum)
+        return number
+
+    def get_int(
+        self,
+        field: str,
+        *,
+        minimum: int | None = None,
+        maximum: int | None = None,
+    ) -> int:
+        """Return an integer field (a JSON integer or a quoted one)."""
+        value = self._get(field)
+        number: int | None = None
+        if isinstance(value, int) and not isinstance(value, bool):
+            number = value
+        elif isinstance(value, str) and _INTEGER.fullmatch(value.strip()):
+            number = int(value)
+        if number is None:
+            raise self.error(f"expected an integer, got {value!r}", field)
+        _check_range(self, field, number, minimum, maximum)
+        return number
+
+    def get_optional_int(
+        self,
+        field: str,
+        *,
+        minimum: int | None = None,
+        maximum: int | None = None,
+        may_be_absent: bool = False,
+    ) -> int | None:
+        """Return an integer field that may be ``null``.
+
+        With ``may_be_absent``, a field missing from the record is ``None``
+        too: for undocumented flags that SWPC may drop without notice.
+        """
+        if (may_be_absent and field not in self.data) or self._get(field) is None:
+            return None
+        return self.get_int(field, minimum=minimum, maximum=maximum)
+
+    def get_optional_float(
+        self,
+        field: str,
+        *,
+        minimum: float | None = None,
+        maximum: float | None = None,
+    ) -> float | None:
+        """Return a number field that may be ``null``."""
+        if self._get(field) is None:
+            return None
+        return self.get_float(field, minimum=minimum, maximum=maximum)
+
+    def get_bool(self, field: str) -> bool:
+        """Return a JSON boolean field (quoted or numeric values are refused)."""
+        value = self._get(field)
+        if not isinstance(value, bool):
+            raise self.error(f"expected a boolean, got {value!r}", field)
+        return value
+
+    def get_optional_bool(
+        self, field: str, *, may_be_absent: bool = False
+    ) -> bool | None:
+        """Return a JSON boolean field that may be ``null`` (or absent)."""
+        if (may_be_absent and field not in self.data) or self._get(field) is None:
+            return None
+        return self.get_bool(field)
+
+
+def _check_range(
+    record: Record,
+    field: str,
+    value: float,
+    minimum: float | None,
+    maximum: float | None,
+) -> None:
+    if (minimum is not None and value < minimum) or (
+        maximum is not None and value > maximum
+    ):
+        raise record.error(
+            f"{value} out of range [{minimum if minimum is not None else '-inf'}, "
+            f"{maximum if maximum is not None else 'inf'}]",
+            field,
+        )
+
+
+class MissingField(ParseError):
+    """A field missing from one record (see :func:`read_records`)."""
+
+    def __init__(self, field: str, index: int | None) -> None:
+        super().__init__("missing field", field=field, index=index)
+
+
+def read_records[V](
+    records: Iterable[Record], build: Callable[[Record], V]
+) -> tuple[list[tuple[int, V]], list[SkippedRecord]]:
+    """Build one value per record, skipping the records that cannot be read.
+
+    Returns:
+        The values with the index of their record, and the records skipped.
+
+    Raises:
+        UnexpectedContent: If no record can be read because the same field
+            is missing from all of them: not this product, or a format no
+            parser knows.
+        ParseError: If no record can be read for any other reason (the
+            error of the first record).
+    """
+    good: list[tuple[int, V]] = []
+    failures: list[tuple[int, ParseError]] = []
+    for record in records:
+        try:
+            good.append((record.index, build(record)))
+        except ParseError as exc:
+            failures.append((record.index, exc))
+    if not good and failures:
+        fields = {exc.field for _, exc in failures if isinstance(exc, MissingField)}
+        if len(fields) == 1 and all(isinstance(e, MissingField) for _, e in failures):
+            raise UnexpectedContent(
+                f"field {fields.pop()!r} missing from every record:"
+                " not this product, or a format no parser knows"
+            )
+        raise failures[0][1]
+    skipped = [
+        SkippedRecord(index=index, field=exc.field, reason=exc.reason)
+        for index, exc in failures
+    ]
+    return good, skipped
+
+
+def json_records(raw: str | bytes) -> list[Record]:
+    """Read a JSON product made of records, in either known shape.
+
+    Returns:
+        One :class:`Record` per record, with ``index`` its position in the
+        JSON array (the header row of the old shape is position 0).
+
+    Raises:
+        SourceNotReady: If the content is empty, truncated or has no records.
+        UnexpectedContent: If the content is not an array of objects nor an
+            array of arrays with a header row.
+        ParseError: If a record does not match the shape of the first one.
+    """
+    document = load_json(raw)
+    if not isinstance(document, list):
+        raise UnexpectedContent(
+            f"expected a JSON array of records, got {type(document).__name__}"
+        )
+    if not document:
+        raise SourceNotReady("no records")
+    first = document[0]
+    if isinstance(first, dict):
+        records = []
+        for index, item in enumerate(document):
+            if not isinstance(item, dict):
+                raise ParseError("expected an object", index=index)
+            records.append(Record(index, item))
+        return records
+    if isinstance(first, list):
+        if not first or not all(isinstance(name, str) for name in first):
+            raise UnexpectedContent("array of arrays without a header row of names")
+        if len(document) == 1:
+            raise SourceNotReady("no records after the header row")
+        rows = []
+        for index, item in enumerate(document[1:], start=1):
+            if not isinstance(item, list):
+                raise ParseError("expected an array", index=index)
+            if len(item) != len(first):
+                raise ParseError(
+                    f"{len(item)} values for {len(first)} columns", index=index
+                )
+            rows.append(Record(index, dict(zip(first, item, strict=True))))
+        return rows
+    raise UnexpectedContent(
+        f"expected objects or arrays in the JSON array, got {type(first).__name__}"
+    )

swpc_kit/parse/_text.py

@@ -0,0 +1,334 @@
+"""Reader shared by the SWPC text products: header, lines and errors.
+
+A text product starts with a header::
+
+    :Product: 3-Day Forecast
+    :Issued: 2026 Oct 15 1230 UTC
+    # Prepared by the U.S. Dept. of Commerce, NOAA, ...
+
+:func:`open_text_product` checks the header and returns a
+:class:`TextReader` placed after the ``#`` comments. The reader walks the
+remaining lines and classifies what goes wrong:
+
+- the content ends before an expected element: :class:`SourceNotReady` (SWPC
+  served the file while rewriting it);
+- an element is there but not as expected: :class:`ParseError` with its
+  1-based line number;
+- except when that element is on the last line and the content does not end
+  with a line break: the line was probably cut, so :class:`SourceNotReady`.
+"""
+
+import re
+from bisect import bisect_right
+from collections.abc import Sequence
+from dataclasses import dataclass
+from datetime import UTC, date, datetime
+
+from swpc_kit.errors import ParseError, SourceNotReady, SwpcError, UnexpectedContent
+from swpc_kit.parse._common import decode
+
+MONTHS = {
+    name: number
+    for number, name in enumerate(
+        (
+            "Jan",
+            "Feb",
+            "Mar",
+            "Apr",
+            "May",
+            "Jun",
+            "Jul",
+            "Aug",
+            "Sep",
+            "Oct",
+            "Nov",
+            "Dec",
+        ),
+        start=1,
+    )
+}
+"""English month abbreviations as SWPC writes them, to month numbers."""
+
+_PRODUCT = re.compile(r":Product:\s*(.*)")
+_ISSUED = re.compile(
+    r":Issued:\s*(\d{4})\s+([A-Za-z]{3})\s+(\d{1,2})\s+(\d{2})(\d{2})\s+UTC"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class Line:
+    """One line of a text product.
+
+    Attributes:
+        number: 1-based line number.
+        text: The line without leading and trailing whitespace.
+    """
+
+    number: int
+    text: str
+
+
+@dataclass(frozen=True, slots=True)
+class Unwrapped:
+    """Free text with its line wrapping undone, mapped back to its lines.
+
+    Attributes:
+        text: Wrapped lines joined with a space, paragraphs with a blank line.
+        starts: Offset in ``text`` where each source line starts.
+        numbers: Line number of each source line, aligned with ``starts``.
+    """
+
+    text: str
+    starts: tuple[int, ...]
+    numbers: tuple[int, ...]
+
+    def line_at(self, offset: int) -> int:
+        """Return the line number of the character at ``offset``."""
+        index = max(bisect_right(self.starts, offset) - 1, 0)
+        return self.numbers[index]
+
+
+def unwrap(lines: Sequence[Line]) -> Unwrapped:
+    """Join wrapped lines into text, keeping blank lines as paragraph breaks."""
+    parts: list[str] = []
+    starts: list[int] = []
+    numbers: list[int] = []
+    length = 0
+    pending_break = False
+    for line in lines:
+        if not line.text:
+            pending_break = bool(parts)
+            continue
+        if parts:
+            separator = "\n\n" if pending_break else " "
+            parts.append(separator)
+            length += len(separator)
+        pending_break = False
+        starts.append(length)
+        numbers.append(line.number)
+        parts.append(line.text)
+        length += len(line.text)
+    return Unwrapped("".join(parts), tuple(starts), tuple(numbers))
+
+
+def nearest_date(month: int, day: int, near: date) -> date:
+    """Place a month and day without a year in the year closest to ``near``.
+
+    A forecast issued on 31 December for ``Jan 01`` gets the next year; an
+    observation of ``31 Dec`` issued on 1 January gets the previous one.
+
+    Raises:
+        ValueError: If no year has that day (such as 31 April).
+    """
+    candidates = []
+    for year in (near.year - 1, near.year, near.year + 1):
+        try:
+            candidates.append(date(year, month, day))
+        except ValueError:
+            continue
+    if not candidates:
+        raise ValueError(f"no such day: month {month}, day {day}")
+    return min(candidates, key=lambda d: abs(d - near))
+
+
+class TextReader:
+    """Walks the lines of a text product, raising the right error.
+
+    Args:
+        text: The decoded content.
+    """
+
+    __slots__ = ("_complete", "_lines", "_pos")
+
+    def __init__(self, text: str) -> None:
+        self._lines = text.splitlines()
+        self._complete = text.endswith(("\n", "\r"))
+        self._pos = 0
+
+    def is_cut(self, line: int) -> bool:
+        """Whether ``line`` is the last line and has no line break after it."""
+        return line >= len(self._lines) and not self._complete
+
+    def error(self, message: str, line: int, field: str | None = None) -> SwpcError:
+        """Return the error for an unexpected ``line``.
+
+        Returns:
+            :class:`SourceNotReady` if the line looks cut (see :meth:`is_cut`),
+            otherwise :class:`ParseError` naming ``field`` and ``line``.
+        """
+        if self.is_cut(line):
+            return SourceNotReady(f"content ends inside line {line}: {message}")
+        return ParseError(message, field=field, line=line)
+
+    @staticmethod
+    def ended(what: str) -> SourceNotReady:
+        """Return the error for content that ends before ``what``."""
+        return SourceNotReady(f"content ends before {what}")
+
+    def _line(self) -> Line:
+        return Line(self._pos + 1, self._lines[self._pos].strip())
+
+    def peek(self) -> Line | None:
+        """Return the next non-blank line without consuming it, or ``None``."""
+        while self._pos < len(self._lines) and not self._lines[self._pos].strip():
+            self._pos += 1
+        if self._pos >= len(self._lines):
+            return None
+        return self._line()
+
+    def next(self, what: str) -> Line:
+        """Consume and return the next non-blank line.
+
+        Raises:
+            SourceNotReady: If the content ends first.
+        """
+        line = self.peek()
+        if line is None:
+            raise self.ended(what)
+        self._pos += 1
+        return line
+
+    def expect(
+        self, pattern: re.Pattern[str], what: str, field: str | None = None
+    ) -> tuple[Line, re.Match[str]]:
+        """Consume the next non-blank line, which must match ``pattern`` fully.
+
+        Raises:
+            SourceNotReady: If the content ends first, or the line looks cut.
+            ParseError: If the line does not match.
+        """
+        line = self.next(what)
+        match = pattern.fullmatch(line.text)
+        if match is None:
+            raise self.error(f"expected {what}, got {line.text!r}", line.number, field)
+        return line, match
+
+    def skip_comments(self) -> None:
+        """Skip blank lines and ``#`` comment lines."""
+        while self._pos < len(self._lines):
+            text = self._lines[self._pos].strip()
+            if text and not text.startswith("#"):
+                return
+            self._pos += 1
+
+    def take_until(
+        self,
+        stop: re.Pattern[str],
+        what: str,
+        *,
+        guard: re.Pattern[str] | None = None,
+        field: str | None = None,
+    ) -> list[Line]:
+        """Consume lines, blank ones included, up to a line matching ``stop``.
+
+        The ``stop`` line is left for the next call.
+
+        Raises:
+            SourceNotReady: If the content ends first.
+            ParseError: If a line matching ``guard`` comes first.
+        """
+        lines: list[Line] = []
+        while self._pos < len(self._lines):
+            line = self._line()
+            if stop.fullmatch(line.text):
+                return lines
+            if guard is not None and guard.fullmatch(line.text):
+                raise self.error(
+                    f"expected {what} before {line.text!r}", line.number, field
+                )
+            lines.append(line)
+            self._pos += 1
+        raise self.ended(what)
+
+    def take_rest(self) -> list[Line]:
+        """Consume and return every remaining line, blank ones included."""
+        rest = [
+            Line(n + 1, self._lines[n].strip())
+            for n in range(self._pos, len(self._lines))
+        ]
+        self._pos = len(self._lines)
+        return rest
+
+    def finish(self, after: str) -> None:
+        """Check that only blank lines remain.
+
+        Raises:
+            ParseError: If there is more content after ``after``.
+        """
+        line = self.peek()
+        if line is not None:
+            raise self.error(
+                f"unexpected content after {after}: {line.text!r}", line.number
+            )
+
+    def date(
+        self, month: str, day: str, near: date, line: int, field: str | None
+    ) -> date:
+        """Return the date of a month abbreviation and day, near ``near``.
+
+        Raises:
+            ParseError: If the month or day is not valid.
+        """
+        number = MONTHS.get(month.title())
+        if number is None:
+            raise self.error(f"unknown month {month!r}", line, field)
+        try:
+            return nearest_date(number, int(day), near)
+        except ValueError as exc:
+            raise self.error(str(exc), line, field) from None
+
+
+def open_text_product(
+    raw: str | bytes, products: Sequence[str]
+) -> tuple[TextReader, datetime]:
+    """Check the header of a text product and return a reader after it.
+
+    Args:
+        raw: The content.
+        products: Accepted values of ``:Product:``, compared ignoring case and
+            repeated spaces.
+
+    Returns:
+        A reader placed after the header and its ``#`` comments, and the
+        ``:Issued:`` time.
+
+    Raises:
+        SourceNotReady: If the content is empty or ends inside the header.
+        UnexpectedContent: If the content is HTML, not a text product, or a
+            different product.
+        ParseError: If the ``:Issued:`` line is not valid.
+    """
+    text = decode(raw)
+    if not text.strip():
+        raise SourceNotReady("empty content")
+    if text.lstrip().startswith("<"):
+        raise UnexpectedContent("got HTML or XML instead of a text product")
+    reader = TextReader(text)
+    first = reader.next("the ':Product:' line")
+    match = _PRODUCT.fullmatch(first.text)
+    if match is None:
+        if reader.is_cut(first.number) and ":Product:".startswith(first.text):
+            raise reader.ended("the end of the ':Product:' line")
+        raise UnexpectedContent(f"expected a ':Product:' line, got {first.text[:80]!r}")
+    name = " ".join(match[1].split())
+    accepted = [" ".join(p.split()) for p in products]
+    if name.casefold() not in (p.casefold() for p in accepted):
+        if reader.is_cut(first.number) and any(
+            p.casefold().startswith(name.casefold()) for p in accepted
+        ):
+            raise reader.ended("the end of the ':Product:' line")
+        raise UnexpectedContent(f"expected product {accepted[0]!r}, got {name!r}")
+
+    line, match = reader.expect(_ISSUED, "':Issued: YYYY Mon DD HHMM UTC'", "Issued")
+    year, month, day, hour, minute = match.groups()
+    number = MONTHS.get(month.title())
+    if number is None:
+        raise reader.error(f"unknown month {month!r}", line.number, "Issued")
+    try:
+        issued = datetime(
+            int(year), number, int(day), int(hour), int(minute), tzinfo=UTC
+        )
+    except ValueError as exc:
+        raise reader.error(f"not a valid time: {exc}", line.number, "Issued") from None
+    reader.skip_comments()
+    return reader, issued