satnogs-orbital-data 0.1__py3-none-any.whl

satnogs_orbital_data/__init__.py

@@ -0,0 +1 @@
+"""SATNOGS orbital data package."""

satnogs_orbital_data/domain/__init__.py

@@ -0,0 +1,26 @@
+"""Domain models for SATNOGS orbital data."""
+
+from satnogs_orbital_data.domain.fetching import (
+    Credentials,
+    FetchAllResult,
+    FetchLatestResult,
+    OrbitalDataFormat,
+    SourceCollectionResult,
+    SourceConfig,
+    SourcedOrbitalData,
+    SourceFailure,
+)
+from satnogs_orbital_data.domain.orbital_data import OrbitalData, TleLines
+
+__all__ = [
+    "Credentials",
+    "FetchAllResult",
+    "FetchLatestResult",
+    "OrbitalData",
+    "OrbitalDataFormat",
+    "SourceConfig",
+    "SourceCollectionResult",
+    "SourceFailure",
+    "SourcedOrbitalData",
+    "TleLines",
+]

satnogs_orbital_data/domain/fetching.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import StrEnum
+
+from satnogs_orbital_data.domain.orbital_data import OrbitalData
+
+
+class OrbitalDataFormat(StrEnum):
+    OMM_CSV = "omm_csv"
+    OMM_JSON = "omm_json"
+    OMM_XML = "omm_xml"
+    TLE = "tle"
+
+
+@dataclass(frozen=True, slots=True)
+class Credentials:
+    username: str | None = None
+    password: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class SourceConfig:
+    name: str
+    uri: str
+    data_format: OrbitalDataFormat
+    credentials: Credentials | None = None
+    min_request_interval_seconds: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class SourcedOrbitalData:
+    source_name: str
+    orbital_data: OrbitalData
+
+
+@dataclass(frozen=True, slots=True)
+class SourceFailure:
+    source_name: str
+    uri: str
+    message: str
+    exception_type: str
+
+
+@dataclass(slots=True)
+class FetchAllResult:
+    records: dict[int, list[SourcedOrbitalData]] = field(default_factory=dict)
+    failures: list[SourceFailure] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class FetchLatestResult:
+    records: dict[int, SourcedOrbitalData] = field(default_factory=dict)
+    failures: list[SourceFailure] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class SourceCollectionResult:
+    records: dict[int, list[SourcedOrbitalData]] = field(default_factory=dict)
+    failures: list[SourceFailure] = field(default_factory=list)

satnogs_orbital_data/domain/orbital_data.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from datetime import datetime
+
+from satnogs_orbital_data.settings import PLACEHOLDER_NORAD_ID
+
+Primitive = str | int | float | bool
+OmmJson = dict[str, Primitive | datetime]
+ExtraFields = dict[str, Primitive]
+TleLines = tuple[str, str, str]
+
+
+@dataclass(frozen=True, slots=True)
+class OrbitalData:
+    object_name: str
+    norad_cat_id: int
+    classification_type: str
+    object_id: str
+    epoch: datetime
+    mean_motion_dot: float
+    mean_motion_ddot: float
+    bstar: float
+    ephemeris_type: int
+    element_set_no: int
+    inclination: float
+    ra_of_asc_node: float
+    eccentricity: float
+    arg_of_pericenter: float
+    mean_anomaly: float
+    mean_motion: float
+    rev_at_epoch: int
+    center_name: str
+    ref_frame: str
+    time_system: str
+    mean_element_theory: str
+    gm: float | None = None
+    extra_fields: ExtraFields = field(default_factory=dict)
+
+    @staticmethod
+    def required_omm_field_names() -> list[str]:
+        return [
+            "OBJECT_NAME",
+            "NORAD_CAT_ID",
+            "CLASSIFICATION_TYPE",
+            "OBJECT_ID",
+            "EPOCH",
+            "MEAN_MOTION_DOT",
+            "MEAN_MOTION_DDOT",
+            "BSTAR",
+            "EPHEMERIS_TYPE",
+            "ELEMENT_SET_NO",
+            "INCLINATION",
+            "RA_OF_ASC_NODE",
+            "ECCENTRICITY",
+            "ARG_OF_PERICENTER",
+            "MEAN_ANOMALY",
+            "MEAN_MOTION",
+            "REV_AT_EPOCH",
+            "CENTER_NAME",
+            "REF_FRAME",
+            "TIME_SYSTEM",
+            "MEAN_ELEMENT_THEORY",
+        ]
+
+    @staticmethod
+    def optional_omm_field_names() -> list[str]:
+        return ["GM"]
+
+    @classmethod
+    def from_omm_json(cls, data: Mapping[str, object]) -> OrbitalData:
+        from satnogs_orbital_data.formats.omm_json import orbital_data_from_omm_json
+
+        return orbital_data_from_omm_json(dict(data))
+
+    @classmethod
+    def from_tle(cls, tle: TleLines) -> OrbitalData:
+        from satnogs_orbital_data.formats.tle import orbital_data_from_tle
+
+        return orbital_data_from_tle(tle)
+
+    def to_tle(self) -> TleLines:
+        from satnogs_orbital_data.formats.tle import orbital_data_to_tle
+
+        return orbital_data_to_tle(self)
+
+    def to_dummy_tle(self, placeholder_norad_id: int = PLACEHOLDER_NORAD_ID) -> TleLines:
+        from satnogs_orbital_data.formats.tle import orbital_data_to_dummy_tle
+
+        return orbital_data_to_dummy_tle(self, placeholder_norad_id)
+
+    def to_omm_json(self) -> OmmJson:
+        from satnogs_orbital_data.formats.omm_json import orbital_data_to_omm_json
+
+        return orbital_data_to_omm_json(self)

satnogs_orbital_data/fetching/__init__.py

@@ -0,0 +1,25 @@
+"""Public fetching API."""
+
+from satnogs_orbital_data.fetching.errors import (
+    SourceError,
+    SourceFetchError,
+    SourceParseError,
+)
+from satnogs_orbital_data.fetching.fetcher import (
+    fetch_all,
+    fetch_latest,
+)
+from satnogs_orbital_data.fetching.sources import (
+    get_default_sources,
+    get_failsafe_source,
+)
+
+__all__ = [
+    "SourceError",
+    "SourceFetchError",
+    "SourceParseError",
+    "fetch_all",
+    "fetch_latest",
+    "get_default_sources",
+    "get_failsafe_source",
+]

satnogs_orbital_data/fetching/errors.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+
+class SourceError(Exception):
+    pass
+
+
+class SourceFetchError(SourceError):
+    pass
+
+
+class SourceParseError(SourceError):
+    pass

satnogs_orbital_data/fetching/fetcher.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterable, Sequence
+from typing import Protocol
+
+from satnogs_orbital_data.domain import (
+    FetchAllResult,
+    FetchLatestResult,
+    OrbitalDataFormat,
+    SourceCollectionResult,
+    SourceConfig,
+    SourcedOrbitalData,
+    SourceFailure,
+)
+from satnogs_orbital_data.domain.orbital_data import OrbitalData
+from satnogs_orbital_data.fetching.errors import SourceError, SourceFetchError, SourceParseError
+from satnogs_orbital_data.fetching.handlers import (
+    FileSourceHandler,
+    HttpSourceHandler,
+    SourceHandler,
+    SpaceTrackSourceHandler,
+)
+from satnogs_orbital_data.fetching.sources import (
+    DEFAULT_FAILSAFE_SOURCE,
+    get_default_sources,
+    get_failsafe_source,
+)
+from satnogs_orbital_data.formats.omm_csv import OmmCsvParser
+from satnogs_orbital_data.formats.omm_json import OmmJsonParser
+from satnogs_orbital_data.formats.tle import TleParser
+from satnogs_orbital_data.settings import DEFAULT_TIMEOUT_SECONDS
+
+logger = logging.getLogger(__name__)
+
+
+class OrbitalDataParser(Protocol):
+    def parse(self, raw: bytes) -> Iterable[OrbitalData]: ...
+
+
+_PARSERS: dict[OrbitalDataFormat, OrbitalDataParser] = {
+    OrbitalDataFormat.OMM_CSV: OmmCsvParser(),
+    OrbitalDataFormat.OMM_JSON: OmmJsonParser(),
+    OrbitalDataFormat.TLE: TleParser(),
+}
+_HANDLERS: tuple[SourceHandler, ...] = (
+    SpaceTrackSourceHandler(),
+    FileSourceHandler(),
+    HttpSourceHandler(),
+)
+
+
+def _collect_all(
+    norad_ids: Iterable[int],
+    *,
+    sources: Sequence[SourceConfig] | None = None,
+    include_default_sources: bool = True,
+    include_failsafe: bool = True,
+    failsafe_source: SourceConfig = DEFAULT_FAILSAFE_SOURCE,
+    timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
+    tls_verify: bool = True,
+) -> SourceCollectionResult:
+    requested_ids = set(norad_ids)
+    result = SourceCollectionResult()
+    if not requested_ids:
+        logger.debug("No NORAD IDs requested; skipping fetch")
+        return result
+
+    effective_sources = []
+    if include_default_sources:
+        effective_sources.extend(get_default_sources())
+    if sources is not None:
+        effective_sources.extend(sources)
+
+    logger.info(
+        "Fetching orbital data for %d NORAD IDs from %d sources",
+        len(requested_ids),
+        len(effective_sources),
+    )
+
+    for source in effective_sources:
+        try:
+            _collect_source(
+                source,
+                requested_ids,
+                timeout_seconds,
+                tls_verify,
+                result,
+            )
+        except SourceError as exc:
+            logger.warning(
+                "Failed to collect orbital data from %s: %s",
+                source.name,
+                exc,
+            )
+            result.failures.append(
+                SourceFailure(
+                    source_name=source.name,
+                    uri=source.uri,
+                    message=str(exc),
+                    exception_type=type(exc).__name__,
+                )
+            )
+
+    if include_failsafe:
+        missing_norad_ids = requested_ids - set(result.records.keys())
+        if missing_norad_ids:
+            logger.info(
+                "Trying failsafe source for %d missing NORAD IDs",
+                len(missing_norad_ids),
+            )
+            if "{norad_id}" not in failsafe_source.uri:
+                message = "Failsafe source URI must contain {norad_id}"
+                logger.warning(
+                    "Failsafe source %s is not usable: %s",
+                    failsafe_source.name,
+                    message,
+                )
+                result.failures.append(
+                    SourceFailure(
+                        source_name=failsafe_source.name,
+                        uri=failsafe_source.uri,
+                        message=message,
+                        exception_type=SourceFetchError.__name__,
+                    )
+                )
+            else:
+                for norad_id in missing_norad_ids:
+                    source = get_failsafe_source(norad_id, failsafe_source)
+                    try:
+                        _collect_source(
+                            source,
+                            {norad_id},
+                            timeout_seconds,
+                            tls_verify,
+                            result,
+                        )
+                    except SourceError as exc:
+                        logger.warning(
+                            "Failsafe source failed for NORAD %s: %s",
+                            norad_id,
+                            exc,
+                        )
+                        result.failures.append(
+                            SourceFailure(
+                                source_name=source.name,
+                                uri=source.uri,
+                                message=str(exc),
+                                exception_type=type(exc).__name__,
+                            )
+                        )
+
+    record_count = sum(len(records) for records in result.records.values())
+    logger.info(
+        "Fetched %d matching orbital data records for %d NORAD IDs with %d source failures",
+        record_count,
+        len(result.records),
+        len(result.failures),
+    )
+    return result
+
+
+def fetch_all(
+    norad_ids: Iterable[int],
+    *,
+    sources: Sequence[SourceConfig] | None = None,
+    include_default_sources: bool = True,
+    include_failsafe: bool = True,
+    failsafe_source: SourceConfig = DEFAULT_FAILSAFE_SOURCE,
+    timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
+    tls_verify: bool = True,
+) -> FetchAllResult:
+    result = _collect_all(
+        norad_ids,
+        sources=sources,
+        include_default_sources=include_default_sources,
+        include_failsafe=include_failsafe,
+        failsafe_source=failsafe_source,
+        timeout_seconds=timeout_seconds,
+        tls_verify=tls_verify,
+    )
+    return FetchAllResult(records=result.records, failures=result.failures)
+
+
+def fetch_latest(
+    norad_ids: Iterable[int],
+    *,
+    sources: Sequence[SourceConfig] | None = None,
+    include_default_sources: bool = True,
+    include_failsafe: bool = True,
+    failsafe_source: SourceConfig = DEFAULT_FAILSAFE_SOURCE,
+    timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS,
+    tls_verify: bool = True,
+) -> FetchLatestResult:
+    result = _collect_all(
+        norad_ids,
+        sources=sources,
+        include_default_sources=include_default_sources,
+        include_failsafe=include_failsafe,
+        failsafe_source=failsafe_source,
+        timeout_seconds=timeout_seconds,
+        tls_verify=tls_verify,
+    )
+    return FetchLatestResult(
+        records={
+            norad_id: max(candidates, key=lambda candidate: candidate.orbital_data.epoch)
+            for norad_id, candidates in result.records.items()
+            if candidates
+        },
+        failures=result.failures,
+    )
+
+
+def _collect_source(
+    source: SourceConfig,
+    requested_ids: set[int],
+    timeout_seconds: float,
+    tls_verify: bool,
+    result: SourceCollectionResult,
+) -> None:
+    handler = _handler_for(source)
+    parser = _parser_for(source.data_format)
+    logger.info("Fetching orbital data from %s", source.name)
+    raw = handler.fetch(source, timeout_seconds, tls_verify)
+
+    try:
+        orbital_data_records = parser.parse(raw)
+    except (KeyError, TypeError, ValueError) as exc:
+        logger.debug("Failed to parse orbital data from %s", source.name, exc_info=True)
+        raise SourceParseError(str(exc)) from exc
+
+    matched_records = 0
+    for orbital_data in orbital_data_records:
+        if orbital_data.norad_cat_id not in requested_ids:
+            continue
+
+        matched_records += 1
+        result.records.setdefault(orbital_data.norad_cat_id, []).append(
+            SourcedOrbitalData(
+                source_name=source.name,
+                orbital_data=orbital_data,
+            )
+        )
+    logger.debug(
+        "Collected %d matching orbital data records from %s",
+        matched_records,
+        source.name,
+    )
+
+
+def _handler_for(source: SourceConfig) -> SourceHandler:
+    for handler in _HANDLERS:
+        if handler.can_handle(source):
+            return handler
+    raise SourceFetchError(f"No source handler configured for source: {source.name}")
+
+
+def _parser_for(data_format: OrbitalDataFormat) -> OrbitalDataParser:
+    try:
+        return _PARSERS[data_format]
+    except KeyError as exc:
+        raise SourceParseError(f"No parser configured for data format: {data_format}") from exc

satnogs_orbital_data/fetching/handlers.py

@@ -0,0 +1,165 @@
+from __future__ import annotations
+
+import json
+import logging
+from io import BytesIO
+from pathlib import Path
+from typing import Protocol
+from urllib.parse import unquote, urlparse
+from zipfile import BadZipFile, ZipFile, is_zipfile
+
+import requests  # type: ignore[import-untyped]
+
+from satnogs_orbital_data.domain import OrbitalDataFormat, SourceConfig
+from satnogs_orbital_data.fetching.errors import SourceFetchError
+
+logger = logging.getLogger(__name__)
+
+_SPACE_TRACK_GP_FORMATS = {
+    OrbitalDataFormat.TLE: "3le",
+    OrbitalDataFormat.OMM_CSV: "csv",
+    OrbitalDataFormat.OMM_JSON: "json",
+}
+
+
+class SourceHandler(Protocol):
+    def can_handle(self, source: SourceConfig) -> bool: ...
+
+    def fetch(self, source: SourceConfig, timeout_seconds: float, tls_verify: bool) -> bytes: ...
+
+
+class FileSourceHandler:
+    def can_handle(self, source: SourceConfig) -> bool:
+        scheme = urlparse(source.uri).scheme
+        return scheme in {"", "file"}
+
+    def fetch(self, source: SourceConfig, timeout_seconds: float, tls_verify: bool) -> bytes:
+        try:
+            path = _file_path_from_uri(source.uri)
+            logger.debug("Reading orbital data file source %s", path)
+            return _extract_zip_if_needed(path.read_bytes())
+        except (OSError, BadZipFile, ValueError) as exc:
+            raise SourceFetchError(str(exc)) from exc
+
+
+class HttpSourceHandler:
+    def can_handle(self, source: SourceConfig) -> bool:
+        scheme = urlparse(source.uri).scheme
+        return scheme in {"http", "https"}
+
+    def fetch(self, source: SourceConfig, timeout_seconds: float, tls_verify: bool) -> bytes:
+        try:
+            logger.debug("Requesting orbital data source %s", source.uri)
+            response = requests.get(
+                source.uri,
+                timeout=timeout_seconds,
+                verify=tls_verify,
+            )
+            response.raise_for_status()
+            logger.debug(
+                "Fetched %d bytes from %s",
+                len(response.content),
+                source.name,
+            )
+            return _extract_zip_if_needed(response.content)
+        except (requests.RequestException, BadZipFile, ValueError) as exc:
+            raise SourceFetchError(str(exc)) from exc
+
+
+class SpaceTrackSourceHandler:
+    def can_handle(self, source: SourceConfig) -> bool:
+        parsed = urlparse(source.uri)
+        return parsed.scheme in {"http", "https"} and "space-track.org" in parsed.netloc
+
+    def fetch(self, source: SourceConfig, timeout_seconds: float, tls_verify: bool) -> bytes:
+        credentials = source.credentials
+        if not credentials or not credentials.username or not credentials.password:
+            raise SourceFetchError("Space-Track source requires username and password")
+
+        try:
+            logger.debug(
+                "Requesting Space-Track GP data from %s as %s",
+                _space_track_base_url(source.uri),
+                source.data_format,
+            )
+            payload = _fetch_space_track_gp(
+                credentials.username,
+                credentials.password,
+                _space_track_base_url(source.uri),
+                _space_track_gp_format(source.data_format),
+            )
+        except SourceFetchError:
+            raise
+        except Exception as exc:
+            raise SourceFetchError(str(exc)) from exc
+
+        raw = _payload_to_bytes(payload)
+        logger.debug("Fetched %d bytes from Space-Track source %s", len(raw), source.name)
+        return raw
+
+
+def _file_path_from_uri(uri: str) -> Path:
+    parsed = urlparse(uri)
+    if parsed.scheme == "":
+        return Path(uri)
+    if parsed.scheme != "file":
+        raise ValueError(f"Unsupported file source URI scheme: {parsed.scheme}")
+    if parsed.netloc not in {"", "localhost"}:
+        raise ValueError(f"Unsupported file source host: {parsed.netloc}")
+    return Path(unquote(parsed.path))
+
+
+def _fetch_space_track_gp(
+    username: str,
+    password: str,
+    base_url: str,
+    space_track_format: str,
+) -> object:
+    try:
+        from spacetrack import SpaceTrackClient  # type: ignore[import-not-found, import-untyped]
+    except ModuleNotFoundError as exc:
+        raise SourceFetchError("spacetrack package is required for Space-Track sources") from exc
+
+    with SpaceTrackClient(username, password, base_url=base_url) as client:
+        return client.gp(epoch=">now-30", orderby=["norad_cat_id"], format=space_track_format)
+
+
+def _space_track_base_url(uri: str) -> str:
+    parsed = urlparse(uri)
+    return f"{parsed.scheme}://{parsed.netloc}"
+
+
+def _space_track_gp_format(data_format: OrbitalDataFormat) -> str:
+    try:
+        return _SPACE_TRACK_GP_FORMATS[data_format]
+    except KeyError as exc:
+        raise SourceFetchError(f"Space-Track does not support data format: {data_format}") from exc
+
+
+def _payload_to_bytes(payload: object) -> bytes:
+    if isinstance(payload, bytes):
+        return payload
+    if isinstance(payload, str):
+        return payload.encode("utf-8")
+    return json.dumps(payload, separators=(",", ":")).encode("utf-8")
+
+
+def _extract_zip_if_needed(raw: bytes) -> bytes:
+    with BytesIO(raw) as buffer:
+        if not is_zipfile(buffer):
+            return raw
+        buffer.seek(0)
+        logger.debug("Extracting zip orbital data source")
+        return _read_zip_files(buffer)
+
+
+def _read_zip_files(buffer: BytesIO) -> bytes:
+    parts: list[bytes] = []
+    with ZipFile(buffer) as archive:
+        for member in archive.infolist():
+            if not member.is_dir():
+                logger.debug("Reading zip member %s", member.filename)
+                parts.append(archive.read(member).replace(b"\r\n", b"\n"))
+    if not parts:
+        raise ValueError("Zip source contains no files")
+    return b"\n".join(parts)