earthscope-sdk 1.0.0b1__py3-none-any.whl → 1.2.0b0__py3-none-any.whl

earthscope_sdk/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.0.0b1"
+__version__ = "1.2.0b0"
 
 from earthscope_sdk.client import AsyncEarthScopeClient, EarthScopeClient
 

earthscope_sdk/auth/auth_flow.py

@@ -311,9 +311,10 @@ class AuthFlow(httpx.Auth):
         """
         super().async_auth_flow
         if request.headers.get("authorization") is None:
-            await self.async_refresh_if_necessary()
-            access_token = self.access_token
-            request.headers["authorization"] = f"Bearer {access_token}"
+            if self._settings.is_host_allowed(request.url.host):
+                await self.async_refresh_if_necessary()
+                access_token = self.access_token
+                request.headers["authorization"] = f"Bearer {access_token}"
 
         yield request
 
@@ -326,8 +327,9 @@ class AuthFlow(httpx.Auth):
         # NOTE: we explicitly redefine this sync method because ctx.syncify()
         # does not support generators
         if request.headers.get("authorization") is None:
-            self.refresh_if_necessary()
-            access_token = self.access_token
-            request.headers["authorization"] = f"Bearer {access_token}"
+            if self._settings.is_host_allowed(request.url.host):
+                self.refresh_if_necessary()
+                access_token = self.access_token
+                request.headers["authorization"] = f"Bearer {access_token}"
 
         yield request

earthscope_sdk/auth/client_credentials_flow.py

@@ -1,5 +1,4 @@
 import logging
-from dataclasses import dataclass
 from json import JSONDecodeError
 
 from earthscope_sdk.auth.auth_flow import AuthFlow
@@ -9,12 +8,6 @@ from earthscope_sdk.common.context import SdkContext
 logger = logging.getLogger(__name__)
 
 
-@dataclass
-class GetTokensErrorResponse:
-    error: str
-    error_description: str
-
-
 class ClientCredentialsFlow(AuthFlow):
     """
     Implements the oauth2 Client Credentials "machine-to-machine" (M2M) flow.
@@ -69,12 +62,9 @@ class ClientCredentialsFlow(AuthFlow):
 
         # Unauthorized
         if r.status_code == 401:
-            err = GetTokensErrorResponse(**resp)
-            if err.error == "access_denied":
-                if err.error_description == "Unauthorized":
-                    raise UnauthorizedError(
-                        f"m2m client '{self._settings.client_id}' is not authorized"
-                    )
+            raise UnauthorizedError(
+                f"m2m client '{self._settings.client_id}' is not authorized. IdP response: {resp}"
+            )
 
         # Unhandled
         raise ClientCredentialsFlowError("client credentials flow failed", r.content)

earthscope_sdk/auth/device_code_flow.py

@@ -1,11 +1,12 @@
 import logging
+from asyncio import sleep
 from contextlib import asynccontextmanager, contextmanager
-from dataclasses import dataclass
 from enum import Enum
 from json import JSONDecodeError
-from time import sleep
 from typing import Optional
 
+from pydantic import BaseModel, ValidationError
+
 from earthscope_sdk.auth.auth_flow import AuthFlow
 from earthscope_sdk.auth.error import (
     DeviceCodePollingError,
@@ -25,18 +26,16 @@ class PollingErrorType(str, Enum):
     ACCESS_DENIED = "access_denied"
 
 
-@dataclass
-class GetDeviceCodeResponse:
+class GetDeviceCodeResponse(BaseModel):
     device_code: str
     user_code: str
     verification_uri: str
     verification_uri_complete: str
     expires_in: int
-    interval: int
+    interval: float
 
 
-@dataclass
-class PollingErrorResponse:
+class PollingErrorResponse(BaseModel):
     error: PollingErrorType
     error_description: str
 
@@ -157,7 +156,7 @@ class DeviceCodeFlow(AuthFlow):
         try:
             while True:
                 # IdP-provided poll interval
-                sleep(codes.interval)
+                await sleep(codes.interval)
 
                 r = await self._ctx.httpx_client.post(
                     f"{self._settings.domain}oauth/token",
@@ -185,7 +184,12 @@ class DeviceCodeFlow(AuthFlow):
                     return self
 
                 # Keep polling
-                poll_err = PollingErrorResponse(**resp)
+                try:
+                    poll_err = PollingErrorResponse.model_validate(resp)
+                except ValidationError as e:
+                    raise DeviceCodePollingError(
+                        f"Failed to unpack polling response: {r.text}"
+                    ) from e
                 if poll_err.error in [
                     PollingErrorType.AUTHORIZATION_PENDING,
                     PollingErrorType.SLOW_DOWN,
@@ -235,7 +239,12 @@ class DeviceCodeFlow(AuthFlow):
                 f"Failed to get a device code: {r.content}"
             )
 
-        codes = GetDeviceCodeResponse(**r.json())
+        try:
+            codes = GetDeviceCodeResponse.model_validate_json(r.content)
+        except ValidationError as e:
+            raise DeviceCodeRequestDeviceCodeError(
+                f"Failed to unpack device code response: {r.text}"
+            ) from e
 
         logger.debug(f"Got device code response: {codes}")
         return codes

earthscope_sdk/client/_client.py

@@ -8,6 +8,26 @@ class AsyncEarthScopeClient(AsyncSdkClient):
     An async client for interacting with api.earthscope.org
     """
 
+    @cached_property
+    def data(self):
+        """
+        Data access functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.data_access._service import AsyncDataAccessService
+
+        return AsyncDataAccessService(self)
+
+    @cached_property
+    def discover(self):
+        """
+        Data discovery functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.discovery._service import AsyncDiscoveryService
+
+        return AsyncDiscoveryService(self._ctx)
+
     @cached_property
     def user(self):
         """
@@ -24,6 +44,33 @@ class EarthScopeClient(SdkClient):
     A client for interacting with api.earthscope.org
     """
 
+    @cached_property
+    def _async_client(self):
+        """
+        An async client for interacting with api.earthscope.org
+        """
+        return AsyncEarthScopeClient(ctx=self._ctx)
+
+    @cached_property
+    def data(self):
+        """
+        Data access functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.data_access._service import DataAccessService
+
+        return DataAccessService(self._async_client)
+
+    @cached_property
+    def discover(self):
+        """
+        Data discovery functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.discovery._service import DiscoveryService
+
+        return DiscoveryService(self._ctx)
+
     @cached_property
     def user(self):
         """

earthscope_sdk/client/data_access/_arrow/_common.py

@@ -0,0 +1,94 @@
+try:
+    import pyarrow as pa
+except ModuleNotFoundError as e:
+    raise ModuleNotFoundError(
+        "Optional dependency 'pyarrow' is required for this feature. "
+        "Install it with: pip install earthscope-sdk[arrow]"
+    ) from e
+
+
+from earthscope_sdk.client.discovery.models import DatasourceBaseModel
+
+
+def convert_time(
+    table: pa.Table,
+    *,
+    source_field: str = "time",
+    target_field: str = "timestamp",
+    unit: str = "ms",
+    tz: str = "UTC",
+) -> pa.Table:
+    """
+    Convert a unix timestamp (integer since epoch) field to a timestamp type.
+
+    Args:
+        table: The table to convert.
+        source_field: The name of the field to convert.
+        target_field: The name of the field to store the result.
+        unit: The unit of the time field.
+        tz: The timezone of the time field.
+
+    Returns:
+        The converted table.
+    """
+    if unit not in {"s", "ms", "us", "ns"}:
+        raise ValueError(f"Invalid time unit: {unit}")
+
+    time_idx = table.schema.get_field_index(source_field)
+    if time_idx < 0:
+        return table
+
+    return table.set_column(
+        time_idx,
+        target_field,
+        table[source_field].cast(pa.timestamp(unit, tz)),
+    )
+
+
+def get_datasource_metadata_table(
+    datasources: list[DatasourceBaseModel],
+    *,
+    fields: list[str],
+    namespaces: list[str],
+) -> pa.Table:
+    """
+    Get a pyarrow table containing metadata columns for a list of datasources.
+
+    Args:
+        datasources: The list of datasources to get metadata for.
+        fields: The fields to include in the metadata table.
+        namespaces: The namespaces to include in the metadata table.
+
+    Returns:
+        The metadata table.
+    """
+    return pa.Table.from_pylist(
+        [s.to_arrow_columns(fields=fields, namespaces=namespaces) for s in datasources]
+    )
+
+
+def load_table_with_extra(content: bytes, *, is_stream=True, **kwargs) -> pa.Table:
+    """
+    Load a table from an arrow stream or file. Optionally add extra metadata columns.
+
+    Args:
+        content: The content of the stream or file.
+        is_stream: Whether the content is an arrow stream or an arrow file.
+        **kwargs: The extra metadata columns to add to the table.
+
+    Returns:
+        The loaded table.
+    """
+    if is_stream:
+        reader = pa.ipc.open_stream(content)
+    else:
+        reader = pa.ipc.open_file(content)
+
+    table = reader.read_all()
+
+    if kwargs:
+        count = len(table)
+        for key, value in kwargs.items():
+            table = table.append_column(key, pa.repeat(value, count))
+
+    return table

earthscope_sdk/client/data_access/_arrow/_gnss.py

@@ -0,0 +1,116 @@
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from earthscope_sdk.client.data_access._arrow._common import convert_time
+
+_SYSTEM_ID = "system_id"
+_SYSTEM_TABLE = pa.Table.from_pylist(
+    [
+        {_SYSTEM_ID: 1, "system": "G"},  # GPS
+        {_SYSTEM_ID: 2, "system": "R"},  # GLONASS
+        {_SYSTEM_ID: 3, "system": "S"},  # SBAS
+        {_SYSTEM_ID: 4, "system": "E"},  # Galileo
+        {_SYSTEM_ID: 5, "system": "C"},  # BeiDou
+        {_SYSTEM_ID: 6, "system": "J"},  # QZSS
+        {_SYSTEM_ID: 7, "system": "I"},  # IRNSS/NavIC
+    ],
+    schema=pa.schema(
+        [
+            pa.field(_SYSTEM_ID, pa.uint8()),
+            pa.field("system", pa.string()),
+        ]
+    ),
+)
+
+
+def _convert_obs_code(
+    table: pa.Table,
+    *,
+    source_field: str = "obs",
+    target_field: str = "obs_code",
+) -> pa.Table:
+    """
+    Convert the obs_code field from packed chars to a string.
+
+    Args:
+        table: The table to convert.
+        source_field: The name of the field to convert.
+        target_field: The name of the field to store the result.
+
+    Returns:
+        The converted table.
+    """
+    # extract obs_code (encoded as packed chars)
+    result_chunks = [
+        pc.cast(
+            pa.FixedSizeBinaryArray.from_buffers(
+                type=pa.binary(2),
+                length=len(chunk),
+                buffers=[None, chunk.buffers()[1]],
+            ),
+            pa.string(),
+        )
+        for chunk in table[source_field].chunks
+    ]
+
+    return table.set_column(
+        table.schema.get_field_index(source_field),
+        target_field,
+        # can't easily swap endianness of packed chars, so reverse the string
+        pc.utf8_reverse(pa.chunked_array(result_chunks, type=pa.string())),
+    )
+
+
+def _convert_system(table: pa.Table, *, field: str) -> pa.Table:
+    """
+    Convert the sys field from integer to string.
+
+    Args:
+        table: The table to convert.
+        field: The name of the field to convert.
+
+    Returns:
+        The converted table.
+    """
+    return table.join(_SYSTEM_TABLE, field, right_keys=_SYSTEM_ID).drop(field)
+
+
+def prettify_observations_table(table: pa.Table) -> pa.Table:
+    """
+    Clean up a raw observations table (as stored in TileDB) for easier use.
+
+    Args:
+        table: The table to prettify.
+
+    Returns:
+        The prettified table.
+    """
+    # Cast time column (integer milliseconds since epoch) to timestamp type
+    table = convert_time(table)
+
+    # Expose friendlier column names
+    table = table.rename_columns({"sat": "satellite"})
+
+    # Convert system column from integer to string
+    table = _convert_system(table, field="sys")
+
+    # Convert obs_code column from packed chars to string
+    table = _convert_obs_code(table)
+
+    return table
+
+
+def prettify_positions_table(table: pa.Table) -> pa.Table:
+    """
+    Clean up a raw positions table (as stored in TileDB) for easier use.
+
+    Args:
+        table: The table to prettify.
+
+    Returns:
+        The prettified table.
+    """
+    # Convert time column (integer milliseconds since epoch) to timestamp type
+    table = convert_time(table)
+
+    return table

earthscope_sdk/client/data_access/_base.py

@@ -0,0 +1,85 @@
+import datetime as dt
+from typing import TYPE_CHECKING, Union
+
+from earthscope_sdk.client.data_access._arrow._common import load_table_with_extra
+from earthscope_sdk.common.service import SdkService
+from earthscope_sdk.util._time import to_utc_dt
+
+if TYPE_CHECKING:
+    from pyarrow import Table
+
+
+class DataAccessBaseService(SdkService):
+    """
+    L1 data access service functionality
+    """
+
+    async def _get_gnss_observations(
+        self,
+        *,
+        session_edid: str,
+        start_datetime: dt.datetime,
+        end_datetime: dt.datetime,
+        system: Union[str, list[str]] = [],
+        satellite: Union[str, list[str]] = [],
+        obs_code: Union[str, list[str]] = [],
+        field: Union[str, list[str]] = [],
+    ) -> "Table":
+        """
+        Retrieve GNSS observations.
+
+        This method retrieves GNSS observations for a given session.
+
+        Args:
+            session_edid: The session EDID to retrieve observations for.
+            start_datetime: The start datetime to retrieve positions for.
+            end_datetime: The end datetime to retrieve positions for.
+            system: The system(s) to retrieve observations for.
+            satellite: The satellite(s) to retrieve observations for.
+            obs_code: The observation code(s) to retrieve observations for.
+            field: The field(s) to retrieve observations for.
+
+        Returns:
+            A pyarrow table containing the GNSS observations.
+        """
+
+        # lazy import
+
+        headers = {
+            "accept": "application/vnd.apache.arrow.stream",
+        }
+
+        params = {
+            "session_id": session_edid,
+            "start_datetime": to_utc_dt(start_datetime).isoformat(),
+            "end_datetime": to_utc_dt(end_datetime).isoformat(),
+        }
+
+        if system:
+            params["system"] = system
+
+        if satellite:
+            params["satellite"] = satellite
+
+        if obs_code:
+            params["obs_code"] = obs_code
+
+        if field:
+            params["field"] = field
+
+        req = self.ctx.httpx_client.build_request(
+            method="GET",
+            url=f"{self.resources.api_url}beta/data-products/gnss/observations",
+            headers=headers,
+            params=params,
+            timeout=30,
+        )
+
+        resp = await self._send_with_retries(req)
+
+        return await self.ctx.run_in_executor(
+            load_table_with_extra,
+            resp.content,
+            # Add edid column to the table returned by the API
+            edid=session_edid,
+        )