earthscope-sdk 1.0.0b1__tar.gz → 1.2.0b0__tar.gz

{earthscope_sdk-1.0.0b1 → earthscope_sdk-1.2.0b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: earthscope-sdk
-Version: 1.0.0b1
+Version: 1.2.0b0
 Summary: An SDK for EarthScope API
 Author-email: EarthScope <data-help@earthscope.org>
 License:                                  Apache License
@@ -216,12 +216,16 @@ License-File: LICENSE
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: pydantic-settings[toml]>=2.8.0
 Requires-Dist: stamina>=24.3.0
+Provides-Extra: arrow
+Requires-Dist: pyarrow>=20.0.0; extra == "arrow"
+Requires-Dist: pyarrow-stubs>=20.0.0; extra == "arrow"
 Provides-Extra: dev
 Requires-Dist: bumpver; extra == "dev"
 Requires-Dist: build; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: twine; extra == "dev"
 Requires-Dist: pip-tools; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
 Requires-Dist: pytest-httpx; extra == "dev"
 Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
@@ -241,9 +245,16 @@ Install from PyPI
 pip install earthscope-sdk
 ```
 
+Or with optional dependencies:
+
+```shell
+# install arrow dependencies for efficient data access
+pip install earthscope-sdk[arrow]
+```
+
 ### Usage
 
-For detailed usage options and examples, visit [our usage docs](docs/usage.md).
+For detailed usage info and examples, visit [our SDK docs](https://docs.earthscope.org/projects/SDK).
 
 ```py
 # Import and create a client
@@ -300,7 +311,7 @@ Once refreshable credentials are available to the SDK, it will transparently han
 
 ### Same host
 
-If you have the [EarthScope CLI](TODO) installed on the same host that is running your application which uses `earthscope-sdk`, you can simply log in using the CLI. The CLI shares credentials and configuration with this SDK (when running on the same host).
+If you have the [EarthScope CLI](https://docs.earthscope.org/projects/CLI) installed on the same host that is running your application which uses `earthscope-sdk`, you can simply log in using the CLI. The CLI shares credentials and configuration with this SDK (when running on the same host).
 
 Running `es login` will open your browser and prompt you to log in to your EarthScope account.
 
@@ -320,7 +331,7 @@ Now when you run your application, `earthscope-sdk` will find your credentials.
 
 Sometimes your workload runs on different hosts than your main workstation and you cannot feasibly "log in" on all of them. For example, maybe you're running many containers in your workload.
 
-You can still use the [EarthScope CLI](TODO) to facilitate auth for applications on other machines.
+You can still use the [EarthScope CLI](https://docs.earthscope.org/projects/CLI) to facilitate auth for applications on other machines.
 
 1. Use the CLI on your primary workstation [as described above](#same-host) to log in.
 

{earthscope_sdk-1.0.0b1 → earthscope_sdk-1.2.0b0}/README.md

@@ -12,9 +12,16 @@ Install from PyPI
 pip install earthscope-sdk
 ```
 
+Or with optional dependencies:
+
+```shell
+# install arrow dependencies for efficient data access
+pip install earthscope-sdk[arrow]
+```
+
 ### Usage
 
-For detailed usage options and examples, visit [our usage docs](docs/usage.md).
+For detailed usage info and examples, visit [our SDK docs](https://docs.earthscope.org/projects/SDK).
 
 ```py
 # Import and create a client
@@ -71,7 +78,7 @@ Once refreshable credentials are available to the SDK, it will transparently han
 
 ### Same host
 
-If you have the [EarthScope CLI](TODO) installed on the same host that is running your application which uses `earthscope-sdk`, you can simply log in using the CLI. The CLI shares credentials and configuration with this SDK (when running on the same host).
+If you have the [EarthScope CLI](https://docs.earthscope.org/projects/CLI) installed on the same host that is running your application which uses `earthscope-sdk`, you can simply log in using the CLI. The CLI shares credentials and configuration with this SDK (when running on the same host).
 
 Running `es login` will open your browser and prompt you to log in to your EarthScope account.
 
@@ -91,7 +98,7 @@ Now when you run your application, `earthscope-sdk` will find your credentials.
 
 Sometimes your workload runs on different hosts than your main workstation and you cannot feasibly "log in" on all of them. For example, maybe you're running many containers in your workload.
 
-You can still use the [EarthScope CLI](TODO) to facilitate auth for applications on other machines.
+You can still use the [EarthScope CLI](https://docs.earthscope.org/projects/CLI) to facilitate auth for applications on other machines.
 
 1. Use the CLI on your primary workstation [as described above](#same-host) to log in.
 

{earthscope_sdk-1.0.0b1 → earthscope_sdk-1.2.0b0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "earthscope-sdk"
-version = "1.0.0b1"
+version = "1.2.0b0"
 description = "An SDK for EarthScope API"
 readme = "README.md"
 authors = [{ name = "EarthScope", email = "data-help@earthscope.org" }]
@@ -25,12 +25,14 @@ dependencies = [
 requires-python = ">=3.9"
 
 [project.optional-dependencies]
+arrow = ["pyarrow>=20.0.0", "pyarrow-stubs>=20.0.0"]
 dev = [
     "bumpver",
     "build",
     "pytest",
     "twine",
     "pip-tools",
+    "pre-commit",
     "pytest-httpx",
     "pytest-asyncio",
     "ruff",
@@ -40,7 +42,7 @@ dev = [
 Homepage = "https://gitlab.com/earthscope/public/earthscope-sdk"
 
 [tool.bumpver]
-current_version = "1.0.0b1"
+current_version = "1.2.0b0"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "chore: bump version {old_version} -> {new_version}"
 commit = true

{earthscope_sdk-1.0.0b1 → earthscope_sdk-1.2.0b0}/src/earthscope_sdk/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.0.0b1"
+__version__ = "1.2.0b0"
 
 from earthscope_sdk.client import AsyncEarthScopeClient, EarthScopeClient
 

{earthscope_sdk-1.0.0b1 → earthscope_sdk-1.2.0b0}/src/earthscope_sdk/auth/auth_flow.py

@@ -311,9 +311,10 @@ class AuthFlow(httpx.Auth):
         """
         super().async_auth_flow
         if request.headers.get("authorization") is None:
-            await self.async_refresh_if_necessary()
-            access_token = self.access_token
-            request.headers["authorization"] = f"Bearer {access_token}"
+            if self._settings.is_host_allowed(request.url.host):
+                await self.async_refresh_if_necessary()
+                access_token = self.access_token
+                request.headers["authorization"] = f"Bearer {access_token}"
 
         yield request
 
@@ -326,8 +327,9 @@ class AuthFlow(httpx.Auth):
         # NOTE: we explicitly redefine this sync method because ctx.syncify()
         # does not support generators
         if request.headers.get("authorization") is None:
-            self.refresh_if_necessary()
-            access_token = self.access_token
-            request.headers["authorization"] = f"Bearer {access_token}"
+            if self._settings.is_host_allowed(request.url.host):
+                self.refresh_if_necessary()
+                access_token = self.access_token
+                request.headers["authorization"] = f"Bearer {access_token}"
 
         yield request

{earthscope_sdk-1.0.0b1 → earthscope_sdk-1.2.0b0}/src/earthscope_sdk/auth/client_credentials_flow.py

@@ -1,5 +1,4 @@
 import logging
-from dataclasses import dataclass
 from json import JSONDecodeError
 
 from earthscope_sdk.auth.auth_flow import AuthFlow
@@ -9,12 +8,6 @@ from earthscope_sdk.common.context import SdkContext
 logger = logging.getLogger(__name__)
 
 
-@dataclass
-class GetTokensErrorResponse:
-    error: str
-    error_description: str
-
-
 class ClientCredentialsFlow(AuthFlow):
     """
     Implements the oauth2 Client Credentials "machine-to-machine" (M2M) flow.
@@ -69,12 +62,9 @@ class ClientCredentialsFlow(AuthFlow):
 
         # Unauthorized
         if r.status_code == 401:
-            err = GetTokensErrorResponse(**resp)
-            if err.error == "access_denied":
-                if err.error_description == "Unauthorized":
-                    raise UnauthorizedError(
-                        f"m2m client '{self._settings.client_id}' is not authorized"
-                    )
+            raise UnauthorizedError(
+                f"m2m client '{self._settings.client_id}' is not authorized. IdP response: {resp}"
+            )
 
         # Unhandled
         raise ClientCredentialsFlowError("client credentials flow failed", r.content)

{earthscope_sdk-1.0.0b1 → earthscope_sdk-1.2.0b0}/src/earthscope_sdk/auth/device_code_flow.py

@@ -1,11 +1,12 @@
 import logging
+from asyncio import sleep
 from contextlib import asynccontextmanager, contextmanager
-from dataclasses import dataclass
 from enum import Enum
 from json import JSONDecodeError
-from time import sleep
 from typing import Optional
 
+from pydantic import BaseModel, ValidationError
+
 from earthscope_sdk.auth.auth_flow import AuthFlow
 from earthscope_sdk.auth.error import (
     DeviceCodePollingError,
@@ -25,18 +26,16 @@ class PollingErrorType(str, Enum):
     ACCESS_DENIED = "access_denied"
 
 
-@dataclass
-class GetDeviceCodeResponse:
+class GetDeviceCodeResponse(BaseModel):
     device_code: str
     user_code: str
     verification_uri: str
     verification_uri_complete: str
     expires_in: int
-    interval: int
+    interval: float
 
 
-@dataclass
-class PollingErrorResponse:
+class PollingErrorResponse(BaseModel):
     error: PollingErrorType
     error_description: str
 
@@ -157,7 +156,7 @@ class DeviceCodeFlow(AuthFlow):
         try:
             while True:
                 # IdP-provided poll interval
-                sleep(codes.interval)
+                await sleep(codes.interval)
 
                 r = await self._ctx.httpx_client.post(
                     f"{self._settings.domain}oauth/token",
@@ -185,7 +184,12 @@ class DeviceCodeFlow(AuthFlow):
                     return self
 
                 # Keep polling
-                poll_err = PollingErrorResponse(**resp)
+                try:
+                    poll_err = PollingErrorResponse.model_validate(resp)
+                except ValidationError as e:
+                    raise DeviceCodePollingError(
+                        f"Failed to unpack polling response: {r.text}"
+                    ) from e
                 if poll_err.error in [
                     PollingErrorType.AUTHORIZATION_PENDING,
                     PollingErrorType.SLOW_DOWN,
@@ -235,7 +239,12 @@ class DeviceCodeFlow(AuthFlow):
                 f"Failed to get a device code: {r.content}"
             )
 
-        codes = GetDeviceCodeResponse(**r.json())
+        try:
+            codes = GetDeviceCodeResponse.model_validate_json(r.content)
+        except ValidationError as e:
+            raise DeviceCodeRequestDeviceCodeError(
+                f"Failed to unpack device code response: {r.text}"
+            ) from e
 
         logger.debug(f"Got device code response: {codes}")
         return codes

earthscope_sdk-1.2.0b0/src/earthscope_sdk/client/_client.py

@@ -0,0 +1,82 @@
+from functools import cached_property
+
+from earthscope_sdk.common.client import AsyncSdkClient, SdkClient
+
+
+class AsyncEarthScopeClient(AsyncSdkClient):
+    """
+    An async client for interacting with api.earthscope.org
+    """
+
+    @cached_property
+    def data(self):
+        """
+        Data access functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.data_access._service import AsyncDataAccessService
+
+        return AsyncDataAccessService(self)
+
+    @cached_property
+    def discover(self):
+        """
+        Data discovery functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.discovery._service import AsyncDiscoveryService
+
+        return AsyncDiscoveryService(self._ctx)
+
+    @cached_property
+    def user(self):
+        """
+        User and Identity Management functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.user._service import AsyncUserService
+
+        return AsyncUserService(self._ctx)
+
+
+class EarthScopeClient(SdkClient):
+    """
+    A client for interacting with api.earthscope.org
+    """
+
+    @cached_property
+    def _async_client(self):
+        """
+        An async client for interacting with api.earthscope.org
+        """
+        return AsyncEarthScopeClient(ctx=self._ctx)
+
+    @cached_property
+    def data(self):
+        """
+        Data access functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.data_access._service import DataAccessService
+
+        return DataAccessService(self._async_client)
+
+    @cached_property
+    def discover(self):
+        """
+        Data discovery functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.discovery._service import DiscoveryService
+
+        return DiscoveryService(self._ctx)
+
+    @cached_property
+    def user(self):
+        """
+        User and Identity Management functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.user._service import UserService
+
+        return UserService(self._ctx)

earthscope_sdk-1.2.0b0/src/earthscope_sdk/client/data_access/_arrow/_common.py

@@ -0,0 +1,94 @@
+try:
+    import pyarrow as pa
+except ModuleNotFoundError as e:
+    raise ModuleNotFoundError(
+        "Optional dependency 'pyarrow' is required for this feature. "
+        "Install it with: pip install earthscope-sdk[arrow]"
+    ) from e
+
+
+from earthscope_sdk.client.discovery.models import DatasourceBaseModel
+
+
+def convert_time(
+    table: pa.Table,
+    *,
+    source_field: str = "time",
+    target_field: str = "timestamp",
+    unit: str = "ms",
+    tz: str = "UTC",
+) -> pa.Table:
+    """
+    Convert a unix timestamp (integer since epoch) field to a timestamp type.
+
+    Args:
+        table: The table to convert.
+        source_field: The name of the field to convert.
+        target_field: The name of the field to store the result.
+        unit: The unit of the time field.
+        tz: The timezone of the time field.
+
+    Returns:
+        The converted table.
+    """
+    if unit not in {"s", "ms", "us", "ns"}:
+        raise ValueError(f"Invalid time unit: {unit}")
+
+    time_idx = table.schema.get_field_index(source_field)
+    if time_idx < 0:
+        return table
+
+    return table.set_column(
+        time_idx,
+        target_field,
+        table[source_field].cast(pa.timestamp(unit, tz)),
+    )
+
+
+def get_datasource_metadata_table(
+    datasources: list[DatasourceBaseModel],
+    *,
+    fields: list[str],
+    namespaces: list[str],
+) -> pa.Table:
+    """
+    Get a pyarrow table containing metadata columns for a list of datasources.
+
+    Args:
+        datasources: The list of datasources to get metadata for.
+        fields: The fields to include in the metadata table.
+        namespaces: The namespaces to include in the metadata table.
+
+    Returns:
+        The metadata table.
+    """
+    return pa.Table.from_pylist(
+        [s.to_arrow_columns(fields=fields, namespaces=namespaces) for s in datasources]
+    )
+
+
+def load_table_with_extra(content: bytes, *, is_stream=True, **kwargs) -> pa.Table:
+    """
+    Load a table from an arrow stream or file. Optionally add extra metadata columns.
+
+    Args:
+        content: The content of the stream or file.
+        is_stream: Whether the content is an arrow stream or an arrow file.
+        **kwargs: The extra metadata columns to add to the table.
+
+    Returns:
+        The loaded table.
+    """
+    if is_stream:
+        reader = pa.ipc.open_stream(content)
+    else:
+        reader = pa.ipc.open_file(content)
+
+    table = reader.read_all()
+
+    if kwargs:
+        count = len(table)
+        for key, value in kwargs.items():
+            table = table.append_column(key, pa.repeat(value, count))
+
+    return table

earthscope_sdk-1.2.0b0/src/earthscope_sdk/client/data_access/_arrow/_gnss.py

@@ -0,0 +1,116 @@
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from earthscope_sdk.client.data_access._arrow._common import convert_time
+
+_SYSTEM_ID = "system_id"
+_SYSTEM_TABLE = pa.Table.from_pylist(
+    [
+        {_SYSTEM_ID: 1, "system": "G"},  # GPS
+        {_SYSTEM_ID: 2, "system": "R"},  # GLONASS
+        {_SYSTEM_ID: 3, "system": "S"},  # SBAS
+        {_SYSTEM_ID: 4, "system": "E"},  # Galileo
+        {_SYSTEM_ID: 5, "system": "C"},  # BeiDou
+        {_SYSTEM_ID: 6, "system": "J"},  # QZSS
+        {_SYSTEM_ID: 7, "system": "I"},  # IRNSS/NavIC
+    ],
+    schema=pa.schema(
+        [
+            pa.field(_SYSTEM_ID, pa.uint8()),
+            pa.field("system", pa.string()),
+        ]
+    ),
+)
+
+
+def _convert_obs_code(
+    table: pa.Table,
+    *,
+    source_field: str = "obs",
+    target_field: str = "obs_code",
+) -> pa.Table:
+    """
+    Convert the obs_code field from packed chars to a string.
+
+    Args:
+        table: The table to convert.
+        source_field: The name of the field to convert.
+        target_field: The name of the field to store the result.
+
+    Returns:
+        The converted table.
+    """
+    # extract obs_code (encoded as packed chars)
+    result_chunks = [
+        pc.cast(
+            pa.FixedSizeBinaryArray.from_buffers(
+                type=pa.binary(2),
+                length=len(chunk),
+                buffers=[None, chunk.buffers()[1]],
+            ),
+            pa.string(),
+        )
+        for chunk in table[source_field].chunks
+    ]
+
+    return table.set_column(
+        table.schema.get_field_index(source_field),
+        target_field,
+        # can't easily swap endianness of packed chars, so reverse the string
+        pc.utf8_reverse(pa.chunked_array(result_chunks, type=pa.string())),
+    )
+
+
+def _convert_system(table: pa.Table, *, field: str) -> pa.Table:
+    """
+    Convert the sys field from integer to string.
+
+    Args:
+        table: The table to convert.
+        field: The name of the field to convert.
+
+    Returns:
+        The converted table.
+    """
+    return table.join(_SYSTEM_TABLE, field, right_keys=_SYSTEM_ID).drop(field)
+
+
+def prettify_observations_table(table: pa.Table) -> pa.Table:
+    """
+    Clean up a raw observations table (as stored in TileDB) for easier use.
+
+    Args:
+        table: The table to prettify.
+
+    Returns:
+        The prettified table.
+    """
+    # Cast time column (integer milliseconds since epoch) to timestamp type
+    table = convert_time(table)
+
+    # Expose friendlier column names
+    table = table.rename_columns({"sat": "satellite"})
+
+    # Convert system column from integer to string
+    table = _convert_system(table, field="sys")
+
+    # Convert obs_code column from packed chars to string
+    table = _convert_obs_code(table)
+
+    return table
+
+
+def prettify_positions_table(table: pa.Table) -> pa.Table:
+    """
+    Clean up a raw positions table (as stored in TileDB) for easier use.
+
+    Args:
+        table: The table to prettify.
+
+    Returns:
+        The prettified table.
+    """
+    # Convert time column (integer milliseconds since epoch) to timestamp type
+    table = convert_time(table)
+
+    return table

earthscope_sdk-1.2.0b0/src/earthscope_sdk/client/data_access/_base.py

@@ -0,0 +1,85 @@
+import datetime as dt
+from typing import TYPE_CHECKING, Union
+
+from earthscope_sdk.client.data_access._arrow._common import load_table_with_extra
+from earthscope_sdk.common.service import SdkService
+from earthscope_sdk.util._time import to_utc_dt
+
+if TYPE_CHECKING:
+    from pyarrow import Table
+
+
+class DataAccessBaseService(SdkService):
+    """
+    L1 data access service functionality
+    """
+
+    async def _get_gnss_observations(
+        self,
+        *,
+        session_edid: str,
+        start_datetime: dt.datetime,
+        end_datetime: dt.datetime,
+        system: Union[str, list[str]] = [],
+        satellite: Union[str, list[str]] = [],
+        obs_code: Union[str, list[str]] = [],
+        field: Union[str, list[str]] = [],
+    ) -> "Table":
+        """
+        Retrieve GNSS observations.
+
+        This method retrieves GNSS observations for a given session.
+
+        Args:
+            session_edid: The session EDID to retrieve observations for.
+            start_datetime: The start datetime to retrieve positions for.
+            end_datetime: The end datetime to retrieve positions for.
+            system: The system(s) to retrieve observations for.
+            satellite: The satellite(s) to retrieve observations for.
+            obs_code: The observation code(s) to retrieve observations for.
+            field: The field(s) to retrieve observations for.
+
+        Returns:
+            A pyarrow table containing the GNSS observations.
+        """
+
+        # lazy import
+
+        headers = {
+            "accept": "application/vnd.apache.arrow.stream",
+        }
+
+        params = {
+            "session_id": session_edid,
+            "start_datetime": to_utc_dt(start_datetime).isoformat(),
+            "end_datetime": to_utc_dt(end_datetime).isoformat(),
+        }
+
+        if system:
+            params["system"] = system
+
+        if satellite:
+            params["satellite"] = satellite
+
+        if obs_code:
+            params["obs_code"] = obs_code
+
+        if field:
+            params["field"] = field
+
+        req = self.ctx.httpx_client.build_request(
+            method="GET",
+            url=f"{self.resources.api_url}beta/data-products/gnss/observations",
+            headers=headers,
+            params=params,
+            timeout=30,
+        )
+
+        resp = await self._send_with_retries(req)
+
+        return await self.ctx.run_in_executor(
+            load_table_with_extra,
+            resp.content,
+            # Add edid column to the table returned by the API
+            edid=session_edid,
+        )