earthscope-sdk 1.1.0__tar.gz → 1.2.0b0__tar.gz

{earthscope_sdk-1.1.0 → earthscope_sdk-1.2.0b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: earthscope-sdk
-Version: 1.1.0
+Version: 1.2.0b0
 Summary: An SDK for EarthScope API
 Author-email: EarthScope <data-help@earthscope.org>
 License:                                  Apache License
@@ -216,6 +216,9 @@ License-File: LICENSE
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: pydantic-settings[toml]>=2.8.0
 Requires-Dist: stamina>=24.3.0
+Provides-Extra: arrow
+Requires-Dist: pyarrow>=20.0.0; extra == "arrow"
+Requires-Dist: pyarrow-stubs>=20.0.0; extra == "arrow"
 Provides-Extra: dev
 Requires-Dist: bumpver; extra == "dev"
 Requires-Dist: build; extra == "dev"
@@ -242,6 +245,13 @@ Install from PyPI
 pip install earthscope-sdk
 ```
 
+Or with optional dependencies:
+
+```shell
+# install arrow dependencies for efficient data access
+pip install earthscope-sdk[arrow]
+```
+
 ### Usage
 
 For detailed usage info and examples, visit [our SDK docs](https://docs.earthscope.org/projects/SDK).

{earthscope_sdk-1.1.0 → earthscope_sdk-1.2.0b0}/README.md

@@ -12,6 +12,13 @@ Install from PyPI
 pip install earthscope-sdk
 ```
 
+Or with optional dependencies:
+
+```shell
+# install arrow dependencies for efficient data access
+pip install earthscope-sdk[arrow]
+```
+
 ### Usage
 
 For detailed usage info and examples, visit [our SDK docs](https://docs.earthscope.org/projects/SDK).

{earthscope_sdk-1.1.0 → earthscope_sdk-1.2.0b0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "earthscope-sdk"
-version = "1.1.0"
+version = "1.2.0b0"
 description = "An SDK for EarthScope API"
 readme = "README.md"
 authors = [{ name = "EarthScope", email = "data-help@earthscope.org" }]
@@ -25,6 +25,7 @@ dependencies = [
 requires-python = ">=3.9"
 
 [project.optional-dependencies]
+arrow = ["pyarrow>=20.0.0", "pyarrow-stubs>=20.0.0"]
 dev = [
     "bumpver",
     "build",
@@ -41,7 +42,7 @@ dev = [
 Homepage = "https://gitlab.com/earthscope/public/earthscope-sdk"
 
 [tool.bumpver]
-current_version = "1.1.0"
+current_version = "1.2.0b0"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "chore: bump version {old_version} -> {new_version}"
 commit = true

{earthscope_sdk-1.1.0 → earthscope_sdk-1.2.0b0}/src/earthscope_sdk/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.1.0"
+__version__ = "1.2.0b0"
 
 from earthscope_sdk.client import AsyncEarthScopeClient, EarthScopeClient
 

earthscope_sdk-1.2.0b0/src/earthscope_sdk/client/_client.py

@@ -0,0 +1,82 @@
+from functools import cached_property
+
+from earthscope_sdk.common.client import AsyncSdkClient, SdkClient
+
+
+class AsyncEarthScopeClient(AsyncSdkClient):
+    """
+    An async client for interacting with api.earthscope.org
+    """
+
+    @cached_property
+    def data(self):
+        """
+        Data access functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.data_access._service import AsyncDataAccessService
+
+        return AsyncDataAccessService(self)
+
+    @cached_property
+    def discover(self):
+        """
+        Data discovery functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.discovery._service import AsyncDiscoveryService
+
+        return AsyncDiscoveryService(self._ctx)
+
+    @cached_property
+    def user(self):
+        """
+        User and Identity Management functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.user._service import AsyncUserService
+
+        return AsyncUserService(self._ctx)
+
+
+class EarthScopeClient(SdkClient):
+    """
+    A client for interacting with api.earthscope.org
+    """
+
+    @cached_property
+    def _async_client(self):
+        """
+        An async client for interacting with api.earthscope.org
+        """
+        return AsyncEarthScopeClient(ctx=self._ctx)
+
+    @cached_property
+    def data(self):
+        """
+        Data access functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.data_access._service import DataAccessService
+
+        return DataAccessService(self._async_client)
+
+    @cached_property
+    def discover(self):
+        """
+        Data discovery functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.discovery._service import DiscoveryService
+
+        return DiscoveryService(self._ctx)
+
+    @cached_property
+    def user(self):
+        """
+        User and Identity Management functionality
+        """
+        # lazy load
+        from earthscope_sdk.client.user._service import UserService
+
+        return UserService(self._ctx)

earthscope_sdk-1.2.0b0/src/earthscope_sdk/client/data_access/_arrow/_common.py

@@ -0,0 +1,94 @@
+try:
+    import pyarrow as pa
+except ModuleNotFoundError as e:
+    raise ModuleNotFoundError(
+        "Optional dependency 'pyarrow' is required for this feature. "
+        "Install it with: pip install earthscope-sdk[arrow]"
+    ) from e
+
+
+from earthscope_sdk.client.discovery.models import DatasourceBaseModel
+
+
+def convert_time(
+    table: pa.Table,
+    *,
+    source_field: str = "time",
+    target_field: str = "timestamp",
+    unit: str = "ms",
+    tz: str = "UTC",
+) -> pa.Table:
+    """
+    Convert a unix timestamp (integer since epoch) field to a timestamp type.
+
+    Args:
+        table: The table to convert.
+        source_field: The name of the field to convert.
+        target_field: The name of the field to store the result.
+        unit: The unit of the time field.
+        tz: The timezone of the time field.
+
+    Returns:
+        The converted table.
+    """
+    if unit not in {"s", "ms", "us", "ns"}:
+        raise ValueError(f"Invalid time unit: {unit}")
+
+    time_idx = table.schema.get_field_index(source_field)
+    if time_idx < 0:
+        return table
+
+    return table.set_column(
+        time_idx,
+        target_field,
+        table[source_field].cast(pa.timestamp(unit, tz)),
+    )
+
+
+def get_datasource_metadata_table(
+    datasources: list[DatasourceBaseModel],
+    *,
+    fields: list[str],
+    namespaces: list[str],
+) -> pa.Table:
+    """
+    Get a pyarrow table containing metadata columns for a list of datasources.
+
+    Args:
+        datasources: The list of datasources to get metadata for.
+        fields: The fields to include in the metadata table.
+        namespaces: The namespaces to include in the metadata table.
+
+    Returns:
+        The metadata table.
+    """
+    return pa.Table.from_pylist(
+        [s.to_arrow_columns(fields=fields, namespaces=namespaces) for s in datasources]
+    )
+
+
+def load_table_with_extra(content: bytes, *, is_stream=True, **kwargs) -> pa.Table:
+    """
+    Load a table from an arrow stream or file. Optionally add extra metadata columns.
+
+    Args:
+        content: The content of the stream or file.
+        is_stream: Whether the content is an arrow stream or an arrow file.
+        **kwargs: The extra metadata columns to add to the table.
+
+    Returns:
+        The loaded table.
+    """
+    if is_stream:
+        reader = pa.ipc.open_stream(content)
+    else:
+        reader = pa.ipc.open_file(content)
+
+    table = reader.read_all()
+
+    if kwargs:
+        count = len(table)
+        for key, value in kwargs.items():
+            table = table.append_column(key, pa.repeat(value, count))
+
+    return table

earthscope_sdk-1.2.0b0/src/earthscope_sdk/client/data_access/_arrow/_gnss.py

@@ -0,0 +1,116 @@
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from earthscope_sdk.client.data_access._arrow._common import convert_time
+
+_SYSTEM_ID = "system_id"
+_SYSTEM_TABLE = pa.Table.from_pylist(
+    [
+        {_SYSTEM_ID: 1, "system": "G"},  # GPS
+        {_SYSTEM_ID: 2, "system": "R"},  # GLONASS
+        {_SYSTEM_ID: 3, "system": "S"},  # SBAS
+        {_SYSTEM_ID: 4, "system": "E"},  # Galileo
+        {_SYSTEM_ID: 5, "system": "C"},  # BeiDou
+        {_SYSTEM_ID: 6, "system": "J"},  # QZSS
+        {_SYSTEM_ID: 7, "system": "I"},  # IRNSS/NavIC
+    ],
+    schema=pa.schema(
+        [
+            pa.field(_SYSTEM_ID, pa.uint8()),
+            pa.field("system", pa.string()),
+        ]
+    ),
+)
+
+
+def _convert_obs_code(
+    table: pa.Table,
+    *,
+    source_field: str = "obs",
+    target_field: str = "obs_code",
+) -> pa.Table:
+    """
+    Convert the obs_code field from packed chars to a string.
+
+    Args:
+        table: The table to convert.
+        source_field: The name of the field to convert.
+        target_field: The name of the field to store the result.
+
+    Returns:
+        The converted table.
+    """
+    # extract obs_code (encoded as packed chars)
+    result_chunks = [
+        pc.cast(
+            pa.FixedSizeBinaryArray.from_buffers(
+                type=pa.binary(2),
+                length=len(chunk),
+                buffers=[None, chunk.buffers()[1]],
+            ),
+            pa.string(),
+        )
+        for chunk in table[source_field].chunks
+    ]
+
+    return table.set_column(
+        table.schema.get_field_index(source_field),
+        target_field,
+        # can't easily swap endianness of packed chars, so reverse the string
+        pc.utf8_reverse(pa.chunked_array(result_chunks, type=pa.string())),
+    )
+
+
+def _convert_system(table: pa.Table, *, field: str) -> pa.Table:
+    """
+    Convert the sys field from integer to string.
+
+    Args:
+        table: The table to convert.
+        field: The name of the field to convert.
+
+    Returns:
+        The converted table.
+    """
+    return table.join(_SYSTEM_TABLE, field, right_keys=_SYSTEM_ID).drop(field)
+
+
+def prettify_observations_table(table: pa.Table) -> pa.Table:
+    """
+    Clean up a raw observations table (as stored in TileDB) for easier use.
+
+    Args:
+        table: The table to prettify.
+
+    Returns:
+        The prettified table.
+    """
+    # Cast time column (integer milliseconds since epoch) to timestamp type
+    table = convert_time(table)
+
+    # Expose friendlier column names
+    table = table.rename_columns({"sat": "satellite"})
+
+    # Convert system column from integer to string
+    table = _convert_system(table, field="sys")
+
+    # Convert obs_code column from packed chars to string
+    table = _convert_obs_code(table)
+
+    return table
+
+
+def prettify_positions_table(table: pa.Table) -> pa.Table:
+    """
+    Clean up a raw positions table (as stored in TileDB) for easier use.
+
+    Args:
+        table: The table to prettify.
+
+    Returns:
+        The prettified table.
+    """
+    # Convert time column (integer milliseconds since epoch) to timestamp type
+    table = convert_time(table)
+
+    return table

earthscope_sdk-1.2.0b0/src/earthscope_sdk/client/data_access/_base.py

@@ -0,0 +1,85 @@
+import datetime as dt
+from typing import TYPE_CHECKING, Union
+
+from earthscope_sdk.client.data_access._arrow._common import load_table_with_extra
+from earthscope_sdk.common.service import SdkService
+from earthscope_sdk.util._time import to_utc_dt
+
+if TYPE_CHECKING:
+    from pyarrow import Table
+
+
+class DataAccessBaseService(SdkService):
+    """
+    L1 data access service functionality
+    """
+
+    async def _get_gnss_observations(
+        self,
+        *,
+        session_edid: str,
+        start_datetime: dt.datetime,
+        end_datetime: dt.datetime,
+        system: Union[str, list[str]] = [],
+        satellite: Union[str, list[str]] = [],
+        obs_code: Union[str, list[str]] = [],
+        field: Union[str, list[str]] = [],
+    ) -> "Table":
+        """
+        Retrieve GNSS observations.
+
+        This method retrieves GNSS observations for a given session.
+
+        Args:
+            session_edid: The session EDID to retrieve observations for.
+            start_datetime: The start datetime to retrieve positions for.
+            end_datetime: The end datetime to retrieve positions for.
+            system: The system(s) to retrieve observations for.
+            satellite: The satellite(s) to retrieve observations for.
+            obs_code: The observation code(s) to retrieve observations for.
+            field: The field(s) to retrieve observations for.
+
+        Returns:
+            A pyarrow table containing the GNSS observations.
+        """
+
+        # lazy import
+
+        headers = {
+            "accept": "application/vnd.apache.arrow.stream",
+        }
+
+        params = {
+            "session_id": session_edid,
+            "start_datetime": to_utc_dt(start_datetime).isoformat(),
+            "end_datetime": to_utc_dt(end_datetime).isoformat(),
+        }
+
+        if system:
+            params["system"] = system
+
+        if satellite:
+            params["satellite"] = satellite
+
+        if obs_code:
+            params["obs_code"] = obs_code
+
+        if field:
+            params["field"] = field
+
+        req = self.ctx.httpx_client.build_request(
+            method="GET",
+            url=f"{self.resources.api_url}beta/data-products/gnss/observations",
+            headers=headers,
+            params=params,
+            timeout=30,
+        )
+
+        resp = await self._send_with_retries(req)
+
+        return await self.ctx.run_in_executor(
+            load_table_with_extra,
+            resp.content,
+            # Add edid column to the table returned by the API
+            edid=session_edid,
+        )