tilebox-datasets 0.35.0__py3-none-any.whl → 0.36.0__py3-none-any.whl

tilebox/datasets/aio/dataset.py

@@ -7,17 +7,18 @@ from warnings import warn
 import xarray as xr
 from tqdm.auto import tqdm
 
+from _tilebox.grpc.aio.pagination import Pagination as PaginationProtocol
+from _tilebox.grpc.aio.pagination import paginated_request
 from _tilebox.grpc.aio.producer_consumer import async_producer_consumer
 from _tilebox.grpc.error import ArgumentError, NotFoundError
 from tilebox.datasets.aio.pagination import (
-    paginated_request,
     with_progressbar,
     with_time_progress_callback,
     with_time_progressbar,
 )
 from tilebox.datasets.data.collection import CollectionInfo
 from tilebox.datasets.data.data_access import QueryFilters, SpatialFilter, SpatialFilterLike
-from tilebox.datasets.data.datapoint import DatapointInterval, DatapointPage, QueryResultPage
+from tilebox.datasets.data.datapoint import DatapointInterval, DatapointIntervalLike, DatapointPage, QueryResultPage
 from tilebox.datasets.data.datasets import Dataset
 from tilebox.datasets.data.pagination import Pagination
 from tilebox.datasets.data.time_interval import TimeInterval, TimeIntervalLike
@@ -242,7 +243,7 @@ class CollectionClient:
 
     async def _find_interval(
         self,
-        datapoint_id_interval: tuple[str, str] | tuple[UUID, UUID],
+        datapoint_id_interval: DatapointIntervalLike,
         end_inclusive: bool = True,
         *,
         skip_data: bool = False,
@@ -266,18 +267,13 @@ class CollectionClient:
                 datapoint_id_interval, end_inclusive, skip_data=skip_data, show_progress=show_progress
             )
 
-        start_id, end_id = datapoint_id_interval
-
         filters = QueryFilters(
-            temporal_extent=DatapointInterval(
-                start_id=as_uuid(start_id),
-                end_id=as_uuid(end_id),
-                start_exclusive=False,
-                end_inclusive=end_inclusive,
-            )
+            temporal_extent=DatapointInterval.parse(datapoint_id_interval, end_inclusive=end_inclusive)
         )
 
-        request = partial(self._dataset._service.query, [self._collection.id], filters, skip_data)
+        async def request(page: PaginationProtocol) -> QueryResultPage:
+            query_page = Pagination(page.limit, page.starting_after)
+            return await self._dataset._service.query([self._collection.id], filters, skip_data, query_page)
 
         initial_page = Pagination()
         pages = paginated_request(request, initial_page)
@@ -288,27 +284,19 @@ class CollectionClient:
 
     async def _find_interval_legacy(
         self,
-        datapoint_id_interval: tuple[str, str] | tuple[UUID, UUID],
+        datapoint_id_interval: DatapointIntervalLike,
         end_inclusive: bool = True,
         *,
         skip_data: bool = False,
         show_progress: bool = False,
     ) -> xr.Dataset:
-        start_id, end_id = datapoint_id_interval
+        datapoint_interval = DatapointInterval.parse(datapoint_id_interval, end_inclusive=end_inclusive)
 
-        datapoint_interval = DatapointInterval(
-            start_id=as_uuid(start_id),
-            end_id=as_uuid(end_id),
-            start_exclusive=False,
-            end_inclusive=end_inclusive,
-        )
-        request = partial(
-            self._dataset._service.get_dataset_for_datapoint_interval,
-            str(self._collection.id),
-            datapoint_interval,
-            skip_data,
-            False,
-        )
+        async def request(page: PaginationProtocol) -> DatapointPage:
+            query_page = Pagination(page.limit, page.starting_after)
+            return await self._dataset._service.get_dataset_for_datapoint_interval(
+                str(self._collection.id), datapoint_interval, skip_data, False, query_page
+            )
 
         initial_page = Pagination()
         pages = paginated_request(request, initial_page)
@@ -427,9 +415,10 @@ class CollectionClient:
             yield page
 
     async def _load_page(
-        self, filters: QueryFilters, skip_data: bool, page: Pagination | None = None
+        self, filters: QueryFilters, skip_data: bool, page: PaginationProtocol | None = None
     ) -> QueryResultPage:
-        return await self._dataset._service.query([self._collection.id], filters, skip_data, page)
+        query_page = Pagination(page.limit, page.starting_after) if page else Pagination()
+        return await self._dataset._service.query([self._collection.id], filters, skip_data, query_page)
 
     async def _load_legacy(
         self,
@@ -472,10 +461,11 @@ class CollectionClient:
             yield page
 
     async def _load_page_legacy(
-        self, time_interval: TimeInterval, skip_data: bool, skip_meta: bool, page: Pagination | None = None
+        self, time_interval: TimeInterval, skip_data: bool, skip_meta: bool, page: PaginationProtocol | None = None
     ) -> DatapointPage:
+        query_page = Pagination(page.limit, page.starting_after) if page else Pagination()
         return await self._dataset._service.get_dataset_for_time_interval(
-            str(self._collection.id), time_interval, skip_data, skip_meta, page
+            str(self._collection.id), time_interval, skip_data, skip_meta, query_page
         )
 
     async def ingest(

tilebox/datasets/aio/pagination.py

@@ -1,5 +1,5 @@
 import time
-from collections.abc import AsyncIterator, Awaitable, Callable
+from collections.abc import AsyncIterator
 from datetime import datetime, timezone
 from typing import TypeVar
 
@@ -9,41 +9,11 @@ from tilebox.datasets.data import (
     TimeInterval,
 )
 from tilebox.datasets.data.datapoint import DatapointPage, QueryResultPage
-from tilebox.datasets.data.pagination import Pagination
 from tilebox.datasets.progress import ProgressCallback, TimeIntervalProgressBar
 
 ResultPage = TypeVar("ResultPage", bound=DatapointPage | QueryResultPage)
 
 
-async def paginated_request(
-    paging_request: Callable[[Pagination], Awaitable[ResultPage]],
-    initial_page: Pagination | None = None,
-) -> AsyncIterator[ResultPage]:
-    """Make a paginated request to a gRPC service endpoint.
-
-    The endpoint is expected to return a next_page field, which is used for subsequent requests. Once no such
-    next_page field is returned, the request is completed.
-
-    Args:
-        paging_request: A function that takes a page as input and returns a Datapoints object
-            Often this will be a functools.partial object that wraps a gRPC service endpoint
-            and only leaves the page argument remaining
-        initial_page: The initial page to request
-
-    Yields:
-        Datapoints: The individual pages of the response
-    """
-    if initial_page is None:
-        initial_page = Pagination()
-
-    response = await paging_request(initial_page)
-    yield response
-
-    while response.next_page.starting_after is not None:
-        response = await paging_request(response.next_page)
-        yield response
-
-
 async def with_progressbar(
     paginated_request: AsyncIterator[ResultPage],
     progress_description: str,

tilebox/datasets/data/datapoint.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 from datetime import datetime
-from typing import Any
+from typing import Any, TypeAlias
 from uuid import UUID
 
 from tilebox.datasets.data.pagination import Pagination
@@ -9,6 +9,8 @@ from tilebox.datasets.data.uuid import uuid_message_to_uuid, uuid_to_uuid_messag
 from tilebox.datasets.datasetsv1 import core_pb2, data_access_pb2, data_ingestion_pb2
 from tilebox.datasets.message_pool import get_message_type
 
+DatapointIntervalLike: TypeAlias = "tuple[str, str] | tuple[UUID, UUID] | DatapointInterval"
+
 
 @dataclass(frozen=True)
 class DatapointInterval:
@@ -34,6 +36,47 @@ class DatapointInterval:
             end_inclusive=self.end_inclusive,
         )
 
+    @classmethod
+    def parse(
+        cls, arg: DatapointIntervalLike, start_exclusive: bool = False, end_inclusive: bool = True
+    ) -> "DatapointInterval":
+        """
+        Convert a variety of input types to a DatapointInterval.
+
+        Supported input types:
+        - DatapointInterval: Return the input as is
+        - tuple of two UUIDs: Return an DatapointInterval with start and end id set to the given values
+        - tuple of two strings: Return an DatapointInterval with start and end id set to the UUIDs parsed from the given strings
+
+        Args:
+            arg: The input to convert
+            start_exclusive: Whether the start id is exclusive
+            end_inclusive: Whether the end id is inclusive
+
+        Returns:
+            DatapointInterval: The parsed ID interval
+        """
+
+        match arg:
+            case DatapointInterval(_, _, _, _):
+                return arg
+            case (UUID(), UUID()):
+                start, end = arg
+                return DatapointInterval(
+                    start_id=start,
+                    end_id=end,
+                    start_exclusive=start_exclusive,
+                    end_inclusive=end_inclusive,
+                )
+            case (str(), str()):
+                start, end = arg
+                return DatapointInterval(
+                    start_id=UUID(start),
+                    end_id=UUID(end),
+                    start_exclusive=start_exclusive,
+                    end_inclusive=end_inclusive,
+                )
+
 
 @dataclass(frozen=True)
 class AnyMessage:

tilebox/datasets/datasetsv1/core_pb2.py

@@ -26,7 +26,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__
 from tilebox.datasets.datasetsv1 import dataset_type_pb2 as datasets_dot_v1_dot_dataset__type__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16\x64\x61tasets/v1/core.proto\x12\x0b\x64\x61tasets.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1e\x64\x61tasets/v1/dataset_type.proto\"\x18\n\x02ID\x12\x12\n\x04uuid\x18\x01 \x01(\x0cR\x04uuid\"\xce\x01\n\x0cTimeInterval\x12\x39\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12\x35\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x07\x65ndTime\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"\xb5\x01\n\x11\x44\x61tapointInterval\x12*\n\x08start_id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x07startId\x12&\n\x06\x65nd_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x05\x65ndId\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"v\n\x10LegacyPagination\x12\x19\n\x05limit\x18\x01 \x01(\x03H\x00R\x05limit\x88\x01\x01\x12*\n\x0estarting_after\x18\x02 \x01(\tH\x01R\rstartingAfter\x88\x01\x01\x42\x08\n\x06_limitB\x11\n\x0f_starting_after\"\x81\x01\n\nPagination\x12\x19\n\x05limit\x18\x01 \x01(\x03H\x00R\x05limit\x88\x01\x01\x12;\n\x0estarting_after\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDH\x01R\rstartingAfter\x88\x01\x01\x42\x08\n\x06_limitB\x11\n\x0f_starting_after\"6\n\x03\x41ny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05value\">\n\x0bRepeatedAny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x03(\x0cR\x05value\"\xad\x01\n\x11\x44\x61tapointMetadata\x12\x39\n\nevent_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\teventTime\x12\x41\n\x0eingestion_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\ringestionTime\x12\x13\n\x02id\x18\x03 \x01(\tH\x00R\x02id\x88\x01\x01\x42\x05\n\x03_id\"n\n\nDatapoints\x12\x32\n\x04meta\x18\x01 \x03(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x18.datasets.v1.RepeatedAnyR\x04\x64\x61ta\"\xc0\x01\n\rDatapointPage\x12\x32\n\x04meta\x18\x01 \x03(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x18.datasets.v1.RepeatedAnyR\x04\x64\x61ta\x12?\n\tnext_page\x18\x03 \x01(\x0b\x32\x1d.datasets.v1.LegacyPaginationH\x00R\x08nextPage\x88\x01\x01\x42\x0c\n\n_next_page\"e\n\tDatapoint\x12\x32\n\x04meta\x18\x01 \x01(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12$\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x10.datasets.v1.AnyR\x04\x64\x61ta\"^\n\nCollection\x12\x1b\n\tlegacy_id\x18\x01 \x01(\tR\x08legacyId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1f\n\x02id\x18\x03 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\"\xc3\x01\n\x0e\x43ollectionInfo\x12\x37\n\ncollection\x18\x01 \x01(\x0b\x32\x17.datasets.v1.CollectionR\ncollection\x12\x42\n\x0c\x61vailability\x18\x02 \x01(\x0b\x32\x19.datasets.v1.TimeIntervalH\x00R\x0c\x61vailability\x88\x01\x01\x12\x19\n\x05\x63ount\x18\x03 \x01(\x04H\x01R\x05\x63ount\x88\x01\x01\x42\x0f\n\r_availabilityB\x08\n\x06_count\"B\n\x0f\x43ollectionInfos\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x1b.datasets.v1.CollectionInfoR\x04\x64\x61ta\"\x82\x03\n\x07\x44\x61taset\x12\x1f\n\x02id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\x12*\n\x08group_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x07groupId\x12.\n\x04type\x18\x03 \x01(\x0b\x32\x1a.datasets.v1.AnnotatedTypeR\x04type\x12\x1b\n\tcode_name\x18\x04 \x01(\tR\x08\x63odeName\x12\x12\n\x04name\x18\x05 \x01(\tR\x04name\x12\x18\n\x07summary\x18\x06 \x01(\tR\x07summary\x12\x12\n\x04icon\x18\x07 \x01(\tR\x04icon\x12 \n\x0b\x64\x65scription\x18\x08 \x01(\tR\x0b\x64\x65scription\x12@\n\x0bpermissions\x18\n \x03(\x0e\x32\x1e.datasets.v1.DatasetPermissionR\x0bpermissions\x12\x37\n\nvisibility\x18\x0b \x01(\x0e\x32\x17.datasets.v1.VisibilityR\nvisibility\"\xa2\x01\n\x0c\x44\x61tasetGroup\x12\x1f\n\x02id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\x12,\n\tparent_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x08parentId\x12\x1b\n\tcode_name\x18\x03 \x01(\tR\x08\x63odeName\x12\x12\n\x04name\x18\x04 \x01(\tR\x04name\x12\x12\n\x04icon\x18\x05 \x01(\tR\x04icon*\x9b\x01\n\x11\x44\x61tasetPermission\x12\"\n\x1e\x44\x41TASET_PERMISSION_UNSPECIFIED\x10\x00\x12\"\n\x1e\x44\x41TASET_PERMISSION_ACCESS_DATA\x10\x01\x12!\n\x1d\x44\x41TASET_PERMISSION_WRITE_DATA\x10\x02\x12\x1b\n\x17\x44\x41TASET_PERMISSION_EDIT\x10\x03*v\n\nVisibility\x12\x1a\n\x16VISIBILITY_UNSPECIFIED\x10\x00\x12\x16\n\x12VISIBILITY_PRIVATE\x10\x01\x12\x1d\n\x19VISIBILITY_SHARED_WITH_ME\x10\x02\x12\x15\n\x11VISIBILITY_PUBLIC\x10\x03\x42\xab\x01\n\x0f\x63om.datasets.v1B\tCoreProtoP\x01Z@github.com/tilebox/tilebox-go/protogen/go/datasets/v1;datasetsv1\xa2\x02\x03\x44XX\xaa\x02\x0b\x44\x61tasets.V1\xca\x02\x0b\x44\x61tasets\\V1\xe2\x02\x17\x44\x61tasets\\V1\\GPBMetadata\xea\x02\x0c\x44\x61tasets::V1b\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16\x64\x61tasets/v1/core.proto\x12\x0b\x64\x61tasets.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1e\x64\x61tasets/v1/dataset_type.proto\"\x18\n\x02ID\x12\x12\n\x04uuid\x18\x01 \x01(\x0cR\x04uuid\"\xce\x01\n\x0cTimeInterval\x12\x39\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12\x35\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x07\x65ndTime\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"\xb5\x01\n\x11\x44\x61tapointInterval\x12*\n\x08start_id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x07startId\x12&\n\x06\x65nd_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x05\x65ndId\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"v\n\x10LegacyPagination\x12\x19\n\x05limit\x18\x01 \x01(\x03H\x00R\x05limit\x88\x01\x01\x12*\n\x0estarting_after\x18\x02 \x01(\tH\x01R\rstartingAfter\x88\x01\x01\x42\x08\n\x06_limitB\x11\n\x0f_starting_after\"\x81\x01\n\nPagination\x12\x19\n\x05limit\x18\x01 \x01(\x03H\x00R\x05limit\x88\x01\x01\x12;\n\x0estarting_after\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDH\x01R\rstartingAfter\x88\x01\x01\x42\x08\n\x06_limitB\x11\n\x0f_starting_after\"6\n\x03\x41ny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05value\">\n\x0bRepeatedAny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x03(\x0cR\x05value\"\xad\x01\n\x11\x44\x61tapointMetadata\x12\x39\n\nevent_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\teventTime\x12\x41\n\x0eingestion_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\ringestionTime\x12\x13\n\x02id\x18\x03 \x01(\tH\x00R\x02id\x88\x01\x01\x42\x05\n\x03_id\"n\n\nDatapoints\x12\x32\n\x04meta\x18\x01 \x03(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x18.datasets.v1.RepeatedAnyR\x04\x64\x61ta\"\xc0\x01\n\rDatapointPage\x12\x32\n\x04meta\x18\x01 \x03(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x18.datasets.v1.RepeatedAnyR\x04\x64\x61ta\x12?\n\tnext_page\x18\x03 \x01(\x0b\x32\x1d.datasets.v1.LegacyPaginationH\x00R\x08nextPage\x88\x01\x01\x42\x0c\n\n_next_page\"e\n\tDatapoint\x12\x32\n\x04meta\x18\x01 \x01(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12$\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x10.datasets.v1.AnyR\x04\x64\x61ta\"^\n\nCollection\x12\x1b\n\tlegacy_id\x18\x01 \x01(\tR\x08legacyId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1f\n\x02id\x18\x03 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\"\xc3\x01\n\x0e\x43ollectionInfo\x12\x37\n\ncollection\x18\x01 \x01(\x0b\x32\x17.datasets.v1.CollectionR\ncollection\x12\x42\n\x0c\x61vailability\x18\x02 \x01(\x0b\x32\x19.datasets.v1.TimeIntervalH\x00R\x0c\x61vailability\x88\x01\x01\x12\x19\n\x05\x63ount\x18\x03 \x01(\x04H\x01R\x05\x63ount\x88\x01\x01\x42\x0f\n\r_availabilityB\x08\n\x06_count\"B\n\x0f\x43ollectionInfos\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x1b.datasets.v1.CollectionInfoR\x04\x64\x61ta\"\x96\x03\n\x07\x44\x61taset\x12\x1f\n\x02id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\x12*\n\x08group_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x07groupId\x12.\n\x04type\x18\x03 \x01(\x0b\x32\x1a.datasets.v1.AnnotatedTypeR\x04type\x12\x1b\n\tcode_name\x18\x04 \x01(\tR\x08\x63odeName\x12\x12\n\x04name\x18\x05 \x01(\tR\x04name\x12\x18\n\x07summary\x18\x06 \x01(\tR\x07summary\x12\x12\n\x04icon\x18\x07 \x01(\tR\x04icon\x12 \n\x0b\x64\x65scription\x18\x08 \x01(\tR\x0b\x64\x65scription\x12@\n\x0bpermissions\x18\n \x03(\x0e\x32\x1e.datasets.v1.DatasetPermissionR\x0bpermissions\x12\x37\n\nvisibility\x18\x0b \x01(\x0e\x32\x17.datasets.v1.VisibilityR\nvisibility\x12\x12\n\x04slug\x18\x0c \x01(\tR\x04slug\"\xa2\x01\n\x0c\x44\x61tasetGroup\x12\x1f\n\x02id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\x12,\n\tparent_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x08parentId\x12\x1b\n\tcode_name\x18\x03 \x01(\tR\x08\x63odeName\x12\x12\n\x04name\x18\x04 \x01(\tR\x04name\x12\x12\n\x04icon\x18\x05 \x01(\tR\x04icon*\x9b\x01\n\x11\x44\x61tasetPermission\x12\"\n\x1e\x44\x41TASET_PERMISSION_UNSPECIFIED\x10\x00\x12\"\n\x1e\x44\x41TASET_PERMISSION_ACCESS_DATA\x10\x01\x12!\n\x1d\x44\x41TASET_PERMISSION_WRITE_DATA\x10\x02\x12\x1b\n\x17\x44\x41TASET_PERMISSION_EDIT\x10\x03*v\n\nVisibility\x12\x1a\n\x16VISIBILITY_UNSPECIFIED\x10\x00\x12\x16\n\x12VISIBILITY_PRIVATE\x10\x01\x12\x1d\n\x19VISIBILITY_SHARED_WITH_ME\x10\x02\x12\x15\n\x11VISIBILITY_PUBLIC\x10\x03\x42\xab\x01\n\x0f\x63om.datasets.v1B\tCoreProtoP\x01Z@github.com/tilebox/tilebox-go/protogen/go/datasets/v1;datasetsv1\xa2\x02\x03\x44XX\xaa\x02\x0b\x44\x61tasets.V1\xca\x02\x0b\x44\x61tasets\\V1\xe2\x02\x17\x44\x61tasets\\V1\\GPBMetadata\xea\x02\x0c\x44\x61tasets::V1b\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -34,10 +34,10 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'datasets.v1.core_pb2', _glo
 if not _descriptor._USE_C_DESCRIPTORS:
   _globals['DESCRIPTOR']._loaded_options = None
   _globals['DESCRIPTOR']._serialized_options = b'\n\017com.datasets.v1B\tCoreProtoP\001Z@github.com/tilebox/tilebox-go/protogen/go/datasets/v1;datasetsv1\242\002\003DXX\252\002\013Datasets.V1\312\002\013Datasets\\V1\342\002\027Datasets\\V1\\GPBMetadata\352\002\014Datasets::V1'
-  _globals['_DATASETPERMISSION']._serialized_start=2398
-  _globals['_DATASETPERMISSION']._serialized_end=2553
-  _globals['_VISIBILITY']._serialized_start=2555
-  _globals['_VISIBILITY']._serialized_end=2673
+  _globals['_DATASETPERMISSION']._serialized_start=2418
+  _globals['_DATASETPERMISSION']._serialized_end=2573
+  _globals['_VISIBILITY']._serialized_start=2575
+  _globals['_VISIBILITY']._serialized_end=2693
   _globals['_ID']._serialized_start=104
   _globals['_ID']._serialized_end=128
   _globals['_TIMEINTERVAL']._serialized_start=131
@@ -67,7 +67,7 @@ if not _descriptor._USE_C_DESCRIPTORS:
   _globals['_COLLECTIONINFOS']._serialized_start=1775
   _globals['_COLLECTIONINFOS']._serialized_end=1841
   _globals['_DATASET']._serialized_start=1844
-  _globals['_DATASET']._serialized_end=2230
-  _globals['_DATASETGROUP']._serialized_start=2233
-  _globals['_DATASETGROUP']._serialized_end=2395
+  _globals['_DATASET']._serialized_end=2250
+  _globals['_DATASETGROUP']._serialized_start=2253
+  _globals['_DATASETGROUP']._serialized_end=2415
 # @@protoc_insertion_point(module_scope)

tilebox/datasets/datasetsv1/core_pb2.pyi

@@ -155,7 +155,7 @@ class CollectionInfos(_message.Message):
     def __init__(self, data: _Optional[_Iterable[_Union[CollectionInfo, _Mapping]]] = ...) -> None: ...
 
 class Dataset(_message.Message):
-    __slots__ = ("id", "group_id", "type", "code_name", "name", "summary", "icon", "description", "permissions", "visibility")
+    __slots__ = ("id", "group_id", "type", "code_name", "name", "summary", "icon", "description", "permissions", "visibility", "slug")
     ID_FIELD_NUMBER: _ClassVar[int]
     GROUP_ID_FIELD_NUMBER: _ClassVar[int]
     TYPE_FIELD_NUMBER: _ClassVar[int]
@@ -166,6 +166,7 @@ class Dataset(_message.Message):
     DESCRIPTION_FIELD_NUMBER: _ClassVar[int]
     PERMISSIONS_FIELD_NUMBER: _ClassVar[int]
     VISIBILITY_FIELD_NUMBER: _ClassVar[int]
+    SLUG_FIELD_NUMBER: _ClassVar[int]
     id: ID
     group_id: ID
     type: _dataset_type_pb2.AnnotatedType
@@ -176,7 +177,8 @@ class Dataset(_message.Message):
     description: str
     permissions: _containers.RepeatedScalarFieldContainer[DatasetPermission]
     visibility: Visibility
-    def __init__(self, id: _Optional[_Union[ID, _Mapping]] = ..., group_id: _Optional[_Union[ID, _Mapping]] = ..., type: _Optional[_Union[_dataset_type_pb2.AnnotatedType, _Mapping]] = ..., code_name: _Optional[str] = ..., name: _Optional[str] = ..., summary: _Optional[str] = ..., icon: _Optional[str] = ..., description: _Optional[str] = ..., permissions: _Optional[_Iterable[_Union[DatasetPermission, str]]] = ..., visibility: _Optional[_Union[Visibility, str]] = ...) -> None: ...
+    slug: str
+    def __init__(self, id: _Optional[_Union[ID, _Mapping]] = ..., group_id: _Optional[_Union[ID, _Mapping]] = ..., type: _Optional[_Union[_dataset_type_pb2.AnnotatedType, _Mapping]] = ..., code_name: _Optional[str] = ..., name: _Optional[str] = ..., summary: _Optional[str] = ..., icon: _Optional[str] = ..., description: _Optional[str] = ..., permissions: _Optional[_Iterable[_Union[DatasetPermission, str]]] = ..., visibility: _Optional[_Union[Visibility, str]] = ..., slug: _Optional[str] = ...) -> None: ...
 
 class DatasetGroup(_message.Message):
     __slots__ = ("id", "parent_id", "code_name", "name", "icon")

tilebox/datasets/sync/dataset.py

@@ -7,10 +7,12 @@ import xarray as xr
 from tqdm.auto import tqdm
 
 from _tilebox.grpc.error import ArgumentError, NotFoundError
+from _tilebox.grpc.pagination import Pagination as PaginationProtocol
+from _tilebox.grpc.pagination import paginated_request
 from _tilebox.grpc.producer_consumer import concurrent_producer_consumer
 from tilebox.datasets.data.collection import CollectionInfo
 from tilebox.datasets.data.data_access import QueryFilters, SpatialFilter, SpatialFilterLike
-from tilebox.datasets.data.datapoint import DatapointInterval, DatapointPage, QueryResultPage
+from tilebox.datasets.data.datapoint import DatapointInterval, DatapointIntervalLike, DatapointPage, QueryResultPage
 from tilebox.datasets.data.datasets import Dataset
 from tilebox.datasets.data.pagination import Pagination
 from tilebox.datasets.data.time_interval import TimeInterval, TimeIntervalLike
@@ -27,7 +29,6 @@ from tilebox.datasets.protobuf_conversion.to_protobuf import (
 )
 from tilebox.datasets.service import TileboxDatasetService
 from tilebox.datasets.sync.pagination import (
-    paginated_request,
     with_progressbar,
     with_time_progress_callback,
     with_time_progressbar,
@@ -232,7 +233,7 @@ class CollectionClient:
 
     def _find_interval(
         self,
-        datapoint_id_interval: tuple[str, str] | tuple[UUID, UUID],
+        datapoint_id_interval: DatapointIntervalLike,
         end_inclusive: bool = True,
         *,
         skip_data: bool = False,
@@ -256,19 +257,13 @@ class CollectionClient:
                 datapoint_id_interval, end_inclusive, skip_data=skip_data, show_progress=show_progress
             )
 
-        start_id, end_id = datapoint_id_interval
-
         filters = QueryFilters(
-            temporal_extent=DatapointInterval(
-                start_id=as_uuid(start_id),
-                end_id=as_uuid(end_id),
-                start_exclusive=False,
-                end_inclusive=end_inclusive,
-            )
+            temporal_extent=DatapointInterval.parse(datapoint_id_interval, end_inclusive=end_inclusive)
         )
 
-        def request(page: Pagination) -> QueryResultPage:
-            return self._dataset._service.query([self._collection.id], filters, skip_data, page).get()
+        def request(page: PaginationProtocol) -> QueryResultPage:
+            query_page = Pagination(page.limit, page.starting_after)
+            return self._dataset._service.query([self._collection.id], filters, skip_data, query_page).get()
 
         initial_page = Pagination()
         pages = paginated_request(request, initial_page)
@@ -279,7 +274,7 @@ class CollectionClient:
 
     def _find_interval_legacy(
         self,
-        datapoint_id_interval: tuple[str, str] | tuple[UUID, UUID],
+        datapoint_id_interval: DatapointIntervalLike,
         end_inclusive: bool = True,
         *,
         skip_data: bool = False,
@@ -298,18 +293,12 @@ class CollectionClient:
         Returns:
             The datapoints in the given interval as an xarray dataset
         """
-        start_id, end_id = datapoint_id_interval
-
-        datapoint_interval = DatapointInterval(
-            start_id=as_uuid(start_id),
-            end_id=as_uuid(end_id),
-            start_exclusive=False,
-            end_inclusive=end_inclusive,
-        )
+        datapoint_interval = DatapointInterval.parse(datapoint_id_interval, end_inclusive=end_inclusive)
 
-        def request(page: Pagination) -> DatapointPage:
+        def request(page: PaginationProtocol) -> DatapointPage:
+            query_page = Pagination(page.limit, page.starting_after)
             return self._dataset._service.get_dataset_for_datapoint_interval(
-                str(self._collection.id), datapoint_interval, skip_data, False, page
+                str(self._collection.id), datapoint_interval, skip_data, False, query_page
             ).get()
 
         initial_page = Pagination()
@@ -427,8 +416,11 @@ class CollectionClient:
 
         yield from pages
 
-    def _load_page(self, filters: QueryFilters, skip_data: bool, page: Pagination | None = None) -> QueryResultPage:
-        return self._dataset._service.query([self._collection.id], filters, skip_data, page).get()
+    def _load_page(
+        self, filters: QueryFilters, skip_data: bool, page: PaginationProtocol | None = None
+    ) -> QueryResultPage:
+        query_page = Pagination(page.limit, page.starting_after) if page else Pagination()
+        return self._dataset._service.query([self._collection.id], filters, skip_data, query_page).get()
 
     def _load_legacy(
         self,
@@ -470,10 +462,11 @@ class CollectionClient:
         yield from pages
 
     def _load_page_legacy(
-        self, time_interval: TimeInterval, skip_data: bool, skip_meta: bool, page: Pagination | None = None
+        self, time_interval: TimeInterval, skip_data: bool, skip_meta: bool, page: PaginationProtocol | None = None
     ) -> DatapointPage:
+        query_page = Pagination(page.limit, page.starting_after) if page else Pagination()
         return self._dataset._service.get_dataset_for_time_interval(
-            str(self._collection.id), time_interval, skip_data, skip_meta, page
+            str(self._collection.id), time_interval, skip_data, skip_meta, query_page
         ).get()
 
     def ingest(

tilebox/datasets/sync/pagination.py

@@ -1,5 +1,5 @@
 import time
-from collections.abc import Callable, Iterator
+from collections.abc import Iterator
 from datetime import datetime, timezone
 from typing import TypeVar
 
@@ -9,41 +9,11 @@ from tilebox.datasets.data import (
     TimeInterval,
 )
 from tilebox.datasets.data.datapoint import DatapointPage, QueryResultPage
-from tilebox.datasets.data.pagination import Pagination
 from tilebox.datasets.progress import ProgressCallback, TimeIntervalProgressBar
 
 ResultPage = TypeVar("ResultPage", bound=DatapointPage | QueryResultPage)
 
 
-def paginated_request(
-    paging_request: Callable[[Pagination], ResultPage],
-    initial_page: Pagination | None = None,
-) -> Iterator[ResultPage]:
-    """Make a paginated request to a gRPC service endpoint.
-
-    The endpoint is expected to return a next_page field, which is used for subsequent requests. Once no such
-    next_page field is returned, the request is completed.
-
-    Args:
-        paging_request: A function that takes a page as input and returns a Datapoints object
-            Often this will be a functools.partial object that wraps a gRPC service endpoint
-            and only leaves the page argument remaining
-        initial_page: The initial page to request
-
-    Yields:
-        Datapoints: The individual pages of the response
-    """
-    if initial_page is None:
-        initial_page = Pagination()
-
-    response = paging_request(initial_page)
-    yield response
-
-    while response.next_page.starting_after is not None:
-        response = paging_request(response.next_page)
-        yield response
-
-
 def with_progressbar(
     paginated_request: Iterator[ResultPage],
     progress_description: str,

{tilebox_datasets-0.35.0.dist-info → tilebox_datasets-0.36.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tilebox-datasets
-Version: 0.35.0
+Version: 0.36.0
 Summary: Access Tilebox datasets from Python
 Project-URL: Homepage, https://tilebox.com
 Project-URL: Documentation, https://docs.tilebox.com/datasets/introduction

{tilebox_datasets-0.35.0.dist-info → tilebox_datasets-0.36.0.dist-info}/RECORD

@@ -6,13 +6,13 @@ tilebox/datasets/progress.py,sha256=NC3YFnK4EqspCH0QPKsAHmkzoAl5KHedPWYvdS37Tp8,
 tilebox/datasets/service.py,sha256=ooF8QIUFaTBxmyMEPoHk2J7lwj5AzvGP_2dTWQkyi3g,10230
 tilebox/datasets/aio/__init__.py,sha256=0x_gddLgDsUCdl8MMZj4MPH1lp4HuOrExMHTjIFmM6s,405
 tilebox/datasets/aio/client.py,sha256=NLA59PvFVRJtAhfYbRNYOvzJ9KnL6Uptji7pBYDNRcc,2126
-tilebox/datasets/aio/dataset.py,sha256=gY6IFVOeyl-MWqK-qHJczuPmOmCg3mu8tfZdf1CY7rY,26944
-tilebox/datasets/aio/pagination.py,sha256=mRzK-zVGyOCc9u-mNefSIu4F9TTOahN6Ikhunddc8gA,7287
+tilebox/datasets/aio/dataset.py,sha256=5WW8UhI3JpDlCtVzvxOnSxF7htEDH2LkQEfDe65XuKk,27189
+tilebox/datasets/aio/pagination.py,sha256=C9MNcxnD6Hycsx0mLYXrMC76qehic_ca0R6Q_VpiCS8,6148
 tilebox/datasets/aio/timeseries.py,sha256=iQqIyh9TPL_gJz18GCxmtFJEwObR9S2rPsUohFYM8wQ,301
 tilebox/datasets/data/__init__.py,sha256=vgHSaWXMuC5NLlMwC_VhwKd4nvpOn1NIL4YzaD58goQ,401
 tilebox/datasets/data/collection.py,sha256=u9O8vjLYLoe888sZq4yPs3NFlGem5QfdE-wZQW4OK2w,2670
 tilebox/datasets/data/data_access.py,sha256=Z0PuUem9I2TOdAiHiaLrZ-pC1UiNhr9dES0PP3K85aE,5622
-tilebox/datasets/data/datapoint.py,sha256=gVfHpHysISJCcUoYkyRDvm9alBRMfKO57c69WM0SCwc,7010
+tilebox/datasets/data/datapoint.py,sha256=h89tQ2Vs08dj7TlF_GSf9Od0ekOHbIOJY7px_rxiNgk,8631
 tilebox/datasets/data/datasets.py,sha256=AXFrTvDgytox3lGoO-fUG1Ob7eYFtig1p2DAQ4bWxts,5246
 tilebox/datasets/data/pagination.py,sha256=ouFO2pWf1BFsDVVY37QDxPN1TbYRSWXfQF1p6sHeZY0,1332
 tilebox/datasets/data/time_interval.py,sha256=CADKc0xxf5PYwF2VsQ2GVi4JapwBQ6-JXrVV8VHv-so,9737
@@ -21,8 +21,8 @@ tilebox/datasets/data/uuid.py,sha256=uc9P1wCowkdsxnJMhcDGmjWT2Py8Tiqh3LSsAMJngdo
 tilebox/datasets/datasetsv1/collections_pb2.py,sha256=8Ax9D85AnfchVjmuTlvE6WATy0DNr_37uDAwh8i3ZB0,3415
 tilebox/datasets/datasetsv1/collections_pb2.pyi,sha256=CcjNu9HSi2zyKcC3GWYPRBFeDTjfkd36SEO-WYIiBQQ,1749
 tilebox/datasets/datasetsv1/collections_pb2_grpc.py,sha256=oFR5lUwWEdxVA33cQIsjve92NUAU7VS9QmsbT5flMoo,7040
-tilebox/datasets/datasetsv1/core_pb2.py,sha256=ww1t-FTGIxyUZE9gJMpD441nXAqEsG9DqxsokFTlWPI,8159
-tilebox/datasets/datasetsv1/core_pb2.pyi,sha256=UiXlInWZ-Q1-7-Wl028M963Mdm3KcAt4OgDKOVKQ2IQ,9062
+tilebox/datasets/datasetsv1/core_pb2.py,sha256=k88K48SIWV4DtDNLGUd2q7qzQqXNbtSE1yc8tL3FbFA,8202
+tilebox/datasets/datasetsv1/core_pb2.pyi,sha256=n6gV3BiHrcCgmRHvrMmjxRyhulcVFdsGJsMcXSSJUJk,9150
 tilebox/datasets/datasetsv1/core_pb2_grpc.py,sha256=xYOs94SXiNYAlFodACnsXW5QovLsHY5tCk3p76RH5Zc,158
 tilebox/datasets/datasetsv1/data_access_pb2.py,sha256=fOgy5ZQb9ojGUbFr6_NFDowYQfegXUidi9ljdJeIyKY,6182
 tilebox/datasets/datasetsv1/data_access_pb2.pyi,sha256=maP2FKtta1nmQRyNLKvTPOlqA-LDEV-x8N6ZgMJQTak,5743
@@ -49,9 +49,9 @@ tilebox/datasets/protobuf_conversion/protobuf_xarray.py,sha256=xTHN69r6GrXtDlOPu
 tilebox/datasets/protobuf_conversion/to_protobuf.py,sha256=Q-RaVsLrRgSzQYI4nHjB3ppliyHG0jR9EB2sfT1riYg,7640
 tilebox/datasets/sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tilebox/datasets/sync/client.py,sha256=d6d2ABi9Lo9oIgpnq_1O0v4JqJrppaP4hCT5TWWXHyE,2101
-tilebox/datasets/sync/dataset.py,sha256=v4B5LLJISc-LVgBETOHRKPd1beOovYuRZQU5uwxsaSg,27242
-tilebox/datasets/sync/pagination.py,sha256=mGR9UkIPA8mUVW5aSVCgVVuwHRVszpUqiA-zTBv67Cg,7156
+tilebox/datasets/sync/dataset.py,sha256=88FhFRP4tUq0cxeaWGouz8v3fbeKLuOdhx6NyWvYzbQ,27399
+tilebox/datasets/sync/pagination.py,sha256=aWE9iBfPd6Gj-wT2Pfg2Z-owRe53fFpuG5HN-pporeI,6062
 tilebox/datasets/sync/timeseries.py,sha256=4nTP8_tmv6V7PXTUNzzlbzlxv0OXo_IqVLtSdJpUOW0,303
-tilebox_datasets-0.35.0.dist-info/METADATA,sha256=QKoxtkzVXKjEjCgMbTWVkN8g5B6esOCki9FPX3ZRHC0,4048
-tilebox_datasets-0.35.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-tilebox_datasets-0.35.0.dist-info/RECORD,,
+tilebox_datasets-0.36.0.dist-info/METADATA,sha256=LSIcvqFmQGwUAURNAZIAkRBJxOKJnz34kMbSTBkn8JM,4048
+tilebox_datasets-0.36.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+tilebox_datasets-0.36.0.dist-info/RECORD,,