tilebox-datasets 0.36.0__py3-none-any.whl → 0.37.0__py3-none-any.whl

tilebox/datasets/aio/dataset.py

@@ -18,14 +18,14 @@ from tilebox.datasets.aio.pagination import (
 )
 from tilebox.datasets.data.collection import CollectionInfo
 from tilebox.datasets.data.data_access import QueryFilters, SpatialFilter, SpatialFilterLike
-from tilebox.datasets.data.datapoint import DatapointInterval, DatapointIntervalLike, DatapointPage, QueryResultPage
+from tilebox.datasets.data.datapoint import DatapointInterval, DatapointIntervalLike, QueryResultPage
 from tilebox.datasets.data.datasets import Dataset
 from tilebox.datasets.data.pagination import Pagination
 from tilebox.datasets.data.time_interval import TimeInterval, TimeIntervalLike
 from tilebox.datasets.data.uuid import as_uuid
 from tilebox.datasets.message_pool import get_message_type
 from tilebox.datasets.progress import ProgressCallback
-from tilebox.datasets.protobuf_conversion.protobuf_xarray import MessageToXarrayConverter, TimeseriesToXarrayConverter
+from tilebox.datasets.protobuf_conversion.protobuf_xarray import MessageToXarrayConverter
 from tilebox.datasets.protobuf_conversion.to_protobuf import (
     DatapointIDs,
     IngestionData,
@@ -145,7 +145,6 @@ class CollectionClient:
         info: CollectionInfo,
     ) -> None:
         self._dataset = dataset
-        self._use_legacy_api = dataset._dataset.is_legacy_type
         self._collection = info.collection
         self._info: CollectionInfo | None = info
 
@@ -208,9 +207,6 @@ class CollectionClient:
         Returns:
             The datapoint as an xarray dataset
         """
-        if self._use_legacy_api:  # remove this once all datasets are fully migrated to the new endpoints
-            return await self._find_legacy(str(datapoint_id), skip_data)
-
         try:
             datapoint = await self._dataset._service.query_by_id(
                 [self._collection.id], as_uuid(datapoint_id), skip_data
@@ -225,21 +221,7 @@ class CollectionClient:
 
         converter = MessageToXarrayConverter(initial_capacity=1)
         converter.convert(data)
-        return converter.finalize("time").isel(time=0)
-
-    async def _find_legacy(self, datapoint_id: str, skip_data: bool = False) -> xr.Dataset:
-        try:
-            datapoint = await self._dataset._service.get_datapoint_by_id(
-                str(self._collection.id), datapoint_id, skip_data
-            )
-        except ArgumentError:
-            raise ValueError(f"Invalid datapoint id: {datapoint_id} is not a valid UUID") from None
-        except NotFoundError:
-            raise NotFoundError(f"No such datapoint {datapoint_id}") from None
-
-        converter = TimeseriesToXarrayConverter(initial_capacity=1)
-        converter.convert(datapoint)
-        return converter.finalize().isel(time=0)
+        return converter.finalize("time", skip_empty_fields=skip_data).isel(time=0)
 
     async def _find_interval(
         self,
@@ -262,11 +244,6 @@ class CollectionClient:
         Returns:
             The datapoints in the given interval as an xarray dataset
         """
-        if self._use_legacy_api:  # remove this once all datasets are fully migrated to the new endpoints
-            return await self._find_interval_legacy(
-                datapoint_id_interval, end_inclusive, skip_data=skip_data, show_progress=show_progress
-            )
-
         filters = QueryFilters(
             temporal_extent=DatapointInterval.parse(datapoint_id_interval, end_inclusive=end_inclusive)
         )
@@ -280,30 +257,7 @@ class CollectionClient:
         if show_progress:
             pages = with_progressbar(pages, f"Fetching {self._dataset.name}")
 
-        return await _convert_to_dataset(pages)
-
-    async def _find_interval_legacy(
-        self,
-        datapoint_id_interval: DatapointIntervalLike,
-        end_inclusive: bool = True,
-        *,
-        skip_data: bool = False,
-        show_progress: bool = False,
-    ) -> xr.Dataset:
-        datapoint_interval = DatapointInterval.parse(datapoint_id_interval, end_inclusive=end_inclusive)
-
-        async def request(page: PaginationProtocol) -> DatapointPage:
-            query_page = Pagination(page.limit, page.starting_after)
-            return await self._dataset._service.get_dataset_for_datapoint_interval(
-                str(self._collection.id), datapoint_interval, skip_data, False, query_page
-            )
-
-        initial_page = Pagination()
-        pages = paginated_request(request, initial_page)
-        if show_progress:
-            pages = with_progressbar(pages, f"Fetching {self._dataset.name}")
-
-        return await _convert_to_dataset_legacy(pages)
+        return await _convert_to_dataset(pages, skip_empty_fields=skip_data)
 
     async def load(
         self,
@@ -336,8 +290,11 @@ class CollectionClient:
         Returns:
             Matching datapoints in the given temporal extent as an xarray dataset
         """
-        if self._use_legacy_api:  # remove this once all datasets are fully migrated to the new endpoints
-            return await self._load_legacy(temporal_extent, skip_data=skip_data, show_progress=show_progress)
+        warn(
+            "collection.load(interval) is deprecated. Please use collection.query(temporal_extent=interval) instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
 
         return await self.query(temporal_extent=temporal_extent, skip_data=skip_data, show_progress=show_progress)
 
@@ -380,14 +337,11 @@ class CollectionClient:
         Returns:
             Matching datapoints in the given temporal and spatial extent as an xarray dataset
         """
-        if self._use_legacy_api:
-            raise ValueError("Querying is not supported for this dataset. Please use load() instead.")
-
         if temporal_extent is None:
             raise ValueError("A temporal_extent for your query must be specified")
 
         pages = self._iter_pages(temporal_extent, spatial_extent, skip_data, show_progress=show_progress)
-        return await _convert_to_dataset(pages)
+        return await _convert_to_dataset(pages, skip_empty_fields=skip_data)
 
     async def _iter_pages(
         self,
@@ -400,7 +354,7 @@ class CollectionClient:
         time_interval = TimeInterval.parse(temporal_extent)
         filters = QueryFilters(time_interval, SpatialFilter.parse(spatial_extent) if spatial_extent else None)
 
-        request = partial(self._load_page, filters, skip_data)
+        request = partial(self._query_page, filters, skip_data)
 
         initial_page = Pagination(limit=page_size)
         pages = paginated_request(request, initial_page)
@@ -414,60 +368,12 @@ class CollectionClient:
         async for page in pages:
             yield page
 
-    async def _load_page(
+    async def _query_page(
         self, filters: QueryFilters, skip_data: bool, page: PaginationProtocol | None = None
     ) -> QueryResultPage:
         query_page = Pagination(page.limit, page.starting_after) if page else Pagination()
         return await self._dataset._service.query([self._collection.id], filters, skip_data, query_page)
 
-    async def _load_legacy(
-        self,
-        time_or_interval: TimeIntervalLike,
-        *,
-        skip_data: bool = False,
-        show_progress: bool | ProgressCallback = False,
-    ) -> xr.Dataset:
-        pages = self._iter_pages_legacy(time_or_interval, skip_data, show_progress=show_progress)
-        return await _convert_to_dataset_legacy(pages)
-
-    async def _iter_pages_legacy(
-        self,
-        time_or_interval: TimeIntervalLike,
-        skip_data: bool = False,
-        skip_meta: bool = False,
-        show_progress: bool | ProgressCallback = False,
-        page_size: int | None = None,
-    ) -> AsyncIterator[DatapointPage]:
-        time_interval = TimeInterval.parse(time_or_interval)
-
-        request = partial(self._load_page_legacy, time_interval, skip_data, skip_meta)
-
-        initial_page = Pagination(limit=page_size)
-        pages = paginated_request(request, initial_page)
-
-        if callable(show_progress):
-            if skip_meta:
-                raise ValueError("Progress callback requires datapoint metadata, but skip_meta is True")
-            else:
-                pages = with_time_progress_callback(pages, time_interval, show_progress)
-        elif show_progress:
-            message = f"Fetching {self._dataset.name}"
-            if skip_meta:  # without metadata we can't estimate progress based on event time (since it is not returned)
-                pages = with_progressbar(pages, message)
-            else:
-                pages = with_time_progressbar(pages, time_interval, message)
-
-        async for page in pages:
-            yield page
-
-    async def _load_page_legacy(
-        self, time_interval: TimeInterval, skip_data: bool, skip_meta: bool, page: PaginationProtocol | None = None
-    ) -> DatapointPage:
-        query_page = Pagination(page.limit, page.starting_after) if page else Pagination()
-        return await self._dataset._service.get_dataset_for_time_interval(
-            str(self._collection.id), time_interval, skip_data, skip_meta, query_page
-        )
-
     async def ingest(
         self,
         data: IngestionData,
@@ -493,9 +399,6 @@ class CollectionClient:
         Returns:
             List of datapoint ids that were ingested.
         """
-        if self._use_legacy_api:  # remove this once all datasets are fully migrated to the new endpoints
-            raise ValueError("Ingestion is not supported for this dataset. Please create a new dataset.")
-
         message_type = get_message_type(self._dataset._dataset.type.type_url)
         messages = marshal_messages(
             to_messages(data, message_type, required_fields=["time"], ignore_fields=["id", "ingestion_time"])
@@ -564,7 +467,7 @@ class CollectionClient:
         return num_deleted
 
 
-async def _convert_to_dataset(pages: AsyncIterator[QueryResultPage]) -> xr.Dataset:
+async def _convert_to_dataset(pages: AsyncIterator[QueryResultPage], skip_empty_fields: bool = False) -> xr.Dataset:
     """
     Convert an async iterator of QueryResultPages into a single xarray Dataset
 
@@ -572,6 +475,7 @@ async def _convert_to_dataset(pages: AsyncIterator[QueryResultPage]) -> xr.Datas
 
     Args:
         pages: Async iterator of QueryResultPages to convert
+        skip_empty_fields: Whether to omit fields from the output dataset in case no values are set
 
     Returns:
         The datapoints from the individual pages converted and combined into a single xarray dataset
@@ -588,26 +492,4 @@ async def _convert_to_dataset(pages: AsyncIterator[QueryResultPage]) -> xr.Datas
     # this would also account for the case where the server sends pages faster than we are converting
     # them to xarray
     await async_producer_consumer(pages, convert_page)
-    return converter.finalize("time")
-
-
-async def _convert_to_dataset_legacy(pages: AsyncIterator[DatapointPage]) -> xr.Dataset:
-    """
-    Convert an async iterator of DatasetIntervals (pages) into a single xarray Dataset
-
-    Parses each incoming page while in parallel already requesting and waiting for the next page from the server.
-
-    Args:
-        pages: Async iterator of DatasetIntervals (pages) to convert
-
-    Returns:
-        The datapoints from the individual pages converted and combined into a single xarray dataset
-    """
-
-    converter = TimeseriesToXarrayConverter()
-    # lets parse the incoming pages already while we wait for the next page from the server
-    # we solve this using a classic producer/consumer with a queue of pages for communication
-    # this would also account for the case where the server sends pages faster than we are converting
-    # them to xarray
-    await async_producer_consumer(pages, lambda page: converter.convert_all(page))
-    return converter.finalize()
+    return converter.finalize("time", skip_empty_fields=skip_empty_fields)

tilebox/datasets/aio/pagination.py

@@ -8,10 +8,10 @@ from tqdm.auto import tqdm
 from tilebox.datasets.data import (
     TimeInterval,
 )
-from tilebox.datasets.data.datapoint import DatapointPage, QueryResultPage
+from tilebox.datasets.data.datapoint import QueryResultPage
 from tilebox.datasets.progress import ProgressCallback, TimeIntervalProgressBar
 
-ResultPage = TypeVar("ResultPage", bound=DatapointPage | QueryResultPage)
+ResultPage = TypeVar("ResultPage", bound=QueryResultPage)
 
 
 async def with_progressbar(
@@ -81,14 +81,14 @@ async def with_time_progressbar(
 
     # we have more pages, so lets set up a progress bar
     actual_interval = TimeInterval(
-        start=max(interval.start, first_page.min_time()),
+        start=max(interval.start, first_page.min_time),
         end=min(interval.end, datetime.now(tz=timezone.utc)),
     )
 
     with TimeIntervalProgressBar(
         interval=actual_interval,
         description=progress_description,
-        initial_time=first_page.max_time(),
+        initial_time=first_page.max_time,
         actual_start_time=actual_start_time,
     ) as progress_bar:
         # provide download information for the first page
@@ -99,7 +99,7 @@ async def with_time_progressbar(
         async for page in paginated_request:  # now loop over the remaining pages
             now = time.time()
             if page.n_datapoints > 0:
-                progress_bar.set_progress(page.max_time())
+                progress_bar.set_progress(page.max_time)
                 progress_bar.set_download_info(page.n_datapoints, page.byte_size, now - before)
             yield page
             before = now
@@ -134,17 +134,17 @@ async def with_time_progress_callback(
 
     # we have more pages, so lets set up a progress bar
     actual_interval = TimeInterval(
-        start=max(interval.start, first_page.min_time()),
+        start=max(interval.start, first_page.min_time),
         end=min(interval.end, datetime.now(tz=timezone.utc)),
     )
 
     total = (actual_interval.end - actual_interval.start).total_seconds()
     if first_page.n_datapoints > 0:
-        current = (first_page.max_time() - actual_interval.start).total_seconds()
+        current = (first_page.max_time - actual_interval.start).total_seconds()
         progress_callback(current / total)
     async for page in paginated_request:  # now loop over the remaining pages
         if page.n_datapoints > 0:
-            current = (page.max_time() - actual_interval.start).total_seconds()
+            current = (page.max_time - actual_interval.start).total_seconds()
             progress_callback(current / total)
         yield page
 

tilebox/datasets/data/datapoint.py

@@ -110,83 +110,6 @@ class RepeatedAny:
         return core_pb2.RepeatedAny(type_url=self.type_url, value=self.value)
 
 
-@dataclass(frozen=True)
-class Datapoint:
-    """Datapoint contains the metadata for a single data point."""
-
-    meta: core_pb2.DatapointMetadata  # we keep this as protobuf message to easily convert to/from xarray
-    data: AnyMessage
-
-    @classmethod
-    def from_message(
-        cls, datapoint: core_pb2.Datapoint
-    ) -> "Datapoint":  # lets use typing.Self once we require python >= 3.11
-        """Convert a Datapoint protobuf message to a Datapoint object."""
-        return cls(
-            meta=datapoint.meta,
-            data=AnyMessage.from_message(datapoint.data),
-        )
-
-    def to_message(self) -> core_pb2.Datapoint:
-        return core_pb2.Datapoint(
-            meta=self.meta,
-            data=self.data.to_message(),
-        )
-
-
-@dataclass(frozen=True)
-class Datapoints:
-    meta: list[core_pb2.DatapointMetadata]  # we keep this as protobuf message to easily convert to/from xarray
-    data: RepeatedAny
-
-    @classmethod
-    def from_message(cls, datapoints: core_pb2.Datapoints) -> "Datapoints":
-        return cls(meta=list(datapoints.meta), data=RepeatedAny.from_message(datapoints.data))
-
-    def to_message(self) -> core_pb2.Datapoints:
-        return core_pb2.Datapoints(meta=self.meta, data=self.data.to_message())
-
-
-@dataclass(frozen=True)
-class DatapointPage:
-    meta: list[core_pb2.DatapointMetadata]  # we keep this as protobuf message to easily convert to/from xarray
-    data: RepeatedAny
-    next_page: Pagination
-    byte_size: int = field(compare=False)
-
-    @classmethod
-    def from_message(cls, datapoints: core_pb2.DatapointPage) -> "DatapointPage":
-        return cls(
-            meta=list(datapoints.meta),
-            data=RepeatedAny.from_message(datapoints.data),
-            next_page=Pagination.from_legacy_message(datapoints.next_page),
-            byte_size=datapoints.ByteSize(),  # useful for progress bars
-        )
-
-    def to_message(self) -> core_pb2.DatapointPage:
-        return core_pb2.DatapointPage(
-            meta=self.meta,
-            data=self.data.to_message(),
-            next_page=self.next_page.to_legacy_message() if self.next_page else None,
-        )
-
-    @property
-    def n_datapoints(self) -> int:
-        return len(self.data.value)
-
-    def min_id(self) -> UUID:
-        return UUID(self.meta[0].id)
-
-    def max_id(self) -> UUID:
-        return UUID(self.meta[-1].id)
-
-    def min_time(self) -> datetime:
-        return timestamp_to_datetime(self.meta[0].event_time)
-
-    def max_time(self) -> datetime:
-        return timestamp_to_datetime(self.meta[-1].event_time)
-
-
 @dataclass(frozen=True)
 class QueryResultPage:
     data: RepeatedAny
@@ -211,15 +134,19 @@ class QueryResultPage:
     def n_datapoints(self) -> int:
         return len(self.data.value)
 
+    @property
     def min_id(self) -> UUID:
         return uuid_message_to_uuid(self._parse_message(0).id)
 
+    @property
     def max_id(self) -> UUID:
         return uuid_message_to_uuid(self._parse_message(-1).id)
 
+    @property
     def min_time(self) -> datetime:
         return timestamp_to_datetime(self._parse_message(0).time)
 
+    @property
     def max_time(self) -> datetime:
         return timestamp_to_datetime(self._parse_message(-1).time)
 

tilebox/datasets/data/datasets.py

@@ -80,29 +80,6 @@ class Dataset:
             description=self.description,
         )
 
-    @property
-    def is_legacy_type(self) -> bool:
-        """
-        Check if the dataset type is a legacy type (without the meta fields in the proto message).
-
-        For those types, we use the legacy query API, until all datasets are fully migrated to the new endpoints.
-        """
-        # helper function to check if the type is a legacy type (without the meta fields in the proto message)
-        files = self.type.descriptor_set.file
-        if not files or len(files) != 1:
-            return False
-        file = files[0]
-        messages = file.message_type
-        if not messages or len(messages) != 1:
-            return False
-        message = messages[0]
-        fields = message.field
-        if not fields or len(fields) < 3:
-            return True  # new style types have at least three fields (time, id, ingestion_time)
-
-        has_new_type_fields = fields[0].name == "time" and fields[1].name == "id" and fields[2].name == "ingestion_time"
-        return not has_new_type_fields
-
 
 @dataclass
 class DatasetGroup:

tilebox/datasets/data/pagination.py

@@ -19,15 +19,3 @@ class Pagination:
 
     def to_message(self) -> core_pb2.Pagination:
         return core_pb2.Pagination(limit=self.limit, starting_after=uuid_to_uuid_message(self.starting_after))
-
-    @classmethod
-    def from_legacy_message(cls, page: core_pb2.LegacyPagination | None) -> "Pagination":
-        if page is None:
-            return cls()
-        # convert falsish values (0 or empty string) to None
-        return cls(limit=page.limit or None, starting_after=UUID(page.starting_after) if page.starting_after else None)
-
-    def to_legacy_message(self) -> core_pb2.LegacyPagination:
-        return core_pb2.LegacyPagination(
-            limit=self.limit, starting_after=str(self.starting_after) if self.starting_after else None
-        )

tilebox/datasets/data/uuid.py

@@ -1,18 +1,18 @@
 # allow the uuid module name which shadows the builtin:
 from uuid import UUID
 
-from tilebox.datasets.datasetsv1 import core_pb2
+from tilebox.datasets.datasetsv1 import core_pb2, well_known_types_pb2
 
 _NIL_UUID = UUID(int=0)
 
 
-def uuid_message_to_uuid(uuid_message: core_pb2.ID) -> UUID:
+def uuid_message_to_uuid(uuid_message: core_pb2.ID | well_known_types_pb2.UUID) -> UUID:
     if uuid_message.uuid == b"":
         return _NIL_UUID
     return UUID(bytes=uuid_message.uuid)
 
 
-def uuid_message_to_optional_uuid(uuid_message: core_pb2.ID | None) -> UUID | None:
+def uuid_message_to_optional_uuid(uuid_message: core_pb2.ID | well_known_types_pb2.UUID | None) -> UUID | None:
     if uuid_message is None:
         return None
     if uuid_message.uuid == b"":

tilebox/datasets/datasetsv1/core_pb2.py

@@ -22,11 +22,11 @@ _runtime_version.ValidateProtobufRuntimeVersion(
 _sym_db = _symbol_database.Default()
 
 
-from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
 from tilebox.datasets.datasetsv1 import dataset_type_pb2 as datasets_dot_v1_dot_dataset__type__pb2
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16\x64\x61tasets/v1/core.proto\x12\x0b\x64\x61tasets.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1e\x64\x61tasets/v1/dataset_type.proto\"\x18\n\x02ID\x12\x12\n\x04uuid\x18\x01 \x01(\x0cR\x04uuid\"\xce\x01\n\x0cTimeInterval\x12\x39\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12\x35\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x07\x65ndTime\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"\xb5\x01\n\x11\x44\x61tapointInterval\x12*\n\x08start_id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x07startId\x12&\n\x06\x65nd_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x05\x65ndId\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"v\n\x10LegacyPagination\x12\x19\n\x05limit\x18\x01 \x01(\x03H\x00R\x05limit\x88\x01\x01\x12*\n\x0estarting_after\x18\x02 \x01(\tH\x01R\rstartingAfter\x88\x01\x01\x42\x08\n\x06_limitB\x11\n\x0f_starting_after\"\x81\x01\n\nPagination\x12\x19\n\x05limit\x18\x01 \x01(\x03H\x00R\x05limit\x88\x01\x01\x12;\n\x0estarting_after\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDH\x01R\rstartingAfter\x88\x01\x01\x42\x08\n\x06_limitB\x11\n\x0f_starting_after\"6\n\x03\x41ny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05value\">\n\x0bRepeatedAny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x03(\x0cR\x05value\"\xad\x01\n\x11\x44\x61tapointMetadata\x12\x39\n\nevent_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\teventTime\x12\x41\n\x0eingestion_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\ringestionTime\x12\x13\n\x02id\x18\x03 \x01(\tH\x00R\x02id\x88\x01\x01\x42\x05\n\x03_id\"n\n\nDatapoints\x12\x32\n\x04meta\x18\x01 \x03(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x18.datasets.v1.RepeatedAnyR\x04\x64\x61ta\"\xc0\x01\n\rDatapointPage\x12\x32\n\x04meta\x18\x01 \x03(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x18.datasets.v1.RepeatedAnyR\x04\x64\x61ta\x12?\n\tnext_page\x18\x03 \x01(\x0b\x32\x1d.datasets.v1.LegacyPaginationH\x00R\x08nextPage\x88\x01\x01\x42\x0c\n\n_next_page\"e\n\tDatapoint\x12\x32\n\x04meta\x18\x01 \x01(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12$\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x10.datasets.v1.AnyR\x04\x64\x61ta\"^\n\nCollection\x12\x1b\n\tlegacy_id\x18\x01 \x01(\tR\x08legacyId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1f\n\x02id\x18\x03 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\"\xc3\x01\n\x0e\x43ollectionInfo\x12\x37\n\ncollection\x18\x01 \x01(\x0b\x32\x17.datasets.v1.CollectionR\ncollection\x12\x42\n\x0c\x61vailability\x18\x02 \x01(\x0b\x32\x19.datasets.v1.TimeIntervalH\x00R\x0c\x61vailability\x88\x01\x01\x12\x19\n\x05\x63ount\x18\x03 \x01(\x04H\x01R\x05\x63ount\x88\x01\x01\x42\x0f\n\r_availabilityB\x08\n\x06_count\"B\n\x0f\x43ollectionInfos\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x1b.datasets.v1.CollectionInfoR\x04\x64\x61ta\"\x96\x03\n\x07\x44\x61taset\x12\x1f\n\x02id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\x12*\n\x08group_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x07groupId\x12.\n\x04type\x18\x03 \x01(\x0b\x32\x1a.datasets.v1.AnnotatedTypeR\x04type\x12\x1b\n\tcode_name\x18\x04 \x01(\tR\x08\x63odeName\x12\x12\n\x04name\x18\x05 \x01(\tR\x04name\x12\x18\n\x07summary\x18\x06 \x01(\tR\x07summary\x12\x12\n\x04icon\x18\x07 \x01(\tR\x04icon\x12 \n\x0b\x64\x65scription\x18\x08 \x01(\tR\x0b\x64\x65scription\x12@\n\x0bpermissions\x18\n \x03(\x0e\x32\x1e.datasets.v1.DatasetPermissionR\x0bpermissions\x12\x37\n\nvisibility\x18\x0b \x01(\x0e\x32\x17.datasets.v1.VisibilityR\nvisibility\x12\x12\n\x04slug\x18\x0c \x01(\tR\x04slug\"\xa2\x01\n\x0c\x44\x61tasetGroup\x12\x1f\n\x02id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\x12,\n\tparent_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x08parentId\x12\x1b\n\tcode_name\x18\x03 \x01(\tR\x08\x63odeName\x12\x12\n\x04name\x18\x04 \x01(\tR\x04name\x12\x12\n\x04icon\x18\x05 \x01(\tR\x04icon*\x9b\x01\n\x11\x44\x61tasetPermission\x12\"\n\x1e\x44\x41TASET_PERMISSION_UNSPECIFIED\x10\x00\x12\"\n\x1e\x44\x41TASET_PERMISSION_ACCESS_DATA\x10\x01\x12!\n\x1d\x44\x41TASET_PERMISSION_WRITE_DATA\x10\x02\x12\x1b\n\x17\x44\x41TASET_PERMISSION_EDIT\x10\x03*v\n\nVisibility\x12\x1a\n\x16VISIBILITY_UNSPECIFIED\x10\x00\x12\x16\n\x12VISIBILITY_PRIVATE\x10\x01\x12\x1d\n\x19VISIBILITY_SHARED_WITH_ME\x10\x02\x12\x15\n\x11VISIBILITY_PUBLIC\x10\x03\x42\xab\x01\n\x0f\x63om.datasets.v1B\tCoreProtoP\x01Z@github.com/tilebox/tilebox-go/protogen/go/datasets/v1;datasetsv1\xa2\x02\x03\x44XX\xaa\x02\x0b\x44\x61tasets.V1\xca\x02\x0b\x44\x61tasets\\V1\xe2\x02\x17\x44\x61tasets\\V1\\GPBMetadata\xea\x02\x0c\x44\x61tasets::V1b\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16\x64\x61tasets/v1/core.proto\x12\x0b\x64\x61tasets.v1\x1a\x1e\x64\x61tasets/v1/dataset_type.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x18\n\x02ID\x12\x12\n\x04uuid\x18\x01 \x01(\x0cR\x04uuid\"\xce\x01\n\x0cTimeInterval\x12\x39\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12\x35\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x07\x65ndTime\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"\xb5\x01\n\x11\x44\x61tapointInterval\x12*\n\x08start_id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x07startId\x12&\n\x06\x65nd_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x05\x65ndId\x12\'\n\x0fstart_exclusive\x18\x03 \x01(\x08R\x0estartExclusive\x12#\n\rend_inclusive\x18\x04 \x01(\x08R\x0c\x65ndInclusive\"v\n\x10LegacyPagination\x12\x19\n\x05limit\x18\x01 \x01(\x03H\x00R\x05limit\x88\x01\x01\x12*\n\x0estarting_after\x18\x02 \x01(\tH\x01R\rstartingAfter\x88\x01\x01\x42\x08\n\x06_limitB\x11\n\x0f_starting_after\"\x81\x01\n\nPagination\x12\x19\n\x05limit\x18\x01 \x01(\x03H\x00R\x05limit\x88\x01\x01\x12;\n\x0estarting_after\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDH\x01R\rstartingAfter\x88\x01\x01\x42\x08\n\x06_limitB\x11\n\x0f_starting_after\"6\n\x03\x41ny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05value\">\n\x0bRepeatedAny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x03(\x0cR\x05value\"\xad\x01\n\x11\x44\x61tapointMetadata\x12\x39\n\nevent_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\teventTime\x12\x41\n\x0eingestion_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\ringestionTime\x12\x13\n\x02id\x18\x03 \x01(\tH\x00R\x02id\x88\x01\x01\x42\x05\n\x03_id\"n\n\nDatapoints\x12\x32\n\x04meta\x18\x01 \x03(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x18.datasets.v1.RepeatedAnyR\x04\x64\x61ta\"\xc0\x01\n\rDatapointPage\x12\x32\n\x04meta\x18\x01 \x03(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12,\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x18.datasets.v1.RepeatedAnyR\x04\x64\x61ta\x12?\n\tnext_page\x18\x03 \x01(\x0b\x32\x1d.datasets.v1.LegacyPaginationH\x00R\x08nextPage\x88\x01\x01\x42\x0c\n\n_next_page\"e\n\tDatapoint\x12\x32\n\x04meta\x18\x01 \x01(\x0b\x32\x1e.datasets.v1.DatapointMetadataR\x04meta\x12$\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32\x10.datasets.v1.AnyR\x04\x64\x61ta\"^\n\nCollection\x12\x1b\n\tlegacy_id\x18\x01 \x01(\tR\x08legacyId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1f\n\x02id\x18\x03 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\"\xc3\x01\n\x0e\x43ollectionInfo\x12\x37\n\ncollection\x18\x01 \x01(\x0b\x32\x17.datasets.v1.CollectionR\ncollection\x12\x42\n\x0c\x61vailability\x18\x02 \x01(\x0b\x32\x19.datasets.v1.TimeIntervalH\x00R\x0c\x61vailability\x88\x01\x01\x12\x19\n\x05\x63ount\x18\x03 \x01(\x04H\x01R\x05\x63ount\x88\x01\x01\x42\x0f\n\r_availabilityB\x08\n\x06_count\"B\n\x0f\x43ollectionInfos\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x1b.datasets.v1.CollectionInfoR\x04\x64\x61ta\"\x96\x03\n\x07\x44\x61taset\x12\x1f\n\x02id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\x12*\n\x08group_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x07groupId\x12.\n\x04type\x18\x03 \x01(\x0b\x32\x1a.datasets.v1.AnnotatedTypeR\x04type\x12\x1b\n\tcode_name\x18\x04 \x01(\tR\x08\x63odeName\x12\x12\n\x04name\x18\x05 \x01(\tR\x04name\x12\x18\n\x07summary\x18\x06 \x01(\tR\x07summary\x12\x12\n\x04icon\x18\x07 \x01(\tR\x04icon\x12 \n\x0b\x64\x65scription\x18\x08 \x01(\tR\x0b\x64\x65scription\x12@\n\x0bpermissions\x18\n \x03(\x0e\x32\x1e.datasets.v1.DatasetPermissionR\x0bpermissions\x12\x37\n\nvisibility\x18\x0b \x01(\x0e\x32\x17.datasets.v1.VisibilityR\nvisibility\x12\x12\n\x04slug\x18\x0c \x01(\tR\x04slug\"\xa2\x01\n\x0c\x44\x61tasetGroup\x12\x1f\n\x02id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x02id\x12,\n\tparent_id\x18\x02 \x01(\x0b\x32\x0f.datasets.v1.IDR\x08parentId\x12\x1b\n\tcode_name\x18\x03 \x01(\tR\x08\x63odeName\x12\x12\n\x04name\x18\x04 \x01(\tR\x04name\x12\x12\n\x04icon\x18\x05 \x01(\tR\x04icon*\x9b\x01\n\x11\x44\x61tasetPermission\x12\"\n\x1e\x44\x41TASET_PERMISSION_UNSPECIFIED\x10\x00\x12\"\n\x1e\x44\x41TASET_PERMISSION_ACCESS_DATA\x10\x01\x12!\n\x1d\x44\x41TASET_PERMISSION_WRITE_DATA\x10\x02\x12\x1b\n\x17\x44\x41TASET_PERMISSION_EDIT\x10\x03*v\n\nVisibility\x12\x1a\n\x16VISIBILITY_UNSPECIFIED\x10\x00\x12\x16\n\x12VISIBILITY_PRIVATE\x10\x01\x12\x1d\n\x19VISIBILITY_SHARED_WITH_ME\x10\x02\x12\x15\n\x11VISIBILITY_PUBLIC\x10\x03\x42\xab\x01\n\x0f\x63om.datasets.v1B\tCoreProtoP\x01Z@github.com/tilebox/tilebox-go/protogen/go/datasets/v1;datasetsv1\xa2\x02\x03\x44XX\xaa\x02\x0b\x44\x61tasets.V1\xca\x02\x0b\x44\x61tasets\\V1\xe2\x02\x17\x44\x61tasets\\V1\\GPBMetadata\xea\x02\x0c\x44\x61tasets::V1b\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)

tilebox/datasets/datasetsv1/core_pb2.pyi

@@ -1,5 +1,5 @@
-from google.protobuf import timestamp_pb2 as _timestamp_pb2
 from tilebox.datasets.datasetsv1 import dataset_type_pb2 as _dataset_type_pb2
+from google.protobuf import timestamp_pb2 as _timestamp_pb2
 from google.protobuf.internal import containers as _containers
 from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
 from google.protobuf import descriptor as _descriptor

tilebox/datasets/datasetsv1/data_access_pb2_grpc.py

@@ -43,14 +43,14 @@ class DataAccessServiceServicer(object):
     """
 
     def GetDatasetForInterval(self, request, context):
-        """GetDatasetForInterval returns a list of data points for a given time interval and collection.
+        """legacy endpoint, kept around for backwards compatibility with older python clients for now
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
     def GetDatapointByID(self, request, context):
-        """GetDatapointByID returns a single datapoint by its ID.
+        """legacy endpoint, kept around for backwards compatibility with older python clients for now
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')

tilebox/datasets/datasetsv1/data_ingestion_pb2.py

@@ -25,7 +25,7 @@ _sym_db = _symbol_database.Default()
 from tilebox.datasets.datasetsv1 import core_pb2 as datasets_dot_v1_dot_core__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n datasets/v1/data_ingestion.proto\x12\x0b\x64\x61tasets.v1\x1a\x16\x64\x61tasets/v1/core.proto\"\xaf\x01\n\x17IngestDatapointsRequest\x12\x34\n\rcollection_id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x0c\x63ollectionId\x12\x37\n\ndatapoints\x18\x02 \x01(\x0b\x32\x17.datasets.v1.DatapointsR\ndatapoints\x12%\n\x0e\x61llow_existing\x18\x03 \x01(\x08R\rallowExisting\"\x84\x01\n\rIngestRequest\x12\x34\n\rcollection_id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x0c\x63ollectionId\x12\x16\n\x06values\x18\x02 \x03(\x0cR\x06values\x12%\n\x0e\x61llow_existing\x18\x03 \x01(\x08R\rallowExisting\"\x8a\x01\n\x0eIngestResponse\x12\x1f\n\x0bnum_created\x18\x01 \x01(\x03R\nnumCreated\x12!\n\x0cnum_existing\x18\x02 \x01(\x03R\x0bnumExisting\x12\x34\n\rdatapoint_ids\x18\x03 \x03(\x0b\x32\x0f.datasets.v1.IDR\x0c\x64\x61tapointIds\"{\n\rDeleteRequest\x12\x34\n\rcollection_id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x0c\x63ollectionId\x12\x34\n\rdatapoint_ids\x18\x02 \x03(\x0b\x32\x0f.datasets.v1.IDR\x0c\x64\x61tapointIds\"1\n\x0e\x44\x65leteResponse\x12\x1f\n\x0bnum_deleted\x18\x01 \x01(\x03R\nnumDeleted2\xf9\x01\n\x14\x44\x61taIngestionService\x12W\n\x10IngestDatapoints\x12$.datasets.v1.IngestDatapointsRequest\x1a\x1b.datasets.v1.IngestResponse\"\x00\x12\x43\n\x06Ingest\x12\x1a.datasets.v1.IngestRequest\x1a\x1b.datasets.v1.IngestResponse\"\x00\x12\x43\n\x06\x44\x65lete\x12\x1a.datasets.v1.DeleteRequest\x1a\x1b.datasets.v1.DeleteResponse\"\x00\x42\xb4\x01\n\x0f\x63om.datasets.v1B\x12\x44\x61taIngestionProtoP\x01Z@github.com/tilebox/tilebox-go/protogen/go/datasets/v1;datasetsv1\xa2\x02\x03\x44XX\xaa\x02\x0b\x44\x61tasets.V1\xca\x02\x0b\x44\x61tasets\\V1\xe2\x02\x17\x44\x61tasets\\V1\\GPBMetadata\xea\x02\x0c\x44\x61tasets::V1b\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n datasets/v1/data_ingestion.proto\x12\x0b\x64\x61tasets.v1\x1a\x16\x64\x61tasets/v1/core.proto\"\xaf\x01\n\x17IngestDatapointsRequest\x12\x34\n\rcollection_id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x0c\x63ollectionId\x12\x37\n\ndatapoints\x18\x02 \x01(\x0b\x32\x17.datasets.v1.DatapointsR\ndatapoints\x12%\n\x0e\x61llow_existing\x18\x03 \x01(\x08R\rallowExisting\"\x84\x01\n\rIngestRequest\x12\x34\n\rcollection_id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x0c\x63ollectionId\x12\x16\n\x06values\x18\x02 \x03(\x0cR\x06values\x12%\n\x0e\x61llow_existing\x18\x03 \x01(\x08R\rallowExisting\"\x8a\x01\n\x0eIngestResponse\x12\x1f\n\x0bnum_created\x18\x01 \x01(\x03R\nnumCreated\x12!\n\x0cnum_existing\x18\x02 \x01(\x03R\x0bnumExisting\x12\x34\n\rdatapoint_ids\x18\x03 \x03(\x0b\x32\x0f.datasets.v1.IDR\x0c\x64\x61tapointIds\"{\n\rDeleteRequest\x12\x34\n\rcollection_id\x18\x01 \x01(\x0b\x32\x0f.datasets.v1.IDR\x0c\x63ollectionId\x12\x34\n\rdatapoint_ids\x18\x02 \x03(\x0b\x32\x0f.datasets.v1.IDR\x0c\x64\x61tapointIds\"1\n\x0e\x44\x65leteResponse\x12\x1f\n\x0bnum_deleted\x18\x01 \x01(\x03R\nnumDeleted2\xa0\x01\n\x14\x44\x61taIngestionService\x12\x43\n\x06Ingest\x12\x1a.datasets.v1.IngestRequest\x1a\x1b.datasets.v1.IngestResponse\"\x00\x12\x43\n\x06\x44\x65lete\x12\x1a.datasets.v1.DeleteRequest\x1a\x1b.datasets.v1.DeleteResponse\"\x00\x42\xb4\x01\n\x0f\x63om.datasets.v1B\x12\x44\x61taIngestionProtoP\x01Z@github.com/tilebox/tilebox-go/protogen/go/datasets/v1;datasetsv1\xa2\x02\x03\x44XX\xaa\x02\x0b\x44\x61tasets.V1\xca\x02\x0b\x44\x61tasets\\V1\xe2\x02\x17\x44\x61tasets\\V1\\GPBMetadata\xea\x02\x0c\x44\x61tasets::V1b\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -44,5 +44,5 @@ if not _descriptor._USE_C_DESCRIPTORS:
   _globals['_DELETERESPONSE']._serialized_start=652
   _globals['_DELETERESPONSE']._serialized_end=701
   _globals['_DATAINGESTIONSERVICE']._serialized_start=704
-  _globals['_DATAINGESTIONSERVICE']._serialized_end=953
+  _globals['_DATAINGESTIONSERVICE']._serialized_end=864
 # @@protoc_insertion_point(module_scope)

tilebox/datasets/datasetsv1/data_ingestion_pb2_grpc.py

@@ -15,11 +15,6 @@ class DataIngestionServiceStub(object):
         Args:
             channel: A grpc.Channel.
         """
-        self.IngestDatapoints = channel.unary_unary(
-                '/datasets.v1.DataIngestionService/IngestDatapoints',
-                request_serializer=datasets_dot_v1_dot_data__ingestion__pb2.IngestDatapointsRequest.SerializeToString,
-                response_deserializer=datasets_dot_v1_dot_data__ingestion__pb2.IngestResponse.FromString,
-                _registered_method=True)
         self.Ingest = channel.unary_unary(
                 '/datasets.v1.DataIngestionService/Ingest',
                 request_serializer=datasets_dot_v1_dot_data__ingestion__pb2.IngestRequest.SerializeToString,
@@ -36,13 +31,6 @@ class DataIngestionServiceServicer(object):
     """DataIngestionService provides data ingestion and deletion capabilities for Tilebox datasets.
     """
 
-    def IngestDatapoints(self, request, context):
-        """legacy ingest endpoint, that separates datapoints into meta and data. Will be removed in the future.
-        """
-        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-        context.set_details('Method not implemented!')
-        raise NotImplementedError('Method not implemented!')
-
     def Ingest(self, request, context):
         """Missing associated documentation comment in .proto file."""
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
@@ -58,11 +46,6 @@ class DataIngestionServiceServicer(object):
 
 def add_DataIngestionServiceServicer_to_server(servicer, server):
     rpc_method_handlers = {
-            'IngestDatapoints': grpc.unary_unary_rpc_method_handler(
-                    servicer.IngestDatapoints,
-                    request_deserializer=datasets_dot_v1_dot_data__ingestion__pb2.IngestDatapointsRequest.FromString,
-                    response_serializer=datasets_dot_v1_dot_data__ingestion__pb2.IngestResponse.SerializeToString,
-            ),
             'Ingest': grpc.unary_unary_rpc_method_handler(
                     servicer.Ingest,
                     request_deserializer=datasets_dot_v1_dot_data__ingestion__pb2.IngestRequest.FromString,
@@ -85,33 +68,6 @@ class DataIngestionService(object):
     """DataIngestionService provides data ingestion and deletion capabilities for Tilebox datasets.
     """
 
-    @staticmethod
-    def IngestDatapoints(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
-        return grpc.experimental.unary_unary(
-            request,
-            target,
-            '/datasets.v1.DataIngestionService/IngestDatapoints',
-            datasets_dot_v1_dot_data__ingestion__pb2.IngestDatapointsRequest.SerializeToString,
-            datasets_dot_v1_dot_data__ingestion__pb2.IngestResponse.FromString,
-            options,
-            channel_credentials,
-            insecure,
-            call_credentials,
-            compression,
-            wait_for_ready,
-            timeout,
-            metadata,
-            _registered_method=True)
-
     @staticmethod
     def Ingest(request,
             target,