tilebox-datasets 0.45.0__py3-none-any.whl → 0.47.0__py3-none-any.whl

tilebox/datasets/aio/client.py

@@ -2,9 +2,11 @@ from uuid import UUID
 
 from _tilebox.grpc.aio.channel import open_channel
 from _tilebox.grpc.aio.error import with_pythonic_errors
+from _tilebox.grpc.error import NotFoundError
 from tilebox.datasets.aio.dataset import DatasetClient
 from tilebox.datasets.client import Client as BaseClient
 from tilebox.datasets.client import token_from_env
+from tilebox.datasets.data.datasets import DatasetKind, FieldDict
 from tilebox.datasets.datasets.v1.collections_pb2_grpc import CollectionServiceStub
 from tilebox.datasets.datasets.v1.data_access_pb2_grpc import DataAccessServiceStub
 from tilebox.datasets.datasets.v1.data_ingestion_pb2_grpc import DataIngestionServiceStub
@@ -32,10 +34,52 @@ class Client:
         )
         self._client = BaseClient(service)
 
+    async def create_or_update_dataset(
+        self,
+        kind: DatasetKind,
+        code_name: str,
+        fields: list[FieldDict] | None = None,
+        *,
+        name: str | None = None,
+    ) -> DatasetClient:
+        """Create a new dataset.
+
+        Args:
+            kind: The kind of the dataset.
+            code_name: The code name of the dataset.
+            fields: The custom fields of the dataset.
+            name: The name of the dataset. Defaults to the code name.
+
+        Returns:
+            The created dataset.
+        """
+
+        try:
+            dataset = await self.dataset(code_name)
+        except NotFoundError:
+            return await self._client.create_dataset(kind, code_name, fields or [], name or code_name, DatasetClient)
+
+        return await self._client.update_dataset(
+            kind,
+            dataset._dataset.id,  # noqa: SLF001
+            fields or [],
+            name or dataset._dataset.name,  # noqa: SLF001
+            DatasetClient,
+        )
+
     async def datasets(self) -> Group:
+        """Fetch all available datasets."""
         return await self._client.datasets(DatasetClient)
 
     async def dataset(self, slug: str) -> DatasetClient:
+        """Get a dataset by its slug, e.g. `open_data.copernicus.sentinel1_sar`.
+
+        Args:
+            slug: The slug of the dataset.
+
+        Returns:
+            The dataset if it exists.
+        """
         return await self._client.dataset(slug, DatasetClient)
 
     async def _dataset_by_id(self, dataset_id: str | UUID) -> DatasetClient:

tilebox/datasets/client.py

@@ -7,7 +7,7 @@ from loguru import logger
 from promise import Promise
 
 from _tilebox.grpc.channel import parse_channel_info
-from tilebox.datasets.data.datasets import Dataset, DatasetGroup, ListDatasetsResponse
+from tilebox.datasets.data.datasets import Dataset, DatasetGroup, DatasetKind, FieldDict, ListDatasetsResponse
 from tilebox.datasets.group import Group
 from tilebox.datasets.message_pool import register_once
 from tilebox.datasets.service import TileboxDatasetService
@@ -26,8 +26,35 @@ class Client:
     def __init__(self, service: TileboxDatasetService) -> None:
         self._service = service
 
+    def create_dataset(
+        self,
+        kind: DatasetKind,
+        code_name: str,
+        fields: list[FieldDict] | None,
+        name: str | None,
+        py_dataset_class: type[T],
+    ) -> Promise[T]:
+        return (
+            self._service.create_dataset(kind, code_name, name or code_name, fields or [])
+            .then(_ensure_registered)
+            .then(lambda dataset: py_dataset_class(self._service, dataset))
+        )
+
+    def update_dataset(
+        self,
+        kind: DatasetKind,
+        dataset_id: UUID,
+        fields: list[FieldDict] | None,
+        name: str | None,
+        py_dataset_class: type[T],
+    ) -> Promise[T]:
+        return (
+            self._service.update_dataset(kind, dataset_id, name, fields or [])
+            .then(_ensure_registered)
+            .then(lambda dataset: py_dataset_class(self._service, dataset))
+        )
+
     def datasets(self, dataset_type: type[T]) -> Promise[Group]:
-        """Fetch all available datasets."""
         return (
             self._service.list_datasets()
             .then(_log_server_message)
@@ -40,16 +67,6 @@ class Client:
         )
 
     def dataset(self, slug: str, dataset_type: type[T]) -> Promise[T]:
-        """
-        Get a dataset by its slug, e.g. `open_data.copernicus.sentinel1_sar`.
-
-        Args:
-            slug: The slug of the dataset
-
-        Returns:
-            The dataset if it exists.
-        """
-
         return (
             self._service.get_dataset_by_slug(slug)
             .then(_ensure_registered)

tilebox/datasets/data/datasets.py

@@ -1,12 +1,29 @@
 from dataclasses import dataclass
+from datetime import datetime, timedelta
+from enum import Enum
+from typing import TypedDict, get_args, get_origin
 from uuid import UUID
 
-from google.protobuf.descriptor_pb2 import FileDescriptorSet
+import numpy as np
+from google.protobuf import duration_pb2, timestamp_pb2
+from google.protobuf.descriptor_pb2 import FieldDescriptorProto, FileDescriptorSet
+from shapely import Geometry
+from typing_extensions import NotRequired, Required
 
-from tilebox.datasets.datasets.v1 import core_pb2, dataset_type_pb2, datasets_pb2
+from tilebox.datasets.datasets.v1 import core_pb2, dataset_type_pb2, datasets_pb2, well_known_types_pb2
 from tilebox.datasets.uuid import uuid_message_to_optional_uuid, uuid_message_to_uuid, uuid_to_uuid_message
 
 
+class DatasetKind(Enum):
+    TEMPORAL = dataset_type_pb2.DATASET_KIND_TEMPORAL
+    """A dataset that contains a timestamp field."""
+    SPATIOTEMPORAL = dataset_type_pb2.DATASET_KIND_SPATIOTEMPORAL
+    """A dataset that contains a timestamp field and a geometry field."""
+
+
+_dataset_kind_int_to_enum = {kind.value: kind for kind in DatasetKind}
+
+
 @dataclass(frozen=True)
 class FieldAnnotation:
     description: str
@@ -20,6 +37,116 @@ class FieldAnnotation:
         return dataset_type_pb2.FieldAnnotation(description=self.description, example_value=self.example_value)
 
 
+class FieldDict(TypedDict):
+    name: Required[str]
+    type: Required[
+        type[str]
+        | type[list[str]]
+        | type[bytes]
+        | type[list[bytes]]
+        | type[bool]
+        | type[list[bool]]
+        | type[int]
+        | type[list[int]]
+        | type[np.uint64]
+        | type[list[np.uint64]]
+        | type[float]
+        | type[list[float]]
+        | type[timedelta]
+        | type[list[timedelta]]
+        | type[datetime]
+        | type[list[datetime]]
+        | type[UUID]
+        | type[list[UUID]]
+        | type[Geometry]
+        | type[list[Geometry]]
+    ]
+    description: NotRequired[str]
+    example_value: NotRequired[str]
+
+
+_TYPE_INFO: dict[type, tuple[FieldDescriptorProto.Type.ValueType, str | None]] = {
+    str: (FieldDescriptorProto.TYPE_STRING, None),
+    bytes: (FieldDescriptorProto.TYPE_BYTES, None),
+    bool: (FieldDescriptorProto.TYPE_BOOL, None),
+    int: (FieldDescriptorProto.TYPE_INT64, None),
+    np.uint64: (FieldDescriptorProto.TYPE_UINT64, None),
+    float: (FieldDescriptorProto.TYPE_DOUBLE, None),
+    timedelta: (FieldDescriptorProto.TYPE_MESSAGE, f".{duration_pb2.Duration.DESCRIPTOR.full_name}"),
+    datetime: (FieldDescriptorProto.TYPE_MESSAGE, f".{timestamp_pb2.Timestamp.DESCRIPTOR.full_name}"),
+    UUID: (FieldDescriptorProto.TYPE_MESSAGE, f".{well_known_types_pb2.UUID.DESCRIPTOR.full_name}"),
+    Geometry: (FieldDescriptorProto.TYPE_MESSAGE, f".{well_known_types_pb2.Geometry.DESCRIPTOR.full_name}"),
+}
+
+
+@dataclass(frozen=True)
+class Field:
+    descriptor: FieldDescriptorProto
+    annotation: FieldAnnotation
+    queryable: bool
+
+    @classmethod
+    def from_message(cls, field: dataset_type_pb2.Field) -> "Field":
+        return cls(
+            descriptor=field.descriptor,
+            annotation=FieldAnnotation.from_message(field.annotation),
+            queryable=field.queryable,
+        )
+
+    @classmethod
+    def from_dict(cls, field: FieldDict) -> "Field":
+        origin = get_origin(field["type"])
+        if origin is list:
+            label = FieldDescriptorProto.Label.LABEL_REPEATED
+            args = get_args(field["type"])
+            inner_type = args[0] if args else field["type"]
+        else:
+            label = FieldDescriptorProto.Label.LABEL_OPTIONAL
+            inner_type = field["type"]
+
+        (field_type, field_type_name) = _TYPE_INFO[inner_type]
+
+        return cls(
+            descriptor=FieldDescriptorProto(
+                name=field["name"],
+                type=field_type,
+                type_name=field_type_name,
+                label=label,
+            ),
+            annotation=FieldAnnotation(
+                description=field.get("description", ""),
+                example_value=field.get("example_value", ""),
+            ),
+            queryable=False,
+        )
+
+    def to_message(self) -> dataset_type_pb2.Field:
+        return dataset_type_pb2.Field(
+            descriptor=self.descriptor,
+            annotation=self.annotation.to_message(),
+            queryable=self.queryable,
+        )
+
+
+@dataclass(frozen=True)
+class DatasetType:
+    kind: DatasetKind | None
+    fields: list[Field]
+
+    @classmethod
+    def from_message(cls, dataset_type: dataset_type_pb2.DatasetType) -> "DatasetType":
+        return cls(
+            kind=_dataset_kind_int_to_enum.get(dataset_type.kind, None),
+            fields=[Field.from_message(f) for f in dataset_type.fields],
+        )
+
+    def to_message(self) -> dataset_type_pb2.DatasetType:
+        return dataset_type_pb2.DatasetType(
+            kind=self.kind.value if self.kind else dataset_type_pb2.DATASET_KIND_UNSPECIFIED,
+            fields=[f.to_message() for f in self.fields],
+        )
+
+
 @dataclass(frozen=True)
 class AnnotatedType:
     descriptor_set: FileDescriptorSet

tilebox/datasets/message_pool.py

@@ -1,5 +1,6 @@
 from google.protobuf import descriptor_pb2, duration_pb2, timestamp_pb2
 from google.protobuf.descriptor_pool import Default
+from google.protobuf.message import Message
 from google.protobuf.message_factory import GetMessageClass, GetMessages
 
 from tilebox.datasets.data.datasets import AnnotatedType
@@ -25,5 +26,5 @@ def register_message_types(descriptor_set: descriptor_pb2.FileDescriptorSet) ->
     GetMessages(descriptor_set.file, pool=Default())
 
 
-def get_message_type(type_url: str) -> type:
+def get_message_type(type_url: str) -> type[Message]:
     return GetMessageClass(Default().FindMessageTypeByName(type_url))

tilebox/datasets/progress.py

@@ -3,6 +3,11 @@ from datetime import datetime
 from types import TracebackType
 from typing import Any
 
+try:
+    from typing import Self  # ty: ignore[unresolved-import]
+except ImportError:  # Self is only available in Python 3.11+
+    from typing_extensions import Self
+
 from tqdm.auto import tqdm
 
 from tilebox.datasets.query.time_interval import TimeInterval
@@ -42,7 +47,7 @@ class TimeIntervalProgressBar:
         self._actual_start_time = actual_start_time
         self._total_data_points = 0
 
-    def __enter__(self) -> "TimeIntervalProgressBar":
+    def __enter__(self) -> Self:
         self._progress_bar = tqdm(
             bar_format="{l_bar}{bar}[{elapsed}<{remaining}{postfix}]",
             total=self._calc_progress_seconds(self._interval.end),
@@ -62,7 +67,8 @@ class TimeIntervalProgressBar:
 
     def set_progress(self, time: datetime) -> None:
         """Set the progress of the progress bar to the given time"""
-        done = min(self._calc_progress_seconds(time), self._progress_bar.total)
+        total = self._calc_progress_seconds(self._interval.end)
+        done = min(self._calc_progress_seconds(time), total)
         self._progress_bar.update(done - self._progress_bar.n)
 
     def set_download_info(self, datapoints: int, byte_size: int, download_time: float) -> None:
@@ -79,7 +85,8 @@ class TimeIntervalProgressBar:
     ) -> None:
         try:
             if traceback is None:
-                self._progress_bar.update(self._progress_bar.total - self._progress_bar.n)  # set to 100%
+                total = self._calc_progress_seconds(self._interval.end)
+                self._progress_bar.update(total - self._progress_bar.n)  # set to 100%
 
             self._progress_bar.close()  # mark as completed or failed
         except AttributeError:

tilebox/datasets/protobuf_conversion/field_types.py

@@ -1,9 +1,10 @@
-from collections.abc import Sized
+from collections.abc import Sequence
 from datetime import timedelta
 from typing import Any
 from uuid import UUID
 
 import numpy as np
+import pandas as pd
 from google.protobuf.descriptor import FieldDescriptor
 from google.protobuf.duration_pb2 import Duration
 from google.protobuf.message import Message
@@ -16,9 +17,12 @@ from shapely import from_wkb
 from tilebox.datasets.datasets.v1.well_known_types_pb2 import UUID as UUIDMessage  # noqa: N811
 from tilebox.datasets.datasets.v1.well_known_types_pb2 import Geometry, LatLon, LatLonAlt, Quaternion, Vec3
 
-ProtoFieldValue = Message | float | str | bool | bytes | Sized | None
+ScalarProtoFieldValue = Message | float | str | bool | bytes
 
-_FILL_VALUES_BY_DTYPE = {
+
+ProtoFieldValue = ScalarProtoFieldValue | Sequence[ScalarProtoFieldValue] | None
+
+_FILL_VALUES_BY_DTYPE: dict[type[np.dtype[Any]], Any] = {
     npdtypes.Int8DType: np.int8(0),
     npdtypes.Int16DType: np.int16(0),
     npdtypes.Int32DType: np.int32(0),
@@ -106,7 +110,7 @@ class TimestampField(ProtobufFieldType):
         return value.seconds * 10**9 + value.nanos
 
     def to_proto(self, value: DatetimeScalar) -> Timestamp | None:
-        if value is None or (isinstance(value, np.datetime64) and np.isnat(value)):
+        if is_missing(value) or (isinstance(value, np.datetime64) and np.isnat(value)):
             return None
         # we use pandas to_datetime function to handle a variety of input types that can be coerced to datetimes
         seconds, nanos = divmod(to_datetime(value, utc=True).value, 10**9)
@@ -123,10 +127,10 @@ class TimeDeltaField(ProtobufFieldType):
         return value.seconds * 10**9 + value.nanos
 
     def to_proto(self, value: str | float | timedelta | np.timedelta64) -> Duration | None:
-        if value is None or (isinstance(value, np.timedelta64) and np.isnat(value)):
+        if is_missing(value) or (isinstance(value, np.timedelta64) and np.isnat(value)):
             return None
         # we use pandas to_timedelta function to handle a variety of input types that can be coerced to timedeltas
-        seconds, nanos = divmod(to_timedelta(value).value, 10**9)  # type: ignore[arg-type]
+        seconds, nanos = divmod(to_timedelta(value).value, 10**9)
         return Duration(seconds=seconds, nanos=nanos)
 
 
@@ -140,7 +144,7 @@ class UUIDField(ProtobufFieldType):
         return str(UUID(bytes=value.uuid))
 
     def to_proto(self, value: str | UUID) -> UUIDMessage | None:
-        if not value:  # None or empty string
+        if is_missing(value) or value == "":  # missing or empty string
             return None
 
         if isinstance(value, str):
@@ -159,7 +163,7 @@ class GeometryField(ProtobufFieldType):
         return from_wkb(value.wkb)
 
     def to_proto(self, value: Any) -> Geometry | None:
-        if value is None:
+        if is_missing(value):
             return None
         return Geometry(wkb=value.wkb)
 
@@ -174,7 +178,7 @@ class Vec3Field(ProtobufFieldType):
         return value.x, value.y, value.z
 
     def to_proto(self, value: tuple[float, float, float]) -> Vec3 | None:
-        if value is None or np.all(np.isnan(value)):
+        if is_missing(value) or np.all(np.isnan(value)):
             return None
         return Vec3(x=value[0], y=value[1], z=value[2])
 
@@ -189,7 +193,7 @@ class QuaternionField(ProtobufFieldType):
         return value.q1, value.q2, value.q3, value.q4
 
     def to_proto(self, value: tuple[float, float, float, float]) -> Quaternion | None:
-        if value is None or np.all(np.isnan(value)):
+        if is_missing(value) or np.all(np.isnan(value)):
             return None
         return Quaternion(q1=value[0], q2=value[1], q3=value[2], q4=value[3])
 
@@ -204,7 +208,7 @@ class LatLonField(ProtobufFieldType):
         return value.latitude, value.longitude
 
     def to_proto(self, value: tuple[float, float]) -> LatLon | None:
-        if value is None or np.all(np.isnan(value)):
+        if is_missing(value) or np.all(np.isnan(value)):
             return None
         return LatLon(latitude=value[0], longitude=value[1])
 
@@ -220,7 +224,7 @@ class LatLonAltField(ProtobufFieldType):
         return value.latitude, value.longitude, value.altitude
 
     def to_proto(self, value: tuple[float, float, float]) -> LatLonAlt | None:
-        if value is None or np.all(np.isnan(value)):
+        if is_missing(value) or np.all(np.isnan(value)):
             return None
         return LatLonAlt(latitude=value[0], longitude=value[1], altitude=value[2])
 
@@ -300,3 +304,19 @@ def _camel_to_uppercase(name: str) -> str:
         'PROCESSING_LEVEL'
     """
     return "".join(["_" + c.lower() if c.isupper() else c for c in name]).lstrip("_").upper()
+
+
+def is_missing(value: Any) -> bool:
+    """Check if a value represents a missing/null value.
+
+    Handles None, np.nan, pd.NA, NaT, and other pandas missing value sentinels.
+    This is needed for pandas 3.0+ compatibility where object-dtype columns use
+    np.nan instead of None for missing values.
+    """
+    try:
+        return bool(pd.isna(value))
+    except ValueError:
+        # pd.isna returns either a bool, or an array of bools. In case of an array, converting the result to bool()
+        # will raise a ValueError. For an array, we know it's not a missing value, even an array of all NaNs is not
+        # a missing value.
+        return False

tilebox/datasets/protobuf_conversion/protobuf_xarray.py

@@ -3,7 +3,7 @@ Functionality for converting protobuf messages to xarray datasets.
 """
 
 import contextlib
-from collections.abc import Sized
+from collections.abc import Sequence
 from typing import Any, TypeVar
 
 import numpy as np
@@ -231,10 +231,10 @@ class _SimpleFieldConverter(_FieldConverter):
         elif buffer_size > len(self._data):
             # resize the data buffer to the new capacity, by just padding it with zeros at the end
             missing = buffer_size - len(self._data)
-            self._data = np.pad(
+            self._data = np.pad(  # ty: ignore[no-matching-overload]
                 self._data,
                 ((0, missing), (0, 0)),
-                constant_values=self._type.fill_value,  # type: ignore[arg-type]
+                constant_values=self._type.fill_value,
             )
 
 
@@ -258,13 +258,13 @@ class _ArrayFieldConverter(_FieldConverter):
         self._array_dim: int | None = None
 
     def __call__(self, index: int, value: ProtoFieldValue) -> None:
-        if not isinstance(value, Sized):
+        if not isinstance(value, Sequence):
             raise TypeError(f"Expected array field but got {type(value)}")
 
         if self._array_dim is None or len(value) > self._array_dim:
             self._resize_array_dim(len(value))
 
-        for i, v in enumerate(value):  # type: ignore[arg-type]  # somehow the isinstance(value, Sized) isn't used here
+        for i, v in enumerate(value):  # somehow the isinstance(value, Sized) isn't used here
             self._data[index, i, :] = self._type.from_proto(v)
 
     def finalize(
@@ -309,10 +309,10 @@ class _ArrayFieldConverter(_FieldConverter):
         else:  # resize the data buffer to the new capacity, by just padding it with zeros at the end
             missing_capacity = self._capacity - self._data.shape[0]
             missing_array_dim = self._array_dim - self._data.shape[1]
-            self._data = np.pad(
+            self._data = np.pad(  # ty: ignore[no-matching-overload]
                 self._data,
                 ((0, missing_capacity), (0, missing_array_dim), (0, 0)),
-                constant_values=self._type.fill_value,  # type: ignore[arg-type]
+                constant_values=self._type.fill_value,
             )
 
 
@@ -374,13 +374,13 @@ def _create_field_converter(field: FieldDescriptor) -> _FieldConverter:
     """
     # special handling for enums:
     if field.type == FieldDescriptor.TYPE_ENUM:
-        if field.is_repeated:  # type: ignore[attr-defined]
+        if field.is_repeated:
             raise NotImplementedError("Repeated enum fields are not supported")
 
         return _EnumFieldConverter(field.name, enum_mapping_from_field_descriptor(field))
 
     field_type = infer_field_type(field)
-    if field.is_repeated:  # type: ignore[attr-defined]
+    if field.is_repeated:
         return _ArrayFieldConverter(field.name, field_type)
 
     return _SimpleFieldConverter(field.name, field_type)

tilebox/datasets/protobuf_conversion/to_protobuf.py

@@ -12,6 +12,7 @@ from tilebox.datasets.protobuf_conversion.field_types import (
     ProtobufFieldType,
     ProtoFieldValue,
     infer_field_type,
+    is_missing,
 )
 
 IngestionData = Mapping[str, Collection[Any]] | Iterable[tuple[str, Collection[Any]]] | pd.DataFrame | xr.Dataset
@@ -20,7 +21,7 @@ DatapointIDs = pd.DataFrame | pd.Series | xr.Dataset | xr.DataArray | np.ndarray
 
 def to_messages(  # noqa: C901, PLR0912
     data: IngestionData,
-    message_type: type,
+    message_type: type[Message],
     required_fields: list[str] | None = None,
     ignore_fields: list[str] | None = None,
 ) -> list[Message]:
@@ -44,9 +45,9 @@ def to_messages(  # noqa: C901, PLR0912
     # let's validate our fields, to make sure that they are all known fields for the given protobuf message
     # and that they are all lists of the same length
     field_lengths = defaultdict(list)
-    fields: dict[str, pd.Series | np.ndarray] = {}
+    fields: dict[str, pd.Series | np.ndarray | list[ProtoFieldValue]] = {}
 
-    field_names = list(map(str, data))
+    field_names = [str(field) for field in data]
     if isinstance(data, xr.Dataset):
         # list(dataset) only returns the variables, not the coords, so for xarray we need to add the coords as well
         # but not all coords, we only care abou time for now
@@ -84,7 +85,7 @@ def to_messages(  # noqa: C901, PLR0912
         else:
             values = convert_values_to_proto(values, field_type, filter_none=False)
 
-        fields[field_name] = values  # type: ignore[assignment]
+        fields[field_name] = values
 
     # now convert every datapoint to a protobuf message
     if len(field_lengths) == 0:  # early return, no actual data to convert
@@ -103,7 +104,7 @@ def marshal_messages(messages: list[Message]) -> list[bytes]:
 
 
 def columnar_to_row_based(
-    data: dict[str, pd.Series | np.ndarray],
+    data: dict[str, pd.Series | np.ndarray | list[ProtoFieldValue]],
 ) -> Iterator[dict[str, Any]]:
     if len(data) == 0:
         return
@@ -120,18 +121,18 @@ def convert_values_to_proto(
     values: np.ndarray | pd.Series, field_type: ProtobufFieldType, filter_none: bool = False
 ) -> list[ProtoFieldValue]:
     if filter_none:
-        return [field_type.to_proto(value) for value in values if value is not None]
+        return [field_type.to_proto(value) for value in values if not is_missing(value)]
     return [field_type.to_proto(value) for value in values]
 
 
 def convert_repeated_values_to_proto(
     values: np.ndarray | pd.Series | list[np.ndarray], field_type: ProtobufFieldType
-) -> Any:
+) -> list[ProtoFieldValue]:
     if isinstance(values, np.ndarray):  # it was an xarray, with potentially padded fill values at the end
         values = trim_trailing_fill_values(values, field_type.fill_value)
 
     # since repeated fields can have different lengths between datapoints, we can filter out None values here
-    return [convert_values_to_proto(repeated_values, field_type, filter_none=True) for repeated_values in values]
+    return [convert_values_to_proto(repeated_values, field_type, filter_none=True) for repeated_values in values]  # ty: ignore[invalid-return-type]
 
 
 def trim_trailing_fill_values(values: np.ndarray, fill_value: Any) -> list[np.ndarray]:

tilebox/datasets/query/id_interval.py

@@ -50,23 +50,21 @@ class IDInterval:
         Returns:
             IDInterval: The parsed ID interval
         """
+        if isinstance(arg, IDInterval):
+            return arg
 
-        match arg:
-            case IDInterval(_, _, _, _):
-                return arg
-            case (UUID(), UUID()):
-                start, end = arg
+        if isinstance(arg, tuple) and len(arg) == 2:
+            start, end = arg
+            if isinstance(start, UUID) and isinstance(end, UUID):
                 return IDInterval(
-                    start_id=start,
-                    end_id=end,
-                    start_exclusive=start_exclusive,
-                    end_inclusive=end_inclusive,
+                    start_id=start, end_id=end, start_exclusive=start_exclusive, end_inclusive=end_inclusive
                 )
-            case (str(), str()):
-                start, end = arg
+            if isinstance(start, str) and isinstance(end, str):
                 return IDInterval(
                     start_id=UUID(start),
                     end_id=UUID(end),
                     start_exclusive=start_exclusive,
                     end_inclusive=end_inclusive,
                 )
+
+        raise ValueError(f"Failed to convert {arg} ({type(arg)}) to IDInterval")

tilebox/datasets/query/time_interval.py

@@ -15,8 +15,10 @@ _EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
 
 # A type alias for the different types that can be used to specify a time interval
 TimeIntervalLike: TypeAlias = (
-    DatetimeScalar | tuple[DatetimeScalar, DatetimeScalar] | xr.DataArray | xr.Dataset | "TimeInterval"
+    "DatetimeScalar | tuple[DatetimeScalar, DatetimeScalar] | xr.DataArray | xr.Dataset | TimeInterval"
 )
+# once we require python >= 3.12 we can replace this with a type statement, which doesn't require a string at all
+# type TimeIntervalLike = DatetimeScalar | tuple[DatetimeScalar ...  | TimeInterval
 
 
 @dataclass(frozen=True)

tilebox/datasets/service.py

@@ -3,12 +3,23 @@ import sys
 from importlib.metadata import distributions
 from uuid import UUID
 
+from google.protobuf import timestamp_pb2
+from google.protobuf.descriptor_pb2 import FieldDescriptorProto
 from promise import Promise
 
 from tilebox.datasets.data.collection import CollectionInfo
 from tilebox.datasets.data.data_access import QueryFilters
 from tilebox.datasets.data.datapoint import AnyMessage, IngestResponse, QueryResultPage
-from tilebox.datasets.data.datasets import Dataset, ListDatasetsResponse
+from tilebox.datasets.data.datasets import (
+    Dataset,
+    DatasetKind,
+    DatasetType,
+    Field,
+    FieldAnnotation,
+    FieldDict,
+    ListDatasetsResponse,
+)
+from tilebox.datasets.datasets.v1 import well_known_types_pb2
 from tilebox.datasets.datasets.v1.collections_pb2 import (
     CreateCollectionRequest,
     DeleteCollectionRequest,
@@ -20,7 +31,14 @@ from tilebox.datasets.datasets.v1.data_access_pb2 import QueryByIDRequest, Query
 from tilebox.datasets.datasets.v1.data_access_pb2_grpc import DataAccessServiceStub
 from tilebox.datasets.datasets.v1.data_ingestion_pb2 import DeleteRequest, IngestRequest
 from tilebox.datasets.datasets.v1.data_ingestion_pb2_grpc import DataIngestionServiceStub
-from tilebox.datasets.datasets.v1.datasets_pb2 import ClientInfo, GetDatasetRequest, ListDatasetsRequest, Package
+from tilebox.datasets.datasets.v1.datasets_pb2 import (
+    ClientInfo,
+    CreateDatasetRequest,
+    GetDatasetRequest,
+    ListDatasetsRequest,
+    Package,
+    UpdateDatasetRequest,
+)
 from tilebox.datasets.datasets.v1.datasets_pb2_grpc import DatasetServiceStub
 from tilebox.datasets.query.pagination import Pagination
 from tilebox.datasets.tilebox.v1 import id_pb2
@@ -46,6 +64,71 @@ class TileboxDatasetService:
         self._data_access_service = data_access_service_stub
         self._data_ingestion_service = data_ingestion_service_stub
 
+    def create_dataset(
+        self, kind: DatasetKind, code_name: str, name: str, custom_fields: list[FieldDict]
+    ) -> Promise[Dataset]:
+        """Create a new dataset.
+
+        Args:
+            kind: The kind of the dataset.
+            code_name: The code name of the dataset.
+            name: The name of the dataset.
+            fields: The custom fields of the dataset
+
+        Returns:
+            The created dataset.
+        """
+        dataset_type = DatasetType(
+            kind, _REQUIRED_FIELDS_PER_DATASET_KIND[kind] + [Field.from_dict(f) for f in custom_fields]
+        )
+        req = CreateDatasetRequest(name=name, type=dataset_type.to_message(), code_name=code_name)
+        return Promise.resolve(self._dataset_service.CreateDataset(req)).then(Dataset.from_message)
+
+    def update_dataset(
+        self, kind: DatasetKind, dataset_id: UUID, name: str | None, custom_fields: list[FieldDict]
+    ) -> Promise[Dataset]:
+        """Update a dataset.
+
+        Args:
+            kind: The kind of the dataset to update, cannot be changed.
+            dataset_id: The id of the dataset to update, cannot be changed.
+            name: The new name of the dataset.
+            custom_fields: The new list of custom fields of the dataset.
+
+        Returns:
+            The updated dataset.
+        """
+        dataset_type = DatasetType(
+            kind, _REQUIRED_FIELDS_PER_DATASET_KIND[kind] + [Field.from_dict(f) for f in custom_fields]
+        )
+        req = UpdateDatasetRequest(id=uuid_to_uuid_message(dataset_id), name=name, type=dataset_type.to_message())
+        return Promise.resolve(self._dataset_service.UpdateDataset(req)).then(Dataset.from_message)
+
+    def create_or_update_dataset(
+        self, kind: DatasetKind, code_name: str, name: str, custom_fields: list[FieldDict]
+    ) -> Promise[Dataset]:
+        """Create a new dataset, or update it if it already exists.
+
+        Args:
+            kind: The kind of the dataset.
+            code_name: The code name of the dataset.
+            name: The name of the dataset.
+            custom_fields: The custom fields of the dataset
+
+        Returns:
+            The created or updated dataset.
+        """
+        return (
+            Promise.resolve(self._dataset_service.GetDataset(GetDatasetRequest(slug=code_name)))
+            .then(
+                did_fulfill=lambda dataset: self.update_dataset(
+                    kind, Dataset.from_message(dataset).id, name, custom_fields
+                ),
+                did_reject=lambda _: self.create_dataset(kind, code_name, name, custom_fields),
+            )
+            .then(Dataset.from_message)
+        )
+
     def list_datasets(self) -> Promise[ListDatasetsResponse]:
         """List all datasets and dataset groups."""
         return Promise.resolve(
@@ -188,7 +271,7 @@ def _client_info() -> ClientInfo:
 def _environment_info() -> str:
     python_version = sys.version.split(" ")[0]
     try:
-        shell = str(get_ipython())  # type: ignore[name-defined]
+        shell = str(get_ipython())  # ty: ignore[unresolved-reference]
     except NameError:
         return f"Python {python_version}"  # Probably standard Python interpreter
 
@@ -202,3 +285,65 @@ def _environment_info() -> str:
         return f"Google Colab using python {python_version}"
 
     return f"Unknown IPython using python {python_version}"
+
+
+_time_field = Field(
+    descriptor=FieldDescriptorProto(
+        name="time",
+        label=FieldDescriptorProto.Label.LABEL_OPTIONAL,
+        type=FieldDescriptorProto.TYPE_MESSAGE,
+        type_name=f".{timestamp_pb2.Timestamp.DESCRIPTOR.full_name}",
+    ),
+    annotation=FieldAnnotation(
+        description="The timestamp associated with each data point.",
+        example_value="2022-10-17T14:35:28Z",
+    ),
+    queryable=False,
+)
+
+_id_field = Field(
+    descriptor=FieldDescriptorProto(
+        name="id",
+        label=FieldDescriptorProto.Label.LABEL_OPTIONAL,
+        type=FieldDescriptorProto.TYPE_MESSAGE,
+        type_name=f".{well_known_types_pb2.UUID.DESCRIPTOR.full_name}",
+    ),
+    annotation=FieldAnnotation(
+        description="A universally unique identifier (UUID) that uniquely identifies each data point, automatically generated by Tilebox.",
+        example_value="4e8a2836-72f8-4ac2-a9e9-cbe3492ef60c",
+    ),
+    queryable=False,
+)
+
+_ingestion_time_field = Field(
+    descriptor=FieldDescriptorProto(
+        name="ingestion_time",
+        label=FieldDescriptorProto.Label.LABEL_OPTIONAL,
+        type=FieldDescriptorProto.TYPE_MESSAGE,
+        type_name=f".{timestamp_pb2.Timestamp.DESCRIPTOR.full_name}",
+    ),
+    annotation=FieldAnnotation(
+        description="The time the data point was ingested into the Tilebox API, automatically generated by Tilebox.",
+        example_value="2022-10-17T14:35:28Z",
+    ),
+    queryable=False,
+)
+
+_geometry_field = Field(
+    descriptor=FieldDescriptorProto(
+        name="geometry",
+        label=FieldDescriptorProto.Label.LABEL_OPTIONAL,
+        type=FieldDescriptorProto.TYPE_MESSAGE,
+        type_name=f".{well_known_types_pb2.Geometry.DESCRIPTOR.full_name}",
+    ),
+    annotation=FieldAnnotation(
+        description="The geometry associated with each data point.",
+        example_value="POLYGON ((112.345 -36.789, ...))",
+    ),
+    queryable=False,
+)
+
+_REQUIRED_FIELDS_PER_DATASET_KIND: dict[DatasetKind, list[Field]] = {
+    DatasetKind.TEMPORAL: [_time_field, _id_field, _ingestion_time_field],
+    DatasetKind.SPATIOTEMPORAL: [_time_field, _id_field, _ingestion_time_field, _geometry_field],
+}

tilebox/datasets/sync/client.py

@@ -1,9 +1,10 @@
 from uuid import UUID
 
 from _tilebox.grpc.channel import open_channel
-from _tilebox.grpc.error import with_pythonic_errors
+from _tilebox.grpc.error import NotFoundError, with_pythonic_errors
 from tilebox.datasets.client import Client as BaseClient
 from tilebox.datasets.client import token_from_env
+from tilebox.datasets.data.datasets import DatasetKind, FieldDict
 from tilebox.datasets.datasets.v1.collections_pb2_grpc import CollectionServiceStub
 from tilebox.datasets.datasets.v1.data_access_pb2_grpc import DataAccessServiceStub
 from tilebox.datasets.datasets.v1.data_ingestion_pb2_grpc import DataIngestionServiceStub
@@ -32,10 +33,52 @@ class Client:
         )
         self._client = BaseClient(service)
 
+    def create_or_update_dataset(
+        self,
+        kind: DatasetKind,
+        code_name: str,
+        fields: list[FieldDict] | None = None,
+        *,
+        name: str | None = None,
+    ) -> DatasetClient:
+        """Create a new dataset.
+
+        Args:
+            kind: The kind of the dataset.
+            code_name: The code name of the dataset.
+            fields: The custom fields of the dataset.
+            name: The name of the dataset. Defaults to the code name.
+
+        Returns:
+            The created dataset.
+        """
+
+        try:
+            dataset = self.dataset(code_name)
+        except NotFoundError:
+            return self._client.create_dataset(kind, code_name, fields or [], name or code_name, DatasetClient).get()
+
+        return self._client.update_dataset(
+            kind,
+            dataset._dataset.id,  # noqa: SLF001
+            fields or [],
+            name or dataset._dataset.name,  # noqa: SLF001
+            DatasetClient,
+        ).get()
+
     def datasets(self) -> Group:
+        """Fetch all available datasets."""
         return self._client.datasets(DatasetClient).get()
 
     def dataset(self, slug: str) -> DatasetClient:
+        """Get a dataset by its slug, e.g. `open_data.copernicus.sentinel1_sar`.
+
+        Args:
+            slug: The slug of the dataset.
+
+        Returns:
+            The dataset if it exists.
+        """
         return self._client.dataset(slug, DatasetClient).get()
 
     def _dataset_by_id(self, dataset_id: str | UUID) -> DatasetClient:

{tilebox_datasets-0.45.0.dist-info → tilebox_datasets-0.47.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tilebox-datasets
-Version: 0.45.0
+Version: 0.47.0
 Summary: Access Tilebox datasets from Python
 Project-URL: Homepage, https://tilebox.com
 Project-URL: Documentation, https://docs.tilebox.com/datasets/introduction

{tilebox_datasets-0.45.0.dist-info → tilebox_datasets-0.47.0.dist-info}/RECORD

@@ -1,12 +1,12 @@
 tilebox/datasets/__init__.py,sha256=4fI6ErCVb15KD_iPWIM6TUXTPgU8qrJVx3Cx7m7jeC8,824
-tilebox/datasets/client.py,sha256=RieyTkoUED5cDFsLcakeO4LZ3Yrq7WtRqG5ZSH7mJPM,4076
+tilebox/datasets/client.py,sha256=uWAFr77kJ1HXR8hXsDDl8H6ug-PKUdPomwc4Z85QQ9I,4718
 tilebox/datasets/group.py,sha256=DoGl4w7Viy-l4kE9580dJOymP_B2pj3LRqvMNxvrYmU,1884
-tilebox/datasets/message_pool.py,sha256=5Hkd2xqgcpQsYY4IySNRGnvAyl2tsyhH21fTAu3EVXg,1272
-tilebox/datasets/progress.py,sha256=5w_kmrg_aKcW4qluOJu0bPnMP2tV_JA3EZMgk1GDYJM,3531
-tilebox/datasets/service.py,sha256=w5Vo5EXN6f_moifa9xLz4dS5en6wB5kNRHZSA-CoXcE,8865
+tilebox/datasets/message_pool.py,sha256=nGuwihK2Lhfk-q5cuRtjXxmgPlEU_DEp5uYRueCPWHk,1325
+tilebox/datasets/progress.py,sha256=Mmn1ukjX_lNhgR7jnYFgksAoc2r33DK44VqIWMogI04,3776
+tilebox/datasets/service.py,sha256=32Lw-giFGchbqDcX6vfYJxZfdPUxAtEjPxhf5P93DkE,14043
 tilebox/datasets/uuid.py,sha256=pqtp5GMHM41KEKZHPdwrHVVThY9VDa7BPbCogrM01ZU,1107
 tilebox/datasets/aio/__init__.py,sha256=0x_gddLgDsUCdl8MMZj4MPH1lp4HuOrExMHTjIFmM6s,405
-tilebox/datasets/aio/client.py,sha256=fr7rtBwVmKihfKQ_pmwfbMascGLbop0liYr4hb3o1eg,2130
+tilebox/datasets/aio/client.py,sha256=fJ7AF0cCocc0XT-Z1V3eb1QjnP4FirEixd4IufnEZoo,3499
 tilebox/datasets/aio/dataset.py,sha256=CBiEhT7Pex5JcYulCkKl4DX80oGj6jQB1uUfVV_F-zo,22511
 tilebox/datasets/aio/pagination.py,sha256=dqxnG1UIvXQwhS5XZDlnmtiy_zat5j5E7xeucqI1dZU,6111
 tilebox/datasets/aio/timeseries.py,sha256=iQqIyh9TPL_gJz18GCxmtFJEwObR9S2rPsUohFYM8wQ,301
@@ -17,7 +17,7 @@ tilebox/datasets/data/__init__.py,sha256=0PFUikRaYq-jwJSx0rZOUrGj5LF2NMgQv5BQkqY
 tilebox/datasets/data/collection.py,sha256=OfMsaq02FPk4btVJ3fYF6a8WEVUrjWZAHZffqSwuFbA,2667
 tilebox/datasets/data/data_access.py,sha256=rKxnSjzLvzco-GHJ4sRC4AJWkx5vSK13Y2o90_0sV-Q,5599
 tilebox/datasets/data/datapoint.py,sha256=5FOTzPtxWHt0DDivCGv8ZoWbbETkNMEbET6IFHwD8X0,3712
-tilebox/datasets/data/datasets.py,sha256=sJ1IAysUOizniJCORICq8wUacVMx0YsN9XLW85n3suI,4234
+tilebox/datasets/data/datasets.py,sha256=K6jmP68gpxMF_U--5JYL2NgEJn80S-oKasfOQ2RYbfU,8570
 tilebox/datasets/data/time_interval.py,sha256=Ws0oGOUtG9UlNzRVHWxrJu3aMAfOHvMUbemz4VEpg6E,501
 tilebox/datasets/data/timeseries.py,sha256=eT-J0ySyQZpuctEkG3Otxhd9i-WF4ppDiRlRHLnB-9A,3611
 tilebox/datasets/datasets/v1/collections_pb2.py,sha256=FRFOmFrI9Qg9yDOqrTseoRzEW-jzt7ux8W-as93_Hiw,5834
@@ -45,15 +45,15 @@ tilebox/datasets/datasets/v1/well_known_types_pb2.py,sha256=1Pi83QERn1nFFenZkITH
 tilebox/datasets/datasets/v1/well_known_types_pb2.pyi,sha256=RW-TrAbeFof2x-Qa5UPH0nCg-4KTuJ1RLU6SApnS60E,7216
 tilebox/datasets/datasets/v1/well_known_types_pb2_grpc.py,sha256=xYOs94SXiNYAlFodACnsXW5QovLsHY5tCk3p76RH5Zc,158
 tilebox/datasets/protobuf_conversion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tilebox/datasets/protobuf_conversion/field_types.py,sha256=WBISp5pJ_jUYXLC_mNXw_S98guLlnqBeAr1fJjAk5uo,11977
-tilebox/datasets/protobuf_conversion/protobuf_xarray.py,sha256=kO4Pq9bsH7p7Uocg8V4zUKM8KY0hes8bxByIdb6_s5k,16183
-tilebox/datasets/protobuf_conversion/to_protobuf.py,sha256=Xy_ItOzTBd0nziRUe_CHBTM7VwQi6XY8LbYFE2thjAc,7392
+tilebox/datasets/protobuf_conversion/field_types.py,sha256=vvPy1-8vsHedza-jPOCFXtQXwK6QVRc0BEXESSQ1jgU,12793
+tilebox/datasets/protobuf_conversion/protobuf_xarray.py,sha256=7OTQ3NnTVL52KoCeC8X5yOPG-Y60UpBm5bRuT8IAFAY,16123
+tilebox/datasets/protobuf_conversion/to_protobuf.py,sha256=-rJP954W4ZyONyHQSMtnLSnVIs9j73PCX0hPu7O-fL4,7504
 tilebox/datasets/query/__init__.py,sha256=lR-tzsVyx1QXe-uIHrYkCWcjmLRfKzmRHC7E1TTGroY,245
-tilebox/datasets/query/id_interval.py,sha256=Ha3Rm92hZugQXNzyfdFUROT1pTJ1ZBIISqTJbf13OP4,2508
+tilebox/datasets/query/id_interval.py,sha256=HiaAqn9Hh-07N9MSTjvVO0W0wJOTIP1HLy15PDa1nQQ,2583
 tilebox/datasets/query/pagination.py,sha256=0kaQI6v9sJnDJblP3VJn6erPbkP_LSwegFRSCzINGY0,774
-tilebox/datasets/query/time_interval.py,sha256=1Y_ewpGxIUJ1KSkAhoceH0madfsxS-F-NSHY2yM5GZ8,10018
+tilebox/datasets/query/time_interval.py,sha256=qJbCIjFzPt5oEi6YIEb4viNElJUPp1ZCanoA44eXITg,10216
 tilebox/datasets/sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tilebox/datasets/sync/client.py,sha256=Z_ZbsWxnL5qIvIFvmCT8yP7VEIVIp0MAJK94OtsFP8g,2105
+tilebox/datasets/sync/client.py,sha256=5wRzSAkXtPAfY7ykzqSckzbvvMmBjm0Dft-0mCHkfAU,3431
 tilebox/datasets/sync/dataset.py,sha256=wh8grBQZJAPZ7_X_8Ui67hK6v4uGAPC2gx02PSwbUgE,22174
 tilebox/datasets/sync/pagination.py,sha256=IOSbpNTlv3Fx9QLdBMZHJxZSWeKJNLOVWkmSoKJHIcw,6025
 tilebox/datasets/sync/timeseries.py,sha256=4nTP8_tmv6V7PXTUNzzlbzlxv0OXo_IqVLtSdJpUOW0,303
@@ -63,6 +63,6 @@ tilebox/datasets/tilebox/v1/id_pb2_grpc.py,sha256=xYOs94SXiNYAlFodACnsXW5QovLsHY
 tilebox/datasets/tilebox/v1/query_pb2.py,sha256=l60DA1setyQhdBbZ_jgG8Pw3ourUSxXWU5P8AACYlpk,3444
 tilebox/datasets/tilebox/v1/query_pb2.pyi,sha256=f-u60POkJqzssOmCEbOrD5fam9_86c6MdY_CzpnZZk0,2061
 tilebox/datasets/tilebox/v1/query_pb2_grpc.py,sha256=xYOs94SXiNYAlFodACnsXW5QovLsHY5tCk3p76RH5Zc,158
-tilebox_datasets-0.45.0.dist-info/METADATA,sha256=Gtgt3w-DipEQ5KYnux3b1rKY2W2xwggOF5lnAeOIZfo,4234
-tilebox_datasets-0.45.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-tilebox_datasets-0.45.0.dist-info/RECORD,,
+tilebox_datasets-0.47.0.dist-info/METADATA,sha256=-s7X2rgCr-xJHUZ51xMdPIgeOCjceUHadhaYpt-q-NM,4234
+tilebox_datasets-0.47.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+tilebox_datasets-0.47.0.dist-info/RECORD,,

{tilebox_datasets-0.45.0.dist-info → tilebox_datasets-0.47.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.27.0
+Generator: hatchling 1.28.0
 Root-Is-Purelib: true
 Tag: py3-none-any