tilebox-datasets 0.38.0__py3-none-any.whl → 0.39.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tilebox/datasets/aio/client.py +4 -4
- tilebox/datasets/aio/dataset.py +8 -13
- tilebox/datasets/aio/pagination.py +1 -3
- tilebox/datasets/client.py +1 -1
- tilebox/datasets/data/__init__.py +27 -15
- tilebox/datasets/data/collection.py +3 -3
- tilebox/datasets/data/data_access.py +6 -6
- tilebox/datasets/data/datapoint.py +7 -74
- tilebox/datasets/data/datasets.py +2 -2
- tilebox/datasets/data/time_interval.py +13 -232
- tilebox/datasets/data/timeseries.py +6 -6
- tilebox/datasets/datasets/v1/collections_pb2.py +49 -0
- tilebox/datasets/{datasetsv1 → datasets/v1}/collections_pb2.pyi +11 -10
- tilebox/datasets/{datasetsv1 → datasets/v1}/collections_pb2_grpc.py +2 -2
- tilebox/datasets/datasets/v1/core_pb2.py +79 -0
- tilebox/datasets/{datasetsv1 → datasets/v1}/core_pb2.pyi +13 -49
- tilebox/datasets/datasets/v1/data_access_pb2.py +65 -0
- tilebox/datasets/{datasetsv1 → datasets/v1}/data_access_pb2.pyi +18 -16
- tilebox/datasets/{datasetsv1 → datasets/v1}/data_access_pb2_grpc.py +2 -2
- tilebox/datasets/datasets/v1/data_ingestion_pb2.py +49 -0
- tilebox/datasets/{datasetsv1 → datasets/v1}/data_ingestion_pb2.pyi +11 -10
- tilebox/datasets/{datasetsv1 → datasets/v1}/data_ingestion_pb2_grpc.py +1 -1
- tilebox/datasets/datasets/v1/dataset_type_pb2.py +53 -0
- tilebox/datasets/{datasetsv1 → datasets/v1}/dataset_type_pb2.pyi +6 -5
- tilebox/datasets/datasets/v1/datasets_pb2.py +62 -0
- tilebox/datasets/{datasetsv1 → datasets/v1}/datasets_pb2.pyi +21 -8
- tilebox/datasets/{datasetsv1 → datasets/v1}/datasets_pb2_grpc.py +45 -2
- tilebox/datasets/datasets/v1/timeseries_pb2.py +42 -0
- tilebox/datasets/{datasetsv1 → datasets/v1}/timeseries_pb2.pyi +9 -8
- tilebox/datasets/{datasetsv1 → datasets/v1}/well_known_types_pb2.py +2 -2
- tilebox/datasets/message_pool.py +1 -1
- tilebox/datasets/progress.py +1 -1
- tilebox/datasets/protobuf_conversion/field_types.py +2 -2
- tilebox/datasets/query/__init__.py +8 -0
- tilebox/datasets/query/id_interval.py +72 -0
- tilebox/datasets/{data → query}/pagination.py +5 -5
- tilebox/datasets/query/time_interval.py +237 -0
- tilebox/datasets/service.py +13 -20
- tilebox/datasets/sync/client.py +4 -4
- tilebox/datasets/sync/dataset.py +7 -8
- tilebox/datasets/sync/pagination.py +1 -3
- tilebox/datasets/tilebox/v1/id_pb2.py +37 -0
- tilebox/datasets/tilebox/v1/id_pb2.pyi +11 -0
- tilebox/datasets/tilebox/v1/id_pb2_grpc.py +3 -0
- tilebox/datasets/tilebox/v1/query_pb2.py +47 -0
- tilebox/datasets/tilebox/v1/query_pb2.pyi +39 -0
- tilebox/datasets/tilebox/v1/query_pb2_grpc.py +3 -0
- tilebox/datasets/{data/uuid.py → uuid.py} +8 -7
- {tilebox_datasets-0.38.0.dist-info → tilebox_datasets-0.39.0.dist-info}/METADATA +1 -1
- tilebox_datasets-0.39.0.dist-info/RECORD +65 -0
- tilebox/datasets/datasetsv1/collections_pb2.py +0 -48
- tilebox/datasets/datasetsv1/core_pb2.py +0 -73
- tilebox/datasets/datasetsv1/data_access_pb2.py +0 -57
- tilebox/datasets/datasetsv1/data_ingestion_pb2.py +0 -48
- tilebox/datasets/datasetsv1/dataset_type_pb2.py +0 -52
- tilebox/datasets/datasetsv1/datasets_pb2.py +0 -55
- tilebox/datasets/datasetsv1/timeseries_pb2.py +0 -41
- tilebox_datasets-0.38.0.dist-info/RECORD +0 -56
- /tilebox/datasets/{datasetsv1 → datasets/v1}/core_pb2_grpc.py +0 -0
- /tilebox/datasets/{datasetsv1 → datasets/v1}/dataset_type_pb2_grpc.py +0 -0
- /tilebox/datasets/{datasetsv1 → datasets/v1}/timeseries_pb2_grpc.py +0 -0
- /tilebox/datasets/{datasetsv1 → datasets/v1}/well_known_types_pb2.pyi +0 -0
- /tilebox/datasets/{datasetsv1 → datasets/v1}/well_known_types_pb2_grpc.py +0 -0
- {tilebox_datasets-0.38.0.dist-info → tilebox_datasets-0.39.0.dist-info}/WHEEL +0 -0
tilebox/datasets/aio/client.py
CHANGED
|
@@ -5,10 +5,10 @@ from _tilebox.grpc.aio.error import with_pythonic_errors
|
|
|
5
5
|
from tilebox.datasets.aio.dataset import DatasetClient
|
|
6
6
|
from tilebox.datasets.client import Client as BaseClient
|
|
7
7
|
from tilebox.datasets.client import token_from_env
|
|
8
|
-
from tilebox.datasets.
|
|
9
|
-
from tilebox.datasets.
|
|
10
|
-
from tilebox.datasets.
|
|
11
|
-
from tilebox.datasets.
|
|
8
|
+
from tilebox.datasets.datasets.v1.collections_pb2_grpc import CollectionServiceStub
|
|
9
|
+
from tilebox.datasets.datasets.v1.data_access_pb2_grpc import DataAccessServiceStub
|
|
10
|
+
from tilebox.datasets.datasets.v1.data_ingestion_pb2_grpc import DataIngestionServiceStub
|
|
11
|
+
from tilebox.datasets.datasets.v1.datasets_pb2_grpc import DatasetServiceStub
|
|
12
12
|
from tilebox.datasets.group import Group
|
|
13
13
|
from tilebox.datasets.service import TileboxDatasetService
|
|
14
14
|
|
tilebox/datasets/aio/dataset.py
CHANGED
|
@@ -11,18 +11,11 @@ from _tilebox.grpc.aio.pagination import Pagination as PaginationProtocol
|
|
|
11
11
|
from _tilebox.grpc.aio.pagination import paginated_request
|
|
12
12
|
from _tilebox.grpc.aio.producer_consumer import async_producer_consumer
|
|
13
13
|
from _tilebox.grpc.error import ArgumentError, NotFoundError
|
|
14
|
-
from tilebox.datasets.aio.pagination import
|
|
15
|
-
with_progressbar,
|
|
16
|
-
with_time_progress_callback,
|
|
17
|
-
with_time_progressbar,
|
|
18
|
-
)
|
|
14
|
+
from tilebox.datasets.aio.pagination import with_progressbar, with_time_progress_callback, with_time_progressbar
|
|
19
15
|
from tilebox.datasets.data.collection import CollectionInfo
|
|
20
16
|
from tilebox.datasets.data.data_access import QueryFilters, SpatialFilter, SpatialFilterLike
|
|
21
|
-
from tilebox.datasets.data.datapoint import
|
|
17
|
+
from tilebox.datasets.data.datapoint import QueryResultPage
|
|
22
18
|
from tilebox.datasets.data.datasets import Dataset
|
|
23
|
-
from tilebox.datasets.data.pagination import Pagination
|
|
24
|
-
from tilebox.datasets.data.time_interval import TimeInterval, TimeIntervalLike
|
|
25
|
-
from tilebox.datasets.data.uuid import as_uuid
|
|
26
19
|
from tilebox.datasets.message_pool import get_message_type
|
|
27
20
|
from tilebox.datasets.progress import ProgressCallback
|
|
28
21
|
from tilebox.datasets.protobuf_conversion.protobuf_xarray import MessageToXarrayConverter
|
|
@@ -33,7 +26,11 @@ from tilebox.datasets.protobuf_conversion.to_protobuf import (
|
|
|
33
26
|
marshal_messages,
|
|
34
27
|
to_messages,
|
|
35
28
|
)
|
|
29
|
+
from tilebox.datasets.query.id_interval import IDInterval, IDIntervalLike
|
|
30
|
+
from tilebox.datasets.query.pagination import Pagination
|
|
31
|
+
from tilebox.datasets.query.time_interval import TimeInterval, TimeIntervalLike
|
|
36
32
|
from tilebox.datasets.service import TileboxDatasetService
|
|
33
|
+
from tilebox.datasets.uuid import as_uuid
|
|
37
34
|
|
|
38
35
|
# allow private member access: we allow it here because we want to make as much private as possible so that we can
|
|
39
36
|
# minimize the publicly facing API (which allows us to change internals later, and also limits to auto-completion)
|
|
@@ -240,7 +237,7 @@ class CollectionClient:
|
|
|
240
237
|
|
|
241
238
|
async def _find_interval(
|
|
242
239
|
self,
|
|
243
|
-
datapoint_id_interval:
|
|
240
|
+
datapoint_id_interval: IDIntervalLike,
|
|
244
241
|
end_inclusive: bool = True,
|
|
245
242
|
*,
|
|
246
243
|
skip_data: bool = False,
|
|
@@ -259,9 +256,7 @@ class CollectionClient:
|
|
|
259
256
|
Returns:
|
|
260
257
|
The datapoints in the given interval as an xarray dataset
|
|
261
258
|
"""
|
|
262
|
-
filters = QueryFilters(
|
|
263
|
-
temporal_extent=DatapointInterval.parse(datapoint_id_interval, end_inclusive=end_inclusive)
|
|
264
|
-
)
|
|
259
|
+
filters = QueryFilters(temporal_extent=IDInterval.parse(datapoint_id_interval, end_inclusive=end_inclusive))
|
|
265
260
|
|
|
266
261
|
async def request(page: PaginationProtocol) -> QueryResultPage:
|
|
267
262
|
query_page = Pagination(page.limit, page.starting_after)
|
|
@@ -5,11 +5,9 @@ from typing import TypeVar
|
|
|
5
5
|
|
|
6
6
|
from tqdm.auto import tqdm
|
|
7
7
|
|
|
8
|
-
from tilebox.datasets.data import (
|
|
9
|
-
TimeInterval,
|
|
10
|
-
)
|
|
11
8
|
from tilebox.datasets.data.datapoint import QueryResultPage
|
|
12
9
|
from tilebox.datasets.progress import ProgressCallback, TimeIntervalProgressBar
|
|
10
|
+
from tilebox.datasets.query.time_interval import TimeInterval
|
|
13
11
|
|
|
14
12
|
ResultPage = TypeVar("ResultPage", bound=QueryResultPage)
|
|
15
13
|
|
tilebox/datasets/client.py
CHANGED
|
@@ -8,10 +8,10 @@ from promise import Promise
|
|
|
8
8
|
|
|
9
9
|
from _tilebox.grpc.channel import parse_channel_info
|
|
10
10
|
from tilebox.datasets.data.datasets import Dataset, DatasetGroup, ListDatasetsResponse
|
|
11
|
-
from tilebox.datasets.data.uuid import as_uuid
|
|
12
11
|
from tilebox.datasets.group import Group
|
|
13
12
|
from tilebox.datasets.message_pool import register_once
|
|
14
13
|
from tilebox.datasets.service import TileboxDatasetService
|
|
14
|
+
from tilebox.datasets.uuid import as_uuid
|
|
15
15
|
|
|
16
16
|
|
|
17
17
|
class TimeseriesDatasetLike(Protocol):
|
|
@@ -1,16 +1,28 @@
|
|
|
1
|
-
|
|
2
|
-
TimeInterval,
|
|
3
|
-
TimeIntervalLike,
|
|
4
|
-
datetime_to_timestamp,
|
|
5
|
-
timestamp_to_datetime,
|
|
6
|
-
)
|
|
7
|
-
from tilebox.datasets.data.uuid import uuid_message_to_uuid, uuid_to_uuid_message
|
|
1
|
+
# for backwards compatibility, we can remove this hack in the future
|
|
8
2
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
3
|
+
from warnings import warn
|
|
4
|
+
|
|
5
|
+
from tilebox.datasets.query.time_interval import TimeInterval as _TimeInterval
|
|
6
|
+
from tilebox.datasets.query.time_interval import TimeIntervalLike
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class TimeInterval(_TimeInterval):
|
|
10
|
+
def __post_init__(self) -> None:
|
|
11
|
+
warn(
|
|
12
|
+
"The TimeInterval class has been deprecated, import from tilebox.datasets.query instead.",
|
|
13
|
+
DeprecationWarning,
|
|
14
|
+
stacklevel=2,
|
|
15
|
+
)
|
|
16
|
+
super().__post_init__()
|
|
17
|
+
|
|
18
|
+
@classmethod
|
|
19
|
+
def parse(cls, arg: TimeIntervalLike) -> "_TimeInterval":
|
|
20
|
+
warn(
|
|
21
|
+
"The TimeInterval class has been deprecated, import from tilebox.datasets.query instead.",
|
|
22
|
+
DeprecationWarning,
|
|
23
|
+
stacklevel=2,
|
|
24
|
+
)
|
|
25
|
+
return super().parse(arg)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
__all__ = ["TimeInterval"]
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
from dataclasses import dataclass
|
|
2
2
|
from uuid import UUID
|
|
3
3
|
|
|
4
|
-
from tilebox.datasets.
|
|
5
|
-
from tilebox.datasets.
|
|
6
|
-
from tilebox.datasets.
|
|
4
|
+
from tilebox.datasets.datasets.v1 import core_pb2
|
|
5
|
+
from tilebox.datasets.query.time_interval import TimeInterval
|
|
6
|
+
from tilebox.datasets.uuid import uuid_message_to_uuid, uuid_to_uuid_message
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
@dataclass
|
|
@@ -7,9 +7,9 @@ from shapely import Geometry, from_wkb, to_wkb
|
|
|
7
7
|
# from python 3.11 onwards this is available as typing.NotRequired:
|
|
8
8
|
from typing_extensions import NotRequired
|
|
9
9
|
|
|
10
|
-
from tilebox.datasets.
|
|
11
|
-
from tilebox.datasets.
|
|
12
|
-
from tilebox.datasets.
|
|
10
|
+
from tilebox.datasets.datasets.v1 import data_access_pb2, well_known_types_pb2
|
|
11
|
+
from tilebox.datasets.query.id_interval import IDInterval
|
|
12
|
+
from tilebox.datasets.query.time_interval import TimeInterval
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
class SpatialFilterMode(Enum):
|
|
@@ -104,16 +104,16 @@ class SpatialFilter:
|
|
|
104
104
|
|
|
105
105
|
@dataclass(frozen=True)
|
|
106
106
|
class QueryFilters:
|
|
107
|
-
temporal_extent: TimeInterval |
|
|
107
|
+
temporal_extent: TimeInterval | IDInterval
|
|
108
108
|
spatial_extent: SpatialFilter | None = None
|
|
109
109
|
|
|
110
110
|
@classmethod
|
|
111
111
|
def from_message(cls, filters: data_access_pb2.QueryFilters) -> "QueryFilters":
|
|
112
|
-
temporal_extent: TimeInterval |
|
|
112
|
+
temporal_extent: TimeInterval | IDInterval | None = None
|
|
113
113
|
if filters.HasField("time_interval"):
|
|
114
114
|
temporal_extent = TimeInterval.from_message(filters.time_interval)
|
|
115
115
|
if filters.HasField("datapoint_interval"):
|
|
116
|
-
temporal_extent =
|
|
116
|
+
temporal_extent = IDInterval.from_message(filters.datapoint_interval)
|
|
117
117
|
|
|
118
118
|
if temporal_extent is None:
|
|
119
119
|
raise ValueError("Invalid filter: time or datapoint interval must be set")
|
|
@@ -1,81 +1,14 @@
|
|
|
1
1
|
from dataclasses import dataclass, field
|
|
2
2
|
from datetime import datetime
|
|
3
|
-
from typing import Any
|
|
3
|
+
from typing import Any
|
|
4
4
|
from uuid import UUID
|
|
5
5
|
|
|
6
|
-
from tilebox.datasets.
|
|
7
|
-
from tilebox.datasets.data.time_interval import timestamp_to_datetime
|
|
8
|
-
from tilebox.datasets.data.uuid import uuid_message_to_uuid, uuid_to_uuid_message
|
|
9
|
-
from tilebox.datasets.datasetsv1 import core_pb2, data_access_pb2, data_ingestion_pb2
|
|
6
|
+
from tilebox.datasets.datasets.v1 import core_pb2, data_access_pb2, data_ingestion_pb2
|
|
10
7
|
from tilebox.datasets.message_pool import get_message_type
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
@dataclass(frozen=True)
|
|
16
|
-
class DatapointInterval:
|
|
17
|
-
start_id: UUID
|
|
18
|
-
end_id: UUID
|
|
19
|
-
start_exclusive: bool
|
|
20
|
-
end_inclusive: bool
|
|
21
|
-
|
|
22
|
-
@classmethod
|
|
23
|
-
def from_message(cls, interval: core_pb2.DatapointInterval) -> "DatapointInterval":
|
|
24
|
-
return cls(
|
|
25
|
-
start_id=uuid_message_to_uuid(interval.start_id),
|
|
26
|
-
end_id=uuid_message_to_uuid(interval.end_id),
|
|
27
|
-
start_exclusive=interval.start_exclusive,
|
|
28
|
-
end_inclusive=interval.end_inclusive,
|
|
29
|
-
)
|
|
30
|
-
|
|
31
|
-
def to_message(self) -> core_pb2.DatapointInterval:
|
|
32
|
-
return core_pb2.DatapointInterval(
|
|
33
|
-
start_id=uuid_to_uuid_message(self.start_id),
|
|
34
|
-
end_id=uuid_to_uuid_message(self.end_id),
|
|
35
|
-
start_exclusive=self.start_exclusive,
|
|
36
|
-
end_inclusive=self.end_inclusive,
|
|
37
|
-
)
|
|
38
|
-
|
|
39
|
-
@classmethod
|
|
40
|
-
def parse(
|
|
41
|
-
cls, arg: DatapointIntervalLike, start_exclusive: bool = False, end_inclusive: bool = True
|
|
42
|
-
) -> "DatapointInterval":
|
|
43
|
-
"""
|
|
44
|
-
Convert a variety of input types to a DatapointInterval.
|
|
45
|
-
|
|
46
|
-
Supported input types:
|
|
47
|
-
- DatapointInterval: Return the input as is
|
|
48
|
-
- tuple of two UUIDs: Return an DatapointInterval with start and end id set to the given values
|
|
49
|
-
- tuple of two strings: Return an DatapointInterval with start and end id set to the UUIDs parsed from the given strings
|
|
50
|
-
|
|
51
|
-
Args:
|
|
52
|
-
arg: The input to convert
|
|
53
|
-
start_exclusive: Whether the start id is exclusive
|
|
54
|
-
end_inclusive: Whether the end id is inclusive
|
|
55
|
-
|
|
56
|
-
Returns:
|
|
57
|
-
DatapointInterval: The parsed ID interval
|
|
58
|
-
"""
|
|
59
|
-
|
|
60
|
-
match arg:
|
|
61
|
-
case DatapointInterval(_, _, _, _):
|
|
62
|
-
return arg
|
|
63
|
-
case (UUID(), UUID()):
|
|
64
|
-
start, end = arg
|
|
65
|
-
return DatapointInterval(
|
|
66
|
-
start_id=start,
|
|
67
|
-
end_id=end,
|
|
68
|
-
start_exclusive=start_exclusive,
|
|
69
|
-
end_inclusive=end_inclusive,
|
|
70
|
-
)
|
|
71
|
-
case (str(), str()):
|
|
72
|
-
start, end = arg
|
|
73
|
-
return DatapointInterval(
|
|
74
|
-
start_id=UUID(start),
|
|
75
|
-
end_id=UUID(end),
|
|
76
|
-
start_exclusive=start_exclusive,
|
|
77
|
-
end_inclusive=end_inclusive,
|
|
78
|
-
)
|
|
8
|
+
from tilebox.datasets.query.pagination import Pagination
|
|
9
|
+
from tilebox.datasets.query.time_interval import timestamp_to_datetime
|
|
10
|
+
from tilebox.datasets.tilebox.v1 import id_pb2
|
|
11
|
+
from tilebox.datasets.uuid import uuid_message_to_uuid
|
|
79
12
|
|
|
80
13
|
|
|
81
14
|
@dataclass(frozen=True)
|
|
@@ -173,5 +106,5 @@ class IngestResponse:
|
|
|
173
106
|
return data_ingestion_pb2.IngestResponse(
|
|
174
107
|
num_created=self.num_created,
|
|
175
108
|
num_existing=self.num_existing,
|
|
176
|
-
datapoint_ids=[
|
|
109
|
+
datapoint_ids=[id_pb2.ID(uuid=datapoint_id.bytes) for datapoint_id in self.datapoint_ids],
|
|
177
110
|
)
|
|
@@ -3,8 +3,8 @@ from uuid import UUID
|
|
|
3
3
|
|
|
4
4
|
from google.protobuf.descriptor_pb2 import FileDescriptorSet
|
|
5
5
|
|
|
6
|
-
from tilebox.datasets.
|
|
7
|
-
from tilebox.datasets.
|
|
6
|
+
from tilebox.datasets.datasets.v1 import core_pb2, dataset_type_pb2, datasets_pb2
|
|
7
|
+
from tilebox.datasets.uuid import uuid_message_to_optional_uuid, uuid_message_to_uuid, uuid_to_uuid_message
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
@dataclass(frozen=True)
|
|
@@ -1,237 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
from datetime import datetime, timedelta, timezone
|
|
3
|
-
from typing import TypeAlias
|
|
1
|
+
# kept for backwards compatibility, we can remove this whole file in the future
|
|
4
2
|
|
|
5
|
-
|
|
6
|
-
import xarray as xr
|
|
7
|
-
from google.protobuf.duration_pb2 import Duration
|
|
8
|
-
from google.protobuf.timestamp_pb2 import Timestamp
|
|
9
|
-
from pandas.core.tools.datetimes import DatetimeScalar, to_datetime
|
|
3
|
+
from warnings import warn
|
|
10
4
|
|
|
11
|
-
from tilebox.datasets.
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
# A type alias for the different types that can be used to specify a time interval
|
|
17
|
-
TimeIntervalLike: TypeAlias = (
|
|
18
|
-
DatetimeScalar | tuple[DatetimeScalar, DatetimeScalar] | xr.DataArray | xr.Dataset | "TimeInterval"
|
|
5
|
+
from tilebox.datasets.query.time_interval import (
|
|
6
|
+
TimeInterval,
|
|
7
|
+
TimeIntervalLike,
|
|
8
|
+
datetime_to_timestamp,
|
|
9
|
+
timestamp_to_datetime,
|
|
19
10
|
)
|
|
20
11
|
|
|
12
|
+
warn(
|
|
13
|
+
"The time_interval module has been deprecated, import from tilebox.datasets.query instead.",
|
|
14
|
+
DeprecationWarning,
|
|
15
|
+
stacklevel=2,
|
|
16
|
+
)
|
|
21
17
|
|
|
22
|
-
|
|
23
|
-
class TimeInterval:
|
|
24
|
-
"""
|
|
25
|
-
A time interval from a given start to a given end time.
|
|
26
|
-
Both the start and end time can be exclusive or inclusive.
|
|
27
|
-
"""
|
|
28
|
-
|
|
29
|
-
start: datetime
|
|
30
|
-
end: datetime
|
|
31
|
-
|
|
32
|
-
# We use exclusive for start and inclusive for end, because that way when both are false
|
|
33
|
-
# we have a half-open interval [start, end) which is the default behaviour we want to achieve.
|
|
34
|
-
start_exclusive: bool = False
|
|
35
|
-
end_inclusive: bool = False
|
|
36
|
-
|
|
37
|
-
def __post_init__(self) -> None:
|
|
38
|
-
"""Validate the time interval"""
|
|
39
|
-
if not isinstance(self.start, datetime):
|
|
40
|
-
raise TypeError(f"Start time {self.start} must be a datetime object")
|
|
41
|
-
if not isinstance(self.end, datetime):
|
|
42
|
-
raise TypeError(f"End time {self.end} must be a datetime object")
|
|
43
|
-
|
|
44
|
-
# in case datetime objects are timezone naive, we assume they are in UTC
|
|
45
|
-
if self.start.tzinfo is None:
|
|
46
|
-
object.__setattr__(self, "start", self.start.replace(tzinfo=timezone.utc)) # since self is frozen
|
|
47
|
-
if self.end.tzinfo is None:
|
|
48
|
-
object.__setattr__(self, "end", self.end.replace(tzinfo=timezone.utc)) # since self is frozen
|
|
49
|
-
|
|
50
|
-
def to_half_open(self) -> "TimeInterval":
|
|
51
|
-
"""Convert the time interval to a half-open interval [start, end)"""
|
|
52
|
-
return TimeInterval(
|
|
53
|
-
start=self.start + int(self.start_exclusive) * _SMALLEST_POSSIBLE_TIMEDELTA,
|
|
54
|
-
end=self.end + int(self.end_inclusive) * _SMALLEST_POSSIBLE_TIMEDELTA,
|
|
55
|
-
start_exclusive=False,
|
|
56
|
-
end_inclusive=False,
|
|
57
|
-
)
|
|
58
|
-
|
|
59
|
-
def astimezone(self, tzinfo: timezone) -> "TimeInterval":
|
|
60
|
-
"""Convert start and end time to the given timezone"""
|
|
61
|
-
return TimeInterval(
|
|
62
|
-
start=self.start.astimezone(tzinfo),
|
|
63
|
-
end=self.end.astimezone(tzinfo),
|
|
64
|
-
start_exclusive=self.start_exclusive,
|
|
65
|
-
end_inclusive=self.end_inclusive,
|
|
66
|
-
)
|
|
67
|
-
|
|
68
|
-
def __eq__(self, other: object) -> bool:
|
|
69
|
-
"""Check whether two intervals are equal"""
|
|
70
|
-
if not isinstance(other, TimeInterval):
|
|
71
|
-
return False
|
|
72
|
-
|
|
73
|
-
a, b = self.to_half_open(), other.to_half_open()
|
|
74
|
-
return (a.start, a.end) == (b.start, b.end)
|
|
75
|
-
|
|
76
|
-
def __hash__(self) -> int:
|
|
77
|
-
"""Hash the time interval"""
|
|
78
|
-
|
|
79
|
-
# if two intervals are equal, they should have the same hash, so we convert to half-open intervals first
|
|
80
|
-
half_open = self.to_half_open()
|
|
81
|
-
return hash((half_open.start, half_open.end))
|
|
82
|
-
|
|
83
|
-
def __repr__(self) -> str:
|
|
84
|
-
return self.format()
|
|
85
|
-
|
|
86
|
-
def format(self, endpoints: bool = True, sep: str = ", ", timespec: str = "milliseconds") -> str:
|
|
87
|
-
"""Human readable representation of the time interval
|
|
88
|
-
|
|
89
|
-
Args:
|
|
90
|
-
endpoints: Whether to format as interval using scientific interval notation [start, end].
|
|
91
|
-
"[", "]" for closed intervals and "(", ")" for open intervals.
|
|
92
|
-
sep: The separator to use between the start and end of the interval.
|
|
93
|
-
timespec: Specifies the number of additional terms of the time to include. Valid options are 'auto',
|
|
94
|
-
'hours', 'minutes', 'seconds', 'milliseconds' and 'microseconds'.
|
|
95
|
-
"""
|
|
96
|
-
if self == _EMPTY_TIME_INTERVAL:
|
|
97
|
-
return "<empty>"
|
|
98
|
-
|
|
99
|
-
start = self.start.isoformat(timespec=timespec).replace("+00:00", " UTC")
|
|
100
|
-
end = self.end.isoformat(timespec=timespec).replace("+00:00", " UTC")
|
|
101
|
-
|
|
102
|
-
formatted = f"{start}{sep}{end}"
|
|
103
|
-
if endpoints:
|
|
104
|
-
start_ch = "[" if not self.start_exclusive else "("
|
|
105
|
-
end_ch = "]" if self.end_inclusive else ")"
|
|
106
|
-
formatted = f"{start_ch}{formatted}{end_ch}"
|
|
107
|
-
return formatted
|
|
108
|
-
|
|
109
|
-
def __str__(self) -> str:
|
|
110
|
-
"""Human readable representation of the time interval"""
|
|
111
|
-
return self.format()
|
|
112
|
-
|
|
113
|
-
@classmethod
|
|
114
|
-
def parse(cls, arg: TimeIntervalLike) -> "TimeInterval":
|
|
115
|
-
"""
|
|
116
|
-
Convert a variety of input types to a TimeInterval.
|
|
117
|
-
|
|
118
|
-
Supported input types:
|
|
119
|
-
- TimeInterval: Return the input as is
|
|
120
|
-
- DatetimeScalar: Return a TimeInterval with start and end time set to the same value and the end time inclusive
|
|
121
|
-
- tuple of two DatetimeScalar: Return a TimeInterval with start and end time
|
|
122
|
-
- xr.DataArray: Return a TimeInterval with start and end time set to the first and last value in the array and
|
|
123
|
-
the end time inclusive
|
|
124
|
-
- xr.Dataset: Return a TimeInterval with start and end time set to the first and last value in the time
|
|
125
|
-
coordinate of the dataset and the end time inclusive
|
|
126
|
-
|
|
127
|
-
Args:
|
|
128
|
-
arg: The input to convert
|
|
129
|
-
|
|
130
|
-
Returns:
|
|
131
|
-
TimeInterval: The parsed time interval
|
|
132
|
-
"""
|
|
133
|
-
|
|
134
|
-
match arg:
|
|
135
|
-
case TimeInterval(_, _, _, _):
|
|
136
|
-
return arg
|
|
137
|
-
case (start, end):
|
|
138
|
-
return TimeInterval(start=_convert_to_datetime(start), end=_convert_to_datetime(end))
|
|
139
|
-
case point_in_time if isinstance(point_in_time, DatetimeScalar | int):
|
|
140
|
-
dt = _convert_to_datetime(point_in_time)
|
|
141
|
-
return TimeInterval(start=dt, end=dt, start_exclusive=False, end_inclusive=True)
|
|
142
|
-
case arr if (
|
|
143
|
-
isinstance(arr, xr.DataArray)
|
|
144
|
-
and arr.ndim == 1
|
|
145
|
-
and arr.size > 0
|
|
146
|
-
and arr.dtype == np.dtype("datetime64[ns]")
|
|
147
|
-
):
|
|
148
|
-
start = arr.data[0]
|
|
149
|
-
end = arr.data[-1]
|
|
150
|
-
return TimeInterval(
|
|
151
|
-
start=_convert_to_datetime(start),
|
|
152
|
-
end=_convert_to_datetime(end),
|
|
153
|
-
start_exclusive=False,
|
|
154
|
-
end_inclusive=True,
|
|
155
|
-
)
|
|
156
|
-
case ds if isinstance(ds, xr.Dataset) and "time" in ds.coords:
|
|
157
|
-
return cls.parse(ds.time)
|
|
158
|
-
|
|
159
|
-
raise ValueError(f"Failed to convert {arg} ({type(arg)}) to TimeInterval)")
|
|
160
|
-
|
|
161
|
-
@classmethod
|
|
162
|
-
def from_message(
|
|
163
|
-
cls, interval: core_pb2.TimeInterval
|
|
164
|
-
) -> "TimeInterval": # lets use typing.Self once we require python >= 3.11
|
|
165
|
-
"""Convert a TimeInterval protobuf message to a TimeInterval object."""
|
|
166
|
-
|
|
167
|
-
start = timestamp_to_datetime(interval.start_time)
|
|
168
|
-
end = timestamp_to_datetime(interval.end_time)
|
|
169
|
-
if start == _EPOCH and end == _EPOCH and not interval.start_exclusive and not interval.end_inclusive:
|
|
170
|
-
return _EMPTY_TIME_INTERVAL
|
|
171
|
-
|
|
172
|
-
return cls(
|
|
173
|
-
start=timestamp_to_datetime(interval.start_time),
|
|
174
|
-
end=timestamp_to_datetime(interval.end_time),
|
|
175
|
-
start_exclusive=interval.start_exclusive,
|
|
176
|
-
end_inclusive=interval.end_inclusive,
|
|
177
|
-
)
|
|
178
|
-
|
|
179
|
-
def to_message(self) -> core_pb2.TimeInterval:
|
|
180
|
-
"""Convert a TimeInterval object to a TimeInterval protobuf message."""
|
|
181
|
-
return core_pb2.TimeInterval(
|
|
182
|
-
start_time=datetime_to_timestamp(self.start),
|
|
183
|
-
end_time=datetime_to_timestamp(self.end),
|
|
184
|
-
start_exclusive=self.start_exclusive,
|
|
185
|
-
end_inclusive=self.end_inclusive,
|
|
186
|
-
)
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
# A sentinel value to use for time interval that are empty
|
|
190
|
-
_EMPTY_TIME_INTERVAL = TimeInterval(_EPOCH, _EPOCH, start_exclusive=True, end_inclusive=False)
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
def _convert_to_datetime(arg: DatetimeScalar) -> datetime:
|
|
194
|
-
"""Convert the given datetime scalar to a datetime object in the UTC timezone"""
|
|
195
|
-
dt: datetime = to_datetime(arg, utc=True).to_pydatetime()
|
|
196
|
-
if dt.tzinfo is None:
|
|
197
|
-
dt = dt.replace(tzinfo=timezone.utc)
|
|
198
|
-
return dt.astimezone(timezone.utc)
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
def timestamp_to_datetime(timestamp: Timestamp) -> datetime:
|
|
202
|
-
"""Convert a protobuf timestamp to a datetime object."""
|
|
203
|
-
# datetime.fromtimestamp() expects a timestamp in seconds, not microseconds
|
|
204
|
-
# if we pass it as a floating point number, we will run into rounding errors
|
|
205
|
-
offset = timedelta(seconds=timestamp.seconds, microseconds=timestamp.nanos // 1000)
|
|
206
|
-
return _EPOCH + offset
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
def datetime_to_timestamp(dt: datetime) -> Timestamp:
|
|
210
|
-
"""Convert a datetime object to a protobuf timestamp."""
|
|
211
|
-
# manual epoch offset calculation to avoid rounding errors and support negative timestamps (before 1970)
|
|
212
|
-
offset_us = datetime_to_us(dt.astimezone(timezone.utc))
|
|
213
|
-
seconds, us = divmod(offset_us, 10**6)
|
|
214
|
-
return Timestamp(seconds=seconds, nanos=us * 10**3)
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
def datetime_to_us(dt: datetime) -> int:
|
|
218
|
-
"""Convert a datetime object to a timestamp in microseconds since the epoch."""
|
|
219
|
-
offset = dt - _EPOCH
|
|
220
|
-
# implementation taken from timedelta.total_seconds() but without dividing by 10**6 at the end to avoid floating
|
|
221
|
-
# point rounding errors
|
|
222
|
-
return (offset.days * 86400 + offset.seconds) * 10**6 + offset.microseconds
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
def us_to_datetime(us: int) -> datetime:
|
|
226
|
-
"""Convert a timestamp in microseconds since the epoch to a datetime object."""
|
|
227
|
-
return _EPOCH + timedelta(microseconds=us)
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
def timedelta_to_duration(td: timedelta) -> Duration:
|
|
231
|
-
"""Convert a timedelta to a duration protobuf message."""
|
|
232
|
-
return Duration(seconds=int(td.total_seconds()), nanos=int(td.microseconds * 1000))
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
def duration_to_timedelta(duration: Duration) -> timedelta:
|
|
236
|
-
"""Convert a duration protobuf message to a timedelta."""
|
|
237
|
-
return timedelta(seconds=duration.seconds, microseconds=duration.nanos // 1000)
|
|
18
|
+
__all__ = ["TimeInterval", "TimeIntervalLike", "datetime_to_timestamp", "timestamp_to_datetime"]
|
|
@@ -2,10 +2,10 @@ from dataclasses import dataclass
|
|
|
2
2
|
from datetime import timedelta
|
|
3
3
|
from uuid import UUID
|
|
4
4
|
|
|
5
|
-
from tilebox.datasets.
|
|
6
|
-
from tilebox.datasets.
|
|
7
|
-
from tilebox.datasets.
|
|
8
|
-
from tilebox.datasets.
|
|
5
|
+
from tilebox.datasets.datasets.v1 import timeseries_pb2
|
|
6
|
+
from tilebox.datasets.query.id_interval import IDInterval
|
|
7
|
+
from tilebox.datasets.query.time_interval import TimeInterval, duration_to_timedelta, timedelta_to_duration
|
|
8
|
+
from tilebox.datasets.uuid import uuid_message_to_uuid, uuid_to_uuid_message
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
@dataclass
|
|
@@ -13,7 +13,7 @@ class TimeseriesDatasetChunk:
|
|
|
13
13
|
dataset_id: UUID
|
|
14
14
|
collection_id: UUID
|
|
15
15
|
time_interval: TimeInterval | None
|
|
16
|
-
datapoint_interval:
|
|
16
|
+
datapoint_interval: IDInterval | None
|
|
17
17
|
branch_factor: int
|
|
18
18
|
chunk_size: int
|
|
19
19
|
datapoints_per_365_days: int
|
|
@@ -28,7 +28,7 @@ class TimeseriesDatasetChunk:
|
|
|
28
28
|
and chunk.datapoint_interval.start_id.uuid
|
|
29
29
|
and chunk.datapoint_interval.end_id.uuid
|
|
30
30
|
):
|
|
31
|
-
datapoint_interval =
|
|
31
|
+
datapoint_interval = IDInterval.from_message(chunk.datapoint_interval)
|
|
32
32
|
|
|
33
33
|
time_interval = None
|
|
34
34
|
if chunk.time_interval and chunk.time_interval.start_time and chunk.time_interval.end_time:
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
|
+
# NO CHECKED-IN PROTOBUF GENCODE
|
|
4
|
+
# source: datasets/v1/collections.proto
|
|
5
|
+
# Protobuf Python Version: 5.29.3
|
|
6
|
+
"""Generated protocol buffer code."""
|
|
7
|
+
from google.protobuf import descriptor as _descriptor
|
|
8
|
+
from google.protobuf import descriptor_pool as _descriptor_pool
|
|
9
|
+
from google.protobuf import runtime_version as _runtime_version
|
|
10
|
+
from google.protobuf import symbol_database as _symbol_database
|
|
11
|
+
from google.protobuf.internal import builder as _builder
|
|
12
|
+
_runtime_version.ValidateProtobufRuntimeVersion(
|
|
13
|
+
_runtime_version.Domain.PUBLIC,
|
|
14
|
+
5,
|
|
15
|
+
29,
|
|
16
|
+
3,
|
|
17
|
+
'',
|
|
18
|
+
'datasets/v1/collections.proto'
|
|
19
|
+
)
|
|
20
|
+
# @@protoc_insertion_point(imports)
|
|
21
|
+
|
|
22
|
+
_sym_db = _symbol_database.Default()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
from tilebox.datasets.datasets.v1 import core_pb2 as datasets_dot_v1_dot_core__pb2
|
|
26
|
+
from tilebox.datasets.tilebox.v1 import id_pb2 as tilebox_dot_v1_dot_id__pb2
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x64\x61tasets/v1/collections.proto\x12\x0b\x64\x61tasets.v1\x1a\x16\x64\x61tasets/v1/core.proto\x1a\x13tilebox/v1/id.proto\"\\\n\x17\x43reateCollectionRequest\x12-\n\ndataset_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\tdatasetId\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\"\xc0\x01\n\x1aGetCollectionByNameRequest\x12\'\n\x0f\x63ollection_name\x18\x01 \x01(\tR\x0e\x63ollectionName\x12+\n\x11with_availability\x18\x02 \x01(\x08R\x10withAvailability\x12\x1d\n\nwith_count\x18\x03 \x01(\x08R\twithCount\x12-\n\ndataset_id\x18\x04 \x01(\x0b\x32\x0e.tilebox.v1.IDR\tdatasetId\"}\n\x17\x44\x65leteCollectionRequest\x12\x33\n\rcollection_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\x0c\x63ollectionId\x12-\n\ndataset_id\x18\x02 \x01(\x0b\x32\x0e.tilebox.v1.IDR\tdatasetId\"\x1a\n\x18\x44\x65leteCollectionResponse\"\x93\x01\n\x16ListCollectionsRequest\x12-\n\ndataset_id\x18\x01 \x01(\x0b\x32\x0e.tilebox.v1.IDR\tdatasetId\x12+\n\x11with_availability\x18\x02 \x01(\x08R\x10withAvailability\x12\x1d\n\nwith_count\x18\x03 \x01(\x08R\twithCount2\x86\x03\n\x11\x43ollectionService\x12W\n\x10\x43reateCollection\x12$.datasets.v1.CreateCollectionRequest\x1a\x1b.datasets.v1.CollectionInfo\"\x00\x12]\n\x13GetCollectionByName\x12\'.datasets.v1.GetCollectionByNameRequest\x1a\x1b.datasets.v1.CollectionInfo\"\x00\x12\x61\n\x10\x44\x65leteCollection\x12$.datasets.v1.DeleteCollectionRequest\x1a%.datasets.v1.DeleteCollectionResponse\"\x00\x12V\n\x0fListCollections\x12#.datasets.v1.ListCollectionsRequest\x1a\x1c.datasets.v1.CollectionInfos\"\x00\x42u\n\x0f\x63om.datasets.v1B\x10\x43ollectionsProtoP\x01\xa2\x02\x03\x44XX\xaa\x02\x0b\x44\x61tasets.V1\xca\x02\x0b\x44\x61tasets\\V1\xe2\x02\x17\x44\x61tasets\\V1\\GPBMetadata\xea\x02\x0c\x44\x61tasets::V1\x92\x03\x02\x08\x02\x62\x08\x65\x64itionsp\xe8\x07')
|
|
30
|
+
|
|
31
|
+
_globals = globals()
|
|
32
|
+
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
|
33
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'datasets.v1.collections_pb2', _globals)
|
|
34
|
+
if not _descriptor._USE_C_DESCRIPTORS:
|
|
35
|
+
_globals['DESCRIPTOR']._loaded_options = None
|
|
36
|
+
_globals['DESCRIPTOR']._serialized_options = b'\n\017com.datasets.v1B\020CollectionsProtoP\001\242\002\003DXX\252\002\013Datasets.V1\312\002\013Datasets\\V1\342\002\027Datasets\\V1\\GPBMetadata\352\002\014Datasets::V1\222\003\002\010\002'
|
|
37
|
+
_globals['_CREATECOLLECTIONREQUEST']._serialized_start=91
|
|
38
|
+
_globals['_CREATECOLLECTIONREQUEST']._serialized_end=183
|
|
39
|
+
_globals['_GETCOLLECTIONBYNAMEREQUEST']._serialized_start=186
|
|
40
|
+
_globals['_GETCOLLECTIONBYNAMEREQUEST']._serialized_end=378
|
|
41
|
+
_globals['_DELETECOLLECTIONREQUEST']._serialized_start=380
|
|
42
|
+
_globals['_DELETECOLLECTIONREQUEST']._serialized_end=505
|
|
43
|
+
_globals['_DELETECOLLECTIONRESPONSE']._serialized_start=507
|
|
44
|
+
_globals['_DELETECOLLECTIONRESPONSE']._serialized_end=533
|
|
45
|
+
_globals['_LISTCOLLECTIONSREQUEST']._serialized_start=536
|
|
46
|
+
_globals['_LISTCOLLECTIONSREQUEST']._serialized_end=683
|
|
47
|
+
_globals['_COLLECTIONSERVICE']._serialized_start=686
|
|
48
|
+
_globals['_COLLECTIONSERVICE']._serialized_end=1076
|
|
49
|
+
# @@protoc_insertion_point(module_scope)
|