polytope-python 1.0.31__py3-none-any.whl

polytope_feature/datacube/backends/xarray.py

@@ -0,0 +1,142 @@
+from copy import deepcopy
+
+import numpy as np
+import xarray as xr
+
+from .datacube import Datacube
+
+
+class XArrayDatacube(Datacube):
+    """Xarray arrays are labelled, axes can be defined as strings or integers (e.g. "time" or 0)."""
+
+    def __init__(self, dataarray: xr.DataArray, axis_options=None, compressed_axes_options=[], context=None):
+        super().__init__(axis_options, compressed_axes_options)
+        if axis_options is None:
+            axis_options = {}
+        self.axis_options = axis_options
+        self.axis_counter = 0
+        self._axes = None
+        self.dataarray = dataarray
+
+        for name, values in dataarray.coords.variables.items():
+            options = None
+            for opt in self.axis_options:
+                if opt.axis_name == name:
+                    options = opt
+            if name in dataarray.dims:
+                self._check_and_add_axes(options, name, values)
+                self.treated_axes.append(name)
+                self.complete_axes.append(name)
+            else:
+                if self.dataarray[name].dims == ():
+                    self._check_and_add_axes(options, name, values)
+                    self.treated_axes.append(name)
+        for name in dataarray.dims:
+            if name not in self.treated_axes:
+                options = None
+                for opt in self.axis_options:
+                    if opt.axis_name == name:
+                        options = opt
+                val = dataarray[name].values[0]
+                self._check_and_add_axes(options, name, val)
+                self.treated_axes.append(name)
+        # add other options to axis which were just created above like "lat" for the mapper transformations for eg
+        for name in self._axes:
+            if name not in self.treated_axes:
+                options = None
+                for opt in self.axis_options:
+                    if opt.axis_name == name:
+                        options = opt
+                val = self._axes[name].type
+                self._check_and_add_axes(options, name, val)
+
+    def get(self, requests, context=None, leaf_path=None, axis_counter=0):
+        if context is None:
+            context = {}
+        if leaf_path is None:
+            leaf_path = {}
+        if requests.axis.name == "root":
+            for c in requests.children:
+                self.get(c, context, leaf_path, axis_counter + 1)
+        else:
+            key_value_path = {requests.axis.name: requests.values}
+            ax = requests.axis
+            (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key(
+                key_value_path, leaf_path, self.unwanted_path
+            )
+            leaf_path.update(key_value_path)
+            if len(requests.children) != 0:
+                # We are not a leaf and we loop over
+                for c in requests.children:
+                    self.get(c, context, leaf_path, axis_counter + 1)
+            else:
+                if self.axis_counter != axis_counter:
+                    requests.remove_branch()
+                else:
+                    # We are at a leaf and need to assign value to it
+                    leaf_path_copy = deepcopy(leaf_path)
+                    unmapped_path = {}
+                    self.refit_path(leaf_path_copy, unmapped_path, leaf_path)
+                    for key in leaf_path_copy:
+                        leaf_path_copy[key] = list(leaf_path_copy[key])
+                    for key in unmapped_path:
+                        if isinstance(unmapped_path[key], tuple):
+                            unmapped_path[key] = list(unmapped_path[key])
+                    subxarray = self.dataarray.sel(leaf_path_copy, method="nearest")
+                    subxarray = subxarray.sel(unmapped_path)
+                    value = subxarray.values
+                    key = subxarray.name
+                    requests.result = (key, value)
+
+    def datacube_natural_indexes(self, axis, subarray):
+        if axis.name in self.complete_axes:
+            indexes = next(iter(subarray.xindexes.values())).to_pandas_index()
+        else:
+            if subarray[axis.name].values.ndim == 0:
+                # NOTE how we handle the two special datetime and timedelta cases to conform with numpy arrays
+                if np.issubdtype(subarray[axis.name].values.dtype, np.datetime64):
+                    indexes = [subarray[axis.name].astype("datetime64[us]").values]
+                elif np.issubdtype(subarray[axis.name].values.dtype, np.timedelta64):
+                    indexes = [subarray[axis.name].astype("timedelta64[us]").values]
+                else:
+                    indexes = [subarray[axis.name].values.tolist()]
+            else:
+                indexes = subarray[axis.name].values
+        return indexes
+
+    def refit_path(self, path_copy, unmapped_path, path):
+        for key in path.keys():
+            if key not in self.dataarray.dims:
+                path_copy.pop(key)
+            if key not in self.dataarray.coords.dtypes:
+                unmapped_path.update({key: path[key]})
+                path_copy.pop(key)
+            for key in self.dataarray.coords.dtypes:
+                key_dtype = self.dataarray.coords.dtypes[key]
+                if key_dtype.type is np.str_ and key in path.keys():
+                    unmapped_path.update({key: path[key]})
+                    path_copy.pop(key, None)
+
+    def select(self, path, unmapped_path):
+        for key in path:
+            key_value = path[key][0]
+            path[key] = key_value
+        for key in unmapped_path:
+            key_value = unmapped_path[key][0]
+            unmapped_path[key] = key_value
+        path_copy = deepcopy(path)
+        self.refit_path(path_copy, unmapped_path, path)
+        subarray = self.dataarray.sel(path_copy, method="nearest")
+        subarray = subarray.sel(unmapped_path)
+        return subarray
+
+    def ax_vals(self, name):
+        treated_axes = []
+        for _name, values in self.dataarray.coords.variables.items():
+            treated_axes.append(_name)
+            if _name == name:
+                return values.values
+        for _name in self.dataarray.dims:
+            if _name not in treated_axes:
+                if _name == name:
+                    return self.dataarray[name].values[0]

polytope_feature/datacube/datacube_axis.py

@@ -0,0 +1,332 @@
+import bisect
+from abc import ABC, abstractmethod
+from copy import deepcopy
+from typing import Any, List
+
+import numpy as np
+import pandas as pd
+import xarray as xr
+
+from .transformations.datacube_cyclic.datacube_cyclic import DatacubeAxisCyclic
+from .transformations.datacube_mappers.datacube_mappers import DatacubeMapper
+from .transformations.datacube_merger.datacube_merger import DatacubeAxisMerger
+from .transformations.datacube_reverse.datacube_reverse import DatacubeAxisReverse
+from .transformations.datacube_type_change.datacube_type_change import (
+    DatacubeAxisTypeChange,
+)
+
+
+class DatacubeAxis(ABC):
+    is_cyclic = False
+    has_mapper = False
+    has_merger = False
+    reorder = False
+    type_change = False
+
+    def order_tranformations(self):
+        self.transformations = sorted(self.transformations, key=lambda x: transformations_order[type(x)])
+
+    def give_transformations_parents(self):
+        for i, transform in enumerate(self.transformations[1:]):
+            transform.parent = self.transformations[i - 1]
+
+    # Convert from user-provided value to CONTINUOUS type (e.g. float, pd.timestamp)
+    @abstractmethod
+    def parse(self, value: Any) -> Any:
+        pass
+
+    # Convert from CONTINUOUS type to FLOAT
+    @abstractmethod
+    def to_float(self, value: Any) -> float:
+        pass
+
+    # Convert from FLOAT type to CONTINUOUS type
+    @abstractmethod
+    def from_float(self, value: float) -> Any:
+        pass
+
+    def serialize(self, value: Any) -> Any:
+        pass
+
+    def to_intervals(self, range):
+        intervals = [range]
+        for transformation in self.transformations[::-1]:
+            intervals = transformation.to_intervals(range, intervals, self)
+        return intervals
+
+    def remap(self, range: List) -> Any:
+        ranges = [range]
+        for transformation in self.transformations[::-1]:
+            ranges = transformation.remap(range, ranges, self)
+        return ranges
+
+    def unmap_to_datacube(self, path, unmapped_path):
+        return (path, unmapped_path)
+
+    def find_standard_indexes(self, path, datacube):
+        unmapped_path = {}
+        path_copy = deepcopy(path)
+        for key in path_copy:
+            axis = datacube._axes[key]
+            (path, unmapped_path) = axis.unmap_to_datacube(path, unmapped_path)
+        subarray = datacube.select(path, unmapped_path)
+        return datacube.datacube_natural_indexes(self, subarray)
+
+    def find_indexes(self, path, datacube):
+        indexes = self.find_standard_indexes(path, datacube)
+        for transformation in self.transformations[::-1]:
+            indexes = transformation.find_modified_indexes(indexes, path, datacube, self)
+        return indexes
+
+    def offset(self, value):
+        offset = 0
+        for transformation in self.transformations[::-1]:
+            offset = transformation.offset(value, self, offset)
+        return offset
+
+    def unmap_path_key(self, key_value_path, leaf_path, unwanted_path):
+        for transformation in self.transformations[::-1]:
+            (key_value_path, leaf_path, unwanted_path) = transformation.unmap_path_key(
+                key_value_path, leaf_path, unwanted_path, self
+            )
+        return (key_value_path, leaf_path, unwanted_path)
+
+    def unmap_tree_node(self, node, unwanted_path):
+        for transformation in self.transformations[::-1]:
+            (node, unwanted_path) = transformation.unmap_tree_node(node, unwanted_path)
+        return (node, unwanted_path)
+
+    def _remap_val_to_axis_range(self, value):
+        for transformation in self.transformations[::-1]:
+            value = transformation._remap_val_to_axis_range(value, self)
+        return value
+
+    def find_standard_indices_between(self, indexes, low, up, datacube, method=None):
+        indexes_between_ranges = []
+
+        if self.name in datacube.complete_axes and self.name not in datacube.transformed_axes:
+            # Find the range of indexes between lower and upper
+            # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html
+            # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing
+            if method == "surrounding" or method == "nearest":
+                start = indexes.searchsorted(low, "left")
+                end = indexes.searchsorted(up, "right")
+                start = max(start - 1, 0)
+                end = min(end + 1, len(indexes))
+                indexes_between = indexes[start:end].to_list()
+                indexes_between_ranges.extend(indexes_between)
+            else:
+                start = indexes.searchsorted(low, "left")
+                end = indexes.searchsorted(up, "right")
+                indexes_between = indexes[start:end].to_list()
+                indexes_between_ranges.extend(indexes_between)
+        else:
+            if method == "surrounding" or method == "nearest":
+                start = bisect.bisect_left(indexes, low)
+                end = bisect.bisect_right(indexes, up)
+                start = max(start - 1, 0)
+                end = min(end + 1, len(indexes))
+                indexes_between = indexes[start:end]
+                indexes_between_ranges.extend(indexes_between)
+            else:
+                lower_idx = bisect.bisect_left(indexes, low)
+                upper_idx = bisect.bisect_right(indexes, up)
+                indexes_between = indexes[lower_idx:upper_idx]
+                indexes_between_ranges.extend(indexes_between)
+        return indexes_between_ranges
+
+    def find_indices_between(self, indexes_ranges, low, up, datacube, method=None):
+        indexes_between_ranges = self.find_standard_indices_between(indexes_ranges, low, up, datacube, method)
+        for transformation in self.transformations[::-1]:
+            indexes_between_ranges = transformation.find_indices_between(
+                indexes_ranges, low, up, datacube, method, indexes_between_ranges, self
+            )
+        return indexes_between_ranges
+
+    @staticmethod
+    def values_type(values):
+        type_ = None
+        if isinstance(values, xr.core.variable.IndexVariable) or isinstance(values, xr.core.variable.Variable):
+            # If we have some xarray variable, transform them to actual variable type
+            values = np.array(values)
+            type_ = values.dtype.type
+        else:
+            if len(values) == 0:
+                # If we have no values (newly created axis), default to a float
+                values = np.array(values)
+                type_ = values.dtype.type
+            else:
+                type_ = type(values[0])
+        return type_
+
+    @staticmethod
+    def create_standard(name, values, datacube):
+        val_type = DatacubeAxis.values_type(values)
+
+        DatacubeAxis.check_axis_type(name, val_type)
+        if datacube._axes is None:
+            datacube._axes = {name: deepcopy(_type_to_axis_lookup[val_type])}
+        else:
+            datacube._axes[name] = deepcopy(_type_to_axis_lookup[val_type])
+
+        datacube._axes[name].name = name
+        datacube.axis_counter += 1
+
+    @staticmethod
+    def check_axis_type(name, val_type):
+        if val_type not in _type_to_axis_lookup:
+            raise ValueError(f"Could not create a mapper for index type {val_type} for axis {name}")
+
+
+transformations_order = [
+    DatacubeAxisMerger,
+    DatacubeAxisReverse,
+    DatacubeAxisCyclic,
+    DatacubeMapper,
+    DatacubeAxisTypeChange,
+]
+transformations_order = {key: i for i, key in enumerate(transformations_order)}
+
+
+class IntDatacubeAxis(DatacubeAxis):
+    def __init__(self):
+        self.name = None
+        self.tol = 1e-12
+        self.range = None
+        # TODO: Maybe here, store transformations as a dico instead
+        self.transformations = []
+        self.type = 0
+        self.can_round = True
+
+    def parse(self, value: Any) -> Any:
+        return float(value)
+
+    def to_float(self, value):
+        return float(value)
+
+    def from_float(self, value):
+        return float(value)
+
+    def serialize(self, value):
+        return value
+
+
+class FloatDatacubeAxis(DatacubeAxis):
+    def __init__(self):
+        self.name = None
+        self.tol = 1e-12
+        self.range = None
+        self.transformations = []
+        self.type = 0.0
+        self.can_round = True
+
+    def parse(self, value: Any) -> Any:
+        return float(value)
+
+    def to_float(self, value):
+        return float(value)
+
+    def from_float(self, value):
+        return float(value)
+
+    def serialize(self, value):
+        return value
+
+
+class PandasTimestampDatacubeAxis(DatacubeAxis):
+    def __init__(self):
+        self.name = None
+        self.tol = 1e-12
+        self.range = None
+        self.transformations = []
+        self.type = pd.Timestamp("2000-01-01T00:00:00")
+        self.can_round = False
+
+    def parse(self, value: Any) -> Any:
+        if isinstance(value, np.str_):
+            value = str(value)
+        return pd.Timestamp(value)
+
+    def to_float(self, value: pd.Timestamp):
+        if isinstance(value, np.datetime64):
+            return float((value - np.datetime64("1970-01-01T00:00:00")).astype("int"))
+        else:
+            return float(value.value / 10**9)
+
+    def from_float(self, value):
+        return pd.Timestamp(int(value), unit="s")
+
+    def serialize(self, value):
+        return str(value)
+
+    def offset(self, value):
+        return None
+
+
+class PandasTimedeltaDatacubeAxis(DatacubeAxis):
+    def __init__(self):
+        self.name = None
+        self.tol = 1e-12
+        self.range = None
+        self.transformations = []
+        self.type = np.timedelta64(0, "s")
+        self.can_round = False
+
+    def parse(self, value: Any) -> Any:
+        if isinstance(value, np.str_):
+            value = str(value)
+        return pd.Timedelta(value)
+
+    def to_float(self, value: pd.Timedelta):
+        if isinstance(value, np.timedelta64):
+            return value.astype("timedelta64[s]").astype(int)
+        else:
+            return float(value.value / 10**9)
+
+    def from_float(self, value):
+        return pd.Timedelta(int(value), unit="s")
+
+    def serialize(self, value):
+        return str(value)
+
+    def offset(self, value):
+        return None
+
+
+class UnsliceableDatacubeAxis(DatacubeAxis):
+    def __init__(self):
+        self.name = None
+        self.tol = float("NaN")
+        self.range = None
+        self.transformations = []
+        self.can_round = False
+
+    def parse(self, value: Any) -> Any:
+        return value
+
+    def to_float(self, value: pd.Timedelta):
+        raise TypeError("Tried to slice unsliceable axis")
+
+    def from_float(self, value):
+        raise TypeError("Tried to slice unsliceable axis")
+
+    def serialize(self, value):
+        raise TypeError("Tried to slice unsliceable axis")
+
+
+_type_to_axis_lookup = {
+    pd.Int64Dtype: IntDatacubeAxis(),
+    pd.Timestamp: PandasTimestampDatacubeAxis(),
+    np.int64: IntDatacubeAxis(),
+    np.datetime64: PandasTimestampDatacubeAxis(),
+    np.timedelta64: PandasTimedeltaDatacubeAxis(),
+    pd.Timedelta: PandasTimedeltaDatacubeAxis(),
+    np.float64: FloatDatacubeAxis(),
+    np.float32: FloatDatacubeAxis(),
+    np.int32: IntDatacubeAxis(),
+    np.str_: UnsliceableDatacubeAxis(),
+    str: UnsliceableDatacubeAxis(),
+    np.object_: UnsliceableDatacubeAxis(),
+    int: IntDatacubeAxis(),
+    float: FloatDatacubeAxis(),
+}

polytope_feature/datacube/index_tree_pb2.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: index_tree.proto
+# Protobuf Python Version: 5.26.1
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
+    b'\n\x10index_tree.proto\x12\nindex_tree"\x9a\x01\n\x04Node\x12\x0c\n\x04\x61xis\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x03(\t\x12\x0f\n\x07indexes\x18\x03 \x03(\x03\x12\x0e\n\x06result\x18\x04 \x03(\x01\x12\x13\n\x0bsize_result\x18\x05 \x03(\x03\x12"\n\x08\x63hildren\x18\x06 \x03(\x0b\x32\x10.index_tree.Node\x12\x1b\n\x13size_indexes_branch\x18\x07 \x03(\x03\x62\x06proto3'
+)
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "index_tree_pb2", _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+    DESCRIPTOR._loaded_options = None
+    _globals["_NODE"]._serialized_start = 33
+    _globals["_NODE"]._serialized_end = 187
+# @@protoc_insertion_point(module_scope)