polytope-python 1.0.31__py3-none-any.whl

polytope_feature/datacube/transformations/datacube_mappers/mapper_types/regular.py

@@ -0,0 +1,75 @@
+import bisect
+
+from ..datacube_mappers import DatacubeMapper
+
+
+class RegularGridMapper(DatacubeMapper):
+    def __init__(self, base_axis, mapped_axes, resolution, md5_hash=None, local_area=[], axis_reversed=None):
+        # TODO: if local area is not empty list, raise NotImplemented
+        self._mapped_axes = mapped_axes
+        self._base_axis = base_axis
+        self._resolution = resolution
+        self.deg_increment = 90 / self._resolution
+        if axis_reversed is None:
+            self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False}
+        else:
+            assert set(axis_reversed.keys()) == set(mapped_axes)
+            self._axis_reversed = axis_reversed
+        self._first_axis_vals = self.first_axis_vals()
+        self.compressed_grid_axes = [self._mapped_axes[1]]
+        if md5_hash is not None:
+            self.md5_hash = md5_hash
+        else:
+            self.md5_hash = _md5_hash.get(resolution, None)
+        if self._axis_reversed[mapped_axes[1]]:
+            raise NotImplementedError("Regular grid with second axis in decreasing order is not supported")
+
+    def first_axis_vals(self):
+        if self._axis_reversed[self._mapped_axes[0]]:
+            first_ax_vals = [90 - i * self.deg_increment for i in range(2 * self._resolution)]
+        else:
+            first_ax_vals = [-90 + i * self.deg_increment for i in range(2 * self._resolution)]
+        return first_ax_vals
+
+    def map_first_axis(self, lower, upper):
+        axis_lines = self._first_axis_vals
+        return_vals = [val for val in axis_lines if lower <= val <= upper]
+        return return_vals
+
+    def second_axis_vals(self, first_val):
+        second_ax_vals = [i * self.deg_increment for i in range(4 * self._resolution)]
+        return second_ax_vals
+
+    def map_second_axis(self, first_val, lower, upper):
+        axis_lines = self.second_axis_vals(first_val)
+        return_vals = [val for val in axis_lines if lower <= val <= upper]
+        return return_vals
+
+    def axes_idx_to_regular_idx(self, first_idx, second_idx):
+        final_idx = first_idx * 4 * self._resolution + second_idx
+        return final_idx
+
+    def find_second_idx(self, first_val, second_val):
+        tol = 1e-10
+        second_axis_vals = self.second_axis_vals(first_val)
+        second_idx = bisect.bisect_left(second_axis_vals, second_val - tol)
+        return second_idx
+
+    def unmap_first_val_to_start_line_idx(self, first_val):
+        tol = 1e-8
+        first_val = [i for i in self._first_axis_vals if first_val - tol <= i <= first_val + tol][0]
+        first_idx = self._first_axis_vals.index(first_val)
+        return first_idx * 4 * self._resolution
+
+    def unmap(self, first_val, second_val):
+        tol = 1e-8
+        first_val = [i for i in self._first_axis_vals if first_val[0] - tol <= i <= first_val[0] + tol][0]
+        first_idx = self._first_axis_vals.index(first_val)
+        second_val = [i for i in self.second_axis_vals(first_val) if second_val[0] - tol <= i <= second_val[0] + tol][0]
+        second_idx = self.second_axis_vals(first_val).index(second_val)
+        final_index = self.axes_idx_to_regular_idx(first_idx, second_idx)
+        return final_index
+
+
+# md5 grid hash in form {resolution : hash}
+_md5_hash = {}

polytope_feature/datacube/transformations/datacube_merger/__init__.py

@@ -0,0 +1 @@
+from .datacube_merger import *

polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py

@@ -0,0 +1,95 @@
+import logging
+
+import numpy as np
+import pandas as pd
+
+from ..datacube_transformations import DatacubeAxisTransformation
+
+
+class DatacubeAxisMerger(DatacubeAxisTransformation):
+    def __init__(self, name, merge_options, datacube=None):
+        self.transformation_options = merge_options
+        self.name = name
+        self._first_axis = name
+        self._second_axis = merge_options.other_axis
+        self._linkers = merge_options.linkers
+        self._merged_values = self.merged_values(datacube)
+
+    def blocked_axes(self):
+        return [self._second_axis]
+
+    def unwanted_axes(self):
+        return []
+
+    def _mapped_axes(self):
+        return self._first_axis
+
+    def merged_values(self, datacube):
+        first_ax_vals = np.array(datacube.ax_vals(self.name))
+        second_ax_name = self._second_axis
+        second_ax_vals = np.array(datacube.ax_vals(second_ax_name))
+        linkers = self._linkers
+        first_grid, second_grid = np.meshgrid(first_ax_vals, second_ax_vals, indexing="ij")
+        combined_strings = np.char.add(
+            np.char.add(first_grid.ravel(), linkers[0]), np.char.add(second_grid.ravel(), linkers[1])
+        )
+        merged_values = pd.to_datetime(combined_strings).to_numpy().astype("datetime64[s]")
+        merged_values = np.array(merged_values)
+        merged_values.sort()
+        logging.info(
+            f"Merged values {first_ax_vals} on axis {self.name} and \
+                     values {second_ax_vals} on axis {second_ax_name} to values {merged_values}"
+        )
+        return merged_values
+
+    def transformation_axes_final(self):
+        return [self._first_axis]
+
+    def generate_final_transformation(self):
+        return self
+
+    def unmerge(self, merged_val):
+        first_values = []
+        second_values = []
+        for merged_value in merged_val:
+            merged_val = str(merged_value)
+            first_idx = merged_val.find(self._linkers[0])
+            first_val = merged_val[:first_idx]
+            first_linker_size = len(self._linkers[0])
+            second_linked_size = len(self._linkers[1])
+            second_val = merged_val[first_idx + first_linker_size : -second_linked_size]
+
+            # TODO: maybe replacing like this is too specific to time/dates?
+            first_val = str(first_val).replace("-", "")
+            second_val = second_val.replace(":", "")
+            logging.info(
+                f"Unmerged value {merged_val} to values {first_val} on axis {self.name} \
+                        and {second_val} on axis {self._second_axis}"
+            )
+            first_values.append(first_val)
+            second_values.append(second_val)
+        return (tuple(first_values), tuple(second_values))
+
+    def change_val_type(self, axis_name, values):
+        new_values = pd.to_datetime(values)
+        return new_values
+
+    def find_modified_indexes(self, indexes, path, datacube, axis):
+        if axis.name == self._first_axis:
+            return self._merged_values
+
+    def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis):
+        new_key_value_path = {}
+        value = key_value_path[axis.name]
+        if axis.name == self._first_axis:
+            (first_val, second_val) = self.unmerge(value)
+            new_key_value_path[self._first_axis] = first_val
+            new_key_value_path[self._second_axis] = second_val
+        return (new_key_value_path, leaf_path, unwanted_path)
+
+    def unmap_tree_node(self, node, unwanted_path):
+        if node.axis.name == self._first_axis:
+            (new_first_vals, new_second_vals) = self.unmerge(node.values)
+            node.values = new_first_vals
+            interm_node = node.add_node_layer_after(self._second_axis, new_second_vals)
+        return (interm_node, unwanted_path)

polytope_feature/datacube/transformations/datacube_reverse/__init__.py

@@ -0,0 +1 @@
+from .datacube_reverse import *

polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py

@@ -0,0 +1,65 @@
+from ....utility.list_tools import bisect_left_cmp, bisect_right_cmp
+from ..datacube_transformations import DatacubeAxisTransformation
+
+
+class DatacubeAxisReverse(DatacubeAxisTransformation):
+    def __init__(self, name, mapper_options, datacube=None):
+        self.name = name
+        self.transformation_options = mapper_options
+
+    def generate_final_transformation(self):
+        return self
+
+    def transformation_axes_final(self):
+        return [self.name]
+
+    def change_val_type(self, axis_name, values):
+        return values
+
+    def blocked_axes(self):
+        return []
+
+    def unwanted_axes(self):
+        return []
+
+    def find_modified_indexes(self, indexes, path, datacube, axis):
+        if axis.name in datacube.complete_axes:
+            ordered_indices = indexes.sort_values()
+        else:
+            ordered_indices = indexes
+        return ordered_indices
+
+    def find_indices_between(self, indexes, low, up, datacube, method, indexes_between_ranges, axis):
+        indexes_between_ranges = []
+        if axis.name == self.name:
+            if axis.name in datacube.complete_axes:
+                # Find the range of indexes between lower and upper
+                # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html
+                # Assumes the indexes are already sorted (could sort to be sure) and monotonically
+                # increasing
+                if method == "surrounding" or method == "nearest":
+                    start = indexes.searchsorted(low, "left")
+                    end = indexes.searchsorted(up, "right")
+                    start = max(start - 1, 0)
+                    end = min(end + 1, len(indexes))
+                    indexes_between = indexes[start:end].to_list()
+                    indexes_between_ranges.extend(indexes_between)
+                else:
+                    start = indexes.searchsorted(low, "left")
+                    end = indexes.searchsorted(up, "right")
+                    indexes_between = indexes[start:end].to_list()
+                    indexes_between_ranges.extend(indexes_between)
+            else:
+                if method == "surrounding" or method == "nearest":
+                    end_idx = bisect_left_cmp(indexes, low, cmp=lambda x, y: x > y) + 1
+                    start_idx = bisect_right_cmp(indexes, up, cmp=lambda x, y: x > y)
+                    start = max(start_idx - 1, 0)
+                    end = min(end_idx + 1, len(indexes))
+                    indexes_between = indexes[start:end]
+                    indexes_between_ranges.extend(indexes_between)
+                else:
+                    end_idx = bisect_left_cmp(indexes, low, cmp=lambda x, y: x > y) + 1
+                    start_idx = bisect_right_cmp(indexes, up, cmp=lambda x, y: x > y)
+                    indexes_between = indexes[start_idx:end_idx]
+                    indexes_between_ranges.extend(indexes_between)
+        return indexes_between_ranges

polytope_feature/datacube/transformations/datacube_transformations.py

@@ -0,0 +1,96 @@
+from abc import ABC, abstractmethod
+from copy import deepcopy
+from importlib import import_module
+
+
+class DatacubeAxisTransformation(ABC):
+    def __init__(self):
+        self.parent = None
+
+    @staticmethod
+    def create_transform(name, transformation_type_key, transformation_options, datacube):
+        transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key]
+        transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key]
+        file_name = ".datacube_" + transformation_file_name
+        module = import_module("polytope_feature.datacube.transformations" + file_name + file_name)
+        constructor = getattr(module, transformation_type)
+        transformation_type_option = transformation_options
+        new_transformation = deepcopy(constructor(name, transformation_type_option, datacube))
+
+        new_transformation.name = name
+        return new_transformation
+
+    @staticmethod
+    def get_final_axes(name, transformation_type_key, transformation_options, datacube):
+        new_transformation = DatacubeAxisTransformation.create_transform(
+            name, transformation_type_key, transformation_options, datacube
+        )
+        transformation_axis_names = new_transformation.transformation_axes_final()
+        return (transformation_axis_names, new_transformation)
+
+    def name(self):
+        pass
+
+    def transformation_options(self):
+        pass
+
+    @abstractmethod
+    def generate_final_transformation(self):
+        pass
+
+    @abstractmethod
+    def transformation_axes_final(self):
+        pass
+
+    @abstractmethod
+    def change_val_type(self, axis_name, values):
+        pass
+
+    def find_modified_indexes(self, indexes, path, datacube, axis):
+        return indexes
+
+    def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis):
+        return (key_value_path, leaf_path, unwanted_path)
+
+    def unmap_tree_node(self, node, unwanted_path):
+        return (node, unwanted_path)
+
+    def find_indices_between(self, indexes_ranges, low, up, datacube, method, indexes_between_ranges, axis):
+        return indexes_between_ranges
+
+    def _remap_val_to_axis_range(self, value, axis):
+        return value
+
+    def offset(self, range, axis, offset):
+        return offset
+
+    def remap(self, range, ranges, axis):
+        return ranges
+
+    def to_intervals(self, range, intervals, axis):
+        return intervals
+
+
+_type_to_datacube_transformation_lookup = {
+    "mapper": "DatacubeMapper",
+    "cyclic": "DatacubeAxisCyclic",
+    "merge": "DatacubeAxisMerger",
+    "reverse": "DatacubeAxisReverse",
+    "type_change": "DatacubeAxisTypeChange",
+}
+
+_type_to_transformation_file_lookup = {
+    "mapper": "mappers",
+    "cyclic": "cyclic",
+    "merge": "merger",
+    "reverse": "reverse",
+    "type_change": "type_change",
+}
+
+has_transform = {
+    "mapper": "has_mapper",
+    "cyclic": "is_cyclic",
+    "merge": "has_merger",
+    "reverse": "reorder",
+    "type_change": "type_change",
+}

polytope_feature/datacube/transformations/datacube_type_change/__init__.py

@@ -0,0 +1 @@
+from .datacube_type_change import *

polytope_feature/datacube/transformations/datacube_type_change/datacube_type_change.py

@@ -0,0 +1,124 @@
+from copy import deepcopy
+from importlib import import_module
+
+import pandas as pd
+
+from ..datacube_transformations import DatacubeAxisTransformation
+
+
+class DatacubeAxisTypeChange(DatacubeAxisTransformation):
+    # The transformation here will be to point the old axes to the new cyclic axes
+
+    def __init__(self, name, type_options, datacube=None):
+        self.name = name
+        self.transformation_options = type_options
+        self.new_type = type_options.type
+        self._final_transformation = self.generate_final_transformation()
+
+    def generate_final_transformation(self):
+        map_type = _type_to_datacube_type_change_lookup[self.new_type]
+        module = import_module("polytope_feature.datacube.transformations.datacube_type_change.datacube_type_change")
+        constructor = getattr(module, map_type)
+        transformation = deepcopy(constructor(self.name, self.new_type))
+        return transformation
+
+    def transformation_axes_final(self):
+        return [self._final_transformation.axis_name]
+
+    def change_val_type(self, axis_name, values):
+        return_idx = [self._final_transformation.transform_type(val) for val in values]
+        if None in return_idx:
+            return None
+        return_idx.sort()
+        return return_idx
+
+    def make_str(self, value):
+        return self._final_transformation.make_str(value)
+
+    def blocked_axes(self):
+        return []
+
+    def unwanted_axes(self):
+        return []
+
+    def find_modified_indexes(self, indexes, path, datacube, axis):
+        if axis.name == self.name:
+            return self.change_val_type(axis.name, indexes)
+
+    def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis):
+        value = key_value_path[axis.name]
+        if axis.name == self.name:
+            unchanged_val = self.make_str(value)
+            key_value_path[axis.name] = unchanged_val
+        return (key_value_path, leaf_path, unwanted_path)
+
+    def unmap_tree_node(self, node, unwanted_path):
+        if node.axis.name == self.name:
+            new_node_vals = self.make_str(node.values)
+            node.values = new_node_vals
+        return (node, unwanted_path)
+
+
+class TypeChangeStrToInt(DatacubeAxisTypeChange):
+    def __init__(self, axis_name, new_type):
+        self.axis_name = axis_name
+        self._new_type = new_type
+
+    def transform_type(self, value):
+        try:
+            return int(value)
+        except ValueError:
+            return None
+
+    def make_str(self, value):
+        values = []
+        for val in value:
+            values.append(str(val))
+        return tuple(values)
+
+
+class TypeChangeStrToTimestamp(DatacubeAxisTypeChange):
+    def __init__(self, axis_name, new_type):
+        self.axis_name = axis_name
+        self._new_type = new_type
+
+    def transform_type(self, value):
+        try:
+            return pd.Timestamp(value)
+        except ValueError:
+            return None
+
+    def make_str(self, value):
+        values = []
+        for val in value:
+            values.append(val.strftime("%Y%m%d"))
+        return tuple(values)
+
+
+class TypeChangeStrToTimedelta(DatacubeAxisTypeChange):
+    def __init__(self, axis_name, new_type):
+        self.axis_name = axis_name
+        self._new_type = new_type
+
+    def transform_type(self, value):
+        try:
+            hours = int(value[:2])
+            mins = int(value[2:])
+            return pd.Timedelta(hours=hours, minutes=mins)
+        except ValueError:
+            return None
+
+    def make_str(self, value):
+        values = []
+        for val in value:
+            hours = int(val.total_seconds() // 3600)
+            mins = int((val.total_seconds() % 3600) // 60)
+            values.append(f"{hours:02d}{mins:02d}")
+        return tuple(values)
+
+
+_type_to_datacube_type_change_lookup = {
+    "int": "TypeChangeStrToInt",
+    "date": "TypeChangeStrToTimestamp",
+    "time": "TypeChangeStrToTimedelta",
+}

polytope_feature/datacube/tree_encoding.py

@@ -0,0 +1,132 @@
+import math
+from copy import deepcopy
+
+from . import index_tree_pb2 as pb2
+from .datacube_axis import IntDatacubeAxis
+from .tensor_index_tree import TensorIndexTree
+
+
+def encode_tree(tree: TensorIndexTree):
+    node = pb2.Node()
+
+    node.axis = tree.axis.name
+
+    # NOTE: do we need this if we parse the tree before it has values?
+    if tree.result is not None:
+        for result in tree.result:
+            node.result.append(result)
+
+    # Nest children in protobuf root tree node
+    for c in tree.children:
+        encode_child(tree, c, node)
+
+    # Write to file
+    return node.SerializeToString()
+
+
+def write_encoded_tree_to_file(tree_bytes):
+    with open("encodedTree", "wb") as fs:
+        fs.write(tree_bytes)
+
+
+def encode_child(tree: TensorIndexTree, child: TensorIndexTree, node, result_size=[]):
+    child_node = pb2.Node()
+
+    new_result_size = deepcopy(result_size)
+    # new_result_size = result_size
+    new_result_size.append(len(child.values))
+
+    if child.hidden:
+        # add indexes to parent and add also indexes size...
+        node.indexes.extend(tree.indexes)
+        break_tag = False
+        return break_tag
+
+    # need to add axis and children etc to the encoded node only if the tree node isn't hidden
+    else:
+        child_node.axis = child.axis.name
+        child_node.value.extend(child.values)
+        child_node.size_result.extend(new_result_size)
+
+        for c in child.children:
+            breaking = encode_child(child, c, child_node, new_result_size)
+            if not breaking:
+                for c_ in child.children:
+                    child_node.size_indexes_branch.append(len(c_.children))
+                break
+
+        # we append the children once their branch has been completed until the leaf
+        if not child.hidden:
+            node.children.append(child_node)
+
+
+def decode_tree(datacube, bytearray):
+    node = pb2.Node()
+    node.ParseFromString(bytearray)
+
+    tree = TensorIndexTree()
+
+    if node.axis == "root":
+        root = IntDatacubeAxis()
+        root.name = "root"
+        tree.axis = root
+    else:
+        tree.axis = datacube._axes[node.axis]
+
+    # Put contents of node children into tree
+    decode_child(node, tree, datacube)
+
+    return tree
+
+
+def decode_child(node, tree, datacube):
+    if len(node.children) == 0:
+        tree.result = node.result
+        tree.result_size = node.size_result
+        tree.indexes = node.indexes
+        tree.indexes_size = node.size_indexes_branch
+    for child in node.children:
+        if child.axis in datacube._axes.keys():
+            child_axis = datacube._axes[child.axis]
+            child_vals = tuple(child.value)
+            child_node = TensorIndexTree(child_axis, child_vals)
+            tree.add_child(child_node)
+            decode_child(child, child_node, datacube)
+        else:
+            grandchild_axis = datacube._axes[child.children[0].axis]
+            for c in child.children:
+                grandchild_vals = tuple(c.value)
+                grandchild_node = TensorIndexTree(grandchild_axis, grandchild_vals)
+                tree.add_child(grandchild_node)
+                decode_child(c, grandchild_node, datacube)
+
+
+def decode_into_tree(tree, bytearray):
+    # TODO: write a decoder that decodes the bytearray (ie results) from gribjump directly into the tree instance
+    node = pb2.Node()
+    node.ParseFromString(bytearray)
+
+    decode_child_into_tree(tree, node)
+
+    return tree
+
+
+def decode_child_into_tree(tree, node):
+    if not tree.hidden:
+        # iterate through tree
+        for i, child in enumerate(tree.children):
+            node_c = node.children[i]
+            decode_child_into_tree(child, node_c)
+    else:
+        # TODO: if it's hidden, use the sizes to attribute to the hidden lat/lon nodes...
+        num_results = math.prod(node.size_result)
+        num_lat_branches = len(tree.children)
+        start_result_idx = 0
+        for i in range(num_lat_branches):
+            lat_node = tree.children[i]
+            num_lon_branches = len(lat_node.children)
+            for j in range(num_lon_branches):
+                lon_node = lat_node.children[j]
+                next_result_idx = start_result_idx + num_results
+                lon_node.result = node.result[start_result_idx:next_result_idx]
+                start_result_idx = next_result_idx

polytope_feature/engine/__init__.py

@@ -0,0 +1 @@
+from .engine import *

polytope_feature/engine/engine.py

@@ -0,0 +1,19 @@
+from typing import List
+
+from ..datacube.backends.datacube import Datacube
+from ..datacube.tensor_index_tree import TensorIndexTree
+from ..shapes import ConvexPolytope
+
+
+class Engine:
+    def __init__(self):
+        pass
+
+    def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]) -> TensorIndexTree:
+        pass
+
+    @staticmethod
+    def default():
+        from .hullslicer import HullSlicer
+
+        return HullSlicer()