polytope-python 2.0.4__tar.gz → 2.1.1__tar.gz

{polytope_python-2.0.4 → polytope_python-2.1.1}/.gitignore

@@ -57,6 +57,7 @@ target
 rust_deployment_venv
 *.so
 *.nc
+*.grb
 
 test_w_qubed
 rust_venv
@@ -75,4 +76,6 @@ performance_plots_qubed
 *.nc
 **/*venv*
 
+bugfix*
+
 icon_grids

{polytope_python-2.0.4/polytope_python.egg-info → polytope_python-2.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytope-python
-Version: 2.0.4
+Version: 2.1.1
 Summary: Polytope datacube feature extraction library
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 Maintainer-email: James Hawkes <James.Hawkes@ecmwf.int>, Mathilde Leuridan <Mathilde.Leuridan@ecmwf.int>

{polytope_python-2.0.4 → polytope_python-2.1.1}/polytope_feature/_version.py

@@ -1,2 +1,2 @@
 # Do not change! Do not track in version control!
-__version__ = "2.0.4"
+__version__ = "2.1.1"

{polytope_python-2.0.4 → polytope_python-2.1.1}/polytope_feature/datacube/backends/fdb.py

@@ -219,28 +219,32 @@ class FDBDatacube(Datacube):
 
     def remove_duplicates_in_request_ranges(self, fdb_node_ranges, current_start_idxs):
         seen_indices = set()
+        new_fdb_node_ranges = []
+        new_current_start_idxs = []
         for i, idxs_list in enumerate(current_start_idxs):
+            new_idx_group = []
+            new_fdb_group = []
             for k, sub_lat_idxs in enumerate(idxs_list):
                 actual_fdb_node = fdb_node_ranges[i][k]
-                original_fdb_node_range_vals = []
-                new_current_start_idx = []
+                original_vals = []
+                filtered_idxs = []
                 for j, idx in enumerate(sub_lat_idxs):
                     if idx not in seen_indices:
-                        # NOTE: need to remove it from the values in the corresponding tree node
-                        # NOTE: need to read just the range we give to gj
-                        original_fdb_node_range_vals.append(actual_fdb_node[0].values[j])
                         seen_indices.add(idx)
-                        new_current_start_idx.append(idx)
-                if original_fdb_node_range_vals != []:
-                    actual_fdb_node[0].values = tuple(original_fdb_node_range_vals)
+                        filtered_idxs.append(idx)
+                        original_vals.append(actual_fdb_node[0].values[j])
+                if filtered_idxs:
+                    # keep only if we had values still
+                    actual_fdb_node[0].values = tuple(original_vals)
+                    new_idx_group.append(filtered_idxs)
+                    new_fdb_group.append(actual_fdb_node)
                 else:
-                    # there are no values on this node anymore so can remove it
+                    # remove this node because we removed all values
                     actual_fdb_node[0].remove_branch()
-                if len(new_current_start_idx) == 0:
-                    current_start_idxs[i].pop(k)
-                else:
-                    current_start_idxs[i][k] = new_current_start_idx
-        return (fdb_node_ranges, current_start_idxs)
+            new_current_start_idxs.append(new_idx_group)
+            new_fdb_node_ranges.append(new_fdb_group)
+
+        return new_fdb_node_ranges, new_current_start_idxs
 
     def nearest_lat_lon_search(self, requests):
         if len(self.nearest_search) != 0:
@@ -341,6 +345,9 @@ class FDBDatacube(Datacube):
             # TODO: change this to accommodate non consecutive indexes being compressed too
             current_idx[i].extend(key_value_path["values"])
             fdb_range_n[i].append(c)
+        assert len(current_idx) == len(fdb_range_n)
+        for i, node in enumerate(fdb_range_n):
+            assert len(node[0].values) == len(current_idx[i])
         return (current_idx, fdb_range_n)
 
     def assign_fdb_output_to_nodes(self, output_iterator, fdb_requests_decoding_info):
@@ -363,6 +370,8 @@ class FDBDatacube(Datacube):
         (new_fdb_node_ranges, new_current_start_idx) = self.remove_duplicates_in_request_ranges(
             fdb_node_ranges, current_start_idx
         )
+        current_start_idx = new_current_start_idx
+        fdb_node_ranges = new_fdb_node_ranges
         interm_request_ranges = []
         # TODO: modify the start indexes to have as many arrays as the request ranges
         new_fdb_node_ranges = []

{polytope_python-2.0.4 → polytope_python-2.1.1}/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/local_regular.py

@@ -1,5 +1,7 @@
 import bisect
 
+import numpy as np
+
 from ..datacube_mappers import DatacubeMapper
 
 
@@ -45,15 +47,28 @@ class LocalRegularGridMapper(DatacubeMapper):
         if self._axis_reversed[mapped_axes[1]]:
             raise NotImplementedError("Local regular grid with second axis in decreasing order is not supported")
 
+        self._second_axis_vals = np.arange(
+            self._second_axis_min,
+            self._second_axis_min + self._second_deg_increment * (self.second_resolution + 1),
+            self._second_deg_increment,
+            dtype=np.float64,
+        )
+
     def first_axis_vals(self):
         if self._axis_reversed[self._mapped_axes[0]]:
-            first_ax_vals = [
-                self._first_axis_max - i * self._first_deg_increment for i in range(self.first_resolution + 1)
-            ]
+            first_ax_vals = np.arange(
+                self._first_axis_max,
+                self._first_axis_max - self._first_deg_increment * (self.first_resolution + 1),
+                -self._first_deg_increment,
+                dtype=np.float64,
+            )
         else:
-            first_ax_vals = [
-                self._first_axis_min + i * self._first_deg_increment for i in range(self.first_resolution + 1)
-            ]
+            first_ax_vals = np.arange(
+                self._first_axis_min,
+                self._first_axis_min + self._first_deg_increment * (self.first_resolution + 1),
+                self._first_deg_increment,
+                dtype=np.float64,
+            )
         return first_ax_vals
 
     def map_first_axis(self, lower, upper):
@@ -62,10 +77,7 @@ class LocalRegularGridMapper(DatacubeMapper):
         return return_vals
 
     def second_axis_vals(self, first_val):
-        second_ax_vals = [
-            self._second_axis_min + i * self._second_deg_increment for i in range(self.second_resolution + 1)
-        ]
-        return second_ax_vals
+        return self._second_axis_vals
 
     def map_second_axis(self, first_val, lower, upper):
         axis_lines = self.second_axis_vals(first_val)
@@ -89,16 +101,26 @@ class LocalRegularGridMapper(DatacubeMapper):
         return first_idx * self.second_resolution
 
     def unmap(self, first_val, second_vals, unmapped_idx=None):
-        tol = 1e-8
-        first_val = [i for i in self._first_axis_vals if first_val[0] - tol <= i <= first_val[0] + tol][0]
-        first_idx = self._first_axis_vals.index(first_val)
-        return_idxs = []
-        for second_val in second_vals:
-            second_val = [i for i in self.second_axis_vals(first_val) if second_val - tol <= i <= second_val + tol][0]
-            second_idx = self.second_axis_vals(first_val).index(second_val)
-            final_index = self.axes_idx_to_regular_idx(first_idx, second_idx)
-            return_idxs.append(final_index)
-        return return_idxs
+        first_array = self._first_axis_vals
+        second_array = self._second_axis_vals
+        second_vals = np.asarray(second_vals)
+        descending = self._axis_reversed[self._mapped_axes[0]]
+        if descending:
+            # right descending order for searchsorted
+            first_idx = np.searchsorted(first_array[::-1], first_val[0])
+            first_idx = len(first_array) - 1 - first_idx
+        else:
+            first_idx = np.searchsorted(first_array, first_val[0])
+        if first_idx > 0 and first_idx < len(first_array):
+            left_val = first_array[first_idx - 1]
+            right_val = first_array[first_idx]
+            if abs(first_val[0] - left_val) < abs(first_val[0] - right_val):
+                first_idx -= 1
+        second_idxs = np.searchsorted(second_array, second_vals)
+
+        # map to grid idx
+        final_idxs = first_idx * (self.second_resolution + 1) + second_idxs
+        return final_idxs
 
 
 # md5 grid hash in form {resolution : hash}

{polytope_python-2.0.4 → polytope_python-2.1.1/polytope_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytope-python
-Version: 2.0.4
+Version: 2.1.1
 Summary: Polytope datacube feature extraction library
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 Maintainer-email: James Hawkes <James.Hawkes@ecmwf.int>, Mathilde Leuridan <Mathilde.Leuridan@ecmwf.int>