openms-insight 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

openms_insight/components/heatmap.py

@@ -229,6 +229,8 @@ class Heatmap(BaseComponent):
         render time, the resulting data has ~min_points regardless of the
         filter value selected.
 
+        Data is sorted by x, y columns for efficient range query predicate pushdown.
+
         Example: For im_dimension with values [0, 1, 2, 3], creates:
         - cat_level_im_dimension_0_0: 20K points with im_id=0
         - cat_level_im_dimension_0_1: 20K points with im_id=1
@@ -314,14 +316,19 @@ class Heatmap(BaseComponent):
                             y_range=y_range,
                         )
 
+                    # Sort by x, y for efficient range query predicate pushdown
+                    level = level.sort([self._x_column, self._y_column])
                     # Store LazyFrame for streaming to disk
                     level_key = f'cat_level_{filter_id}_{filter_value}_{level_idx}'
                     self._preprocessed_data[level_key] = level  # Keep lazy
 
                 # Add full resolution as final level (for zoom fallback)
+                # Also sorted for consistent predicate pushdown behavior
                 num_compressed = len(level_sizes)
                 full_res_key = f'cat_level_{filter_id}_{filter_value}_{num_compressed}'
-                self._preprocessed_data[full_res_key] = filtered_data
+                self._preprocessed_data[full_res_key] = filtered_data.sort(
+                    [self._x_column, self._y_column]
+                )
                 self._preprocessed_data[f'cat_num_levels_{filter_id}_{filter_value}'] = num_compressed + 1
 
         # Also create global levels for when no categorical filter is selected
@@ -351,11 +358,16 @@ class Heatmap(BaseComponent):
                     x_range=x_range,
                     y_range=y_range,
                 )
+            # Sort by x, y for efficient range query predicate pushdown
+            level = level.sort([self._x_column, self._y_column])
             self._preprocessed_data[f'level_{i}'] = level  # Keep lazy
 
         # Add full resolution as final level (for zoom fallback)
+        # Also sorted for consistent predicate pushdown behavior
         num_compressed = len(level_sizes)
-        self._preprocessed_data[f'level_{num_compressed}'] = self._raw_data
+        self._preprocessed_data[f'level_{num_compressed}'] = self._raw_data.sort(
+            [self._x_column, self._y_column]
+        )
         self._preprocessed_data['num_levels'] = num_compressed + 1
 
     def _preprocess_streaming(self) -> None:
@@ -363,6 +375,7 @@ class Heatmap(BaseComponent):
         Streaming preprocessing - levels stay lazy through caching.
 
         Builds lazy query plans that are streamed to disk via sink_parquet().
+        Data is sorted by x, y columns for efficient range query predicate pushdown.
         """
         # Get data ranges (minimal collect - just 4 values)
         x_range, y_range = get_data_range(
@@ -406,13 +419,19 @@ class Heatmap(BaseComponent):
                     x_range=x_range,
                     y_range=y_range,
                 )
+            # Sort by x, y for efficient range query predicate pushdown
+            # This clusters spatially close points together in row groups
+            level = level.sort([self._x_column, self._y_column])
             # Store LazyFrame for streaming to disk
             # Base class will use sink_parquet() to stream without full materialization
             self._preprocessed_data[f'level_{i}'] = level  # Keep lazy
 
         # Add full resolution as final level (for zoom fallback)
+        # Also sorted for consistent predicate pushdown behavior
         num_compressed = len(level_sizes)
-        self._preprocessed_data[f'level_{num_compressed}'] = self._raw_data
+        self._preprocessed_data[f'level_{num_compressed}'] = self._raw_data.sort(
+            [self._x_column, self._y_column]
+        )
 
         # Store number of levels for reconstruction (includes full resolution)
         self._preprocessed_data['num_levels'] = num_compressed + 1
@@ -423,6 +442,7 @@ class Heatmap(BaseComponent):
 
         Uses more memory at init but faster rendering. Uses scipy-based
         downsampling for better spatial distribution.
+        Data is sorted by x, y columns for efficient range query predicate pushdown.
         """
         # Get data ranges
         x_range, y_range = get_data_range(
@@ -465,6 +485,11 @@ class Heatmap(BaseComponent):
                         x_bins=self._x_bins,
                         y_bins=self._y_bins,
                     )
+                # Sort by x, y for efficient range query predicate pushdown
+                if isinstance(downsampled, pl.LazyFrame):
+                    downsampled = downsampled.sort([self._x_column, self._y_column])
+                else:
+                    downsampled = downsampled.sort([self._x_column, self._y_column])
                 # Store LazyFrame for streaming to disk
                 level_idx = len(level_sizes) - 1 - i
                 if isinstance(downsampled, pl.LazyFrame):
@@ -475,8 +500,11 @@ class Heatmap(BaseComponent):
                 current = downsampled
 
         # Add full resolution as final level (for zoom fallback)
+        # Also sorted for consistent predicate pushdown behavior
         num_compressed = len(level_sizes)
-        self._preprocessed_data[f'level_{num_compressed}'] = self._raw_data
+        self._preprocessed_data[f'level_{num_compressed}'] = self._raw_data.sort(
+            [self._x_column, self._y_column]
+        )
 
         # Store number of levels for reconstruction (includes full resolution)
         self._preprocessed_data['num_levels'] = num_compressed + 1

openms_insight/core/base.py

@@ -15,7 +15,8 @@ if TYPE_CHECKING:
 
 # Cache format version - increment when cache structure changes
 # Version 2: Added sorting by filter columns + smaller row groups for predicate pushdown
-CACHE_VERSION = 2
+# Version 3: Downcast numeric types (Int64→Int32, Float64→Float32) for efficient transfer
+CACHE_VERSION = 3
 
 
 class BaseComponent(ABC):
@@ -236,6 +237,8 @@ class BaseComponent(ABC):
 
     def _save_to_cache(self) -> None:
         """Save preprocessed data to cache."""
+        from ..preprocessing.filtering import optimize_for_transfer, optimize_for_transfer_lazy
+
         # Create directories
         self._cache_dir.mkdir(parents=True, exist_ok=True)
         preprocessed_dir = self._get_preprocessed_dir()
@@ -254,17 +257,23 @@ class BaseComponent(ABC):
             "data_values": {},
         }
 
-        # Save preprocessed data - stream LazyFrames directly to disk
+        # Save preprocessed data with type optimization for efficient transfer
+        # Float64→Float32 reduces Arrow payload size
+        # Int64→Int32 (when safe) avoids BigInt overhead in JavaScript
         for key, value in self._preprocessed_data.items():
             if isinstance(value, pl.LazyFrame):
                 filename = f"{key}.parquet"
                 filepath = preprocessed_dir / filename
-                # Stream directly to disk without full materialization
+                # Apply streaming-safe optimization (Float64→Float32 only)
+                # Int64 bounds checking would require collect(), breaking streaming
+                value = optimize_for_transfer_lazy(value)
                 value.sink_parquet(filepath, compression='zstd')
                 manifest["data_files"][key] = filename
             elif isinstance(value, pl.DataFrame):
                 filename = f"{key}.parquet"
                 filepath = preprocessed_dir / filename
+                # Full optimization including Int64→Int32 with bounds checking
+                value = optimize_for_transfer(value)
                 value.write_parquet(filepath, compression='zstd')
                 manifest["data_files"][key] = filename
             elif self._is_json_serializable(value):