openms-insight 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

openms_insight/preprocessing/filtering.py

@@ -8,6 +8,88 @@ import polars as pl
 import streamlit as st
 
 
+def optimize_for_transfer(df: pl.DataFrame) -> pl.DataFrame:
+    """
+    Optimize DataFrame types for efficient Arrow transfer to frontend.
+
+    This function downcasts numeric types to reduce Arrow payload size and
+    avoid BigInt overhead in JavaScript:
+    - Int64 → Int32 (if values fit): Avoids BigInt conversion in JS
+    - Float64 → Float32: Sufficient precision for visualization
+
+    Args:
+        df: Polars DataFrame to optimize
+
+    Returns:
+        DataFrame with optimized types
+    """
+    if len(df) == 0:
+        return df
+
+    casts = []
+
+    for col in df.columns:
+        dtype = df[col].dtype
+
+        # Downcast Int64 to Int32 to avoid BigInt in JavaScript
+        # JS safe integer is 2^53, but Int32 range is simpler and sufficient for most data
+        if dtype == pl.Int64:
+            # Get min/max in a single pass
+            stats = df.select([
+                pl.col(col).min().alias('min'),
+                pl.col(col).max().alias('max'),
+            ]).row(0)
+            col_min, col_max = stats
+
+            if col_min is not None and col_max is not None:
+                # Int32 range: -2,147,483,648 to 2,147,483,647
+                if col_min >= -2147483648 and col_max <= 2147483647:
+                    casts.append(pl.col(col).cast(pl.Int32))
+
+        # Downcast Float64 to Float32 (sufficient for display)
+        # Float32 has ~7 significant digits - enough for visualization
+        elif dtype == pl.Float64:
+            casts.append(pl.col(col).cast(pl.Float32))
+
+    if casts:
+        df = df.with_columns(casts)
+
+    return df
+
+
+def optimize_for_transfer_lazy(lf: pl.LazyFrame) -> pl.LazyFrame:
+    """
+    Optimize LazyFrame types for efficient Arrow transfer (streaming-safe).
+
+    Unlike optimize_for_transfer(), this only applies optimizations that don't
+    require knowing the data values, preserving the ability to stream via sink_parquet().
+
+    Currently applies:
+    - Float64 → Float32: Always safe, no bounds check needed
+
+    Int64 → Int32 is NOT applied here because it requires bounds checking.
+    Use optimize_for_transfer() on collected DataFrames for full optimization.
+
+    Args:
+        lf: Polars LazyFrame to optimize
+
+    Returns:
+        LazyFrame with Float64 columns cast to Float32
+    """
+    schema = lf.collect_schema()
+    casts = []
+
+    for col, dtype in zip(schema.names(), schema.dtypes()):
+        # Only Float64 → Float32 is safe without bounds checking
+        if dtype == pl.Float64:
+            casts.append(pl.col(col).cast(pl.Float32))
+
+    if casts:
+        lf = lf.with_columns(casts)
+
+    return lf
+
+
 def _make_cache_key(
     filters: Dict[str, str],
     state: Dict[str, Any],
@@ -81,35 +163,30 @@ def compute_dataframe_hash(df: pl.DataFrame) -> str:
     return hashlib.sha256(hash_input).hexdigest()
 
 
-@st.cache_data(ttl=300, max_entries=100)
-def _cached_filter_and_collect(
-    _data: pl.LazyFrame,
+def _filter_and_collect(
+    data: pl.LazyFrame,
     filters_tuple: Tuple[Tuple[str, str], ...],
     state_tuple: Tuple[Tuple[str, Any], ...],
     columns_tuple: Optional[Tuple[str, ...]] = None,
-    filter_defaults_tuple: Optional[Tuple[Tuple[str, Any], ...]] = None,
 ) -> Tuple[pd.DataFrame, str]:
     """
-    Filter data and collect with caching.
+    Filter data and collect.
 
-    This function is cached by Streamlit, so repeated calls with the same
-    filter state will return cached results without re-executing the query.
+    This function executes the filter query. Caching is handled at a higher
+    level (per-component in bridge.py) to ensure memory = O(num_components).
 
     Returns pandas DataFrame for efficient Arrow serialization to frontend.
 
     Args:
-        _data: LazyFrame to filter (underscore prefix tells st.cache_data to hash by id)
+        data: LazyFrame to filter
         filters_tuple: Tuple of (identifier, column) pairs from filters dict
         state_tuple: Tuple of (identifier, value) pairs for current selection state
             (already has defaults applied from _make_cache_key)
         columns_tuple: Optional tuple of column names to select (projection)
-        filter_defaults_tuple: Optional tuple of (identifier, default_value) pairs
-            (included for cache key differentiation)
 
     Returns:
         Tuple of (pandas DataFrame, hash string)
     """
-    data = _data
     filters = dict(filters_tuple)
     state = dict(state_tuple)  # Already has defaults applied
 
@@ -138,6 +215,8 @@ def _cached_filter_and_collect(
         data = data.filter(pl.col(column) == selected_value)
 
     # Collect to Polars DataFrame
+    # Note: Type optimization (Int64→Int32, Float64→Float32) is applied at cache
+    # creation time in base.py._save_to_cache(), so data is already optimized
     df_polars = data.collect()
 
     # Compute hash efficiently (no pickle)
@@ -182,19 +261,17 @@ def filter_and_collect_cached(
     if isinstance(data, pl.DataFrame):
         data = data.lazy()
 
-    # Convert to tuples for caching (dicts aren't hashable)
+    # Convert to tuples for consistent processing
     filters_tuple = tuple(sorted(filters.items()))
     # Pass filter_defaults to _make_cache_key so defaults are applied to state
     state_tuple = _make_cache_key(filters, state, filter_defaults)
     columns_tuple = tuple(columns) if columns else None
-    filter_defaults_tuple = tuple(sorted(filter_defaults.items())) if filter_defaults else None
 
-    return _cached_filter_and_collect(
+    return _filter_and_collect(
         data,
         filters_tuple,
         state_tuple,
         columns_tuple,
-        filter_defaults_tuple,
     )
 
 

openms_insight/rendering/bridge.py

@@ -42,43 +42,107 @@ _vue_component_func = None
 # Vue's echoed hash doesn't match the current data hash
 _VUE_ECHOED_HASH_KEY = "_svc_vue_echoed_hashes"
 
+# Session state key for per-component data cache
+# Each component stores exactly one entry (current filter state)
+_COMPONENT_DATA_CACHE_KEY = "_svc_component_data_cache"
 
 
-@st.cache_data(max_entries=10, show_spinner=False)
-def _cached_prepare_vue_data(
-    _component: 'BaseComponent',
+def _get_component_cache() -> Dict[str, Any]:
+    """Get per-component data cache from session state."""
+    if _COMPONENT_DATA_CACHE_KEY not in st.session_state:
+        st.session_state[_COMPONENT_DATA_CACHE_KEY] = {}
+    return st.session_state[_COMPONENT_DATA_CACHE_KEY]
+
+
+def clear_component_cache() -> None:
+    """
+    Clear all per-component cached data.
+
+    Call this when loading a new file to ensure fresh data is sent to Vue.
+    """
+    if _COMPONENT_DATA_CACHE_KEY in st.session_state:
+        st.session_state[_COMPONENT_DATA_CACHE_KEY].clear()
+
+
+def _get_cached_vue_data(
     component_id: str,
     filter_state_hashable: Tuple[Tuple[str, Any], ...],
-    _data_id: int,
-    _state_dict: Dict[str, Any],
-) -> Tuple[Dict[str, Any], str]:
+) -> Optional[Tuple[Dict[str, Any], str]]:
     """
-    Cached wrapper for _prepare_vue_data.
+    Get cached Vue data for component if filter state matches.
 
-    Cache key is based on:
-    - component_id: unique key for this component instance
-    - filter_state_hashable: hashable version of state values (for cache key only)
+    Each component has exactly one cached entry. If filter state changed,
+    returns None (cache miss).
 
-    The _component, _data_id, and _state_dict parameters are prefixed with underscore
-    so they are not hashed (component instances are not hashable, and state_dict
-    may contain unhashable values like dicts).
+    Args:
+        component_id: Unique identifier for this component
+        filter_state_hashable: Current filter state (for cache validation)
+
+    Returns:
+        Tuple of (vue_data, data_hash) if cache hit, None otherwise
+    """
+    cache = _get_component_cache()
+    if component_id in cache:
+        cached_state, vue_data, data_hash = cache[component_id]
+        if cached_state == filter_state_hashable:
+            return (vue_data, data_hash)
+    return None
+
+
+def _set_cached_vue_data(
+    component_id: str,
+    filter_state_hashable: Tuple[Tuple[str, Any], ...],
+    vue_data: Dict[str, Any],
+    data_hash: str,
+) -> None:
+    """
+    Cache Vue data for component, replacing any previous entry.
+
+    Each component stores exactly one entry, so memory = O(num_components).
+
+    Args:
+        component_id: Unique identifier for this component
+        filter_state_hashable: Current filter state
+        vue_data: Data to cache
+        data_hash: Hash of the data
+    """
+    cache = _get_component_cache()
+    cache[component_id] = (filter_state_hashable, vue_data, data_hash)
+
+
+def _prepare_vue_data_cached(
+    component: 'BaseComponent',
+    component_id: str,
+    filter_state_hashable: Tuple[Tuple[str, Any], ...],
+    state_dict: Dict[str, Any],
+) -> Tuple[Dict[str, Any], str]:
+    """
+    Prepare Vue data with per-component caching.
+
+    Each component caches exactly one entry (its current filter state).
+    When filter state changes, old entry is replaced - memory stays bounded.
 
     Args:
-        _component: The component to prepare data for (not hashed)
+        component: The component to prepare data for
         component_id: Unique identifier for this component
-        filter_state_hashable: Tuple of (identifier, hashable_value) for cache key
-        _data_id: id() of the raw data object (not used in cache key)
-        _state_dict: Original state dict with actual values (not hashed)
+        filter_state_hashable: Hashable version of filter state (for cache key)
+        state_dict: Original state dict with actual values
 
     Returns:
         Tuple of (vue_data dict, data_hash string)
     """
-    # Use the original state dict (not the hashable version)
-    vue_data = _component._prepare_vue_data(_state_dict)
+    # Check cache first
+    cached = _get_cached_vue_data(component_id, filter_state_hashable)
+    if cached is not None:
+        return cached
 
-    # Compute hash before any conversion
+    # Cache miss - compute data
+    vue_data = component._prepare_vue_data(state_dict)
     data_hash = _hash_data(vue_data)
 
+    # Store in cache (replaces any previous entry for this component)
+    _set_cached_vue_data(component_id, filter_state_hashable, vue_data, data_hash)
+
     return vue_data, data_hash
 
 
@@ -177,15 +241,12 @@ def render_component(
     component_type = component._get_vue_component_name()
     component_id = f"{component_type}:{key}"
 
-    # Get data identity - cache invalidates if raw data object changes
-    data_id = id(component._raw_data)
-
-    # Get component data using cached function
-    # Cache key: (component_id, filter_state_hashable, data_id)
+    # Get component data using per-component cache
+    # Each component stores exactly one entry (current filter state)
     # - Filterless components: filter_state=() always → always cache hit
     # - Filtered components: cache hit when filter values unchanged
-    vue_data, data_hash = _cached_prepare_vue_data(
-        component, component_id, filter_state_hashable, data_id, relevant_state
+    vue_data, data_hash = _prepare_vue_data_cached(
+        component, component_id, filter_state_hashable, relevant_state
     )
 
     component_args = component._get_component_args()

{openms_insight-0.1.0.dist-info → openms_insight-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openms-insight
-Version: 0.1.0
+Version: 0.1.2
 Summary: Interactive visualization components for mass spectrometry data in Streamlit
 Project-URL: Homepage, https://github.com/t0mdavid-m/OpenMS-Insight
 Project-URL: Documentation, https://github.com/t0mdavid-m/OpenMS-Insight#readme
@@ -43,7 +43,7 @@ Interactive visualization components for mass spectrometry data in Streamlit, ba
 ## Features
 
 - **Cross-component selection linking** via shared identifiers
-- **Polars LazyFrame support** for efficient data handling
+- **Memory-efficient preprocessing** via subprocess isolation
 - **Automatic disk caching** with config-based invalidation
 - **Table component** (Tabulator.js) with filtering, sorting, go-to, pagination
 - **Line plot component** (Plotly.js) with highlighting, annotations, zoom
@@ -60,7 +60,6 @@ pip install openms-insight
 
 ```python
 import streamlit as st
-import polars as pl
 from openms_insight import Table, LinePlot, StateManager
 
 # Create state manager for cross-component linking
@@ -69,7 +68,7 @@ state_manager = StateManager()
 # Create a table - clicking a row sets the 'item' selection
 table = Table(
     cache_id="items_table",
-    data=pl.scan_parquet("items.parquet"),
+    data_path="items.parquet",
     interactivity={'item': 'item_id'},
     column_definitions=[
         {'field': 'item_id', 'title': 'ID', 'sorter': 'number'},
@@ -81,7 +80,7 @@ table(state_manager=state_manager)
 # Create a linked plot - filters by the selected 'item'
 plot = LinePlot(
     cache_id="values_plot",
-    data=pl.scan_parquet("values.parquet"),
+    data_path="values.parquet",
     filters={'item': 'item_id'},
     x_column='x',
     y_column='y',
@@ -100,14 +99,14 @@ Components communicate through **identifiers** using two mechanisms:
 # Master table: no filters, sets 'spectrum' on click
 master = Table(
     cache_id="spectra",
-    data=spectra_data,
+    data_path="spectra.parquet",
     interactivity={'spectrum': 'scan_id'},  # Click -> sets spectrum=scan_id
 )
 
 # Detail table: filters by 'spectrum', sets 'peak' on click
 detail = Table(
     cache_id="peaks",
-    data=peaks_data,
+    data_path="peaks.parquet",
     filters={'spectrum': 'scan_id'},        # Filters where scan_id = selected spectrum
     interactivity={'peak': 'peak_id'},      # Click -> sets peak=peak_id
 )
@@ -115,7 +114,7 @@ detail = Table(
 # Plot: filters by 'spectrum', highlights selected 'peak'
 plot = LinePlot(
     cache_id="plot",
-    data=peaks_data,
+    data_path="peaks.parquet",
     filters={'spectrum': 'scan_id'},
     interactivity={'peak': 'peak_id'},
     x_column='mass',
@@ -134,7 +133,7 @@ Interactive table using Tabulator.js with filtering dialogs, sorting, pagination
 ```python
 Table(
     cache_id="spectra_table",
-    data=pl.scan_parquet("spectra.parquet"),
+    data_path="spectra.parquet",
     interactivity={'spectrum': 'scan_id'},
     column_definitions=[
         {'field': 'scan_id', 'title': 'Scan', 'sorter': 'number'},
@@ -156,7 +155,7 @@ Stick-style line plot using Plotly.js for mass spectra visualization.
 ```python
 LinePlot(
     cache_id="spectrum_plot",
-    data=pl.scan_parquet("peaks.parquet"),
+    data_path="peaks.parquet",
     filters={'spectrum': 'scan_id'},
     interactivity={'peak': 'peak_id'},
     x_column='mass',
@@ -176,7 +175,7 @@ LinePlot(
 ```python
 Heatmap(
     cache_id="peaks_heatmap",
-    data=pl.scan_parquet("all_peaks.parquet"),
+    data_path="all_peaks.parquet",
     x_column='retention_time',
     y_column='mass',
     intensity_column='intensity',
@@ -213,7 +212,8 @@ All components accept these common arguments:
 | Argument | Type | Default | Description |
 |----------|------|---------|-------------|
 | `cache_id` | `str` | **Required** | Unique identifier for disk cache |
-| `data` | `pl.LazyFrame` | `None` | Polars LazyFrame with source data |
+| `data_path` | `str` | `None` | Path to parquet file (preferred - uses subprocess for memory efficiency) |
+| `data` | `pl.LazyFrame` | `None` | Polars LazyFrame (alternative to data_path, in-process preprocessing) |
 | `filters` | `Dict[str, str]` | `None` | Map identifier -> column for filtering |
 | `interactivity` | `Dict[str, str]` | `None` | Map identifier -> column for click actions |
 | `cache_path` | `str` | `"."` | Base directory for cache storage |

{openms_insight-0.1.0.dist-info → openms_insight-0.1.2.dist-info}/RECORD

@@ -1,27 +1,28 @@
 openms_insight/__init__.py,sha256=-QMPUfCk1DwwlQKW6NFSZodRz-r82R4rJPOiWDf_mwA,830
 openms_insight/components/__init__.py,sha256=aNy1E8ttqQiiA5Ka9sviRy17gQT8lfc_EHsl1v1o-UQ,177
-openms_insight/components/heatmap.py,sha256=wqNgmccxI5PeTC08kVf06UUtVEg8YhUf_eY9CN4wAMw,33125
-openms_insight/components/lineplot.py,sha256=klGzpGO-gi76sIocr9T4TYUTV-ztfIIAtdd5k-U-aGE,19382
+openms_insight/components/heatmap.py,sha256=VNlc3wgrzkujj0jmX4RN93yKhPPOvGXdo4YoyQ_elJs,36223
+openms_insight/components/lineplot.py,sha256=fcul7W2RZlTQBse6pDbTAS14_HE14gLDdsUZ-YLjgAQ,19948
 openms_insight/components/sequenceview.py,sha256=IPLyUiXTLYwdZ79B3pZDH1intUsjBIknTSLUQ2ftdD4,14359
-openms_insight/components/table.py,sha256=VaUbn2lckeT3shqn9quDvw1HGMsiQrIPPK4w1XQFzzc,14795
+openms_insight/components/table.py,sha256=P2VIGcPf2iuVJ07rl1el_kTjUKBjlJ_PQeFQAO9sgbw,15361
 openms_insight/core/__init__.py,sha256=yKRgLfRm1CQm6ZGqQiyDwblzw2Qh3jw57U-ZSd63PYk,338
-openms_insight/core/base.py,sha256=e939N2cu0TEllj09Uw7AKS0FqxE2n_XecwF3ckM_YM0,14746
+openms_insight/core/base.py,sha256=fkxAM9KVvzD6T8Guk3AeFWkNK4jeDgM3MbYPbmma8mw,16364
 openms_insight/core/cache.py,sha256=3fnPDWjuWUnxazK2XflcUIeRZZPQ3N45kAKYu-xGBKw,1197
 openms_insight/core/registry.py,sha256=iBb9xWvgOisxjVX74wtCvqWO20eBW5xH-XcNcVuW-Dg,2107
 openms_insight/core/state.py,sha256=VhY_3Yg-zbx9VsdqHElsNyD2UbJj6ApE4TRcUywDrjQ,6871
+openms_insight/core/subprocess_preprocess.py,sha256=E08qEyVinwRM_PXq0bZk-lHJcYeDHmdr3ok4yTRC63I,3155
 openms_insight/preprocessing/__init__.py,sha256=XFnxlvG-VRMrFbreGmFSIDqfYhev8WPUyZluadMmCgY,462
 openms_insight/preprocessing/compression.py,sha256=8oe-vqeSX-Xf1okPX0C0km2DRnl6Iz53Y1UgsLdMUPI,10499
-openms_insight/preprocessing/filtering.py,sha256=chvmCqLLsMe5mXwG_S4KUagNqJZFLI_iLKKr9g-MkLM,10907
+openms_insight/preprocessing/filtering.py,sha256=ZG8P1RLzwJVGdhElXD6UOny0GFz-aNr0LesSKZPVJTA,13191
 openms_insight/rendering/__init__.py,sha256=i9MRFrAEAX5kjbOnMjw2woP_6mEaI6HcxtbrNwUxNaM,198
-openms_insight/rendering/bridge.py,sha256=yzXPyK_oKVGBymeDWxgzXeI_jOP7ntJ4p_npOQ1-d1A,11279
+openms_insight/rendering/bridge.py,sha256=cPZWMz07jKOcWkkL7FdCY8w1lNk8bME_1xuulCO3VgY,13146
 openms_insight/js-component/dist/index.html,sha256=LSJ3B_YmGUrCCdZ1UaZO2p6Wqsih6nTH62Z_0uZxpD8,430
-openms_insight/js-component/dist/assets/index.css,sha256=h0LrX503uebeJrJB0LtoPCVwA4QfDBMmOOR_tU2Wkic,883995
-openms_insight/js-component/dist/assets/index.js,sha256=pakZe7BdeEjiDCPhzGYqEFumB3WtbdnnXYnBuTDVfWo,6060653
+openms_insight/js-component/dist/assets/index.css,sha256=ACP0LGYUGQusvfDX9igw5DfoQr60P4IFgSqWhACY46A,884228
+openms_insight/js-component/dist/assets/index.js,sha256=9fkS8XuO6U_3-qnGhjpeiLnkacKZ9eDUGTbWyNgHwTc,6061688
 openms_insight/js-component/dist/assets/materialdesignicons-webfont.eot,sha256=CxgxBNL8XyYZbnc8d72vLgVQn9QlnS0V7O3Kebh-hPk,1307880
 openms_insight/js-component/dist/assets/materialdesignicons-webfont.ttf,sha256=YeirpaTpgf4iz3yOi82-oAR251xiw38Bv37jM2HWhCg,1307660
 openms_insight/js-component/dist/assets/materialdesignicons-webfont.woff,sha256=pZKKDVwvYk5G-Y2bFcL2AEU3f3xZTdeKF1kTLqO0Y-s,587984
 openms_insight/js-component/dist/assets/materialdesignicons-webfont.woff2,sha256=Zi_vqPL4qVwYWI0hd0eJwQfGTnccvmWmmvRikcQxGvw,403216
-openms_insight-0.1.0.dist-info/METADATA,sha256=S7rH1PAk9nJDomrQSDnJFmiGC7v8JcwnfrGcBqe55iM,7288
-openms_insight-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-openms_insight-0.1.0.dist-info/licenses/LICENSE,sha256=INFF4rOMmpah7Oi14hLqu7NTOsx56KRRNChAAUcfh2E,1823
-openms_insight-0.1.0.dist-info/RECORD,,
+openms_insight-0.1.2.dist-info/METADATA,sha256=PZwmhlJVjzVeI23Ow3axp28YYg_84VxxsH9FrkFuRmg,7385
+openms_insight-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+openms_insight-0.1.2.dist-info/licenses/LICENSE,sha256=INFF4rOMmpah7Oi14hLqu7NTOsx56KRRNChAAUcfh2E,1823
+openms_insight-0.1.2.dist-info/RECORD,,