figpack 0.2.6__py3-none-any.whl → 0.2.8__py3-none-any.whl

figpack/spike_sorting/views/SpikeAmplitudes.py

@@ -2,13 +2,15 @@
 SpikeAmplitudes view for figpack - displays spike amplitudes over time
 """
 
-from typing import List, Optional
+from typing import List
 
 import numpy as np
 import zarr
 
 from ...core.figpack_view import FigpackView
 from .SpikeAmplitudesItem import SpikeAmplitudesItem
+from .UnitsTable import UnitsTable, UnitsTableColumn, UnitsTableRow
+from ...views.Box import Box, LayoutItem
 
 
 class SpikeAmplitudes(FigpackView):
@@ -22,8 +24,6 @@ class SpikeAmplitudes(FigpackView):
         start_time_sec: float,
         end_time_sec: float,
         plots: List[SpikeAmplitudesItem],
-        hide_unit_selector: bool = False,
-        height: int = 500,
     ):
         """
         Initialize a SpikeAmplitudes view
@@ -32,18 +32,89 @@ class SpikeAmplitudes(FigpackView):
             start_time_sec: Start time of the view in seconds
             end_time_sec: End time of the view in seconds
             plots: List of SpikeAmplitudesItem objects
-            hide_unit_selector: Whether to hide the unit selector
-            height: Height of the view in pixels
         """
         self.start_time_sec = start_time_sec
         self.end_time_sec = end_time_sec
         self.plots = plots
-        self.hide_unit_selector = hide_unit_selector
-        self.height = height
+
+    @staticmethod
+    def from_nwb_units_table(
+        nwb_url_or_path_or_h5py,
+        *,
+        units_path: str,
+        include_units_selector: bool = False,
+    ):
+        if isinstance(nwb_url_or_path_or_h5py, str):
+            import lindi
+
+            f = lindi.LindiH5pyFile.from_hdf5_file(nwb_url_or_path_or_h5py)
+        else:
+            f = nwb_url_or_path_or_h5py
+        X = f[units_path]
+        spike_amplitudes = X["spike_amplitudes"]
+        # spike_amplitudes_index = X["spike_amplitudes_index"] # presumably the same as spike_times_index
+        spike_times = X["spike_times"]
+        spike_times_index = X["spike_times_index"]
+        id = X["id"]
+        plots = []
+        num_units = len(spike_times_index)
+        start_times = []
+        end_times = []
+        for unit_index in range(num_units):
+            unit_id = id[unit_index]
+            if unit_index > 0:
+                start_index = spike_times_index[unit_index - 1]
+            else:
+                start_index = 0
+            end_index = spike_times_index[unit_index]
+            unit_spike_amplitudes = spike_amplitudes[start_index:end_index]
+            unit_spike_times = spike_times[start_index:end_index]
+            if len(unit_spike_times) == 0:
+                continue
+            start_times.append(unit_spike_times[0])
+            end_times.append(unit_spike_times[-1])
+            plots.append(
+                SpikeAmplitudesItem(
+                    unit_id=str(unit_id),
+                    spike_times_sec=unit_spike_times,
+                    spike_amplitudes=unit_spike_amplitudes,
+                )
+            )
+        view = SpikeAmplitudes(
+            start_time_sec=min(start_times),
+            end_time_sec=max(end_times),
+            plots=plots,
+        )
+        if include_units_selector:
+            columns: List[UnitsTableColumn] = [
+                UnitsTableColumn(key="unitId", label="Unit", dtype="int"),
+            ]
+            rows: List[UnitsTableRow] = []
+            for unit_id in id:
+                rows.append(
+                    UnitsTableRow(
+                        unit_id=str(unit_id),
+                        values={},
+                    )
+                )
+            units_table = UnitsTable(
+                columns=columns,
+                rows=rows,
+            )
+            layout = Box(
+                direction="horizontal",
+                items=[
+                    LayoutItem(view=units_table, max_size=150, title="Units"),
+                    LayoutItem(view=view, title="Spike Amplitudes"),
+                ],
+            )
+            return layout
+        else:
+            return view
 
     def _write_to_zarr_group(self, group: zarr.Group) -> None:
         """
-        Write the SpikeAmplitudes data to a Zarr group
+        Write the SpikeAmplitudes data to a Zarr group using unified storage format
 
         Args:
             group: Zarr group to write data into
@@ -54,36 +125,250 @@ class SpikeAmplitudes(FigpackView):
         # Store view parameters
         group.attrs["start_time_sec"] = self.start_time_sec
         group.attrs["end_time_sec"] = self.end_time_sec
-        group.attrs["hide_unit_selector"] = self.hide_unit_selector
-        group.attrs["height"] = self.height
-
-        # Store the number of plots
-        group.attrs["num_plots"] = len(self.plots)
-
-        # Store metadata for each plot
-        plot_metadata = []
-        for i, plot in enumerate(self.plots):
-            plot_name = f"plot_{i}"
-
-            # Store metadata
-            metadata = {
-                "name": plot_name,
-                "unit_id": str(plot.unit_id),
-                "num_spikes": len(plot.spike_times_sec),
-            }
-            plot_metadata.append(metadata)
 
-            # Create arrays for this plot
+        # Prepare unified data arrays
+        unified_data = self._prepare_unified_data()
+
+        if unified_data["total_spikes"] == 0:
+            # Handle empty data case
+            group.create_dataset("timestamps", data=np.array([], dtype=np.float32))
+            group.create_dataset("unit_indices", data=np.array([], dtype=np.uint16))
+            group.create_dataset("amplitudes", data=np.array([], dtype=np.float32))
+            group.create_dataset("reference_times", data=np.array([], dtype=np.float32))
             group.create_dataset(
-                f"{plot_name}/spike_times_sec",
-                data=plot.spike_times_sec,
-                dtype=np.float32,
+                "reference_indices", data=np.array([], dtype=np.uint32)
             )
-            group.create_dataset(
-                f"{plot_name}/spike_amplitudes",
-                data=plot.spike_amplitudes,
-                dtype=np.float32,
+            group.attrs["unit_ids"] = []
+            group.attrs["total_spikes"] = 0
+            return
+
+        chunks = (
+            (2_000_000,)
+            if unified_data["total_spikes"] > 2_000_000
+            else (len(unified_data["timestamps"]),)
+        )
+        # Store main data arrays
+        group.create_dataset(
+            "timestamps",
+            data=unified_data["timestamps"],
+            dtype=np.float32,
+            chunks=chunks,
+        )
+        group.create_dataset(
+            "unit_indices",
+            data=unified_data["unit_indices"],
+            dtype=np.uint16,
+            chunks=chunks,
+        )
+        group.create_dataset(
+            "amplitudes",
+            data=unified_data["amplitudes"],
+            dtype=np.float32,
+            chunks=chunks,
+        )
+        group.create_dataset(
+            "reference_times",
+            data=unified_data["reference_times"],
+            dtype=np.float32,
+            chunks=(len(unified_data["reference_times"]),),
+        )
+        group.create_dataset(
+            "reference_indices",
+            data=unified_data["reference_indices"],
+            dtype=np.uint32,
+            chunks=(len(unified_data["reference_indices"]),),
+        )
+
+        # Store unit ID mapping
+        group.attrs["unit_ids"] = unified_data["unit_ids"]
+        group.attrs["total_spikes"] = unified_data["total_spikes"]
+
+        # Create subsampled data
+        subsampled_data = self._create_subsampled_data(
+            unified_data["timestamps"],
+            unified_data["unit_indices"],
+            unified_data["amplitudes"],
+        )
+
+        if subsampled_data:
+            subsampled_group = group.create_group("subsampled_data")
+            for factor_name, data in subsampled_data.items():
+                chunks = (
+                    (2_000_000,)
+                    if len(data["timestamps"]) > 2_000_000
+                    else (len(data["timestamps"]),)
+                )
+                factor_group = subsampled_group.create_group(factor_name)
+                factor_group.create_dataset(
+                    "timestamps",
+                    data=data["timestamps"],
+                    dtype=np.float32,
+                    chunks=chunks,
+                )
+                factor_group.create_dataset(
+                    "unit_indices",
+                    data=data["unit_indices"],
+                    dtype=np.uint16,
+                    chunks=chunks,
+                )
+                factor_group.create_dataset(
+                    "amplitudes",
+                    data=data["amplitudes"],
+                    dtype=np.float32,
+                    chunks=chunks,
+                )
+                factor_group.create_dataset(
+                    "reference_times",
+                    data=data["reference_times"],
+                    dtype=np.float32,
+                    chunks=(len(data["reference_times"]),),
+                )
+                factor_group.create_dataset(
+                    "reference_indices",
+                    data=data["reference_indices"],
+                    dtype=np.uint32,
+                    chunks=(len(data["reference_indices"]),),
+                )
+
+    def _prepare_unified_data(self) -> dict:
+        """
+        Prepare unified data arrays from all plots
+
+        Returns:
+            Dictionary containing unified arrays and metadata
+        """
+        if not self.plots:
+            return {
+                "timestamps": np.array([], dtype=np.float32),
+                "unit_indices": np.array([], dtype=np.uint16),
+                "amplitudes": np.array([], dtype=np.float32),
+                "reference_times": np.array([], dtype=np.float32),
+                "reference_indices": np.array([], dtype=np.uint32),
+                "unit_ids": [],
+                "total_spikes": 0,
+            }
+
+        # Create unit ID mapping
+        unit_ids = [str(plot.unit_id) for plot in self.plots]
+        unit_id_to_index = {unit_id: i for i, unit_id in enumerate(unit_ids)}
+
+        # Collect all spikes with their unit indices
+        all_spikes = []
+        for plot in self.plots:
+            unit_index = unit_id_to_index[str(plot.unit_id)]
+            for time, amplitude in zip(plot.spike_times_sec, plot.spike_amplitudes):
+                all_spikes.append((float(time), unit_index, float(amplitude)))
+
+        if not all_spikes:
+            return {
+                "timestamps": np.array([], dtype=np.float32),
+                "unit_indices": np.array([], dtype=np.uint16),
+                "amplitudes": np.array([], dtype=np.float32),
+                "reference_times": np.array([], dtype=np.float32),
+                "reference_indices": np.array([], dtype=np.uint32),
+                "unit_ids": unit_ids,
+                "total_spikes": 0,
+            }
+
+        # Sort by timestamp
+        all_spikes.sort(key=lambda x: x[0])
+
+        # Extract sorted arrays
+        timestamps = np.array([spike[0] for spike in all_spikes], dtype=np.float32)
+        unit_indices = np.array([spike[1] for spike in all_spikes], dtype=np.uint16)
+        amplitudes = np.array([spike[2] for spike in all_spikes], dtype=np.float32)
+
+        # Generate reference arrays
+        reference_times, reference_indices = self._generate_reference_arrays(timestamps)
+
+        return {
+            "timestamps": timestamps,
+            "unit_indices": unit_indices,
+            "amplitudes": amplitudes,
+            "reference_times": reference_times,
+            "reference_indices": reference_indices,
+            "unit_ids": unit_ids,
+            "total_spikes": len(all_spikes),
+        }
+
+    def _generate_reference_arrays(
+        self, timestamps: np.ndarray, interval_sec: float = 1.0
+    ) -> tuple:
+        """
+        Generate reference arrays using actual timestamps from the data
+
+        Args:
+            timestamps: Sorted array of timestamps
+            interval_sec: Minimum interval between reference points
+
+        Returns:
+            Tuple of (reference_times, reference_indices)
+        """
+        if len(timestamps) == 0:
+            return np.array([], dtype=np.float32), np.array([], dtype=np.uint32)
+
+        reference_times = []
+        reference_indices = []
+
+        current_ref_time = timestamps[0]
+        reference_times.append(current_ref_time)
+        reference_indices.append(0)
+
+        # Find the next reference point at least interval_sec later
+        for i, timestamp in enumerate(timestamps):
+            if timestamp >= current_ref_time + interval_sec:
+                reference_times.append(timestamp)
+                reference_indices.append(i)
+                current_ref_time = timestamp
+
+        return np.array(reference_times, dtype=np.float32), np.array(
+            reference_indices, dtype=np.uint32
+        )
+
+    def _create_subsampled_data(
+        self, timestamps: np.ndarray, unit_indices: np.ndarray, amplitudes: np.ndarray
+    ) -> dict:
+        """
+        Create subsampled data with geometric progression factors
+
+        Args:
+            timestamps: Original timestamps array
+            unit_indices: Original unit indices array
+            amplitudes: Original amplitudes array
+
+        Returns:
+            Dictionary of subsampled data by factor
+        """
+        subsampled_data = {}
+        factor = 4
+        current_timestamps = timestamps
+        current_unit_indices = unit_indices
+        current_amplitudes = amplitudes
+
+        while len(current_timestamps) >= 500000:
+            # Create subsampled version by taking every Nth spike
+            subsampled_indices = np.arange(0, len(current_timestamps), factor)
+            subsampled_timestamps = current_timestamps[subsampled_indices]
+            subsampled_unit_indices = current_unit_indices[subsampled_indices]
+            subsampled_amplitudes = current_amplitudes[subsampled_indices]
+
+            # Generate reference arrays for this subsampled level
+            ref_times, ref_indices = self._generate_reference_arrays(
+                subsampled_timestamps
             )
 
-        # Store the plot metadata
-        group.attrs["plots"] = plot_metadata
+            subsampled_data[f"factor_{factor}"] = {
+                "timestamps": subsampled_timestamps,
+                "unit_indices": subsampled_unit_indices,
+                "amplitudes": subsampled_amplitudes,
+                "reference_times": ref_times,
+                "reference_indices": ref_indices,
+            }
+
+            # Prepare for next iteration
+            current_timestamps = subsampled_timestamps
+            current_unit_indices = subsampled_unit_indices
+            current_amplitudes = subsampled_amplitudes
+            factor *= 4  # Geometric progression: 4, 16, 64, 256, ...
+
+        return subsampled_data

figpack/views/DataFrame.py

@@ -0,0 +1,109 @@
+"""
+DataFrame view for figpack - displays pandas DataFrames as interactive tables
+"""
+
+import json
+from typing import Any, Dict, Union
+
+import numpy as np
+import pandas as pd
+import zarr
+
+from ..core.figpack_view import FigpackView
+
+
+class DataFrame(FigpackView):
+    """
+    A DataFrame visualization component for displaying pandas DataFrames as interactive tables
+    """
+
+    def __init__(self, df: pd.DataFrame):
+        """
+        Initialize a DataFrame view
+
+        Args:
+            df: The pandas DataFrame to display
+
+        Raises:
+            ValueError: If df is not a pandas DataFrame
+        """
+        if not isinstance(df, pd.DataFrame):
+            raise ValueError("df must be a pandas DataFrame")
+
+        self.df = df
+
+    def _write_to_zarr_group(self, group: zarr.Group) -> None:
+        """
+        Write the DataFrame data to a Zarr group
+
+        Args:
+            group: Zarr group to write data into
+        """
+        # Set the view type
+        group.attrs["view_type"] = "DataFrame"
+
+        try:
+            # Convert DataFrame to CSV string
+            csv_string = self.df.to_csv(index=False)
+
+            # Convert CSV string to bytes and store in numpy array
+            csv_bytes = csv_string.encode("utf-8")
+            csv_array = np.frombuffer(csv_bytes, dtype=np.uint8)
+
+            # Store the CSV data as compressed array
+            group.create_dataset(
+                "csv_data",
+                data=csv_array,
+                dtype=np.uint8,
+                chunks=True,
+                compressor=zarr.Blosc(
+                    cname="zstd", clevel=3, shuffle=zarr.Blosc.SHUFFLE
+                ),
+            )
+
+            # Store metadata about the DataFrame
+            group.attrs["data_size"] = len(csv_bytes)
+            group.attrs["row_count"] = len(self.df)
+            group.attrs["column_count"] = len(self.df.columns)
+
+            # Store column information
+            column_info = []
+            for col in self.df.columns:
+                dtype_str = str(self.df[col].dtype)
+                # Simplify dtype names for frontend
+                if dtype_str.startswith("int"):
+                    simple_dtype = "integer"
+                elif dtype_str.startswith("float"):
+                    simple_dtype = "float"
+                elif dtype_str.startswith("bool"):
+                    simple_dtype = "boolean"
+                elif dtype_str.startswith("datetime"):
+                    simple_dtype = "datetime"
+                elif dtype_str == "object":
+                    # Check if it's actually strings
+                    if self.df[col].dtype == "object":
+                        simple_dtype = "string"
+                    else:
+                        simple_dtype = "object"
+                else:
+                    simple_dtype = "string"
+
+                column_info.append(
+                    {"name": str(col), "dtype": dtype_str, "simple_dtype": simple_dtype}
+                )
+
+            # Store column info as JSON string
+            column_info_json = json.dumps(column_info)
+            group.attrs["column_info"] = column_info_json
+
+        except Exception as e:
+            # If DataFrame processing fails, store error information
+            group.attrs["error"] = f"Failed to process DataFrame: {str(e)}"
+            group.attrs["row_count"] = 0
+            group.attrs["column_count"] = 0
+            group.attrs["data_size"] = 0
+            group.attrs["column_info"] = "[]"
+            # Create empty array as placeholder
+            group.create_dataset(
+                "csv_data", data=np.array([], dtype=np.uint8), dtype=np.uint8
+            )