masster 0.4.20__py3-none-any.whl → 0.4.22__py3-none-any.whl

masster/study/load.py

@@ -1257,17 +1257,53 @@ def load_features(self):
         feature_map = oms.FeatureMap()
 
         # Convert DataFrame features to OpenMS Features
+        # Keep track of next available feature_id for this sample
+        next_feature_id = 1
+        used_feature_ids = set()
+        
+        # First pass: collect existing feature_ids to avoid conflicts
+        for feature_row in sample_features.iter_rows(named=True):
+            if feature_row["feature_id"] is not None:
+                used_feature_ids.add(int(feature_row["feature_id"]))
+                
+        # Find the next available feature_id
+        while next_feature_id in used_feature_ids:
+            next_feature_id += 1
+        
         for feature_row in sample_features.iter_rows(named=True):
             feature = oms.Feature()
 
             # Set properties from DataFrame (handle missing values gracefully)
             try:
-                feature.setUniqueId(int(feature_row["feature_id"]))
+                # Skip features with missing critical data
+                if feature_row["mz"] is None:
+                    self.logger.warning("Skipping feature due to missing mz")
+                    continue
+                if feature_row["rt"] is None:
+                    self.logger.warning("Skipping feature due to missing rt")
+                    continue
+                if feature_row["inty"] is None:
+                    self.logger.warning("Skipping feature due to missing inty")
+                    continue
+                
+                # Handle missing feature_id by generating a new one
+                if feature_row["feature_id"] is None:
+                    feature_id = next_feature_id
+                    next_feature_id += 1
+                    self.logger.debug(f"Generated new feature_id {feature_id} for feature with missing ID")
+                else:
+                    feature_id = int(feature_row["feature_id"])
+                
+                feature.setUniqueId(feature_id)
                 feature.setMZ(float(feature_row["mz"]))
                 feature.setRT(float(feature_row["rt"]))
                 feature.setIntensity(float(feature_row["inty"]))
-                feature.setOverallQuality(float(feature_row["quality"]))
-                feature.setCharge(int(feature_row["charge"]))
+                
+                # Handle optional fields that might be None
+                if feature_row.get("quality") is not None:
+                    feature.setOverallQuality(float(feature_row["quality"]))
+                if feature_row.get("charge") is not None:
+                    feature.setCharge(int(feature_row["charge"]))
 
                 # Add to feature map
                 feature_map.push_back(feature)

masster/study/merge.py

@@ -400,7 +400,7 @@ def merge(self, **kwargs) -> None:
         # Feature maps will be generated on-demand within each merge method
     
     self.logger.info(
-        f"Merge: {params.method}, samples={params.min_samples}, rt_tol={params.rt_tol}s, mz_tol={params.mz_tol}Da, min_rel_cc_size={params.min_rel_cc_size}, max_pairwise_log_fc={params.max_pairwise_log_fc}, max_nr_conflicts={params.max_nr_conflicts}"
+        f"Merge: {params.method}, samples={params.min_samples}, rt_tol={params.rt_tol}s, mz_tol={params.mz_tol}Da"
     )
     
     # Initialize
@@ -446,7 +446,7 @@ def merge(self, **kwargs) -> None:
         # Note: _merge_qt_chunked populates consensus_df directly, no need to extract
     
     # Enhanced post-clustering to merge over-segmented features (for qt and kd methods)
-    if params.method in ['qt', 'sensitivity', 'qt_chunked', 'kd_chunked']:
+    if params.method in ['qt', 'sensitivity', 'qt_chunked', 'kd_chunked', 'quality']:
         self._consensus_cleanup(params.rt_tol, params.mz_tol)
     
     # Perform adduct grouping
@@ -705,11 +705,11 @@ def _merge_kd_strict(self, params: merge_defaults) -> oms.ConsensusMap:
         optimized_params = params
     
     # Phase 1: Standard KD clustering
-    self.logger.info("Initial KD clustering")
+    self.logger.debug("Initial KD clustering")
     consensus_map = _merge_kd(self, optimized_params)
     
     # Phase 2: Post-processing quality control
-    self.logger.info("Post-processing quality control")
+    self.logger.debug("Post-processing quality control")
     consensus_map = _apply_kd_strict_postprocessing(self, consensus_map, optimized_params)
     
     return consensus_map
@@ -911,7 +911,7 @@ def _apply_kd_strict_postprocessing(self, consensus_map: oms.ConsensusMap, param
     final_feature_count = len(self.consensus_df)
     reduction_pct = ((initial_feature_count - final_feature_count) / initial_feature_count * 100) if initial_feature_count > 0 else 0
     
-    self.logger.info(f"Post-processing complete: {initial_feature_count} → {final_feature_count} features ({reduction_pct:.1f}% reduction)")
+    self.logger.info(f"Consensus cleanup complete: {initial_feature_count} → {final_feature_count} features ({reduction_pct:.1f}% reduction)")
     
     # Create a new consensus map for compatibility (the processed data is in consensus_df)
     processed_consensus_map = oms.ConsensusMap()
@@ -1691,8 +1691,12 @@ def _merge_chunk_results(self, chunk_consensus_maps: list, params: merge_default
                 mz_min_local = mz_max_local = consensus_mz
                 
             # Store chunk consensus with feature tracking
+            # Generate unique 16-character consensus_id string
+            import uuid
+            consensus_id_str = str(uuid.uuid4()).replace('-', '')[:16]
+            
             chunk_consensus_data = {
-                'consensus_id': consensus_id_counter,
+                'consensus_id': consensus_id_str,
                 'chunk_idx': chunk_idx,
                 'chunk_start_idx': chunk_start_idx,
                 'mz': consensus_mz,
@@ -1710,7 +1714,6 @@ def _merge_chunk_results(self, chunk_consensus_maps: list, params: merge_default
             }
             
             all_chunk_consensus.append(chunk_consensus_data)
-            consensus_id_counter += 1
 
     if not all_chunk_consensus:
         # No valid consensus features found
@@ -2094,9 +2097,13 @@ def _calculate_consensus_statistics(study_obj, consensus_uid: int, feature_data_
             ms2_count += len(ms2_scans)
     
     # Build consensus metadata
+    # Generate unique 16-character consensus_id string
+    import uuid
+    consensus_id_str = str(uuid.uuid4()).replace('-', '')[:16]
+    
     return {
         "consensus_uid": int(consensus_uid),
-        "consensus_id": str(consensus_uid),  # Use simple string ID
+        "consensus_id": consensus_id_str,  # Use unique 16-char string ID
         "quality": round(float(np.mean(quality_values)), 3) if len(quality_values) > 0 else 1.0,
         "number_samples": number_samples if number_samples is not None else len(feature_data_list),
         "rt": round(float(np.mean(rt_values)), 4) if len(rt_values) > 0 else 0.0,
@@ -2118,6 +2125,7 @@ def _calculate_consensus_statistics(study_obj, consensus_uid: int, feature_data_
         "chrom_prominence_mean": round(float(np.mean(prominence_values)), 0) if len(prominence_values) > 0 else 0.0,
         "chrom_prominence_scaled_mean": round(float(np.mean(prominence_scaled_values)), 3) if len(prominence_scaled_values) > 0 else 0.0,
         "chrom_height_scaled_mean": round(float(np.mean(height_scaled_values)), 3) if len(height_scaled_values) > 0 else 0.0,
+        "iso": None,  # Will be filled by find_iso() function
         "iso_mean": round(float(np.mean(iso_values)), 2) if len(iso_values) > 0 else 0.0,
         "charge_mean": round(float(np.mean(charge_values)), 2) if len(charge_values) > 0 else 0.0,
         "number_ms2": int(ms2_count),
@@ -2509,10 +2517,14 @@ def _extract_consensus_features(self, consensus_map, min_samples, cached_adducts
             if ms2_scans is not None:
                 ms2_count += len(ms2_scans)
 
+        # Generate unique 16-character consensus_id string (UUID-based)
+        import uuid
+        consensus_id_str = str(uuid.uuid4()).replace('-', '')[:16]
+
         metadata_list.append(
             {
                 "consensus_uid": int(i),  # "consensus_id": i,
-                "consensus_id": str(feature.getUniqueId()),
+                "consensus_id": consensus_id_str,  # Use unique 16-char string ID
                 "quality": round(float(feature.getQuality()), 3),
                 "number_samples": len(feature_data_list),
                 # "number_ext": int(len(features_list)),
@@ -2577,6 +2589,7 @@ def _extract_consensus_features(self, consensus_map, min_samples, cached_adducts
                 )
                 if len(height_scaled_values) > 0
                 else 0.0,
+                "iso": None,  # Will be filled by find_iso() function
                 "iso_mean": round(float(np.mean(iso_values)), 2)
                 if len(iso_values) > 0
                 else 0.0,
@@ -3325,7 +3338,9 @@ def _finalize_merge(self, link_ms2, min_samples):
             f"Merging completed with empty result. Consensus features: {len(self.consensus_df)}. "
             f"This may be due to min_samples ({min_samples}) being too high for the available data.",
         )
-    
+
+    # add iso data from raw files.
+    self.find_iso()
     if link_ms2:
         self.find_ms2()
 

masster/study/plot.py

@@ -42,9 +42,10 @@ def _isolated_save_plot(plot_object, filename, abs_filename, logger, plot_title=
             from bokeh.io.export import export_png
             export_png(plot_object, filename=filename)
             logger.info(f"Plot saved to: {abs_filename}")
-        except Exception:
+        except Exception as e:
             # Fall back to HTML if PNG export not available
             html_filename = filename.replace('.png', '.html')
+            abs_html_filename = html_filename if abs_filename == filename else abs_filename.replace('.png', '.html')
             from bokeh.resources import Resources
             from bokeh.embed import file_html
 
@@ -54,7 +55,7 @@ def _isolated_save_plot(plot_object, filename, abs_filename, logger, plot_title=
             with open(html_filename, 'w', encoding='utf-8') as f:
                 f.write(html)
 
-            logger.warning(f"PNG export not available, saved as HTML instead: {html_filename}")
+            logger.warning(f"PNG export not available ({str(e)}). Use export_png. Saved as HTML instead: {abs_html_filename}")
     elif filename.endswith(".pdf"):
         # Try to save as PDF, fall back to HTML if not available
         try:
@@ -74,6 +75,26 @@ def _isolated_save_plot(plot_object, filename, abs_filename, logger, plot_title=
                 f.write(html)
 
             logger.warning(f"PDF export not available, saved as HTML instead: {html_filename}")
+    elif filename.endswith(".svg"):
+        # Try to save as SVG, fall back to HTML if not available
+        try:
+            from bokeh.io.export import export_svg
+            export_svg(plot_object, filename=filename)
+            logger.info(f"Plot saved to: {abs_filename}")
+        except Exception as e:
+            # Fall back to HTML if SVG export not available
+            html_filename = filename.replace('.svg', '.html')
+            abs_html_filename = html_filename if abs_filename == filename else abs_filename.replace('.svg', '.html')
+            from bokeh.resources import Resources
+            from bokeh.embed import file_html
+
+            resources = Resources(mode='cdn')
+            html = file_html(plot_object, resources, title=plot_title)
+
+            with open(html_filename, 'w', encoding='utf-8') as f:
+                f.write(html)
+
+            logger.warning(f"SVG export not available ({str(e)}). Saved as HTML instead: {abs_html_filename}")
     else:
         # Default to HTML for unknown extensions using isolated approach
         from bokeh.resources import Resources
@@ -181,6 +202,22 @@ def _isolated_save_panel_plot(panel_obj, filename, abs_filename, logger, plot_ti
                 logger.warning(f"PDF export not available, saved as HTML instead: {abs_html_filename}")
             except Exception as e:
                 logger.error(f"Failed to save {plot_title} as HTML fallback: {e}")
+    elif filename.endswith(".svg"):
+        # Try to save as SVG, fall back to HTML if not available
+        try:
+            from bokeh.io.export import export_svg
+            bokeh_layout = panel_obj.get_root()
+            export_svg(bokeh_layout, filename=filename)
+            logger.info(f"{plot_title} saved to: {abs_filename}")
+        except Exception as e:
+            # Fall back to HTML if SVG export not available
+            html_filename = filename.replace('.svg', '.html')
+            abs_html_filename = os.path.abspath(html_filename)
+            try:
+                panel_obj.save(html_filename, embed=True)
+                logger.warning(f"SVG export not available ({str(e)}), saved as HTML instead: {abs_html_filename}")
+            except Exception as e:
+                logger.error(f"Failed to save {plot_title} as HTML fallback: {e}")
     else:
         # Default to HTML for unknown extensions
         try:
@@ -1687,221 +1724,154 @@ def plot_consensus_stats(
     self,
     filename=None,
     width=1200,
-    height=1200,
+    height=None,
     alpha=0.6,
-    markersize=3,
+    bins=30,
+    n_cols=4,
 ):
     """
-    Plot a scatter plot matrix (SPLOM) of consensus statistics using Bokeh.
-
+    Plot histograms/distributions for all numeric columns in consensus_df.
+    
     Parameters:
         filename (str, optional): Output filename for saving the plot
         width (int): Overall width of the plot (default: 1200)
-        height (int): Overall height of the plot (default: 1200)
-        alpha (float): Point transparency (default: 0.6)
-        markersize (int): Size of points (default: 5)
+        height (int, optional): Overall height of the plot (auto-calculated if None)
+        alpha (float): Histogram transparency (default: 0.6)
+        bins (int): Number of histogram bins (default: 30)
+        n_cols (int): Number of columns in the grid layout (default: 4)
     """
     from bokeh.layouts import gridplot
-    from bokeh.models import ColumnDataSource, HoverTool
-    from bokeh.plotting import figure, show, output_file
+    from bokeh.plotting import figure
+    import polars as pl
+    import numpy as np
 
     # Check if consensus_df exists and has data
     if self.consensus_df is None or self.consensus_df.is_empty():
         self.logger.error("No consensus data available. Run merge/find_consensus first.")
         return
 
-    # Define the columns to plot
-    columns = [
-        "rt",
-        "mz",
-        "number_samples",
-        "log10_quality",
-        "mz_delta_mean",
-        "rt_delta_mean",
-        "chrom_coherence_mean",
-        "chrom_prominence_scaled_mean",
-        "inty_mean",
-        "number_ms2",
-    ]
-
-    # Check which columns exist in the dataframe and compute missing ones
-    available_columns = self.consensus_df.columns
+    # Get all columns and their data types - work with original dataframe
     data_df = self.consensus_df.clone()
 
-    # Add log10_quality if quality exists
-    if "quality" in available_columns and "log10_quality" not in available_columns:
-        data_df = data_df.with_columns(
-            pl.col("quality").log10().alias("log10_quality"),
-        )
-
-    # Filter columns that actually exist
-    final_columns = [col for col in columns if col in data_df.columns]
-
-    if len(final_columns) < 2:
-        self.logger.error(f"Need at least 2 columns for SPLOM. Available: {final_columns}")
+    # Identify numeric columns (excluding ID columns that are typically strings)
+    id_columns = ["consensus_uid", "consensus_id", "uid", "id"]
+    numeric_columns = []
+    
+    for col in data_df.columns:
+        if col not in id_columns:
+            dtype = data_df[col].dtype
+            # Check if column is numeric (int, float, or can be converted to numeric)
+            if dtype in [pl.Int8, pl.Int16, pl.Int32, pl.Int64, 
+                        pl.UInt8, pl.UInt16, pl.UInt32, pl.UInt64,
+                        pl.Float32, pl.Float64]:
+                numeric_columns.append(col)
+
+    if len(numeric_columns) == 0:
+        self.logger.error("No numeric columns found in consensus_df for plotting distributions.")
         return
 
-    self.logger.debug(f"Creating SPLOM with columns: {final_columns}")
-
-    # Add important ID columns for tooltips even if not plotting them
-    tooltip_columns = []
-    for id_col in ["consensus_uid", "consensus_id"]:
-        if id_col in data_df.columns and id_col not in final_columns:
-            tooltip_columns.append(id_col)
+    self.logger.debug(f"Creating distribution plots for {len(numeric_columns)} numeric columns: {numeric_columns}")
 
-    # Select plotting columns plus tooltip columns
-    all_columns = final_columns + tooltip_columns
-    data_pd = data_df.select(all_columns).to_pandas()
+    # Work directly with Polars - no conversion to pandas needed
+    data_df_clean = data_df.select(numeric_columns)
 
-    # Remove any infinite or NaN values
-    data_pd = data_pd.replace([np.inf, -np.inf], np.nan).dropna()
-
-    if data_pd.empty:
-        self.logger.error("No valid data after removing NaN/infinite values.")
+    # Check if all numeric columns are empty
+    all_columns_empty = True
+    for col in numeric_columns:
+        # Check if column has any non-null, finite values
+        non_null_count = data_df_clean[col].filter(
+            data_df_clean[col].is_not_null() & 
+            (data_df_clean[col].is_finite() if data_df_clean[col].dtype in [pl.Float32, pl.Float64] else pl.lit(True))
+        ).len()
+        
+        if non_null_count > 0:
+            all_columns_empty = False
+            break
+    
+    if all_columns_empty:
+        self.logger.error("All numeric columns contain only NaN/infinite values.")
         return
 
-    source = ColumnDataSource(data_pd)
-
-    n_vars = len(final_columns)
-
-    # Fixed dimensions - override user input to ensure consistent layout
-    total_width = 1200
-    total_height = 1200
-
-    # Calculate plot sizes to ensure uniform inner plot areas
-    # First column needs extra width for y-axis labels
-    plot_width_first = 180  # Wider to account for y-axis labels
-    plot_width_others = 120  # Standard width for other columns
-    plot_height_normal = 120  # Standard height
-    plot_height_last = 155  # Taller last row to accommodate x-axis labels while keeping inner plot area same size
+    # Calculate grid dimensions
+    n_plots = len(numeric_columns)
+    n_rows = (n_plots + n_cols - 1) // n_cols  # Ceiling division
+    
+    # Auto-calculate height if not provided
+    if height is None:
+        plot_height = 300
+        height = plot_height * n_rows + 100  # Add some padding
+    else:
+        plot_height = (height - 100) // n_rows  # Subtract padding and divide
+    
+    plot_width = (width - 100) // n_cols  # Subtract padding and divide
 
-    # Create grid of plots with variable outer sizes but equal inner areas
+    # Create plots grid
     plots = []
-
-    for i, y_var in enumerate(final_columns):
-        row = []
-        for j, x_var in enumerate(final_columns):
-            # Determine if this plot needs axis labels
-            has_x_label = i == n_vars - 1  # bottom row
-            has_y_label = j == 0  # left column
-
-            # First column wider to accommodate y-axis labels, ensuring equal inner plot areas
-            current_width = plot_width_first if has_y_label else plot_width_others
-            current_height = plot_height_last if has_x_label else plot_height_normal
-
-            p = figure(
-                width=current_width,
-                height=current_height,
-                title=None,  # No title on any plot
-                toolbar_location=None,
-                # Adjusted borders - first column has more space, others minimal
-                min_border_left=70 if has_y_label else 15,
-                min_border_bottom=50 if has_x_label else 15,
-                min_border_right=15,
-                min_border_top=15,
-            )
-
-            # Ensure subplot background and border are explicitly white so the plot looks
-            # correct in dark and light themes.
-            p.outline_line_color = None
-            p.border_fill_color = "white"
-            p.border_fill_alpha = 1.0
-            p.background_fill_color = "white"
-
-            # Remove axis lines to eliminate black lines between plots
-            p.xaxis.axis_line_color = None
-            p.yaxis.axis_line_color = None
-
-            # Keep subtle grid lines for data reference
-            p.grid.visible = True
-            p.grid.grid_line_color = "#E0E0E0"  # Light gray grid lines
-
-            # Set axis labels and formatting
-            if has_x_label:  # bottom row
-                p.xaxis.axis_label = x_var
-                p.xaxis.axis_label_text_font_size = "12pt"
-                p.xaxis.major_label_text_font_size = "9pt"
-                p.xaxis.axis_label_standoff = 15
-            else:
-                p.xaxis.major_label_text_font_size = "0pt"
-                p.xaxis.minor_tick_line_color = None
-                p.xaxis.major_tick_line_color = None
-
-            if has_y_label:  # left column
-                p.yaxis.axis_label = y_var
-                p.yaxis.axis_label_text_font_size = "10pt"  # Smaller y-axis title
-                p.yaxis.major_label_text_font_size = "8pt"
-                p.yaxis.axis_label_standoff = 12
-            else:
-                p.yaxis.major_label_text_font_size = "0pt"
-                p.yaxis.minor_tick_line_color = None
-                p.yaxis.major_tick_line_color = None
-
-            if i == j:
-                # Diagonal: histogram
-                hist, edges = np.histogram(data_pd[x_var], bins=30)
-                p.quad(
-                    top=hist,
-                    bottom=0,
-                    left=edges[:-1],
-                    right=edges[1:],
-                    fill_color="green",
-                    line_color="white",
-                    alpha=alpha,
-                )
-            else:
-                # Off-diagonal: scatter plot
-                scatter = p.scatter(
-                    x=x_var,
-                    y=y_var,
-                    size=markersize,
-                    alpha=alpha,
-                    color="blue",
-                    source=source,
-                )
-
-                # Add hover tool
-                hover = HoverTool(
-                    tooltips=[
-                        (x_var, f"@{x_var}{{0.0000}}"),
-                        (y_var, f"@{y_var}{{0.0000}}"),
-                        (
-                            "consensus_uid",
-                            "@consensus_uid"
-                            if "consensus_uid" in data_pd.columns
-                            else "@consensus_id"
-                            if "consensus_id" in data_pd.columns
-                            else "N/A",
-                        ),
-                        ("rt", "@rt{0.00}" if "rt" in data_pd.columns else "N/A"),
-                        ("mz", "@mz{0.0000}" if "mz" in data_pd.columns else "N/A"),
-                    ],
-                    renderers=[scatter],
-                )
-                p.add_tools(hover)
-
-            row.append(p)
-        plots.append(row)
-
-    # Link axes for same variables
-    for i in range(n_vars):
-        for j in range(n_vars):
-            if i != j:  # Don't link diagonal plots
-                # Link x-axis to other plots in same column
-                for k in range(n_vars):
-                    if k != i and k != j:
-                        plots[i][j].x_range = plots[k][j].x_range
-
-                # Link y-axis to other plots in same row
-                for k in range(n_vars):
-                    if k != j and k != i:
-                        plots[i][j].y_range = plots[i][k].y_range
-
-    # Create grid layout and force overall background/border to white so the outer
-    # container doesn't show dark UI colors in night mode.
+    current_row = []
+    
+    for i, col in enumerate(numeric_columns):
+        # Create histogram for this column
+        p = figure(
+            width=plot_width,
+            height=plot_height,
+            title=col,
+            toolbar_location="above",
+            tools="pan,wheel_zoom,box_zoom,reset,save"
+        )
+        
+        # Set white background
+        p.background_fill_color = "white"
+        p.border_fill_color = "white"
+        
+        # Calculate histogram using Polars
+        # Get valid (non-null, finite) values for this column
+        if data_df_clean[col].dtype in [pl.Float32, pl.Float64]:
+            valid_values = data_df_clean.filter(
+                data_df_clean[col].is_not_null() & data_df_clean[col].is_finite()
+            )[col]
+        else:
+            valid_values = data_df_clean.filter(data_df_clean[col].is_not_null())[col]
+        
+        if valid_values.len() == 0:
+            self.logger.warning(f"No valid values for column {col}")
+            continue
+        
+        # Convert to numpy for histogram calculation
+        values_array = valid_values.to_numpy()
+        hist, edges = np.histogram(values_array, bins=bins)
+        
+        # Create histogram bars
+        p.quad(
+            top=hist,
+            bottom=0,
+            left=edges[:-1],
+            right=edges[1:],
+            fill_color="steelblue",
+            line_color="white",
+            alpha=alpha,
+        )
+        
+        # Style the plot
+        p.title.text_font_size = "12pt"
+        p.xaxis.axis_label = col
+        p.yaxis.axis_label = "Count"
+        p.grid.visible = True
+        p.grid.grid_line_color = "#E0E0E0"
+        
+        current_row.append(p)
+        
+        # If we've filled a row or reached the end, add the row to plots
+        if len(current_row) == n_cols or i == n_plots - 1:
+            # Fill remaining spots in the last row with None if needed
+            while len(current_row) < n_cols and i == n_plots - 1:
+                current_row.append(None)
+            plots.append(current_row)
+            current_row = []
+
+    # Create grid layout
     grid = gridplot(plots)
-
-    # Set overall background and border to white when supported
+    
+    # Set overall background to white
     if hasattr(grid, "background_fill_color"):
         grid.background_fill_color = "white"
     if hasattr(grid, "border_fill_color"):