M3Drop 0.4.35__tar.gz → 0.4.37__tar.gz

{m3drop-0.4.35 → m3drop-0.4.37/M3Drop.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: M3Drop
-Version: 0.4.35
+Version: 0.4.37
 Summary: A Python implementation of the M3Drop single-cell RNA-seq analysis tool.
 Home-page: https://github.com/PragalvhaSharma/m3DropNew
 Author: Tallulah Andrews

{m3drop-0.4.35/M3Drop.egg-info → m3drop-0.4.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: M3Drop
-Version: 0.4.35
+Version: 0.4.37
 Summary: A Python implementation of the M3Drop single-cell RNA-seq analysis tool.
 Home-page: https://github.com/PragalvhaSharma/m3DropNew
 Author: Tallulah Andrews

{m3drop-0.4.35 → m3drop-0.4.37}/m3Drop/diagnosticsGPU.py

@@ -10,17 +10,14 @@ import psutil
 import gc
 from scipy import sparse
 from scipy import stats
-import anndata  # <--- FIXED: Added missing import
+import anndata
 
-# [GOVERNOR INTEGRATION] Added get_optimal_chunk_size
+# [GOVERNOR INTEGRATION]
 from .coreGPU import hidden_calc_valsGPU, NBumiFitModelGPU, NBumiFitDispVsMeanGPU, get_optimal_chunk_size
 from cupy.sparse import csr_matrix as cp_csr_matrix
 import scipy.sparse as sp
 from scipy.sparse import csr_matrix as sp_csr_matrix
-
 import statsmodels.api as sm
-from scipy.stats import norm
-from statsmodels.stats.multitest import multipletests
 
 def NBumiFitBasicModelGPU(
     cleaned_filename: str,
@@ -140,7 +137,7 @@ def NBumiCheckFitFSGPU(
     plot_filename=None
 ) -> dict:
     """
-    FIXED VERSION - No cupy.errstate, proper GPU computation.
+    Calculates the fit errors (gene_error, cell_error) for a given model.
     """
     start_time = time.perf_counter()
     print(f"FUNCTION: NBumiCheckFitFS() | FILE: {cleaned_filename}")
@@ -171,14 +168,10 @@ def NBumiCheckFitFSGPU(
     print("Phase [1/2]: COMPLETE")
 
     # --- Phase 2: Calculate Expected Dropouts ---
-    print("Phase [2/2]: Calculating expected dropouts from data chunks...")
-    
-    # [GOVERNOR INTEGRATION] Removed naive calculation, utilizing Governor's chunk_size
-    optimal_chunk = chunk_size
-    print(f"  Using governor chunk size: {optimal_chunk}")
+    print(f"Phase [2/2]: Calculating expected dropouts (Chunk: {chunk_size})...")
     
-    for i in range(0, nc, optimal_chunk):
-        end_col = min(i + optimal_chunk, nc)
+    for i in range(0, nc, chunk_size):
+        end_col = min(i + chunk_size, nc)
         print(f"Phase [2/2]: Processing: {end_col} of {nc} cells.", end='\r')
 
         tis_chunk_gpu = tis_gpu[i:end_col]
@@ -199,11 +192,7 @@ def NBumiCheckFitFSGPU(
         
         # Clean up
         del mu_chunk_gpu, p_is_chunk_gpu, base, tis_chunk_gpu
-        
-        # Periodic memory cleanup
-        mempool = cp.get_default_memory_pool()
-        if (i // optimal_chunk) % 10 == 0:
-            mempool.free_all_blocks()
+        cp.get_default_memory_pool().free_all_blocks()
 
     print(f"Phase [2/2]: COMPLETE{' ' * 50}")
 
@@ -256,7 +245,7 @@ def NBumiCheckFitFSGPU(
     }
 
 def NBumiCompareModelsGPU(
-    raw_filename: str,
+    raw_filename: str, # Kept for API compatibility, but functionally we use cleaned_filename for indices
     cleaned_filename: str,
     stats: dict,
     fit_adjust: dict,
@@ -265,131 +254,118 @@ def NBumiCompareModelsGPU(
     plot_filename=None
 ) -> dict:
     """
-    OPTIMIZED VERSION - Faster normalization and sparse matrix writing.
+    OPTIMIZED VERSION (IN-MEMORY):
+    - Eliminates the 46GB '_basic_norm.h5ad' temporary file.
+    - Performs depth normalization and variance calculation on-the-fly in GPU VRAM.
+    - PRESERVED SCIENTIFIC LOGIC: Var(X) = E[X^2] - (E[X])^2 on normalized data.
     """
     pipeline_start_time = time.time()
     print(f"FUNCTION: NBumiCompareModels() | Comparing models for {cleaned_filename}")
 
-    # [GOVERNOR INTEGRATION] Calculate chunk size for normalization phase (heavy IO)
+    # [GOVERNOR] High multiplier (12.0) because we hold Raw + Norm + Square in VRAM
     if chunk_size is None:
-        # Multiplier 10.0 for safety during normalization of massive dense expansion
-        chunk_size = get_optimal_chunk_size(cleaned_filename, multiplier=10.0, is_dense=True)
-
-    # --- Phase 1: OPTIMIZED Normalization ---
-    print("Phase [1/4]: Creating temporary 'basic' normalized data file...")
-    basic_norm_filename = cleaned_filename.replace('.h5ad', '_basic_norm.h5ad')
-
-    # Read metadata. In 'backed' mode, this keeps a file handle open.
-    adata_meta = anndata.read_h5ad(cleaned_filename, backed='r')
-    nc, ng = adata_meta.shape
-    obs_df = adata_meta.obs.copy()
-    var_df = adata_meta.var.copy()
+        chunk_size = get_optimal_chunk_size(cleaned_filename, multiplier=12.0, is_dense=False)
+
+    # --- Phase 1: In-Memory "Basic Fit" (Normalization + Variance) ---
+    print("Phase [1/3]: Calculating Basic Model (Depth-Normalized) variance on-the-fly...")
     
-    cell_sums = stats['tis'].values
-    median_sum = np.median(cell_sums[cell_sums > 0])
+    # 1. Prepare Size Factors (CPU)
+    tjs = stats['tjs'].values # Gene sums
+    tis = stats['tis'].values # Cell sums
+    nc, ng = stats['nc'], stats['ng']
     
-    # Avoid division by zero for cells with zero counts
-    size_factors = np.ones_like(cell_sums, dtype=np.float32)
-    non_zero_mask = cell_sums > 0
-    size_factors[non_zero_mask] = cell_sums[non_zero_mask] / median_sum
-
-    adata_out = anndata.AnnData(obs=obs_df, var=var_df)
-    adata_out.write_h5ad(basic_norm_filename, compression="gzip")
-
-    with h5py.File(basic_norm_filename, 'a') as f_out:
-        if 'X' in f_out:
-            del f_out['X']
-        x_group_out = f_out.create_group('X')
-        x_group_out.attrs['encoding-type'] = 'csr_matrix'
-        x_group_out.attrs['encoding-version'] = '0.1.0'
-        x_group_out.attrs['shape'] = np.array([nc, ng], dtype='int64')
-
-        out_data = x_group_out.create_dataset('data', shape=(0,), maxshape=(None,), dtype='float32')
-        out_indices = x_group_out.create_dataset('indices', shape=(0,), maxshape=(None,), dtype='int32')
-        out_indptr = x_group_out.create_dataset('indptr', shape=(nc + 1,), dtype='int64')
-        out_indptr[0] = 0
-        current_nnz = 0
-
-        with h5py.File(cleaned_filename, 'r') as f_in:
-            h5_indptr = f_in['X']['indptr']
-            h5_data = f_in['X']['data']
-            h5_indices = f_in['X']['indices']
-
-            for i in range(0, nc, chunk_size):
-                end_row = min(i + chunk_size, nc)
-                print(f"Phase [1/4]: Normalizing: {end_row} of {nc} cells.", end='\r')
-                
-                start_idx, end_idx = h5_indptr[i], h5_indptr[end_row]
-                if start_idx == end_idx:
-                    out_indptr[i + 1 : end_row + 1] = current_nnz
-                    continue
-
-                # Read data for the chunk
-                data_slice = h5_data[start_idx:end_idx]
-                indices_slice = h5_indices[start_idx:end_idx]
-                indptr_slice = h5_indptr[i:end_row + 1] - start_idx
-                
-                # Move to GPU for fast normalization
-                data_gpu = cp.asarray(data_slice.copy(), dtype=cp.float32)
-                
-                indptr_gpu = cp.asarray(indptr_slice.copy())
-                nnz_in_chunk = indptr_gpu[-1].item()
-                cell_boundary_markers = cp.zeros(nnz_in_chunk, dtype=cp.int32)
-                if len(indptr_gpu) > 1:
-                    cell_boundary_markers[indptr_gpu[:-1]] = 1
-                row_indices = cp.cumsum(cell_boundary_markers, axis=0) - 1
-                
-                size_factors_for_chunk = cp.asarray(size_factors[i:end_row])
-                
-                data_gpu /= size_factors_for_chunk[row_indices]
-                
-                data_cpu = np.round(data_gpu.get())
-
-                num_cells_in_chunk = end_row - i
-                chunk_sp = sp_csr_matrix((data_cpu, indices_slice, indptr_slice), 
-                                         shape=(num_cells_in_chunk, ng))
-
-                nnz_chunk = chunk_sp.nnz
-                out_data.resize(current_nnz + nnz_chunk, axis=0)
-                out_data[current_nnz:] = chunk_sp.data
-
-                out_indices.resize(current_nnz + nnz_chunk, axis=0)
-                out_indices[current_nnz:] = chunk_sp.indices
-
-                new_indptr_list = chunk_sp.indptr[1:].astype(np.int64) + current_nnz
-                out_indptr[i + 1 : end_row + 1] = new_indptr_list
-                
-                current_nnz += nnz_chunk
+    median_sum = np.median(tis[tis > 0])
+    size_factors = np.ones_like(tis, dtype=np.float32)
+    non_zero_mask = tis > 0
+    size_factors[non_zero_mask] = tis[non_zero_mask] / median_sum
+    
+    # 2. Prepare GPU Arrays
+    sum_x_sq_gpu = cp.zeros(ng, dtype=cp.float64)
+    sum_x_gpu = cp.zeros(ng, dtype=cp.float64) # Need sum(x) to calc mean(x) for variance
+    
+    # 3. GPU Loop (Raw Data -> Normalize -> Accumulate)
+    # CRITICAL: We read CLEANED_FILENAME to ensure indices match 'stats'
+    with h5py.File(cleaned_filename, 'r') as f_in:
+        h5_indptr = f_in['X']['indptr']
+        h5_data = f_in['X']['data']
+        h5_indices = f_in['X']['indices']
 
-                del data_gpu, row_indices, size_factors_for_chunk, indptr_gpu
-                cp.get_default_memory_pool().free_all_blocks()
+        for i in range(0, nc, chunk_size):
+            end_row = min(i + chunk_size, nc)
+            print(f"Phase [1/3]: Processing: {end_row} of {nc} cells.", end='\r')
+            
+            start_idx, end_idx = h5_indptr[i], h5_indptr[end_row]
+            if start_idx == end_idx: continue
+
+            # Load Raw Chunk
+            data_gpu = cp.asarray(h5_data[start_idx:end_idx], dtype=cp.float32)
+            indices_gpu = cp.asarray(h5_indices[start_idx:end_idx])
+            indptr_gpu = cp.asarray(h5_indptr[i:end_row + 1] - start_idx)
+
+            # Expand Size Factors to match Data Structure
+            nnz_in_chunk = indptr_gpu[-1].item()
+            cell_boundary_markers = cp.zeros(nnz_in_chunk, dtype=cp.int32)
+            if len(indptr_gpu) > 1:
+                cell_boundary_markers[indptr_gpu[:-1]] = 1
+            # row_indices maps every data point to its cell index (0 to chunk_size)
+            row_indices = cp.cumsum(cell_boundary_markers, axis=0) - 1
+            
+            # Get size factors for this chunk
+            sf_chunk = cp.asarray(size_factors[i:end_row])
+            
+            # --- THE MAGIC: On-the-Fly Normalization ---
+            # data_norm = data_raw / size_factor
+            data_gpu /= sf_chunk[row_indices]
+            
+            # Accumulate for Variance: E[X^2] and E[X]
+            cp.add.at(sum_x_sq_gpu, indices_gpu, data_gpu**2)
+            cp.add.at(sum_x_gpu, indices_gpu, data_gpu)
+            
+            # Clean up VRAM
+            del data_gpu, indices_gpu, indptr_gpu, row_indices, sf_chunk, cell_boundary_markers
+            cp.get_default_memory_pool().free_all_blocks()
 
-    print(f"Phase [1/4]: COMPLETE{' '*50}")
+    print(f"Phase [1/3]: COMPLETE{' '*50}")
 
-    print("Phase [2/4]: Fitting Basic Model on normalized data...")
-    
-    # [GOVERNOR INTEGRATION] Calculate chunk size for basic fit on the heavy normalized file
-    chunk_size_basic = get_optimal_chunk_size(basic_norm_filename, multiplier=10.0, is_dense=True)
+    # 4. Finalize Basic Statistics
+    # Var(X) = E[X^2] - (E[X])^2
+    mean_x_sq_gpu = sum_x_sq_gpu / nc
+    mean_mu_gpu = sum_x_gpu / nc
+    my_rowvar_gpu = mean_x_sq_gpu - mean_mu_gpu**2
     
-    stats_basic = hidden_calc_valsGPU(basic_norm_filename) # hidden_calc uses its own governor internally
-    fit_basic = NBumiFitBasicModelGPU(basic_norm_filename, stats_basic, chunk_size=chunk_size_basic)
-    print("Phase [2/4]: COMPLETE")
+    # Dispersion = Mean^2 / (Var - Mean)
+    size_gpu = mean_mu_gpu**2 / (my_rowvar_gpu - mean_mu_gpu)
     
-    print("Phase [3/4]: Evaluating fits of both models on ORIGINAL data...")
-    # [GOVERNOR INTEGRATION] Chunk size for check fit
-    chunk_size_check = get_optimal_chunk_size(cleaned_filename, multiplier=5.0, is_dense=True)
-    
-    check_adjust = NBumiCheckFitFSGPU(cleaned_filename, fit_adjust, suppress_plot=True, chunk_size=chunk_size_check)
+    # Safety Clamping
+    max_size_val = cp.nanmax(size_gpu) * 10
+    if cp.isnan(max_size_val): max_size_val = 1000
+    size_gpu[cp.isnan(size_gpu) | (size_gpu <= 0)] = max_size_val
+    size_gpu[size_gpu < 1e-10] = 1e-10
     
-    fit_basic_for_eval = {
-        'sizes': fit_basic['sizes'],
+    # Construct "Basic Fit" Object
+    fit_basic = {
+        'sizes': pd.Series(size_gpu.get(), index=stats['tjs'].index),
         'vals': stats,
-        'var_obs': fit_basic['var_obs']
+        'var_obs': pd.Series(my_rowvar_gpu.get(), index=stats['tjs'].index)
     }
-    check_basic = NBumiCheckFitFSGPU(cleaned_filename, fit_basic_for_eval, suppress_plot=True, chunk_size=chunk_size_check)
-    print("Phase [3/4]: COMPLETE")
+    
+    # --- Phase 2: Check Fit (Calculate Errors) ---
+    print("Phase [2/3]: Evaluating fit errors on ORIGINAL data...")
+    
+    # Check Adjust (M3Drop) - uses its own governor
+    check_adjust = NBumiCheckFitFSGPU(
+        cleaned_filename, fit_adjust, suppress_plot=True
+    )
+    
+    # Check Basic (Depth-Norm) - uses its own governor
+    check_basic = NBumiCheckFitFSGPU(
+        cleaned_filename, fit_basic, suppress_plot=True
+    )
+    print("Phase [2/3]: COMPLETE")
 
-    print("Phase [4/4]: Generating final comparison...")
+    # --- Phase 3: Plotting & Comparison ---
+    print("Phase [3/3]: Generating comparison...")
+    
     nc_data = stats['nc']
     mean_expr = stats['tjs'] / nc_data
     observed_dropout = stats['djs'] / nc_data
@@ -432,15 +408,8 @@ def NBumiCompareModelsGPU(
         plt.show()
     
     plt.close()
-    print("Phase [4/4]: COMPLETE")
-
-    pipeline_end_time = time.time()
     
-    # --- ADD THIS LINE TO FIX THE ERROR ---
-    adata_meta.file.close() # Explicitly close the file handle
-    
-    os.remove(basic_norm_filename)
-    print(f"STATUS: Temporary file '{basic_norm_filename}' removed.")
+    pipeline_end_time = time.time()
     print(f"Total time: {pipeline_end_time - pipeline_start_time:.2f} seconds.\n")
     
     return {
@@ -455,11 +424,6 @@ def NBumiPlotDispVsMeanGPU(
 ):
     """
     Generates a diagnostic plot of the dispersion vs. mean expression.
-
-    Args:
-        fit (dict): The 'fit' object from NBumiFitModelGPU.
-        suppress_plot (bool): If True, the plot will not be displayed on screen.
-        plot_filename (str, optional): Path to save the plot. If None, not saved.
     """
     print("FUNCTION: NBumiPlotDispVsMean()")
 
@@ -470,7 +434,6 @@ def NBumiPlotDispVsMeanGPU(
     intercept, slope = coeffs[0], coeffs[1]
 
     # --- 2. Calculate the fitted line for plotting ---
-    # Create a smooth, continuous line using the regression coefficients
     log_mean_expr_range = np.linspace(
         np.log(mean_expression[mean_expression > 0].min()),
         np.log(mean_expression.max()),

{m3drop-0.4.35 → m3drop-0.4.37}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setuptools.setup(
     name="M3Drop",
-    version="0.4.35",  # Version bump
+    version="0.4.37",  # Version bump
     author="Tallulah Andrews",
     author_email="tandrew6@uwo.ca",
     description="A Python implementation of the M3Drop single-cell RNA-seq analysis tool.",