rectanglepy 1.4.2__tar.gz → 1.6.0__tar.gz

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.4.2
+current_version = 1.6.0
 tag = True
 commit = True
 

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/.readthedocs.yaml

@@ -1,7 +1,7 @@
 # https://docs.readthedocs.io/en/stable/config-file/v2.html
 version: 2
 build:
-    os: ubuntu-20.04
+    os: ubuntu-24.04
     tools:
         python: "3.10"
 sphinx:

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: rectanglepy
-Version: 1.4.2
+Version: 1.6.0
 Summary: Hierarchical deconvolution of bulk  transcriptomics
 Project-URL: Documentation, https://rectanglepy.readthedocs.io/
 Project-URL: Source, https://github.com/ComputationalBiomedicineGroup/Rectangle
@@ -72,6 +72,12 @@ Install the latest release of `Rectangle` from `PyPI` <https://pypi.org/project/
 pip install rectanglepy
 ```
 
+## How Rectangle works
+
+![Overview of the Rectangle signature-building and multiscale deconvolution workflow](docs/_static/rectangle_workflow.png)
+
+Rectangle performs robust multiscale deconvolution informed by single-cell transcriptomics. Rectangle takes annotated scRNA-seq data and a bulk RNA-seq mixture as input. In the signature-building phase, it performs bootstrapped pseudobulking of the scRNA-seq data, followed by signature-gene selection based on log-fold-change (logFC) and p-value, and signature-matrix optimization (minimizing the condition number and correcting for cell-type-specific mRNA content bias). Based on this, it constructs two signature matrices: a “direct signature”, resolving individual cell types, and a “clustered signature” grouping transcriptionally similar cell types. In the deconvolution phase, Rectangle first uses the clustered signature to deconvolve the bulk data into coarse cell-type estimates. These are then used to constrain a second deconvolution step that leverages the direct signature to resolve individual cell-type fractions. Finally, the resulting estimates are scaled to account for unknown cellular content not represented in the reference.
+
 ## License
 
 Rectangle is dual-licensed: BSD-3-Clause OR Commercial.
@@ -100,7 +106,9 @@ For commercial licensing: **innovation-psb@uibk.ac.at**
 
 ## Citation
 
-> If you use Rectangle in your project, please cite: (TBA)
+If you use Rectangle in your project, please cite:
+
+> Eder B, Rigato I, Dietrich A, Merotto L, Sturm G, Treis T, List M, Theis FJ, Finotello F. Rectangle: robust and scalable multiscale deconvolution informed by single-cell RNA sequencing data. bioRxiv. 2026. doi:[10.64898/2026.07.07.736950](https://doi.org/10.64898/2026.07.07.736950)
 
 [scverse-discourse]: https://discourse.scverse.org/
 [issue-tracker]: https://github.com/ComputationalBiomedicineGroup/Rectangle/issues

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/README.md

@@ -33,6 +33,12 @@ Install the latest release of `Rectangle` from `PyPI` <https://pypi.org/project/
 pip install rectanglepy
 ```
 
+## How Rectangle works
+
+![Overview of the Rectangle signature-building and multiscale deconvolution workflow](docs/_static/rectangle_workflow.png)
+
+Rectangle performs robust multiscale deconvolution informed by single-cell transcriptomics. Rectangle takes annotated scRNA-seq data and a bulk RNA-seq mixture as input. In the signature-building phase, it performs bootstrapped pseudobulking of the scRNA-seq data, followed by signature-gene selection based on log-fold-change (logFC) and p-value, and signature-matrix optimization (minimizing the condition number and correcting for cell-type-specific mRNA content bias). Based on this, it constructs two signature matrices: a “direct signature”, resolving individual cell types, and a “clustered signature” grouping transcriptionally similar cell types. In the deconvolution phase, Rectangle first uses the clustered signature to deconvolve the bulk data into coarse cell-type estimates. These are then used to constrain a second deconvolution step that leverages the direct signature to resolve individual cell-type fractions. Finally, the resulting estimates are scaled to account for unknown cellular content not represented in the reference.
+
 ## License
 
 Rectangle is dual-licensed: BSD-3-Clause OR Commercial.
@@ -61,7 +67,9 @@ For commercial licensing: **innovation-psb@uibk.ac.at**
 
 ## Citation
 
-> If you use Rectangle in your project, please cite: (TBA)
+If you use Rectangle in your project, please cite:
+
+> Eder B, Rigato I, Dietrich A, Merotto L, Sturm G, Treis T, List M, Theis FJ, Finotello F. Rectangle: robust and scalable multiscale deconvolution informed by single-cell RNA sequencing data. bioRxiv. 2026. doi:[10.64898/2026.07.07.736950](https://doi.org/10.64898/2026.07.07.736950)
 
 [scverse-discourse]: https://discourse.scverse.org/
 [issue-tracker]: https://github.com/ComputationalBiomedicineGroup/Rectangle/issues

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/docs/api.md

@@ -12,4 +12,5 @@
     pp.build_rectangle_signatures
     tl.deconvolution
     pp.RectangleSignatureResult
+    RectangleAdvancedParameters
 ```

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/docs/index.md

@@ -1,4 +1,5 @@
 ```{include} ../README.md
+:relative-images:
 
 ```
 

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/docs/references.bib

@@ -16,6 +16,15 @@
   url      = {https://doi.org/10.1186/s13059-017-1382-0}
 }
 
+@article{Eder2026,
+  author  = {Bernhard Eder and Irene Rigato and Alexander Dietrich and Lorenzo Merotto and Gregor Sturm and Tim Treis and Markus List and Fabian J. Theis and Francesca Finotello},
+  title   = {Rectangle: robust and scalable multiscale deconvolution informed by single-cell RNA sequencing data},
+  journal = {bioRxiv},
+  year    = {2026},
+  doi     = {10.64898/2026.07.07.736950},
+  url     = {https://doi.org/10.64898/2026.07.07.736950}
+}
+
 @article{Finotello2019,
   author   = {Francesca Finotello and Gregor Mayer and Christian Plattner and Cornelia Laschober and Dietmar Rieder and Peter Hackl and Lukas Krogsdam and Michael L. T. Helmberg and Zlatko Trajanoski},
   title    = {Molecular and pharmacological modulators of the tumor immune contexture revealed by deconvolution of RNA-seq data},

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ requires = ["hatchling"]
 
 [project]
 name = "rectanglepy"
-version = "1.4.2"
+version = "1.6.0"
 description = "Hierarchical deconvolution of bulk  transcriptomics"
 readme = "README.md"
 requires-python = ">=3.10"

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/src/rectanglepy/__init__.py

@@ -1,8 +1,9 @@
 from importlib.metadata import version
 
 from . import pp, tl
+from .parameters import RectangleAdvancedParameters
 from .rectangle import load_tutorial_data, rectangle
 
-__all__ = ["pp", "tl", "load_tutorial_data", "rectangle"]
+__all__ = ["pp", "tl", "RectangleAdvancedParameters", "load_tutorial_data", "rectangle"]
 
 __version__ = version("rectanglepy")

rectanglepy-1.6.0/src/rectanglepy/parameters.py

@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: BSD-3-Clause OR LicenseRef-Rectangle-Commercial
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class RectangleAdvancedParameters:
+    """Advanced parameters for Rectangle internals."""
+
+    number_of_bootstraps: int = 20
+    grid_search_split_size: int = 50

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/src/rectanglepy/pp/create_signature.py

@@ -13,6 +13,7 @@ from scipy.cluster.hierarchy import fcluster, linkage
 from scipy.stats import pearsonr
 from sklearn.metrics import silhouette_score
 
+from rectanglepy.parameters import RectangleAdvancedParameters
 from rectanglepy.tl.deconvolution import solve_qp
 
 from .rectangle_signature import RectangleSignatureResult
@@ -129,24 +130,25 @@ def _filter_de_analysis_results(de_analysis_result, p, logfc):
 
 
 def _run_deseq2(
-    countsig: pd.DataFrame, sc_data, annotations: pd.Series, n_cpus: int = None, gene_expression_threshold=0.5
+    countsig: pd.DataFrame,
+    sc_data,
+    annotations: pd.Series,
+    n_cpus: int = None,
+    gene_expression_threshold=0.4,
+    number_of_bootstraps: int = 20,
 ) -> dict[str | int, pd.DataFrame]:
     results = {}
     inference = DefaultInference(n_cpus=n_cpus)
-    bootstrapped_signature = _create_bootstrap_signature(countsig, sc_data, annotations)
+    bootstrapped_signature = _create_bootstrap_signature(countsig, sc_data, annotations, number_of_bootstraps)
     np.random.seed(42)
     for _i, cell_type in enumerate(countsig.columns):
-        bootstrapped_signature_copy = bootstrapped_signature.copy()
-        countsig_copy = countsig.copy()
         sc_data_filtered = sc_data.T[annotations == cell_type]
         expressed_cells = (sc_data_filtered > 0).sum(axis=0)
         if expressed_cells.ndim > 1:  # needed for sparse matrices
             expressed_cells = np.squeeze(np.asarray(expressed_cells))
-            # make dense out of sparse
-            sc_data_filtered = sc_data_filtered.toarray()
         threshold = gene_expression_threshold * sc_data_filtered.shape[0]
-        genes = countsig_copy.index[expressed_cells > threshold].tolist()
-        bootstrapped_signature_copy = bootstrapped_signature_copy.loc[genes].T
+        genes = countsig.index[expressed_cells > threshold].tolist()
+        bootstrapped_signature_copy = bootstrapped_signature.loc[genes].T
         logger.info(f"Running DE analysis for {cell_type}")
         condition = ["B" if (cell_type + "_") in x else "A" for x in bootstrapped_signature_copy.index]
         clinical_df = pd.DataFrame({"condition": condition}, index=bootstrapped_signature_copy.index)
@@ -167,25 +169,29 @@ def _run_deseq2(
     return results
 
 
-def _create_bootstrap_signature(countsig, sc_data, annotations) -> pd.DataFrame:
-    if scipy.sparse.issparse(sc_data):
-        sc_data = sc_data.toarray()
+def _create_bootstrap_signature(countsig, sc_data, annotations, number_of_bootstraps: int = 20) -> pd.DataFrame:
+    # Cells are only ever summed here, never read one by one, so sparse input is kept sparse:
+    # densifying the whole matrix costs genes * cells * 8 bytes, which is gigabytes on an
+    # atlas-sized reference. Dense input deliberately stays dense - gathering rows out of a
+    # dense array is faster than routing them through a sparse format.
+    cells_by_gene = sc_data.T.tocsr() if scipy.sparse.issparse(sc_data) else sc_data.T
     celltypes = countsig.columns
-    bootstrapped_signature = pd.DataFrame()
-    number_of_bootstraps = 7
-    samples_per_bootstrap = 500
+    columns = {}
+    samples_per_bootstrap = 1000
     np.random.seed(42)
     for celltype in celltypes:
-        sc_data_filtered = sc_data.T[annotations == celltype]
+        sc_data_filtered = cells_by_gene[(annotations == celltype).to_numpy()]
+        # sparse matrices do not support len(); shape[0] is the cell count for both formats
+        number_of_cells = sc_data_filtered.shape[0]
         for i in range(number_of_bootstraps):
-            selected_rows = np.random.choice(len(sc_data_filtered), samples_per_bootstrap, replace=True)
+            selected_rows = np.random.choice(number_of_cells, samples_per_bootstrap, replace=True)
             summed_rows = sc_data_filtered[selected_rows].sum(axis=0)
-            bootstrapped_signature[f"{celltype}_{i}"] = list(summed_rows)
-    bootstrapped_signature.index = countsig.index
+            # a sparse sum is a (1, genes) matrix, a dense one a flat array
+            columns[f"{celltype}_{i}"] = np.asarray(summed_rows).ravel()
+    # built in one go: assigning column by column repeatedly reallocates the frame
+    bootstrapped_signature = pd.DataFrame(columns, index=countsig.index)
     # to int
-    samples_per_bootstrap = samples_per_bootstrap / 2.5
-    bootstrapped_signature = bootstrapped_signature.astype(int)
-    return bootstrapped_signature
+    return bootstrapped_signature.astype(int)
 
 
 def _de_analysis(
@@ -197,15 +203,31 @@ def _de_analysis(
     optimize_cutoffs: bool,
     n_cpus: int = None,
     genes=None,
-    gene_expression_threshold=0.5,
+    gene_expression_threshold=0.4,
+    advanced_parameters: RectangleAdvancedParameters = None,
 ) -> tuple[Series, dict[str, [str]] :, DataFrame | None]:
     logger.info("Starting DE analysis")
-    deseq_results = _run_deseq2(pseudo_count_sig, sc_data, annotations, n_cpus, gene_expression_threshold)
+    advanced_parameters = advanced_parameters or RectangleAdvancedParameters()
+    deseq_results = _run_deseq2(
+        pseudo_count_sig,
+        sc_data,
+        annotations,
+        n_cpus,
+        gene_expression_threshold,
+        advanced_parameters.number_of_bootstraps,
+    )
     optimization_results = None
 
     if optimize_cutoffs:
         logger.info("Optimizing cutoff parameters p and lfc")
-        optimization_results = _optimize_parameters(sc_data, annotations, pseudo_count_sig, deseq_results, genes)
+        optimization_results = _optimize_parameters(
+            sc_data,
+            annotations,
+            pseudo_count_sig,
+            deseq_results,
+            genes,
+            advanced_parameters.grid_search_split_size,
+        )
         p, lfc = optimization_results.iloc[0, 0:2]
         logger.info(f"Optimization done\n Best cutoffs  p: {p} and lfc: {lfc}")
 
@@ -286,7 +308,8 @@ def build_rectangle_signatures(
     p=0.015,
     lfc=1.5,
     n_cpus: int = None,
-    gene_expression_threshold=0.5,
+    gene_expression_threshold=0.4,
+    advanced_parameters: RectangleAdvancedParameters = None,
 ) -> RectangleSignatureResult:
     r"""Builds rectangle signatures based on single-cell  count data and annotations.
 
@@ -303,7 +326,7 @@ def build_rectangle_signatures(
     raw
         A flag indicating whether to use the raw Anndata data. Defaults to False.
     optimize_cutoffs
-        Indicates whether to optimize the p-value and log fold change cutoffs using gridsearch. Defaults to True.
+        Indicates whether to optimize the log fold change cutoffs using gridsearch. Defaults to True.
     p
         The p-value threshold for the DE analysis (only used if optimize_cutoffs is False).
     lfc
@@ -311,12 +334,15 @@ def build_rectangle_signatures(
     n_cpus
         The number of cpus to use for the DE analysis. Defaults to the number of cpus available.
     gene_expression_threshold
-        The gene expression threshold for the DE analysis. How many cells need to express a gene to be considered in DGE
+        The gene expression threshold for the DE analysis. The fraction of cells that must express a gene to be considered in DGE. Defaults to 0.4
+    advanced_parameters
+        Optional advanced Rectangle parameters. Defaults are used when not provided.
 
     Returns
     -------
     The result of the rectangle signature analysis which is of type RectangleSignatureResult.
     """
+    advanced_parameters = advanced_parameters or RectangleAdvancedParameters()
     annotations = adata.obs[cell_type_col]
     adata = adata[:, adata.X.sum(axis=0) > len(annotations.value_counts())]
     assert adata.var_names.is_unique, "Duplicate gene found in adata"
@@ -344,7 +370,16 @@ def build_rectangle_signatures(
     m_rna_biasfactors = _create_bias_factors(pseudo_sig_counts, sc_counts, annotations)
 
     marker_genes, marker_genes_per_cell_type, optimization_result = _de_analysis(
-        pseudo_sig_counts, sc_counts, annotations, p, lfc, optimize_cutoffs, n_cpus, genes, gene_expression_threshold
+        pseudo_sig_counts,
+        sc_counts,
+        annotations,
+        p,
+        lfc,
+        optimize_cutoffs,
+        n_cpus,
+        genes,
+        gene_expression_threshold,
+        advanced_parameters,
     )
     pseudo_sig_cpm = _convert_to_cpm(pseudo_sig_counts)
     logger.info("Starting rectangle cluster analysis")
@@ -371,6 +406,7 @@ def build_rectangle_signatures(
         lfc,
         False,
         gene_expression_threshold=gene_expression_threshold,
+        advanced_parameters=advanced_parameters,
     )
     clustered_signature = _convert_to_cpm(clustered_signature)
     return RectangleSignatureResult(
@@ -390,9 +426,10 @@ def build_rectangle_signatures(
 def _create_pseudo_count_sig(sc_counts: np.ndarray, annotations: pd.Series, var_names) -> pd.DataFrame:
     unique_labels, label_indices = np.unique(annotations, return_inverse=True)
     grouped_sum = np.zeros((len(unique_labels), sc_counts.shape[0]))
+    cells_by_gene = sc_counts.T
     for i, _label in enumerate(unique_labels):
         label_columns = label_indices == i
-        grouped_sum[i, :] = np.sum(sc_counts.T[label_columns, :], axis=0)
+        grouped_sum[i, :] = np.sum(cells_by_gene[label_columns, :], axis=0)
     grouped_sum = grouped_sum.T
 
     grouped_sum = pd.DataFrame(grouped_sum, index=var_names, columns=unique_labels).astype(int)
@@ -400,15 +437,20 @@ def _create_pseudo_count_sig(sc_counts: np.ndarray, annotations: pd.Series, var_
 
 
 def _optimize_parameters(
-    sc_data: pd.DataFrame, annotations: pd.Series, pseudo_signature_counts: pd.DataFrame, de_results, genes=None
+    sc_data: pd.DataFrame,
+    annotations: pd.Series,
+    pseudo_signature_counts: pd.DataFrame,
+    de_results,
+    genes=None,
+    grid_search_split_size: int = 50,
 ) -> pd.DataFrame:
     # search space for p and lfc
-    lfcs = [x / 100 for x in range(160, 230, 10)]
+    lfcs = [1.5, 1.75, 2.0, 2.25, 2.5, 2.75, 3.0]
     ps = [x / 1000 for x in range(50, 51, 1)]
 
     results = []
     logger.info("generating pseudo bulks")
-    bulks, real_fractions = _generate_pseudo_bulks(sc_data, annotations, genes)
+    bulks, real_fractions = _generate_pseudo_bulks(sc_data, annotations, genes, grid_search_split_size)
     for p in ps:
         for lfc in lfcs:
             try:
@@ -441,9 +483,8 @@ def _assess_parameter_fit(
     return rsme, pearson_r
 
 
-def _generate_pseudo_bulks(sc_data, annotations, genes=None):
+def _generate_pseudo_bulks(sc_data, annotations, genes=None, split_size: int = 50):
     number_of_bulks = 50
-    split_size = 50
     bulks = []
     real_fractions = []
     np.random.seed(42)

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/src/rectanglepy/rectangle.py

@@ -7,6 +7,7 @@ from anndata import AnnData
 from loguru import logger
 from pandas import DataFrame
 
+from .parameters import RectangleAdvancedParameters
 from .pp import RectangleSignatureResult, build_rectangle_signatures
 from .tl import deconvolution
 
@@ -23,7 +24,8 @@ def rectangle(
     p=0.015,
     lfc=1.5,
     n_cpus: int = None,
-    gene_expression_threshold=0.5,
+    gene_expression_threshold=0.4,
+    advanced_parameters: RectangleAdvancedParameters = None,
 ) -> tuple[DataFrame, RectangleSignatureResult]:
     r"""All in one deconvolution method. Creates signatures and deconvolutes the bulk data. Has options for subsampling and consensus runs.
 
@@ -40,7 +42,7 @@ def rectangle(
     raw
         A flag indicating whether to use the raw Anndata data.
     optimize_cutoffs
-        Indicates whether to optimize the p-value and log fold change cutoffs using gridsearch.
+        Indicates whether to optimize the log fold change cutoffs using gridsearch.
     p
         The p-value threshold for the DE analysis (only used if optimize_cutoffs is False).
     lfc
@@ -50,7 +52,9 @@ def rectangle(
     correct_mrna_bias : bool
         A flag indicating whether to correct for mRNA bias. Defaults to True.
     gene_expression_threshold : float
-        The threshold for gene expression. Genes with expression below this threshold are removed from the analysis.
+        The threshold for gene expression. Genes must be expressed in at least this fraction of cells. Defaults to 0.4.
+    advanced_parameters
+        Optional advanced Rectangle parameters. Defaults are used when not provided.
 
     Returns
     -------
@@ -71,6 +75,7 @@ def rectangle(
         lfc=lfc,
         n_cpus=n_cpus,
         gene_expression_threshold=gene_expression_threshold,
+        advanced_parameters=advanced_parameters,
     )
 
     estimations, bulk_err = deconvolution(signatures, bulks, correct_mrna_bias, n_cpus)

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/src/rectanglepy/tl/deconvolution.py

@@ -7,18 +7,166 @@ import numpy as np
 import osqp
 import pandas as pd
 import scipy.sparse as sp
-import statsmodels.api as sm
 from joblib import Parallel, delayed, parallel_backend
 from loguru import logger
 
 from rectanglepy.pp.rectangle_signature import RectangleSignatureResult
 
+# OSQP can return noticeably different solutions from active-set QP when P is singular/ill-conditioned.
+# A tiny ridge makes the problem strictly convex and closer to quadprog behavior.
+QP_RIDGE = 1e-8
+
 
 def _scale_weights(weights: np.ndarray) -> np.ndarray:
     min_weight = np.nextafter(min(weights), np.float64(1.0))  # prevent division by zero
     return weights / min_weight
 
 
+def _upper_triangular_csc_pattern(n_vars: int) -> tuple[np.ndarray, np.ndarray]:
+    """Returns the (indices, indptr) of a fully populated upper triangular CSC matrix.
+
+    OSQP only stores the upper triangle of P, so handing it the triangle directly saves the
+    conversion of the full symmetric matrix on every solve.
+    """
+    indices = np.concatenate([np.arange(column + 1) for column in range(n_vars)]).astype(np.int32)
+    indptr = np.concatenate(([0], np.cumsum(np.arange(1, n_vars + 1)))).astype(np.int32)
+    return indices, indptr
+
+
+class _QuadraticProgram:
+    """A deconvolution QP for one (signature, bulk) pair, solvable at several dampening weights.
+
+    Everything that does not depend on the weights - the numpy views of the inputs and the constraint
+    matrix with its bounds - is built once and shared by all solves, which keeps it out of the
+    dampened least squares iteration.
+
+    Note that each solve still gets its own OSQP workspace: reusing one across solves makes OSQP
+    keep the problem scaling of the first objective, which perturbs the fixed point the iteration
+    converges to (and is not faster at these problem sizes).
+    """
+
+    def __init__(
+        self,
+        signature: pd.DataFrame,
+        bulk: pd.Series,
+        prev_assignments: list[int | str] = None,
+        prev_solution: pd.Series = None,
+    ):
+        if not signature.index.equals(bulk.index):
+            bulk = bulk.reindex(signature.index)
+        self.signature = signature.to_numpy(dtype=np.float64)
+        self.bulk = bulk.to_numpy(dtype=np.float64)
+        self.n_vars = self.signature.shape[1]  # number of cell types / fractions
+
+        self._triu_indices, self._triu_indptr = _upper_triangular_csc_pattern(self.n_vars)
+        self._triu_selection = np.tril_indices(self.n_vars)  # reads P.T column-major upper triangle
+        self._constraints, self._lower_bounds = self._build_constraints(prev_assignments, prev_solution)
+        self._upper_bounds = np.full_like(self._lower_bounds, np.inf, dtype=np.float64)
+
+    def _build_constraints(
+        self, prev_assignments: list[int | str], prev_solution: pd.Series
+    ) -> tuple[sp.csc_matrix, np.ndarray]:
+        # ----- Constraints in the original quadprog form: C.T x >= b
+        # We build C (n_vars, n_constraints) then map to OSQP: A = C.T, l=b, u=+inf
+        C_cols = []
+        b_list = []
+
+        # Constraint 1: sum(x) <= 1  ->  -sum(x) >= -1
+        C_cols.append(-np.ones((self.n_vars, 1), dtype=np.float64))
+        b_list.append(np.array([-1.0], dtype=np.float64))
+
+        # Constraint 2: x >= 0  -> I x >= 0
+        C_cols.append(np.eye(self.n_vars, dtype=np.float64))
+        b_list.append(np.zeros(self.n_vars, dtype=np.float64))
+
+        # Constraint 3: keep close to prev_solution (this encoding already turns upper bounds into >= via negation)
+        if prev_solution is not None:
+            if prev_assignments is None:
+                raise ValueError("prev_assignments must be provided when prev_solution is provided.")
+
+            for cluster in prev_solution.index:
+                # x_cluster <= upper  ->  -x_cluster >= -upper
+                C_upper = np.array(
+                    [-1.0 if str(x) == str(cluster) else 0.0 for x in prev_assignments],
+                    dtype=np.float64,
+                ).reshape(-1, 1)
+
+                # x_cluster >= lower  ->  +x_cluster >= +lower
+                C_lower = np.array(
+                    [1.0 if str(x) == str(cluster) else 0.0 for x in prev_assignments],
+                    dtype=np.float64,
+                ).reshape(-1, 1)
+
+                prev_weight = float(prev_solution.loc[cluster])
+                upper = min(1.0, prev_weight + 0.03)
+                lower = max(0.0, prev_weight - 0.03)
+
+                C_cols.append(C_upper)
+                b_list.append(np.array([-upper], dtype=np.float64))
+
+                C_cols.append(C_lower)
+                b_list.append(np.array([lower], dtype=np.float64))
+
+        C = np.concatenate(C_cols, axis=1)  # (n_vars, n_constraints)
+        b = np.concatenate(b_list, axis=0).astype(np.float64)  # (n_constraints,)
+
+        # OSQP uses l <= A x <= u
+        return sp.csc_matrix(C.T), b
+
+    def _objective(self, gld: np.ndarray, multiplier: int) -> tuple[np.ndarray, np.ndarray]:
+        # Minimize     1/2 x^T G x - a^T x
+        if multiplier is None:
+            G = self.signature.T @ self.signature
+            a = self.signature.T @ self.bulk
+        else:
+            weights = np.square(1 / (self.signature @ gld))
+            weights_dampened = np.clip(_scale_weights(weights), None, multiplier)
+            # equivalent to signature.T @ diag(weights) @ signature, without the dense (genes, genes) matrix
+            G = self.signature.T @ (weights_dampened[:, None] * self.signature)
+            a = self.signature.T @ (weights_dampened * self.bulk)
+
+        # ----- Map to OSQP: minimize 1/2 x^T P x + q^T x  ->  q = -a
+        P = G
+        q = -a
+
+        # Optional scaling
+        scale = np.linalg.norm(P)
+        if scale > 0:
+            P = P / scale
+            q = q / scale
+
+        P = ((P + P.T) / 2.0) + QP_RIDGE * np.eye(self.n_vars, dtype=np.float64)
+        return P, q
+
+    def solve(self, gld: np.ndarray = None, multiplier: int = None) -> np.ndarray:
+        P, q = self._objective(gld, multiplier)
+        P_data = P.T[self._triu_selection]  # upper triangle in CSC (column major) order
+        P_sp = sp.csc_matrix((P_data, self._triu_indices, self._triu_indptr), shape=(self.n_vars, self.n_vars))
+
+        solver = osqp.OSQP()
+        solver.setup(
+            P=P_sp,
+            q=q,
+            A=self._constraints,
+            l=self._lower_bounds,
+            u=self._upper_bounds,
+            verbose=False,
+            eps_abs=1e-7,
+            eps_rel=1e-7,
+            max_iter=50000,
+            polish=True,
+            warm_start=False,
+            scaled_termination=False,
+        )
+
+        res = solver.solve()
+
+        if res.info.status_val not in (1,):  # 1 = solved
+            raise RuntimeError(f"OSQP did not solve the problem: {res.info.status}")
+
+        return res.x
+
+
 def solve_qp(
     signature: pd.DataFrame,
     bulk: pd.Series,
@@ -52,143 +200,89 @@ def solve_qp(
     Notes
     -----
     This function uses quadratic programming to solve the deconvolution problem. The objective is to minimize the difference between the observed bulk data and the data predicted by the signature matrix and the cell fractions. The function also includes constraints to ensure that the cell fractions are non-negative and sum to 1, and to make the solution similar to the previous assignments and weights if they are provided.
+
+    Callers that solve the same problem at several dampening weights (e.g. the dampened least squares
+    iteration) should use :class:`_QuadraticProgram` directly, which shares the weight independent
+    parts of the problem across solves.
     """
-    # ------------------ QP-based deconvolution
-    # Minimize     1/2 x^T G x - a^T x
-    # Subject to   C.T x >= b
-    if multiplier is None:
-        a = (signature.T @ bulk).to_numpy(dtype=np.float64)
-        G = (signature.T @ signature).to_numpy(dtype=np.float64)
-    else:
-        weights = np.square(1 / (signature @ gld))
-        weights_dampened = np.clip(_scale_weights(weights), None, multiplier)
-        W = np.diag(np.asarray(weights_dampened, dtype=np.float64))
-        G = (signature.T.to_numpy() @ (W @ signature.to_numpy())).astype(np.float64)
-        a = (signature.T.to_numpy() @ (W @ bulk.to_numpy())).astype(np.float64)
-
-    n_vars = G.shape[0]  # number of cell types / fractions
-
-    # ----- Constraints in your original quadprog form: C.T x >= b
-    # We'll build C (n_vars, n_constraints) then map to OSQP: A = C.T, l=b, u=+inf
-    C_cols = []
-    b_list = []
-
-    # Constraint 1: sum(x) <= 1  ->  -sum(x) >= -1
-    C_cols.append(-np.ones((n_vars, 1), dtype=np.float64))
-    b_list.append(np.array([-1.0], dtype=np.float64))
-
-    # Constraint 2: x >= 0  -> I x >= 0
-    C_cols.append(np.eye(n_vars, dtype=np.float64))
-    b_list.append(np.zeros(n_vars, dtype=np.float64))
-
-    # Constraint 3: keep close to prev_solution (your encoding already turns upper bounds into >= via negation)
-    if prev_solution is not None:
-        if prev_assignments is None:
-            raise ValueError("prev_assignments must be provided when prev_solution is provided.")
-
-        for cluster in prev_solution.index:
-            # x_cluster <= upper  ->  -x_cluster >= -upper
-            C_upper = np.array(
-                [-1.0 if str(x) == str(cluster) else 0.0 for x in prev_assignments],
-                dtype=np.float64,
-            ).reshape(-1, 1)
-
-            # x_cluster >= lower  ->  +x_cluster >= +lower
-            C_lower = np.array(
-                [1.0 if str(x) == str(cluster) else 0.0 for x in prev_assignments],
-                dtype=np.float64,
-            ).reshape(-1, 1)
-
-            prev_weight = float(prev_solution.loc[cluster])
-            upper = min(1.0, prev_weight + 0.03)
-            lower = max(0.0, prev_weight - 0.03)
-
-            C_cols.append(C_upper)
-            b_list.append(np.array([-upper], dtype=np.float64))
-
-            C_cols.append(C_lower)
-            b_list.append(np.array([lower], dtype=np.float64))
-
-    C = np.concatenate(C_cols, axis=1)  # (n_vars, n_constraints)
-    b = np.concatenate(b_list, axis=0).astype(np.float64)  # (n_constraints,)
-
-    # ----- Map to OSQP: minimize 1/2 x^T P x + q^T x
-    # Your objective: 1/2 x^T G x - a^T x  -> q = -a
-    P = G
-    q = -a
-
-    # Optional scaling
-    scale = np.linalg.norm(P)
-    if scale > 0:
-        P = P / scale
-        q = q / scale
-
-    # OSQP can return noticeably different solutions from active-set QP when P is singular/ill-conditioned.
-    # Add tiny ridge to make the problem strictly convex and closer to quadprog behavior.
-    ridge = 1e-8
-    P = ((P + P.T) / 2.0) + ridge * np.eye(n_vars, dtype=np.float64)
-
-    # OSQP uses l <= A x <= u
-    A = C.T  # (n_constraints, n_vars)
-    l = b
-    u = np.full_like(l, np.inf, dtype=np.float64)
-
-    # Sparse matrices (required/expected)
-    P_sp = sp.csc_matrix(P)
-    A_sp = sp.csc_matrix(A)
-
-    solver = osqp.OSQP()
-    solver.setup(
-        P=P_sp,
-        q=q,
-        A=A_sp,
-        l=l,
-        u=u,
-        verbose=False,
-        eps_abs=1e-7,
-        eps_rel=1e-7,
-        max_iter=50000,
-        polish=True,
-        warm_start=False,
-        scaled_termination=False,
-    )
+    problem = _QuadraticProgram(signature, bulk, prev_assignments, prev_solution)
+    return problem.solve(gld, multiplier)
+
+
+def _batched_wls(
+    design: np.ndarray,
+    response: np.ndarray,
+    weights: np.ndarray,
+    subsets: np.ndarray,
+    max_bytes: int = 64 * 1024**2,
+) -> np.ndarray:
+    """Fits one weighted least squares model per gene subset in batch.
+
+    Mirrors ``statsmodels.api.WLS(...).fit()``: the design and the response are whitened with the
+    square root of the weights and the parameters are read off the pseudo inverse of the whitened
+    design. Stacking the subsets lets numpy loop over them in C instead of Python.
 
-    res = solver.solve()
+    Parameters
+    ----------
+    design : np.ndarray
+        The (genes, cell types) design matrix.
+    response : np.ndarray
+        The (genes,) response vector.
+    weights : np.ndarray
+        The (genes,) regression weights.
+    subsets : np.ndarray
+        A (n_subsets, subset size) array of gene indices, one row per model.
+    max_bytes : int
+        Upper bound on the size of a single whitened design batch, used to chunk the subsets.
+
+    Returns
+    -------
+    np.ndarray
+        A (n_subsets, cell types) array of fitted parameters.
+    """
+    n_subsets, subset_size = subsets.shape
+    n_params = design.shape[1]
+    params = np.empty((n_subsets, n_params), dtype=np.float64)
 
-    if res.info.status_val not in (1,):  # 1 = solved
-        raise RuntimeError(f"OSQP did not solve the problem: {res.info.status}")
+    chunk_size = max(1, int(max_bytes // (subset_size * n_params * design.itemsize)))
+    for start in range(0, n_subsets, chunk_size):
+        chunk = subsets[start : start + chunk_size]
+        sqrt_weights = np.sqrt(weights[chunk])  # (chunk, subset size)
+        whitened_design = design[chunk] * sqrt_weights[:, :, None]  # (chunk, subset size, cell types)
+        whitened_response = response[chunk] * sqrt_weights
+        params[start : start + chunk_size] = np.einsum("bpg,bg->bp", np.linalg.pinv(whitened_design), whitened_response)
 
-    return res.x
+    return params
 
 
 def _calculate_dampening_constant(signature: pd.DataFrame, bulk: pd.Series, qp_gld: np.ndarray) -> int:
     solutions_std = []
     np.random.seed(1)
-    weights = np.square(1 / (np.dot(signature, qp_gld)))
+    signature_values = np.asarray(signature, dtype=np.float64)
+    bulk_values = np.asarray(bulk, dtype=np.float64)
+    n_genes = signature_values.shape[0]
+    weights = np.square(1 / (signature_values @ qp_gld))
     weights_scaled = _scale_weights(weights)
     weights_scaled_no_inf = weights_scaled[weights_scaled != np.inf]
     qp_gld_sum = sum(qp_gld)
+    subset_size = n_genes // 2
+    n_subsets = 100
     # try multiple values of the dampening constant (multiplier)
     # for each, calculate the variance of the dampened weighted solution for a subset of genes
     max_range = 40
     multiplier_range = min(max_range, math.ceil(np.log2(max(weights_scaled_no_inf))))
     for i in range(multiplier_range):
-        solutions = []
         multiplier = 2**i
-        weights_dampened = np.array([multiplier if multiplier <= x else x for x in weights_scaled]).astype("double")
-        for _ in range(100):
-            subset = np.random.choice(len(signature), size=len(signature) // 2, replace=False)
-            bulk_subset = bulk.iloc[list(subset)]
-            signature_subset = signature.iloc[subset, :]
-            fit = sm.WLS(bulk_subset, -1 + signature_subset, weights=weights_dampened[subset]).fit()
-            solution = fit.params * qp_gld_sum / sum(fit.params)
-            solutions.append(solution)
-        solutions_df = pd.DataFrame(solutions)
-
-        solutions_std.append(solutions_df.std(axis=0))
-    solutions_std_df = pd.DataFrame(solutions_std)
-    means = solutions_std_df.apply(lambda x: np.mean(x**2), axis=1)
-    best_dampening_constant = means.idxmin()
+        weights_dampened = np.minimum(weights_scaled, multiplier)
+        subsets = np.array([np.random.choice(n_genes, size=subset_size, replace=False) for _ in range(n_subsets)])
+        # WLS uses the signature directly; no intercept column is added.
+        params = _batched_wls(signature_values, bulk_values, weights_dampened, subsets)
+        solutions = params * qp_gld_sum / params.sum(axis=1, keepdims=True)
+
+        solutions_std.append(np.nanstd(solutions, axis=0, ddof=1))
+    solutions_std_array = np.array(solutions_std)
+    means = np.nanmean(np.square(solutions_std_array), axis=1)
+    best_dampening_constant = int(np.nanargmin(means))
     return best_dampening_constant
 
 
@@ -202,7 +296,8 @@ def _calculate_ls(
     signature = signature.loc[genes].sort_index()
     bulk = bulk.loc[genes].sort_index().astype("double")
 
-    approximate_solution = solve_qp(signature, bulk, prev_assignments, prev_weights)
+    problem = _QuadraticProgram(signature, bulk, prev_assignments, prev_weights)
+    approximate_solution = problem.solve()
     dampening_constant = _calculate_dampening_constant(signature, bulk, approximate_solution)
     multiplier = 2**dampening_constant
 
@@ -212,7 +307,7 @@ def _calculate_ls(
     iterations = 2
     solutions_sum = approximate_solution
     while (change > convergence_threshold) and (iterations < max_iterations):
-        dampened_solution = solve_qp(signature, bulk, prev_assignments, prev_weights, approximate_solution, multiplier)
+        dampened_solution = problem.solve(approximate_solution, multiplier)
         solutions_sum += dampened_solution
         solution_averages = solutions_sum / iterations
         change = np.linalg.norm(solution_averages - approximate_solution, 1)

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/tests/test_pp.py

@@ -6,6 +6,7 @@ import pytest
 from anndata import AnnData
 
 import rectanglepy as rectangle
+from rectanglepy import RectangleAdvancedParameters
 from rectanglepy.pp.create_signature import (
     _assess_parameter_fit,
     _calculate_cluster_range,
@@ -18,6 +19,7 @@ from rectanglepy.pp.create_signature import (
     _de_analysis,
     _generate_pseudo_bulks,
     _get_fcluster_assignments,
+    _optimize_parameters,
     _run_deseq2,
     build_rectangle_signatures,
 )
@@ -154,6 +156,7 @@ def test_generate_pseudo_bulks(small_data):
     sc_counts = sc_counts.astype("int")
     adata = AnnData(sc_counts.T, obs=annotations.to_frame(name="cell_type"))
     result, _ = _generate_pseudo_bulks(adata.X.T, annotations, adata.var_names)
+    custom_result, _ = _generate_pseudo_bulks(adata.X.T, annotations, adata.var_names, split_size=10)
 
     sc_data = sc_counts.astype(pd.SparseDtype("int"))
     csr_sparse_matrix = sc_data.sparse.to_coo().tocsr()
@@ -162,11 +165,40 @@ def test_generate_pseudo_bulks(small_data):
     result_sparse, _ = _generate_pseudo_bulks(adata_sparse.X.T, annotations, adata_sparse.var_names)
 
     assert len(result) == 1000 and len(result.columns) == 50
+    assert custom_result.shape == result.shape
+    assert not np.allclose(result, custom_result)
     # first gene should have all 0s
     assert result.iloc[0, :].sum() == 0
     assert np.allclose(result, result_sparse)
 
 
+def test_optimize_parameters_uses_grid_search_split_size(monkeypatch):
+    seen = {}
+
+    def fake_generate_pseudo_bulks(sc_data, annotations, genes=None, split_size=50):
+        seen["split_size"] = split_size
+        bulks = pd.DataFrame([[1.0]], index=["gene"], columns=["bulk"])
+        real_fractions = pd.DataFrame([[1.0]], index=["cell_type"], columns=["bulk"])
+        return bulks, real_fractions
+
+    def fake_assess_parameter_fit(lfc, p, bulks, real_fractions, pseudo_signature_counts, de_results):
+        return 0.0, 1.0
+
+    monkeypatch.setattr(rectangle.pp.create_signature, "_generate_pseudo_bulks", fake_generate_pseudo_bulks)
+    monkeypatch.setattr(rectangle.pp.create_signature, "_assess_parameter_fit", fake_assess_parameter_fit)
+
+    results = _optimize_parameters(
+        pd.DataFrame(),
+        pd.Series(dtype=str),
+        pd.DataFrame(),
+        {},
+        grid_search_split_size=13,
+    )
+
+    assert seen["split_size"] == 13
+    assert results.iloc[0]["pearson_r"] == 1.0
+
+
 def test_asses_fit(small_data):
     sc_counts, annotations, bulk = small_data
     sc_counts = sc_counts.astype("int")
@@ -204,12 +236,12 @@ def test_de_analysis(small_data):
     # test with sparse matrix
     _ = _de_analysis(sc_pseudo, adata_sparse.X.T, annotations, 0.4, 0.1, False, None, adata.var_names)
 
-    assert 5 < len(r1) < 50
+    assert 5 < len(r1) < 100
     assert len(r2) == 3
 
 
 def test_create_bootstrap_signature(small_data):
-    bootstraps_per_cell = 7
+    bootstraps_per_cell = RectangleAdvancedParameters().number_of_bootstraps
     sc_counts, annotations, bulk = small_data
     sc_counts = sc_counts.astype("int")
     sc_pseudo = sc_counts.groupby(annotations.values, axis=1).sum()
@@ -217,3 +249,20 @@ def test_create_bootstrap_signature(small_data):
     bootstrap = _create_bootstrap_signature(sc_pseudo, adata.X.T, annotations)
 
     assert len(bootstrap.columns) == len(sc_pseudo.columns) * bootstraps_per_cell
+
+
+def test_create_bootstrap_signature_with_advanced_parameter(small_data):
+    bootstraps_per_cell = 3
+    advanced_parameters = RectangleAdvancedParameters(number_of_bootstraps=bootstraps_per_cell)
+    sc_counts, annotations, bulk = small_data
+    sc_counts = sc_counts.astype("int")
+    sc_pseudo = sc_counts.groupby(annotations.values, axis=1).sum()
+    adata = AnnData(sc_counts.T, obs=annotations.to_frame(name="cell_type"))
+    bootstrap = _create_bootstrap_signature(
+        sc_pseudo,
+        adata.X.T,
+        annotations,
+        advanced_parameters.number_of_bootstraps,
+    )
+
+    assert len(bootstrap.columns) == len(sc_pseudo.columns) * bootstraps_per_cell

{rectanglepy-1.4.2 → rectanglepy-1.6.0}/tests/test_tl.py

@@ -62,7 +62,7 @@ def test_simple_weighted_dampened_deconvolution(quantiseq_data):
     corr = np.corrcoef(result, expected)[0, 1]
     rsme = np.sqrt(np.mean((result - expected) ** 2))
 
-    assert corr > 0.815 and rsme < 0.011
+    assert corr > 0.81 and rsme < 0.01
 
 
 def test_correct_for_unknown_cell_content(small_data, quantiseq_data):