crispyx 0.2.6__tar.gz → 0.2.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. {crispyx-0.2.6/src/crispyx.egg-info → crispyx-0.2.7}/PKG-INFO +1 -1
  2. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_grouping.py +22 -0
  3. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_kernels.py +25 -4
  4. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_provenance.py +1 -1
  5. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_statistics.py +26 -2
  6. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_version.py +1 -1
  7. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/data.py +59 -114
  8. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/de.py +29 -50
  9. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/qc.py +10 -15
  10. {crispyx-0.2.6 → crispyx-0.2.7/src/crispyx.egg-info}/PKG-INFO +1 -1
  11. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx.egg-info/SOURCES.txt +1 -0
  12. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_nb_glm.py +4 -6
  13. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_qc_parity.py +12 -9
  14. crispyx-0.2.7/tests/test_sort_by_perturbation.py +149 -0
  15. crispyx-0.2.7/tests/test_statistics.py +44 -0
  16. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_two_group_deseq2.py +5 -6
  17. crispyx-0.2.6/tests/test_sort_by_perturbation.py +0 -58
  18. {crispyx-0.2.6 → crispyx-0.2.7}/LICENSE +0 -0
  19. {crispyx-0.2.6 → crispyx-0.2.7}/README.md +0 -0
  20. {crispyx-0.2.6 → crispyx-0.2.7}/pyproject.toml +0 -0
  21. {crispyx-0.2.6 → crispyx-0.2.7}/setup.cfg +0 -0
  22. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/__init__.py +0 -0
  23. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_checkpoint.py +0 -0
  24. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_disk.py +0 -0
  25. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_h5codec.py +0 -0
  26. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_irls.py +0 -0
  27. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_memory.py +0 -0
  28. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_messages.py +0 -0
  29. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_namespaces.py +0 -0
  30. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_preflight.py +0 -0
  31. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/_size_factors.py +0 -0
  32. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/batch.py +0 -0
  33. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/dimred.py +0 -0
  34. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/glm.py +0 -0
  35. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/hvg.py +0 -0
  36. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/plotting.py +0 -0
  37. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/profiling.py +0 -0
  38. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/pseudobulk.py +0 -0
  39. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/residual.py +0 -0
  40. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx/sample.py +0 -0
  41. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx.egg-info/dependency_links.txt +0 -0
  42. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx.egg-info/requires.txt +0 -0
  43. {crispyx-0.2.6 → crispyx-0.2.7}/src/crispyx.egg-info/top_level.txt +0 -0
  44. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_assign_guides.py +0 -0
  45. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_atomic_outputs.py +0 -0
  46. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_batch_process.py +0 -0
  47. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_cgroup_memory_budget.py +0 -0
  48. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_check_data.py +0 -0
  49. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_collapse_genes.py +0 -0
  50. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_compare_controls.py +0 -0
  51. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_compress_h5ad.py +0 -0
  52. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_concat_h5ad.py +0 -0
  53. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_convert_format.py +0 -0
  54. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_data_helpers.py +0 -0
  55. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_de_edge_cases.py +0 -0
  56. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_de_lowexpr_filter.py +0 -0
  57. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_de_resume.py +0 -0
  58. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_dimred.py +0 -0
  59. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_disk_dispatch.py +0 -0
  60. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_downsample_counts.py +0 -0
  61. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_glm_structured.py +0 -0
  62. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_groupby_alias.py +0 -0
  63. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_hvg.py +0 -0
  64. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_in_memory_input.py +0 -0
  65. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_irls.py +0 -0
  66. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_memory_dispatch.py +0 -0
  67. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_messages.py +0 -0
  68. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_nb_offset.py +0 -0
  69. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_normalisation.py +0 -0
  70. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_normalized_effects.py +0 -0
  71. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_normalized_effects_se.py +0 -0
  72. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_obs_writes.py +0 -0
  73. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_pickle.py +0 -0
  74. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_plotting.py +0 -0
  75. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_preflight_dispatch.py +0 -0
  76. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_profiling.py +0 -0
  77. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_pseudobulk_batch.py +0 -0
  78. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_pseudobulk_memory.py +0 -0
  79. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_pseudobulk_profiles.py +0 -0
  80. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_pseudobulk_sparse_output.py +0 -0
  81. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_rank_genes_groups_outputs.py +0 -0
  82. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_read_backed.py +0 -0
  83. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_recover_counts.py +0 -0
  84. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_residual_effects.py +0 -0
  85. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_result_provenance.py +0 -0
  86. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_scanpy_parity.py +0 -0
  87. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_shrink_lfc_arrays.py +0 -0
  88. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_standardize.py +0 -0
  89. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_stream_on_fast_axis.py +0 -0
  90. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_subsample.py +0 -0
  91. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_temp_file_cleanup.py +0 -0
  92. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_wilcoxon_dispatch.py +0 -0
  93. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_wilcoxon_hypergeometric.py +0 -0
  94. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_wilcoxon_labels.py +0 -0
  95. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_wilcoxon_stratified.py +0 -0
  96. {crispyx-0.2.6 → crispyx-0.2.7}/tests/test_workflow.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: crispyx
3
- Version: 0.2.6
3
+ Version: 0.2.7
4
4
  Summary: Memory-efficient streaming analysis of large-scale CRISPR and Perturb-seq screens on disk-backed AnnData files
5
5
  Author: Jin-Hong Du
6
6
  License: Modified MIT License
@@ -103,3 +103,25 @@ def cell_labels(obs: pd.DataFrame, labels: str | pd.Series) -> tuple[pd.Series,
103
103
  def label_strings(labels: pd.Series) -> np.ndarray:
104
104
  """Per-cell labels as strings, missing ones as :data:`UNLABELLED`."""
105
105
  return labels.astype(object).where(labels.notna(), UNLABELLED).astype(str).to_numpy()
106
+
107
+
108
+ def contiguous_ranges(labels: np.ndarray) -> dict[str, slice]:
109
+ """Each label whose cells form one contiguous run of rows, as that run's
110
+ ``slice``. Labels split over several runs are left out, so a file is
111
+ sorted by ``labels`` exactly when every distinct label is a key.
112
+
113
+ Derived from the labels rather than stored with the file: a map stored
114
+ in ``uns`` is one HDF5 dataset per label, which every backed open reads
115
+ (over a second for a genome-wide screen) and which can go stale."""
116
+ codes, uniques = pd.factorize(np.asarray(labels), use_na_sentinel=False)
117
+ if not codes.size:
118
+ return {}
119
+ starts = np.flatnonzero(np.r_[True, codes[1:] != codes[:-1]])
120
+ stops = np.r_[starts[1:], codes.size]
121
+ run_codes = codes[starts]
122
+ single = np.bincount(run_codes, minlength=len(uniques)) == 1
123
+ return {
124
+ str(uniques[code]): slice(int(start), int(stop))
125
+ for code, start, stop in zip(run_codes, starts, stops)
126
+ if single[code]
127
+ }
@@ -1,8 +1,8 @@
1
- """Numba-accelerated kernels for NB-GLM differential expression.
1
+ """Numba-accelerated kernels for differential expression.
2
2
 
3
- This module contains all Numba JIT-compiled functions used by the GLM module.
4
- Separating these kernels improves code organization and allows for easier
5
- maintenance of the performance-critical code paths.
3
+ It holds the Numba JIT-compiled functions of the NB-GLM fits, the Wilcoxon
4
+ rank sums and the t-test's per-group moments, kept apart from the Python
5
+ modules that call them.
6
6
  """
7
7
 
8
8
  from __future__ import annotations
@@ -2277,3 +2277,24 @@ def _structured_nb_irls(X, labels, offset, Y, alpha, n_groups, const_col, const_
2277
2277
  converged[g] = conv
2278
2278
  n_iter[g] = it
2279
2279
  return coef, se, dev_out, converged, n_iter
2280
+
2281
+
2282
+ @nb.njit(cache=True, nogil=True)
2283
+ def _group_moments_csr(indptr, indices, data, codes, sums, sumsq, expr_counts, counts):
2284
+ """Add each group's per-gene sum and sum of squares (in float64), its
2285
+ stored-entry count and its cell count over CSR rows into the
2286
+ accumulators; row ``i`` is in group ``codes[i]``, ``-1`` rows left out.
2287
+
2288
+ One pass over the stored entries whatever the row order, and each group
2289
+ summed in row order -- the same on a file sorted by group as unsorted."""
2290
+ for row in range(codes.shape[0]):
2291
+ code = codes[row]
2292
+ if code < 0:
2293
+ continue
2294
+ counts[code] += 1
2295
+ for entry in range(indptr[row], indptr[row + 1]):
2296
+ gene = indices[entry]
2297
+ value = np.float64(data[entry])
2298
+ sums[code, gene] += value
2299
+ sumsq[code, gene] += value * value
2300
+ expr_counts[code, gene] += 1
@@ -35,7 +35,7 @@ SCHEMAS = {
35
35
  "pseudobulk": 2, # 2: profiles named by group key, constant obs columns carried over
36
36
  "pseudobulk_effects": 1,
37
37
  "batch": 1,
38
- "sorted": 1,
38
+ "sorted": 2, # 2: no per-label row map in uns (read off obs)
39
39
  "standardized": 1,
40
40
  "concat": 1,
41
41
  "residual_effects": 1,
@@ -1,7 +1,7 @@
1
1
  """Statistical utilities for differential expression testing.
2
2
 
3
- This module provides functions for tie correction, p-value adjustment and
4
- the low-expression and non-estimability masks.
3
+ This module provides functions for tie correction, p-value adjustment, the
4
+ low-expression and non-estimability masks, and per-group moment sums.
5
5
  """
6
6
 
7
7
  from __future__ import annotations
@@ -9,6 +9,9 @@ from __future__ import annotations
9
9
  from typing import Literal
10
10
 
11
11
  import numpy as np
12
+ import scipy.sparse as sp
13
+
14
+ from ._kernels import _group_moments_csr
12
15
 
13
16
 
14
17
  def _low_expr_in_both_mask(
@@ -235,3 +238,24 @@ def _nonestimable_glm_mask(
235
238
  if pert_floor:
236
239
  flagged |= np.asarray(pert_expr_counts) < pert_floor
237
240
  return flagged
241
+
242
+
243
+ def _add_group_moments(
244
+ block: sp.csr_matrix,
245
+ codes: np.ndarray,
246
+ sums: np.ndarray,
247
+ sumsq: np.ndarray,
248
+ expr_counts: np.ndarray,
249
+ counts: np.ndarray,
250
+ ) -> None:
251
+ """Add each group's per-gene sum, sum of squares and stored-entry count,
252
+ and its cell count, over the rows of ``block`` (group ``codes[i]`` for
253
+ row ``i``; ``-1`` rows are left out) into the accumulators, in place.
254
+
255
+ One pass over the block's stored entries in a numba kernel, where one
256
+ mask scan and fancy index per group took ~3x longer on an unsorted
257
+ chunk (thousands of groups in 4,096 rows)."""
258
+ _group_moments_csr(
259
+ block.indptr, block.indices, block.data, np.asarray(codes, dtype=np.int64),
260
+ sums, sumsq, expr_counts, counts,
261
+ )
@@ -2,4 +2,4 @@
2
2
  the distribution's metadata reads it at build time, so a source checkout
3
3
  reports the code it runs, whatever version is installed."""
4
4
 
5
- __version__ = "0.2.6"
5
+ __version__ = "0.2.7"
@@ -30,6 +30,7 @@ from ._memory import (
30
30
  _resolve_n_jobs,
31
31
  )
32
32
  from ._checkpoint import _create_progress_context, run_fingerprint
33
+ from ._grouping import contiguous_ranges, label_strings
33
34
  from ._provenance import KEY as _PROVENANCE_KEY, reusable, reuse_mismatch, stamp, write_stamp
34
35
  from ._disk import (
35
36
  _free_bytes_at,
@@ -1637,8 +1638,11 @@ def iter_matrix_chunks(
1637
1638
  ``warn_slow_axis=False`` suppresses the one-time logger warning about
1638
1639
  streaming a compressed matrix off its fast axis; callers that already
1639
1640
  resolved that decision through :func:`stream_on_fast_axis` pass it.
1640
- ``selected``, a boolean mask along ``axis``, skips (without reading) the
1641
- chunks that hold no selected entry; the chunks yielded are whole.
1641
+ ``selected``, a boolean mask along ``axis``, leaves unselected stretches
1642
+ unread: each chunk starts at the first selected entry the previous one
1643
+ left uncovered (:func:`chunk_starts`), so chunks are off the fixed grid
1644
+ unless every entry is selected, and ``start_chunk`` cannot be combined
1645
+ with it.
1642
1646
  ``read_ahead=True`` reads the next chunk on a background thread while the
1643
1647
  caller works on the current one -- worth it when that work releases the
1644
1648
  GIL (a numba kernel), so reading and computing overlap.
@@ -1695,14 +1699,31 @@ def chunk_starts(
1695
1699
  length: int, chunk_size: int, *, start_chunk: int = 0, selected: np.ndarray | None = None,
1696
1700
  ) -> list[int]:
1697
1701
  """First index of each chunk :func:`iter_matrix_chunks` reads: every
1698
- ``chunk_size``-th from chunk ``start_chunk`` on, less those holding no
1699
- ``selected`` entry."""
1700
- starts = range(start_chunk * chunk_size, length, chunk_size)
1702
+ ``chunk_size``-th from chunk ``start_chunk`` on; with ``selected``, each
1703
+ chunk starts at the first selected entry the previous one left uncovered.
1704
+
1705
+ Unselected stretches are never read, and chunks align with selected runs
1706
+ -- a few perturbations of a file sorted by perturbation are read as
1707
+ their own row ranges, not as the whole fixed-grid chunks they touch.
1708
+ Covering the selected entries greedily this way takes the fewest chunks
1709
+ of ``chunk_size`` possible, so never more than the fixed grid; when every
1710
+ entry is selected it is the grid."""
1711
+ offset = start_chunk * chunk_size
1701
1712
  if selected is None:
1702
- return list(starts)
1713
+ return list(range(offset, length, chunk_size))
1714
+ if start_chunk:
1715
+ # Chunk k of a selection is not rows k * chunk_size on: no grid to resume on.
1716
+ raise ValueError("start_chunk counts fixed-grid chunks and cannot be combined with selected")
1703
1717
  if len(selected) != length:
1704
1718
  raise ValueError(f"selected has {len(selected)} entries for an axis of {length}")
1705
- return [start for start in starts if selected[start : start + chunk_size].any()]
1719
+ rows = np.flatnonzero(np.asarray(selected, dtype=bool))
1720
+ starts: list[int] = []
1721
+ position = 0
1722
+ while position < rows.size:
1723
+ start = int(rows[position])
1724
+ starts.append(start)
1725
+ position = int(np.searchsorted(rows, start + chunk_size, side="left"))
1726
+ return starts
1706
1727
 
1707
1728
 
1708
1729
  def _decoded_block_reader(matrix, fmt: str | None, axis: int, n_obs: int, n_vars: int):
@@ -2264,7 +2285,7 @@ def _write_filtered_subset(
2264
2285
  )
2265
2286
  var[key] = np.asarray(values)
2266
2287
  if row_order is None:
2267
- uns = filtered_sorting_metadata(meta["uns"], obs, cell_mask)
2288
+ uns = filtered_sorting_metadata(meta["uns"], obs)
2268
2289
  else: # the caller records the new order
2269
2290
  uns = {k: v for k, v in meta["uns"].items() if k != "sorting_metadata"}
2270
2291
  layer_keys, obsm_keys, varm_keys, obsp_keys, varp_keys = (
@@ -5775,13 +5796,7 @@ def needs_sorting_for_nbglm(
5775
5796
  """
5776
5797
  backed = read_backed(path)
5777
5798
  try:
5778
- # First check if file is already sorted
5779
- if "sorting_metadata" in backed.uns:
5780
- metadata = backed.uns["sorting_metadata"]
5781
- if metadata.get("sorted_by") == perturbation_column:
5782
- logger.debug(f"Dataset is already sorted by '{perturbation_column}'")
5783
- return False
5784
-
5799
+ # A sorted file passes the contiguity check below.
5785
5800
  n_cells = backed.n_obs
5786
5801
 
5787
5802
  # Check cell count threshold
@@ -5797,6 +5812,9 @@ def needs_sorting_for_nbglm(
5797
5812
  labels = backed.obs[perturbation_column].astype(str).to_numpy()
5798
5813
  unique_labels = np.unique(labels)
5799
5814
  n_perts = len(unique_labels)
5815
+ if len(contiguous_ranges(labels)) == n_perts: # already sorted: one exact pass
5816
+ logger.debug(f"Dataset is already sorted by '{perturbation_column}'")
5817
+ return False
5800
5818
 
5801
5819
  # Check perturbation count threshold
5802
5820
  if n_perts < min_perturbations:
@@ -5841,60 +5859,40 @@ def perturbation_sort_order(labels: np.ndarray, control_label: str) -> np.ndarra
5841
5859
  return np.argsort(keys, kind="stable")
5842
5860
 
5843
5861
 
5844
- def sorting_metadata(
5845
- sorted_labels: np.ndarray, sort_indices: np.ndarray, *, perturbation_column: str, control_label: str,
5846
- original_path: Path,
5847
- ) -> dict:
5862
+ _SORTING_KEYS = ("original_path", "sorted_by", "control_label", "timestamp")
5863
+
5864
+
5865
+ def sorting_metadata(*, perturbation_column: str, control_label: str, original_path: Path) -> dict:
5848
5866
  """``uns["sorting_metadata"]`` of a file whose cells are in perturbation
5849
- order: each label's ``[start, end)`` rows, the column and control, and
5850
- the order itself for files under 100,000 cells (``sort_indices``, rows
5851
- of the unsorted cells)."""
5867
+ order (controls first): the column, the control and the source. Where
5868
+ each label's rows are is read off ``obs``
5869
+ (:func:`~crispyx._grouping.contiguous_ranges`), not stored."""
5852
5870
  import datetime
5853
5871
 
5854
- boundaries = _label_boundaries(sorted_labels)
5855
- metadata = {
5872
+ return {
5856
5873
  "original_path": str(original_path),
5857
5874
  "sorted_by": perturbation_column,
5858
5875
  "control_label": control_label,
5859
5876
  "timestamp": datetime.datetime.now().isoformat(),
5860
- "n_perturbations": len(boundaries),
5861
- "perturbation_boundaries": boundaries,
5862
5877
  }
5863
- if len(sort_indices) < 100000:
5864
- metadata["sort_order"] = np.asarray(sort_indices).tolist()
5865
- return metadata
5866
-
5867
-
5868
- def _label_boundaries(sorted_labels: np.ndarray) -> dict[str, list[int]] | None:
5869
- """Each label's ``[start, end)`` rows in ``sorted_labels``, or None when
5870
- a label's rows are not contiguous."""
5871
- sorted_labels = np.asarray(sorted_labels).astype(str)
5872
- if not sorted_labels.size:
5873
- return {}
5874
- starts = np.flatnonzero(np.r_[True, sorted_labels[1:] != sorted_labels[:-1]])
5875
- stops = np.r_[starts[1:], sorted_labels.size]
5876
- boundaries = {str(sorted_labels[a]): [int(a), int(b)] for a, b in zip(starts, stops)}
5877
- return boundaries if len(boundaries) == len(starts) else None
5878
-
5879
-
5880
- def filtered_sorting_metadata(uns: Mapping, obs: pd.DataFrame, cell_mask: np.ndarray) -> dict:
5881
- """``uns`` of a file whose rows are ``cell_mask``'s rows of a source with
5882
- ``uns``, in the source's order: a filter keeps a sorted file sorted, so its
5883
- ``sorting_metadata`` is rebuilt from the kept cells' labels in ``obs``
5884
- (dropped if they are no longer contiguous)."""
5878
+
5879
+
5880
+ def filtered_sorting_metadata(uns: Mapping, obs: pd.DataFrame) -> dict:
5881
+ """``uns`` of a file whose rows are a subset of a source with ``uns``,
5882
+ in the source's order: a filter keeps a sorted file sorted, so its
5883
+ ``sorting_metadata`` is kept unless the kept cells' labels in ``obs`` are
5884
+ no longer contiguous (a label dropped from between two runs of another)."""
5885
5885
  uns = dict(uns)
5886
5886
  metadata = uns.pop("sorting_metadata", None)
5887
5887
  if metadata is None:
5888
5888
  return uns
5889
5889
  column = metadata.get("sorted_by")
5890
- boundaries = _label_boundaries(obs[column].to_numpy()) if column in obs.columns else None
5891
- if boundaries is not None:
5892
- metadata = dict(metadata)
5893
- metadata["perturbation_boundaries"] = boundaries
5894
- metadata["n_perturbations"] = len(boundaries)
5895
- if "sort_order" in metadata:
5896
- metadata["sort_order"] = np.asarray(metadata["sort_order"])[np.asarray(cell_mask, dtype=bool)].tolist()
5897
- uns["sorting_metadata"] = metadata
5890
+ if column in obs.columns:
5891
+ labels = label_strings(obs[column])
5892
+ if len(contiguous_ranges(labels)) == len(pd.unique(labels)):
5893
+ # Only the stamp's own keys: a source sorted by an earlier release
5894
+ # also has its row map, in the source's row positions.
5895
+ uns["sorting_metadata"] = {key: metadata[key] for key in _SORTING_KEYS if key in metadata}
5898
5896
  return uns
5899
5897
 
5900
5898
 
@@ -5958,12 +5956,10 @@ def sort_by_perturbation(
5958
5956
 
5959
5957
  Notes
5960
5958
  -----
5961
- The sorted file contains additional metadata in uns['sorting_metadata']:
5962
- - original_path: Path to the original unsorted file
5963
- - sort_order: Array mapping new indices to original indices
5964
- - perturbation_boundaries: Dict mapping perturbation labels to (start, end) indices
5965
- - sorted_by: The column used for sorting
5966
- - timestamp: When the sorting was performed
5959
+ The sorted file records in uns['sorting_metadata'] the column it is
5960
+ sorted by (``sorted_by``), the ``control_label``, the
5961
+ ``original_path`` and a ``timestamp``. Each perturbation's rows are not
5962
+ stored: readers find them from ``obs`` in one pass over the labels.
5967
5963
 
5968
5964
  For sparse inputs the output is always written in CSR format, since sorting
5969
5965
  benefits row-wise (per-perturbation) access patterns used by NB-GLM. CSC
@@ -6027,8 +6023,7 @@ def sort_by_perturbation(
6027
6023
  backed.file.close()
6028
6024
 
6029
6025
  uns["sorting_metadata"] = sorting_metadata(
6030
- labels[sort_indices], sort_indices, perturbation_column=perturbation_column,
6031
- control_label=control_label, original_path=path,
6026
+ perturbation_column=perturbation_column, control_label=control_label, original_path=path,
6032
6027
  )
6033
6028
  uns[_PROVENANCE_KEY] = stamp("sorted", fingerprint)
6034
6029
 
@@ -6198,56 +6193,6 @@ def _write_sorted_sparse(
6198
6193
  )
6199
6194
 
6200
6195
 
6201
- def get_perturbation_slice(
6202
- adata_or_path: str | Path | ad.AnnData,
6203
- perturbation_label: str,
6204
- perturbation_column: str = "perturbation",
6205
- ) -> tuple[slice | None, bool]:
6206
- """Get slice for a perturbation's cells, checking if file is sorted.
6207
-
6208
- If the file is sorted by perturbation, returns a contiguous slice.
6209
- Otherwise, returns None for the slice (caller should use boolean mask).
6210
-
6211
- Parameters
6212
- ----------
6213
- adata_or_path
6214
- Path to h5ad file, or an already-opened AnnData object.
6215
- perturbation_label
6216
- Label of the perturbation to get slice for.
6217
- perturbation_column
6218
- Column containing perturbation labels.
6219
-
6220
- Returns
6221
- -------
6222
- tuple[slice | None, bool]
6223
- (slice object or None, is_sorted flag).
6224
- If is_sorted is True, slice is valid for contiguous access.
6225
- If is_sorted is False, slice is None and caller should use mask.
6226
- """
6227
- # Handle both path and AnnData input
6228
- if isinstance(adata_or_path, ad.AnnData):
6229
- adata = adata_or_path
6230
- should_close = False
6231
- else:
6232
- adata = read_backed(adata_or_path)
6233
- should_close = True
6234
-
6235
- try:
6236
- # Check for sorting metadata
6237
- if "sorting_metadata" in adata.uns:
6238
- metadata = adata.uns["sorting_metadata"]
6239
- if metadata.get("sorted_by") == perturbation_column:
6240
- boundaries = metadata.get("perturbation_boundaries", {})
6241
- if perturbation_label in boundaries:
6242
- start, end = boundaries[perturbation_label]
6243
- return slice(start, end), True
6244
-
6245
- return None, False
6246
- finally:
6247
- if should_close:
6248
- adata.file.close()
6249
-
6250
-
6251
6196
  # =============================================================================
6252
6197
  # Feature 1 — Backed metadata helpers (load/write obs and var without X)
6253
6198
  # =============================================================================
@@ -39,7 +39,6 @@ from .data import (
39
39
  calculate_wilcoxon_chunk_size,
40
40
  ensure_gene_symbol_column,
41
41
  get_matrix_storage_format,
42
- get_perturbation_slice,
43
42
  iter_matrix_chunks,
44
43
  needs_sorting_for_nbglm,
45
44
  read_backed,
@@ -92,7 +91,9 @@ from . import _messages
92
91
  from ._version import __version__
93
92
  from ._provenance import KEY as _PROVENANCE_KEY, SCHEMAS, call_fingerprint, reusable, stamp, write_stamp
94
93
  from ._disk import estimate_bytes, warn_if_disk_space_low
95
- from ._grouping import UNLABELLED, canonical_call_args, cell_labels, label_strings, resolve_group_reference_aliases
94
+ from ._grouping import (
95
+ UNLABELLED, canonical_call_args, cell_labels, contiguous_ranges, label_strings, resolve_group_reference_aliases,
96
+ )
96
97
  from ._memory import _resolve_memory_limit_bytes, _resolve_n_jobs, drop_file_cache
97
98
  from ._irls import ETA_MAX, ETA_MIN
98
99
  from ._size_factors import (
@@ -102,6 +103,7 @@ from ._size_factors import (
102
103
  _control_reference_size_factors,
103
104
  )
104
105
  from ._statistics import (
106
+ _add_group_moments,
105
107
  _tie_correction,
106
108
  _adjust_pvalue_matrix,
107
109
  _low_expr_in_both_mask,
@@ -619,6 +621,18 @@ def _group_row_indices(labels: np.ndarray, groups: Iterable[str]) -> dict[str, n
619
621
  return out
620
622
 
621
623
 
624
+ def _group_rows(labels: np.ndarray, groups: Iterable[str]) -> "dict[str, slice | np.ndarray]":
625
+ """Each of ``groups``' rows: a ``slice`` when its cells are one
626
+ contiguous run (every group of a file sorted by perturbation; one read
627
+ of the matrix's data and indices, and small to send to a worker), else
628
+ their ascending indices."""
629
+ groups = list(groups)
630
+ ranges = contiguous_ranges(labels)
631
+ scattered = [label for label in groups if label not in ranges]
632
+ indices = _group_row_indices(labels, scattered) if scattered else {}
633
+ return {label: ranges[label] if label in ranges else indices[label] for label in groups}
634
+
635
+
622
636
  #: Arguments that change how a DE call runs but not what it writes; a
623
637
  #: finished result or a checkpoint from a call differing only in these is
624
638
  #: still reused. The memory and chunking knobs belong here because rerunning
@@ -1062,6 +1076,9 @@ def t_test(
1062
1076
  Column in `adata.var` with gene symbols. If None, uses `adata.var_names`.
1063
1077
  perturbations
1064
1078
  Specific perturbations to test. If None, tests all non-control groups.
1079
+ Stretches of rows holding none of these or the controls are not
1080
+ read: on a file sorted by perturbation (:func:`sort_by_perturbation`)
1081
+ only their own row ranges are.
1065
1082
  min_cells_expressed
1066
1083
  Minimum total cells (control + perturbation) expressing a gene for
1067
1084
  testing. Genes below this threshold are untested: ``NaN`` in
@@ -1241,24 +1258,15 @@ def t_test(
1241
1258
  sumsq = np.zeros((n_groups_total, n_genes), dtype=np.float64)
1242
1259
  counts = np.zeros(n_groups_total, dtype=np.int64)
1243
1260
  expr_counts = np.zeros((n_groups_total, n_genes), dtype=np.int32)
1261
+ # With a subset of perturbations, the cells of the others (coded -1)
1262
+ # are not read where they fill whole stretches: all but the tested
1263
+ # groups' row ranges of a file sorted by perturbation.
1264
+ tested = label_codes >= 0
1244
1265
  for slc, block in iter_matrix_chunks(
1245
- backed, axis=0, chunk_size=cell_chunk_size, convert_to_dense=False
1266
+ backed, axis=0, chunk_size=cell_chunk_size, convert_to_dense=False,
1267
+ selected=None if tested.all() else tested, read_ahead=True,
1246
1268
  ):
1247
- slice_codes = label_codes[slc]
1248
- csr = sp.csr_matrix(block)
1249
- for code in np.unique(slice_codes):
1250
- # Cells of a perturbation this run does not test are coded -1,
1251
- # which would index the last group's row.
1252
- if code < 0:
1253
- continue
1254
- row_mask = slice_codes == code
1255
- group_block = csr[row_mask, :]
1256
- # Expression count: number of nonzero per gene
1257
- expr_counts[code] += np.asarray(group_block.getnnz(axis=0), dtype=np.int32)
1258
- # Sum and sumsq using sparse ops
1259
- sums[code] += group_block.sum(axis=0).A1.astype(np.float64)
1260
- sumsq[code] += group_block.power(2).sum(axis=0).A1.astype(np.float64)
1261
- counts[code] += row_mask.sum()
1269
+ _add_group_moments(sp.csr_matrix(block), label_codes[slc], sums, sumsq, expr_counts, counts)
1262
1270
  finally:
1263
1271
  backed.file.close()
1264
1272
 
@@ -1949,17 +1957,6 @@ def nb_glm_test(
1949
1957
  for label in candidates:
1950
1958
  if label not in observed:
1951
1959
  raise ValueError(f"Perturbation '{label}' contains no cells")
1952
-
1953
- # Check if file is sorted by perturbation for efficient I/O
1954
- perturbation_boundaries = None
1955
- if "sorting_metadata" in backed.uns:
1956
- metadata = backed.uns["sorting_metadata"]
1957
- if metadata.get("sorted_by") == perturbation_column:
1958
- perturbation_boundaries = metadata.get("perturbation_boundaries", {})
1959
- if perturbation_boundaries:
1960
- logger.debug(
1961
- f"Using sorted file with {len(perturbation_boundaries)} contiguous perturbation groups"
1962
- )
1963
1960
  finally:
1964
1961
  backed.file.close()
1965
1962
 
@@ -2312,15 +2309,8 @@ def nb_glm_test(
2312
2309
  if run.resumed:
2313
2310
  _messages.vprint(verbose, "nb_glm_test", f"Resuming: {len(completed_labels)}/{n_groups} perturbations already done")
2314
2311
 
2315
- # Each comparison's rows: a slice of a file sorted by perturbation, else
2316
- # their indices. A worker gets only its own, not the per-cell labels.
2317
- boundaries = perturbation_boundaries or {}
2318
- unsorted = [label for label in candidates if label not in boundaries]
2319
- row_indices = _group_row_indices(labels, unsorted) if unsorted else {}
2320
- group_rows = {
2321
- label: slice(*boundaries[label]) if label in boundaries else row_indices[label]
2322
- for label in candidates
2323
- }
2312
+ # A worker gets only its own comparison's rows, not the per-cell labels.
2313
+ group_rows = _group_rows(labels, candidates)
2324
2314
 
2325
2315
  # =====================================================================
2326
2316
  # Workers: as many as the budget holds
@@ -3193,11 +3183,6 @@ def nb_offset_test(
3193
3183
  "dispersion cannot be estimated; pass dispersion=, or use nb_glm_test, which estimates it "
3194
3184
  "from every unit."
3195
3185
  )
3196
- boundaries = {}
3197
- if "sorting_metadata" in backed.uns:
3198
- metadata = backed.uns["sorting_metadata"]
3199
- if metadata.get("sorted_by") == perturbation_column:
3200
- boundaries = metadata.get("perturbation_boundaries", {}) or {}
3201
3186
  finally:
3202
3187
  backed.file.close()
3203
3188
  n_strata = len(stratum_names)
@@ -3246,13 +3231,7 @@ def nb_offset_test(
3246
3231
  n_control = baseline.n_cells
3247
3232
  pts_rest = (baseline.nnz / max(n_control, 1)).astype(np.float32)
3248
3233
 
3249
- # Each perturbation's rows: a slice of a file sorted by perturbation
3250
- # (one read of the matrix's data and indices), else their indices.
3251
- unsorted = [label for label in candidates if label not in boundaries]
3252
- row_indices = _group_row_indices(labels, unsorted) if unsorted else {}
3253
- group_rows = {
3254
- label: slice(*boundaries[label]) if label in boundaries else row_indices[label] for label in candidates
3255
- }
3234
+ group_rows = _group_rows(labels, candidates)
3256
3235
  # A fit holds a block of cells dense in about five cells x genes arrays
3257
3236
  # (baseline, mean, weight, their products), and every thread fits at
3258
3237
  # once: the threads' blocks together take a tenth of the budget, so
@@ -57,16 +57,12 @@ def _kept_order(labels: np.ndarray, cell_mask: np.ndarray, control_label: str, s
57
57
  return perturbation_sort_order(labels[cell_mask], control_label) if sort else None
58
58
 
59
59
 
60
- def _stamp_sorted(
61
- output_path: Path, kept_labels: np.ndarray, order: np.ndarray, *, perturbation_column: str,
62
- control_label: str, source_path: Path,
63
- ) -> None:
64
- """Record in a filtered file written in ``order`` that its cells are in
65
- perturbation order, as :func:`~crispyx.data.sort_by_perturbation` does."""
60
+ def _stamp_sorted(output_path: Path, *, perturbation_column: str, control_label: str, source_path: Path) -> None:
61
+ """Record in a filtered file written in perturbation order that its
62
+ cells are, as :func:`~crispyx.data.sort_by_perturbation` does."""
66
63
  with h5py.File(output_path, "r+") as handle:
67
64
  write_elem(handle["uns"], "sorting_metadata", sorting_metadata(
68
- kept_labels[order], order, perturbation_column=perturbation_column, control_label=control_label,
69
- original_path=source_path,
65
+ perturbation_column=perturbation_column, control_label=control_label, original_path=source_path,
70
66
  ))
71
67
 
72
68
 
@@ -1032,11 +1028,10 @@ def _qc_in_memory(
1032
1028
  rows = rows[order]
1033
1029
  adata_filtered = adata[rows, gene_mask].copy()
1034
1030
  if order is None:
1035
- adata_filtered.uns = filtered_sorting_metadata(adata_filtered.uns, adata_filtered.obs, combined_cell_mask)
1031
+ adata_filtered.uns = filtered_sorting_metadata(adata_filtered.uns, adata_filtered.obs)
1036
1032
  else:
1037
1033
  adata_filtered.uns["sorting_metadata"] = sorting_metadata(
1038
- labels_full[rows], order, perturbation_column=perturbation_column, control_label=control_label,
1039
- original_path=Path(path),
1034
+ perturbation_column=perturbation_column, control_label=control_label, original_path=Path(path),
1040
1035
  )
1041
1036
 
1042
1037
  # Free original data immediately
@@ -1218,8 +1213,8 @@ def _qc_column_oriented(
1218
1213
  memory_limit_gb=memory_limit_gb,
1219
1214
  )
1220
1215
  if order is not None:
1221
- _stamp_sorted(output_path, labels[combined_cell_mask], order, perturbation_column=perturbation_column,
1222
- control_label=control_label, source_path=Path(path))
1216
+ _stamp_sorted(output_path, perturbation_column=perturbation_column, control_label=control_label,
1217
+ source_path=Path(path))
1223
1218
 
1224
1219
  filtered = AnnData(output_path)
1225
1220
  filtered_labels = filtered.obs[perturbation_column].astype(str)
@@ -1414,8 +1409,8 @@ def _qc_row_oriented(
1414
1409
  if chunk_cache is not None:
1415
1410
  chunk_cache.cleanup()
1416
1411
  if order is not None:
1417
- _stamp_sorted(output_path, labels[combined_cell_mask], order, perturbation_column=perturbation_column,
1418
- control_label=control_label, source_path=Path(path))
1412
+ _stamp_sorted(output_path, perturbation_column=perturbation_column, control_label=control_label,
1413
+ source_path=Path(path))
1419
1414
 
1420
1415
  filtered = AnnData(output_path)
1421
1416
  filtered_labels = filtered.obs[perturbation_column].astype(str)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: crispyx
3
- Version: 0.2.6
3
+ Version: 0.2.7
4
4
  Summary: Memory-efficient streaming analysis of large-scale CRISPR and Perturb-seq screens on disk-backed AnnData files
5
5
  Author: Jin-Hong Du
6
6
  License: Modified MIT License
@@ -81,6 +81,7 @@ tests/test_scanpy_parity.py
81
81
  tests/test_shrink_lfc_arrays.py
82
82
  tests/test_sort_by_perturbation.py
83
83
  tests/test_standardize.py
84
+ tests/test_statistics.py
84
85
  tests/test_stream_on_fast_axis.py
85
86
  tests/test_subsample.py
86
87
  tests/test_temp_file_cleanup.py
@@ -930,7 +930,7 @@ def test_the_mean_floor_does_not_make_genes_report_as_non_convergent():
930
930
  np.testing.assert_allclose(again.coef, result.coef, rtol=1e-10, atol=1e-12)
931
931
 
932
932
 
933
- def test_floored_genes_match_pydeseq2_irls():
933
+ def test_floored_genes_match_pydeseq2_irls(pydeseq2_helper):
934
934
  """The floor exists for DESeq2 compatibility, so that is what it is worth
935
935
  measuring: on the genes it binds on, the coefficients must be PyDESeq2's.
936
936
 
@@ -938,8 +938,7 @@ def test_floored_genes_match_pydeseq2_irls():
938
938
  a couple of steps from the answer -- 2.8e-02 away in coefficient, against
939
939
  1.4e-06 on the genes the floor never touches.
940
940
  """
941
- pytest.importorskip("pydeseq2")
942
- from pydeseq2.utils import irls_solver
941
+ irls_solver = pydeseq2_helper("irls_solver")
943
942
 
944
943
  counts, design, size_factors, alpha = _floored_fixture()
945
944
  offset = np.log(size_factors)
@@ -957,7 +956,7 @@ def test_floored_genes_match_pydeseq2_irls():
957
956
  assert np.max(np.abs(result.coef[binds] - reference[binds])) < 1e-4
958
957
 
959
958
 
960
- def test_standard_errors_match_pydeseq2_wald_test():
959
+ def test_standard_errors_match_pydeseq2_wald_test(pydeseq2_helper):
961
960
  """``min_mu`` steadies the iteration and has no place in the covariance.
962
961
 
963
962
  DESeq2 keeps it out: ``irls_solver`` returns an unthresholded ``mu`` and
@@ -965,8 +964,7 @@ def test_standard_errors_match_pydeseq2_wald_test():
965
964
  floored mean instead made the standard errors a median of 24% too small on
966
965
  floor-binding genes, and the Wald statistics correspondingly too large.
967
966
  """
968
- pytest.importorskip("pydeseq2")
969
- from pydeseq2.utils import wald_test
967
+ wald_test = pydeseq2_helper("wald_test")
970
968
 
971
969
  counts, design, size_factors, alpha = _floored_fixture()
972
970
  offset = np.log(size_factors)