vima-spatial 0.2.8__tar.gz → 0.2.9__tar.gz

{vima_spatial-0.2.8 → vima_spatial-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vima-spatial
-Version: 0.2.8
+Version: 0.2.9
 Summary: variational inference-based microniche analysis
 Home-page: https://github.com/yakirr/vima
 Author: Yakir Reshef

{vima_spatial-0.2.8 → vima_spatial-0.2.9}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vima-spatial
-version = 0.2.8
+version = 0.2.9
 author = Yakir Reshef
 author_email = yreshef@broadinstitute.org
 description = variational inference-based microniche analysis

{vima_spatial-0.2.8 → vima_spatial-0.2.9}/src/vima/ingest/ingest.py

@@ -1,5 +1,6 @@
 import numpy as numpy
 import numpy as np
+import pandas as pd
 import xarray as xr
 import cv2 as cv2
 import scanpy as sc
@@ -105,6 +106,161 @@ def add_covs(pca, sid_to_covs):
         pca[cov_name] = pca['sid'].map(sid_to_covs[cov_name])
     return ['sid'] + cov_names
 
+def collapse_markers(normeddir, markers, pseudomarker, outdir, masksdir=None):
+    """
+    Keep a subset of markers and fold all the others into one pseudomarker.
+
+    Reads the normalized pixel matrices written by `prepare_merfish`,
+    `prepare_xenium5k`, or `nonst.prepare` (for transcriptomic data the markers
+    are genes), retains `markers`, and replaces every other marker with a single
+    channel named `pseudomarker` holding their combined signal. Because the
+    stored values are log-normalized, they are exponentiated, summed, and
+    log-normalized again, so the pseudomarker is on the same scale as a real
+    marker. The dataset-wide means and stds stored on each file are rewritten to
+    match the new marker set: retained markers keep their existing values and the
+    pseudomarker's are computed from the collapsed data.
+
+    Parameters
+    ----------
+    normeddir
+        Directory of normalized ``.nc`` files, e.g. ``{outdir}/normalized``.
+    markers
+        Markers to retain, in the order they should appear; the pseudomarker is
+        appended after them. Markers absent from the data are skipped with a
+        warning.
+    pseudomarker
+        Name for the new channel, e.g. ``'nonimmune'``.
+    outdir
+        Directory to write the collapsed ``.nc`` files to. This is a
+        ``normalized``-style directory, so to build a dataset that
+        `pca_pixels` can be pointed at, pass ``f'{newroot}/normalized'`` and copy
+        the masks to ``f'{newroot}/masks'``.
+    masksdir
+        Directory of tissue masks, used to compute the pseudomarker's moments
+        over non-empty pixels only. Defaults to ``masks`` alongside `normeddir`.
+    """
+    import netCDF4
+
+    if masksdir is None:
+        masksdir = os.path.join(os.path.dirname(os.path.normpath(normeddir)), 'masks')
+
+    sids = [os.path.splitext(f)[0]
+        for f in os.listdir(normeddir) if f.endswith('.nc') and not f.startswith('.')]
+    if len(sids) == 0:
+        logger.warning(f'No .nc files found in {normeddir}. Check your path and try again.')
+        return
+    os.makedirs(outdir, exist_ok=True)
+
+    # decide on the new marker set, using the first sample as the reference
+    da = xr.open_dataarray(f'{normeddir}/{sids[0]}.nc')
+    ref_markers = list(da.marker.values)
+    ref_means, ref_stds = da.attrs['means'], da.attrs['stds']
+    da.close(); del da
+
+    if pseudomarker in ref_markers:
+        raise ValueError(f"'{pseudomarker}' is already a marker; pick a name for the "
+                         f"pseudomarker that isn't in the data.")
+    missing = [m for m in markers if m not in ref_markers]
+    if missing:
+        logger.warning(f'{len(missing)} of the {len(markers)} requested markers are not in the '
+                       f'data and will be skipped: {missing}')
+    keep = [m for m in markers if m in ref_markers]
+    if len(keep) == 0:
+        raise ValueError('None of the requested markers are in the data.')
+    if len(keep) == len(ref_markers):
+        logger.warning(f'All {len(ref_markers)} markers were retained, so {pseudomarker} will be '
+                       f'empty.')
+    logger.info(f'Keeping {len(keep)} markers and collapsing the other '
+                f'{len(ref_markers) - len(keep)} into {pseudomarker}.')
+
+    # collapse each sample, accumulating the pseudomarker's per-sample moments as we go
+    sample_sids, sample_means, sample_stds, sample_npixels = [], [], [], []
+    for sid in settings.progress(sids, name='collapsing markers'):
+        da = xr.open_dataarray(f'{normeddir}/{sid}.nc')
+        mask_da = xr.open_dataarray(f'{masksdir}/{sid}.nc')
+
+        sid_markers = list(da.marker.values)
+        if sid_markers != ref_markers:
+            logger.warning(f'{sid} has different markers ({len(sid_markers)}) than {sids[0]} '
+                           f'({len(ref_markers)}); matching by name.')
+        absent = [m for m in keep if m not in sid_markers]
+        if absent:
+            raise ValueError(f'{sid} is missing {len(absent)} of the markers to retain: {absent}')
+        ix = {m: i for i, m in enumerate(sid_markers)}
+        keep_ix = [ix[m] for m in keep]
+        keep_ix_set = set(keep_ix)
+        drop_ix = [i for i in range(len(sid_markers)) if i not in keep_ix_set]
+
+        # load raw arrays and close before doing any work, so we never hold two
+        # full (H x W x n_markers) copies simultaneously
+        x = da.x.values; y = da.y.values
+        data = da.values
+        mask = mask_da.values
+        da.close(); mask_da.close()
+        del da, mask_da; gc.collect()
+
+        # accumulate the dropped markers one at a time: fancy-indexing them all at
+        # once would materialize the (y, x, n_dropped) copy this function exists to
+        # avoid. log-normalized values are exponentiated before summing so the
+        # pseudomarker ends up on the same scale as the markers we kept.
+        acc = np.zeros(data.shape[:2], dtype=np.float64)
+        for j in drop_ix:
+            acc += np.expm1(data[..., j].astype(np.float64))
+        other = np.log1p(acc)
+        del acc
+
+        # empty pixels are zero in the input, and log1p(sum(expm1(0))) is zero, so
+        # they stay empty without any special handling
+        s = xr.DataArray(
+            np.concatenate([data[..., keep_ix], other[..., None].astype(np.float32)], axis=-1),
+            dims=['y', 'x', 'marker'],
+            coords={'x': x, 'y': y, 'marker': keep + [pseudomarker]})
+        s.name = sid
+        util.write_xarray(s, f'{outdir}/{sid}.nc')
+        del data, s; gc.collect()
+
+        # a sample with an empty mask contributes nothing rather than a nan that
+        # would poison the pooled moments
+        vals = other[mask]
+        if len(vals) == 0:
+            logger.warning(f'{sid} has no non-empty pixels; excluding it from the '
+                           f'{pseudomarker} moments.')
+        else:
+            sample_sids.append(sid)
+            sample_means.append(vals.mean(dtype=np.float64))
+            sample_stds.append(vals.std(dtype=np.float64))
+            sample_npixels.append(len(vals))
+        del other, mask, vals; gc.collect()
+
+    if len(sample_sids) == 0:
+        raise ValueError(f'No non-empty pixels in any sample, so {pseudomarker} has no moments. '
+                         f'Check that {masksdir} holds the masks for {normeddir}.')
+
+    # pool the pseudomarker's moments across samples the same way get_sumstats does
+    pseudo_mean, pseudo_std = util.pool_moments(
+        pd.DataFrame([sample_means], index=[pseudomarker], columns=sample_sids),
+        pd.DataFrame([sample_stds],  index=[pseudomarker], columns=sample_sids),
+        sample_npixels)
+    pseudo_mean, pseudo_std = pseudo_mean.iloc[0], pseudo_std.iloc[0]
+    if pseudo_std == 0:
+        # a constant channel stays exactly 0 after standardization rather than
+        # dividing by zero downstream
+        logger.warning(f'{pseudomarker} has zero variance; setting its std to 1.')
+        pseudo_std = 1.
+    logger.info(f'{pseudomarker}: mean {pseudo_mean:.3f}, std {pseudo_std:.3f}')
+
+    # the pseudomarker's moments aren't known until every sample has been read, so
+    # the files are written above without them and stamped here; this rewrites
+    # metadata only, rather than re-collapsing every sample a second time
+    keep_ix_ref = [ref_markers.index(m) for m in keep]
+    means = np.append(np.asarray(ref_means)[keep_ix_ref], pseudo_mean).astype(np.float32)
+    stds  = np.append(np.asarray(ref_stds)[keep_ix_ref],  pseudo_std).astype(np.float32)
+    for sid in sids:
+        with netCDF4.Dataset(f'{outdir}/{sid}.nc', 'a') as ds:
+            v = ds.variables[sid]
+            v.setncattr('means', means)
+            v.setncattr('stds', stds)
+
 def pca_pixels(outdir, repname, nmetamarkers=10, npixels_to_plot=50000,
                total_n_metapixels=2_000_000, sid_to_covs=None):
     """

{vima_spatial-0.2.8 → vima_spatial-0.2.9}/src/vima/ingest/nonst.py

@@ -1,5 +1,6 @@
 import os, glob, gc
 import numpy as np
+import pandas as pd
 import cv2 as cv2
 import xarray as xr
 from . import util
@@ -134,12 +135,14 @@ def prepare(load, filepaths, orig_pixel_size, markers, get_foreground, norm_by_b
             )
         for sid in settings.progress(sids)])
     gc.collect()
-    _, pixels = norm_by_background(pixels)
+    goodmarkers, pixels = norm_by_background(pixels)
     ntranscripts = pixels.sum(axis=1, dtype=np.float64)
     med_ntranscripts = np.median(ntranscripts)
     pixels = np.log1p(med_ntranscripts * pixels / (ntranscripts[:,None] + 1e-6)) # adding to denominator in case pixel is all 0s
-    means = pixels.mean(axis=0, dtype=np.float64)
-    stds = pixels.std(axis=0, dtype=np.float64)
+    # indexed by marker name so each sample's moments are aligned by name below,
+    # rather than positionally against whatever markers that sample kept
+    means = pd.Series(pixels.mean(axis=0, dtype=np.float64), index=goodmarkers)
+    stds = pd.Series(pixels.std(axis=0, dtype=np.float64), index=goodmarkers)
     del pixels; gc.collect()
     
     logger.info('Normalizing and writing')
@@ -153,6 +156,7 @@ def prepare(load, filepaths, orig_pixel_size, markers, get_foreground, norm_by_b
         pl = np.log1p(med_ntranscripts * pl / (pl.sum(axis=1)[:,None] + 1e-6)) # adding to denominator in case pixel is all 0s
         s = s.sel(marker=goodmarkers)
         util.set_pixels(s, mask, pl)
-        s.attrs['means'] = means.reindex(s.marker.values, fill_value=1).values.astype(np.float32)
-        s.attrs['stds'] = stds.reindex(s.marker.values, fill_value=0).values.astype(np.float32)
+        # a marker with no moments standardizes to a no-op rather than dividing by zero
+        s.attrs['means'] = means.reindex(s.marker.values, fill_value=0).values.astype(np.float32)
+        s.attrs['stds'] = stds.reindex(s.marker.values, fill_value=1).values.astype(np.float32)
         util.write_xarray(s, f'{normeddir}/{sid}.nc')

{vima_spatial-0.2.8 → vima_spatial-0.2.9}/src/vima/ingest/st.py

@@ -6,50 +6,72 @@ from matplotlib import pyplot as plt
 from . import util
 from .._settings import logger, settings
 
-def med_ntranscripts(load, filepaths, x_col, y_col, pixel_size=10):
+def rasterize_all(load, filepaths, x_col, y_col, gene_col, pixel_size=10):
+    """
+    Bin every sample's transcripts to a pixel grid, once.
+
+    Ingest makes three passes over the samples -- normalization factor, then
+    HVGs and dataset-wide moments, then the final matrices -- and each pass
+    needs the same binned counts. Binning once and holding the sparse result
+    trades memory for two fewer reads and two fewer binning passes per sample;
+    the counts are ~1-2% dense, so a 34-sample Xenium 5K cohort costs a few GB.
+
+    Returns
+    -------
+    list
+        ``(sample_id, SparsePixels)`` per file, in ``filepaths`` order.
+    """
+    rasterized = []
+    for filepath in settings.progress(filepaths, name='binning transcripts'):
+        sid, data = load(filepath)
+        pixels = util.transcriptlist_to_sparsepixels(data, x_col, y_col, gene_col,
+                                                     pixel_size=pixel_size)
+        del data; gc.collect()
+        logger.info(f'\t{sid}: {pixels.ntranscripts/1e6:.2f}M tx → '
+                    f'{len(pixels.nonempty())} non-empty pixels '
+                    f'({pixels.nbytes()/1e9:.2f} GB sparse)')
+        rasterized.append((sid, pixels))
+
+    held = sum(p.nbytes() for _, p in rasterized)
+    logger.info(f'\tHolding {held/1e9:.2f} GB of binned counts for reuse across passes.')
+    return rasterized
+
+def med_ntranscripts(rasterized):
     """
     Compute the normalization target: median transcripts per dense pixel.
 
-    Bins each sample's transcripts into ``pixel_size`` square pixels and takes
-    the median transcript count over pixels with at least 10 transcripts; returns
-    the mean of these per-sample medians, used as the total-count normalization
-    target.
+    Takes the median transcript count over pixels with at least 10 transcripts
+    in each sample, and returns the mean of those per-sample medians; this is
+    the total-count normalization target. Counts come from the same binning the
+    rest of ingest uses, so any transcript with no gene assigned is excluded.
     """
     medians = []
-    for i, filepath in enumerate(settings.progress(filepaths, name='median #transcripts')):
-        sid, df = load(filepath)
-
-        df["px"] = (df[x_col] // pixel_size).astype(np.int32)
-        df["py"] = (df[y_col] // pixel_size).astype(np.int32)
-        pg = (
-            df
-            .groupby(["px", "py"])
-            .size()
-            .reset_index(name="txcount")
-        )
-        med = pg[pg.txcount > 10].txcount.median()
-        logger.info(f"\tSample {i+1}/{len(filepaths)}: {sid} ({len(df)/1e6:.2f}M tx) "
+    for i, (sid, pixels) in enumerate(settings.progress(rasterized, name='median #transcripts')):
+        # float64 so the median (and the normalization factor built from it)
+        # carries full precision, as it did when counts were an int64 table
+        totals = np.asarray(pixels.counts.sum(axis=1), dtype=np.float64).ravel()
+        med = np.median(totals[totals > 10])
+        logger.info(f"\tSample {i+1}/{len(rasterized)}: {sid} ({pixels.ntranscripts/1e6:.2f}M tx) "
                     f"→ median {med} transcripts per pixel with >=10 transcripts")
         medians.append(med)
-        del df, pg; gc.collect()
     return np.mean(np.array(medians))
 
-def get_sumstats(load, filepaths, target_sum, x_col, y_col, gene_col, n_top_genes_per_sample=200, genes_to_add=[],
-                 pixel_size=10, min_mean=0.01, min_ntranscripts_per_pixel=10, min_ngenes_per_pixel=1,
+def get_sumstats(rasterized, target_sum, n_top_genes_per_sample=200, genes_to_add=[],
+                 min_mean=0.01, min_ntranscripts_per_pixel=10, min_ngenes_per_pixel=1,
                  min_npixels=20, min_totalcounts=500):
     """
     Select highly variable genes and compute dataset-wide per-gene moments.
 
-    For each sample, rasterizes transcripts to pixels, QC-filters pixels,
-    log-normalizes to ``target_sum``, and selects the top highly variable genes;
-    the union of HVGs across samples (plus any ``genes_to_add``) is returned.
-    Also returns the pixel-count-weighted grand mean and grand std per gene
-    across the whole dataset (pooling within- and between-sample variance).
+    For each sample, QC-filters the binned pixels, log-normalizes them to
+    ``target_sum``, and selects the top highly variable genes; the union of HVGs
+    across samples (plus any ``genes_to_add``) is returned. Also returns the
+    pixel-count-weighted grand mean and grand std per gene across the whole
+    dataset (pooling within- and between-sample variance).
 
     Parameters
     ----------
-    load
-        Callable mapping a file path to ``(sample_id, transcript_dataframe)``.
+    rasterized
+        ``(sample_id, SparsePixels)`` per sample, from `rasterize_all`.
     target_sum
         Per-pixel total-count normalization target (from `med_ntranscripts`).
     n_top_genes_per_sample
@@ -71,29 +93,31 @@ def get_sumstats(load, filepaths, target_sum, x_col, y_col, gene_col, n_top_gene
     stds = []
     npixels = []
 
-    for i, filepath in enumerate(settings.progress(filepaths, name='HVGs & moments')):
-        sid, transcripts = load(filepath)
-        ntranscripts = len(transcripts)
-
-        # create pixellist
-        pl = util.transcriptlist_to_pixellist(transcripts, x_col, y_col, gene_col, pixel_size=pixel_size)
-        del transcripts; gc.collect()
-        
-        # create scanpy object and filter empty/near-empty pixels
-        genes = pl.columns[2:]
-        obs = pl[['pixel_x', 'pixel_y']].copy()
-        obs.index = obs.index.astype(str)
-        pl = sc.AnnData(pl[genes].values.astype(np.float32), var=pd.DataFrame(index=genes), obs=obs)
-        sc.pp.filter_cells(pl, min_counts=min_ntranscripts_per_pixel, inplace=True)
-        sc.pp.filter_cells(pl, min_genes=min_ngenes_per_pixel, inplace=True)
-        
+    for i, (sid, pixels) in enumerate(settings.progress(rasterized, name='HVGs & moments')):
+        ntranscripts = pixels.ntranscripts
+
+        # filter empty/near-empty pixels; both thresholds are per-pixel, so
+        # applying them together matches filtering on one and then the other
+        genes = pixels.genes
+        keep, totals = util.qc_pixels(pixels.counts, min_ntranscripts_per_pixel, min_ngenes_per_pixel)
+        rows = np.flatnonzero(keep)
+        counts = pixels.counts[keep]
+
         # compute moments
-        X = pl.X / pl.X.sum(axis=1, keepdims=True) * target_sum
-        X = np.log1p(X)
-        means.append(pd.Series(np.array(X.mean(axis=0, dtype=np.float64)).squeeze(), index=genes))
-        stds.append(pd.Series(np.array(X.std(axis=0, dtype=np.float64)).squeeze(), index=genes))
-        npixels.append(pl.n_obs)
+        X = util.lognormalize(counts, totals[keep], target_sum)
+        mean, std = util.sparse_moments(X)
+        means.append(pd.Series(mean, index=genes))
+        stds.append(pd.Series(std, index=genes))
+        npixels.append(len(rows))
         union_allgenes.update(genes)
+        del X; gc.collect()
+
+        # create scanpy object over the surviving pixels
+        pixel_x, pixel_y = pixels.coords_of(rows)
+        obs = pd.DataFrame({'pixel_x': pixel_x, 'pixel_y': pixel_y})
+        obs.index = obs.index.astype(str)
+        pl = sc.AnnData(counts, var=pd.DataFrame(index=genes), obs=obs)
+        del counts; gc.collect()
 
         # QC genes and compute HVGs for this sample
         sc.pp.filter_genes(pl, min_cells=min_npixels, inplace=True)
@@ -149,43 +173,36 @@ def get_sumstats(load, filepaths, target_sum, x_col, y_col, gene_col, n_top_gene
                     plt.close(fig)
 
                 union_hvgs.update(hvgs)
-                logger.info(f'\tSample {i+1}/{len(filepaths)}: {sid} ({ntranscripts/1e6:.2f}M tx) → {len(union_hvgs)} HVGs across all samples so far.')
+                logger.info(f'\tSample {i+1}/{len(rasterized)}: {sid} ({ntranscripts/1e6:.2f}M tx) → {len(union_hvgs)} HVGs across all samples so far.')
         else:
             union_hvgs.update(genes)
-            logger.info(f'\tSample {i+1}/{len(filepaths)}: {sid} ({ntranscripts/1e6:.2f}M tx) → Using all {len(genes)} genes.')
+            logger.info(f'\tSample {i+1}/{len(rasterized)}: {sid} ({ntranscripts/1e6:.2f}M tx) → Using all {len(genes)} genes.')
 
-        pl.X = None; del pl; del X; gc.collect()
+        pl.X = None; del pl; gc.collect()
 
     means_df = pd.concat([m.reindex(index=union_allgenes, fill_value=0) for m in means], axis=1)
     stds_df  = pd.concat([s.reindex(index=union_allgenes, fill_value=0) for s in stds],  axis=1)
-    w = np.array(npixels, dtype=np.float64)
-    W = w.sum()
-
-    grand_mean   = np.sum((means_df * w).values, axis=1, dtype=np.float64) / W
-    mean_of_vars = np.sum((stds_df ** 2 * w).values, axis=1, dtype=np.float64) / W
-    var_of_means = ((means_df.subtract(grand_mean, axis=0).values.astype(np.float64) ** 2) * w).sum(axis=1) / W
-    grand_mean = pd.Series(grand_mean, index=means_df.index)
-    grand_std  = pd.Series(np.sqrt(mean_of_vars + var_of_means), index=means_df.index)
+    grand_mean, grand_std = util.pool_moments(means_df, stds_df, npixels)
 
     return list(union_hvgs), grand_mean, grand_std
 
-def transcriptlist_to_normedpixelmatrix(sid, data, x_col, y_col, gene_col, pixel_size, target_sum, means, stds,
-                                  genes, min_ngenes_per_pixel, min_ntranscripts_per_pixel):
+def pixels_to_normedpixelmatrix(sid, pixels, target_sum, means, stds,
+                                genes, min_ngenes_per_pixel, min_ntranscripts_per_pixel):
     """
-    Rasterize one sample's transcripts to a log-normalized ``(y, x, gene)`` matrix.
+    Turn one sample's binned pixels into a log-normalized ``(y, x, gene)`` matrix.
 
-    Bins transcripts into square pixels, drops pixels failing the QC thresholds,
-    log-normalizes surviving pixels to ``target_sum``, restricts to ``genes``
-    (filling absent genes with zeros), and returns the pixel matrix alongside its
-    tissue mask. The dataset-wide ``means`` and ``stds`` are stored as attributes
-    on the matrix for later standardization (they are not applied here).
+    Drops pixels failing the QC thresholds, log-normalizes the survivors to
+    ``target_sum``, restricts to ``genes`` (filling absent genes with zeros), and
+    returns the pixel matrix alongside its tissue mask. The dataset-wide
+    ``means`` and ``stds`` are stored as attributes on the matrix for later
+    standardization (they are not applied here).
 
     Parameters
     ----------
     sid
         Sample ID; used to name the output arrays.
-    data
-        Transcript table for this sample.
+    pixels
+        This sample's `util.SparsePixels`, from `rasterize_all`.
     target_sum
         Per-pixel total-count normalization target.
     means, stds
@@ -199,49 +216,39 @@ def transcriptlist_to_normedpixelmatrix(sid, data, x_col, y_col, gene_col, pixel
         ``(mask, matrix)``: the boolean tissue mask and the normalized
         ``(y, x, gene)`` DataArray.
     """
-    logger.info(f'\tNumber of transcripts: {len(data)/1e6:.2f}M')
-
-    # process data
-    pl = util.transcriptlist_to_pixellist(
-        data,
-        x_col,
-        y_col,
-        gene_col,
-        pixel_size=pixel_size
-    )
-    markers = pl.columns[2:]
-    all_x = np.sort(pl['pixel_x'].unique())
-    all_y = np.sort(pl['pixel_y'].unique())
-    logger.info(f'\tMaking pixel list... {len(pl)} pixels.')
+    logger.info(f'\tNumber of transcripts: {pixels.ntranscripts/1e6:.2f}M')
+    logger.info(f'\t{len(pixels.nonempty())} non-empty pixels.')
 
     if settings.show_plots():
+        occupied_x, occupied_y = pixels.coords_of(pixels.nonempty())
         plt.figure(figsize=(5,5))
-        plt.scatter(pl.pixel_x, pl.pixel_y, c='gray', s=0.1, alpha=0.2)
-    pl = pl[(pl[markers] != 0).sum(axis=1) >= min_ngenes_per_pixel]
-    pl = pl[(pl[markers].sum(axis=1) >= min_ntranscripts_per_pixel) ]#& (pl[list(set(markers) & set(genes))].sum(axis=1) > 0)]
+        plt.scatter(occupied_x, occupied_y, c='gray', s=0.1, alpha=0.2)
+    # both thresholds are per-pixel, so applying them together matches
+    # filtering on one and then the other
+    keep, totals = util.qc_pixels(pixels.counts, min_ntranscripts_per_pixel, min_ngenes_per_pixel)
+    rows = np.flatnonzero(keep)
     if settings.show_plots():
-        plt.scatter(pl.pixel_x, pl.pixel_y, c=pl[markers].sum(axis=1), s=0.1, alpha=0.8, vmin=0, vmax=100)
+        kept_x, kept_y = pixels.coords_of(rows)
+        plt.scatter(kept_x, kept_y, c=totals[rows], s=0.1, alpha=0.8, vmin=0, vmax=100)
         plt.gca().set_aspect('equal'); plt.title('transcript density (gray = failed qc)'); plt.axis('off')
         settings.show(f'transcript_density_{sid}')
-    logger.info(f'\t{len(pl)} pixels after QC.')
+    logger.info(f'\t{len(rows)} pixels after QC.')
 
     logger.info('\tLog-normalizing...')
-    pl[markers] = pl[markers].div(pl[markers].sum(axis=1), axis=0).fillna(0) * target_sum
-    pl[markers] = np.log1p(pl[markers])
+    X = util.lognormalize(pixels.counts[keep], totals[keep], target_sum)
 
     logger.info(f'\trestricting to {len(genes)} genes')
-    pl = pl.reindex(columns=['pixel_x', 'pixel_y'] + genes, fill_value=0) # zeros for genes not present in this sample
-    mask_pl = pl[['pixel_x', 'pixel_y']].copy()
-    mask_pl['nonempty'] = 1
-    
-    s = util.pixellist_to_pixelmatrix(pl, genes).reindex({'x': all_x, 'y': all_y}, fill_value=0)
+    # sorted to match the marker order the pandas pivot used to impose; genes
+    # absent from this sample are scattered in as zeros
+    s = util.sparsepixels_to_pixelmatrix(X, pixels.genes, rows, pixels.x, pixels.y, sorted(genes))
+    del X; gc.collect()
     s.attrs['means'] = means.reindex(s.marker.values).values.astype(np.float32)
     s.attrs['stds'] = stds.reindex(s.marker.values).values.astype(np.float32)
-    mask = util.pixellist_to_pixelmatrix(mask_pl, ['nonempty']).squeeze().astype(bool).reindex({'x': all_x, 'y': all_y}, fill_value=False)
+    mask = util.sparsepixels_to_mask(rows, pixels.x, pixels.y)
     s.name = sid; mask.name = sid
     gc.collect()
     logger.info(f'\tMaking pixel matrix... done. shape: {s.shape}')
-    
+
     return mask, s.astype(np.float32)
 
 def rasterize_and_normalize_generic(load, filepaths, x_col, y_col, gene_col, n_top_genes_per_sample, pixel_size, outdir,
@@ -250,10 +257,12 @@ def rasterize_and_normalize_generic(load, filepaths, x_col, y_col, gene_col, n_t
     """
     Shared driver that rasterizes and normalizes every sample to disk.
 
-    Backs `prepare_merfish` and `prepare_xenium5k`: computes the normalization
-    factor and dataset-wide gene set/moments once, then rasterizes and
-    normalizes each sample, writing its normalized matrix and mask under
-    ``outdir/normalized`` and ``outdir/masks``.
+    Backs `prepare_merfish` and `prepare_xenium5k`: bins every sample's
+    transcripts once, computes the normalization factor and dataset-wide gene
+    set/moments from those bins, then normalizes each sample, writing its matrix
+    and mask under ``outdir/normalized`` and ``outdir/masks``. The binned counts
+    are held in memory across all three passes; each sample's are released as
+    soon as its matrix has been written.
 
     Parameters
     ----------
@@ -268,31 +277,36 @@ def rasterize_and_normalize_generic(load, filepaths, x_col, y_col, gene_col, n_t
         logger.warning('No files found. Check your filepaths and try again.')
         return
 
+    logger.info('Binning transcripts to pixels...')
+    rasterized = rasterize_all(load, filepaths, x_col, y_col, gene_col, pixel_size=pixel_size)
+
     logger.info('Computing normalization factor...')
-    normfactor = med_ntranscripts(load, filepaths, x_col, y_col, pixel_size=pixel_size)
+    normfactor = med_ntranscripts(rasterized)
     logger.info('Finding HVGs and dataset-wide mean and variance per gene...')
-    hvgs, means, stds = get_sumstats(load, filepaths, normfactor, x_col, y_col, gene_col,
+    hvgs, means, stds = get_sumstats(rasterized, normfactor,
                                      n_top_genes_per_sample=n_top_genes_per_sample,
-                                     genes_to_add=genes_to_add, pixel_size=pixel_size,
+                                     genes_to_add=genes_to_add,
                                      min_ntranscripts_per_pixel=min_ntranscripts_per_pixel,
                                      min_ngenes_per_pixel=min_ngenes_per_pixel)
     logger.info(f'Final number of genes used = {len(hvgs)}')
 
-    logger.info('Rasterizing and normalizing...')
+    logger.info('Normalizing...')
     normdir = f'{outdir}/normalized'
     masksdir = f'{outdir}/masks'
     os.makedirs(normdir, exist_ok=True)
     os.makedirs(masksdir, exist_ok=True)
-    for i, filepath in enumerate(settings.progress(filepaths, name='rasterize & normalize')):
-        sid, data = load(filepath)
-        logger.info(f'Processing sample {i+1}/{len(filepaths)}: {sid}')
-        mask, pm = transcriptlist_to_normedpixelmatrix(sid, data, x_col, y_col, gene_col, pixel_size,
-                                                       normfactor, means=means, stds=stds, genes=hvgs,
-                                                       min_ntranscripts_per_pixel=min_ntranscripts_per_pixel,
-                                                       min_ngenes_per_pixel=min_ngenes_per_pixel)
-        del data; gc.collect()
+    for i in settings.progress(range(len(rasterized)), name='normalize & write'):
+        sid, pixels = rasterized[i]
+        rasterized[i] = None   # this sample's bins are no longer needed after here
+        logger.info(f'Processing sample {i+1}/{len(rasterized)}: {sid}')
+        mask, pm = pixels_to_normedpixelmatrix(sid, pixels, normfactor, means=means, stds=stds,
+                                               genes=hvgs,
+                                               min_ntranscripts_per_pixel=min_ntranscripts_per_pixel,
+                                               min_ngenes_per_pixel=min_ngenes_per_pixel)
+        del pixels; gc.collect()
         util.write_xarray(mask, f'{masksdir}/{pm.name}.nc')
         util.write_xarray(pm, f'{normdir}/{pm.name}.nc')
+        del mask, pm; gc.collect()
 
 def prepare_xenium5k(load, filepaths, x_col, y_col, gene_col, n_top_genes_per_sample, outdir,
                      pixel_size=10, genes_to_add=[],