segger 1.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- segger/__init__.py +68 -0
- segger/__main__.py +4 -0
- segger/api.py +64 -0
- segger/cli/__init__.py +0 -0
- segger/cli/config.yaml +42 -0
- segger/cli/debug.py +49 -0
- segger/cli/export.py +433 -0
- segger/cli/main.py +13 -0
- segger/cli/registry.py +562 -0
- segger/cli/segment.py +569 -0
- segger/data/__init__.py +2 -0
- segger/data/csr.py +179 -0
- segger/data/data_module.py +503 -0
- segger/data/partition/__init__.py +2 -0
- segger/data/partition/dataset.py +618 -0
- segger/data/partition/sampler.py +408 -0
- segger/data/tile_dataset.py +402 -0
- segger/data/tiling.py +324 -0
- segger/data/utils/__init__.py +3 -0
- segger/data/utils/anndata.py +316 -0
- segger/data/utils/heterodata.py +195 -0
- segger/data/utils/neighbors.py +238 -0
- segger/data/utils/threshold.py +13 -0
- segger/data/writer.py +268 -0
- segger/debug/prediction.py +95 -0
- segger/debug/segmentation.py +34 -0
- segger/export/__init__.py +8 -0
- segger/export/anndata_writer.py +54 -0
- segger/export/boundary.py +233 -0
- segger/geometry/__init__.py +8 -0
- segger/geometry/morphology.py +43 -0
- segger/geometry/quadtree.py +98 -0
- segger/geometry/query.py +145 -0
- segger/io/__init__.py +7 -0
- segger/io/cosmx.py +170 -0
- segger/io/fields.py +169 -0
- segger/io/preprocessor.py +1420 -0
- segger/io/utils.py +159 -0
- segger/metrics/reference_contamination.py +0 -0
- segger/models/__init__.py +1 -0
- segger/models/_triplet_loss_dev.py +362 -0
- segger/models/ist_encoder.py +337 -0
- segger/models/lightning_model.py +300 -0
- segger/models/torch_scatter_plugin.py +48 -0
- segger/models/triplet_loss.py +205 -0
- segger/utils.py +44 -0
- segger/validation/contamination.py +408 -0
- segger-1.0.0rc1.dist-info/METADATA +150 -0
- segger-1.0.0rc1.dist-info/RECORD +51 -0
- segger-1.0.0rc1.dist-info/WHEEL +4 -0
- segger-1.0.0rc1.dist-info/entry_points.txt +2 -0
segger/__init__.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
import warnings
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
# torch_geometric uses torch.jit. issue here: https://github.com/pyg-team/pytorch_geometric/issues/10697
|
|
7
|
+
warnings.filterwarnings("ignore", message=".*torch.jit.script.*is deprecated", category=FutureWarning)
|
|
8
|
+
|
|
9
|
+
# ignore lightning's pytree warning: https://github.com/Lightning-AI/pytorch-lightning/issues/21326
|
|
10
|
+
warnings.filterwarnings("ignore", message=".*isinstance\\(treespec, LeafSpec\\).*is deprecated")
|
|
11
|
+
|
|
12
|
+
# ignore batch sampler warning. it returns every tile
|
|
13
|
+
warnings.filterwarnings("ignore", message=".*custom batch sampler `DynamicBatchSamplerPatch` for prediction.*")
|
|
14
|
+
|
|
15
|
+
# the lazily built model has uninitialized parameters, so lightning's parameter count is approximate
|
|
16
|
+
warnings.filterwarnings("ignore", message=".*total number of parameters detected may be inaccurate.*`UninitializedParameter`.*")
|
|
17
|
+
|
|
18
|
+
# small datasets only have a few training batches per epoch
|
|
19
|
+
warnings.filterwarnings("ignore", message=".*number of training batches.*is smaller than the logging interval.*")
|
|
20
|
+
|
|
21
|
+
import cupy as cp
|
|
22
|
+
import torch
|
|
23
|
+
import rmm
|
|
24
|
+
from rmm.allocators.cupy import rmm_cupy_allocator
|
|
25
|
+
from rmm.allocators.torch import rmm_torch_allocator
|
|
26
|
+
from rmm.statistics import enable_statistics, get_statistics
|
|
27
|
+
|
|
28
|
+
logger = logging.getLogger(__name__)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def configure_memory(force: bool = False) -> None:
|
|
32
|
+
"""Point CuPy/cuDF/cuspa and PyTorch at a single shared RMM pool.
|
|
33
|
+
|
|
34
|
+
Must run before any CUDA tensor is created. Run by default from the CLI.
|
|
35
|
+
Importing segger as a library should not change already set allocators.
|
|
36
|
+
|
|
37
|
+
Check if cp allocators are already set to RMM's pool, if not, set them.
|
|
38
|
+
|
|
39
|
+
Note: rmm.is_initialized() is True as soon as `rmm` is imported, thus
|
|
40
|
+
we can't use this to check if memory allocators are already initialised.
|
|
41
|
+
"""
|
|
42
|
+
if not force and os.environ.get("SEGGER_SKIP_ALLOCATOR_INIT") == "1":
|
|
43
|
+
logger.info("Allocators not configured: SEGGER_SKIP_ALLOCATOR_INIT is set.")
|
|
44
|
+
return
|
|
45
|
+
if not force and cp.cuda.get_allocator() is rmm_cupy_allocator:
|
|
46
|
+
logger.info("Allocators not configured: RMM pool already active.")
|
|
47
|
+
return
|
|
48
|
+
# Note: cuspa's CUDA kernels currently reject managed-memory pointers.
|
|
49
|
+
rmm.reinitialize(pool_allocator=True, managed_memory=True)
|
|
50
|
+
cp.cuda.set_allocator(rmm_cupy_allocator)
|
|
51
|
+
torch.cuda.memory.change_current_allocator(rmm_torch_allocator)
|
|
52
|
+
enable_statistics()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def free_mem_str() -> str:
|
|
56
|
+
stats = get_statistics()
|
|
57
|
+
if stats is None:
|
|
58
|
+
return "GPU: stats not enabled (call segger.configure_memory() first)"
|
|
59
|
+
return (
|
|
60
|
+
f"GPU: {stats.current_bytes / 1e9:.2f} GB "
|
|
61
|
+
f"(peak {stats.peak_bytes / 1e9:.2f} GB)"
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def print_free_mem():
|
|
66
|
+
print(free_mem_str())
|
|
67
|
+
|
|
68
|
+
from .api import segment
|
segger/__main__.py
ADDED
segger/api.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Python entry point for running segger from other tools (e.g. Sopa)."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import geopandas as gpd
|
|
6
|
+
import pandas as pd
|
|
7
|
+
import polars as pl
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def segment(
|
|
11
|
+
output_directory: Path,
|
|
12
|
+
input_directory: Path | None = None,
|
|
13
|
+
transcripts: pl.DataFrame | pd.DataFrame | str | Path | None = None,
|
|
14
|
+
boundaries: gpd.GeoDataFrame | str | Path | None = None,
|
|
15
|
+
**options,
|
|
16
|
+
) -> pl.DataFrame:
|
|
17
|
+
"""Run segger segmentation and return the per-transcript assignments.
|
|
18
|
+
|
|
19
|
+
Give `input_directory`, or `transcripts` together with `boundaries`
|
|
20
|
+
(standard input format, see `segger.io.fields.SopaTranscriptFields`).
|
|
21
|
+
`input_directory` can be omitted with in-memory inputs, or with
|
|
22
|
+
`platform="standard"` and `transcripts` and `boundaries` given as paths.
|
|
23
|
+
Unlike the CLI, this does not configure GPU memory allocators; call
|
|
24
|
+
`segger.configure_memory()` first if needed.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
output_directory : Path
|
|
29
|
+
Directory to write outputs to. Created if missing.
|
|
30
|
+
input_directory : Path or None
|
|
31
|
+
IST data directory, as for `segger segment -i`. Not needed for the
|
|
32
|
+
cases above.
|
|
33
|
+
transcripts : pl.DataFrame or pd.DataFrame or str or Path or None
|
|
34
|
+
In-memory transcripts with columns `x`, `y`, `feature_name`,
|
|
35
|
+
`cell_id` (0 = unassigned) and optionally `cell_compartment`
|
|
36
|
+
or the path to a custom transcripts file. Requires `boundaries` of
|
|
37
|
+
the same kind, and `platform` if given as paths.
|
|
38
|
+
boundaries : gpd.GeoDataFrame or str or Path or None
|
|
39
|
+
In-memory boundaries with columns `cell_id`, `boundary_type`
|
|
40
|
+
('cell' or 'nucleus') and `geometry`, or the path to a custom
|
|
41
|
+
boundaries file, used as the cell boundaries of the prediction
|
|
42
|
+
graph (sets `prediction_mode="cell"`).
|
|
43
|
+
**options
|
|
44
|
+
Any `segger segment` option, using its Python name
|
|
45
|
+
(e.g. `n_epochs=20`, `segmentation_mode="cell"`).
|
|
46
|
+
|
|
47
|
+
Returns
|
|
48
|
+
-------
|
|
49
|
+
pl.DataFrame
|
|
50
|
+
Contents of `segger_segmentation.parquet`: one row per transcript,
|
|
51
|
+
joined to the input by `row_index`.
|
|
52
|
+
"""
|
|
53
|
+
from .cli.segment import segment as _segment
|
|
54
|
+
|
|
55
|
+
output_directory = Path(output_directory)
|
|
56
|
+
output_directory.mkdir(parents=True, exist_ok=True)
|
|
57
|
+
_segment(
|
|
58
|
+
input_directory=input_directory,
|
|
59
|
+
output_directory=output_directory,
|
|
60
|
+
transcripts=transcripts,
|
|
61
|
+
boundaries=boundaries,
|
|
62
|
+
**options,
|
|
63
|
+
)
|
|
64
|
+
return pl.read_parquet(output_directory / "segger_segmentation.parquet")
|
segger/cli/__init__.py
ADDED
|
File without changes
|
segger/cli/config.yaml
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# I/O
|
|
2
|
+
data_directory_inputs: ~
|
|
3
|
+
data_directory_outputs: ~
|
|
4
|
+
# Cell representation parameters
|
|
5
|
+
cells_min_counts: 10.
|
|
6
|
+
cells_cluster_n_neighbors: 10.
|
|
7
|
+
cells_cluster_resolution: 3.
|
|
8
|
+
cells_representation: 'pca' # BENCHMARKING: other option is 'morphology'
|
|
9
|
+
# Gene representation parameters
|
|
10
|
+
genes_clusters_n_neighbors: 5.
|
|
11
|
+
genes_clusters_resolution: 3.
|
|
12
|
+
# Transcript-transcript graph parameters
|
|
13
|
+
transcripts_max_k: 5.
|
|
14
|
+
transcripts_max_dist: 5.
|
|
15
|
+
# Segmentation (prediction) graph parameters
|
|
16
|
+
prediction_mode: 'buffer_cell' # Other options are 'buffer_nucleus' or 'uniform'
|
|
17
|
+
prediction_buffer_dist: 3.
|
|
18
|
+
prediction_max_k: ~
|
|
19
|
+
prediction_max_dist: ~
|
|
20
|
+
# Tiling parameters
|
|
21
|
+
tiling_margin_training: 10.
|
|
22
|
+
tiling_margin_prediction: 20.
|
|
23
|
+
max_nodes_per_tile: 50000
|
|
24
|
+
max_edges_per_batch: 1000000
|
|
25
|
+
# Model parameters
|
|
26
|
+
n_mid_layers: 2
|
|
27
|
+
n_heads: 2
|
|
28
|
+
in_channels: 128 # This is also the embedding dimension for cells/genes
|
|
29
|
+
hidden_channels: 64
|
|
30
|
+
out_channels: 64
|
|
31
|
+
learning_rate: 0.001
|
|
32
|
+
n_epochs: 10
|
|
33
|
+
# Loss parameters
|
|
34
|
+
segmentation_loss: 'triplet' # BENCHMARKING: Other option is 'bce'
|
|
35
|
+
transcripts_margin: 0.2
|
|
36
|
+
segmentation_margin: 0.4
|
|
37
|
+
transcripts_loss_weight_start: 1.
|
|
38
|
+
transcripts_loss_weight_end: 1.
|
|
39
|
+
cells_loss_weight_start: 1.
|
|
40
|
+
cells_loss_weight_start: 1.
|
|
41
|
+
segmentation_loss_weight_start: 0.
|
|
42
|
+
segmentation_loss_weight_end: 0.1
|
segger/cli/debug.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Debugging utilities for Segger.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing_extensions import Annotated
|
|
8
|
+
from cyclopts import App, Parameter
|
|
9
|
+
|
|
10
|
+
from ..debug.segmentation import run_segmentation_only
|
|
11
|
+
from ..debug.prediction import run_prediction_only
|
|
12
|
+
from ..utils import setup_logging
|
|
13
|
+
|
|
14
|
+
debug = App(name="debug", help="Utilities for debugging and testing individual components.")
|
|
15
|
+
|
|
16
|
+
@debug.command(name="segment-only")
|
|
17
|
+
def segment_only_cli(
|
|
18
|
+
path_adata: Annotated[Path, Parameter(
|
|
19
|
+
help="Path to input AnnData object.",
|
|
20
|
+
)],
|
|
21
|
+
path_predictions: Annotated[Path, Parameter(
|
|
22
|
+
help="Path to write predictions.",
|
|
23
|
+
)],
|
|
24
|
+
path_outputs: Annotated[Path, Parameter(
|
|
25
|
+
help="Path to write outputs.",
|
|
26
|
+
)],
|
|
27
|
+
):
|
|
28
|
+
"""Run prediction only."""
|
|
29
|
+
run_segmentation_only(
|
|
30
|
+
path_adata=path_adata,
|
|
31
|
+
path_predictions=path_predictions,
|
|
32
|
+
path_outputs=path_outputs,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
@debug.command(name="predict-only")
|
|
36
|
+
def predict_only_cli(
|
|
37
|
+
path_checkpoint: Annotated[Path, Parameter(
|
|
38
|
+
help="Path to trainer checkpoint object.",
|
|
39
|
+
)],
|
|
40
|
+
path_outputs: Annotated[Path, Parameter(
|
|
41
|
+
help="Path to write outputs.",
|
|
42
|
+
)],
|
|
43
|
+
):
|
|
44
|
+
"""Run prediction only."""
|
|
45
|
+
setup_logging(level="DEBUG", debug=True)
|
|
46
|
+
run_prediction_only(
|
|
47
|
+
path_checkpoint=path_checkpoint,
|
|
48
|
+
path_outputs=path_outputs,
|
|
49
|
+
)
|
segger/cli/export.py
ADDED
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
"""``segger export``: write a segger segmentation as scverse-compatible files.
|
|
2
|
+
|
|
3
|
+
Example usage:
|
|
4
|
+
# save anndata and boundaries
|
|
5
|
+
segger export anndata \
|
|
6
|
+
-s $PATH_OUTPUT/segger_segmentation.parquet \
|
|
7
|
+
-o $PATH_OUTPUT/adata_export
|
|
8
|
+
|
|
9
|
+
# save spatialdata - adds elements directly to an existing sdata object, no -o needed
|
|
10
|
+
segger export spatialdata \
|
|
11
|
+
-s $PATH_OUTPUT/segger_segmentation.parquet \
|
|
12
|
+
--sdata $PATH_INPUT/sdata.zarr
|
|
13
|
+
|
|
14
|
+
# save transcript-assignment CSV + boundaries GeoJSON for `xeniumranger import-segmentation`
|
|
15
|
+
segger export xeniumranger \
|
|
16
|
+
-s $PATH_OUTPUT/segger_segmentation.parquet \
|
|
17
|
+
-i $PATH_INPUT/xenium_bundle \
|
|
18
|
+
-o $PATH_OUTPUT/xenium_export
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import logging
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Annotated, Literal, Optional
|
|
26
|
+
from cyclopts import Parameter, Group, validators
|
|
27
|
+
|
|
28
|
+
import polars as pl
|
|
29
|
+
from ..io import StandardTranscriptFields
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
_group_io = Group(name="I/O", sort_key=0)
|
|
34
|
+
_group_opts = Group(name="Options", sort_key=1)
|
|
35
|
+
|
|
36
|
+
_Element = Literal["anndata", "transcripts-filtered", "boundaries", "spatialdata", "xeniumranger"]
|
|
37
|
+
_DEFAULT_ELEMENTS = ("anndata", "boundaries")
|
|
38
|
+
|
|
39
|
+
_Seg = Annotated[Path, Parameter(alias="-s", group=_group_io, validator=validators.Path(exists=True, dir_okay=False))]
|
|
40
|
+
_Source = Annotated[
|
|
41
|
+
Optional[Path],
|
|
42
|
+
Parameter(
|
|
43
|
+
alias="-i",
|
|
44
|
+
group=_group_io,
|
|
45
|
+
validator=validators.Path(exists=True, dir_okay=True),
|
|
46
|
+
help="Source transcripts directory. Needed for segger v0.2.0 (before x/y/feature_name were included in outputs), "
|
|
47
|
+
"and for 'xeniumranger' (raw transcript_id and the import command).",
|
|
48
|
+
),
|
|
49
|
+
]
|
|
50
|
+
_Out = Annotated[
|
|
51
|
+
Optional[Path],
|
|
52
|
+
Parameter(alias="-o", group=_group_io, help="Required unless the only element being exported is 'spatialdata'."),
|
|
53
|
+
]
|
|
54
|
+
_Sdata = Annotated[
|
|
55
|
+
Optional[Path],
|
|
56
|
+
Parameter(
|
|
57
|
+
alias="--sdata",
|
|
58
|
+
group=_group_io,
|
|
59
|
+
validator=validators.Path(exists=True, dir_okay=True),
|
|
60
|
+
help="Existing SpatialData Zarr store to add elements to in place (required for 'spatialdata').",
|
|
61
|
+
),
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
_IncludeAll = Annotated[
|
|
65
|
+
Optional[bool],
|
|
66
|
+
Parameter(
|
|
67
|
+
group=_group_opts,
|
|
68
|
+
help="Override each element's default transcript filtering. Unset (default): 'anndata' and 'boundaries' "
|
|
69
|
+
"use only filtered transcripts, 'transcripts' and 'spatialdata' use all of them. Pass this flag to force "
|
|
70
|
+
"every element to include all transcripts, or --no-include-all-transcripts to restrict every element to "
|
|
71
|
+
"filtered ones.",
|
|
72
|
+
),
|
|
73
|
+
]
|
|
74
|
+
_SdataTranscriptsName = Annotated[
|
|
75
|
+
str,
|
|
76
|
+
Parameter(
|
|
77
|
+
group=_group_opts,
|
|
78
|
+
help="Name of the existing points element in --sdata to append segger's columns to.",
|
|
79
|
+
),
|
|
80
|
+
]
|
|
81
|
+
_SdataCellBoundariesName = Annotated[
|
|
82
|
+
str,
|
|
83
|
+
Parameter(
|
|
84
|
+
group=_group_opts,
|
|
85
|
+
help="Name of the shapes element segger's cell boundaries are written to "
|
|
86
|
+
"(avoids colliding with an existing same-named element, e.g. from the raw Xenium sdata).",
|
|
87
|
+
),
|
|
88
|
+
]
|
|
89
|
+
_SdataTableName = Annotated[
|
|
90
|
+
str,
|
|
91
|
+
Parameter(
|
|
92
|
+
group=_group_opts,
|
|
93
|
+
help="Name of the table element segger's AnnData is written to.",
|
|
94
|
+
),
|
|
95
|
+
]
|
|
96
|
+
_MinCounts = Annotated[
|
|
97
|
+
int,
|
|
98
|
+
Parameter(
|
|
99
|
+
group=_group_opts,
|
|
100
|
+
help="Drop cells with fewer than this many assigned transcripts from the table/anndata "
|
|
101
|
+
"(must be >= 3 for spatialdata, since boundaries need >= 3 points).",
|
|
102
|
+
),
|
|
103
|
+
]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# -- LOAD TRANSCRIPTS --
|
|
107
|
+
def load_transcripts(
|
|
108
|
+
segmentation_path: Path,
|
|
109
|
+
source_path: Optional[Path] = None,
|
|
110
|
+
):
|
|
111
|
+
# read transcripts
|
|
112
|
+
tx = pl.read_parquet(segmentation_path)
|
|
113
|
+
|
|
114
|
+
# check if legacy result (before v0.2.0)
|
|
115
|
+
std = StandardTranscriptFields()
|
|
116
|
+
if not {std.x, std.y, std.feature, "filtered"} <= set(tx.columns):
|
|
117
|
+
tx = _legacy_join(tx, source_path=source_path, std=std)
|
|
118
|
+
|
|
119
|
+
# older outputs have no "converged" column; treat all features as converged
|
|
120
|
+
if "converged" not in tx.columns:
|
|
121
|
+
logger.warning("Segmentation has no 'converged' column (older output); assuming converged=True for all transcripts.")
|
|
122
|
+
tx = tx.with_columns(pl.lit(True).alias("converged"))
|
|
123
|
+
|
|
124
|
+
# subset
|
|
125
|
+
coord_cols = [pl.col(std.x).alias("x"), pl.col(std.y).alias("y")]
|
|
126
|
+
if std.z in tx.columns:
|
|
127
|
+
coord_cols.append(pl.col(std.z).alias("z"))
|
|
128
|
+
|
|
129
|
+
# select
|
|
130
|
+
tx = tx.select(
|
|
131
|
+
pl.col(std.row_index),
|
|
132
|
+
pl.col("segger_cell_id").cast(pl.String),
|
|
133
|
+
pl.col(std.feature).alias("feature_name"),
|
|
134
|
+
pl.col("segger_similarity"),
|
|
135
|
+
pl.col("similarity_threshold"),
|
|
136
|
+
pl.col("converged"),
|
|
137
|
+
pl.col("filtered"),
|
|
138
|
+
*coord_cols,
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
return tx
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _legacy_join(tx: pl.DataFrame, source_path: Optional[Path], std) -> pl.DataFrame:
|
|
145
|
+
"""Join x/y/feature_name onto a segmentation output written before they were included inline."""
|
|
146
|
+
if source_path is None:
|
|
147
|
+
raise ValueError("This segmentation output predates inline x/y/feature_name; pass -i/--source-path to join them from the source transcripts.")
|
|
148
|
+
|
|
149
|
+
from ..io import get_preprocessor
|
|
150
|
+
|
|
151
|
+
# load and merge
|
|
152
|
+
tx_all = get_preprocessor(source_path).transcripts
|
|
153
|
+
pred_cols = [c for c in (std.row_index, "segger_cell_id", "segger_similarity", "similarity_threshold") if c in tx.columns]
|
|
154
|
+
tx = tx.select(pred_cols).join(tx_all, on=std.row_index, how="left")
|
|
155
|
+
|
|
156
|
+
# add "filtered"
|
|
157
|
+
tx = tx.with_columns((
|
|
158
|
+
(pl.col("segger_cell_id").is_not_null()) & (pl.col("segger_similarity") >= pl.col("similarity_threshold"))
|
|
159
|
+
).alias("filtered"))
|
|
160
|
+
|
|
161
|
+
# if "converged" exists, also require this to be true
|
|
162
|
+
if "converged" in tx.columns:
|
|
163
|
+
tx = tx.with_columns(
|
|
164
|
+
(pl.col("filtered") & pl.col("converged")).alias("filtered")
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
return tx
|
|
168
|
+
|
|
169
|
+
# -- XENIUMRANGER IMPORT-SEGMENTATION SUPPORT --
|
|
170
|
+
def _build_xenium_transcript_csv(tx: "pl.DataFrame", source_path: Path, cell_labels: dict) -> "pl.DataFrame":
|
|
171
|
+
"""Columns for `xeniumranger import-segmentation`: transcript_id,cell,is_noise
|
|
172
|
+
|
|
173
|
+
`row_index` is the row position in the raw Xenium transcripts file, so the real
|
|
174
|
+
`transcript_id` is joined back from `source_path`. Takes `tx` (not `assigned`) since
|
|
175
|
+
`filtered` is only kept there. `cell_labels` maps segger cell IDs to the xeniumranger
|
|
176
|
+
labels of cells with a polygon. A transcript is assigned if it is filtered and its cell has
|
|
177
|
+
a label; all others get an empty `cell` and `is_noise=true`.
|
|
178
|
+
"""
|
|
179
|
+
from ..io.fields import XeniumTranscriptFields
|
|
180
|
+
|
|
181
|
+
std = StandardTranscriptFields()
|
|
182
|
+
ids = pl.read_parquet(source_path / XeniumTranscriptFields().filename, columns=["transcript_id"]).with_row_index(std.row_index)
|
|
183
|
+
labels = pl.DataFrame({"segger_cell_id": list(cell_labels), "cell": list(cell_labels.values())})
|
|
184
|
+
assigned = pl.col("filtered") & pl.col("cell").is_not_null()
|
|
185
|
+
return (
|
|
186
|
+
tx.join(ids, on=std.row_index, how="left")
|
|
187
|
+
.join(labels, on="segger_cell_id", how="left")
|
|
188
|
+
.select(
|
|
189
|
+
"transcript_id",
|
|
190
|
+
pl.when(assigned).then(pl.col("cell")).alias("cell"),
|
|
191
|
+
assigned.not_().alias("is_noise"),
|
|
192
|
+
)
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
# -- SPATIAL DATA SUPPORT --
|
|
197
|
+
def _check_sdata_elements(
|
|
198
|
+
sdata,
|
|
199
|
+
transcripts_element: str,
|
|
200
|
+
cell_boundaries_element: str,
|
|
201
|
+
table_element: str,
|
|
202
|
+
) -> None:
|
|
203
|
+
"""Fail fast if the transcripts element is missing, or any target element name already exists."""
|
|
204
|
+
if transcripts_element not in sdata.points:
|
|
205
|
+
raise KeyError(f"{transcripts_element!r} not found in sdata.points; pass --sdata-transcripts-name to point at the right one.")
|
|
206
|
+
if cell_boundaries_element in sdata.shapes:
|
|
207
|
+
raise FileExistsError(f"{cell_boundaries_element!r} already exists in sdata.shapes; pass --sdata-cell-boundaries-name to specify a different name.")
|
|
208
|
+
if table_element in sdata.tables:
|
|
209
|
+
raise FileExistsError(f"{table_element!r} already exists in sdata.tables; pass --sdata-table-name to specify a different name.")
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _merge_sdata_transcripts(sdata_tx: dd.DataFrame, tx: pl.DataFrame) -> dd.DataFrame:
|
|
214
|
+
"""Segger's per-transcript outputs, joined onto the existing (dask-backed) transcripts table. Stays lazy throughout."""
|
|
215
|
+
row_index = StandardTranscriptFields().row_index
|
|
216
|
+
|
|
217
|
+
# rename; these names are hardcoded. make sure to update this if any name should change going forward
|
|
218
|
+
map_columns = {
|
|
219
|
+
"segger_cell_id": "segger_cell_id",
|
|
220
|
+
"segger_similarity": "segger_similarity",
|
|
221
|
+
"similarity_threshold": "segger_similarity_threshold",
|
|
222
|
+
"converged": "segger_converged",
|
|
223
|
+
"filtered": "segger_filtered",
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
# rename
|
|
227
|
+
tx = tx.rename(map_columns).select(row_index, *map_columns.values()).to_pandas()
|
|
228
|
+
|
|
229
|
+
# sdata_tx has no row_index column; construct it
|
|
230
|
+
sdata_tx = sdata_tx.assign(**{row_index: 1})
|
|
231
|
+
sdata_tx[row_index] = sdata_tx[row_index].cumsum() - 1
|
|
232
|
+
|
|
233
|
+
# delete existing (overwrite is not exactly silent - export would stop if boundaries or table already exists)
|
|
234
|
+
sdata_tx = sdata_tx.drop(columns=[col for col in map_columns.values() if col in sdata_tx.columns])
|
|
235
|
+
|
|
236
|
+
# merge (tx is small enough to broadcast onto every partition; no shuffle needed)
|
|
237
|
+
sdata_tx = sdata_tx.merge(tx, on=row_index, how="left")
|
|
238
|
+
sdata_tx["segger_seen"] = sdata_tx["segger_filtered"].notnull()
|
|
239
|
+
|
|
240
|
+
# row_index is only a join key; drop it from the output
|
|
241
|
+
sdata_tx = sdata_tx.drop(columns=[row_index])
|
|
242
|
+
|
|
243
|
+
return sdata_tx
|
|
244
|
+
|
|
245
|
+
def _write_to_sdata(
|
|
246
|
+
sdata,
|
|
247
|
+
tx: pl.DataFrame,
|
|
248
|
+
gdf: gpd.GeoDataFrame,
|
|
249
|
+
adata: AnnData,
|
|
250
|
+
transcripts_element: str = "transcripts",
|
|
251
|
+
cell_boundaries_element: str = "cell_boundaries_segger",
|
|
252
|
+
table_element: str = "table_segger",
|
|
253
|
+
) -> None:
|
|
254
|
+
"""Append segger's columns to the existing transcripts element, and add its boundaries/table elements to the given SpatialData store."""
|
|
255
|
+
from spatialdata.models import PointsModel, ShapesModel, TableModel
|
|
256
|
+
from spatialdata.transformations import get_transformation
|
|
257
|
+
|
|
258
|
+
# get current transcripts
|
|
259
|
+
base_transcripts = sdata.points[transcripts_element]
|
|
260
|
+
attrs = base_transcripts.attrs.get("spatialdata_attrs", {})
|
|
261
|
+
transformations = get_transformation(base_transcripts, get_all=True)
|
|
262
|
+
|
|
263
|
+
# new element fully in memory, required for overwriting
|
|
264
|
+
tx = _merge_sdata_transcripts(base_transcripts, tx).compute().reset_index(drop=True)
|
|
265
|
+
|
|
266
|
+
coordinates = {"x": "x", "y": "y"}
|
|
267
|
+
if "z" in tx.columns:
|
|
268
|
+
coordinates["z"] = "z"
|
|
269
|
+
|
|
270
|
+
# build the new (in-memory, unbacked) elements
|
|
271
|
+
new_transcripts = PointsModel.parse(
|
|
272
|
+
tx,
|
|
273
|
+
coordinates=coordinates,
|
|
274
|
+
feature_key=attrs.get("feature_key"),
|
|
275
|
+
instance_key=attrs.get("instance_key"),
|
|
276
|
+
transformations=transformations,
|
|
277
|
+
)
|
|
278
|
+
new_boundaries = ShapesModel.parse(gdf, transformations=transformations)
|
|
279
|
+
new_table = TableModel.parse(adata)
|
|
280
|
+
|
|
281
|
+
# SpatialData can't overwrite (#520). Do 1) backup, 2) drop in-memory handles, 3) delete original, 4) write new elements, 5) drop backup
|
|
282
|
+
# 1 backup
|
|
283
|
+
backup_element = f"{transcripts_element}_backup"
|
|
284
|
+
sdata[backup_element] = base_transcripts
|
|
285
|
+
sdata.write_element([backup_element])
|
|
286
|
+
|
|
287
|
+
# 2 detach in-memory
|
|
288
|
+
del sdata.points[backup_element]
|
|
289
|
+
sdata[transcripts_element] = new_transcripts
|
|
290
|
+
sdata[cell_boundaries_element] = new_boundaries
|
|
291
|
+
sdata[table_element] = new_table
|
|
292
|
+
|
|
293
|
+
# 3, 4 delete and write
|
|
294
|
+
print(f"Writing {transcripts_element}, {cell_boundaries_element}, {table_element} to {sdata.path}...")
|
|
295
|
+
try:
|
|
296
|
+
sdata.delete_element_from_disk(transcripts_element)
|
|
297
|
+
sdata.write_element([transcripts_element, cell_boundaries_element, table_element])
|
|
298
|
+
except Exception:
|
|
299
|
+
print(
|
|
300
|
+
f"Write failed. The original {transcripts_element!r} is preserved on disk as "
|
|
301
|
+
f"{backup_element!r}; restore it from there."
|
|
302
|
+
)
|
|
303
|
+
raise
|
|
304
|
+
else:
|
|
305
|
+
# 5 drop backup
|
|
306
|
+
sdata.delete_element_from_disk(backup_element)
|
|
307
|
+
if sdata.has_consolidated_metadata():
|
|
308
|
+
sdata.write_consolidated_metadata()
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def export(
|
|
312
|
+
*elements: Annotated[_Element, Parameter(help="Elements to write (default: anndata boundaries).")],
|
|
313
|
+
segmentation_path: _Seg,
|
|
314
|
+
output_directory: _Out = None,
|
|
315
|
+
source_path: _Source = None,
|
|
316
|
+
sdata_path: _Sdata = None,
|
|
317
|
+
method: Annotated[
|
|
318
|
+
Literal["delaunay", "convex_hull"],
|
|
319
|
+
Parameter(group=_group_opts, help="Cell-polygon method for boundaries."),
|
|
320
|
+
] = "delaunay",
|
|
321
|
+
chaikin_iterations: Annotated[
|
|
322
|
+
int, Parameter(group=_group_opts, help="Chaikin corner-cutting iterations to round boundaries (0 disables).")
|
|
323
|
+
] = 0,
|
|
324
|
+
include_all_transcripts: _IncludeAll = None,
|
|
325
|
+
sdata_transcripts_name: _SdataTranscriptsName = "transcripts",
|
|
326
|
+
sdata_cell_boundaries_name: _SdataCellBoundariesName = "cell_boundaries_segger",
|
|
327
|
+
sdata_table_name: _SdataTableName = "table_segger",
|
|
328
|
+
min_counts: _MinCounts = 10,
|
|
329
|
+
):
|
|
330
|
+
"""Write a segger segmentation as scverse SpatialData elements (anndata, transcripts-filtered, boundaries, spatialdata, xeniumranger)."""
|
|
331
|
+
selected = elements or _DEFAULT_ELEMENTS
|
|
332
|
+
|
|
333
|
+
sdata = None
|
|
334
|
+
if "spatialdata" in selected:
|
|
335
|
+
if min_counts < 3:
|
|
336
|
+
# cell boundaries need >= 3 transcripts to form a polygon; a lower table cutoff leaves cells without one
|
|
337
|
+
raise ValueError("--min-counts must be >= 3 for spatialdata: boundaries need >= 3 transcripts to form a polygon.")
|
|
338
|
+
if sdata_path is None:
|
|
339
|
+
raise ValueError("--sdata is required when exporting 'spatialdata'.")
|
|
340
|
+
import spatialdata as sd
|
|
341
|
+
sdata = sd.read_zarr(sdata_path)
|
|
342
|
+
_check_sdata_elements(sdata, sdata_transcripts_name, sdata_cell_boundaries_name, sdata_table_name)
|
|
343
|
+
|
|
344
|
+
if set(selected) - {"spatialdata"} and output_directory is None:
|
|
345
|
+
raise ValueError("-o/--output-directory is required unless the only element being exported is 'spatialdata'.")
|
|
346
|
+
|
|
347
|
+
if "xeniumranger" in selected and source_path is None:
|
|
348
|
+
raise ValueError("-i/--source-path is required for 'xeniumranger': it provides the raw Xenium transcript_id and the --xenium-bundle path.")
|
|
349
|
+
|
|
350
|
+
# load tx
|
|
351
|
+
tx = load_transcripts(segmentation_path, source_path)
|
|
352
|
+
tx_filtered = tx.filter(pl.col("filtered"))
|
|
353
|
+
|
|
354
|
+
# None (default): tx stays unfiltered, tx_filtered stays filtered; explicit True/False overrides both
|
|
355
|
+
if include_all_transcripts is False:
|
|
356
|
+
tx = tx.filter(pl.col("filtered"))
|
|
357
|
+
if include_all_transcripts:
|
|
358
|
+
tx_filtered = tx
|
|
359
|
+
|
|
360
|
+
if output_directory is not None:
|
|
361
|
+
output_directory.mkdir(parents=True, exist_ok=True)
|
|
362
|
+
|
|
363
|
+
# compute outputs
|
|
364
|
+
gdf = None
|
|
365
|
+
if "boundaries" in selected or "spatialdata" in selected or "xeniumranger" in selected:
|
|
366
|
+
from ..export import generate_boundaries
|
|
367
|
+
gdf = generate_boundaries(tx_filtered, cell_id="segger_cell_id", method=method, smoothing=chaikin_iterations)
|
|
368
|
+
|
|
369
|
+
adata = None
|
|
370
|
+
if "anndata" in selected or "spatialdata" in selected:
|
|
371
|
+
from ..export import build_anndata
|
|
372
|
+
adata = build_anndata(
|
|
373
|
+
tx_filtered,
|
|
374
|
+
cell_id="segger_cell_id",
|
|
375
|
+
z="z",
|
|
376
|
+
area=gdf.geometry.area if gdf is not None else None,
|
|
377
|
+
region=sdata_cell_boundaries_name,
|
|
378
|
+
min_counts=min_counts,
|
|
379
|
+
)
|
|
380
|
+
# keep only cells that produced a boundary polygon, so the table annotates only real shapes
|
|
381
|
+
if gdf is not None:
|
|
382
|
+
adata = adata[adata.obs_names.isin(gdf.index)].copy()
|
|
383
|
+
|
|
384
|
+
# save outputs
|
|
385
|
+
if "transcripts-filtered" in selected:
|
|
386
|
+
tx.write_parquet(output_directory / "transcripts.parquet")
|
|
387
|
+
print(f"Wrote {tx.height} assigned transcripts: {output_directory / 'transcripts.parquet'}")
|
|
388
|
+
|
|
389
|
+
if "boundaries" in selected:
|
|
390
|
+
gdf.to_parquet(output_directory / "cell_boundaries.parquet")
|
|
391
|
+
print(f"Wrote {len(gdf)} {method} cell boundaries: {output_directory / 'cell_boundaries.parquet'}")
|
|
392
|
+
|
|
393
|
+
if "anndata" in selected:
|
|
394
|
+
adata.write_h5ad(output_directory / "adata.h5ad")
|
|
395
|
+
print(f"Wrote AnnData ({adata.n_obs} cells x {adata.n_vars} genes): {output_directory / 'adata.h5ad'}")
|
|
396
|
+
|
|
397
|
+
if "spatialdata" in selected:
|
|
398
|
+
_write_to_sdata(
|
|
399
|
+
sdata,
|
|
400
|
+
tx,
|
|
401
|
+
gdf,
|
|
402
|
+
adata,
|
|
403
|
+
transcripts_element=sdata_transcripts_name,
|
|
404
|
+
cell_boundaries_element=sdata_cell_boundaries_name,
|
|
405
|
+
table_element=sdata_table_name,
|
|
406
|
+
)
|
|
407
|
+
|
|
408
|
+
if "xeniumranger" in selected:
|
|
409
|
+
# xeniumranger needs "<prefix>-<n>" cell IDs with one shared prefix and a unique n,
|
|
410
|
+
# for cells that have both a polygon and assigned transcripts
|
|
411
|
+
assigned_cells = set(tx.filter(pl.col("filtered"))["segger_cell_id"].drop_nulls())
|
|
412
|
+
gdf_xr = gdf[gdf.index.isin(assigned_cells)]
|
|
413
|
+
cell_labels = {cell_id: f"segger-{i}" for i, cell_id in enumerate(gdf_xr.index, start=1)}
|
|
414
|
+
xenium_tx = _build_xenium_transcript_csv(tx, source_path, cell_labels)
|
|
415
|
+
transcript_assignment_path = output_directory / "xenium_transcript_assignment.csv"
|
|
416
|
+
xenium_tx.write_csv(transcript_assignment_path)
|
|
417
|
+
print(f"Wrote {xenium_tx.height} transcripts for xeniumranger: {transcript_assignment_path}")
|
|
418
|
+
|
|
419
|
+
viz_polygons_path = output_directory / "xenium_cell_boundaries.geojson"
|
|
420
|
+
# features carry "id" = cell label, as xeniumranger expects; `cell_id` keeps the segger ID
|
|
421
|
+
viz_polygons_path.write_text(gdf_xr.rename(index=cell_labels).to_json())
|
|
422
|
+
print(f"Wrote {len(gdf_xr)} cell boundaries for xeniumranger: {viz_polygons_path}")
|
|
423
|
+
|
|
424
|
+
print(
|
|
425
|
+
"\nRun this to import the segmentation into a Xenium bundle "
|
|
426
|
+
f"(writes its output under {output_directory}/xeniumranger_import):\n"
|
|
427
|
+
f"\tcd {output_directory} && xeniumranger import-segmentation \\\n"
|
|
428
|
+
f"\t\t--id=xeniumranger_import \\\n"
|
|
429
|
+
f"\t\t--xenium-bundle={source_path} \\\n"
|
|
430
|
+
f"\t\t--transcript-assignment={transcript_assignment_path} \\\n"
|
|
431
|
+
f"\t\t--viz-polygons={viz_polygons_path} \\\n"
|
|
432
|
+
f"\t\t--units=microns"
|
|
433
|
+
)
|
segger/cli/main.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from cyclopts import App
|
|
2
|
+
from segger import configure_memory
|
|
3
|
+
from .segment import segment
|
|
4
|
+
from .debug import debug
|
|
5
|
+
from .export import export
|
|
6
|
+
|
|
7
|
+
configure_memory()
|
|
8
|
+
|
|
9
|
+
app = App(name="Segger")
|
|
10
|
+
|
|
11
|
+
app.command(segment)
|
|
12
|
+
app.command(debug)
|
|
13
|
+
app.command(export)
|