patchworks 3.0.0__tar.gz → 3.1.0__tar.gz

{patchworks-3.0.0 → patchworks-3.1.0}/.github/min-versions.txt

@@ -4,3 +4,4 @@ dask==2024.10.0
 numpy==1.26.0
 zarr==3.1.3
 scipy==1.11.1
+pandas==2.0.3

{patchworks-3.0.0 → patchworks-3.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: patchworks
-Version: 3.0.0
+Version: 3.1.0
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues
@@ -62,6 +62,7 @@ Provides-Extra: dev
 Requires-Dist: jsonschema>=4.18; extra == 'dev'
 Requires-Dist: mypy>=1.10; extra == 'dev'
 Requires-Dist: openpyxl; extra == 'dev'
+Requires-Dist: pandas>=2.0; extra == 'dev'
 Requires-Dist: psutil; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
 Requires-Dist: pytest-cov; extra == 'dev'
@@ -92,8 +93,12 @@ Requires-Dist: numpy<2.5; extra == 'napari'
 Requires-Dist: pyqt6<6.10; extra == 'napari'
 Provides-Extra: remote
 Requires-Dist: fsspec[gcs,http,s3]; extra == 'remote'
+Provides-Extra: review
+Requires-Dist: openpyxl; extra == 'review'
+Requires-Dist: pandas>=2.0; extra == 'review'
 Provides-Extra: workflow
 Requires-Dist: openpyxl; extra == 'workflow'
+Requires-Dist: pandas>=2.0; extra == 'workflow'
 Requires-Dist: snakemake-executor-plugin-slurm; extra == 'workflow'
 Requires-Dist: snakemake>=8; extra == 'workflow'
 Description-Content-Type: text/markdown
@@ -284,6 +289,16 @@ Pyramids downsample **X/Y only** (Z kept full-res) and are built level-by-level
 from disk, so terabyte volumes convert in bounded RAM. See the
 [OME-ZARR & napari guide](https://imcf.one/patchworks/guide/ome_zarr_napari/).
 
+### Check and correct the result
+
+Every label image comes with an object table (size, position, which cell
+each nucleus or cilium is in). `patchworks review scan.zarr` opens napari on
+the objects most likely to be wrong — a cilium in no cell, a cell with two
+nuclei, a nucleus cut in two at a tile seam — one at a time; one key
+accepts, rejects, reassigns or joins. The corrections flow into the tables
+and workbooks. See
+[Reviewing and correcting results](https://imcf.one/patchworks/guide/review/).
+
 ---
 
 ## From the command line

{patchworks-3.0.0 → patchworks-3.1.0}/README.md

@@ -184,6 +184,16 @@ Pyramids downsample **X/Y only** (Z kept full-res) and are built level-by-level
 from disk, so terabyte volumes convert in bounded RAM. See the
 [OME-ZARR & napari guide](https://imcf.one/patchworks/guide/ome_zarr_napari/).
 
+### Check and correct the result
+
+Every label image comes with an object table (size, position, which cell
+each nucleus or cilium is in). `patchworks review scan.zarr` opens napari on
+the objects most likely to be wrong — a cilium in no cell, a cell with two
+nuclei, a nucleus cut in two at a tile seam — one at a time; one key
+accepts, rejects, reassigns or joins. The corrections flow into the tables
+and workbooks. See
+[Reviewing and correcting results](https://imcf.one/patchworks/guide/review/).
+
 ---
 
 ## From the command line

patchworks-3.1.0/docs/api/review.md

@@ -0,0 +1,19 @@
+# Object tables and review
+
+See [Reviewing and correcting results](../guide/review.md) for the workflow.
+
+::: patchworks.measure_objects
+
+::: patchworks.compute_table
+
+::: patchworks.relate_tables
+
+::: patchworks._tables.nearest_parents
+
+::: patchworks._tables.shape_columns
+
+::: patchworks.read_table
+
+::: patchworks.Review
+
+::: patchworks.plugins.review.review_in_napari

{patchworks-3.0.0 → patchworks-3.1.0}/docs/guide/cli.md

@@ -19,8 +19,23 @@ patchworks seams scan.zarr/labels/labels --tile-shape 16,1024,1024
 
 # Look at the result (needs patchworks[napari])
 patchworks view scan.zarr
+
+# One row per object; which cell each nucleus is in
+patchworks tables scan.zarr --relate nuclei:cells
+
+# Look at the likely mistakes one by one and fix them (napari)
+patchworks review scan.zarr --expect cells:nuclei=1
 ```
 
+`patchworks review` without a window: `--summary` (counts and error
+estimate), `--export DIR --format csv|xlsx|parquet` (corrected tables),
+`--workbooks DIR` (relation workbooks), `--write-labels NAME` (a label image
+with the corrections applied). `--position CHILD:PARENT=APICAL` classifies
+children as apical/basal/lateral/central (APICAL: `+z`, or a label image to
+point away from, such as the nuclei); `patchworks tables --max-distance UM`
+gives a child touching no parent the nearest one. See
+[Reviewing and correcting results](review.md).
+
 ## Segmentation methods
 
 | `--method` | What it does | Main flags |

{patchworks-3.0.0 → patchworks-3.1.0}/docs/guide/label_relations.md

@@ -53,6 +53,14 @@ with open("nuclei_to_cell.csv", "w", newline="") as f:
         w.writerow([nucleus_id, m["match"], m["overlap_voxels"], m["overlap_fraction"]])
 ```
 
+To keep the result with the labels instead, as columns of the object
+tables (the corrections made in
+[`patchworks review`](review.md) then apply to it):
+
+```bash
+patchworks tables results/image.zarr --relate nuclei_labels:cyto_labels
+```
+
 On the cluster, producing the two label stores in the first place is a
 matter of running the workflow twice against the same `work_dir` — see
 [Running two segmentations](snakemake.md#running-two-segmentations-eg-nuclei-cytoplasm).

{patchworks-3.0.0 → patchworks-3.1.0}/docs/guide/measurements.md

@@ -3,6 +3,26 @@
 `skimage.measure.regionprops` needs the full labelled + intensity array in
 RAM — fine for one tile, not for a hundred-thousand-object OME-ZARR.
 
+## Already measured: the object tables
+
+The workflow measures every object once, right after the merge: size
+(`area_voxels`, `area_um3`), centroid, bounding box and spread (second
+moments, from which the corrected view derives `length_um`, `elongation`
+and the main axis), plus mean/std intensity for the channels listed in
+`table_channels`. The table is stored
+with the labels (`image.zarr/labels/<name>/table`):
+
+```python
+from patchworks import read_table
+
+cells = read_table("results/image.zarr/labels/cyto_labels")  # pandas
+```
+
+For a store without one, `patchworks tables results/image.zarr --channels 0,1`
+adds them. With review corrections applied, and as csv files that
+napari-chunked-regionprops loads directly: `patchworks review
+results/image.zarr --export tables/` (see [Reviewing](review.md)).
+
 ## Interactively, in napari
 
 [napari-chunked-regionprops](https://github.com/imcf/napari-chunked-regionprops)

patchworks-3.1.0/docs/guide/review.md

@@ -0,0 +1,261 @@
+# Reviewing and correcting results
+
+A segmentation of a whole tissue has tens of thousands of objects, and some
+of them are wrong. Scrolling through the volume hoping to spot them doesn't
+work. `patchworks review` shows you the objects **most likely to be wrong**,
+one at a time, and lets you fix each one with a single key. Your
+corrections go straight into the result tables and workbooks, and no voxel
+is rewritten unless you ask.
+
+```bash
+pip install "patchworks[napari]"      # on the machine you review on
+patchworks review results/image.zarr
+```
+
+![The review panel](../assets/review_panel.png)
+
+*A cilium only 25% inside its cell. The view jumps to it, shows it with its
+cell outlined, and hides everything else. Press G if it is right, H to give
+it the right cell, W if it isn't a cilium at all.*
+
+## What gets flagged
+
+Each object in the queue shows why it is there:
+
+| Flag | Means | Typical cause |
+| --- | --- | --- |
+| *not inside any cell* | the object overlaps no parent object | debris, a missed cell, a cilium on the lumen side |
+| *only 30% inside cell #88* | less than `min_overlap` of it is inside its parent | wrong cell picked at a boundary, or a merge of two objects |
+| *0 nuclei (expected 1)* | a parent holds an unexpected number of children | a cell split in two, two cells merged, a missed nucleus |
+| *meets #412 exactly at a tile seam (z)* | two objects touch face to face on a tile boundary | one object the tiling cut in two, or two neighbours; press J to join them |
+| *unusually large (6.2x the median)* | the size is far from the rest (robust z-score > 3.5) | two objects merged, or a fragment |
+| *outside cell #12, 0.8 µm away* | touches no parent, but one is within `max_distance_um` | a cilium beside its cell (worth a look, lower priority) |
+| *position unclear: its cell has no nucleus to orient it* | a position rule needs the nucleus, and the cell has none | a missed nucleus, or a cell cut by the image edge |
+
+The most suspicious objects come first. "Children" and "parents" come from
+the relations of a multi run (e.g. `cilia_labels → cyto_labels`); the size
+and seam flags apply to every label image.
+
+**Expected counts** are yours to state, since only you know that a cell has
+one nucleus and zero to two cilia. Put them in `multi.yaml`, and the
+workflow checks them before it starts and stores them with the results:
+
+```yaml
+review:
+  expect:
+    cyto_labels:
+      nuclei_labels: 1         # exactly one
+      cilia_labels: [0, 2]     # zero to two
+  min_overlap: 0.5             # or per pair: {cilia_labels: {cyto_labels: 0.3}}
+```
+
+Or pass them when you open the review:
+`patchworks review image.zarr --expect cyto_labels:nuclei_labels=1 cyto_labels:cilia_labels=0-2`.
+
+## Deciding
+
+| Key | Decision | Effect on the results |
+| --- | --- | --- |
+| **G** | correct as it is | kept; marked `ok` |
+| **W** | wrong: not a real object | dropped from every table and count |
+| **H**, then click | belongs to another parent: click the right one (the background means "none") | its parent id changes; both parents' counts follow |
+| **J** | join with the suggested object (a seam split) | the two become one object: sizes added, centroid and intensities weighted |
+| **Shift+J**, then click | join with the object you click | same |
+| **N** / **Shift+N** | skip / go back | nothing |
+| **U** | undo the decision about this object | back to unreviewed |
+
+Every decision is saved immediately, in the store, next to the object
+table. Close napari whenever you like: the next `patchworks review`
+continues where you stopped, and the queue leaves out what has been
+decided. Corrections chain as you would expect. For example, joining two
+halves of a cell moves the cilia of both halves to the joined cell.
+
+## Seeing what belongs to what
+
+Outside the queue, you can look at any object:
+
+- **Click any object** in the image: the panel switches to it and shows it
+  with its parent (outlined) and its children. Click a cell to see its
+  nuclei and cilia, a cilium to see its cell. A click reads the image at
+  full resolution, and a thin object is hit even a couple of voxels off,
+  so cilia are easy to pick.
+- **Go to #**: type an object's id and press Enter.
+- **Hover** over any object: napari's status bar shows its parent, its
+  number of children, its position and its review status.
+
+Options in the panel change the view:
+
+- **Show only this object and its relatives** (on by default) hides
+  everything else. Untick it to see the neighbourhood.
+- **Colour these objects by their parent** adds a layer where every child
+  has its parent's colour: all cilia of one cell share a colour, so an
+  assignment error stands out as a different colour.
+- **Colour these objects by position** colours each cilium by its class:
+  apical, basal, lateral or central.
+- **Side view** shows z against x, with z up, through the object. Apical
+  and basal are then seen at a glance.
+
+Orange rings mark the flagged objects still open. They stay visible in 3D,
+where napari's coarse 3D level hides small objects.
+
+## Where a cilium sits: apical, basal, lateral, central
+
+![A cilium classified as apical, in side view](../assets/review_position.png)
+
+*Side view, coloured by position: an apical cilium (green) standing out of
+the top of its cell, the nucleus at the bottom; next door, a basal cilium
+(blue).*
+
+Each cilium is classified by the cell surface its **base** is nearest to:
+
+| Class | The base is nearest to |
+| --- | --- |
+| `apical` | the top of the cell |
+| `basal` | the bottom of the cell |
+| `lateral` | the side wall |
+| `central` | none of them: deeper than half-way from every surface |
+
+"Top" needs an apical direction per cell. It can be a fixed direction:
+`+z` if apical is up the stack, as for a monolayer imaged from below. Or it
+can point **away from the nucleus**, for epithelia whose nuclei sit
+basally; then each cell gets its own axis, whatever its tilt. The base of
+a cilium is its end nearer the cell's centre, since a cilium grows out
+from its base.
+
+```yaml
+# multi.yaml
+review:
+  position:
+    cilia_labels:
+      parent: cyto_labels
+      apical: nuclei_labels     # away from the nucleus; or "+z", "-z", ...
+      central_depth: 0.5        # optional
+```
+
+Or when opening the review:
+`patchworks review image.zarr --position cilia_labels:cyto_labels=nuclei_labels`.
+
+The corrected tables get the class (`position`), where the base sits
+(`position_axial`: -1 basal … +1 apical; `position_radial`: 0 on the axis …
+1 at the side) and the cilium's angle to the apical axis
+(`angle_to_axis_deg`: 0 along it, 90 across it). Each cell gets its counts
+per class (`n_cilia_labels_apical`, …), as does the relation workbook.
+
+The cell's shape comes from its moments, i.e. an equivalent cylinder, so
+this is a classification, not a surface distance. It is reliable for
+columnar and cuboidal cells, and less so for very irregular ones. Check it
+the usual way: **Colour these objects by position** plus **Side view**. If
+one is wrong, the **Position is:** buttons correct it. A correction counts
+as a classification fix, not a segmentation error, so it does not enter
+the error rate. A cell without a nucleus cannot be oriented; its cilia are
+`unknown` and flagged.
+
+## Cilia next to their cell, not on it
+
+A cilium can lie against its cell without overlapping it, and would then
+count as belonging to no cell. With `max_distance_um` on a relation, such
+an object gets the **nearest** cell within that distance. The distance is
+exact, in µm, with anisotropic voxels taken into account, and is recorded
+in `cyto_labels_distance_um`. These objects are flagged with a lower
+priority ("outside cell #12, 0.8 µm away").
+
+```yaml
+relations:
+- a: cilia_labels
+  b: cyto_labels
+  output: cilia_to_cell.xlsx
+  max_distance_um: 1.0
+```
+
+For an existing store:
+`patchworks tables image.zarr --relate cilia_labels:cyto_labels --max-distance 1`.
+
+## How good is the segmentation?
+
+Pick the queue **Random sample (error rate)** and review objects in the
+order it gives. The panel reports the fraction found wrong, with a 95%
+confidence interval, e.g. *Error rate 3.0% (95% CI 1.0–8.5%) from 100
+random objects*. Stop when the interval is narrow enough for what you need.
+The estimate is unbiased because the order is random and fixed. Objects you
+already decided from the flagged queue count too: a decision about an
+object is true however it came up.
+
+## Using the corrections
+
+The workbooks and tables are always read *with* the decisions applied:
+
+- **The relation workbooks** of a multi run:
+  `patchworks review image.zarr --workbooks results/` writes every one
+  (`<child>_to_<parent>.xlsx`) from the corrected tables, with a `qc`
+  column (`ok`, `fixed`, or blank if not reviewed). Re-running the same
+  `run_multi` command does it too: a workbook older than your decisions is
+  rewritten, straight from the tables, without re-reading any labels.
+- **Export** the corrected tables, one file per label image, from the
+  panel or with `patchworks review image.zarr --export results/tables
+  --format xlsx` (or `csv`, `parquet`). A csv loads straight into
+  [napari-chunked-regionprops](measurements.md) ("Reload previous
+  results").
+- **Write corrected labels** (panel button, or `--write-labels
+  nuclei_labels`) writes `labels/nuclei_labels_reviewed`, the label image
+  with the decisions applied to the voxels. You only need this for figures,
+  or for tools that read label images only.
+
+`patchworks review image.zarr --summary` prints the counts and the error
+estimate without opening napari.
+
+## Where the tables come from
+
+Every label image the workflow writes gets an **object table**: one row per
+object with its size (`area_voxels`, `area_um3`), centroid, bounding box,
+its spread (`cov_*`, the second moments) and, for a multi run, the parent
+it sits in (`cyto_labels_id`, `cyto_labels_overlap`). The corrected view
+adds each object's shape from its spread: `length_um` (for a straight rod
+the true length; shorter for a curved one), `elongation` (1 round, large
+rod-like) and its main axis (`axis_z`, `axis_y`, `axis_x`). The table lives
+inside the label group:
+
+```text
+image.zarr/labels/cilia_labels/
+  0/ 1/ 2/ …     the label pyramid
+  table/         the object table (one zarr array per column)
+```
+
+So it travels with the labels, including in a zip bundle, and it is
+replaced whenever the labels are. A table computed from older labels is
+recognised and ignored, never shown against the wrong segmentation.
+Measuring costs one read of the labels, after the merge. Add intensity
+columns with `table_channels: [0, 2]`, or turn tables off with
+`object_table: false` (see the [workflow config](snakemake.md)).
+
+For a store that has none, for example a run made before tables existed,
+or labels from elsewhere:
+
+```bash
+patchworks tables image.zarr --relate nuclei_labels:cyto_labels cilia_labels:cyto_labels
+```
+
+In Python:
+
+```python
+from patchworks import Review, read_table
+
+cells = read_table("image.zarr/labels/cyto_labels")      # pandas, as computed
+rv = Review("image.zarr", expect={"cyto_labels": {"nuclei_labels": 1}})
+rv.flags("cyto_labels")[:5]                               # the worst five
+rv.decide("nuclei_labels", 17, "wrong")
+rv.effective("cyto_labels")                               # with corrections
+```
+
+## Where to review
+
+On any machine with a screen and napari, reading the store the cluster
+wrote:
+
+- **Copy the store** to your computer (tables and decisions live in it), or
+  open it on a mounted cluster filesystem. napari reads only what is on
+  screen, so a network mount works.
+- The store must be **writable**, since that is where the decisions are
+  saved. A `.zip` bundle is read-only, so unzip it first.
+- The decisions are small. If you reviewed a copy, `--workbooks` and
+  `--export` work on that copy directly; there is no need to copy anything
+  back.

{patchworks-3.0.0 → patchworks-3.1.0}/docs/guide/snakemake.md

@@ -609,10 +609,15 @@ Everything is under `work_dir`:
 
 ```text
 results/
-  image.zarr/                 # converted, pyramidal OME-ZARR
-  image.zarr/labels/<name>/   # the segmentation (multi-scale, calibrated)
+  image.zarr/                       # converted, pyramidal OME-ZARR
+  image.zarr/labels/<name>/         # the segmentation (multi-scale, calibrated)
+  image.zarr/labels/<name>/table/   # one row per object (object_table: true)
 ```
 
+To look at the likely mistakes and correct them, on any machine with napari:
+`patchworks review /scratch/results/image.zarr` — see
+[Reviewing and correcting results](review.md).
+
 The labels live **inside** the image store. View image + labels together:
 
 ```python
@@ -916,7 +921,9 @@ abort the others; you get a per-config status and a non-zero exit.
     that would exceed your cluster quota.
 
 Each `output:` is an Excel workbook (`openpyxl`, part of the `workflow`
-extra) with two sheets:
+extra), written from the [object tables](review.md#where-the-tables-come-from)
+with any review corrections applied, with two sheets (plus a `qc` column
+each):
 
 | Sheet | One row per | Columns |
 | --- | --- | --- |
@@ -943,6 +950,18 @@ cyto_labels` and `cilia_labels -> nuclei_labels`) so you can use whichever
 fits a given dataset. See `config/config_cilia.yaml`. Its deconvolution step
 needs `pip install "patchworks[dog]"` in the segment jobs' environment.
 
+A `review:` block in `multi.yaml` states what `patchworks review` should
+flag: how many of each child a parent should hold, and how far inside it a
+child must be. It can also classify children by where they sit in their
+parent: apical, basal, lateral or central, see
+[Where a cilium sits](review.md#where-a-cilium-sits-apical-basal-lateral-central).
+A relation with `max_distance_um` gives a child touching no parent the
+nearest one. All of it is checked before anything runs, and stored with
+the results. See [Reviewing](review.md#what-gets-flagged).
+
+A sheet that would exceed Excel's 1,048,576 rows is written as a csv file
+instead (`<stem>_<name>.csv`), and the log says so.
+
 ## Email notifications
 
 Set an address and the workflow mails you when the long steps finish or fail:

{patchworks-3.0.0 → patchworks-3.1.0}/mkdocs.yml

@@ -43,6 +43,7 @@ nav:
   - Performance & memory: guide/performance.md
   - Custom segmentation function: guide/custom_segmentation.md
   - Relating labels across segmentations: guide/label_relations.md
+  - Reviewing and correcting results: guide/review.md
   - Measurements: guide/measurements.md
   - OME-ZARR & napari: guide/ome_zarr_napari.md
   - Cluster workflow (Snakemake): guide/snakemake.md
@@ -65,6 +66,7 @@ nav:
   - I/O helpers: api/io.md
   - Relabelling: api/relabel.md
   - Seam report: api/seams.md
+  - Object tables and review: api/review.md
   - Provenance: api/provenance.md
   - Cluster helpers: api/cluster.md
   - Plugins:

{patchworks-3.0.0 → patchworks-3.1.0}/pyproject.toml

@@ -110,9 +110,17 @@ napari = [
     "pyqt6<6.10",
 ]
 # workflow runs the Snakemake pipeline (per-tile SLURM jobs across GPUs).
-# openpyxl -> scripts/run_multi.py writes label_relations() output as an
-# Excel workbook (per-object + per-container sheets), not a plain CSV.
-workflow = ["snakemake>=8", "snakemake-executor-plugin-slurm", "openpyxl"]
+# pandas + openpyxl -> the relate step writes each relation as an Excel
+# workbook (per-object + per-container sheets) from the object tables.
+workflow = [
+    "snakemake>=8",
+    "snakemake-executor-plugin-slurm",
+    "openpyxl",
+    "pandas>=2.0",
+]
+# review: the object tables' corrected view and exports, without napari
+# (`patchworks review --summary/--export`). The napari panel needs [napari].
+review = ["pandas>=2.0", "openpyxl"]
 # jsonschema validates what we write against the vendored official
 # OME-NGFF schemas (tests/ngff_schemas/); without it that one test skips.
 # openpyxl -> tests/test_run_multi.py checks the relations workbook.
@@ -126,6 +134,8 @@ dev = [
     "tqdm",
     "jsonschema>=4.18",
     "openpyxl",
+    # pandas -> tests/test_review.py (object tables, the corrected view)
+    "pandas>=2.0",
     "ruff==0.15.18",
     "mypy>=1.10",
 ]

{patchworks-3.0.0 → patchworks-3.1.0}/src/patchworks/__init__.py

@@ -63,7 +63,14 @@ from ._postprocess import dilate_labels, fill_holes, open_labels
 from ._provenance import provenance, read_provenance
 from ._relabel import relabel_sequential_array, relabel_sequential_zarr
 from ._relations import label_relations
+from ._review import Review
 from ._seams import seam_report
+from ._tables import (
+    compute_table,
+    measure_objects,
+    read_table,
+    relate_tables,
+)
 from ._volume_filter import (
     filter_labels_by_size,
     max_voxels_for_volume,
@@ -97,6 +104,11 @@ __all__ = [
     "relabel_sequential_array",
     "relabel_sequential_zarr",
     "label_relations",
+    "measure_objects",
+    "compute_table",
+    "read_table",
+    "relate_tables",
+    "Review",
     "seam_report",
     "suggest_overlap",
     "object_f1",