patchworks 0.11.1__tar.gz → 0.11.3__tar.gz

{patchworks-0.11.1 → patchworks-0.11.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: patchworks
-Version: 0.11.1
+Version: 0.11.3
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues

{patchworks-0.11.1 → patchworks-0.11.3}/docs/guide/snakemake.md

@@ -56,6 +56,7 @@ shard: false                   # true → pack chunks into shards (fewer files)
 channel: 0                     # channel to segment (null = keep all)
 level: 0                       # pyramid level (0 = full resolution)
 tile_shape: "auto"             # "auto", or e.g. [16, 1024, 1024] (zyx)
+gpu_memory_gb: null            # for "auto" on SLURM: your segment GPU's VRAM
 overlap: 30                    # halo ≈ one object diameter
 skip_empty: true               # skip background tiles
 empty_threshold: null          # null → Otsu
@@ -81,6 +82,11 @@ sequential_labels: true        # renumber labels to a contiguous 1..N
     more (faster) jobs; very large 3-D tiles are slow. Keep `do_3D: false` (2-D
     per slice) if your objects segment fine per slice — it is much faster.
 
+    Tile planning runs on a **CPU** node, which cannot see the segment GPU, so
+    it logs `GPU memory query failed … using 8 GiB default` and sizes tiles for
+    8 GiB. Harmless, but to size for the real GPU set `gpu_memory_gb:` to its
+    VRAM (e.g. `24`, `40`, `80`) — or just set `tile_shape` explicitly.
+
 ## 4. Dry-run (always do this first)
 
 Check the plan without running anything:
@@ -174,6 +180,11 @@ Snakemake is resumable — if jobs fail or you cancel, just relaunch the same
 command and it picks up only the missing tiles. To force a clean rerun, delete
 `work_dir` (or the relevant outputs).
 
+The OME-ZARR conversion is **not redone** once `image.zarr` exists: the
+`convert` rule's output is a marker file inside the store, so Snakemake skips it
+on every later run. To force a fresh conversion, delete `image.zarr` or run
+`snakemake --forcerun convert`.
+
 ## pixi (instead of conda)
 
 Conda is **not** required — Snakemake runs in whatever environment launches it.

{patchworks-0.11.1 → patchworks-0.11.3}/workflow/config/config.yaml

@@ -15,11 +15,15 @@ work_dir: "/path/to/results" # everything is written under here
 reuse_pyramid: true # for .ims: copy its own pyramid (fast); else rebuild
 convert_chunks: null # null → patchworks' bounded auto chunks; or [c,z,y,x]
 shard: false # true → pack chunks into shards (zarr v3; fewer files)
+# Re-running reuses an existing image.zarr automatically (skips conversion).
+# To force a rebuild: delete image.zarr, or `snakemake --forcerun convert`.
 
 # ---- tiling -----------------------------------------------------------------
 channel: 0 # channel to segment
 level: 0 # pyramid level to segment (0 = full res)
 tile_shape: "auto" # "auto", or e.g. [16, 1024, 1024] (zyx)
+gpu_memory_gb: null # for "auto" on a CPU prepare node: your segment GPU's VRAM
+#                     (e.g. 24, 40, 80). null → 8 GiB guess + a harmless warning
 overlap: 30 # halo (≈ one object diameter)
 skip_empty: true # skip background tiles
 empty_threshold: null # null → Otsu; or a number

{patchworks-0.11.1 → patchworks-0.11.3}/workflow/rules/common.smk

@@ -2,6 +2,11 @@
 
 WORK = config["work_dir"]
 IMAGE = f"{WORK}/image.zarr"
+# A single file inside the store, used as the convert rule's output and as the
+# dependency marker for downstream rules. Tracking a leaf file (not the
+# directory) lets Snakemake skip conversion when the store already exists and
+# avoids wiping the whole store on a re-run (same trick as imcf/sopa).
+IMAGE_OK = f"{IMAGE}/zarr.json"
 TILES = f"{WORK}/tiles.json"
 STAGE = f"{WORK}/stage.zarr"
 

{patchworks-0.11.1 → patchworks-0.11.3}/workflow/rules/convert.smk

@@ -2,6 +2,7 @@
 
 rule convert:
     output:
-        directory(IMAGE),
+        # marker file inside the store; existence => skip re-conversion.
+        IMAGE_OK,
     script:
         "../scripts/convert.py"

{patchworks-0.11.1 → patchworks-0.11.3}/workflow/rules/segment.smk

@@ -2,7 +2,7 @@
 
 checkpoint prepare:
     input:
-        IMAGE,
+        IMAGE_OK,
     output:
         tiles=TILES,
         stage=directory(STAGE),
@@ -15,7 +15,7 @@ rule segment:
     input:
         tiles=TILES,
         stage=STAGE,
-        image=IMAGE,
+        image=IMAGE_OK,
     output:
         f"{WORK}/seg/{{index}}.done",
     script:

patchworks-0.11.3/workflow/scripts/convert.py

@@ -0,0 +1,21 @@
+"""Snakemake script: convert the input to a pyramidal OME-ZARR.
+
+The rule's output is a marker file inside the store (``image.zarr/zarr.json``),
+so Snakemake skips this step entirely when the store already exists — the
+conversion is not redone. To force a fresh conversion, delete ``image.zarr``
+(or run ``snakemake --forcerun convert``).
+"""
+
+from patchworks.plugins.ome_zarr import to_ome_zarr
+
+cfg = snakemake.config  # noqa: F821  (injected by Snakemake)
+chunks = cfg.get("convert_chunks")
+to_ome_zarr(
+    cfg["input"],
+    str(snakemake.output[0]).removesuffix("/zarr.json"),  # noqa: F821
+    chunks=tuple(chunks) if chunks else None,
+    shard=bool(cfg.get("shard", False)),
+    reuse_pyramid=bool(cfg.get("reuse_pyramid", False)),
+    progress=False,
+    overwrite=True,
+)

{patchworks-0.11.1 → patchworks-0.11.3}/workflow/scripts/prepare_tiles.py

@@ -20,12 +20,17 @@ image = open_image(work_dir, cfg["channel"], cfg["level"])
 ts = cfg.get("tile_shape", "auto")
 if ts == "auto":
     cp = cfg["cellpose"]
+    # prepare runs on a CPU node, so the segment GPU's VRAM can't be queried
+    # here; pass gpu_memory_gb from the config to size tiles for it (avoids the
+    # "GPU memory query failed" fallback). None => the built-in 8 GiB default.
+    gpu_gb = cfg.get("gpu_memory_gb")
     tile_shape = tuple(
         partial(
             auto_tile_shape_cellpose,
             do_3D=cp.get("do_3D", False),
             use_gpu=cp.get("gpu", True),
             diameter=cp.get("diameter"),
+            gpu_memory=int(gpu_gb * 1024**3) if gpu_gb else None,
         )(image.shape, image.dtype)
     )
 else:

patchworks-0.11.1/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L2ltYWdlLnphcnI=

@@ -1 +0,0 @@
-{"rule": "convert", "input": [], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/convert.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968324357465, "starttime": 1782285191.3091571, "endtime": 1782285191.3090107, "incomplete": false, "external_jobid": null, "input_checksums": {}}

patchworks-0.11.1/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L2xhYmVscy5kb25l

@@ -1 +0,0 @@
-{"rule": "merge", "input": ["/tmp/toy/out/seg/0.done", "/tmp/toy/out/seg/1.done", "/tmp/toy/out/seg/3.done", "/tmp/toy/out/seg/4.done"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/merge.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968324177309, "starttime": 1782285198.9120235, "endtime": 1782285198.9118745, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/toy/out/seg/0.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/toy/out/seg/1.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/toy/out/seg/3.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/toy/out/seg/4.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}

patchworks-0.11.1/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3N0YWdlLnphcnI=

@@ -1 +0,0 @@
-{"rule": "prepare", "input": ["/tmp/toy/out/image.zarr"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/prepare_tiles.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968324357329, "starttime": 1782285193.0925362, "endtime": 1782285193.0922585, "incomplete": false, "external_jobid": null, "input_checksums": {}}

patchworks-0.11.1/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy80LmRvbmU=

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/toy/out/image.zarr", "/tmp/toy/out/stage.zarr", "/tmp/toy/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968316803128, "starttime": 1782285194.9512916, "endtime": 1782285194.9498596, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/toy/out/tiles.json": "sha256:a7ba7de7a100b9eed2f69158d779580251e48da2c4bdb722e28c4d9a79abb411"}}

patchworks-0.11.1/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8wLmRvbmU=

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/toy/out/image.zarr", "/tmp/toy/out/stage.zarr", "/tmp/toy/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968316911209, "starttime": 1782285194.9791708, "endtime": 1782285194.9778593, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/toy/out/tiles.json": "sha256:a7ba7de7a100b9eed2f69158d779580251e48da2c4bdb722e28c4d9a79abb411"}}

patchworks-0.11.1/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8xLmRvbmU=

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/toy/out/image.zarr", "/tmp/toy/out/stage.zarr", "/tmp/toy/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968316911362, "starttime": 1782285196.8732357, "endtime": 1782285196.871846, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/toy/out/tiles.json": "sha256:a7ba7de7a100b9eed2f69158d779580251e48da2c4bdb722e28c4d9a79abb411"}}

patchworks-0.11.1/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3NlZy8zLmRvbmU=

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/toy/out/image.zarr", "/tmp/toy/out/stage.zarr", "/tmp/toy/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968316911515, "starttime": 1782285196.9122667, "endtime": 1782285196.9108458, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/toy/out/tiles.json": "sha256:a7ba7de7a100b9eed2f69158d779580251e48da2c4bdb722e28c4d9a79abb411"}}

patchworks-0.11.1/workflow/.snakemake/metadata/L3RtcC90b3kvb3V0L3RpbGVzLmpzb24=

@@ -1 +0,0 @@
-{"rule": "prepare", "input": ["/tmp/toy/out/image.zarr"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/prepare_tiles.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 7968324357329, "starttime": 1782285193.0925362, "endtime": 1782285193.0908728, "incomplete": false, "external_jobid": null, "input_checksums": {}}

patchworks-0.11.1/workflow/scripts/convert.py

@@ -1,15 +0,0 @@
-"""Snakemake script: convert the input to a pyramidal OME-ZARR."""
-
-from patchworks.plugins.ome_zarr import to_ome_zarr
-
-cfg = snakemake.config  # noqa: F821  (injected by Snakemake)
-chunks = cfg.get("convert_chunks")
-to_ome_zarr(
-    cfg["input"],
-    snakemake.output[0],  # noqa: F821
-    chunks=tuple(chunks) if chunks else None,
-    shard=bool(cfg.get("shard", False)),
-    reuse_pyramid=bool(cfg.get("reuse_pyramid", False)),
-    progress=False,
-    overwrite=True,
-)