patchworks 0.11.2__tar.gz → 0.11.4__tar.gz

{patchworks-0.11.2 → patchworks-0.11.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: patchworks
-Version: 0.11.2
+Version: 0.11.4
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues

{patchworks-0.11.2 → patchworks-0.11.4}/docs/guide/snakemake.md

@@ -56,6 +56,7 @@ shard: false                   # true → pack chunks into shards (fewer files)
 channel: 0                     # channel to segment (null = keep all)
 level: 0                       # pyramid level (0 = full resolution)
 tile_shape: "auto"             # "auto", or e.g. [16, 1024, 1024] (zyx)
+gpu_memory_gb: null            # for "auto" on SLURM: your segment GPU's VRAM
 overlap: 30                    # halo ≈ one object diameter
 skip_empty: true               # skip background tiles
 empty_threshold: null          # null → Otsu
@@ -81,6 +82,11 @@ sequential_labels: true        # renumber labels to a contiguous 1..N
     more (faster) jobs; very large 3-D tiles are slow. Keep `do_3D: false` (2-D
     per slice) if your objects segment fine per slice — it is much faster.
 
+    Tile planning runs on a **CPU** node, which cannot see the segment GPU, so
+    it logs `GPU memory query failed … using 8 GiB default` and sizes tiles for
+    8 GiB. Harmless, but to size for the real GPU set `gpu_memory_gb:` to its
+    VRAM (e.g. `24`, `40`, `80`) — or just set `tile_shape` explicitly.
+
 ## 4. Dry-run (always do this first)
 
 Check the plan without running anything:
@@ -96,11 +102,14 @@ known after `prepare` decides which tiles are non-empty.)
 ## 5a. Run locally (single machine)
 
 ```bash
-python -m snakemake -s Snakefile --configfile config/config.yaml --cores 8
+python -m snakemake -s Snakefile --configfile config/config.yaml \
+    --rerun-triggers mtime --cores 8
 ```
 
 Tiles run on the local machine (one at a time on the GPU). Good for a small
-image or a smoke test.
+image or a smoke test. `--rerun-triggers mtime` re-runs only steps whose output
+is missing/stale — so upgrading patchworks won't redo the conversion (the SLURM
+profile sets this for you).
 
 ## 5b. Run on SLURM (one GPU job per tile)
 
@@ -179,6 +188,12 @@ The OME-ZARR conversion is **not redone** once `image.zarr` exists: the
 on every later run. To force a fresh conversion, delete `image.zarr` or run
 `snakemake --forcerun convert`.
 
+This relies on `--rerun-triggers mtime` (set in the SLURM profile and the pixi
+tasks; add it on the command line for ad-hoc local runs). Without it, Snakemake
+also re-runs a step when its **code, params or software environment** change —
+so upgrading patchworks would re-do the conversion and overwrite an existing
+result. Keep `mtime` and reruns happen only when an output is missing or stale.
+
 ## pixi (instead of conda)
 
 Conda is **not** required — Snakemake runs in whatever environment launches it.

{patchworks-0.11.2 → patchworks-0.11.4}/workflow/README.md

@@ -56,10 +56,11 @@ account, GPU request).
 ## Run
 
 ```bash
-# locally (single machine)
-snakemake --cores 8 --configfile config/config.yaml
+# locally (single machine) — mtime triggers => upgrades don't redo conversion
+snakemake --cores 8 --configfile config/config.yaml --rerun-triggers mtime
 
 # on SLURM — one GPU job per tile, up to `jobs:` in parallel
+# (the profile already sets --rerun-triggers mtime)
 snakemake --workflow-profile profile/slurm --configfile config/config.yaml
 ```
 

{patchworks-0.11.2 → patchworks-0.11.4}/workflow/config/config.yaml

@@ -22,6 +22,8 @@ shard: false # true → pack chunks into shards (zarr v3; fewer files)
 channel: 0 # channel to segment
 level: 0 # pyramid level to segment (0 = full res)
 tile_shape: "auto" # "auto", or e.g. [16, 1024, 1024] (zyx)
+gpu_memory_gb: null # for "auto" on a CPU prepare node: your segment GPU's VRAM
+#                     (e.g. 24, 40, 80). null → 8 GiB guess + a harmless warning
 overlap: 30 # halo (≈ one object diameter)
 skip_empty: true # skip background tiles
 empty_threshold: null # null → Otsu; or a number

{patchworks-0.11.2 → patchworks-0.11.4}/workflow/pixi.toml

@@ -19,6 +19,8 @@ python = "3.12.*"
 patchworks = { version = "*", extras = ["workflow", "cellpose", "imaris", "bioio"] }
 
 [tasks]
-dry = "snakemake -s Snakefile --configfile config/config.yaml -n -p"
-go = "snakemake -s Snakefile --configfile config/config.yaml --cores 8"
+# --rerun-triggers mtime: don't redo finished steps (e.g. the conversion) just
+# because the patchworks version/code changed — only when an output is missing.
+dry = "snakemake -s Snakefile --configfile config/config.yaml --rerun-triggers mtime -n -p"
+go = "snakemake -s Snakefile --configfile config/config.yaml --rerun-triggers mtime --cores 8"
 slurm = "snakemake --workflow-profile profile/slurm --configfile config/config.yaml"

{patchworks-0.11.2 → patchworks-0.11.4}/workflow/profile/slurm/config.yaml

@@ -8,6 +8,10 @@ executor: slurm
 jobs: 64 # max concurrent SLURM jobs (≈ how many GPUs you can grab at once)
 latency-wait: 60
 rerun-incomplete: true
+# Only re-run when an output is missing/stale — NOT when the patchworks code,
+# params or software env change. Stops an upgrade from re-doing the expensive
+# conversion / re-segmenting an existing image.zarr.
+rerun-triggers: mtime
 printshellcmds: true
 keep-going: true
 

{patchworks-0.11.2 → patchworks-0.11.4}/workflow/scripts/prepare_tiles.py

@@ -20,12 +20,17 @@ image = open_image(work_dir, cfg["channel"], cfg["level"])
 ts = cfg.get("tile_shape", "auto")
 if ts == "auto":
     cp = cfg["cellpose"]
+    # prepare runs on a CPU node, so the segment GPU's VRAM can't be queried
+    # here; pass gpu_memory_gb from the config to size tiles for it (avoids the
+    # "GPU memory query failed" fallback). None => the built-in 8 GiB default.
+    gpu_gb = cfg.get("gpu_memory_gb")
     tile_shape = tuple(
         partial(
             auto_tile_shape_cellpose,
             do_3D=cp.get("do_3D", False),
             use_gpu=cp.get("gpu", True),
             diameter=cp.get("diameter"),
+            gpu_memory=int(gpu_gb * 1024**3) if gpu_gb else None,
         )(image.shape, image.dtype)
     )
 else:

patchworks-0.11.2/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvbGFiZWxzLmRvbmU=

@@ -1 +0,0 @@
-{"rule": "merge", "input": ["/tmp/pw_wf/out/seg/0.done", "/tmp/pw_wf/out/seg/1.done", "/tmp/pw_wf/out/seg/2.done", "/tmp/pw_wf/out/seg/3.done", "/tmp/pw_wf/out/seg/4.done", "/tmp/pw_wf/out/seg/5.done", "/tmp/pw_wf/out/seg/6.done", "/tmp/pw_wf/out/seg/7.done"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/merge.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289946198395, "starttime": 1782306826.05809, "endtime": 1782306826.057891, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/seg/0.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/1.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/2.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/3.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/4.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/5.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/6.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "/tmp/pw_wf/out/seg/7.done": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}}

patchworks-0.11.2/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzAuZG9uZQ==

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938895482, "starttime": 1782306823.2564273, "endtime": 1782306823.255227, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}}

patchworks-0.11.2/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzEuZG9uZQ==

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938895635, "starttime": 1782306823.2361996, "endtime": 1782306823.2352273, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}}

patchworks-0.11.2/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzIuZG9uZQ==

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938895788, "starttime": 1782306820.7251186, "endtime": 1782306820.724245, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}}

patchworks-0.11.2/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzMuZG9uZQ==

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792521, "starttime": 1782306820.8450873, "endtime": 1782306820.8442442, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}}

patchworks-0.11.2/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzQuZG9uZQ==

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792674, "starttime": 1782306820.7828398, "endtime": 1782306820.7822447, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}}

patchworks-0.11.2/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzUuZG9uZQ==

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792827, "starttime": 1782306823.5588164, "endtime": 1782306823.558225, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}}

patchworks-0.11.2/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzYuZG9uZQ==

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938792980, "starttime": 1782306823.0890763, "endtime": 1782306823.0882282, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}}

patchworks-0.11.2/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc2VnLzcuZG9uZQ==

@@ -1 +0,0 @@
-{"rule": "segment", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json", "/tmp/pw_wf/out/stage.zarr", "/tmp/pw_wf/out/tiles.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/segment_tile.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289938793133, "starttime": 1782306821.0048127, "endtime": 1782306821.0042431, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/tiles.json": "sha256:d5abe0a9301e6274b02cdae545a8b5e414d3b51c1396d18ae9bcb1ea935bfe67", "/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}}

patchworks-0.11.2/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvc3RhZ2UuemFycg==

@@ -1 +0,0 @@
-{"rule": "prepare", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/prepare_tiles.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289946375343, "starttime": 1782306818.2703326, "endtime": 1782306818.2701185, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}}

patchworks-0.11.2/workflow/.snakemake/metadata/L3RtcC9wd193Zi9vdXQvdGlsZXMuanNvbg==

@@ -1 +0,0 @@
-{"rule": "prepare", "input": ["/tmp/pw_wf/out/image.zarr/zarr.json"], "log": [], "shellcmd": null, "params": [], "code": "        \"../scripts/prepare_tiles.py\"\n", "record_format_version": 6, "conda_env": null, "container_img_url": null, "software_stack_hash": "d41d8cd98f00b204e9800998ecf8427e", "job_hash": 8289946375343, "starttime": 1782306818.2703326, "endtime": 1782306818.2692626, "incomplete": false, "external_jobid": null, "input_checksums": {"/tmp/pw_wf/out/image.zarr/zarr.json": "sha256:8baa9bb439818b6e4a5358464300e6239dae6fddb2b31fa26717a7c56eaf1458"}}