chipstream 0.2.2__tar.gz → 0.3.0__tar.gz

{chipstream-0.2.2 → chipstream-0.3.0}/.github/workflows/deploy_github.yml

@@ -19,6 +19,7 @@ jobs:
         os: [macos-latest, windows-latest]
     steps:
     - name: Set env
+      shell: bash
       run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}

chipstream-0.3.0/CHANGELOG

@@ -0,0 +1,23 @@
+0.3.0
+ - BREAKING CHANGE: major changes in dcnum postprocessing
+ - feat: allow to select whether volume should be computed
+ - feat: allow to select whether flickering correction should be done
+ - feat: allow to specify number of events to analyze in CLI
+ - enh: expose background computation parameters in CLI
+ - ref: move `manager` module to `gui` module
+ - setup: bump dcnum from 0.17.2 to 0.19.1
+0.2.2
+ - maintenance release
+0.2.1
+ - fix: cast string to paths when adding a new path to the manager (#6)
+ - setup: bump dcnum from 0.16.8 to 0.17.2
+0.2.0
+ - feat: allow to specify an output directory in the GUI (#3)
+ - feat: allow to select the pixel size in the GUI (#4)
+ - feat: introduce PathCache for remembering previous output directories
+ - ref: support latest dcnum version
+ - ci: do not build mac app during checks
+0.1.9
+  - initial pre-release
+0.0.1
+ - skeleton release

{chipstream-0.2.2/chipstream.egg-info → chipstream-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chipstream
-Version: 0.2.2
+Version: 0.3.0
 Summary: GUI for DC data postprocessing
 Author: Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -17,7 +17,7 @@ Classifier: Intended Audience :: Science/Research
 Requires-Python: <4,>=3.10
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: dcnum>=0.17.2
+Requires-Dist: dcnum>=0.19.1
 Requires-Dist: h5py<4,>=3.0.0
 Requires-Dist: numpy<2,>=1.21
 Provides-Extra: cli

{chipstream-0.2.2 → chipstream-0.3.0}/chipstream/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.2'
-__version_tuple__ = version_tuple = (0, 2, 2)
+__version__ = version = '0.3.0'
+__version_tuple__ = version_tuple = (0, 3, 0)

{chipstream-0.2.2 → chipstream-0.3.0}/chipstream/cli/cli_main.py

@@ -62,6 +62,17 @@ Recursively analyze a directory containing .rtdc and .avi files::
                                 resolve_path=True,
                                 path_type=pathlib.Path),
                 )
+@click.option("-b", "--background-method",
+              type=click.Choice(sorted(cm.bg_methods.keys()),
+                                case_sensitive=False),
+              default="sparsemed",
+              help="Background computation method to use.")
+@click.option("-kb", "background_kwargs",
+              multiple=True,
+              help="Optional ``KEY=VALUE`` argument for the specified "
+                   "background method",
+              metavar="KEY=VALUE",
+              )
 @click.option("-s", "--segmentation-method",
               type=click.Choice(sorted(cm.seg_methods.keys()),
                                 case_sensitive=False),
@@ -87,6 +98,12 @@ Recursively analyze a directory containing .rtdc and .avi files::
 @click.option("-p", "--pixel-size", type=float, default=0,
               help="Set/override the pixel size for feature "
                    "extraction [µm].")
+@click.option("--limit-events", type=str, default="0",
+              help="Limit events of events to analyze. This can be either "
+                   "a number (e.g. '5000') or a range (e.g. '5000-7000'). "
+                   "You can also specify a step size (e.g. '5000-7000-2' for "
+                   "every second event). The convention follows Python slices "
+                   "with 'n' substituting for 'None'.")
 @click.option("-r", "--recursive", is_flag=True,
               help="Recurse into subdirectories.")
 @click.option("--num-cpus",
@@ -104,11 +121,14 @@ Recursively analyze a directory containing .rtdc and .avi files::
 def chipstream_cli(
     path_in,
     path_out=None,
+    background_method="sparsemed",
+    background_kwargs=None,
     segmentation_method="thresh",
     segmentation_kwargs=None,
     feature_kwargs=None,
     gate_kwargs=None,
     pixel_size=0,
+    limit_events="0",
     recursive=False,
     num_cpus=None,
     dry_run=False,
@@ -121,6 +141,23 @@ def chipstream_cli(
                     fg="yellow")
         verbose = True
 
+    # Parse limit_frames to get the HDF5Data index_mapping
+
+    if limit_events == "0":
+        index_mapping = None
+    elif limit_events.count("-"):
+        vals = limit_events.split("-")
+        assert len(vals) in [2, 3], "slice definition must have length 2 or 3"
+        start = None if vals[0] == "n" else int(vals[0])
+        stop = None if vals[1] == "n" else int(vals[1])
+        if len(vals) == 3:
+            step = None if vals[2] == "n" else int(vals[2])
+        else:
+            step = None
+        index_mapping = slice(start, stop, step)
+    else:
+        index_mapping = int(limit_events)
+
     # Tell the root logger to pretty-print logs
     root_logger = logging.getLogger()
     handler = logging.StreamHandler()
@@ -131,12 +168,15 @@ def chipstream_cli(
     mp.freeze_support()
 
     process_kwargs = dict(
+        background_method=background_method,
+        background_kwargs=background_kwargs,
         segmentation_method=segmentation_method,
         segmentation_kwargs=segmentation_kwargs,
         feature_kwargs=feature_kwargs,
         gate_kwargs=gate_kwargs,
         pixel_size=pixel_size,
         # Below this line are arguments that do not define the pipeline ID
+        index_mapping=index_mapping,
         num_cpus=num_cpus or mp.cpu_count(),
         dry_run=dry_run,
         debug=debug,

{chipstream-0.2.2 → chipstream-0.3.0}/chipstream/cli/cli_proc.py

@@ -10,20 +10,23 @@ import dcnum.read
 
 from . import cli_common as cm
 from .cli_valid import (
-    validate_feature_kwargs, validate_gate_kwargs, validate_pixel_size,
-    validate_segmentation_kwargs
+    validate_background_kwargs, validate_feature_kwargs, validate_gate_kwargs,
+    validate_pixel_size, validate_segmentation_kwargs
 )
 
 
 def process_dataset(
     path_in: pathlib.Path,
     path_out: pathlib.Path,
+    background_method: str,
+    background_kwargs: List[str],
     segmentation_method: str,
     segmentation_kwargs: List[str],
     feature_kwargs: List[str],
     gate_kwargs: List[str],
     pixel_size: float,
     # Below this line are arguments that do not affect the pipeline ID
+    index_mapping: int | slice | None,
     num_cpus: int,
     dry_run: bool,
     debug: bool,
@@ -42,9 +45,10 @@ def process_dataset(
     click.echo(f"Data ID:\t{dat_id}")
 
     # background keyword arguments
-    bg_cls = cm.bg_methods["sparsemed"]
-    bg_kwargs = {}  # use defaults
-    bg_id = bg_cls.get_ppid_from_ppkw(bg_kwargs)  # use defaults
+    bg_kwargs = validate_background_kwargs(background_method,
+                                           background_kwargs)
+    bg_cls = cm.bg_methods[background_method]
+    bg_id = bg_cls.get_ppid_from_ppkw(bg_kwargs)
     click.echo(f"Background ID:\t{bg_id}")
 
     # segmenter keyword arguments
@@ -84,7 +88,8 @@ def process_dataset(
         path_in=path_in,
         path_out=path_out,
         data_code="hdf",
-        data_kwargs={"pixel_size": pixel_size},
+        data_kwargs={"pixel_size": pixel_size,
+                     "index_mapping": index_mapping},
         background_code=bg_cls.get_ppid_code(),
         background_kwargs=bg_kwargs,
         segmenter_code=seg_cls.get_ppid_code(),

{chipstream-0.2.2 → chipstream-0.3.0}/chipstream/cli/cli_valid.py

@@ -8,6 +8,24 @@ import h5py
 from . import cli_common as cm
 
 
+def validate_background_kwargs(bg_method, args):
+    """Parse background keyword arguments"""
+    # Get list of valid keyword arguments
+    bg_cls = cm.bg_methods[bg_method]
+    spec = inspect.getfullargspec(bg_cls.check_user_kwargs)
+    valid_kw = spec.kwonlyargs
+    annot = spec.annotations
+    # Convert the input args to key-value pairs
+    kwargs = {}
+    for key, value in [a.split("=") for a in args]:
+        if key not in valid_kw:
+            raise ValueError(f"Invalid keyword '{key}' for {bg_method}. "
+                             + f"Allowed keywords are {valid_kw}!")
+        # Convert to correct dtype (default to string)
+        kwargs[key] = ppid.convert_to_dtype(value, annot[key])
+    return kwargs
+
+
 def validate_feature_kwargs(args):
     # Get list of valid keyword arguments
     feat_cls = cm.QueueEventExtractor

{chipstream-0.2.2 → chipstream-0.3.0}/chipstream/gui/main_window.py

@@ -150,6 +150,10 @@ class ChipStream(QtWidgets.QMainWindow):
 
         # default background computer is "sparsemed"
         bg_default = feat_background.BackgroundSparseMed
+        bg_kwargs = inspect.getfullargspec(
+            bg_default.check_user_kwargs).kwonlydefaults
+        bg_kwargs["offset_correction"] = \
+            self.checkBox_bg_flickering.isChecked()
 
         # populate segmenter and its kwargs
         segmenter = self.comboBox_segmenter.currentData()
@@ -161,14 +165,14 @@ class ChipStream(QtWidgets.QMainWindow):
             "data_code": "hdf",
             "data_kwargs": data_kwargs,
             "background_code": bg_default.get_ppid_code(),
-            "background_kwargs": inspect.getfullargspec(
-                bg_default.check_user_kwargs).kwonlydefaults,
+            "background_kwargs": bg_kwargs,
             "segmenter_code": segmenter,
             "segmenter_kwargs": segmenter_kwargs,
             "feature_code": "legacy",
             "feature_kwargs": {
                 "brightness": self.checkBox_feat_bright.isChecked(),
                 "haralick": self.checkBox_feat_haralick.isChecked(),
+                "volume": self.checkBox_feat_volume.isChecked(),
                 },
             "gate_code": "norm",
             "gate_kwargs": {},

{chipstream-0.2.2 → chipstream-0.3.0}/chipstream/gui/main_window.ui

@@ -6,8 +6,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>1408</width>
-    <height>754</height>
+    <width>1120</width>
+    <height>680</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -122,6 +122,25 @@
          </property>
         </spacer>
        </item>
+       <item>
+        <widget class="QGroupBox" name="groupBox">
+         <property name="title">
+          <string>Background Image</string>
+         </property>
+         <layout class="QVBoxLayout" name="verticalLayout_2">
+          <item>
+           <widget class="QCheckBox" name="checkBox_bg_flickering">
+            <property name="text">
+             <string>flickering correction</string>
+            </property>
+            <property name="checked">
+             <bool>true</bool>
+            </property>
+           </widget>
+          </item>
+         </layout>
+        </widget>
+       </item>
        <item>
         <widget class="QGroupBox" name="groupBox_4">
          <property name="sizePolicy">
@@ -233,6 +252,16 @@
             </property>
            </widget>
           </item>
+          <item>
+           <widget class="QCheckBox" name="checkBox_feat_volume">
+            <property name="text">
+             <string>Volume</string>
+            </property>
+            <property name="checked">
+             <bool>true</bool>
+            </property>
+           </widget>
+          </item>
           <item>
            <widget class="QCheckBox" name="checkBox_feat_haralick">
             <property name="text">
@@ -335,7 +364,7 @@
     <rect>
      <x>0</x>
      <y>0</y>
-     <width>1408</width>
+     <width>1120</width>
      <height>22</height>
     </rect>
    </property>

{chipstream-0.2.2 → chipstream-0.3.0}/chipstream/gui/table_progress.py

@@ -1,6 +1,6 @@
 from PyQt6 import QtCore, QtWidgets
 
-from ..manager import ChipStreamJobManager
+from .manager import ChipStreamJobManager
 
 
 ItemProgressRole = QtCore.Qt.ItemDataRole.UserRole + 1001

{chipstream-0.2.2 → chipstream-0.3.0/chipstream.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chipstream
-Version: 0.2.2
+Version: 0.3.0
 Summary: GUI for DC data postprocessing
 Author: Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -17,7 +17,7 @@ Classifier: Intended Audience :: Science/Research
 Requires-Python: <4,>=3.10
 Description-Content-Type: text/x-rst
 License-File: LICENSE
-Requires-Dist: dcnum>=0.17.2
+Requires-Dist: dcnum>=0.19.1
 Requires-Dist: h5py<4,>=3.0.0
 Requires-Dist: numpy<2,>=1.21
 Provides-Extra: cli

{chipstream-0.2.2 → chipstream-0.3.0}/chipstream.egg-info/SOURCES.txt

@@ -22,7 +22,6 @@ build-recipes/win_chipstream.iss_dummy
 build-recipes/win_make_iss.py
 chipstream/__init__.py
 chipstream/_version.py
-chipstream/manager.py
 chipstream/path_cache.py
 chipstream.egg-info/PKG-INFO
 chipstream.egg-info/SOURCES.txt
@@ -38,6 +37,7 @@ chipstream/cli/cli_valid.py
 chipstream/gui/__init__.py
 chipstream/gui/main_window.py
 chipstream/gui/main_window.ui
+chipstream/gui/manager.py
 chipstream/gui/splash.py
 chipstream/gui/table_progress.py
 chipstream/gui/img/chipstream_icon.png
@@ -53,6 +53,6 @@ tests/requirements-full.txt
 tests/requirements.txt
 tests/test_cli.py
 tests/test_gui.py
-tests/test_manager.py
+tests/test_gui_manager.py
 tests/test_path_cache.py
 tests/data/fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip

{chipstream-0.2.2 → chipstream-0.3.0}/chipstream.egg-info/requires.txt

@@ -1,4 +1,4 @@
-dcnum>=0.17.2
+dcnum>=0.19.1
 h5py<4,>=3.0.0
 numpy<2,>=1.21
 

{chipstream-0.2.2 → chipstream-0.3.0}/pyproject.toml

@@ -27,7 +27,7 @@ classifiers = [
 ]
 license = {text = "GPL version 3.0 or later"}
 dependencies = [
-    "dcnum>=0.17.2",
+    "dcnum>=0.19.1",
     "h5py>=3.0.0, <4",
     "numpy>=1.21, <2",  # CVE-2021-33430
 ]

chipstream-0.3.0/tests/test_cli.py

@@ -0,0 +1,109 @@
+import dcnum.read
+import h5py
+
+import pytest
+
+from helper_methods import retrieve_data
+
+
+pytest.importorskip("click")
+
+from chipstream.cli import cli_main  # noqa: E402
+
+
+@pytest.mark.parametrize("limit_events,dcnum_mapping,dcnum_yield,f0", [
+    # this is the default
+    ["0", "0", 36, 1],
+    ["1-4", "1-4-n", 6, 2],
+    ["3-10-5", "3-10-5", 5, 4],
+])
+def test_cli_limit_events(cli_runner, limit_events, dcnum_yield,
+                          dcnum_mapping, f0):
+    path_temp = retrieve_data(
+        "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+    path = path_temp.with_name("input_path.rtdc")
+
+    # create a test file for more than 100 events
+    with dcnum.read.concatenated_hdf5_data(
+        paths=3*[path_temp],
+        path_out=path,
+            compute_frame=True):
+        pass
+
+    # sanity check
+    with h5py.File(path) as h5:
+        assert h5["events/frame"][0] == 1
+        assert h5["events/frame"][1] == 2
+
+    path_out = path.with_name("limited_events.rtdc")
+    result = cli_runner.invoke(cli_main.chipstream_cli,
+                               [str(path),
+                                str(path_out),
+                                "-s", "thresh",
+                                "--limit-events", limit_events,
+                                ])
+    assert result.exit_code == 0
+
+    with h5py.File(path_out) as h5:
+        assert h5["events/frame"][0] == f0
+        assert h5.attrs["pipeline:dcnum yield"] == dcnum_yield
+        assert h5.attrs["pipeline:dcnum mapping"] == dcnum_mapping
+        assert h5.attrs["experiment:event count"] == dcnum_yield
+
+
+def test_cli_set_pixel_size(cli_runner):
+    path_temp = retrieve_data(
+        "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+    path = path_temp.with_name("input_path.rtdc")
+
+    # create a test file for more than 100 events
+    with dcnum.read.concatenated_hdf5_data(
+        paths=3*[path_temp],
+        path_out=path,
+            compute_frame=True):
+        pass
+
+    # remove the pixel size from the input data
+    with h5py.File(path, "a") as h5:
+        del h5.attrs["imaging:pixel size"]
+
+    path_out = path.with_name("with_pixel_size_dcn.rtdc")
+    result = cli_runner.invoke(cli_main.chipstream_cli,
+                               [str(path),
+                                str(path_out),
+                                "-s", "thresh",
+                                "-p", "0.266",
+                                ])
+    assert result.exit_code == 0
+
+    with h5py.File(path_out) as h5:
+        assert h5.attrs["imaging:pixel size"] == 0.266
+
+
+@pytest.mark.parametrize("method,kwarg,ppid", [
+    ["sparsemed", "offset_correction=0", "sparsemed:k=200^s=1^t=0^f=0.8^o=0"],
+    ["rollmed", "kernel_size=12", "rollmed:k=12^b=10000"],
+])
+def test_cli_set_background(cli_runner, method, kwarg, ppid):
+    path_temp = retrieve_data(
+        "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+    path = path_temp.with_name("input_path.rtdc")
+
+    # create a test file for more than 100 events
+    with dcnum.read.concatenated_hdf5_data(
+        paths=3*[path_temp],
+        path_out=path,
+            compute_frame=True):
+        pass
+
+    path_out = path.with_name("output.rtdc")
+    result = cli_runner.invoke(cli_main.chipstream_cli,
+                               [str(path),
+                                str(path_out),
+                                "-b", method,
+                                "-kb", kwarg,
+                                ])
+    assert result.exit_code == 0
+
+    with h5py.File(path_out) as h5:
+        assert h5.attrs["pipeline:dcnum background"] == ppid

{chipstream-0.2.2 → chipstream-0.3.0}/tests/test_gui.py

@@ -40,6 +40,27 @@ def test_gui_basic(mw):
     assert len(mw.manager) == 0
 
 
+@pytest.mark.parametrize("correct_offset", [True, False])
+def test_gui_correct_offset(mw, correct_offset):
+    path = retrieve_data(
+        "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+    mw.append_paths([path])
+    mw.checkBox_pixel_size.setChecked(True)
+    mw.checkBox_bg_flickering.setChecked(correct_offset)
+    mw.doubleSpinBox_pixel_size.setValue(0.666)
+    mw.on_run()
+    while mw.manager.is_busy():
+        time.sleep(.1)
+    out_path = path.with_name(path.stem + "_dcn.rtdc")
+    assert out_path.exists()
+
+    with h5py.File(out_path) as h5:
+        assert np.allclose(h5.attrs["imaging:pixel size"],
+                           0.666,
+                           atol=0, rtol=1e-5)
+        assert ("bg_off" in h5["events"]) == correct_offset
+
+
 def test_gui_set_pixel_size(mw):
     path = retrieve_data(
         "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
@@ -56,3 +77,24 @@ def test_gui_set_pixel_size(mw):
         assert np.allclose(h5.attrs["imaging:pixel size"],
                            0.666,
                            atol=0, rtol=1e-5)
+
+
+@pytest.mark.parametrize("use_volume", [True, False])
+def test_gui_use_volume(mw, use_volume):
+    path = retrieve_data(
+        "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
+    mw.append_paths([path])
+    mw.checkBox_pixel_size.setChecked(True)
+    mw.checkBox_feat_volume.setChecked(use_volume)
+    mw.doubleSpinBox_pixel_size.setValue(0.666)
+    mw.on_run()
+    while mw.manager.is_busy():
+        time.sleep(.1)
+    out_path = path.with_name(path.stem + "_dcn.rtdc")
+    assert out_path.exists()
+
+    with h5py.File(out_path) as h5:
+        assert np.allclose(h5.attrs["imaging:pixel size"],
+                           0.666,
+                           atol=0, rtol=1e-5)
+        assert ("volume" in h5["events"]) == use_volume

chipstream-0.2.2/tests/test_manager.py → chipstream-0.3.0/tests/test_gui_manager.py

@@ -1,6 +1,6 @@
 import shutil
 
-from chipstream import manager
+from chipstream.gui import manager
 
 
 from helper_methods import retrieve_data
@@ -80,11 +80,11 @@ def test_manager_run_defaults():
     assert mg.current_index == 0
     assert not mg.is_busy()
     # default pipeline may change in dcnum
-    assert mg.get_runner(0).ppid == ("7|"
-                                     "hdf:p=0.2645|"
-                                     "sparsemed:k=200^s=1^t=0^f=0.8|"
+    assert mg.get_runner(0).ppid == ("8|"
+                                     "hdf:p=0.2645^i=0|"
+                                     "sparsemed:k=200^s=1^t=0^f=0.8^o=1|"
                                      "thresh:t=-6:cle=1^f=1^clo=2|"
-                                     "legacy:b=1^h=1|"
+                                     "legacy:b=1^h=1^v=1|"
                                      "norm:o=0^s=10")
 
 

chipstream-0.2.2/CHANGELOG

@@ -1,15 +0,0 @@
-0.2.2
- - maintenance release
-0.2.1
- - fix: cast string to paths when adding a new path to the manager (#6)
- - setup: bump dcnum from 0.16.8 to 0 17.2
-0.2.0
- - feat: allow to specify an output directory in the GUI (#3)
- - feat: allow to select the pixel size in the GUI (#4)
- - feat: introduce PathCache for remembering previous output directories
- - ref: support latest dcnum version
- - ci: do not build mac app during checks
-0.1.9
-  - initial pre-release
-0.0.1
- - skeleton release

chipstream-0.2.2/tests/test_cli.py

@@ -1,40 +0,0 @@
-import dcnum.read
-import h5py
-
-import pytest
-
-from helper_methods import retrieve_data
-
-
-pytest.importorskip("click")
-
-from chipstream.cli import cli_main  # noqa: E402
-
-
-def test_cli_set_pixel_size(cli_runner):
-    path_temp = retrieve_data(
-        "fmt-hdf5_cytoshot_full-features_legacy_allev_2023.zip")
-    path = path_temp.with_name("input_path.rtdc")
-
-    # create a test file for more than 100 events
-    with dcnum.read.concatenated_hdf5_data(
-        paths=3*[path_temp],
-        path_out=path,
-            compute_frame=True):
-        pass
-
-    # remove the pixel size from the input data
-    with h5py.File(path, "a") as h5:
-        del h5.attrs["imaging:pixel size"]
-
-    path_out = path.with_name("with_pixel_size_dcn.rtdc")
-    result = cli_runner.invoke(cli_main.chipstream_cli,
-                               [str(path),
-                                str(path_out),
-                                "-s", "thresh",
-                                "-p", "0.266",
-                                ])
-    assert result.exit_code == 0
-
-    with h5py.File(path_out) as h5:
-        assert h5.attrs["imaging:pixel size"] == 0.266