PyOPIA 2.13.1__tar.gz → 2.15.0__tar.gz

{pyopia-2.13.1 → pyopia-2.15.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: PyOPIA
-Version: 2.13.1
+Version: 2.15.0
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Project-URL: Repository, https://github.com/sintef/pyopia
 Project-URL: Documentation, https://pyopia.readthedocs.io
 Author-email: Emlyn Davies <emlyn.davies@sintef.no>, "Alex Nimmo Smith@plymouth.ac.uk" <alex.nimmo.smith@plymouth.ac.uk>
 License-File: LICENSE
 Keywords: Imaging,Measurement,Ocean,Particles,Size distribution
-Requires-Python: ~=3.12
+Requires-Python: ~=3.12.0
 Requires-Dist: click<8.2.0
 Requires-Dist: cmocean<4,>=3.0.3
 Requires-Dist: dask>=2024.8.1
@@ -41,6 +41,7 @@ Requires-Dist: toml<0.11,>=0.10.2
 Requires-Dist: tqdm<5,>=4.66.1
 Requires-Dist: typer[all]<0.10,>=0.9.0
 Requires-Dist: urllib3<2.0
+Requires-Dist: watchdog>=6.0.0
 Requires-Dist: xarray<2024,>=2023.12.0
 Provides-Extra: classification
 Requires-Dist: keras==3.9.1; extra == 'classification'

pyopia-2.15.0/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.15.0"

{pyopia-2.13.1 → pyopia-2.15.0}/pyopia/cf_metadata.json

@@ -62,4 +62,52 @@
     "standard_name": "timestamp",
     "long_name": "Timestamp of particle observation",
     "calculation_method": "Recorded during particle observation",
-    "pyopia_process_level": 0}}
+    "pyopia_process_level": 0},
+"filename": {
+    "standard_name": "filename",
+    "long_name": "Filename of the raw image",
+    "units": "",
+    "calculation_method": "Recorded during image loading",
+    "pyopia_process_level": 0},
+"particle_count": {
+    "standard_name": "particle_count",
+    "long_name": "Number of particles detected in the image",
+    "units": "1",
+    "calculation_method": "Count of particles from segmentation",
+    "pyopia_process_level": 1},
+"saturation": {
+    "standard_name": "image_saturation",
+    "long_name": "Percentage saturation of the image",
+    "units": "percent",
+    "calculation_method": "Computed as the percentage of the image covered by particles relative to the maximum acceptable coverage",
+    "pyopia_process_level": 1},
+"d50": {
+    "standard_name": "median_particle_diameter",
+    "long_name": "Median particle diameter (D50) from volume distribution",
+    "units": "micrometers",
+    "calculation_method": "Calculated from volume-based particle size distribution",
+    "pyopia_process_level": 2},
+"nc": {
+    "standard_name": "number_concentration",
+    "long_name": "Number concentration of particles",
+    "units": "L^-1",
+    "calculation_method": "Calculated from particle count and sample volume",
+    "pyopia_process_level": 2},
+"vc": {
+    "standard_name": "volume_concentration",
+    "long_name": "Volume concentration of particles",
+    "units": "uL L^-1",
+    "calculation_method": "Calculated from particle volumes and sample volume",
+    "pyopia_process_level": 2},
+"sample_volume": {
+    "standard_name": "sample_volume",
+    "long_name": "Volume of water sampled",
+    "units": "L",
+    "calculation_method": "Calculated from image area, pixel size, and path length",
+    "pyopia_process_level": 2},
+"junge": {
+    "standard_name": "junge_parameter",
+    "long_name": "Junge parameter (slope of particle size number distribution)",
+    "units": "1",
+    "calculation_method": "Fitted slope of particle size number distribution between 150-350 µm",
+    "pyopia_process_level": 2}}

{pyopia-2.13.1 → pyopia-2.15.0}/pyopia/cli.py

@@ -42,6 +42,7 @@ import pyopia.auxillarydata
 import pyopia.exampledata
 import pyopia.metadata
 import pyopia.dataexport.ecotaxa
+import pyopia.realtime
 
 app = typer.Typer()
 
@@ -350,6 +351,29 @@ def process(config_filename: str, num_chunks: int = 1, strategy: str = "block"):
         progress.console.print(f"[blue]PROCESSING COMPLETED IN {time_total}")
 
 
+@app.command()
+def process_realtime(config_filename: str, watch_folder: str = None):
+    """Run a PyOPIA processing pipeline in realtime by watching a folder.
+
+    Parameters
+    ----------
+    config_filename : str
+        Config filename
+
+    watch_folder : str, optional
+        Folder to monitor. If not provided, inferred from `general.raw_files` in config.
+
+    Notes
+    -----
+    - Single-core only: files are processed sequentially by a single worker thread.
+    - Uses `watchdog` moved events only (rename into place) to avoid half-written files.
+    """
+    # Load config and setup logging
+    pipeline_config = pyopia.io.load_toml(config_filename)
+    setup_logging(pipeline_config)
+    pyopia.realtime.run_realtime(pipeline_config, watch_folder=watch_folder)
+
+
 @app.command()
 def merge_mfdata(
     path_to_data: str,

{pyopia-2.13.1 → pyopia-2.15.0}/pyopia/io.py

@@ -128,6 +128,9 @@ def write_stats(
             # Add auxillary data to ximage_stats
             ximage_stats = auxillary_data.add_auxillary_data_to_xstats(ximage_stats)
 
+            # Add CF-compliant attributes
+            ximage_stats = add_cf_attributes(ximage_stats)
+
             encoding_imagestats = setup_xstats_encoding(ximage_stats)
             ximage_stats.to_netcdf(
                 datafilename + "-STATS.nc",

{pyopia-2.13.1 → pyopia-2.15.0}/pyopia/process.py

@@ -639,7 +639,13 @@ class CalculateImageStats():
 
         path_length = getattr(data['settings']['general'], 'path_length', 40)
         if path_length is not None:
-            nc_vc = pyopia.statistics.nc_vc_from_stats(data['stats'], pixel_size, path_length)
+            # Get image dimensions from the corrected image
+            image = data.get('imraw')
+            if image is not None:
+                imy, imx = image.shape[:2]
+            else:
+                imx, imy = 2048, 2448  # fallback defaults
+            nc_vc = pyopia.statistics.nc_vc_from_stats(data['stats'], pixel_size, path_length, imx=imx, imy=imy)
             for k, v in zip(['nc', 'vc', 'sample_volume', 'junge'], nc_vc):
                 data['image_stats'].loc[data['timestamp'], k] = v
 

pyopia-2.15.0/pyopia/realtime.py

@@ -0,0 +1,229 @@
+"""Utilities for running PyOPIA pipelines in realtime mode."""
+
+import fnmatch
+import logging
+import pathlib
+import queue
+import threading
+import time
+
+import pandas as pd
+from rich import print as rich_print
+from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
+from watchdog.events import FileSystemEventHandler
+from watchdog.observers import Observer
+
+import pyopia.pipeline
+
+
+def _resolve_watch_settings(raw_files_pattern: str, watch_folder: str | None) -> tuple[str, str]:
+    raw_path = pathlib.Path(raw_files_pattern)
+    file_pattern = raw_path.name if raw_path.name else "*"
+    inferred_watch = str(raw_path.parent) if str(raw_path.parent) != "." else "."
+
+    if watch_folder is None:
+        watch_folder = inferred_watch
+
+    return watch_folder, file_pattern
+
+
+def _enqueue_file_if_new(
+    file_path: pathlib.Path,
+    file_queue: queue.Queue,
+    file_pattern: str,
+    seen_files: set[str],
+    seen_lock: threading.Lock,
+) -> bool:
+    if not file_path.exists() or file_path.is_dir():
+        return False
+
+    if not fnmatch.fnmatch(file_path.name, file_pattern):
+        return False
+
+    file_key = file_path.as_posix()
+    with seen_lock:
+        if file_key in seen_files:
+            return False
+        seen_files.add(file_key)
+
+    file_queue.put(file_path)
+    return True
+
+
+def _enqueue_existing_files(
+    watch_folder: str,
+    file_pattern: str,
+    file_queue: queue.Queue,
+    seen_files: set[str],
+    seen_lock: threading.Lock,
+    logger: logging.Logger,
+):
+    for file_path in sorted(pathlib.Path(watch_folder).glob(file_pattern)):
+        if _enqueue_file_if_new(file_path, file_queue, file_pattern, seen_files, seen_lock):
+            logger.info(f"Existing file queued: {file_path}")
+
+
+def _build_event_handler(
+    file_queue: queue.Queue,
+    file_pattern: str,
+    seen_files: set[str],
+    seen_lock: threading.Lock,
+    logger: logging.Logger,
+):
+    class NewFileHandler(FileSystemEventHandler):
+        def _handle_path(self, path):
+            try:
+                file_path = pathlib.Path(path)
+                if _enqueue_file_if_new(
+                    file_path,
+                    file_queue,
+                    file_pattern,
+                    seen_files,
+                    seen_lock,
+                ):
+                    logger.info(f"New file detected: {file_path}")
+            except Exception:
+                logger.exception("Error handling filesystem event")
+
+        def on_moved(self, event):
+            destination = getattr(event, "dest_path", None)
+            if destination:
+                self._handle_path(destination)
+
+    return NewFileHandler()
+
+
+def _worker_loop(
+    stop_event: threading.Event,
+    file_queue: queue.Queue,
+    processing_pipeline: pyopia.pipeline.Pipeline,
+    logger: logging.Logger,
+    runtime_state: dict,
+    state_lock: threading.Lock,
+):
+    while not stop_event.is_set():
+        try:
+            filepath = file_queue.get(timeout=1)
+        except queue.Empty:
+            continue
+
+        try:
+            with state_lock:
+                runtime_state["current_file"] = filepath.name
+
+            start = time.time()
+            logger.info(f"Starting processing: {filepath.name}")
+            processing_pipeline.run(filepath.as_posix())
+            elapsed = time.time() - start
+            logger.info(f"Completed {filepath.name} in {elapsed:.1f}s")
+            with state_lock:
+                runtime_state["processed_count"] += 1
+        except Exception as exc:
+            logger.exception(f"Error processing {filepath}: {exc}")
+        finally:
+            with state_lock:
+                runtime_state["current_file"] = "idle"
+            file_queue.task_done()
+
+
+def run_realtime(pipeline_config: dict, watch_folder: str | None = None):
+    """Run a PyOPIA processing pipeline in realtime by watching a folder.
+
+    Parameters
+    ----------
+    pipeline_config : dict
+        Loaded PyOPIA pipeline config.
+    watch_folder : str, optional
+        Folder to monitor. If not provided, inferred from ``general.raw_files``.
+    """
+    logger = logging.getLogger("rich")
+    logger.info(f"PyOPIA realtime process started {pd.Timestamp.now()}")
+    t1 = time.time()
+
+    raw_files_pattern = pipeline_config["general"].get("raw_files", "*")
+    watch_folder, file_pattern = _resolve_watch_settings(raw_files_pattern, watch_folder)
+
+    processing_pipeline = pyopia.pipeline.Pipeline(pipeline_config)
+
+    file_queue = queue.Queue()
+    stop_event = threading.Event()
+    seen_files: set[str] = set()
+    seen_lock = threading.Lock()
+    runtime_state = {"processed_count": 0, "current_file": "idle"}
+    state_lock = threading.Lock()
+
+    observer = Observer()
+    handler = _build_event_handler(
+        file_queue,
+        file_pattern,
+        seen_files,
+        seen_lock,
+        logger,
+    )
+    observer.schedule(handler, path=watch_folder, recursive=False)
+    observer.start()
+
+    _enqueue_existing_files(
+        watch_folder,
+        file_pattern,
+        file_queue,
+        seen_files,
+        seen_lock,
+        logger,
+    )
+
+    worker_thread = threading.Thread(
+        target=_worker_loop,
+        args=(
+            stop_event,
+            file_queue,
+            processing_pipeline,
+            logger,
+            runtime_state,
+            state_lock,
+        ),
+        daemon=True,
+    )
+    worker_thread.start()
+
+    logger.info(
+        f"Watching folder {watch_folder} for moved files matching '{file_pattern}'"
+    )
+    rich_print(
+        f"[blue]Realtime started. Watching {watch_folder} for moved files matching '{file_pattern}'"
+    )
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("{task.description}"),
+        BarColumn(),
+        TimeElapsedColumn(),
+        transient=True,
+    ) as progress:
+        task_id = progress.add_task("[blue]Realtime active", total=None)
+
+        try:
+            while True:
+                with state_lock:
+                    processed_count = runtime_state["processed_count"]
+                    current_file = runtime_state["current_file"]
+                progress.update(
+                    task_id,
+                    description=(
+                        "[blue]Realtime active"
+                        f" | processed: {processed_count}"
+                        f" | queued: {file_queue.qsize()}"
+                        f" | current: {current_file}"
+                    ),
+                )
+                time.sleep(1)
+        except KeyboardInterrupt:
+            logger.info("Shutdown requested, stopping observer and worker...")
+        finally:
+            stop_event.set()
+            observer.stop()
+            observer.join()
+            worker_thread.join(timeout=5)
+
+        time_total = pd.to_timedelta(time.time() - t1, "seconds")
+        progress.console.print(f"[blue]REALTIME PROCESSING STOPPED AFTER {time_total}")

pyopia-2.15.0/pyopia/tests/test_realtime.py

@@ -0,0 +1,331 @@
+import logging
+import queue
+import threading
+from pathlib import Path
+
+import pyopia.realtime
+
+
+class _DummyObserver:
+    def __init__(self):
+        self.scheduled_path = None
+        self.recursive = None
+        self.started = False
+        self.stopped = False
+        self.joined = False
+
+    def schedule(self, _handler, path, recursive):
+        self.scheduled_path = path
+        self.recursive = recursive
+
+    def start(self):
+        self.started = True
+
+    def stop(self):
+        self.stopped = True
+
+    def join(self):
+        self.joined = True
+
+
+class _DummyThread:
+    def __init__(self, target, args, daemon):
+        self.target = target
+        self.args = args
+        self.daemon = daemon
+        self.started = False
+        self.join_timeout = None
+
+    def start(self):
+        self.started = True
+
+    def join(self, timeout=None):
+        self.join_timeout = timeout
+
+
+class _DummyProgress:
+    def __init__(self, *args, transient=True):
+        self.args = args
+        self.transient = transient
+        self.console = self
+        self.task_id = 1
+        self.descriptions = []
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        return False
+
+    def print(self, *_args, **_kwargs):
+        return None
+
+    def add_task(self, description, total=None):
+        self.descriptions.append(description)
+        return self.task_id
+
+    def update(self, _task_id, description=None):
+        if description is not None:
+            self.descriptions.append(description)
+
+
+class _DummyPipeline:
+    def __init__(self, config):
+        self.config = config
+
+
+def _patch_realtime_runtime(monkeypatch, created):
+    def observer_factory():
+        created["observer"] = _DummyObserver()
+        return created["observer"]
+
+    def thread_factory(target, args, daemon):
+        created["thread"] = _DummyThread(target, args, daemon)
+        return created["thread"]
+
+    def raise_keyboard_interrupt(_seconds):
+        raise KeyboardInterrupt()
+
+    monkeypatch.setattr(pyopia.realtime, "Observer", observer_factory)
+    monkeypatch.setattr(pyopia.realtime.threading, "Thread", thread_factory)
+    monkeypatch.setattr(pyopia.realtime, "Progress", _DummyProgress)
+    monkeypatch.setattr(pyopia.realtime.pyopia.pipeline, "Pipeline", _DummyPipeline)
+    monkeypatch.setattr(pyopia.realtime.time, "sleep", raise_keyboard_interrupt)
+
+
+def test_resolve_watch_settings_infers_from_raw_pattern(tmp_path: Path):
+    images_dir = tmp_path / "images"
+    images_dir.mkdir()
+
+    raw_pattern = str(images_dir / "*.silc")
+    watch_folder, file_pattern = pyopia.realtime._resolve_watch_settings(raw_pattern, None)
+
+    assert watch_folder == str(images_dir)
+    assert file_pattern == "*.silc"
+
+
+def test_resolve_watch_settings_prefers_explicit_watch_folder(tmp_path: Path):
+    images_dir = tmp_path / "images"
+    images_dir.mkdir()
+
+    raw_pattern = str(images_dir / "*.silc")
+    watch_folder, file_pattern = pyopia.realtime._resolve_watch_settings(
+        raw_pattern,
+        watch_folder="/custom/watch",
+    )
+
+    assert watch_folder == "/custom/watch"
+    assert file_pattern == "*.silc"
+
+
+def test_event_handler_enqueues_only_matching_moved_files(tmp_path: Path):
+    file_queue = queue.Queue()
+    logger = logging.getLogger("test")
+    seen_files = set()
+    seen_lock = threading.Lock()
+    handler = pyopia.realtime._build_event_handler(
+        file_queue,
+        "*.silc",
+        seen_files,
+        seen_lock,
+        logger,
+    )
+
+    matched = tmp_path / "image.silc"
+    matched.write_text("ok")
+    ignored = tmp_path / "other.txt"
+    ignored.write_text("skip")
+
+    moved_event_match = type("Event", (), {"dest_path": str(matched)})()
+    moved_event_no_match = type("Event", (), {"dest_path": str(ignored)})()
+
+    handler.on_moved(moved_event_match)
+    handler.on_moved(moved_event_no_match)
+
+    queued = file_queue.get_nowait()
+    assert queued == matched
+    assert file_queue.empty()
+
+
+def test_event_handler_deduplicates_same_moved_file(tmp_path: Path):
+    file_queue = queue.Queue()
+    logger = logging.getLogger("test")
+    seen_files = set()
+    seen_lock = threading.Lock()
+    handler = pyopia.realtime._build_event_handler(
+        file_queue,
+        "*.silc",
+        seen_files,
+        seen_lock,
+        logger,
+    )
+
+    matched = tmp_path / "image.silc"
+    matched.write_text("ok")
+    moved_event = type("Event", (), {"dest_path": str(matched)})()
+
+    handler.on_moved(moved_event)
+    handler.on_moved(moved_event)
+
+    queued = file_queue.get_nowait()
+    assert queued == matched
+    assert file_queue.empty()
+
+
+def test_enqueue_existing_files_matches_pattern_and_deduplicates(tmp_path: Path):
+    file_queue = queue.Queue()
+    seen_files = set()
+    seen_lock = threading.Lock()
+    logger = logging.getLogger("test")
+
+    matched = tmp_path / "a.silc"
+    matched.write_text("ok")
+    ignored = tmp_path / "b.txt"
+    ignored.write_text("skip")
+
+    pyopia.realtime._enqueue_existing_files(
+        str(tmp_path),
+        "*.silc",
+        file_queue,
+        seen_files,
+        seen_lock,
+        logger,
+    )
+    pyopia.realtime._enqueue_existing_files(
+        str(tmp_path),
+        "*.silc",
+        file_queue,
+        seen_files,
+        seen_lock,
+        logger,
+    )
+
+    queued = file_queue.get_nowait()
+    assert queued == matched
+    assert file_queue.empty()
+
+
+def test_integration_existing_then_moved_files_processed_once(tmp_path: Path):
+    file_queue = queue.Queue()
+    seen_files = set()
+    seen_lock = threading.Lock()
+    logger = logging.getLogger("test")
+
+    existing = tmp_path / "existing.silc"
+    existing.write_text("old")
+    moved_in = tmp_path / "moved.silc"
+    moved_in.write_text("new")
+
+    pyopia.realtime._enqueue_existing_files(
+        str(tmp_path),
+        "*.silc",
+        file_queue,
+        seen_files,
+        seen_lock,
+        logger,
+    )
+
+    handler = pyopia.realtime._build_event_handler(
+        file_queue,
+        "*.silc",
+        seen_files,
+        seen_lock,
+        logger,
+    )
+    handler.on_moved(type("Event", (), {"dest_path": str(moved_in)})())
+    handler.on_moved(type("Event", (), {"dest_path": str(existing)})())
+
+    stop_event = threading.Event()
+    processed = []
+    runtime_state = {"processed_count": 0, "current_file": "idle"}
+    state_lock = threading.Lock()
+
+    class DummyPipeline:
+        def run(self, filename):
+            processed.append(filename)
+            if len(processed) == 2:
+                stop_event.set()
+
+    pyopia.realtime._worker_loop(
+        stop_event=stop_event,
+        file_queue=file_queue,
+        processing_pipeline=DummyPipeline(),
+        logger=logger,
+        runtime_state=runtime_state,
+        state_lock=state_lock,
+    )
+
+    assert set(processed) == {existing.as_posix(), moved_in.as_posix()}
+    assert runtime_state["processed_count"] == 2
+
+
+def test_worker_loop_processes_one_file_and_uses_posix_path(tmp_path: Path):
+    class DummyPipeline:
+        def __init__(self):
+            self.processed = []
+
+        def run(self, filename):
+            self.processed.append(filename)
+            stop_event.set()
+
+    image_file = tmp_path / "sample.silc"
+    image_file.write_text("content")
+
+    stop_event = threading.Event()
+    file_queue = queue.Queue()
+    file_queue.put(image_file)
+    pipeline = DummyPipeline()
+    runtime_state = {"processed_count": 0, "current_file": "idle"}
+    state_lock = threading.Lock()
+
+    pyopia.realtime._worker_loop(
+        stop_event=stop_event,
+        file_queue=file_queue,
+        processing_pipeline=pipeline,
+        logger=logging.getLogger("test"),
+        runtime_state=runtime_state,
+        state_lock=state_lock,
+    )
+
+    assert pipeline.processed == [image_file.as_posix()]
+    assert runtime_state["processed_count"] == 1
+    assert runtime_state["current_file"] == "idle"
+
+
+def test_run_realtime_schedules_observer_and_stops_on_keyboard_interrupt(monkeypatch, tmp_path: Path):
+    images_dir = tmp_path / "incoming"
+    images_dir.mkdir()
+    pipeline_config = {
+        "general": {
+            "raw_files": str(images_dir / "*.silc"),
+        },
+        "steps": {},
+    }
+
+    created = {}
+    progress_calls = {"added": 0, "updated": 0}
+
+    class TrackingProgress(_DummyProgress):
+        def add_task(self, description, total=None):
+            progress_calls["added"] += 1
+            return super().add_task(description, total)
+
+        def update(self, _task_id, description=None):
+            progress_calls["updated"] += 1
+            return super().update(_task_id, description)
+
+    _patch_realtime_runtime(monkeypatch, created)
+    monkeypatch.setattr(pyopia.realtime, "Progress", TrackingProgress)
+
+    pyopia.realtime.run_realtime(pipeline_config)
+
+    assert created["observer"].scheduled_path == str(images_dir)
+    assert created["observer"].recursive is False
+    assert created["observer"].started is True
+    assert created["observer"].stopped is True
+    assert created["observer"].joined is True
+    assert created["thread"].started is True
+    assert created["thread"].daemon is True
+    assert created["thread"].join_timeout == 5
+    assert progress_calls["added"] == 1
+    assert progress_calls["updated"] >= 1

{pyopia-2.13.1 → pyopia-2.15.0}/pyproject.toml

@@ -6,7 +6,7 @@ authors = [
     { name = "Emlyn Davies", email = "emlyn.davies@sintef.no" },
     { name = "Alex Nimmo Smith@plymouth.ac.uk", email = "alex.nimmo.smith@plymouth.ac.uk" },
 ]
-requires-python = "~=3.12"
+requires-python = "~=3.12.0"
 readme = "README.md"
 keywords = [
     "Ocean",
@@ -50,6 +50,7 @@ dependencies = [
     "click<8.2.0",
     "seaborn>=0.13.2",
     "pydantic>=2.11.7",
+    "watchdog>=6.0.0",
 ]
 
 [project.optional-dependencies]

pyopia-2.13.1/pyopia/__init__.py

@@ -1 +0,0 @@
-__version__ = "2.13.1"