PyOPIA 2.14.0__tar.gz → 2.16.0__tar.gz

{pyopia-2.14.0 → pyopia-2.16.0}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: PyOPIA
-Version: 2.14.0
+Version: 2.16.0
 Summary: A Python Ocean Particle Image Analysis toolbox.
 Project-URL: Repository, https://github.com/sintef/pyopia
 Project-URL: Documentation, https://pyopia.readthedocs.io
 Author-email: Emlyn Davies <emlyn.davies@sintef.no>, "Alex Nimmo Smith@plymouth.ac.uk" <alex.nimmo.smith@plymouth.ac.uk>
 License-File: LICENSE
 Keywords: Imaging,Measurement,Ocean,Particles,Size distribution
-Requires-Python: ~=3.12
+Requires-Python: ~=3.12.0
 Requires-Dist: click<8.2.0
 Requires-Dist: cmocean<4,>=3.0.3
 Requires-Dist: dask>=2024.8.1
@@ -41,10 +41,13 @@ Requires-Dist: toml<0.11,>=0.10.2
 Requires-Dist: tqdm<5,>=4.66.1
 Requires-Dist: typer[all]<0.10,>=0.9.0
 Requires-Dist: urllib3<2.0
+Requires-Dist: watchdog>=6.0.0
 Requires-Dist: xarray<2024,>=2023.12.0
 Provides-Extra: classification
 Requires-Dist: keras==3.9.1; extra == 'classification'
 Requires-Dist: tensorflow>=2.19.0; extra == 'classification'
+Provides-Extra: classification-torch
+Requires-Dist: torch>=2.1.0; extra == 'classification-torch'
 Description-Content-Type: text/markdown
 
 PyOPIA

pyopia-2.16.0/pyopia/__init__.py

@@ -0,0 +1 @@
+__version__ = "2.16.0"

pyopia-2.16.0/pyopia/classify_torch.py

@@ -0,0 +1,276 @@
+"""
+Classifier for PyOPIA using PyTorch TorchScript models.
+
+Loads a TorchScript archive (``.pt``) containing a traced model and a
+``metadata.json`` with class labels, input dimensions, normalization
+stats, and optional preprocessing config. Compatible with models
+exported by ``pyopia-train export-pyopia-torch`` and by
+``pyopia.train_torch.save_pytorch_model``.
+
+The TorchScript archive is self-contained, so only extra dependency
+is PyTorch (optional dependency).
+"""
+
+import hashlib
+import json
+import logging
+
+import numpy as np
+from skimage.exposure import rescale_intensity
+from skimage.transform import resize
+
+logger = logging.getLogger()
+
+try:
+    import torch
+except ImportError:
+    info_str = "ERROR: Could not import PyTorch. classify_torch will not work"
+    info_str += " until you install torch.\n"
+    info_str += "Use: uv sync --extra classification-torch\n"
+    raise ImportError(info_str)
+
+
+def _get_device():
+    """Select the best available device: CUDA > MPS > CPU."""
+    if torch.cuda.is_available():
+        return torch.device("cuda")
+    elif torch.backends.mps.is_available():
+        return torch.device("mps")
+    return torch.device("cpu")
+
+
+def save_pytorch_model(
+    model, class_labels, img_height, img_width, output_path, preprocess=None
+):
+    """Save a PyTorch model with metadata as a TorchScript archive.
+
+    The model is expected to accept inputs in the [0, 255] range and
+    bake any per-channel normalisation into its own forward pass (see
+    notebook export examples).  This mirrors the convention in
+    :mod:`pyopia.classify` where the TF model has a Rescaling layer
+    as its first op.
+
+    Parameters
+    ----------
+    model : torch.nn.Module
+        Trained PyTorch model.  Must accept ``(B, 3, H, W)`` floats in
+        ``[0, 255]`` and produce class logits.
+    class_labels : list of str
+        Class name strings.
+    img_height : int
+        Expected input image height.
+    img_width : int
+        Expected input image width.
+    output_path : str
+        Path to save the .pt file.
+    preprocess : dict or None
+        Optional preprocessing config (e.g.
+        ``{"type": "percentile", "lower_pct": 1.0, "upper_pct": 99.0}``).
+        Stored in metadata so inference applies matching preprocessing.
+    """
+    model.eval()
+    try:
+        scripted = torch.jit.script(model)
+    except Exception:
+        example_input = torch.randn(1, 3, int(img_height), int(img_width)) * 255
+        scripted = torch.jit.trace(model, example_input)
+
+    meta = {
+        "class_labels": list(class_labels),
+        "img_height": int(img_height),
+        "img_width": int(img_width),
+    }
+    if preprocess is not None:
+        meta["preprocess"] = preprocess
+
+    torch.jit.save(
+        scripted, output_path, _extra_files={"metadata.json": json.dumps(meta)}
+    )
+
+
+class Classify:
+    """PyTorch TorchScript classifier for the PyOPIA pipeline.
+
+    Drop-in replacement for :class:`pyopia.classify.Classify`.
+    Loads a TorchScript ``.pt`` archive with metadata.
+
+    Supports models from:
+    - ``pyopia-train export-pyopia-torch``
+
+    Parameters
+    ----------
+    model_path : str
+        Path to a ``.pt`` TorchScript archive.
+    normalize_intensity : bool
+        Scale input ROI intensity to [0-1] before classification.
+    correct_whitebalance : bool
+        Per-channel histogram correction before classification.
+
+    Example
+    -------
+    .. code-block:: python
+
+        cl = Classify(model_path='model.pt')
+        prediction = cl.proc_predict(roi)
+
+    TOML config:
+
+    .. code-block:: toml
+
+        [steps.classifier]
+        pipeline_class = 'pyopia.classify_torch.Classify'
+        model_path = 'model.pt'
+    """
+
+    def __init__(
+        self,
+        model_path=None,
+        normalize_intensity=True,
+        correct_whitebalance=False,
+    ):
+        self.model_path = model_path
+        self.correct_whitebalance = correct_whitebalance
+        self.normalize_intensity = normalize_intensity
+        self.load_model()
+
+    def __call__(self):
+        return self
+
+    def load_model(self):
+        """Load a TorchScript model and metadata from a .pt archive."""
+        model_path = self.model_path
+
+        extra_files = {"metadata.json": ""}
+        self.model = torch.jit.load(
+            model_path, map_location="cpu", _extra_files=extra_files
+        )
+        self.model.eval()
+
+        metadata = json.loads(extra_files["metadata.json"])
+        self.class_labels = metadata["class_labels"]
+        self.img_height = metadata["img_height"]
+        self.img_width = metadata["img_width"]
+        self.preprocess = metadata.get("preprocess", None)
+
+        self.device = _get_device()
+        self.model = self.model.to(self.device)
+        logger.info(f"PyTorch classifier loaded on device: {self.device}")
+
+        with open(model_path, "rb") as f:
+            digest = hashlib.file_digest(f, "sha256")
+        self.model_hash = digest.hexdigest()
+
+        logger.info(self.class_labels)
+
+    def preprocessing(self, img_input):
+        """Preprocess an ROI for prediction.
+
+        Parameters
+        ----------
+        img_input : ndarray
+            Particle ROI, float, range 0-1, shape (H, W, 3) or (H, W).
+
+        Returns
+        -------
+        img_preprocessed : ndarray
+            Shape (1, 3, img_height, img_width), float32, ready for the model.
+        """
+        whitebalanced = img_input.astype(np.float64)
+
+        if self.correct_whitebalance:
+            p = 99
+            for c in range(3):
+                whitebalanced[:, :, c] += (p / 100) - np.percentile(
+                    whitebalanced[:, :, c], p
+                )
+            whitebalanced[whitebalanced > 1] = 1
+            whitebalanced[whitebalanced < 0] = 0
+
+        if self.normalize_intensity:
+            whitebalanced = rescale_intensity(whitebalanced)
+
+        # Optional percentile preprocessing (from pyopia-train checkpoints).
+        # Applied after rescale_intensity, before resize, matching the
+        # training pipeline's transform order.
+        if self.preprocess is not None:
+            preprocess_type = self.preprocess.get("type", "none")
+            if preprocess_type == "percentile":
+                lower = float(self.preprocess.get("lower_pct", 1.0))
+                upper = float(self.preprocess.get("upper_pct", 99.0))
+                whitebalanced = self._percentile_normalize(whitebalanced, lower, upper)
+
+        # Convert to [0, 255] floats — matches the convention in
+        # pyopia.classify.preprocessing.  Any per-channel normalisation
+        # is baked into the model itself (see notebook export).
+        img = (whitebalanced * 255).astype(np.float32)
+
+        # Bilinear resize without anti-aliasing — matches the original
+        # pyopia.classify (TF) convention and the notebook training
+        # pipeline (PIL bilinear).
+        img = resize(
+            img,
+            (self.img_height, self.img_width),
+            preserve_range=True,
+            anti_aliasing=False,
+        ).astype(np.float32)
+
+        # HWC → CHW, add batch dim
+        img_preprocessed = np.transpose(img, (2, 0, 1))
+        img_preprocessed = np.expand_dims(img_preprocessed, axis=0)
+        return img_preprocessed
+
+    @staticmethod
+    def _percentile_normalize(img, lower_pct=1.0, upper_pct=99.0):
+        """Robust per-image percentile intensity rescaling.
+
+        Matches pyopia-train's ``PercentileNormalize``: computes a single
+        low/high range from the per-pixel RGB mean, then applies it
+        identically to all channels.
+        """
+        arr = img.astype(np.float64)
+        if arr.ndim == 2:
+            intensity = arr
+        else:
+            intensity = arr.mean(axis=2) if arr.shape[2] > 1 else arr[:, :, 0]
+        lo = float(np.percentile(intensity, lower_pct))
+        hi = float(np.percentile(intensity, upper_pct))
+        if hi <= lo + 1e-6:
+            return np.clip(arr, 0.0, 1.0)
+        arr = (arr - lo) / (hi - lo)
+        return np.clip(arr, 0.0, 1.0)
+
+    def predict(self, img_preprocessed):
+        """Run the model on a preprocessed ROI.
+
+        Parameters
+        ----------
+        img_preprocessed : ndarray
+            Shape (1, 3, H, W), float32.
+
+        Returns
+        -------
+        prediction : ndarray
+            Probability distribution, shape (num_classes,).
+        """
+        with torch.no_grad():
+            tensor = torch.from_numpy(img_preprocessed).to(self.device)
+            output = self.model(tensor)
+            prediction = torch.nn.functional.softmax(output[0], dim=0)
+        return prediction.cpu().numpy()
+
+    def proc_predict(self, img_input):
+        """Preprocess and classify a particle ROI.
+
+        Parameters
+        ----------
+        img_input : ndarray
+            Particle ROI, float, range 0-1.
+
+        Returns
+        -------
+        prediction : ndarray
+            Probability distribution, shape (num_classes,).
+        """
+        img_preprocessed = self.preprocessing(img_input)
+        prediction = self.predict(img_preprocessed)
+        return prediction

{pyopia-2.14.0 → pyopia-2.16.0}/pyopia/cli.py

@@ -42,6 +42,7 @@ import pyopia.auxillarydata
 import pyopia.exampledata
 import pyopia.metadata
 import pyopia.dataexport.ecotaxa
+import pyopia.realtime
 
 app = typer.Typer()
 
@@ -350,6 +351,29 @@ def process(config_filename: str, num_chunks: int = 1, strategy: str = "block"):
         progress.console.print(f"[blue]PROCESSING COMPLETED IN {time_total}")
 
 
+@app.command()
+def process_realtime(config_filename: str, watch_folder: str = None):
+    """Run a PyOPIA processing pipeline in realtime by watching a folder.
+
+    Parameters
+    ----------
+    config_filename : str
+        Config filename
+
+    watch_folder : str, optional
+        Folder to monitor. If not provided, inferred from `general.raw_files` in config.
+
+    Notes
+    -----
+    - Single-core only: files are processed sequentially by a single worker thread.
+    - Uses `watchdog` moved events only (rename into place) to avoid half-written files.
+    """
+    # Load config and setup logging
+    pipeline_config = pyopia.io.load_toml(config_filename)
+    setup_logging(pipeline_config)
+    pyopia.realtime.run_realtime(pipeline_config, watch_folder=watch_folder)
+
+
 @app.command()
 def merge_mfdata(
     path_to_data: str,

{pyopia-2.14.0 → pyopia-2.16.0}/pyopia/pipeline.py

@@ -139,7 +139,6 @@ class Pipeline():
             Name of the step defined in the settings
         '''
         if stepname == 'classifier':
-            import pyopia.classify # noqa: E(F410)
             callobj = self.step_callobj(stepname)
             self.data['cl'] = callobj()
         else:
@@ -160,10 +159,18 @@ class Pipeline():
             callable object for use in run_step()
         '''
 
+        import importlib
+
         pipeline_class = self.settings['steps'][stepname]['pipeline_class']
         classname = pipeline_class.split('.')[-1]
         modulename = pipeline_class.replace(classname, '')[:-1]
 
+        # Import the module on demand rather than relying on sys.modules.
+        # This allows pipeline steps to reference any importable module
+        # (e.g. pyopia.classify_torch) without needing a hardcoded import
+        # in cli.py or run_step().
+        importlib.import_module(modulename)
+
         keys = [k for k in self.settings['steps'][stepname] if k != 'pipeline_class']
 
         arguments = dict()

pyopia-2.16.0/pyopia/realtime.py

@@ -0,0 +1,229 @@
+"""Utilities for running PyOPIA pipelines in realtime mode."""
+
+import fnmatch
+import logging
+import pathlib
+import queue
+import threading
+import time
+
+import pandas as pd
+from rich import print as rich_print
+from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeElapsedColumn
+from watchdog.events import FileSystemEventHandler
+from watchdog.observers import Observer
+
+import pyopia.pipeline
+
+
+def _resolve_watch_settings(raw_files_pattern: str, watch_folder: str | None) -> tuple[str, str]:
+    raw_path = pathlib.Path(raw_files_pattern)
+    file_pattern = raw_path.name if raw_path.name else "*"
+    inferred_watch = str(raw_path.parent) if str(raw_path.parent) != "." else "."
+
+    if watch_folder is None:
+        watch_folder = inferred_watch
+
+    return watch_folder, file_pattern
+
+
+def _enqueue_file_if_new(
+    file_path: pathlib.Path,
+    file_queue: queue.Queue,
+    file_pattern: str,
+    seen_files: set[str],
+    seen_lock: threading.Lock,
+) -> bool:
+    if not file_path.exists() or file_path.is_dir():
+        return False
+
+    if not fnmatch.fnmatch(file_path.name, file_pattern):
+        return False
+
+    file_key = file_path.as_posix()
+    with seen_lock:
+        if file_key in seen_files:
+            return False
+        seen_files.add(file_key)
+
+    file_queue.put(file_path)
+    return True
+
+
+def _enqueue_existing_files(
+    watch_folder: str,
+    file_pattern: str,
+    file_queue: queue.Queue,
+    seen_files: set[str],
+    seen_lock: threading.Lock,
+    logger: logging.Logger,
+):
+    for file_path in sorted(pathlib.Path(watch_folder).glob(file_pattern)):
+        if _enqueue_file_if_new(file_path, file_queue, file_pattern, seen_files, seen_lock):
+            logger.info(f"Existing file queued: {file_path}")
+
+
+def _build_event_handler(
+    file_queue: queue.Queue,
+    file_pattern: str,
+    seen_files: set[str],
+    seen_lock: threading.Lock,
+    logger: logging.Logger,
+):
+    class NewFileHandler(FileSystemEventHandler):
+        def _handle_path(self, path):
+            try:
+                file_path = pathlib.Path(path)
+                if _enqueue_file_if_new(
+                    file_path,
+                    file_queue,
+                    file_pattern,
+                    seen_files,
+                    seen_lock,
+                ):
+                    logger.info(f"New file detected: {file_path}")
+            except Exception:
+                logger.exception("Error handling filesystem event")
+
+        def on_moved(self, event):
+            destination = getattr(event, "dest_path", None)
+            if destination:
+                self._handle_path(destination)
+
+    return NewFileHandler()
+
+
+def _worker_loop(
+    stop_event: threading.Event,
+    file_queue: queue.Queue,
+    processing_pipeline: pyopia.pipeline.Pipeline,
+    logger: logging.Logger,
+    runtime_state: dict,
+    state_lock: threading.Lock,
+):
+    while not stop_event.is_set():
+        try:
+            filepath = file_queue.get(timeout=1)
+        except queue.Empty:
+            continue
+
+        try:
+            with state_lock:
+                runtime_state["current_file"] = filepath.name
+
+            start = time.time()
+            logger.info(f"Starting processing: {filepath.name}")
+            processing_pipeline.run(filepath.as_posix())
+            elapsed = time.time() - start
+            logger.info(f"Completed {filepath.name} in {elapsed:.1f}s")
+            with state_lock:
+                runtime_state["processed_count"] += 1
+        except Exception as exc:
+            logger.exception(f"Error processing {filepath}: {exc}")
+        finally:
+            with state_lock:
+                runtime_state["current_file"] = "idle"
+            file_queue.task_done()
+
+
+def run_realtime(pipeline_config: dict, watch_folder: str | None = None):
+    """Run a PyOPIA processing pipeline in realtime by watching a folder.
+
+    Parameters
+    ----------
+    pipeline_config : dict
+        Loaded PyOPIA pipeline config.
+    watch_folder : str, optional
+        Folder to monitor. If not provided, inferred from ``general.raw_files``.
+    """
+    logger = logging.getLogger("rich")
+    logger.info(f"PyOPIA realtime process started {pd.Timestamp.now()}")
+    t1 = time.time()
+
+    raw_files_pattern = pipeline_config["general"].get("raw_files", "*")
+    watch_folder, file_pattern = _resolve_watch_settings(raw_files_pattern, watch_folder)
+
+    processing_pipeline = pyopia.pipeline.Pipeline(pipeline_config)
+
+    file_queue = queue.Queue()
+    stop_event = threading.Event()
+    seen_files: set[str] = set()
+    seen_lock = threading.Lock()
+    runtime_state = {"processed_count": 0, "current_file": "idle"}
+    state_lock = threading.Lock()
+
+    observer = Observer()
+    handler = _build_event_handler(
+        file_queue,
+        file_pattern,
+        seen_files,
+        seen_lock,
+        logger,
+    )
+    observer.schedule(handler, path=watch_folder, recursive=False)
+    observer.start()
+
+    _enqueue_existing_files(
+        watch_folder,
+        file_pattern,
+        file_queue,
+        seen_files,
+        seen_lock,
+        logger,
+    )
+
+    worker_thread = threading.Thread(
+        target=_worker_loop,
+        args=(
+            stop_event,
+            file_queue,
+            processing_pipeline,
+            logger,
+            runtime_state,
+            state_lock,
+        ),
+        daemon=True,
+    )
+    worker_thread.start()
+
+    logger.info(
+        f"Watching folder {watch_folder} for moved files matching '{file_pattern}'"
+    )
+    rich_print(
+        f"[blue]Realtime started. Watching {watch_folder} for moved files matching '{file_pattern}'"
+    )
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("{task.description}"),
+        BarColumn(),
+        TimeElapsedColumn(),
+        transient=True,
+    ) as progress:
+        task_id = progress.add_task("[blue]Realtime active", total=None)
+
+        try:
+            while True:
+                with state_lock:
+                    processed_count = runtime_state["processed_count"]
+                    current_file = runtime_state["current_file"]
+                progress.update(
+                    task_id,
+                    description=(
+                        "[blue]Realtime active"
+                        f" | processed: {processed_count}"
+                        f" | queued: {file_queue.qsize()}"
+                        f" | current: {current_file}"
+                    ),
+                )
+                time.sleep(1)
+        except KeyboardInterrupt:
+            logger.info("Shutdown requested, stopping observer and worker...")
+        finally:
+            stop_event.set()
+            observer.stop()
+            observer.join()
+            worker_thread.join(timeout=5)
+
+        time_total = pd.to_timedelta(time.time() - t1, "seconds")
+        progress.console.print(f"[blue]REALTIME PROCESSING STOPPED AFTER {time_total}")

pyopia-2.16.0/pyopia/tests/test_realtime.py

@@ -0,0 +1,331 @@
+import logging
+import queue
+import threading
+from pathlib import Path
+
+import pyopia.realtime
+
+
+class _DummyObserver:
+    def __init__(self):
+        self.scheduled_path = None
+        self.recursive = None
+        self.started = False
+        self.stopped = False
+        self.joined = False
+
+    def schedule(self, _handler, path, recursive):
+        self.scheduled_path = path
+        self.recursive = recursive
+
+    def start(self):
+        self.started = True
+
+    def stop(self):
+        self.stopped = True
+
+    def join(self):
+        self.joined = True
+
+
+class _DummyThread:
+    def __init__(self, target, args, daemon):
+        self.target = target
+        self.args = args
+        self.daemon = daemon
+        self.started = False
+        self.join_timeout = None
+
+    def start(self):
+        self.started = True
+
+    def join(self, timeout=None):
+        self.join_timeout = timeout
+
+
+class _DummyProgress:
+    def __init__(self, *args, transient=True):
+        self.args = args
+        self.transient = transient
+        self.console = self
+        self.task_id = 1
+        self.descriptions = []
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        return False
+
+    def print(self, *_args, **_kwargs):
+        return None
+
+    def add_task(self, description, total=None):
+        self.descriptions.append(description)
+        return self.task_id
+
+    def update(self, _task_id, description=None):
+        if description is not None:
+            self.descriptions.append(description)
+
+
+class _DummyPipeline:
+    def __init__(self, config):
+        self.config = config
+
+
+def _patch_realtime_runtime(monkeypatch, created):
+    def observer_factory():
+        created["observer"] = _DummyObserver()
+        return created["observer"]
+
+    def thread_factory(target, args, daemon):
+        created["thread"] = _DummyThread(target, args, daemon)
+        return created["thread"]
+
+    def raise_keyboard_interrupt(_seconds):
+        raise KeyboardInterrupt()
+
+    monkeypatch.setattr(pyopia.realtime, "Observer", observer_factory)
+    monkeypatch.setattr(pyopia.realtime.threading, "Thread", thread_factory)
+    monkeypatch.setattr(pyopia.realtime, "Progress", _DummyProgress)
+    monkeypatch.setattr(pyopia.realtime.pyopia.pipeline, "Pipeline", _DummyPipeline)
+    monkeypatch.setattr(pyopia.realtime.time, "sleep", raise_keyboard_interrupt)
+
+
+def test_resolve_watch_settings_infers_from_raw_pattern(tmp_path: Path):
+    images_dir = tmp_path / "images"
+    images_dir.mkdir()
+
+    raw_pattern = str(images_dir / "*.silc")
+    watch_folder, file_pattern = pyopia.realtime._resolve_watch_settings(raw_pattern, None)
+
+    assert watch_folder == str(images_dir)
+    assert file_pattern == "*.silc"
+
+
+def test_resolve_watch_settings_prefers_explicit_watch_folder(tmp_path: Path):
+    images_dir = tmp_path / "images"
+    images_dir.mkdir()
+
+    raw_pattern = str(images_dir / "*.silc")
+    watch_folder, file_pattern = pyopia.realtime._resolve_watch_settings(
+        raw_pattern,
+        watch_folder="/custom/watch",
+    )
+
+    assert watch_folder == "/custom/watch"
+    assert file_pattern == "*.silc"
+
+
+def test_event_handler_enqueues_only_matching_moved_files(tmp_path: Path):
+    file_queue = queue.Queue()
+    logger = logging.getLogger("test")
+    seen_files = set()
+    seen_lock = threading.Lock()
+    handler = pyopia.realtime._build_event_handler(
+        file_queue,
+        "*.silc",
+        seen_files,
+        seen_lock,
+        logger,
+    )
+
+    matched = tmp_path / "image.silc"
+    matched.write_text("ok")
+    ignored = tmp_path / "other.txt"
+    ignored.write_text("skip")
+
+    moved_event_match = type("Event", (), {"dest_path": str(matched)})()
+    moved_event_no_match = type("Event", (), {"dest_path": str(ignored)})()
+
+    handler.on_moved(moved_event_match)
+    handler.on_moved(moved_event_no_match)
+
+    queued = file_queue.get_nowait()
+    assert queued == matched
+    assert file_queue.empty()
+
+
+def test_event_handler_deduplicates_same_moved_file(tmp_path: Path):
+    file_queue = queue.Queue()
+    logger = logging.getLogger("test")
+    seen_files = set()
+    seen_lock = threading.Lock()
+    handler = pyopia.realtime._build_event_handler(
+        file_queue,
+        "*.silc",
+        seen_files,
+        seen_lock,
+        logger,
+    )
+
+    matched = tmp_path / "image.silc"
+    matched.write_text("ok")
+    moved_event = type("Event", (), {"dest_path": str(matched)})()
+
+    handler.on_moved(moved_event)
+    handler.on_moved(moved_event)
+
+    queued = file_queue.get_nowait()
+    assert queued == matched
+    assert file_queue.empty()
+
+
+def test_enqueue_existing_files_matches_pattern_and_deduplicates(tmp_path: Path):
+    file_queue = queue.Queue()
+    seen_files = set()
+    seen_lock = threading.Lock()
+    logger = logging.getLogger("test")
+
+    matched = tmp_path / "a.silc"
+    matched.write_text("ok")
+    ignored = tmp_path / "b.txt"
+    ignored.write_text("skip")
+
+    pyopia.realtime._enqueue_existing_files(
+        str(tmp_path),
+        "*.silc",
+        file_queue,
+        seen_files,
+        seen_lock,
+        logger,
+    )
+    pyopia.realtime._enqueue_existing_files(
+        str(tmp_path),
+        "*.silc",
+        file_queue,
+        seen_files,
+        seen_lock,
+        logger,
+    )
+
+    queued = file_queue.get_nowait()
+    assert queued == matched
+    assert file_queue.empty()
+
+
+def test_integration_existing_then_moved_files_processed_once(tmp_path: Path):
+    file_queue = queue.Queue()
+    seen_files = set()
+    seen_lock = threading.Lock()
+    logger = logging.getLogger("test")
+
+    existing = tmp_path / "existing.silc"
+    existing.write_text("old")
+    moved_in = tmp_path / "moved.silc"
+    moved_in.write_text("new")
+
+    pyopia.realtime._enqueue_existing_files(
+        str(tmp_path),
+        "*.silc",
+        file_queue,
+        seen_files,
+        seen_lock,
+        logger,
+    )
+
+    handler = pyopia.realtime._build_event_handler(
+        file_queue,
+        "*.silc",
+        seen_files,
+        seen_lock,
+        logger,
+    )
+    handler.on_moved(type("Event", (), {"dest_path": str(moved_in)})())
+    handler.on_moved(type("Event", (), {"dest_path": str(existing)})())
+
+    stop_event = threading.Event()
+    processed = []
+    runtime_state = {"processed_count": 0, "current_file": "idle"}
+    state_lock = threading.Lock()
+
+    class DummyPipeline:
+        def run(self, filename):
+            processed.append(filename)
+            if len(processed) == 2:
+                stop_event.set()
+
+    pyopia.realtime._worker_loop(
+        stop_event=stop_event,
+        file_queue=file_queue,
+        processing_pipeline=DummyPipeline(),
+        logger=logger,
+        runtime_state=runtime_state,
+        state_lock=state_lock,
+    )
+
+    assert set(processed) == {existing.as_posix(), moved_in.as_posix()}
+    assert runtime_state["processed_count"] == 2
+
+
+def test_worker_loop_processes_one_file_and_uses_posix_path(tmp_path: Path):
+    class DummyPipeline:
+        def __init__(self):
+            self.processed = []
+
+        def run(self, filename):
+            self.processed.append(filename)
+            stop_event.set()
+
+    image_file = tmp_path / "sample.silc"
+    image_file.write_text("content")
+
+    stop_event = threading.Event()
+    file_queue = queue.Queue()
+    file_queue.put(image_file)
+    pipeline = DummyPipeline()
+    runtime_state = {"processed_count": 0, "current_file": "idle"}
+    state_lock = threading.Lock()
+
+    pyopia.realtime._worker_loop(
+        stop_event=stop_event,
+        file_queue=file_queue,
+        processing_pipeline=pipeline,
+        logger=logging.getLogger("test"),
+        runtime_state=runtime_state,
+        state_lock=state_lock,
+    )
+
+    assert pipeline.processed == [image_file.as_posix()]
+    assert runtime_state["processed_count"] == 1
+    assert runtime_state["current_file"] == "idle"
+
+
+def test_run_realtime_schedules_observer_and_stops_on_keyboard_interrupt(monkeypatch, tmp_path: Path):
+    images_dir = tmp_path / "incoming"
+    images_dir.mkdir()
+    pipeline_config = {
+        "general": {
+            "raw_files": str(images_dir / "*.silc"),
+        },
+        "steps": {},
+    }
+
+    created = {}
+    progress_calls = {"added": 0, "updated": 0}
+
+    class TrackingProgress(_DummyProgress):
+        def add_task(self, description, total=None):
+            progress_calls["added"] += 1
+            return super().add_task(description, total)
+
+        def update(self, _task_id, description=None):
+            progress_calls["updated"] += 1
+            return super().update(_task_id, description)
+
+    _patch_realtime_runtime(monkeypatch, created)
+    monkeypatch.setattr(pyopia.realtime, "Progress", TrackingProgress)
+
+    pyopia.realtime.run_realtime(pipeline_config)
+
+    assert created["observer"].scheduled_path == str(images_dir)
+    assert created["observer"].recursive is False
+    assert created["observer"].started is True
+    assert created["observer"].stopped is True
+    assert created["observer"].joined is True
+    assert created["thread"].started is True
+    assert created["thread"].daemon is True
+    assert created["thread"].join_timeout == 5
+    assert progress_calls["added"] == 1
+    assert progress_calls["updated"] >= 1

{pyopia-2.14.0 → pyopia-2.16.0}/pyproject.toml

@@ -6,7 +6,7 @@ authors = [
     { name = "Emlyn Davies", email = "emlyn.davies@sintef.no" },
     { name = "Alex Nimmo Smith@plymouth.ac.uk", email = "alex.nimmo.smith@plymouth.ac.uk" },
 ]
-requires-python = "~=3.12"
+requires-python = "~=3.12.0"
 readme = "README.md"
 keywords = [
     "Ocean",
@@ -50,6 +50,7 @@ dependencies = [
     "click<8.2.0",
     "seaborn>=0.13.2",
     "pydantic>=2.11.7",
+    "watchdog>=6.0.0",
 ]
 
 [project.optional-dependencies]
@@ -59,6 +60,10 @@ classification = [
     "keras==3.9.1",
 ]
 
+classification-torch = [
+    "torch>=2.1.0",
+]
+
 [project.urls]
 Repository = "https://github.com/sintef/pyopia"
 Documentation = "https://pyopia.readthedocs.io"

pyopia-2.14.0/pyopia/__init__.py

@@ -1 +0,0 @@
-__version__ = "2.14.0"