wsi-toolbox 0.2.0__py3-none-any.whl

wsi_toolbox/utils/progress.py

@@ -0,0 +1,207 @@
+from typing import Iterable, Optional, TypeVar
+
+from tqdm import tqdm
+
+T = TypeVar("T")
+
+
+class BaseProgress:
+    """Base interface for progress bars"""
+
+    def update(self, n: int = 1) -> None:
+        raise NotImplementedError
+
+    def set_description(self, desc: str = None, refresh: bool = True) -> None:
+        raise NotImplementedError
+
+    def set_postfix(self, ordered_dict=None, **kwargs) -> None:
+        raise NotImplementedError
+
+    def refresh(self) -> None:
+        """Force refresh the progress bar display"""
+        pass
+
+    def close(self) -> None:
+        raise NotImplementedError
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+
+class TqdmProgress(BaseProgress):
+    """tqdm wrapper"""
+
+    def __init__(self, iterable: Optional[Iterable[T]] = None, total: Optional[int] = None, desc: str = "", **kwargs):
+        self._pbar = tqdm(iterable=iterable, total=total, desc=desc, **kwargs)
+
+    def update(self, n: int = 1) -> None:
+        self._pbar.update(n)
+
+    def set_description(self, desc: str = None, refresh: bool = True) -> None:
+        self._pbar.set_description(desc, refresh=refresh)
+
+    def set_postfix(self, ordered_dict=None, **kwargs) -> None:
+        self._pbar.set_postfix(ordered_dict, **kwargs)
+
+    def refresh(self) -> None:
+        self._pbar.refresh()
+
+    def close(self) -> None:
+        self._pbar.close()
+
+    def __iter__(self):
+        return iter(self._pbar)
+
+
+class StreamlitProgress(BaseProgress):
+    """Streamlit progress bar wrapper"""
+
+    def __init__(self, iterable: Optional[Iterable[T]] = None, total: Optional[int] = None, desc: str = "", **kwargs):
+        # import streamlit as st はここに置くこと
+        # import streamlit as st はここに置くこと
+        # import streamlit as st はここに置くこと
+        import streamlit as st  # noqa: E402
+
+        self.iterable = iterable
+        self.total = (
+            total
+            if total is not None
+            else (len(iterable) if iterable is not None and hasattr(iterable, "__len__") else None)
+        )
+        self.desc = desc
+        self.n = 0
+        self.kwargs = kwargs
+
+        # 説明テキスト用のコンテナ
+        self.text_container = st.empty()
+        if desc:
+            self.text_container.text(desc)
+        # プログレスバー
+        self.progress_bar = st.progress(0)
+        # 後置テキスト用のコンテナ
+        self.postfix_container = st.empty()
+
+    def update(self, n: int = 1) -> None:
+        """進捗を更新する"""
+        self.n += n
+        if self.total:
+            self.progress_bar.progress(min(self.n / self.total, 1.0))
+
+    def set_description(self, desc: str = None, refresh: bool = True) -> None:
+        """説明テキストを更新する"""
+        if desc is not None:
+            self.desc = desc
+            # self.text_container.text(desc)
+            self.text_container.markdown(
+                '<p style="font-size:14px; color:gray;">' + desc + "</p>", unsafe_allow_html=True
+            )
+
+    def set_postfix(self, ordered_dict=None, **kwargs) -> None:
+        """後置テキストを設定する"""
+        # ordered_dictとkwargsを組み合わせる
+        postfix_dict = {}
+        if ordered_dict:
+            postfix_dict.update(ordered_dict)
+        if kwargs:
+            postfix_dict.update(kwargs)
+
+        if postfix_dict:
+            # 辞書を文字列に変換して表示
+            postfix_str = ", ".join(f"{k}={v}" for k, v in postfix_dict.items())
+            self.postfix_container.text(f"状態: {postfix_str}")
+
+    def close(self) -> None:
+        """プログレスバーを完了状態にする"""
+        if self.total:
+            self.progress_bar.progress(1.0)
+        self.text_container.empty()
+
+    def refresh(self):
+        """不要なので何もしない"""
+        pass
+
+    def __iter__(self):
+        """イテレータとして使用できるようにする"""
+        if self.iterable is None:
+            raise ValueError("このプログレスバーはイテレータとして使用できません")
+
+        for obj in self.iterable:
+            yield obj
+            self.update(1)
+
+        self.close()
+
+    def __enter__(self):
+        """コンテキストマネージャとして使用できるようにする"""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """コンテキスト終了時に呼ばれる"""
+        self.close()
+
+
+class DummyProgress(BaseProgress):
+    """Dummy progress bar (no output)"""
+
+    def __init__(self, iterable: Optional[Iterable[T]] = None, total: Optional[int] = None, desc: str = "", **kwargs):
+        self.iterable = iterable
+        self.total = total
+        self.desc = desc
+        self.n = 0
+
+    def update(self, n: int = 1) -> None:
+        self.n += n
+
+    def set_description(self, desc: str = None, refresh: bool = True) -> None:
+        if desc is not None:
+            self.desc = desc
+
+    def set_postfix(self, ordered_dict=None, **kwargs) -> None:
+        pass
+
+    def close(self) -> None:
+        pass
+
+    def __iter__(self):
+        if self.iterable is None:
+            raise ValueError("No iterable provided")
+        for obj in self.iterable:
+            yield obj
+            self.update(1)
+
+
+def Progress(
+    iterable: Optional[Iterable[T]] = None,
+    backend: str = "tqdm",
+    total: Optional[int] = None,
+    desc: str = "",
+    **kwargs,
+) -> BaseProgress:
+    """
+    Create a progress bar with the specified backend
+
+    Args:
+        iterable: Optional iterable to track
+        backend: Backend type ("tqdm", "streamlit", "dummy")
+        total: Total iterations (required if iterable is None)
+        desc: Description text
+        **kwargs: Additional arguments passed to the backend
+
+    Returns:
+        BaseProgress instance
+    """
+    if backend == "tqdm":
+        return TqdmProgress(iterable=iterable, total=total, desc=desc, **kwargs)
+    elif backend == "streamlit":
+        try:
+            return StreamlitProgress(iterable=iterable, total=total, desc=desc, **kwargs)
+        except ImportError:
+            print("streamlit not found, falling back to dummy progress")
+            return DummyProgress(iterable=iterable, total=total, desc=desc, **kwargs)
+    elif backend == "dummy":
+        return DummyProgress(iterable=iterable, total=total, desc=desc, **kwargs)
+    else:
+        raise ValueError(f"Unknown backend: {backend}")

wsi_toolbox/utils/seed.py

@@ -0,0 +1,26 @@
+import random
+
+import numpy as np
+
+__GLOBAL_SEED = 42
+
+
+def get_global_seed():
+    return __GLOBAL_SEED
+
+
+def fix_global_seed(seed=None):
+    # Lazy import: torch is slow to load (~800ms), defer until needed
+    import torch  # noqa: PLC0415
+
+    if seed is None:
+        seed = get_global_seed()
+    global __GLOBAL_SEED
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.random.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.use_deterministic_algorithms = True
+    __GLOBAL_SEED = seed

wsi_toolbox/utils/st.py

@@ -0,0 +1,55 @@
+from contextlib import contextmanager
+
+import streamlit as st
+
+HORIZONTAL_STYLE = """
+<style class="hide-element">
+    /* Hides the style container and removes the extra spacing */
+    .element-container:has(.hide-element) {
+        display: none;
+    }
+    /*
+        The selector for >.element-container is necessary to avoid selecting the whole
+        body of the streamlit app, which is also a stVerticalBlock.
+    */
+    div[data-testid="stVerticalBlock"]:has(> .element-container .horizontal-marker) {
+        display: flex;
+        flex-direction: row !important;
+        flex-wrap: wrap;
+        gap: 0.5rem;
+        align-items: baseline;
+    }
+    /* Buttons and their parent container all have a width of 704px, which we need to override */
+    div[data-testid="stVerticalBlock"]:has(> .element-container .horizontal-marker) div {
+        width: max-content !important;
+    }
+    /* Selectbox container */
+    div[data-testid="stVerticalBlock"]:has(> .element-container .horizontal-marker) div[data-testid="stSelectbox"] {
+        display: flex !important;
+        flex-direction: row !important;
+        align-items: center !important;
+        gap: 0.5rem !important;
+    }
+    /* Selectbox label */
+    div[data-testid="stVerticalBlock"]:has(> .element-container .horizontal-marker) div[data-testid="stWidgetLabel"] {
+        margin-bottom: 0 !important;
+        padding-right: 0.5rem !important;
+    }
+    /* Selectbox input container */
+    div[data-testid="stVerticalBlock"]:has(> .element-container .horizontal-marker) div[data-baseweb="select"] {
+        min-width: 120px !important;
+    }
+    /* Selectbox dropdown */
+    div[data-testid="stVerticalBlock"]:has(> .element-container .horizontal-marker) div[role="listbox"] {
+        min-width: 120px !important;
+    }
+</style>
+"""
+
+
+@contextmanager
+def st_horizontal():
+    st.markdown(HORIZONTAL_STYLE, unsafe_allow_html=True)
+    with st.container():
+        st.markdown('<span class="hide-element horizontal-marker"></span>', unsafe_allow_html=True)
+        yield

wsi_toolbox/utils/white.py

@@ -0,0 +1,121 @@
+"""
+White patch detection methods for WSI processing
+"""
+
+from typing import Callable
+
+import cv2
+import numpy as np
+
+
+def is_white_patch_ptp(patch, white_ratio_threshold=0.9, rgb_range_threshold=20):
+    """
+    Check if a patch is mostly white/blank using PTP (peak-to-peak) method
+
+    Args:
+        patch: RGB patch (H, W, 3)
+        white_ratio_threshold: Ratio threshold for white pixels (0-1)
+        rgb_range_threshold: Threshold for RGB range (max-min, 0-255)
+
+    Returns:
+        bool: True if patch is considered white/blank
+    """
+    # white: RGB range (max-min) < rgb_range_threshold
+    rgb_range = np.ptp(patch, axis=2)
+    white_pixels = np.sum(rgb_range < rgb_range_threshold)
+    total_pixels = patch.shape[0] * patch.shape[1]
+    white_ratio_calculated = white_pixels / total_pixels
+    return white_ratio_calculated > white_ratio_threshold
+
+
+def is_white_patch_otsu(patch, white_ratio_threshold=0.8):
+    """
+    Check if a patch is mostly white/blank using Otsu method
+
+    Args:
+        patch: RGB patch (H, W, 3)
+        white_ratio_threshold: Ratio threshold for white pixels (0-1)
+
+    Returns:
+        bool: True if patch is considered white/blank
+    """
+    # Convert to grayscale and apply Otsu thresholding
+    gray = cv2.cvtColor(patch, cv2.COLOR_RGB2GRAY)
+    _, binary = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
+    white_pixels = np.sum(binary == 255)
+    total_pixels = patch.shape[0] * patch.shape[1]
+    white_ratio_calculated = white_pixels / total_pixels
+    return white_ratio_calculated > white_ratio_threshold
+
+
+def is_white_patch_std(patch, white_ratio_threshold=0.75, rgb_std_threshold=7.0):
+    """
+    Check if a patch is mostly white/blank using STD method
+
+    Args:
+        patch: RGB patch (H, W, 3)
+        white_ratio_threshold: Ratio threshold for white pixels (0-1)
+        rgb_std_threshold: Threshold for RGB standard deviation (0-255)
+
+    Returns:
+        bool: True if patch is considered white/blank
+    """
+    # white: RGB std < rgb_std_threshold
+    rgb_std_pixels = np.std(patch, axis=2) < rgb_std_threshold
+    white_pixels = np.sum(rgb_std_pixels)
+    total_pixels = patch.shape[0] * patch.shape[1]
+    white_ratio_calculated = white_pixels / total_pixels
+    return white_ratio_calculated > white_ratio_threshold
+
+
+def is_white_patch_green(patch, green_threshold=0.9):
+    """
+    Check if a patch is mostly white/blank using green channel method
+
+    Args:
+        patch: RGB patch (H, W, 3)
+        green_threshold: Threshold for green channel mean
+
+    Returns:
+        bool: True if patch is considered white/blank
+    """
+    # Extract green channel and normalize
+    green = patch[:, :, 1] / 255.0
+    green_mean = np.mean(green)
+    return green_mean > green_threshold
+
+
+def create_white_detector(method: str, threshold: float | None = None) -> Callable[[np.ndarray], bool]:
+    """
+    Create white detection function from method name and threshold
+
+    Args:
+        method: Detection method ('ptp', 'otsu', 'std', 'green')
+        threshold: Threshold value (None for method-specific default)
+
+    Returns:
+        Function that takes (H, W, 3) numpy array and returns bool
+
+    Raises:
+        ValueError: If method is invalid
+
+    Example:
+        >>> detector = create_white_detector('ptp', 0.85)
+        >>> is_white = detector(patch)  # patch is (256, 256, 3) array
+    """
+    # Map method names to functions and default thresholds
+    method_map = {
+        "ptp": (is_white_patch_ptp, 0.9),
+        "otsu": (is_white_patch_otsu, 0.8),
+        "std": (is_white_patch_std, 0.75),
+        "green": (is_white_patch_green, 0.9),
+    }
+
+    if method not in method_map:
+        raise ValueError(f"Invalid method '{method}'. Must be one of {list(method_map.keys())}")
+
+    func, default_threshold = method_map[method]
+    actual_threshold = threshold if threshold is not None else default_threshold
+
+    # Return curried function: patch -> bool
+    return lambda patch: func(patch, actual_threshold)

wsi_toolbox/watcher.py

@@ -0,0 +1,256 @@
+import argparse
+import os
+import time
+from pathlib import Path
+from typing import Callable, Dict, Optional
+
+from rich.console import Console
+
+from . import commands
+from .utils import plot_umap
+
+DEFAULT_MODEL = os.getenv("DEFAULT_MODEL", "uni")
+
+
+class Status:
+    PROCESSING = "PROCESSING"
+    DONE = "DONE"
+    ERROR = "ERROR"
+
+    @classmethod
+    def is_processing_state(cls, status: str) -> bool:
+        """状態が処理中系かどうかを判定"""
+        return status.startswith((cls.PROCESSING, cls.DONE, cls.ERROR))
+
+
+class Task:
+    REQUEST_FILE = "_ROBIEMON.txt"
+    LOG_FILE = "_ROBIEMON_LOG.txt"
+
+    @staticmethod
+    def parse_request_line(line: str) -> tuple[str, bool]:
+        """Parse the request line for model and rotation specifications.
+        Returns (model_name, should_rotate)"""
+        parts = [p.strip() for p in line.split(",")]
+        model_name = parts[0] if parts and parts[0] else DEFAULT_MODEL
+        should_rotate = len(parts) > 1 and parts[1].lower() == "rotate"
+        return model_name, should_rotate
+
+    def __init__(self, folder: Path, options_line: str, on_complete: Optional[Callable[[Path], None]] = None):
+        self.folder = folder
+        self.options_line = options_line
+        self.model_name, self.should_rotate = self.parse_request_line(options_line)
+        self.on_complete = on_complete
+        self.wsi_files = list(folder.glob("**/*.ndpi")) + list(folder.glob("**/*.svs"))
+        self.wsi_files.sort()
+
+        commands.set_default_progress("tqdm")
+        commands.set_default_model_preset(self.model_name)
+
+    def write_banner(self):
+        """処理開始時のバナーをログに書き込み"""
+        self.append_log("=" * 50)
+        self.append_log(f"Processing folder: {self.folder}")
+        self.append_log(f"Request options: {self.options_line}")
+        self.append_log("Parsed options:")
+        self.append_log(f"  - Model: {self.model_name} (default: {DEFAULT_MODEL})")
+        self.append_log(f"  - Rotation: {'enabled' if self.should_rotate else 'disabled'}")
+        self.append_log(f"Found {len(self.wsi_files)} WSI files:")
+        for i, wsi_file in enumerate(self.wsi_files, 1):
+            size_mb = wsi_file.stat().st_size / (1024 * 1024)
+            self.append_log(f"  {i}. {wsi_file.name} ({size_mb:.1f} MB)")
+        self.append_log("=" * 50)
+
+    def run(self):
+        try:
+            # ログファイルをクリア
+            with open(self.folder / self.LOG_FILE, "w") as f:
+                f.write("")
+
+            self.set_status(Status.PROCESSING)
+            self.write_banner()
+
+            # WSIファイルごとの処理
+            for i, wsi_file in enumerate(self.wsi_files):
+                try:
+                    self.append_log(f"Processing [{i + 1}/{len(self.wsi_files)}]: {wsi_file.name}")
+
+                    hdf5_tmp_path = wsi_file.with_suffix(".h5.tmp")
+                    hdf5_file = wsi_file.with_suffix(".h5")
+
+                    # HDF5変換（既存の場合はスキップ）
+                    if not hdf5_file.exists():
+                        self.append_log("Converting to HDF5...")
+                        # Use new command pattern
+                        commands.set_default_progress("tqdm")
+                        cmd = commands.Wsi2HDF5Command(rotate=self.should_rotate)
+                        _ = cmd(str(wsi_file), str(hdf5_tmp_path))
+                        os.rename(hdf5_tmp_path, hdf5_file)
+                        self.append_log("HDF5 conversion completed.")
+
+                    # 特徴量抽出（既存の場合はスキップ）
+                    self.append_log("Extracting features...")
+                    # Use new command pattern
+                    commands.set_default_device("cuda")
+                    emb_cmd = commands.PatchEmbeddingCommand()
+                    _ = emb_cmd(str(hdf5_file))
+                    self.append_log("Feature extraction completed.")
+
+                    # クラスタリングとUMAP生成
+                    self.append_log("Starting clustering ...")
+                    # Use new command pattern
+                    cluster_cmd = commands.ClusteringCommand(resolution=1.0, use_umap=True)
+                    _ = cluster_cmd([hdf5_file])
+                    self.append_log("Clustering completed.")
+
+                    base = str(wsi_file.with_suffix(""))
+
+                    # UMAPプロット生成
+                    self.append_log("Starting UMAP generation...")
+                    umap_path = Path(f"{base}_umap.png")
+                    if not umap_path.exists():
+                        umap_embs = cluster_cmd.get_umap_embeddings()
+                        fig = plot_umap(umap_embs, cluster_cmd.total_clusters)
+                        fig.savefig(umap_path, bbox_inches="tight", pad_inches=0.5)
+                        self.append_log(f"UMAP plot completed. Saved to {os.path.basename(umap_path)}")
+                    else:
+                        self.append_log("UMAP plot already exists. Skipped.")
+
+                    # サムネイル生成
+                    self.append_log("Starting thumbnail generation...")
+                    thumb_path = Path(f"{base}_thumb.jpg")
+                    if not thumb_path.exists():
+                        # Use new command pattern
+                        preview_cmd = commands.PreviewClustersCommand(size=64)
+                        img = preview_cmd(str(hdf5_file), cluster_name="")
+                        img.save(thumb_path)
+                        self.append_log(f"Thumbnail generation completed. Saved to {thumb_path.name}")
+                    else:
+                        self.append_log("Thumbnail already exists. Skipped.")
+
+                    self.append_log("=" * 30)
+
+                except Exception as e:
+                    self.append_log(f"Error processing {wsi_file}: {str(e)}")
+                    self.set_status(Status.ERROR)
+                    if self.on_complete:
+                        self.on_complete(self.folder)
+                    return
+
+            self.set_status(Status.DONE)
+            self.append_log("All processing completed successfully")
+
+        except Exception as e:
+            self.append_log(f"Error: {str(e)}")
+
+        if self.on_complete:
+            self.on_complete(self.folder)
+
+    def set_status(self, status: str):
+        self.status = status
+        with open(self.folder / self.REQUEST_FILE, "w") as f:
+            f.write(f"{status}\n")
+
+    def append_log(self, message: str):
+        with open(self.folder / self.LOG_FILE, "a") as f:
+            f.write(message + "\n")
+            print(message)
+
+
+class Watcher:
+    def __init__(self, base_dir: str):
+        self.base_dir = Path(base_dir)
+        self.running_tasks: Dict[Path, Task] = {}
+        self.console = Console()
+
+    def run(self, interval: int = 60):
+        self.console.print("\n[bold blue]ROBIEMON Watcher started[/]")
+        self.console.print(f"[blue]Watching directory:[/] {self.base_dir}")
+        self.console.print(f"[blue]Polling interval:[/] {interval} seconds")
+        self.console.print("[yellow]Press Ctrl+C to stop[/]\n")
+
+        while True:
+            try:
+                self.check_folders()
+
+                # カウントダウン表示
+                for remaining in range(interval, 0, -1):
+                    print(f"\rNext check in {remaining:2d}s", end="", flush=True)
+                    time.sleep(1)
+                # カウントダウン終了後、同じ行を再利用
+                print("\rNext check in  0s", end="", flush=True)
+
+            except KeyboardInterrupt:
+                self.console.print("\n[yellow]Stopping watcher...[/]")
+                break
+            except Exception as e:
+                self.console.print(f"[red]ERROR:[/] {str(e)}")
+
+    def check_folders(self):
+        for folder in self.base_dir.rglob("*"):
+            if not folder.is_dir():
+                continue
+
+            request_file = folder / Task.REQUEST_FILE
+            if not request_file.exists():
+                continue
+
+            if folder in self.running_tasks:
+                continue
+
+            try:
+                with open(request_file, "r") as f:
+                    content = f.read()
+                    if not content.strip():
+                        continue
+
+                    # First line contains model/rotation specs
+                    options_line = content.split("\n")[0].strip()
+
+                    # Original status check from the entire file
+                    status = content.strip()
+
+            except Exception as e:
+                print(f"Error reading {request_file}: {e}")
+                continue
+
+            if Status.is_processing_state(status):
+                continue
+
+            # \rを含むログから改行するため空白行を挿入
+            print()
+            print()
+            print(f"detected: {folder}")
+            print(f"Request options: {options_line}")
+
+            task = Task(folder, options_line, on_complete=lambda f: self.running_tasks.pop(f, None))
+            self.running_tasks[folder] = task
+            task.run()  # 同期実行に変更
+
+
+BASE_DIR = os.getenv("BASE_DIR", "data")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="ROBIEMON WSI Processor Watcher")
+    parser.add_argument(
+        "--base-dir", type=str, default=BASE_DIR, help="Base directory to watch for WSI processing requests"
+    )
+    parser.add_argument("--interval", type=int, default=60, help="Polling interval in seconds (default: 60)")
+
+    args = parser.parse_args()
+
+    base_dir = Path(args.base_dir)
+    if not base_dir.exists():
+        print(f"Error: Base directory '{args.base_dir}' does not exist")
+        return
+    if not base_dir.is_dir():
+        print(f"Error: '{args.base_dir}' is not a directory")
+        return
+
+    watcher = Watcher(args.base_dir)
+    watcher.run(interval=args.interval)  # asyncio.runを削除
+
+
+if __name__ == "__main__":
+    main()