phenome 1.4.1__py3-none-any.whl

phenome/__init__.py

@@ -0,0 +1,180 @@
+"""
+@section Core Pipeline
+
+PhenoMe Package
+
+A modular, **dataset-agnostic** and **model-agnostic** pipeline for phenotyping
+analysis using deep learning embeddings.
+
+**Overview:**
+PhenoMe orchestrates the full phenotyping workflow: discover images, extract embeddings
+with custom or pretrained models, compute cell/object properties, perform statistical
+analysis, and generate interactive visualizations and reports.
+
+**Public API:**
+
+Core orchestration:
+- `PhenoMe`: Main pipeline class; find images, process embeddings, compute properties, analyze.
+- `PhenoMeResults`: Container for embeddings, properties, metadata, and analysis results.
+
+Metadata helpers:
+- `DefaultMetadata`: Simple key-value metadata.
+- `PathTemplateMetadata`: Extract metadata from file paths using templates.
+- `DataFrameMetadata`: Use a DataFrame as metadata source.
+- `default_metadata_from_path`, `get_metadata_from_path`, `make_dataframe_metadata_fn`:
+  Metadata factory functions.
+
+Model loading:
+- `load_dinov2_model`: Load pretrained DINOv2 model.
+
+Property factories (create custom cell/object properties):
+- `create_regionprops_function`, `create_intensity_function`,
+  `create_masked_intensity_function`, `create_texture_function`,
+  `create_concentric_ring_function`, `create_blur_effect_function`,
+  `create_entropy_function`: Built-in property creators.
+- `get_preset_property_functions`: List all available preset properties.
+
+**Data flow:**
+
+```
+find_files()
+    ↓ returns DataFrame of image paths
+set_file_df() or directly via DataFrame
+    ↓ store file locations
+process_images(model)
+    ↓ extract embeddings, populate results.embeddings
+compute_properties(property_functions)
+    ↓ compute morphological/biological features
+compute_clustering() / detect_outliers() / ...
+    ↓ statistical analysis
+plot_pca() / plot_tsne() / plot_umap() / ...
+    ↓ visualization
+generate_report()
+    ↓ HTML report
+```
+
+**Model compatibility:**
+- **DINOv2:** Use `load_dinov2_model()` or wrap with `DinoV2ModelWrapper`.
+- **Custom:** Subclass `ModelWrapper`, implement `_get_embeddings(images) -> (B, D)`.
+
+**For advanced usage, import from submodules:**
+- `phenome.core`: Math utilities (DR, correlation, validation, export).
+- `phenome.io`: File I/O (`read_image`, `FileDiscovery`, `CheckpointManager`).
+- `phenome.mixins`: Composition classes (dataset, properties, analysis, visualization).
+- `phenome.plugins`: Property registry and custom property registration.
+- `phenome.report`: HTML report generation.
+- `phenome.utils`: Device management, transforms, metadata and property factories.
+
+**Logging:**
+
+Progress messages are visible at INFO level by default:
+
+```python
+import logging
+logging.getLogger("phenome").setLevel(logging.WARNING)  # Suppress
+```
+
+**Example:**
+
+Basic phenotyping workflow:
+
+```python
+from phenome import PhenoMe, load_dinov2_model
+
+# Initialize pipeline
+pm = PhenoMe(device="cuda", seed=42)
+
+# Find images
+pm.find_files("path/to/images/")
+
+# Extract embeddings
+model = load_dinov2_model(model_name="dinov2_vitb14")
+pm.process_images(model, batch_size=32)
+
+# Compute properties
+pm.compute_properties(property_preset="basic")
+
+# Analyze
+pm.compute_clustering(method="kmeans", n_clusters=5)
+pm.compute_pca()
+
+# Visualize
+pm.plot_pca(color_by="clustering")
+```
+
+**See Also:**
+For architecture, design patterns, and extensibility, see the [Architecture guide](https://AAitorG.github.io/PhenoMe/guides/architecture/).
+"""
+
+# Initialise notebook-safe logging on first import
+from importlib.metadata import PackageNotFoundError, version
+
+from ._logging import get_logger as _get_logger
+from .core import PhenoMeResults
+from .metadata import (
+    DataFrameMetadata,
+    DefaultMetadata,
+    MetadataBase,
+    PathTemplateMetadata,
+)
+from .pipeline import PhenoMe
+from .plotly_display import register_plotly_display_defaults
+
+# Metadata functions and classes
+from .utils.metadata import (
+    default_metadata_from_path,
+    get_metadata_from_path,
+    make_dataframe_metadata_fn,
+)
+
+# Core workflow helpers
+from .utils.model_wrapper import load_dinov2_model
+
+# Property factories
+from .utils.property_factories import (
+    create_blur_effect_function,
+    create_concentric_ring_function,
+    create_entropy_function,
+    create_intensity_function,
+    create_masked_intensity_function,
+    create_regionprops_function,
+    create_texture_function,
+    get_preset_property_functions,
+)
+
+try:
+    __version__ = version("phenome")
+except PackageNotFoundError:
+    __version__ = "1.4.1"
+
+__all__ = [
+    "DataFrameMetadata",
+    "DefaultMetadata",
+    "MetadataBase",
+    "PathTemplateMetadata",
+    "PhenoMe",
+    "PhenoMeResults",
+    "__version__",
+    # Property factories
+    "create_blur_effect_function",
+    "create_concentric_ring_function",
+    "create_entropy_function",
+    "create_intensity_function",
+    "create_masked_intensity_function",
+    "create_regionprops_function",
+    "create_texture_function",
+    # Metadata functions
+    "default_metadata_from_path",
+    "get_metadata_from_path",
+    # Property factories
+    "get_preset_property_functions",
+    # Default models
+    "load_dinov2_model",
+    "make_dataframe_metadata_fn",
+]
+
+_get_logger(__name__)
+__author__ = "Aitor González-Marfil"
+__author_github__ = "AAitorG"
+
+register_plotly_display_defaults()

phenome/_logging.py

@@ -0,0 +1,99 @@
+"""
+Notebook-safe logging configuration for phenome.
+
+Problem
+-------
+The standard ``logging.getLogger(__name__)`` pattern silently drops INFO
+messages in Jupyter notebooks because Python's root logger defaults to
+WARNING level and has no handler attached.  If the user manually adds a
+handler, duplicate output appears on every cell re-run.
+
+Solution
+--------
+We configure a single package-level logger (``phenome``) with:
+
+* A ``StreamHandler`` writing to ``sys.stderr`` (rendered by Jupyter as
+  plain text below the cell).
+* INFO as default level, so progress messages are always visible.
+* A guard that prevents adding duplicate handlers on module reload
+  (``%autoreload`` / repeated imports).
+
+All sub-modules should call::
+
+    from ._logging import get_logger
+    logger = get_logger(__name__)
+
+instead of ``logging.getLogger(__name__)``.
+
+Users who want to silence the output can do::
+
+    import logging
+    logging.getLogger("phenome").setLevel(logging.WARNING)
+"""
+
+import logging
+import sys
+
+# Package-level logger name
+_PKG_LOGGER_NAME = "phenome"
+
+# Sentinel attribute on the package logger so we configure at most once.
+_CONFIGURED_ATTR = "_phenome_configured"
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Return a logger configured for notebook-safe output.
+
+    Typical usage at the top of each module::
+
+        from ._logging import get_logger
+        logger = get_logger(__name__)
+
+    Parameters
+    ----------
+    name : str
+        Usually ``__name__`` of the calling module.
+    """
+    _ensure_configured()
+    return logging.getLogger(name)
+
+
+def _ensure_configured() -> None:
+    """Attach a StreamHandler to the package logger if not done already."""
+    pkg_logger = logging.getLogger(_PKG_LOGGER_NAME)
+
+    if getattr(pkg_logger, _CONFIGURED_ATTR, False):
+        return
+
+    # Library best-practice: add NullHandler so that if the user never
+    # configures logging at all, nothing breaks.
+    pkg_logger.addHandler(logging.NullHandler())
+
+    # Only add our StreamHandler if the package logger (or its parents)
+    # have no *effective* handlers yet.  This avoids fighting with a
+    # user who already configured logging before importing us.
+    if not _has_effective_handler(pkg_logger):
+        handler = logging.StreamHandler(sys.stderr)
+        handler.setFormatter(logging.Formatter("%(message)s"))
+        pkg_logger.addHandler(handler)
+
+    # Set INFO as the default level for the package tree.
+    # Only override if the logger is still at NOTSET (i.e. the user hasn't
+    # explicitly set a level).
+    if pkg_logger.level == logging.NOTSET:
+        pkg_logger.setLevel(logging.INFO)
+
+    setattr(pkg_logger, _CONFIGURED_ATTR, True)
+
+
+def _has_effective_handler(logger: logging.Logger) -> bool:
+    """Check if *logger* or any ancestor already has a non-NullHandler."""
+    current: logging.Logger | None = logger
+    while current:
+        for h in current.handlers:
+            if not isinstance(h, logging.NullHandler):
+                return True
+        if not current.propagate:
+            break
+        current = current.parent
+    return False

phenome/core/__init__.py

@@ -0,0 +1,107 @@
+"""
+@section Core Pipeline
+
+Core utilities for math, validation, export, and results management.
+
+This package provides the mathematical algorithms and data structures that underpin
+the phenotyping pipeline. Key components:
+
+**Results container:**
+- `PhenoMeResults`: Typed container for embeddings, properties, metadata, and analysis results.
+
+**Mathematical functions:**
+- `run_dimensionality_reduction`: PCA, t-SNE, UMAP.
+- `compute_pearson_correlation`, `compute_spearman_correlation`: Feature correlations.
+- `compute_distance_correlation`: Distance correlation between features.
+- `compute_mutual_info`: Mutual information between features.
+- `compute_entropy`: Entropy of features.
+- `build_combined_features`: Combine multiple features into single representation.
+
+**Validation and export:**
+- `validate_results`: Check results integrity (embeddings, properties, shapes).
+- `build_export_dataframe`: Format results for export to CSV/Excel.
+- `prepare_embedding_dataframe`: Prepare embeddings with metadata for analysis.
+
+**Metadata handling:**
+- `build_metadata_columns`: Extract and format metadata columns.
+- `filter_indices`: Select rows by metadata criteria.
+- `get_metadata_value`: Retrieve metadata value for a row.
+- `get_all_metadata_keys`: List all available metadata keys.
+- `metadata_to_stable_key`: Convert metadata to file-safe string.
+
+**Protocols:**
+- `PhenoMeProtocol`: Protocol (duck-type interface) for objects implementing the phenotyping API.
+
+**See Also:**
+For top-level analysis pipeline, see `phenome.PhenoMe`.
+"""
+
+from .dataframe_contract import (
+    IMAGE_INDEX,
+    IMAGE_NAME,
+    IMAGE_PATH,
+    TIER_A_EXCLUDE_FROM_PROPERTIES,
+    append_property_columns,
+    is_per_image_df,
+    per_image_dataframe,
+    reorder_tier_a_columns,
+)
+from .math import (
+    build_combined_features,
+    clean_correlation_inputs,
+    compute_distance_correlation,
+    compute_entropy,
+    compute_lasso_interpretability,
+    compute_mutual_info,
+    compute_pearson_correlation,
+    compute_rf_interpretability,
+    compute_spearman_correlation,
+    run_dimensionality_reduction,
+    run_dimensionality_reduction_matrix,
+)
+from .pipeline_results import PhenoMeResults
+from .property_utils import metadata_to_stable_key, optimize_property_types
+from .protocols import PhenoMeProtocol
+from .results_export import build_export_dataframe, prepare_embedding_dataframe
+from .results_metadata import (
+    build_metadata_columns,
+    filter_indices,
+    get_all_metadata_keys,
+    get_metadata_value,
+    get_metadata_value_from_dict,
+)
+from .results_validation import validate_results
+
+__all__ = [
+    "IMAGE_INDEX",
+    "IMAGE_NAME",
+    "IMAGE_PATH",
+    "TIER_A_EXCLUDE_FROM_PROPERTIES",
+    "PhenoMeProtocol",
+    "PhenoMeResults",
+    "append_property_columns",
+    "build_combined_features",
+    "build_export_dataframe",
+    "build_metadata_columns",
+    "clean_correlation_inputs",
+    "compute_distance_correlation",
+    "compute_entropy",
+    "compute_lasso_interpretability",
+    "compute_mutual_info",
+    "compute_pearson_correlation",
+    "compute_rf_interpretability",
+    "compute_spearman_correlation",
+    "filter_indices",
+    "get_all_metadata_keys",
+    "get_metadata_value",
+    "get_metadata_value_from_dict",
+    "is_per_image_df",
+    "metadata_to_stable_key",
+    "optimize_property_types",
+    "per_image_dataframe",
+    "prepare_embedding_dataframe",
+    "reorder_tier_a_columns",
+    "run_dimensionality_reduction",
+    "run_dimensionality_reduction_matrix",
+    "validate_results",
+]

phenome/core/_processing_pipeline_persist.py

@@ -0,0 +1,112 @@
+"""Slim title/detail step payloads for checkpoint persistence.
+
+Kept separate from ``processing_pipeline`` so HDF5 I/O can slim steps without
+importing image preflight (which would cycle through ``phenome.io``).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+__all__ = ["coalesce_pipeline_payload", "slim_pipeline_payload"]
+
+
+def _printed_step_kind(step: Mapping[str, Any]) -> str | None:
+    """Return a persisted pipeline kind, or None when the step has none."""
+    kind = str(step.get("kind") or "").strip().lower()
+    if kind:
+        return kind
+    if str(step.get("id") or "").startswith("mask_"):
+        return "mask"
+    return None
+
+
+def _is_mask_step(step: Mapping[str, Any]) -> bool:
+    """Return True when *step* is a mask-only pipeline step."""
+    return _printed_step_kind(step) == "mask"
+
+
+def _printed_step_fields(step: Any) -> dict[str, str] | None:
+    """Keep title, detail, and kind for printing or persistence."""
+    if not isinstance(step, Mapping):
+        return None
+    title = str(step.get("title") or "").strip()
+    detail = str(step.get("detail") or "").strip()
+    if not title and not detail:
+        return None
+    out: dict[str, str] = {}
+    if title:
+        out["title"] = title
+    if detail:
+        out["detail"] = detail
+    kind = _printed_step_kind(step)
+    if kind:
+        out["kind"] = kind
+    return out
+
+
+def _steps_from_value(value: Any) -> list[dict[str, str]] | None:
+    """Extract ordered title/detail steps from a slim list or a full segment."""
+    raw: Any = value
+    if isinstance(value, Mapping) and "steps" in value:
+        raw = value.get("steps")
+    if not isinstance(raw, list):
+        return None
+    kept = [step for item in raw if (step := _printed_step_fields(item))]
+    return kept or None
+
+
+def slim_pipeline_payload(payload: Mapping[str, Any] | None) -> dict[str, Any] | None:
+    """Return only embeddings/properties step lists suitable for HDF5.
+
+    Accepts a session dict, a slim ``{embeddings, properties}`` payload, a
+    full segment with ``steps``, or a bare step list.
+
+    Args:
+        payload: Session pipeline, slim checkpoint payload, or None.
+
+    Returns:
+        ``{embeddings, properties}`` lists of printed step fields, or None.
+    """
+    if not payload:
+        return None
+    embeddings = _steps_from_value(payload.get("embeddings"))
+    if embeddings is None:
+        embeddings = _steps_from_value(payload.get("base"))
+    properties = _steps_from_value(payload.get("properties"))
+    if embeddings is None and properties is None:
+        embeddings = _steps_from_value(payload)
+    out: dict[str, Any] = {}
+    if embeddings:
+        out["embeddings"] = embeddings
+    if properties:
+        out["properties"] = properties
+    return out or None
+
+
+def coalesce_pipeline_payload(
+    incoming: Mapping[str, Any] | None,
+    existing: Mapping[str, Any] | None,
+) -> dict[str, Any] | None:
+    """Fill missing embeddings/properties steps from *existing*.
+
+    Used when a write has only one segment (for example ``process_images``
+    after ``reset()``) so the other stored segment is not deleted.
+
+    Args:
+        incoming: Session pipeline or slim payload being written.
+        existing: Payload already stored on the checkpoint, or None.
+
+    Returns:
+        Merged slim payload, or None when both sides are empty.
+    """
+    slim_in = slim_pipeline_payload(incoming)
+    slim_ex = slim_pipeline_payload(existing)
+    if slim_in is None:
+        return slim_ex
+    if slim_ex is None:
+        return slim_in
+    merged = dict(slim_ex)
+    merged.update(slim_in)
+    return merged