pixel-patrol-base 0.1.0__tar.gz

pixel_patrol_base-0.1.0/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.4
+Name: pixel-patrol-base
+Version: 0.1.0
+Summary: Image prevalidation tool
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.2.1
+Requires-Dist: dash-ag-grid>=32.3.0
+Requires-Dist: dash-bootstrap-components>=2.0.3
+Requires-Dist: matplotlib>=3.10.3
+Requires-Dist: polars>=1.31.0
+Requires-Dist: pyarrow>=21.0.0
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: statsmodels>=0.14.4
+Requires-Dist: tensorboard>=2.18.0
+Requires-Dist: tensorboardx>=2.6.4
+Requires-Dist: dask>=2025.5.1
+Requires-Dist: tqdm>=4.67.1

pixel_patrol_base-0.1.0/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "pixel-patrol-base"
+version = "0.1.0"
+description = "Image prevalidation tool"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "click>=8.2.1",
+    "dash-ag-grid>=32.3.0",
+    "dash-bootstrap-components>=2.0.3",
+    "matplotlib>=3.10.3",
+    "polars>=1.31.0",
+    "pyarrow>=21.0.0",
+    "pyyaml>=6.0.2",
+    "statsmodels>=0.14.4",
+    "tensorboard>=2.18.0", # TODO: Are we sure we want it in base?
+    "tensorboardx>=2.6.4", # TODO: Are we sure we want it in base?
+    "dask>=2025.5.1",
+    "tqdm>=4.67.1",
+]
+
+[tool.uv]
+package = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/pixel_patrol_base"]
+
+[project.scripts]
+pixel-patrol = "pixel_patrol_base.cli:cli"
+
+[project.entry-points."pixel_patrol.processor_plugins"]
+pixel_patrol_base_processing_builtins = "pixel_patrol_base.plugin_registry:register_processor_plugins"
+
+[project.entry-points."pixel_patrol.widget_plugins"]
+pixel_patrol_base_widget_builtins = "pixel_patrol_base.plugin_registry:register_widget_plugins"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.3.5",
+    "pytest-mock>=3.14.1",
+]

pixel_patrol_base-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pixel_patrol_base-0.1.0/src/pixel_patrol_base/api.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+from typing import Union, Iterable, List, Optional
+import polars as pl
+import logging
+
+from pixel_patrol_base.core.project import Project
+from pixel_patrol_base.core.project_settings import Settings
+from pixel_patrol_base.io.project_io import export_project as _io_export_project
+from pixel_patrol_base.io.project_io import import_project as _io_import_project
+from pixel_patrol_base.report.dashboard_app import create_app
+
+
+logger = logging.getLogger(__name__)
+
+def create_project(name: str, base_dir: Union[str, Path], loader: str = None) -> Project:
+    logger.info(f"API Call: Creating new project '{name}' with base directory '{base_dir}'.")
+    return Project(name, base_dir, loader)
+
+def add_paths(project: Project, paths: Union[str, Path, Iterable[Union[str, Path]]]) -> Project:
+    logger.info(f"API Call: Adding paths to project '{project.name}'.")
+    return project.add_paths(paths)
+
+def delete_path(project: Project, path: str) -> Project:
+    logger.info(f"API Call: deleting paths from project '{project.name}'.")
+    return project.delete_path(path)
+
+def set_settings(project: Project, settings: Settings) -> Project:
+    """
+    Sets the project-specific settings by replacing the entire Settings object.
+    Detailed validation for individual settings is performed within the Project class itself.
+    Args:
+        project: The Project instance to update.
+        settings: An instance of the Settings dataclass containing the desired settings.
+    """
+    logger.info(f"API Call: Attempting to set project settings for '{project.name}'.")
+    return project.set_settings(settings)
+
+def process_files(project: Project) -> Project:
+    logger.info(f"API Call: Processing files and building DataFrame for project '{project.name}'.")
+    return project.process_records()
+
+def show_report(project: Project, host: str = "127.0.0.1", port: int = None, debug: bool = False) -> None:
+    """
+    Run without the Flask debug reloader by default. The debug reloader
+    spawns a second process and will re-import/run the script, which causes
+    the example to execute twice (scan/process files two times). When a
+    developer needs the interactive debugger they can pass `debug=True`,
+    but should also set `use_reloader=False` if they do not want the script
+    re-executed.
+    """
+    logger.info(f"API Call: Showing report for project '{project.name}'.")
+    app = create_app(project)
+    app.run(debug=debug, host=host, port=port, use_reloader=False)
+
+def export_project(project: Project, dest: Path) -> None: # TODO: think about when project can be saved
+    logger.info(f"API Call: Exporting project '{project.name}' to '{dest}'.")
+    _io_export_project(project, dest)
+    logger.info(f"API Call: Project '{project.name}' exported successfully.")
+
+def import_project(src: Path) -> Project:
+    logger.info(f"API Call: Importing project from '{src}'.")
+    project = _io_import_project(src)
+    logger.info(f"API Call: Project '{project.name}' imported successfully from '{src}'.")
+    return project
+
+def get_name(project: Project) -> str:
+    return project.get_name()
+
+def get_base_dir(project: Project) -> Optional[Path]:
+    return project.get_base_dir()
+
+def get_paths(project: Project) -> List[Path]:
+    return project.get_paths()
+
+def get_settings(project: Project) -> Settings:
+    return project.get_settings()
+
+def get_records_df(project: Project) -> Optional[pl.DataFrame]:
+    return project.get_records_df()

pixel_patrol_base-0.1.0/src/pixel_patrol_base/cli.py

@@ -0,0 +1,130 @@
+import os
+import webbrowser
+from pathlib import Path
+from threading import Timer
+
+import click
+
+from pixel_patrol_base.api import (
+    create_project,
+    add_paths,
+    set_settings,
+    process_files,
+    export_project,
+    import_project,
+    show_report,
+)
+from pixel_patrol_base.core.project_settings import Settings
+
+
+@click.group()
+def cli():
+    """
+    A command-line tool for processing image reports with Pixel Patrol.
+
+    This tool facilitates a two-step process:
+    1. Exporting a processed project to a ZIP file.
+    2. Displaying a report from an exported ZIP file.
+    """
+    pass
+
+@cli.command()
+@click.argument('base_directory', type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True, path_type=Path))
+@click.option('--output-zip', '-o', type=click.Path(exists=False, dir_okay=False, writable=True, path_type=Path),
+              help='Required: Name of the output ZIP file for the exported project (e.g., my_project.zip).',
+              required=True)
+@click.option('--name', type=str, required=False,
+              help='Optional: Name of the project. If not provided, derived from BASE_DIRECTORY.')
+@click.option('--paths', '-p', multiple=True, type=str,
+              help='Optional: Paths (subdirectories) to treat as **experimental conditions**, relative to BASE_DIRECTORY. '
+                   'Can be specified multiple times. If omitted, all immediate subdirectories '
+                   'of BASE_DIRECTORY will be included, or if BASE_DIRECTORY has no subdirectories, '
+                   'it is treated as a single condition.')
+@click.option('--loader', '-l', type=str, show_default=True,
+              help='Recommended: Pixel Patrol file loader (e.g., bioio, zarr). If omitted, only basic file info is collected.')
+@click.option('--cmap', type=str, default="rainbow", show_default=True,
+              help='Colormap for report visualization (e.g., viridis, plasma, rainbow).')
+@click.option('--n-example-files', type=int, default=9, show_default=True,
+              help='Number of example files to display in the report.')
+@click.option('--file-extension', '-e', multiple=True,
+              help='Optional: File extensions to include (e.g., png, jpg). Can be specified multiple times. '
+                   'If not specified, all supported extensions will be used.')
+@click.option('--flavor', type=str, default="", show_default=True,
+              help='Name of pixel patrol configuration, will be displayed next to the tool name.')
+def export(base_directory: Path, output_zip: Path, name: str | None, paths: tuple[str, ...],
+           loader: str, cmap: str, n_example_files: int, file_extension: tuple[str, ...], flavor: str):
+    """
+    Exports a Pixel Patrol project to a ZIP file.
+    Processes images from the BASE_DIRECTORY and specified --paths.
+    """
+    # Always operate on an absolute base directory so downstream path resolution is stable.
+    base_directory = base_directory.resolve()
+
+    # Derive project_name if not provided
+    if name is None:
+        name = base_directory.name # Use the name of the base directory
+        click.echo(f"Project name not provided, deriving from base directory: '{name}'")
+
+    click.echo(f"Creating project: '{name}' from base directory '{base_directory}'")
+    my_project = create_project(name, str(base_directory), loader=loader) # Assuming create_project takes string path
+
+    if paths:
+        click.echo(f"Adding explicitly specified paths: {', '.join(paths)}. Resolution will be relative to '{base_directory}'")
+        add_paths(my_project, paths)
+    else:
+        # If no paths, we want to add the base directory itself.
+        click.echo(f"No --paths specified. Processing all images in '{base_directory}'.")
+        add_paths(my_project, base_directory)
+
+    selected_extensions = set(file_extension) if file_extension else "all"
+    initial_settings = Settings(
+        cmap=cmap,
+        n_example_files=n_example_files,
+        selected_file_extensions=selected_extensions,
+        pixel_patrol_flavor=flavor
+    )
+    click.echo(f"Setting project settings: {initial_settings}")
+    set_settings(my_project, initial_settings)
+
+    click.echo("Processing images...")
+    process_files(my_project)
+
+    click.echo(f"Exporting project to: '{output_zip}'")
+    export_project(my_project, Path(output_zip)) # Assuming export_project takes string path
+    click.echo("Export complete.")
+
+
+@cli.command()
+@click.argument('input_zip', type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True, path_type=Path))
+@click.option('--port', type=int, default=8050, show_default=True,
+              help='Port number for the Dash report server.')
+def report(input_zip: Path, port: int):
+    """
+    Displays the report of an exported Pixel Patrol project from a ZIP file.
+    """
+    click.echo(f"Importing project from: '{input_zip}'")
+    my_project = import_project(Path(input_zip))
+    click.echo("Project imported.")
+
+    report_url = f"http://127.0.0.1:{port}"
+    click.echo(f"Dash report will run on {report_url}/")
+    click.echo("Attempting to open report in your default browser...")
+
+    # We don't need a Timer here if show_report is blocking and we open BEFORE
+    # However, in some systems, opening too fast can fail. A small delay is safer.
+    def _open_browser():
+        # This check is still useful if Werkzeug debug mode spawns a second process
+        # and you want to ensure it only opens once from the parent.
+        if not os.environ.get("WERKZEUG_RUN_MAIN"):
+            webbrowser.open_new_tab(report_url)
+
+    # Schedule the browser open for slightly in the future to give the OS a moment
+    # and to potentially allow Dash's own startup messages to appear first.
+    Timer(1, _open_browser).start() # 0.5 seconds delay
+
+    click.echo("Showing report...")
+    # Pass the port to show_report. You must ensure show_report accepts this.
+    show_report(my_project, port=port)
+
+if __name__ == '__main__':
+    cli()

pixel_patrol_base-0.1.0/src/pixel_patrol_base/config.py

@@ -0,0 +1,7 @@
+MIN_N_EXAMPLE_FILES: int = 1
+MAX_N_EXAMPLE_FILES: int = 20
+DEFAULT_N_EXAMPLE_FILES: int = 9
+MAX_ROWS_DISPLAYED = 100
+MAX_COLS_DISPLAYED = 200
+SPRITE_SIZE = 64
+

pixel_patrol_base-0.1.0/src/pixel_patrol_base/core/contracts.py

@@ -0,0 +1,34 @@
+from typing import Protocol, Iterable, Set, Any, Dict, List
+
+import polars as pl
+from pathlib import Path
+
+from pixel_patrol_base.core.record import Record
+from pixel_patrol_base.core.specs import ProcessResult, RecordSpec, ProcessorOutput
+
+
+class PixelPatrolLoader(Protocol):
+    NAME: str
+    SUPPORTED_EXTENSIONS: Set[str]
+    OUTPUT_SCHEMA: Dict[str, Any]
+    OUTPUT_SCHEMA_PATTERNS: List[tuple[str, Any]]
+    FOLDER_EXTENSIONS: Set[str]
+    def is_folder_supported(self, path: Path) -> bool: ...
+    def load(self, source: str) -> Record: ...
+
+
+class PixelPatrolProcessor(Protocol):
+    NAME: str
+    INPUT: RecordSpec
+    OUTPUT: ProcessorOutput            # "features" or "record"
+    def run(self, art: Record) -> ProcessResult: ...
+
+
+class PixelPatrolWidget(Protocol):
+    NAME: str                       # human readable name
+    TAB: str                        # WidgetCategories value
+    REQUIRES: Set[str]              # columns required to render
+    REQUIRES_PATTERNS: Iterable[str] | None  # optional regexes for dynamic cols
+
+    def layout(self) -> list: ...
+    def register(self, app, df_global: pl.DataFrame) -> None: ...

pixel_patrol_base-0.1.0/src/pixel_patrol_base/core/feature_schema.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, Iterable, List, Tuple, Type
+
+from pixel_patrol_base.core.contracts import PixelPatrolLoader
+
+Schema = Dict[str, Any]
+PatternSpec = List[Tuple[str, Any]]
+
+def merge_output_schemas(processors: Iterable[object]) -> Tuple[Schema, PatternSpec]:
+    static: Schema = {}
+    patterns: PatternSpec = []
+    for p in processors:
+        static.update(getattr(p, "OUTPUT_SCHEMA", {}) or {})
+        patterns += getattr(p, "OUTPUT_SCHEMA_PATTERNS", []) or []
+    return static, patterns
+
+def infer_col_type(col: str, static: Schema, patterns: PatternSpec):
+    if col in static:
+        return static[col]
+    for pat, typ in patterns:
+        if re.match(pat, col):
+            return typ
+    return None  # unknown => leave as-is
+
+def coerce_row_types(row: Dict[str, Any], static: Schema, patterns: PatternSpec) -> Dict[str, Any]:
+    out = dict(row)
+    for k, v in row.items():
+        want = infer_col_type(k, static, patterns)
+        if want is None or v is None:
+            continue
+        try:
+            if want is list and not isinstance(v, list): out[k] = [v]
+            elif want is dict and not isinstance(v, dict): pass  # recommend JSON string instead
+            else: out[k] = want(v)  # int/float/str
+        except Exception:
+            # keep original if coercion fails
+            pass
+    return out
+
+
+def get_requirements_as_patterns(component: Type[PixelPatrolLoader]) -> List[str]:
+    """
+    Consolidates a component's (loader or processor) static and dynamic
+    column specifications into a single list of regex patterns.
+
+    Args:
+        component: An instance of a loader or processor.
+
+    Returns:
+        A list of regex strings representing all required columns.
+    """
+    # 1. Get exact keys from the static specification.
+    exact_keys_as_patterns = [
+        f"^{re.escape(key)}$" for key in component.OUTPUT_SCHEMA.keys()
+    ]
+
+    # 2. Get the regex patterns from the dynamic specification.
+    dynamic_patterns = [
+        pattern_tuple[0] for pattern_tuple in component.OUTPUT_SCHEMA_PATTERNS
+    ]
+
+    return exact_keys_as_patterns + dynamic_patterns
+
+
+def get_dynamic_patterns(component: Type[PixelPatrolLoader]) -> List[str]:
+    return [
+        pattern_tuple[0] for pattern_tuple in component.OUTPUT_SCHEMA_PATTERNS
+    ]
+
+def patterns_from_processor(P) -> List[str]:
+    """
+    Extract regex strings from a processor's declarative OUTPUT_SCHEMA_PATTERNS.
+    Accepts either a class or an instance.
+    """
+    schema_patterns = getattr(P, "OUTPUT_SCHEMA_PATTERNS", None)
+    if schema_patterns is None and hasattr(P, "__class__"):
+        schema_patterns = getattr(P.__class__, "OUTPUT_SCHEMA_PATTERNS", None)
+
+    pats: List[str] = []
+    if schema_patterns:
+        for pat, _typ in schema_patterns:
+            pats.append(getattr(pat, "pattern", pat))  # handle compiled or plain string
+    return pats

pixel_patrol_base-0.1.0/src/pixel_patrol_base/core/file_system.py

@@ -0,0 +1,159 @@
+import logging
+from typing import Dict, Any, List, Literal, Optional, Set
+import os
+from datetime import datetime
+from pathlib import Path
+import polars as pl
+
+from pixel_patrol_base.utils.utils import format_bytes_to_human_readable
+from pixel_patrol_base.core.contracts import PixelPatrolLoader
+
+logger = logging.getLogger(__name__)
+
+
+def make_basic_record(path: Path, base: Path, is_folder: bool = False) -> Dict[str, Any]:
+    """
+    Create a basic metadata record for a file or folder,
+    computing depth relative to `base` and normalizing extensions.
+    """
+    try:
+        stat_func = path.stat if not is_folder else lambda: None
+        st = stat_func() if not is_folder else None
+    except Exception as e:
+        logger.warning(f"Failed stat for {path}: {e}")
+        return {}
+
+    depth = len(path.parts) - len(base.parts)
+
+    # TODO: I guess we're missing imported_path_short and modification_month that were created in preprocess_files
+    # common_base = find_common_base(unique_folders) - should be added after
+    # pl.col("modification_date").dt.month().alias("modification_month"),
+    # pl.col("imported_path").str.replace(common_base, "", literal=True).alias("imported_path_short"),
+    record: Dict[str, Any] = {
+        "path": str(path),
+        "name": path.name,
+        "type": "folder" if is_folder else "file",
+        "parent": str(path.parent) if path != base else None,
+        "depth": depth,
+        "size_bytes": 0 if is_folder else st.st_size,
+        "modification_date": datetime.fromtimestamp(os.path.getmtime(path)),
+        "file_extension": None if is_folder else path.suffix.lstrip(".").lower(),
+        "imported_path": str(base),
+    }
+    return record
+
+
+def walk_filesystem(
+    bases: List[Path],
+    accepted_extensions: Set[str] | Literal["all"],
+    loader: Optional[PixelPatrolLoader] = None,
+) -> pl.DataFrame:
+    """
+    - Only include files and loader-supported folder datasets (no plain directories).
+    - accepted_extensions == "all": include all files + any folder datasets supported by the loader.
+    - accepted_extensions is a set: include files with suffix in set; include folder datasets only if they intersect loader.FOLDER_EXTENSIONS.
+    """
+    records: List[dict] = []
+    include_all = accepted_extensions == "all"
+
+    is_folder_check = (loader is not None) and \
+                      hasattr(loader, "is_folder_supported")  and \
+                      (include_all or
+                       not accepted_extensions.isdisjoint(getattr(loader, "FOLDER_EXTENSIONS", set())))
+    folder_support_fn = loader.is_folder_supported if is_folder_check else None
+
+    for base in bases:
+        for root, dirnames, filenames in os.walk(base, topdown=True):
+            dirpath = Path(root)
+
+            keep: List[str] = []
+
+            if is_folder_check:
+                for d in dirnames:
+                    sub = dirpath / d
+                    if folder_support_fn(sub):
+                        records.append(make_basic_record(sub, base, is_folder=False))
+                    else:
+                        keep.append(d)
+                dirnames[:] = keep
+
+            # Files
+            for name in filenames:
+                p = dirpath / name
+                if include_all or p.suffix.lower().lstrip(".") in accepted_extensions:
+                    records.append(make_basic_record(p, base, is_folder=False))
+
+    return pl.DataFrame(records) if records else pl.DataFrame()
+
+
+def _aggregate_folder_sizes(df: pl.DataFrame) -> pl.DataFrame:
+    """
+    Aggregates file sizes up to their parent folders in the DataFrame.
+    Assumes df contains 'path', 'type', 'parent', 'size_bytes', 'depth' columns.
+    This version aims to be more Polars-idiomatic.
+    """
+    if df.is_empty():
+        return df
+
+    # Ensure 'size_bytes' is numerical
+    df = df.with_columns(pl.col("size_bytes").cast(pl.Int64))
+
+    # Initialize a 'current_size' column that will be updated
+    # Files keep their original size. Folders initially have 0 or their own direct size if applicable.
+    # The sum for folders will be calculated from their children.
+    df = df.with_columns(
+        pl.when(pl.col("type") == "file")
+        .then(pl.col("size_bytes"))
+        .otherwise(0)  # Start folder size from 0, or could be initial direct size if it applies
+        .alias("temp_calculated_size")
+    )
+
+    # Get unique depths in reverse order to process from leaves upwards
+    # Filter out folders at depth 0, as they might not have a parent in the dataframe to aggregate to.
+    unique_depths = sorted(df["depth"].unique().to_list(), reverse=True)
+
+    # If your base directory is included as a folder with depth 0 and no parent in the df,
+    # the aggregation will stop there. This is generally desired.
+
+    # Iterate from deepest folders up to the base-level folders
+    for current_depth in unique_depths:
+        # Sum sizes of direct children at (current_depth + 1) for parents at current_depth
+        # We need to compute the sum of 'temp_calculated_size' for all children
+        # grouped by their 'parent' path (which corresponds to the current folder's path).
+
+        # Calculate children sizes to aggregate to parents at current_depth
+        # This aggregates sizes of *all* items (files and subfolders) at depth 'current_depth'
+        # based on their 'parent' column.
+
+        children_sums_for_parents = df.filter(pl.col("depth") == current_depth + 1) \
+            .group_by("parent") \
+            .agg(pl.col("temp_calculated_size").sum().alias("children_total_size"))
+
+        # Now, join these sums back to the main DataFrame
+        # Update the 'temp_calculated_size' for folders at 'current_depth'
+        # by adding the sum of their children.
+
+        df = df.join(
+            children_sums_for_parents,
+            left_on="path",  # Folder's path is the parent for its children
+            right_on="parent",
+            how="left"
+        ).with_columns(
+            pl.when(pl.col("type") == "folder")
+            .then(
+                pl.col("temp_calculated_size") + pl.col("children_total_size").fill_null(0)
+            )
+            .otherwise(pl.col("temp_calculated_size"))  # Files keep their original size
+            .alias("temp_calculated_size")
+        ).drop("children_total_size")  # Drop the temporary join column
+
+    # After aggregation, the 'temp_calculated_size' column contains the final aggregated sizes.
+    # Replace the original 'size_bytes' with this aggregated column.
+    df = df.with_columns(pl.col("temp_calculated_size").alias("size_bytes")).drop("temp_calculated_size")
+
+    # Drop the temporary Path objects if they were created before
+    # (In this revised version, we don't create path_obj/parent_obj explicitly in the DF)
+    # If the initial scan_directory_to_dataframe already returns Path objects and they are stored
+    # as object dtype, they would need to be handled, but it's better to store strings then convert as needed.
+
+    return df