marEx 4.0__tar.gz → 4.0.3__tar.gz

marex-4.0.3/.codecov.yml

@@ -0,0 +1,34 @@
+# Codecov configuration for marEx
+# https://docs.codecov.com/docs/codecov-yaml
+
+coverage:
+  status:
+    # Overall project coverage
+    project:
+      default:
+        target: 95%  # Maintain 95% overall project coverage
+        threshold: 1%
+        if_ci_failed: error
+
+    # Coverage for new code in patches/PRs - disabled
+    patch:
+      default:
+        enabled: false  # Do not enforce patch coverage requirements
+
+# Comment configuration
+comment:
+  layout: "reach,diff,flags,tree"
+  behavior: default
+  require_changes: false
+  require_base: false
+  require_head: true
+
+# Ignore paths
+ignore:
+  - "tests/"
+  - "docs/"
+  - "examples/"
+  - "setup.py"
+  - "**/_dependencies.py"
+  - "**/_coverage_init.py"
+  - "**/plotX/unstructured.py"

{marex-4.0 → marex-4.0.3}/.github/workflows/ci.yml

@@ -64,6 +64,7 @@ jobs:
     name: Test Suite
     runs-on: ${{ matrix.os }}
     needs: code-quality
+    timeout-minutes: 90  # Windows can be significantly slower than Linux/macOS
     strategy:
       fail-fast: false
       matrix:
@@ -144,18 +145,22 @@ jobs:
         pytest tests/test_logging_system.py -n 2 -v --tb=short
 
     - name: Run preprocessing tests (gridded)
+      timeout-minutes: 30  # Windows can be slow for these tests
       run: |
         pytest tests/test_gridded_preprocessing.py -n 2 -v --tb=short
 
     - name: Run preprocessing tests (unstructured)
+      timeout-minutes: 30  # Windows can be slow for these tests
       run: |
         pytest tests/test_unstructured_preprocessing.py -n 2 -v --tb=short
 
     - name: Run tracking tests (gridded)
+      timeout-minutes: 30  # Windows can be slow for these tests
       run: |
         pytest tests/test_gridded_tracking.py -n 2 -v --tb=short
 
     - name: Run tracking tests (unstructured)
+      timeout-minutes: 30  # Windows can be slow for these tests
       run: |
         pytest tests/test_unstructured_tracking.py -n 2 -v --tb=short
 

{marex-4.0/marEx.egg-info → marex-4.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: marEx
-Version: 4.0
+Version: 4.0.3
 Summary: Marine Extremes Detection and Tracking
 Author-email: Aaron Wienkers <aaron@wienkers.com>
 Project-URL: Homepage, https://github.com/wienkers/marEx

marex-4.0.3/examples/batch jobs/run_detect.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+"""
+Global Daily SST (OSTIA) Analysis: Identifying Marine Extremes with MarEx-Detect
+
+Pre-Process with Shifting-Baseline & Hobday et al. (2016) Definition for Extremes
+
+Run from login node - this script will launch SLURM jobs via Dask distributed cluster
+
+NOTE: This script uses a distributed cluster because detect.py does not contain
+Numba JIT-compiled functions that execute locally. Unlike run_track.py, this
+approach works correctly for detection/preprocessing tasks. If you prefer to
+follow HPC best practices and run everything in SLURM jobs, consider creating
+a submit_detect.sh script similar to submit_track.sh.
+"""
+
+import os
+from getpass import getuser
+from pathlib import Path
+
+import xarray as xr
+
+import marEx
+import marEx.helper as hpc
+
+
+def main():
+    """Run MarEx preprocessing and extreme detection pipeline on distributed SLURM cluster."""
+    # Configuration from environment variables
+    n_workers = int(os.getenv("DASK_N_WORKERS", "128"))
+    workers_per_node = int(os.getenv("DASK_WORKERS_PER_NODE", "64"))
+    runtime = int(os.getenv("DASK_RUNTIME", "39"))
+    slurm_account = os.getenv("SLURM_ACCOUNT", "bk1377")
+
+    # Lustre Scratch Directory
+    scratch_dir = Path("/scratch") / getuser()[0] / getuser()
+
+    # Start Distributed Dask Cluster on SLURM
+    print("Starting distributed Dask cluster on SLURM...")
+    print(f"  Workers: {n_workers}")
+    print(f"  Workers per node: {workers_per_node}")
+    print(f"  Runtime: {runtime} minutes")
+    print(f"  SLURM account: {slurm_account}")
+
+    client = hpc.start_distributed_cluster(
+        n_workers=n_workers,
+        workers_per_node=workers_per_node,
+        runtime=runtime,
+        scratch_dir=scratch_dir / "clients",
+        account=slurm_account,
+    )
+
+    # Choose optimal chunk size & load data
+    print("Loading data...")
+    file_name = scratch_dir / "mhws" / "ostia.zarr"
+    chunk_size = {"time": 30, "lat": 360, "lon": 720}
+    sst = xr.open_zarr(str(file_name), chunks=chunk_size).sst
+    print(f"Data loaded: {sst}")
+
+    # Process Data using MarEx-Detect helper functions:
+    print("Processing extremes data...")
+    extremes_ds = marEx.preprocess_data(
+        sst,
+        method_anomaly="shifting_baseline",  # Anomalies from a rolling climatology using previous window_year years
+        method_extreme="hobday_extreme",  # Local day-of-year specific thresholds with windowing
+        threshold_percentile=95,  # Use the 95th percentile as the extremes threshold
+        window_year_baseline=15,
+        smooth_days_baseline=21,  # Defines the rolling climatology window (15 years) and smoothing window (21 days)
+        window_days_hobday=11,  # Defines the window (11 days) of compiled samples collected for extremes detection
+        dimensions={
+            "time": "time",
+            "x": "lon",
+            "y": "lat",
+        },  # Define the dimensions of the data -- if 'y' exists, MarEx-Detect knows this is a gridded dataset
+        dask_chunks={"time": 25},  # Dask chunks for *output* data
+        use_temp_checkpoints=True,  # Enable checkpointing to prevent expensive recomputations
+    )
+    print(f"Processing complete: {extremes_ds}")
+
+    # Save Extremes Data to zarr for more efficient parallel I/O
+    print("Saving extremes data...")
+    output_file = scratch_dir / "mhws" / "extremes_binary_gridded_shifting_hobday_batch.zarr"
+    extremes_ds.to_zarr(output_file, mode="w")
+    print(f"Data saved to: {output_file}")
+
+    # Close the Dask cluster
+    client.close()
+    print("Processing complete!")
+
+
+if __name__ == "__main__":
+    main()

marex-4.0.3/examples/batch jobs/run_track.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+"""
+Global Daily Event Analysis: Marine Heatwave ID & Tracking using MarEx
+
+Performs morphological preprocessing, blob identification, temporal tracking,
+and graph reduction to create globally unique IDs for each tracked extreme event.
+
+This script should be run INSIDE a SLURM job (via sbatch submit_track.sh).
+It uses a local Dask cluster that utilizes the compute resources allocated to the job.
+This ensures that all computation, including Numba JIT-compiled functions, runs on
+compute nodes rather than the login node.
+"""
+
+import os
+from getpass import getuser
+from pathlib import Path
+
+import xarray as xr
+
+import marEx
+import marEx.helper as hpc
+
+
+def main():
+    """Run MarEx event tracking pipeline on local Dask cluster within SLURM job allocation."""
+    # Configuration from environment variables
+    n_workers = int(os.getenv("DASK_N_WORKERS", "32"))
+    threads_per_worker = int(os.getenv("DASK_THREADS_PER_WORKER", "1"))
+
+    # Tracking configuration
+    run_basic = os.getenv("RUN_BASIC_TRACKER", "false").lower() == "true"
+    grid_resolution = float(os.getenv("GRID_RESOLUTION", "0.25"))
+    area_filter = int(os.getenv("AREA_FILTER", "600"))
+    r_fill = int(os.getenv("R_FILL", "12"))
+    t_fill = int(os.getenv("T_FILL", "4"))
+    overlap_threshold = float(os.getenv("OVERLAP_THRESHOLD", "0.25"))
+
+    # Lustre Scratch Directory
+    scratch_dir = Path("/scratch") / getuser()[0] / getuser()
+
+    # Start Local Dask Cluster
+    # This cluster runs within the SLURM job allocation, ensuring all computation
+    # (including Numba JIT-compiled functions) executes on compute nodes
+    print("Starting local Dask cluster within SLURM job allocation...")
+    print(f"  Workers: {n_workers}")
+    print(f"  Threads per worker: {threads_per_worker}")
+
+    client = hpc.start_local_cluster(
+        n_workers=n_workers,
+        threads_per_worker=threads_per_worker,
+        scratch_dir=scratch_dir / "clients",
+    )
+
+    # Choose optimal chunk size & load preprocessed data
+    print("\nLoading preprocessed extremes data...")
+    input_file = scratch_dir / "mhws" / "extremes_binary_gridded_shifting_hobday_batch.zarr"
+    chunk_size = {"time": 25, "lat": -1, "lon": -1}
+    ds = xr.open_zarr(str(input_file), chunks=chunk_size)
+    print(f"Data loaded: {ds}")
+
+    # Run ID, Tracking, & Merging
+    print("\n" + "=" * 80)
+    print("Running Full Tracking with Merge/Split Detection...")
+    print("=" * 80)
+
+    tracker = marEx.tracker(
+        ds.extreme_events,
+        ds.mask.where(
+            (ds.lat < 85) & (ds.lat > -90), other=False
+        ),  # Modify Mask: Anisotropy of lat/lon grid near poles biases ID & Tracking
+        grid_resolution=grid_resolution,  # Grid resolution in degrees, used to calculate object areas on the globe
+        area_filter_absolute=area_filter,  # Remove objects smaller than specified cells
+        R_fill=r_fill,  # Fill small holes with radius < R_fill _cells_
+        T_fill=t_fill,  # Allow gaps of T_fill days and still continue event tracking with same ID
+        allow_merging=True,  # Allow extreme events to split/merge. Keeps track of merge events & unique IDs
+        overlap_threshold=overlap_threshold,  # Overlap threshold for merging events
+        nn_partitioning=True,  # Use new NN method to partition merged children areas
+        verbose=True,
+    )
+
+    extreme_events_ds, merges_ds = tracker.run(return_merges=True)
+    print(f"\nTracking complete: {extreme_events_ds}")
+    print(f"Merges dataset: {merges_ds}")
+
+    # Save IDed/Tracked/Merged Events to zarr for efficient parallel I/O
+    print("\nSaving tracked events data...")
+    output_file = scratch_dir / "mhws" / "extreme_events_merged_gridded_shifting_batch.zarr"
+    extreme_events_ds.to_zarr(output_file, mode="w")
+    print(f"Events data saved to: {output_file}")
+
+    # Save Merges Dataset to netcdf
+    print("Saving merges data...")
+    merges_file = scratch_dir / "mhws" / "extreme_events_merged_gridded_shifting_merges_batch.nc"
+    merges_ds.to_netcdf(merges_file, mode="w")
+    print(f"Merges data saved to: {merges_file}")
+
+    # Optionally run basic tracking for comparison
+    if run_basic:
+        print("\n" + "=" * 80)
+        print("Running Basic Tracking (No Merge/Split, No Temporal Gap Filling)...")
+        print("=" * 80)
+
+        tracker_basic = marEx.tracker(
+            ds.extreme_events,
+            ds.mask.where(
+                (ds.lat < 85) & (ds.lat > -90), other=False
+            ),  # Modify Mask: Anisotropy of lat/lon grid near poles biases ID & Tracking
+            area_filter_absolute=area_filter,  # Remove objects smaller than specified cells
+            R_fill=r_fill,  # Fill small holes with radius < R_fill _cells_
+            T_fill=0,  # No temporal hole filling
+            allow_merging=False,  # Do not allow extreme events to split/merge
+        )
+
+        extreme_events_basic_ds = tracker_basic.run()
+        print(f"\nBasic tracking complete: {extreme_events_basic_ds}")
+
+        # Save Basic IDed Events to zarr
+        print("\nSaving basic tracked events data...")
+        basic_output_file = scratch_dir / "mhws" / "extreme_events_basic_gridded_shifting_batch.zarr"
+        extreme_events_basic_ds.to_zarr(basic_output_file, mode="w")
+        print(f"Basic events data saved to: {basic_output_file}")
+
+    # Close the Dask cluster
+    client.close()
+    print("\n" + "=" * 80)
+    print("Tracking complete!")
+    print("=" * 80)
+
+
+if __name__ == "__main__":
+    main()

marex-4.0.3/examples/batch jobs/submit_track.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+#SBATCH --job-name=marEx_track
+#SBATCH --partition=compute
+#SBATCH --cpus-per-task=128
+#SBATCH --account=bk1377
+#SBATCH --output=slurm_%j.out
+#SBATCH --time=02:59:00
+#SBATCH --mem=0
+#SBATCH --exclusive
+
+# SLURM Batch Script for MarEx Event Tracking
+#
+# This script submits run_track.py as a SLURM job, ensuring all computation
+# (including Numba JIT-compiled functions) runs on allocated compute nodes.
+#
+# Usage:
+#   sbatch submit_track.sh
+#
+# Configuration via environment variables (optional):
+#   export DASK_N_WORKERS=32         # Number of Dask workers
+#   export DASK_THREADS_PER_WORKER=1 # Threads per worker
+#   export RUN_BASIC_TRACKER=true    # Run basic tracker comparison
+#   export GRID_RESOLUTION=0.25      # Grid resolution in degrees
+#   export AREA_FILTER=600           # Minimum object area (cells)
+#   export R_FILL=12                 # Spatial hole filling radius
+#   export T_FILL=4                  # Temporal gap filling (days)
+#   export OVERLAP_THRESHOLD=0.25    # Merge overlap threshold
+#   sbatch submit_track.sh
+
+# Print job information
+echo "========================================="
+echo "SLURM Job Information"
+echo "========================================="
+echo "Job ID: $SLURM_JOB_ID"
+echo "Job Name: $SLURM_JOB_NAME"
+echo "Node: $SLURM_NODELIST"
+echo "Start Time: $(date)"
+echo "Working Directory: $(pwd)"
+echo "========================================="
+echo ""
+
+# Set ulimits for stack size
+ulimit -S -s unlimited
+ulimit -S -c unlimited
+
+# Initialise conda for bash shell
+source $(conda info --base)/etc/profile.d/conda.sh
+conda activate super
+
+# Set default environment variables if not already set
+export DASK_N_WORKERS=${DASK_N_WORKERS:-32}
+export DASK_THREADS_PER_WORKER=${DASK_THREADS_PER_WORKER:-1}
+
+# Print configuration
+echo "Configuration:"
+echo "  DASK_N_WORKERS: $DASK_N_WORKERS"
+echo "  DASK_THREADS_PER_WORKER: $DASK_THREADS_PER_WORKER"
+echo "  RUN_BASIC_TRACKER: ${RUN_BASIC_TRACKER:-false}"
+echo "  GRID_RESOLUTION: ${GRID_RESOLUTION:-0.25}"
+echo "  AREA_FILTER: ${AREA_FILTER:-600}"
+echo "  R_FILL: ${R_FILL:-12}"
+echo "  T_FILL: ${T_FILL:-4}"
+echo "  OVERLAP_THRESHOLD: ${OVERLAP_THRESHOLD:-0.25}"
+echo ""
+
+# Run the tracking script
+echo "Starting MarEx tracking..."
+python run_track.py
+
+# Capture exit status
+EXIT_STATUS=$?
+
+echo ""
+echo "========================================="
+echo "Job completed at: $(date)"
+echo "Exit status: $EXIT_STATUS"
+echo "========================================="
+
+exit $EXIT_STATUS

{marex-4.0 → marex-4.0.3}/marEx/__init__.py

@@ -57,6 +57,9 @@ from .exceptions import (  # Main exception hierarchy; Convenience constructors
     wrap_exception,
 )
 
+# Import HPC helper utilities
+from .helper import configure_dask
+
 # Import logging configuration functions
 from .logging_config import (
     configure_logging,
@@ -116,6 +119,8 @@ __all__ = [
     "is_verbose_mode",
     "is_quiet_mode",
     "get_logger",
+    # HPC helper utilities
+    "configure_dask",
 ]
 
 # Version information