xdiffly 0.2.6__py3-none-any.whl

xdiff/__init__.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import sys
+
+from xdiff import conf as settings
+from xdiff.utils.log import configure_logging
+
+MIN_SUPPORTED_PYTHON = (3, 10)
+MAX_SUPPORTED_PYTHON = (3, 14)
+
+
+def validate_runtime() -> None:
+    version = sys.version_info[:2]
+    if MIN_SUPPORTED_PYTHON <= version < MAX_SUPPORTED_PYTHON:
+        return
+
+    raise RuntimeError(
+        "xdiff supports Python 3.10 through 3.13. "
+        f"The current interpreter is Python {version[0]}.{version[1]}."
+    )
+
+
+def setup():
+    validate_runtime()
+    configure_logging(settings.LOGGING_CONFIG, settings.LOGGING)

xdiff/comparators/__init__.py

@@ -0,0 +1,19 @@
+"""Comparator implementations for each artifact type."""
+
+from importlib import import_module
+
+from xdiff.comparators.base import ArtifactComparator
+
+__all__ = ["ArtifactComparator", "NetcdfComparator"]
+
+
+def __getattr__(name: str):
+    if name == "NetcdfComparator":
+        comparator = getattr(import_module("xdiff.comparators.netcdf"), name)
+        globals()[name] = comparator
+        return comparator
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__() -> list[str]:
+    return sorted(__all__)

xdiff/comparators/base.py

@@ -0,0 +1,20 @@
+"""Base interfaces for artifact comparators."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+from xdiff.model.artifact import ArtifactKind
+from xdiff.model.comparison import Comparison
+from xdiff.model.match import ArtifactMatch
+from xdiff.model.request import CompareRequest
+
+
+class ArtifactComparator(ABC):
+    """Compares a matched pair of artifacts."""
+
+    artifact_kind: ArtifactKind
+
+    @abstractmethod
+    def compare(self, match: ArtifactMatch, request: CompareRequest) -> Comparison:
+        """Return the comparison outcome for a single matched pair."""

xdiff/comparators/netcdf.py

@@ -0,0 +1,286 @@
+"""Comparator for netCDF artifacts and related numeric helpers."""
+
+from __future__ import annotations
+
+import logging
+import warnings
+
+from functools import lru_cache
+from importlib import import_module
+from typing import TYPE_CHECKING, Any, Iterable
+
+import numpy as np
+
+import xdiff.conf as settings
+
+from xdiff.comparators.base import ArtifactComparator
+from xdiff.exceptions import AllNaN, LastTimestepTimeCheckException
+from xdiff.model import CompareResult
+from xdiff.model.artifact import ArtifactKind
+from xdiff.model.comparison import Comparison
+from xdiff.model.match import ArtifactMatch
+from xdiff.model.request import CompareRequest
+
+warnings.filterwarnings("ignore", message="All-NaN slice encountered")
+
+logger = logging.getLogger("xdiff")
+
+if TYPE_CHECKING:
+    import xarray as xr
+
+
+@lru_cache(maxsize=1)
+def load_xarray():
+    """Import xarray only when a netCDF comparison is actually requested."""
+    try:
+        return import_module("xarray")
+    except ImportError as exc:
+        raise RuntimeError(
+            "xarray is required to compare netCDF files. Install the package dependencies before using this feature."
+        ) from exc
+
+
+class NetcdfComparator(ArtifactComparator):
+    """Compare two netCDF files using xarray-backed numeric checks."""
+
+    artifact_kind = ArtifactKind.NETCDF
+
+    def compare(self, match: ArtifactMatch, request: CompareRequest) -> Comparison:
+        comparison = Comparison(
+            reference_artifact=match.reference,
+            comparison_artifact=match.comparison,
+        )
+
+        if match.comparison is None:
+            raise ValueError("A comparison artifact is required for netCDF comparison")
+
+        comparison.extend(
+            compare_files(
+                match.reference.path,
+                match.comparison.path,
+                request.variables,
+                last_time_step=request.last_time_step,
+            )
+        )
+        return comparison
+
+
+def compare_files(
+    file1,
+    file2,
+    variables: tuple[str, ...] | list[str] | object | None,
+    *,
+    last_time_step: bool,
+) -> list[CompareResult]:
+    xr = load_xarray()
+    with xr.open_dataset(file1) as dataset1, xr.open_dataset(file2) as dataset2:
+        variables_to_compare = get_dataset_variables(dataset1, variables)
+        return compare_datasets(
+            dataset1,
+            dataset2,
+            variables_to_compare,
+            last_time_step=last_time_step,
+        )
+
+
+def compare_datasets(
+    reference: xr.Dataset,
+    comparison: xr.Dataset,
+    variables: list[str],
+    *,
+    last_time_step: bool,
+) -> list[CompareResult]:
+    results: list[CompareResult] = []
+
+    for variable in variables:
+        logger.info("Comparing %s", variable)
+        try:
+            reference_field = reference[variable]
+            comparison_field = comparison[variable]
+            results.append(
+                compare_variables(
+                    reference_field,
+                    comparison_field,
+                    variable,
+                    last_time_step=last_time_step,
+                )
+            )
+        except Exception as exc:
+            results.append(CompareResult(variable=variable, description=str(exc)))
+
+    return results
+
+
+def compare_variables(
+    ref_da: xr.DataArray,
+    cmp_da: xr.DataArray,
+    variable: str,
+    *,
+    last_time_step: bool,
+) -> CompareResult:
+    if last_time_step:
+        if is_time_coordinate_variable(variable, ref_da, cmp_da):
+            raise LastTimestepTimeCheckException("Can't compare time if last time step is enabled")
+        ref_da = select_last_time_step(ref_da)
+        cmp_da = select_last_time_step(cmp_da)
+
+    validate_matching_metadata(ref_da, cmp_da)
+
+    reference_values = ref_da.values
+    comparison_values = cmp_da.values
+    reference_masked = ref_da.to_masked_array()
+    comparison_masked = cmp_da.to_masked_array()
+
+    difference_field = reference_values - comparison_values
+
+    if np.isnan(difference_field).all():
+        raise AllNaN("All nan values found")
+
+    mask_is_equal = np.array_equal(reference_masked.mask, comparison_masked.mask)
+
+    return CompareResult(
+        relative_error=compute_relative_error(difference_field, comparison_values),
+        min_diff=np.nanmin(difference_field),
+        max_diff=np.nanmax(difference_field),
+        mask_equal=mask_is_equal,
+        variable=variable,
+    )
+
+
+
+
+def select_last_time_step(field: xr.DataArray) -> xr.DataArray:
+    time_dimension = find_time_dims_name(field.dims)
+    if time_dimension is None:
+        return field
+
+    if field.sizes[time_dimension] > 1:
+        return field.isel({time_dimension: slice(-1, None)})
+    return field
+
+
+def find_time_dims_name(dims: Iterable) -> Any | None:
+    time_dimensions = [dimension for dimension in dims if "time" in dimension]
+    if len(time_dimensions) == 0:
+        return None
+    if len(time_dimensions) > 1:
+        raise ValueError(f"Found more than 1 time dimension: {', '.join(time_dimensions)}")
+    return time_dimensions.pop()
+
+
+def compute_relative_error(diff: np.ndarray, field2: np.ndarray):
+    if np.all(diff == 0.0):
+        return 0.0
+
+    if is_time_dtype(field2.dtype):
+        field2_values = field2.view("int64")
+    else:
+        field2_values = field2
+
+    abs_diff = np.abs(diff)
+    abs_field2 = np.abs(field2_values)
+
+    try:
+        with np.errstate(divide="ignore", invalid="ignore"):
+            rel_err_array = abs_diff / abs_field2
+            if np.isinf(rel_err_array).any():
+                rel_err_array[np.isinf(rel_err_array)] = np.nan
+            rel_err = np.nanmax(rel_err_array)
+    except Exception as exc:
+        logger.debug("An error occurred when computing relative error: %s", exc)
+        rel_err = np.nan
+
+    if is_time_dtype(field2.dtype):
+        return rel_err / np.timedelta64(1, "s")
+    return rel_err
+
+
+def is_time_dtype(dtype) -> bool:
+    normalized_dtype = np.dtype(dtype)
+    return np.issubdtype(normalized_dtype, np.datetime64) or np.issubdtype(normalized_dtype, np.timedelta64)
+
+
+def is_time_coordinate_variable(variable: str, ref_da: xr.DataArray, cmp_da: xr.DataArray) -> bool:
+    time_dimension = find_time_dims_name(ref_da.dims)
+    comparison_time_dimension = find_time_dims_name(cmp_da.dims)
+
+    if time_dimension != comparison_time_dimension or time_dimension is None:
+        return False
+
+    return (
+        variable == time_dimension
+        and ref_da.dims == (time_dimension,)
+        and cmp_da.dims == (time_dimension,)
+        and is_time_dtype(ref_da.dtype)
+        and is_time_dtype(cmp_da.dtype)
+    )
+
+
+def validate_matching_metadata(ref_da: xr.DataArray, cmp_da: xr.DataArray) -> None:
+    if ref_da.dims != cmp_da.dims:
+        raise ValueError(f"Dimension mismatch: '{ref_da.dims}' - '{cmp_da.dims}'")
+
+    reference_sizes = tuple(ref_da.sizes[dimension] for dimension in ref_da.dims)
+    comparison_sizes = tuple(cmp_da.sizes[dimension] for dimension in cmp_da.dims)
+    if reference_sizes != comparison_sizes:
+        raise ValueError(f"Dimension size mismatch: '{reference_sizes}' - '{comparison_sizes}'")
+
+    if np.dtype(ref_da.dtype) != np.dtype(cmp_da.dtype):
+        raise ValueError(f"Data type mismatch: '{ref_da.dtype}' - '{cmp_da.dtype}'")
+
+    reference_coordinates = set(ref_da.coords)
+    comparison_coordinates = set(cmp_da.coords)
+    if reference_coordinates != comparison_coordinates:
+        logger.debug(
+            "Coordinate mismatch: '%s' - '%s'",
+            ", ".join(sorted(reference_coordinates)) or "-",
+            ", ".join(sorted(comparison_coordinates)) or "-",
+        )
+
+    for coordinate_name in sorted(reference_coordinates & comparison_coordinates):
+        reference_coordinate = ref_da.coords[coordinate_name]
+        comparison_coordinate = cmp_da.coords[coordinate_name]
+
+        if reference_coordinate.dims != comparison_coordinate.dims:
+            logger.debug(
+                "Coordinate dimension mismatch for '%s': '%s' - '%s'",
+                coordinate_name,
+                reference_coordinate.dims,
+                comparison_coordinate.dims,
+            )
+            continue
+
+        if np.dtype(reference_coordinate.dtype) != np.dtype(comparison_coordinate.dtype):
+            logger.debug(
+                "Coordinate type mismatch for '%s': '%s' - '%s'",
+                coordinate_name,
+                reference_coordinate.dtype,
+                comparison_coordinate.dtype,
+            )
+            continue
+
+        if not reference_coordinate.equals(comparison_coordinate):
+            logger.debug("Coordinate values mismatch for '%s'", coordinate_name)
+
+
+def get_dataset_variables(dataset: xr.Dataset, variables: tuple[str, ...] | list[str] | object | None) -> list[str]:
+    """Extract comparable variables and dimensions from a dataset."""
+    selected_variables: list[str] = []
+
+    if variables in (None, settings.DEFAULT_VARIABLES_TO_CHECK):
+        variables_to_check = list(dataset.data_vars) + list(dataset.dims)
+    else:
+        variables_to_check = list(variables)
+
+    for variable in variables_to_check:
+        if variable not in dataset:
+            continue
+
+        dtype_kind = dataset[variable].dtype.kind
+        if dtype_kind in ("U", "S", "O", "a"):
+            logger.debug("Skipping variable %s due to datatype %s", variable, dtype_kind)
+            continue
+
+        selected_variables.append(variable)
+
+    return selected_variables

xdiff/compare/__init__.py

@@ -0,0 +1,38 @@
+"""Public comparison exports."""
+
+from importlib import import_module
+
+__all__ = [
+    "compare",
+    "compare_datasets",
+    "compare_files",
+    "compare_variables",
+    "compute_relative_error",
+    "find_time_dims_name",
+    "get_dataset_variables",
+    "select_last_time_step",
+]
+
+_LAZY_EXPORTS = {
+    "compare": ("xdiff.compare.ncdiff", "compare"),
+    "compare_datasets": ("xdiff.comparators.netcdf", "compare_datasets"),
+    "compare_files": ("xdiff.comparators.netcdf", "compare_files"),
+    "compare_variables": ("xdiff.comparators.netcdf", "compare_variables"),
+    "compute_relative_error": ("xdiff.comparators.netcdf", "compute_relative_error"),
+    "find_time_dims_name": ("xdiff.comparators.netcdf", "find_time_dims_name"),
+    "get_dataset_variables": ("xdiff.comparators.netcdf", "get_dataset_variables"),
+    "select_last_time_step": ("xdiff.comparators.netcdf", "select_last_time_step"),
+}
+
+
+def __getattr__(name: str):
+    if name in _LAZY_EXPORTS:
+        module_name, attribute_name = _LAZY_EXPORTS[name]
+        value = getattr(import_module(module_name), attribute_name)
+        globals()[name] = value
+        return value
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__() -> list[str]:
+    return sorted(__all__)

xdiff/compare/ncdiff.py

@@ -0,0 +1,34 @@
+"""NetCDF comparison orchestration for matched file pairs."""
+
+from pathlib import Path
+from typing import Iterable
+
+from xdiff.exceptions import NoMatchFound
+from xdiff.model.comparison import Comparison
+
+
+def compare_files(*args, **kwargs):
+    """Proxy to the netCDF comparator while keeping the import lazy and patchable."""
+    from xdiff.comparators.netcdf import compare_files as compare_files_impl
+
+    return compare_files_impl(*args, **kwargs)
+
+
+def compare(
+    compare_match: dict[Path, list[Path]],
+    variables: Iterable[str] | tuple[str, ...] | list[str] | object | None,
+    last_time_step: bool,
+):
+    for reference, to_compares in compare_match.items():
+        if len(to_compares) == 0:
+            yield Comparison.from_paths(reference, None, NoMatchFound(f"No match found for {reference}"))
+            continue
+
+        try:
+            for to_compare in to_compares:
+                comparison = Comparison.from_paths(reference, to_compare)
+                comparison.extend(compare_files(reference, to_compare, variables, last_time_step=last_time_step))
+                yield comparison
+
+        except Exception as e:
+            yield Comparison.from_paths(reference, None, e)

xdiff/conf/__init__.py

@@ -0,0 +1,26 @@
+"""Public configuration exports."""
+
+from xdiff.conf import global_settings as settings
+
+DEFAULT_COMMON_PATTERN = settings.DEFAULT_COMMON_PATTERN
+DEFAULT_MAXDEPTH = settings.DEFAULT_MAXDEPTH
+DEFAULT_NAME_TO_COMPARE = settings.DEFAULT_NAME_TO_COMPARE
+DEFAULT_VARIABLES_TO_CHECK = settings.DEFAULT_VARIABLES_TO_CHECK
+DTYPE_NOT_CHECKED = settings.DTYPE_NOT_CHECKED
+TIME_DTYPE = settings.TIME_DTYPE
+LOGGING = settings.LOGGING
+LOGGING_CONFIG = settings.LOGGING_CONFIG
+DEBUG = settings.DEBUG
+
+__all__ = [
+    "settings",
+    "DEFAULT_COMMON_PATTERN",
+    "DEFAULT_MAXDEPTH",
+    "DEFAULT_NAME_TO_COMPARE",
+    "DEFAULT_VARIABLES_TO_CHECK",
+    "DTYPE_NOT_CHECKED",
+    "TIME_DTYPE",
+    "LOGGING",
+    "LOGGING_CONFIG",
+    "DEBUG",
+]

xdiff/conf/global_settings.py

@@ -0,0 +1,26 @@
+"""
+Default X-Diff settings.
+"""
+
+########################
+# SETTINGS
+########################
+
+DEFAULT_MAXDEPTH = 1  # negative value remove the limit on
+DEFAULT_NAME_TO_COMPARE = "*.nc"
+DEFAULT_VARIABLES_TO_CHECK = object()
+DEFAULT_COMMON_PATTERN = None
+DTYPE_NOT_CHECKED = ["S8", "S1", "O"]  # S8|S1:char, O: string
+TIME_DTYPE = ["datetime64[ns]", "<M8[ns]"]
+
+########################
+# LOG
+########################
+
+# The callable to use to configure logging
+LOGGING_CONFIG = "logging.config.dictConfig"
+
+# Custom logging configuration.
+LOGGING = {}
+
+DEBUG = False

xdiff/core/__init__.py

@@ -0,0 +1,15 @@
+"""Public core entrypoints for the package."""
+
+import xdiff
+
+from xdiff.core import main as main_module
+
+
+def execute(**kwargs):
+    xdiff.setup()
+    get_version = kwargs.pop("get_version", False)
+
+    if get_version:
+        exit(0)
+
+    return main_module.execute(**kwargs)

xdiff/core/dask_runtime.py

@@ -0,0 +1,114 @@
+"""Helpers for optional Dask execution.
+
+The application still defaults to serial execution. This module is only used
+when the caller explicitly selects a Dask-backed execution mode, which keeps
+the regular CLI startup path lightweight.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+
+from contextlib import contextmanager
+from importlib import import_module
+from typing import TYPE_CHECKING, Iterator
+
+from xdiff.model.request import CompareRequest
+
+if TYPE_CHECKING:
+    from distributed import Client
+
+logger = logging.getLogger("xdiff")
+
+
+def log_local_worker_advisories(request: CompareRequest) -> None:
+    """Report obviously poor local-worker counts without blocking the run."""
+    if request.dask_workers is None:
+        return
+
+    visible_cpus = os.cpu_count()
+    if visible_cpus is not None and request.dask_workers > visible_cpus:
+        logger.warning(
+            "Requested %s Dask worker(s) but only %s CPU(s) are visible on this node.",
+            request.dask_workers,
+            visible_cpus,
+        )
+
+
+def log_file_mode_advisories(request: CompareRequest, comparable_file_pairs: int) -> None:
+    """Report file-mode specific worker advisories without blocking the run."""
+    log_local_worker_advisories(request)
+    if request.dask_workers is None:
+        return
+
+    if comparable_file_pairs < request.dask_workers:
+        logger.warning(
+            "Requested %s Dask worker(s) for %s comparable file pair(s); some workers will remain idle.",
+            request.dask_workers,
+            comparable_file_pairs,
+        )
+
+
+def log_local_cluster_address(cluster, client) -> None:
+    """Emit the local scheduler address through the default visible log channel."""
+    scheduler_address = getattr(cluster, "scheduler_address", None)
+    if scheduler_address is None:
+        scheduler = getattr(client, "scheduler", None)
+        scheduler_address = getattr(scheduler, "address", None)
+
+    if scheduler_address is not None:
+        logger.warning("Local Dask scheduler available at %s", scheduler_address)
+
+    dashboard_link = getattr(cluster, "dashboard_link", None)
+    if dashboard_link is not None:
+        logger.warning("Local Dask dashboard available at %s", dashboard_link)
+
+
+@contextmanager
+def client_from_request(request: CompareRequest) -> Iterator["Client"]:
+    """Attach to an external scheduler or create a local cluster for this request."""
+    distributed = _load_distributed()
+    cluster = None
+
+    if request.uses_external_dask_scheduler:
+        client = _connect_to_scheduler(request, distributed.Client)
+    else:
+        # One worker process per requested slot is the safest default for Dask-backed netCDF I/O.
+        cluster = distributed.LocalCluster(
+            n_workers=request.dask_workers,
+            threads_per_worker=1,
+            processes=True,
+            dashboard_address=":8787",
+        )
+        client = distributed.Client(cluster)
+        log_local_cluster_address(cluster, client)
+
+    try:
+        yield client
+    finally:
+        client.close()
+        if cluster is not None:
+            cluster.close()
+
+
+def iterate_results_as_completed(futures):
+    """Yield futures and results in completion order."""
+    distributed = _load_distributed()
+    yield from distributed.as_completed(futures, with_results=True)
+
+
+def _connect_to_scheduler(request: CompareRequest, client_type):
+    if request.dask_scheduler_file is not None:
+        return client_type(scheduler_file=str(request.dask_scheduler_file))
+    return client_type(request.dask_scheduler)
+
+
+def _load_distributed():
+    try:
+        return import_module("distributed")
+    except ImportError as exc:
+        raise RuntimeError(
+            "Dask support requires the 'distributed' package. "
+            "Install the project dependencies before using this feature."
+        ) from exc

xdiff/core/main.py

@@ -0,0 +1,84 @@
+"""Compatibility layer for the application service entrypoint."""
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Iterable
+
+import xdiff.conf as settings
+
+from xdiff.core.service import ComparisonService
+from xdiff.discovery import FileSystemArtifactDiscovery
+from xdiff.model import CompareMode, CompareRequest, ComparisonReport, ExecutionMode
+
+if TYPE_CHECKING:
+    from xdiff.printlib.progress import ProgressReporter
+
+
+def execute(
+    reference_path: Path,
+    comparison_path: Path,
+    filter_name: str = settings.DEFAULT_NAME_TO_COMPARE,
+    common_pattern: str | None = settings.DEFAULT_COMMON_PATTERN,
+    variables: Iterable[str] | object = settings.DEFAULT_VARIABLES_TO_CHECK,
+    last_time_step: bool = False,
+    input_mode: CompareMode = CompareMode.DIRECTORIES,
+    execution_mode: ExecutionMode | str = ExecutionMode.SERIAL,
+    dask_scheduler: str | None = None,
+    dask_scheduler_file: Path | None = None,
+    dask_workers: int | None = None,
+    progress_reporter: "ProgressReporter | None" = None,
+) -> ComparisonReport:
+    request = build_request(
+        reference_path=reference_path,
+        comparison_path=comparison_path,
+        input_mode=input_mode,
+        filter_name=filter_name,
+        common_pattern=common_pattern,
+        variables=variables,
+        last_time_step=last_time_step,
+        execution_mode=execution_mode,
+        dask_scheduler=dask_scheduler,
+        dask_scheduler_file=dask_scheduler_file,
+        dask_workers=dask_workers,
+    )
+    return ComparisonService.default().run(request, progress_reporter=progress_reporter)
+
+
+def build_request(
+    reference_path: Path,
+    comparison_path: Path,
+    input_mode: CompareMode = CompareMode.DIRECTORIES,
+    filter_name: str = settings.DEFAULT_NAME_TO_COMPARE,
+    common_pattern: str | None = settings.DEFAULT_COMMON_PATTERN,
+    variables: Iterable[str] | object = settings.DEFAULT_VARIABLES_TO_CHECK,
+    last_time_step: bool = False,
+    execution_mode: ExecutionMode | str = ExecutionMode.SERIAL,
+    dask_scheduler: str | None = None,
+    dask_scheduler_file: Path | None = None,
+    dask_workers: int | None = None,
+) -> CompareRequest:
+    """Normalize legacy execute arguments into a service request."""
+    return CompareRequest(
+        input_mode=input_mode,
+        reference_path=reference_path,
+        comparison_path=comparison_path,
+        filter_name=filter_name,
+        common_pattern=common_pattern,
+        variables=normalize_variables(variables),
+        last_time_step=last_time_step,
+        execution_mode=execution_mode,
+        dask_scheduler=dask_scheduler,
+        dask_scheduler_file=dask_scheduler_file,
+        dask_workers=dask_workers,
+    )
+
+
+def normalize_variables(variables: Iterable[str] | object) -> tuple[str, ...] | None:
+    if variables in (None, settings.DEFAULT_VARIABLES_TO_CHECK):
+        return None
+    return tuple(variables)
+
+
+
+def load_files(directory: Path, filter_name: str) -> list[Path]:
+    """Compatibility helper preserved for callers and tests."""
+    return FileSystemArtifactDiscovery().list_paths(directory, filter_name)