atmos-validation 2.0.0__tar.gz → 2.0.2__tar.gz

{atmos_validation-2.0.0 → atmos_validation-2.0.2}/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## [2.0.2](https://github.com/equinor/atmos-validation/compare/v2.0.1...v2.0.2) (2025-09-10)
+
+
+### Bug Fixes
+
+* Fix xarray future warnings ([#66](https://github.com/equinor/atmos-validation/issues/66)) ([2b8ef08](https://github.com/equinor/atmos-validation/commit/2b8ef08248e7f74cf097d179c7dd8d9522bacfe7))
+
+
+### Optimization
+
+* Fix dataset load performance bottlenecks ([#67](https://github.com/equinor/atmos-validation/issues/67)) ([b899558](https://github.com/equinor/atmos-validation/commit/b899558a13dfe3ffa9138f5ef34836d20f9f8fa2))
+
+## [2.0.1](https://github.com/equinor/atmos-validation/compare/v2.0.0...v2.0.1) (2025-09-09)
+
+
+### Optimization
+
+* Adjust compatibility settings for open_mfdataset ([#64](https://github.com/equinor/atmos-validation/issues/64)) ([9e5825f](https://github.com/equinor/atmos-validation/commit/9e5825f0d666d135f32d06242514258b8288ba9e))
+
 ## [2.0.0](https://github.com/equinor/atmos-validation/compare/v1.11.0...v2.0.0) (2025-09-05)
 
 

{atmos_validation-2.0.0 → atmos_validation-2.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atmos_validation
-Version: 2.0.0
+Version: 2.0.2
 Summary: A python library containing automated data validation tools for the Atmos Data Service
 Project-URL: Repository, https://github.com/equinor/atmos-validation
 Project-URL: Documentation, https://github.com/equinor/atmos-validation/tree/main/docs

{atmos_validation-2.0.0 → atmos_validation-2.0.2}/atmos_validation/validate_netcdf/main.py

@@ -83,7 +83,7 @@ def validate(
         injected_logger: pass a logger to be used for validation
         additional_args: see docstring "Options" for available additional_args.
         batch_size: Since open_mfdataset is slow for opening large amount of files,
-        validation can be run in batches. Defaults to 50 files per open_mfdataset/batch.
+        validation can be run in batches. Defaults to 1000 files per open_mfdataset/batch.
 
     Returns:
         ValidationResult containing errors and warning from running validation
@@ -148,14 +148,22 @@ def load_paths(path: str, batch_size: int) -> List[List[str]]:
 
 def open_mf_dataset(paths: List[str]) -> xr.Dataset:
     xr.set_options(use_new_combine_kwarg_defaults=True)
-    return xr.open_mfdataset(
-        paths,
-        compat="equals",
-        engine="h5netcdf",
-        concat_dim="Time",
-        combine="nested",
-        parallel=True,
-    )
+    if len(paths) > 1:
+        log.info("Running open mfdataset for %s files", len(paths))
+        ds = xr.open_mfdataset(
+            paths,
+            engine="h5netcdf",
+            concat_dim="Time",
+            compat="override",
+            data_vars="minimal",
+            combine="nested",
+            chunks="auto",
+            parallel=True,
+        )
+    else:
+        log.info("Running open dataset for single file %s", paths[0])
+        ds = xr.open_dataset(paths[0], engine="h5netcdf")
+    return ds
 
 
 def pretty_print_result(results: List[str], description: str, width: int = 150):

{atmos_validation-2.0.0 → atmos_validation-2.0.2}/atmos_validation/validate_netcdf/validators/dims/dimvars_validator.py

@@ -17,7 +17,7 @@ def dimvars_validator(
     Incidentally, this also verifies that height/depth dimensions are
     correctly attached
     """
-    alldims = {str(key) for key in ds.dims.keys()}
+    alldims = {str(key) for key in ds.sizes.keys()}
     result = []
     for var in ds.keys():
         var = str(var)

{atmos_validation-2.0.0 → atmos_validation-2.0.2}/atmos_validation/validate_netcdf/validators/variables/varinterval_validator.py

@@ -136,6 +136,7 @@ def _check_randomly_selected_intervals_min_max(
 
     slice_tuple = _get_slice_tuple(dims, actual, rand)
     vals = actual[slice_tuple]
+    vals.load()
     result += undermin_validator(actual, expected, slice_tuple, vals)
     result += overmax_validator(actual, expected, slice_tuple, vals)
 

{atmos_validation-2.0.0 → atmos_validation-2.0.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atmos_validation"
-version = "2.0.0"
+version = "2.0.2"
 description = "A python library containing automated data validation tools for the Atmos Data Service"
 authors = [{ name = "Team Atmos (Equinor)" }]
 requires-python = ">=3.10, <3.14"