atmos-validation 2.0.2__tar.gz → 2.0.4__tar.gz

{atmos_validation-2.0.2 → atmos_validation-2.0.4}/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## [2.0.4](https://github.com/equinor/atmos-validation/compare/v2.0.3...v2.0.4) (2025-09-23)
+
+
+### Optimization
+
+* Increase 5d validation performance ([#71](https://github.com/equinor/atmos-validation/issues/71)) ([a19f180](https://github.com/equinor/atmos-validation/commit/a19f180e4a385d7b6ac4bcb9844a91c2c1be4b4a))
+
+## [2.0.3](https://github.com/equinor/atmos-validation/compare/v2.0.2...v2.0.3) (2025-09-12)
+
+
+### Other
+
+* relax version requirement for requests ([#69](https://github.com/equinor/atmos-validation/issues/69)) ([3135386](https://github.com/equinor/atmos-validation/commit/3135386c8eff169e35c4b9c7430c8e4c71826df5))
+
 ## [2.0.2](https://github.com/equinor/atmos-validation/compare/v2.0.1...v2.0.2) (2025-09-10)
 
 

{atmos_validation-2.0.2 → atmos_validation-2.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atmos_validation
-Version: 2.0.2
+Version: 2.0.4
 Summary: A python library containing automated data validation tools for the Atmos Data Service
 Project-URL: Repository, https://github.com/equinor/atmos-validation
 Project-URL: Documentation, https://github.com/equinor/atmos-validation/tree/main/docs
@@ -15,7 +15,7 @@ Requires-Dist: h5netcdf<2,>=1.6.4
 Requires-Dist: h5py<4,>=3.10.0
 Requires-Dist: numpy<3,>=2.0.0
 Requires-Dist: pydantic<3,>=2.8.2
-Requires-Dist: requests<3,>=2.32.5
+Requires-Dist: requests<3,>=2.32.4
 Requires-Dist: xarray<2026,>=2025.0.0
 Description-Content-Type: text/markdown
 

{atmos_validation-2.0.2 → atmos_validation-2.0.4}/atmos_validation/schemas/dim_constants.py

@@ -6,6 +6,7 @@ TIME = "Time"
 HEIGHT_DIM_PREFIX = "height_"
 FREQUENCY = "frequency"
 DIRECTION = "direction"
+GRID_POINT_MASK = "GRID_POINT_MASK"
 
 
 def get_height_dim_from_parameter_key(key: str) -> str:

{atmos_validation-2.0.2 → atmos_validation-2.0.4}/atmos_validation/validate_netcdf/main.py

@@ -49,6 +49,7 @@ def main():
             print(DOCSTRING)
         else:
             result = validate(sys.argv[2], additional_args=sys.argv[2:])
+            log.info("Validation finished")
             if isinstance(result, ValidationResult):
                 if result.errors:
                     pretty_print_result(

{atmos_validation-2.0.2 → atmos_validation-2.0.4}/atmos_validation/validate_netcdf/tests/test_varinterval_validator.py

@@ -34,10 +34,7 @@ def test_over_max_interval():
         ds["P"][:, 1, 0, 4] = 2  # pylint: disable=unsupported-assignment-operation
         data_array = ds["P"]
 
-        dims = ["Time", "height_P", "south_north", "west_east"]
-        errors = _check_randomly_selected_intervals_min_max(
-            data_array, test_config, dims
-        )  # type: ignore
+        errors = _check_randomly_selected_intervals_min_max(ds, data_array, test_config)  # type: ignore
         assert len(errors) == 1
         assert "overmax" in errors[0]
 
@@ -49,9 +46,6 @@ def test_under_min_interval():
         ds["P"][:, 2, 3, 2] = -1  # pylint: disable=unsupported-assignment-operation
         data_array = ds["P"]
 
-        dims = ["Time", "height_P", "south_north", "west_east"]
-        errors = _check_randomly_selected_intervals_min_max(
-            data_array, test_config, dims
-        )  # type: ignore
+        errors = _check_randomly_selected_intervals_min_max(ds, data_array, test_config)  # type: ignore
         assert len(errors) == 1
         assert "undermin" in errors[0]

{atmos_validation-2.0.2 → atmos_validation-2.0.4}/atmos_validation/validate_netcdf/validators/variables/variables_validator.py

@@ -17,7 +17,7 @@ from .varattrs_validator import (
     var_required_attr_values_validator,
 )
 from .vardims_validator import vardims_validator
-from .varinterval_validator import varinterval_validator
+from .varinterval_validator import SEED, varinterval_validator
 
 
 def load_parameter_config_from_endpoint():
@@ -33,6 +33,8 @@ def load_parameter_config_from_endpoint():
 def variables_validator(ds: xr.Dataset) -> List[str]:
     valids = load_parameter_config_from_endpoint().param_dict
     errors = []
+
+    log.info("using random seed: %s", SEED)
     for key in list(ds.keys()):
         if key not in valids:
             errors += [f"{key} is not a valid key"]
@@ -60,7 +62,7 @@ def variable_validator(
         + var_allowed_instruments_validator(
             key, ds, parameter_settings.allowed_instruments
         )
-        + varinterval_validator(var, parameter_settings)
+        + varinterval_validator(ds, var, parameter_settings)
         + sig_dig_validator(var, parameter_settings)
         + var_height_longname_validator(key, ds)
         + var_height_depth_validator(key, ds, parameter_settings.parameter_category)

{atmos_validation-2.0.2 → atmos_validation-2.0.4}/atmos_validation/validate_netcdf/validators/variables/varinterval_validator.py

@@ -2,11 +2,12 @@ import random
 import sys
 from typing import List, Tuple, Union
 
+import numpy as np
 import xarray as xr
 
 from ....schemas import (
-    DIRECTION,
     FREQUENCY,
+    GRID_POINT_MASK,
     SOUTH_NORTH,
     TIME,
     WEST_EAST,
@@ -19,7 +20,6 @@ from ...utils import Severity, validation_node
 from ...validation_logger import log
 
 SEED = random.randrange(sys.maxsize)
-print(f"using random seed: {SEED}")
 
 
 def _get_random_time_slice(actual: xr.DataArray, rand: random.Random) -> slice:
@@ -41,13 +41,36 @@ def _get_random_time_slice(actual: xr.DataArray, rand: random.Random) -> slice:
     return time_slice
 
 
+def _get_random_spatial_point(
+    ds: xr.Dataset, rand: random.Random
+) -> tuple[slice, slice]:
+    """To save processing time we check 1x1 random spatial point where GRID_POINT_MASK == 1"""
+    if GRID_POINT_MASK not in ds:
+        ds.assign({GRID_POINT_MASK: xr.DataArray(1, dims=(SOUTH_NORTH, WEST_EAST))})
+
+    mask = ds[GRID_POINT_MASK].values
+    valids = np.where(mask == 1)
+    indices = list(zip(valids[0], valids[1]))
+    sn_index, we_index = rand.choice(indices)
+
+    return (slice(sn_index, sn_index + 1), slice(we_index, we_index + 1))
+
+
 def _get_slice_tuple(
-    dims: List[str], actual: xr.DataArray, rand: random.Random
+    ds: xr.Dataset, actual: xr.DataArray, rand: random.Random
 ) -> Tuple[Union[int, slice], ...]:
-    result: Tuple[Union[int, slice], ...] = ()
+    if FREQUENCY in actual.dims and len(actual.dims) == 5:
+        result = ()
+        result += (_get_random_time_slice(actual, rand),)
+        result += _get_random_spatial_point(ds, rand)
+        result += (slice(None, None),)
+        result += (slice(None, None),)
+        return result
+
     key = str(actual.name)
     height_dim = get_height_dim_from_parameter_key(key)
-    for dim in dims:
+    result = ()
+    for dim in actual.dims:
         if dim == TIME:
             result += (_get_random_time_slice(actual, rand),)
         elif dim == SOUTH_NORTH:
@@ -56,10 +79,6 @@ def _get_slice_tuple(
             result += (slice(None, None),)
         elif dim == height_dim:
             result += (slice(None, None),)
-        elif dim == FREQUENCY:
-            result += (slice(None, None),)
-        elif dim == DIRECTION:
-            result += (slice(None, None),)
         else:
             raise ValueError(
                 f"Invalid dimension {dim}, cannot validate interval of {key}"
@@ -100,7 +119,9 @@ def none_larger_than_max_validator(
 
 
 @validation_node(severity=Severity.ERROR)
-def varinterval_validator(actual: xr.DataArray, expected: ParameterConfig) -> List[str]:
+def varinterval_validator(
+    ds: xr.Dataset, actual: xr.DataArray, expected: ParameterConfig
+) -> List[str]:
     """
     Take a bunch of random intervals in
     time, height, south_north, west_east
@@ -109,37 +130,36 @@ def varinterval_validator(actual: xr.DataArray, expected: ParameterConfig) -> Li
     """
     log.info("validating interval for %s", actual.name)
     dims = [str(dim) for dim in actual.dims]
-    accept = get_acceptable_dims_from_parameter_key(str(actual.name))
-
-    result = []
-    if dims not in accept:
+    accepted = get_acceptable_dims_from_parameter_key(str(actual.name))
+    if dims not in accepted:
         return [
             f"Unsupported dimensional layout {dims}. Cannot evaluate interval. Accepted dimensional"
-            f" layouts: {accept}"
+            f" layouts: {accepted}"
         ]
+
+    result = []
     if validation_settings.should_check_min_max_full():
         # Long running operation, so have to explicitly request this in args
         result += none_less_than_min_validator(actual, expected)
         result += none_larger_than_max_validator(actual, expected)
         return result
-    # Seed the randomizer. We want what is called here to be reproducible
     if validation_settings.should_skip_min_max_check():
         return []
-    return _check_randomly_selected_intervals_min_max(actual, expected, dims)
+    return _check_randomly_selected_intervals_min_max(ds, actual, expected)
 
 
 def _check_randomly_selected_intervals_min_max(
-    actual: xr.DataArray, expected: ParameterConfig, dims: List[str]
+    ds: xr.Dataset, actual: xr.DataArray, expected: ParameterConfig
 ):
     rand = random.Random(SEED)
-    result = []
 
-    slice_tuple = _get_slice_tuple(dims, actual, rand)
+    slice_tuple = _get_slice_tuple(ds, actual, rand)
     vals = actual[slice_tuple]
     vals.load()
+
+    result = []
     result += undermin_validator(actual, expected, slice_tuple, vals)
     result += overmax_validator(actual, expected, slice_tuple, vals)
-
     return result
 
 

{atmos_validation-2.0.2 → atmos_validation-2.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atmos_validation"
-version = "2.0.2"
+version = "2.0.4"
 description = "A python library containing automated data validation tools for the Atmos Data Service"
 authors = [{ name = "Team Atmos (Equinor)" }]
 requires-python = ">=3.10, <3.14"
@@ -13,7 +13,7 @@ dependencies = [
     "dask>=2025.7.0,<2026",
     "h5py>=3.10.0,<4",
     "h5netcdf>=1.6.4,<2",
-    "requests>=2.32.5,<3",
+    "requests>=2.32.4,<3",
 ]
 
 [project.urls]

{atmos_validation-2.0.2 → atmos_validation-2.0.4}/uv.lock

@@ -19,7 +19,7 @@ wheels = [
 
 [[package]]
 name = "atmos-validation"
-version = "1.10.0"
+version = "2.0.2"
 source = { editable = "." }
 dependencies = [
     { name = "dask" },
@@ -50,7 +50,7 @@ requires-dist = [
     { name = "h5py", specifier = ">=3.10.0,<4" },
     { name = "numpy", specifier = ">=2.0.0,<3" },
     { name = "pydantic", specifier = ">=2.8.2,<3" },
-    { name = "requests", specifier = ">=2.32.5,<3" },
+    { name = "requests", specifier = ">=2.32.4,<3" },
     { name = "xarray", specifier = ">=2025.0.0,<2026" },
 ]