anemoi-datasets 0.3.5__py3-none-any.whl → 0.3.6__py3-none-any.whl

anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.5'
-__version_tuple__ = version_tuple = (0, 3, 5)
+__version__ = version = '0.3.6'
+__version_tuple__ = version_tuple = (0, 3, 6)

anemoi/datasets/commands/compare.py

@@ -14,6 +14,8 @@ from . import Command
 
 
 class Compare(Command):
+    """Compare two datasets. This command compares the variables in two datasets and prints the mean of the common variables. It does not compare the data itself (yet)."""
+
     def add_arguments(self, command_parser):
         command_parser.add_argument("dataset1")
         command_parser.add_argument("dataset2")

anemoi/datasets/compute/recentre.py

@@ -29,7 +29,7 @@ CLIP_VARIABLES = (
     "swl1",
 )
 
-SKIP = ("class", "stream", "type", "number", "expver", "_leg_number", "anoffset")
+SKIP = ("class", "stream", "type", "number", "expver", "_leg_number", "anoffset", "time", "date", "step")
 
 
 def check_compatible(f1, f2, centre_field_as_mars, ensemble_field_as_mars):

anemoi/datasets/create/functions/filters/rename.py

@@ -13,6 +13,14 @@ from climetlab.indexing.fieldset import FieldArray
 
 
 class RenamedFieldMapping:
+    """Rename a field based on the value of another field.
+
+    Args:
+        field (Field): The field to be renamed.
+        what (str): The name of the field that will be used to rename the field.
+        renaming (dict): A dictionary mapping the values of 'what' to the new names.
+    """
+
     def __init__(self, field, what, renaming):
         self.field = field
         self.what = what
@@ -29,6 +37,12 @@ class RenamedFieldMapping:
 
 
 class RenamedFieldFormat:
+    """Rename a field based on a format string.
+
+    Args:
+        format (str): A string that defines the new name of the field.
+    """
+
     def __init__(self, field, format):
         self.field = field
         self.format = format

anemoi/datasets/create/loaders.py

@@ -177,11 +177,14 @@ class InitialiserLoader(Loader):
                     return
                 if isinstance(obj, (dict, DictObj)):
                     if "grid" in obj:
+                        previous = obj["grid"]
                         obj["grid"] = "20./20."
-                        LOG.warn(f"Running in test mode. Setting grid to {obj['grid']}")
+                        LOG.warn(f"Running in test mode. Setting grid to {obj['grid']} instead of {previous}")
                     if "number" in obj:
-                        obj["number"] = obj["number"][0:3]
-                        LOG.warn(f"Running in test mode. Setting number to {obj['number']}")
+                        if isinstance(obj["number"], (list, tuple)):
+                            previous = obj["number"]
+                            obj["number"] = previous[0:3]
+                            LOG.warn(f"Running in test mode. Setting number to {obj['number']} instead of {previous}")
                     for k, v in obj.items():
                         set_to_test_mode(v)
 

anemoi/datasets/data/misc.py

@@ -133,9 +133,9 @@ def _as_date(d, dates, last):
 
         if "-" in d and ":" in d:
             date, time = d.replace(" ", "T").split("T")
-            year, month, day = date.split("-")
-            hour, minute, second = time.split(":")
-            return np.datetime64(f"{year:04}-{month:02}-{day:02}T{hour}:{minute}:{second}")
+            year, month, day = [int(_) for _ in date.split("-")]
+            hour, minute, second = [int(_) for _ in time.split(":")]
+            return np.datetime64(f"{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}")
 
         if "-" in d:
             assert ":" not in d

{anemoi_datasets-0.3.5.dist-info → anemoi_datasets-0.3.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.3.5
+Version: 0.3.6
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License: Apache License
@@ -230,21 +230,14 @@ Requires-Dist: semantic-version
 Requires-Dist: tqdm
 Requires-Dist: zarr <=2.17
 Provides-Extra: all
-Requires-Dist: anemoi-utils[provenance] >=0.3 ; extra == 'all'
 Requires-Dist: boto3 ; extra == 'all'
 Requires-Dist: climetlab >=0.23.2 ; extra == 'all'
 Requires-Dist: earthkit-meteo ; extra == 'all'
 Requires-Dist: ecmwflibs >=0.6.3 ; extra == 'all'
 Requires-Dist: entrypoints ; extra == 'all'
-Requires-Dist: numpy ; extra == 'all'
 Requires-Dist: pyproj ; extra == 'all'
-Requires-Dist: pytest ; extra == 'all'
-Requires-Dist: pyyaml ; extra == 'all'
 Requires-Dist: requests ; extra == 'all'
 Requires-Dist: s3fs ; extra == 'all'
-Requires-Dist: semantic-version ; extra == 'all'
-Requires-Dist: tqdm ; extra == 'all'
-Requires-Dist: zarr <=2.17 ; extra == 'all'
 Provides-Extra: create
 Requires-Dist: climetlab >=0.23.2 ; extra == 'create'
 Requires-Dist: earthkit-meteo ; extra == 'create'
@@ -253,11 +246,14 @@ Requires-Dist: entrypoints ; extra == 'create'
 Requires-Dist: pyproj ; extra == 'create'
 Provides-Extra: dev
 Requires-Dist: boto3 ; extra == 'dev'
+Requires-Dist: climetlab >=0.23.2 ; extra == 'dev'
 Requires-Dist: earthkit-meteo ; extra == 'dev'
 Requires-Dist: ecmwflibs >=0.6.3 ; extra == 'dev'
+Requires-Dist: entrypoints ; extra == 'dev'
 Requires-Dist: nbsphinx ; extra == 'dev'
 Requires-Dist: pandoc ; extra == 'dev'
 Requires-Dist: pyproj ; extra == 'dev'
+Requires-Dist: pytest ; extra == 'dev'
 Requires-Dist: requests ; extra == 'dev'
 Requires-Dist: s3fs ; extra == 'dev'
 Requires-Dist: sphinx ; extra == 'dev'
@@ -273,6 +269,6 @@ Provides-Extra: remote
 Requires-Dist: boto3 ; extra == 'remote'
 Requires-Dist: requests ; extra == 'remote'
 Requires-Dist: s3fs ; extra == 'remote'
-Provides-Extra: test
-Requires-Dist: pytest ; extra == 'test'
+Provides-Extra: tests
+Requires-Dist: pytest ; extra == 'tests'
 

{anemoi_datasets-0.3.5.dist-info → anemoi_datasets-0.3.6.dist-info}/RECORD

@@ -1,21 +1,21 @@
 anemoi/datasets/__init__.py,sha256=DC7ttKT--pmhBQALX_Cn7P28dngsJucKi5y-Ydm28QM,700
 anemoi/datasets/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
-anemoi/datasets/_version.py,sha256=3eLsZPTwWh0zKBhR6n3c4iAL9geCCdJGMojIL6dF0IA,411
+anemoi/datasets/_version.py,sha256=IKAQ4gPrCQ2FWMXOFRqouULC2EQI1zCb4iXHsnfbmTQ,411
 anemoi/datasets/grids.py,sha256=3YBMMJodgYhavarXPAlMZHaMtDT9v2IbTmAXZTqf8Qo,8481
 anemoi/datasets/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
-anemoi/datasets/commands/compare.py,sha256=tN3eqihvnZ0rFc0OUzrfI34PHDlYfc2l90ZIQBE1TDQ,1300
+anemoi/datasets/commands/compare.py,sha256=p2jQOAC3JhScCLF0GjTCO8goYLWLN8p7vzy_gf5fFcI,1473
 anemoi/datasets/commands/copy.py,sha256=fba-zjD0iTHHXHhPEcm8VhDzsXQXDUxlbtTA1TovyT0,9991
 anemoi/datasets/commands/create.py,sha256=POdOsVDlvRrHFFkI3SNXNgNIbSxkVUUPMoo660x7Ma0,987
 anemoi/datasets/commands/inspect.py,sha256=G3fzcgiLaU8jln7GKvgamN7Y06-qC_JnFw2SbNn1_E4,18646
 anemoi/datasets/commands/scan.py,sha256=HxsLdCgBMSdEXjlJfPq5M_9LxXHHQIoZ1ZEHO_AoPgA,2881
 anemoi/datasets/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-anemoi/datasets/compute/recentre.py,sha256=j8LdC8kq1t4PW7WFTXf93hSxok10un8ENIPwCehzbP8,4768
+anemoi/datasets/compute/recentre.py,sha256=GRxI6rY_KyXJnZGPxU_UO9YDb-rY_raK70Fiwv1mjhs,4792
 anemoi/datasets/create/__init__.py,sha256=jji65Zni5aPTvS269fAMix4pN9ukmSoK0z5SVsbpr5E,5807
 anemoi/datasets/create/check.py,sha256=DLjw-eyaCNxPhoKFsP4Yn_l3SIr57YHdyPR-tE5vx80,5791
 anemoi/datasets/create/chunks.py,sha256=YEDcr0K2KiiceSTiBuZzj0TbRbzZ9J546XO7rrrTFQw,2441
 anemoi/datasets/create/config.py,sha256=uLIp1WHg3hbqwwMV9EepMwJQsXJAGImkbo0okBeEVd4,7683
 anemoi/datasets/create/input.py,sha256=3G7sqdn7R1pLBeeswXwwi8VRAHrBnjq1PdRYHJBe594,27741
-anemoi/datasets/create/loaders.py,sha256=BDeb2CI_oWqIGcBnt39nRGIt3r3dK4rIidNO3pBunTk,29865
+anemoi/datasets/create/loaders.py,sha256=Y9HwH_yPDvnrOTpXVwZxB6VKqGYrHC4xMKhizUhR4lU,30081
 anemoi/datasets/create/patch.py,sha256=xjCLhvIQKRqmypsKInRU1CvFh1uoaB3YGSQP1UVZZik,3682
 anemoi/datasets/create/persistent.py,sha256=nT8gvhVPdI1H3zW_F7uViGKIlQQ94jCDrMSWTmhQ2_A,4290
 anemoi/datasets/create/size.py,sha256=A1w6RkaL0L9IlwIdmYsCTJTecmY_QtvbkGf__jvQle0,1068
@@ -27,7 +27,7 @@ anemoi/datasets/create/functions/__init__.py,sha256=v5RRSAK8LiNdPbxy0c9WxqMx3AMH
 anemoi/datasets/create/functions/filters/__init__.py,sha256=Xe9G54CKvCI3ji-7k0R5l0WZZdhlydRgawsXuBcX_hg,379
 anemoi/datasets/create/functions/filters/empty.py,sha256=X7gAZS_jl9X0ztGs3Arn8DkOB6iP4Draq4EAKa4I8hc,617
 anemoi/datasets/create/functions/filters/noop.py,sha256=ZP434Z1rFlqdgXse_1ZzqC2XAqRQlYlXlVfGLx7rK8g,444
-anemoi/datasets/create/functions/filters/rename.py,sha256=Nvmwor6S5E-mhp60hF58KkFthxbYodGcpG-sJPqbkJ0,1673
+anemoi/datasets/create/functions/filters/rename.py,sha256=cGoHr-IS-PhYEtZvXDpH03gtfZ9uT7uEZqu9EDxAxN0,2105
 anemoi/datasets/create/functions/filters/rotate_winds.py,sha256=fUdh8ILcMzMzckGlvwzdgG-c7w5R9NnWfaijp28Bf5M,4092
 anemoi/datasets/create/functions/filters/unrotate_winds.py,sha256=nsa3EHly8ppWd2WH4ROoMczM8WFu5qKaIhO_UFcL9TY,3502
 anemoi/datasets/create/functions/sources/__init__.py,sha256=Xe9G54CKvCI3ji-7k0R5l0WZZdhlydRgawsXuBcX_hg,379
@@ -56,7 +56,7 @@ anemoi/datasets/data/grids.py,sha256=rooOeR6rvjl4U8B4LO3N23fcgxvGE7ZUmhVryk1QS4M
 anemoi/datasets/data/indexing.py,sha256=625m__JG5m_tDMrkz1hB6Vydenwt0oHuyAlc-o3Zwos,4799
 anemoi/datasets/data/join.py,sha256=dtCBbMTicqrRPxfBULi3RwEcQBLhQpIcvCjdN5A3XUU,4892
 anemoi/datasets/data/masked.py,sha256=Fzkehyka70CiS0LYSy_uyVYu2gKLwDSxlbm8GiC_pYs,3742
-anemoi/datasets/data/misc.py,sha256=m_28VIhX546RIoVfGpimPOThl5EwOhkun2UgWMAUxqw,10355
+anemoi/datasets/data/misc.py,sha256=cu2rMNtq8M8yzcR1CGHHQksE0p9C2SLVigOTMH6ilMs,10400
 anemoi/datasets/data/select.py,sha256=U3AEid80mrJKu0SF4lLc-bRWMVcAZwHNUHUHRehvuHU,3680
 anemoi/datasets/data/statistics.py,sha256=PKRgcCiZEb1HjkIveVGhE3TzUy9Qe3AYWGFD72Urah8,1514
 anemoi/datasets/data/stores.py,sha256=oEjUra0zzIysiUvh-RBQRzcbviggejEQiMO5RfpjPyM,10896
@@ -65,9 +65,9 @@ anemoi/datasets/data/unchecked.py,sha256=xhdMg-ToI1UfBWHNsWyn1y2meZWngZtHx-33L0K
 anemoi/datasets/dates/__init__.py,sha256=4ItowfLLh90T8L_JOjtv98lE6M7gAaWt7dV3niUrFvk,4473
 anemoi/datasets/dates/groups.py,sha256=iq310Pi7ullglOhcNblv14MmcT8FPgYCD5s45qAfV_s,3383
 anemoi/datasets/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-anemoi_datasets-0.3.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_datasets-0.3.5.dist-info/METADATA,sha256=rrQ3_Oz3ToYC03NGG2unQIwQ9kCVW6tQOvGwnZzTJ-0,16190
-anemoi_datasets-0.3.5.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-anemoi_datasets-0.3.5.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
-anemoi_datasets-0.3.5.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_datasets-0.3.5.dist-info/RECORD,,
+anemoi_datasets-0.3.6.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_datasets-0.3.6.dist-info/METADATA,sha256=sEnXtS3eh-ix9Z1aR8RFDqBlnmK-XE6A42ga2uE_Utc,16017
+anemoi_datasets-0.3.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+anemoi_datasets-0.3.6.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
+anemoi_datasets-0.3.6.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_datasets-0.3.6.dist-info/RECORD,,