anemoi-datasets 0.5.23__tar.gz → 0.5.25__tar.gz

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/.github/CODEOWNERS

@@ -1,4 +1,3 @@
-
 # Workflows
 /.github/ @ecmwf/AnemoiSecurity
 

anemoi_datasets-0.5.25/.github/pull_request_template.md

@@ -0,0 +1,13 @@
+## Description
+<!-- What issue or task does this change relate to? -->
+
+## What problem does this change solve?
+<!-- Describe if it's a bugfix, new feature, doc update, or breaking change -->
+
+## What issue or task does this change relate to?
+<!-- link to Issue Number -->
+
+##  Additional notes ##
+<!-- Include any additional information, caveats, or considerations that the reviewer should be aware of. -->
+
+***As a contributor to the Anemoi framework, please ensure that your changes include unit tests, updates to any affected dependencies and documentation, and have been tested in a parallel setting  (i.e., with multiple GPUs). As a reviewer, you are also responsible for verifying these aspects and requesting changes if they are not adequately addressed. For guidelines about those please refer to https://anemoi.readthedocs.io/en/latest/***

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/.github/workflows/downstream-ci-hpc.yml

@@ -1,6 +1,6 @@
 # This workflow triggers tests on dependent packages.
 # The dependency tree itself is defined in ecmwf/downstream-ci/
-name: Test downstream dependent packages on HPC
+name: Test downstream dependent packages
 
 on:
   # Trigger the workflow on push to main or develop, except tag creation

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/.github/workflows/pr-conventional-commit.yml

@@ -1,5 +1,5 @@
 # This workflow ensures that the PR title follows the Conventional Commit format.
-name: "[Pull Request] Ensure Conventional Commit in PR title"
+name: "[PR] Ensure Conventional Commit Title"
 
 on:
   pull_request_target:

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/.github/workflows/pr-label-conventional-commits.yml

@@ -1,6 +1,6 @@
 # This workflow assigns labels to a pull request based on the Conventional Commits format.
 # This is necessary for release-please to work properly.
-name: "[Pull Request] Label Conventional Commits"
+name: "[PR] Label Conventional Commits"
 
 on:
   pull_request:

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/.github/workflows/pr-label-file-based.yml

@@ -1,6 +1,6 @@
 # This workflow assigns labels to a pull request based on the files changed in the PR.
 # The labels are defined in the `.github/labels.yml` file.
-name: "[Pull Request] Label File-based"
+name: "[PR] Label File-based"
 on:
   pull_request_target:
     types: [opened, synchronize]

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/.github/workflows/pr-label-public.yml

@@ -1,5 +1,5 @@
 # Manage labels of pull requests that originate from forks
-name: "[Pull Request] Label PRs from public forks"
+name: "[PR] Label Forks"
 
 on:
   pull_request_target:

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/.github/workflows/python-pull-request.yml

@@ -1,5 +1,5 @@
 # This workflow runs pre-commit checks and pytest tests against multiple platforms and Python versions.
-name: Code Quality checks and Testing
+name: Code Quality and Testing
 
 on:
   pull_request:

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/.github/workflows/readthedocs-pr-update.yml

@@ -1,6 +1,6 @@
 # This workflow adds a link to the experimental documentation build to the PR.
 # This does NOT trigger a build of the documentation, this is handled through webhooks.
-name: Read the Docs PR Preview
+name: "[PR] Read the Docs Preview"
 on:
   pull_request_target:
     types:

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/.gitignore

@@ -94,6 +94,11 @@ dmypy.json
 *.csv
 *.xlsx
 *.xls
+*.json
+*.txt
+*.zip
+*.db
+*.tgz
 
 # ML artifacts
 wandb/
@@ -120,7 +125,8 @@ tmp/
 temp/
 logs/
 _dev/
-outputs
+_api/
+./outputs
 *tmp_data/
 
 # Project specific
@@ -135,8 +141,3 @@ _version.py
 *.to_upload
 tempCodeRunnerFile.python
 Untitled-*.py
-*.zip
-*.json
-*.db
-*.tgz
-_api/

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/.pre-commit-config.yaml

@@ -39,8 +39,9 @@ repos:
     - -l 120
     - --force-single-line-imports
     - --profile black
+    - --project anemoi
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.11.8
+  rev: v0.11.12
   hooks:
   - id: ruff
     args:
@@ -64,7 +65,7 @@ repos:
   - id: docconvert
     args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "v2.5.1"
+  rev: "v2.6.0"
   hooks:
   - id: pyproject-fmt
 -   repo: https://github.com/jshwi/docsig # Check docstrings against function sig

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/.release-please-config.json

@@ -10,11 +10,16 @@
   "draft-pull-request": true,
   "pull-request-title-pattern": "chore${scope}: Release${component} ${version}",
   "pull-request-header": ":robot: Automated Release PR\n\nThis PR was created by `release-please` to prepare the next release. Once merged:\n\n1. A new version tag will be created\n2. A GitHub release will be published\n3. The changelog will be updated\n\nChanges to be included in the next release:",
-  "pull-request-footer": "> [!IMPORTANT]\n> :warning: Merging this PR will:\n> - Create a new release\n> - Trigger deployment pipelines\n> - Update package versions\n\n **Before merging:**\n - Ensure all tests pass\n - Review the changelog carefully\n - Get required approvals\n\n [Release-please documentation](https://github.com/googleapis/release-please)",
+  "pull-request-footer": "> [!IMPORTANT]\n> Please do not change the PR title, manifest file, or any other automatically generated content in this PR unless you understand the implications. Changes here can break the release process.\n> :warning: Merging this PR will:\n> - Create a new release\n> - Trigger deployment pipelines\n> - Update package versions\n\n **Before merging:**\n - Ensure all tests pass\n - Review the changelog carefully\n - Get required approvals\n\n [Release-please documentation](https://github.com/googleapis/release-please)",
   "packages": {
     ".": {
         "package-name": "anemoi-datasets"
     }
   },
+  "plugins": [
+    {
+      "type": "sentence-case"
+    }
+  ],
   "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json"
 }

anemoi_datasets-0.5.25/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.5.25"
+}

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/CHANGELOG.md

@@ -8,6 +8,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.5.25](https://github.com/ecmwf/anemoi-datasets/compare/0.5.24...0.5.25) (2025-06-11)
+
+
+### Features
+
+* Integrating non-regular datasets in anemoi for observations. ([#306](https://github.com/ecmwf/anemoi-datasets/issues/306)) ([95a0fe4](https://github.com/ecmwf/anemoi-datasets/commit/95a0fe4bb10dc48469c0be0efad94f4d5e2a9fe8))
+
+
+### Bug Fixes
+
+* Incremental dataset build tasks called regardless of presence of debug flag in CLI code ([#294](https://github.com/ecmwf/anemoi-datasets/issues/294)) ([37afc0d](https://github.com/ecmwf/anemoi-datasets/commit/37afc0d6489f2d6c4b3ce3f9901c40e4cec5c4eb))
+* Regression in accumulations [#354](https://github.com/ecmwf/anemoi-datasets/issues/354) ([#355](https://github.com/ecmwf/anemoi-datasets/issues/355)) ([f9769d7](https://github.com/ecmwf/anemoi-datasets/commit/f9769d7944738ecbedb6b3cc1f78cd26de36a73f))
+* Remove 2 layers of build function ([#348](https://github.com/ecmwf/anemoi-datasets/issues/348)) ([7a904c4](https://github.com/ecmwf/anemoi-datasets/commit/7a904c451772089f120419a9d39bff746e0aeebb))
+
+## [0.5.24](https://github.com/ecmwf/anemoi-datasets/compare/0.5.23...0.5.24) (2025-05-23)
+
+
+### Features
+
+* verify command ([#279](https://github.com/ecmwf/anemoi-datasets/issues/279)) ([aed36d2](https://github.com/ecmwf/anemoi-datasets/commit/aed36d2ea7a39ea1ae6bbd5f8d01ef0ce7523cde))
+
+
+### Bug Fixes
+
+* adapt to earthkit 0.14 (ignore_keys issue) ([#331](https://github.com/ecmwf/anemoi-datasets/issues/331)) ([fb3ab8d](https://github.com/ecmwf/anemoi-datasets/commit/fb3ab8d46b8e00c62d8d7cbb1d1afae0efea2054))
+
 ## [0.5.23](https://github.com/ecmwf/anemoi-datasets/compare/0.5.22...0.5.23) (2025-05-07)
 
 

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-datasets
-Version: 0.5.23
+Version: 0.5.25
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -225,7 +225,7 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: anemoi-transform>=0.1.9
+Requires-Dist: anemoi-transform>=0.1.10
 Requires-Dist: anemoi-utils[provenance]>=0.4.21
 Requires-Dist: cfunits
 Requires-Dist: numcodecs<0.16
@@ -241,7 +241,7 @@ Requires-Dist: prettytable; extra == "comparelam"
 Requires-Dist: termcolor; extra == "comparelam"
 Provides-Extra: create
 Requires-Dist: cachetools; extra == "create"
-Requires-Dist: earthkit-data[mars]<0.14,>=0.12.4; extra == "create"
+Requires-Dist: earthkit-data[mars]>=0.14; extra == "create"
 Requires-Dist: earthkit-geo>=0.3; extra == "create"
 Requires-Dist: earthkit-meteo>=0.3; extra == "create"
 Requires-Dist: eccodes>=2.39.1; extra == "create"

anemoi_datasets-0.5.25/docs/adr/adr-1.md

@@ -0,0 +1,63 @@
+# Support irregular observations datasets
+
+## Status
+
+<!--What is the status? -->
+
+Proposed - 30/04/2025
+
+## Context
+
+<!--What is the issue that we are seeing that is motivating this decision or change?-->
+
+The objective of this change is to support observations data which is not regular.
+
+In contrast with the fields data where each date contain the same number of points,
+in the observations data, the number of points can change for every time window.
+
+The Zarr format fits well the fields data, but does not fit the observations data.
+
+To allow storing data with irregular shape, we need to use another format than the zarr used for fields.
+An experimental implementation using xarray-zarr has been developed and is not optimised for ML training.
+
+## Decision
+
+<!--Describe the change that you are proposing.-->
+
+Add a functionality in anemoi-datasets to read observations datasets and provide the data as dictionary/mapper of numpy arrays.
+
+Mimic as much as possible what is done for field datasets :
+
+`ds = open_dataset(....)`
+`ds[i]` -> provides the data for a given time window, related to a given reference date. As a dictionary-like object.
+`ds.dates` -> list of reference date, `ds.dates[i]` is the reference date for the data provided in `ds[i]`
+
+Also expose the latitudes, longitudes in a sensible way (as `ds.latitudes` and `ds.longitudes` now depend on the dates) and name_to_index and statistics and metadata, etc.
+
+These API choices need to be made on an actual training use case.
+
+Step to achieve this:
+- Implement now a prototype format to allow developing ML training code on observation data.
+- Performing extensive benchmarking with various formats (explore parquet, and other).
+- As the final format is not defined yet, ensure a flexible architecture to allow switching (this will help for benchmarking).
+
+
+## Scope of Change
+
+<!--Specify which Anemoi packages/modules will be affected by this decision.-->
+- anemoi-datasets
+Not a breaking change, this only add functionality to read observations datasets.
+
+Must be in line with the change related to multi-datasets.
+
+## Consequences
+
+<!--Discuss the impact of this decision, including benefits, trade-offs, and potential technical debt.-->
+
+## Alternatives Considered [Optional]
+
+<!--List alternative solutions and why they were not chosen.-->
+
+## References [Optional]
+
+<!--Links to relevant discussions, documentation, or external resources.-->

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/pyproject.toml

@@ -49,7 +49,7 @@ dynamic = [
   "version",
 ]
 dependencies = [
-  "anemoi-transform>=0.1.9",
+  "anemoi-transform>=0.1.10",
   "anemoi-utils[provenance]>=0.4.21",
   "cfunits",
   "numcodecs<0.16",                   # Until we move to zarr3
@@ -71,7 +71,7 @@ optional-dependencies.comparelam = [
 
 optional-dependencies.create = [
   "cachetools",
-  "earthkit-data[mars]>=0.12.4,<0.14",
+  "earthkit-data[mars]>=0.14",
   "earthkit-geo>=0.3",
   "earthkit-meteo>=0.3",
   "eccodes>=2.39.1",

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.5.23'
-__version_tuple__ = version_tuple = (0, 5, 23)
+__version__ = version = '0.5.25'
+__version_tuple__ = version_tuple = (0, 5, 25)

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/commands/finalise-additions.py

@@ -61,7 +61,8 @@ class FinaliseAdditions(Command):
 
         if "debug" in options:
             options.pop("debug")
-            task(step, options)
+
+        task(step, options)
 
         LOG.info(f"Create step '{step}' completed in {seconds_to_human(time.time()-now)}")
 

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/commands/finalise.py

@@ -55,7 +55,8 @@ class Finalise(Command):
 
         if "debug" in options:
             options.pop("debug")
-            task(step, options)
+
+        task(step, options)
 
         LOG.info(f"Create step '{step}' completed in {seconds_to_human(time.time()-now)}")
 

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/commands/init-additions.py

@@ -61,7 +61,8 @@ class InitAdditions(Command):
 
         if "debug" in options:
             options.pop("debug")
-            task(step, options)
+
+        task(step, options)
 
         LOG.info(f"Create step '{step}' completed in {seconds_to_human(time.time()-now)}")
 

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/commands/load-additions.py

@@ -62,7 +62,8 @@ class LoadAdditions(Command):
 
         if "debug" in options:
             options.pop("debug")
-            task(step, options)
+
+        task(step, options)
 
         LOG.info(f"Create step '{step}' completed in {seconds_to_human(time.time()-now)}")
 

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/commands/load.py

@@ -62,7 +62,8 @@ class Load(Command):
 
         if "debug" in options:
             options.pop("debug")
-            task(step, options)
+
+        task(step, options)
 
         LOG.info(f"Create step '{step}' completed in {seconds_to_human(time.time()-now)}")
 

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/create/__init__.py

@@ -44,7 +44,7 @@ from .check import check_data_values
 from .chunks import ChunkFilter
 from .config import build_output
 from .config import loader_config
-from .input import build_input
+from .input import InputBuilder
 from .statistics import Summary
 from .statistics import TmpStatistics
 from .statistics import check_variance
@@ -101,7 +101,9 @@ def json_tidy(o: Any) -> Any:
 
 
 def build_statistics_dates(
-    dates: list[datetime.datetime], start: Optional[datetime.datetime], end: Optional[datetime.datetime]
+    dates: list[datetime.datetime],
+    start: Optional[datetime.datetime],
+    end: Optional[datetime.datetime],
 ) -> tuple[str, str]:
     """Compute the start and end dates for the statistics.
 
@@ -551,36 +553,16 @@ class HasElementForDataMixin:
 
         self.output = build_output(config.output, parent=self)
 
-        self.input = build_input_(main_config=config, output_config=self.output)
-        # LOG.info("%s", self.input)
-
-
-def build_input_(main_config: Any, output_config: Any) -> Any:
-    """Build the input for the dataset.
-
-    Parameters
-    ----------
-    main_config : Any
-        The main configuration.
-    output_config : Any
-        The output configuration.
-
-    Returns
-    -------
-    Any
-        The input builder.
-    """
-    builder = build_input(
-        main_config.input,
-        data_sources=main_config.get("data_sources", {}),
-        order_by=output_config.order_by,
-        flatten_grid=output_config.flatten_grid,
-        remapping=build_remapping(output_config.remapping),
-        use_grib_paramid=main_config.build.use_grib_paramid,
-    )
-    LOG.debug("✅ INPUT_BUILDER")
-    LOG.debug(builder)
-    return builder
+        self.input = InputBuilder(
+            config.input,
+            data_sources=config.get("data_sources", {}),
+            order_by=self.output.order_by,
+            flatten_grid=self.output.flatten_grid,
+            remapping=build_remapping(self.output.remapping),
+            use_grib_paramid=config.build.use_grib_paramid,
+        )
+        LOG.debug("✅ INPUT_BUILDER")
+        LOG.debug(self.input)
 
 
 class Init(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixin):
@@ -1541,7 +1523,16 @@ class Statistics(Actor, HasStatisticTempMixin, HasRegistryMixin):
         if not all(self.registry.get_flags(sync=False)):
             raise Exception(f"❗Zarr {self.path} is not fully built, not writing statistics into dataset.")
 
-        for k in ["mean", "stdev", "minimum", "maximum", "sums", "squares", "count", "has_nans"]:
+        for k in [
+            "mean",
+            "stdev",
+            "minimum",
+            "maximum",
+            "sums",
+            "squares",
+            "count",
+            "has_nans",
+        ]:
             self.dataset.add_dataset(name=k, array=stats[k], dimensions=("variable",))
 
         self.registry.add_to_history("compute_statistics_end")

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/create/input/__init__.py

@@ -104,23 +104,3 @@ class InputBuilder:
             Trace string.
         """
         return f"InputBuilder({group_of_dates})"
-
-
-def build_input(config: dict, data_sources: Union[dict, list], **kwargs: Any) -> InputBuilder:
-    """Build an InputBuilder instance.
-
-    Parameters
-    ----------
-    config : dict
-        Configuration dictionary.
-    data_sources : Union[dict, list]
-        Data sources.
-    **kwargs : Any
-        Additional keyword arguments.
-
-    Returns
-    -------
-    InputBuilder
-        An instance of InputBuilder.
-    """
-    return InputBuilder(config, data_sources, **kwargs)

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/create/sources/accumulations.py

@@ -459,12 +459,13 @@ class AccumulationFromStart(Accumulation):
             A tuple representing the MARS date-time step.
         """
         assert user_date is None, user_date
-        assert not frequency, frequency
 
         steps = (step1 + add_step, step2 + add_step)
         if steps[0] == 0:
             steps = (steps[1],)
 
+        assert frequency == 0 or frequency == (step2 - step1), frequency
+
         return (
             base_date.year * 10000 + base_date.month * 100 + base_date.day,
             base_date.hour * 100 + base_date.minute,
@@ -824,6 +825,11 @@ def _compute_accumulations(
     step1, step2 = user_accumulation_period
     assert step1 < step2, user_accumulation_period
 
+    if accumulations_reset_frequency is not None:
+        AccumulationClass = AccumulationFromLastReset
+    else:
+        AccumulationClass = AccumulationFromStart if data_accumulation_period in (0, None) else AccumulationFromLastStep
+
     if data_accumulation_period is None:
         data_accumulation_period = user_accumulation_period[1] - user_accumulation_period[0]
 
@@ -838,11 +844,6 @@ def _compute_accumulations(
 
     base_times = [t // 100 if t > 100 else t for t in base_times]
 
-    if accumulations_reset_frequency is not None:
-        AccumulationClass = AccumulationFromLastReset
-    else:
-        AccumulationClass = AccumulationFromStart if data_accumulation_period in (0, None) else AccumulationFromLastStep
-
     mars_date_time_steps = AccumulationClass.mars_date_time_steps(
         dates=dates,
         step1=step1,

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/create/sources/grib.py

@@ -124,7 +124,7 @@ def execute(
     dates = [d.isoformat() for d in dates]
 
     for path in given_paths:
-        paths = Pattern(path, ignore_missing_keys=True).substitute(*args, date=dates, **kwargs)
+        paths = Pattern(path).substitute(*args, date=dates, allow_extra=True, **kwargs)
 
         for name in ("grid", "area", "rotation", "frame", "resol", "bitmap"):
             if name in kwargs:

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/create/sources/patterns.py

@@ -79,6 +79,6 @@ def iterate_patterns(
         kwargs["date"] = dates
 
     for path in given_paths:
-        paths = Pattern(path, ignore_missing_keys=True).substitute(**kwargs)
+        paths = Pattern(path).substitute(allow_extra=True, **kwargs)
         for path in _expand(paths):
             yield path, dates

{anemoi_datasets-0.5.23 → anemoi_datasets-0.5.25}/src/anemoi/datasets/data/dataset.py

@@ -179,6 +179,19 @@ class Dataset(ABC, Sized):
         if "start" in kwargs or "end" in kwargs:
             start = kwargs.pop("start", None)
             end = kwargs.pop("end", None)
+            padding = kwargs.pop("padding", None)
+
+            if padding:
+                if padding != "empty":
+                    raise ValueError(f"Only 'empty' padding is supported, got {padding=}")
+                from .padded import Padded
+
+                frequency = kwargs.pop("frequency", self.frequency)
+                return (
+                    Padded(self, start, end, frequency, dict(start=start, end=end, frequency=frequency))
+                    ._subset(**kwargs)
+                    .mutate()
+                )
 
             from .subset import Subset
 
@@ -724,6 +737,9 @@ class Dataset(ABC, Sized):
         """Return the grid shape of the dataset."""
         return (self.shape[-1],)
 
+    def empty_item(self) -> NDArray[Any]:
+        return np.zeros((*self.shape[1:-1], 0), dtype=self.dtype)
+
     def _check(self) -> None:
         """Check for overridden private methods in the dataset."""
         common = Dataset.__dict__.keys() & self.__class__.__dict__.keys()
@@ -1075,3 +1091,16 @@ class Dataset(ABC, Sized):
             The dataset names.
         """
         pass
+
+    def get_latitudes(self, i):
+        return self.get_aux(i)[0]
+
+    def get_longitudes(self, i):
+        return self.get_aux(i)[1]
+
+    def get_timedeltas(self, i):
+        return self.get_aux(i)[2]
+
+    def get_aux(self, i):
+        # need to decide if Fields datasets need to implement this
+        raise NotImplementedError(f"get_aux is not implemented for this dataset, {type(self)}")