OceanDataStore 0.3.0__py3-none-any.whl

OceanDataStore/cli/arg_parser.py

@@ -0,0 +1,182 @@
+# ===================================================================
+# Copyright 2026 National Oceanography Centre
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+# ===================================================================
+"""
+arg_parser.py
+
+Description:
+This module defines the argument parser for the OceanDataStore command
+line interface.
+
+Authors:
+    - Ollie Tooth
+    - Joao Morado
+    - Tobias Ferreira
+"""
+import json
+import argparse
+
+from OceanDataStore import __version__
+
+
+def create_parser():
+    """Create the argument parser."""
+    parser = argparse.ArgumentParser(
+        description=f"OceanDataStore {__version__} command line interface",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    # Send and Update are mutually exclusive operations
+    parser.add_argument(
+        "action",
+        choices=["send_to_zarr", "update_zarr", "send_to_icechunk", "update_icechunk", "list"],
+        help="Specify the action: 'send_to_zarr' or 'send_to_icechunk' to send a file(s) to an object store, "
+        "'update_zarr' or 'update_icechunk' to update an existing object, or 'list' to list the files in a bucket.",
+    )
+
+    # Always required
+    parser.add_argument(
+        "-f",
+        "--filepaths",
+        dest="filepaths",
+        help="Paths to the files to send.",
+        nargs="+",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-c",
+        "--credentials",
+        dest="store_credentials_json",
+        help="Path to the JSON file containing the credentials for the object store.",
+        required=True,
+    )
+
+    parser.add_argument(
+        "-b",
+        "--bucket",
+        dest="bucket",
+        help="Bucket name.",
+        required=True,
+    )
+
+    # Optional arguments
+    parser.add_argument(
+        "-p",
+        "--prefix",
+        dest="object_prefix",
+        help="Object prefix.",
+        default=None,
+    )
+
+    parser.add_argument(
+        "-ad",
+        "--append-dim",
+        dest="append_dim",
+        help="Append dimension.",
+        default="time_counter",
+    )
+
+    parser.add_argument(
+        "-v",
+        "--variables",
+        dest="variables",
+        help="Variables to send to store. Default None will send all variables.",
+        nargs="+",
+        default=None,
+    )
+
+    parser.add_argument(
+        "-cs",
+        "--chunk-strategy",
+        dest="chunk_strategy",
+        help="Chunk strategy as a JSON string. E.g., '{\"time_counter\": 1, \"x\": 100, \"y\": 100}'",
+        type=json.loads,
+        default=None,
+    )
+
+    parser.add_argument(
+        "-dc",
+        "--dask-configuration",
+        dest="dask_config_json",
+        help="Path to the JSON file defining the Dask Local Cluster configuration.",
+        default=None,
+    )
+
+    parser.add_argument(
+        "-gf",
+        "--grid-filepath",
+        dest="grid_filepath",
+        help="File path to model grid file containing domain information.",
+        default=None,
+    )
+
+    parser.add_argument(
+        "-uc",
+        "--update-coords",
+        dest="update_coords",
+        help="Coordinate dimensions to update as a JSON string. E.g., '{\"nav_lon\": \"glamt\", \"nav_lat\": \"gphit\"}'",
+        type=json.loads,
+        default=None,
+    )
+
+    parser.add_argument(
+        "-at",
+        "--attributes",
+        dest="attrs",
+        help="Attributes to add to the dataset as a JSON string. E.g., '{\"title\": \"my_dataset\"}'",
+        type=json.loads,
+        default=None,
+    )
+
+    parser.add_argument(
+        "-zv",
+        "--zarr-version",
+        dest="zarr_version",
+        help="Zarr version used to create the zarr store. Options are 2 (v2) or 3 (v3).",
+        default=3,
+    )
+
+    parser.add_argument(
+        "-br",
+        "--branch",
+        dest="branch",
+        help="Branch of Icechunk repository to commit changes to.",
+        default="main",
+    )
+
+    parser.add_argument(
+        "-cm",
+        "--commit_message",
+        dest="commit_message",
+        help="Commit message to be recorded when committing changes to Icechunk repository.",
+        default="Add new data to my Icechunk repository",
+    )
+
+    parser.add_argument(
+        "-vc",
+        "--variable-commits",
+        dest="var_commits",
+        action="store_true",
+        help="Send variables to Icechunk repository using independent commits.",
+    )
+
+    parser.add_argument(
+        "-ic",
+        "--icechunk-configuration",
+        dest="icechunk_config_json",
+        help="Path to the JSON file defining the Icechunk storage and repository configurations.",
+        default=None,
+    )
+
+    return parser

OceanDataStore/cli/cli.py

@@ -0,0 +1,203 @@
+# ===================================================================
+# Copyright 2026 National Oceanography Centre
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+# ===================================================================
+"""
+cli.py
+
+Description:
+This module defines the command line interface for the OceanDataStore
+package.
+
+Authors:
+    - Ollie Tooth
+    - Joao Morado
+    - Tobias Ferreira
+"""
+import sys
+import json
+import logging
+
+from OceanDataStore.cli import (
+    send_to_zarr,
+    send_to_icechunk,
+    update_zarr,
+    update_icechunk,
+    list_objects
+)
+from OceanDataStore.cli.arg_parser import create_parser
+from OceanDataStore.cli.logging import initialise_logging
+
+logger = logging.getLogger(__name__)
+
+
+def process_action(args):
+    """Process the selected action."""
+    if len(sys.argv) == 1:
+        args.parser.print_help()
+        sys.exit(0)
+
+    # === Process Arguments === #
+    if args.variables is not None:
+        variables = list(args.variables)
+    else:
+        variables = None
+
+    if args.filepaths is not None:
+        if len(args.filepaths) > 1:
+            filepaths = list(args.filepaths)
+        else:
+            filepaths = args.filepaths
+
+    if args.zarr_version is not None:
+        zarr_version = int(args.zarr_version)
+
+    if args.dask_config_json is None:
+        dask_config = {
+            "config_kwargs": None,
+            "cluster_kwargs": None,
+        }
+    else:
+        dask_config = json.load(open(args.dask_config_json))
+        if "config_kwargs" not in dask_config:
+            raise ValueError("config_kwargs not found in Dask configuration.")
+        if "cluster_kwargs" not in dask_config:
+            raise ValueError("cluster_kwargs not found in Dask configuration.")
+
+    if args.icechunk_config_json is None:
+        # Default: use Icechunk configuration for JASMIN OS:
+        icechunk_config = {
+            "storage_config_kwargs": {"region": "", "force_path_style": True},
+            "repository_config_kwargs": {},
+            "storage_settings_kwargs": {"unsafe_use_conditional_update": False, "unsafe_use_conditional_create": False},
+        }
+    else:
+        icechunk_config = json.load(open(args.icechunk_config_json))
+        if "storage_config_kwargs" not in icechunk_config:
+            raise ValueError("storage_config_kwargs not found in Icechunk configuration.")
+        if "repository_config_kwargs" not in icechunk_config:
+            raise ValueError("repository_config_kwargs not found in Icechunk configuration.")
+        if "storage_settings_kwargs" not in icechunk_config:
+            raise ValueError("storage_settings_kwargs not found in Icechunk configuration.")
+
+    # === Process Actions === #
+    if args.action == "send_to_zarr":
+
+        send_to_zarr(
+            file=filepaths,
+            bucket=args.bucket,
+            object_prefix=args.object_prefix,
+            store_credentials_json=args.store_credentials_json,
+            variables=variables,
+            append_dim=args.append_dim,
+            grid_filepath=args.grid_filepath,
+            update_coords=args.update_coords,
+            rechunk=args.chunk_strategy,
+            attrs=args.attrs,
+            client=None,
+            dask_config_kwargs=dask_config["config_kwargs"],
+            dask_cluster_kwargs=dask_config["cluster_kwargs"],
+            zarr_version=zarr_version,
+            )
+    
+    elif args.action == "update_zarr":
+
+        update_zarr(
+            file=filepaths,
+            bucket=args.bucket,
+            object_prefix=args.object_prefix,
+            store_credentials_json=args.store_credentials_json,
+            variables=variables,
+            append_dim=args.append_dim,
+            grid_filepath=args.grid_filepath,
+            update_coords=args.update_coords,
+            rechunk=args.chunk_strategy,
+            attrs=args.attrs,
+            client=None,
+            dask_config_kwargs=dask_config["config_kwargs"],
+            dask_cluster_kwargs=dask_config["cluster_kwargs"],
+            zarr_version=zarr_version,
+            )
+
+    elif args.action == "send_to_icechunk":
+
+        send_to_icechunk(
+            file=filepaths,
+            bucket=args.bucket,
+            object_prefix=args.object_prefix,
+            store_credentials_json=args.store_credentials_json,
+            variables=variables,
+            append_dim=args.append_dim,
+            grid_filepath=args.grid_filepath,
+            update_coords=args.update_coords,
+            rechunk=args.chunk_strategy,
+            attrs=args.attrs,
+            branch=args.branch,
+            commit_message=args.commit_message,
+            variable_commits=args.var_commits,
+            dask_config_kwargs=dask_config["config_kwargs"],
+            dask_cluster_kwargs=dask_config["cluster_kwargs"],
+            icechunk_config=icechunk_config,
+            )
+    
+    elif args.action == "update_icechunk":
+
+        if args.var_commits:
+            logger.warning("The --var-commits flag will be ignored when updating an Icechunk repository.")
+
+        update_icechunk(
+            file=filepaths,
+            bucket=args.bucket,
+            object_prefix=args.object_prefix,
+            store_credentials_json=args.store_credentials_json,
+            variables=variables,
+            append_dim=args.append_dim,
+            grid_filepath=args.grid_filepath,
+            update_coords=args.update_coords,
+            rechunk=args.chunk_strategy,
+            attrs=args.attrs,
+            branch=args.branch,
+            commit_message=args.commit_message,
+            dask_config_kwargs=dask_config["config_kwargs"],
+            dask_cluster_kwargs=dask_config["cluster_kwargs"],
+            icechunk_config=icechunk_config,
+            )
+
+    elif args.action == "list":
+
+        if args.object_prefix is not None:
+            dest = f"{args.bucket}/{args.object_prefix}"
+        else:
+            dest = args.bucket
+
+        list_objects(
+            dest=dest,
+            store_credentials_json=args.store_credentials_json,
+        )
+
+    else:
+        raise NotImplementedError(f"Action {args.action} not implemented.")
+
+
+def ods():
+    """
+    Run the OceanDataStore CLI.
+    """
+    initialise_logging()
+
+    parser = create_parser()
+    args = parser.parse_args()
+
+    process_action(args)
+
+    logging.info("✔ OceanDataStore terminated successfully ✔")
+    sys.exit(0)

OceanDataStore/cli/exceptions.py

@@ -0,0 +1,83 @@
+# ===================================================================
+# Copyright 2026 National Oceanography Centre
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+# ===================================================================
+"""
+exceptions.py
+
+Description:
+This module defines exceptions classes to be
+raised when performing sanity checks.
+
+Authors:
+    - Joao Morado
+    - Tobias Ferreira
+    - Ollie Tooth
+"""
+import logging
+
+
+class ObjectNotFound(Exception):
+    """Exception raised for when an object is not found in the object store."""
+
+    def __init__(self, object_name):
+        """Initialise the exception."""
+        message = f"Object '{object_name}' not found in the object store."
+        logging.warning(message)
+        super().__init__(message)
+
+class AppendDimensionError(Exception):
+    """
+    Exception raised when attempting to modify values along append dimension.
+    """
+
+    def __init__(self, dim):
+        """Initialise the exception."""
+        message = f"Cannot prepend to existing values along append dimension {dim}."
+        logging.warning(message)
+        super().__init__(message)
+
+class DimensionNotFound(Exception):
+    """Exception raised when a dimension is missing."""
+
+    def __init__(self, dim, object_name):
+        """Initialise the exception."""
+        message = f"Dimension {dim} is not found in {object_name}."
+        logging.warning(message)
+        super().__init__(message)
+
+class DimensionSizeError(Exception):
+    """Exception raised when a dimension has incorrect size."""
+
+    def __init__(self, dim, size, expected_size):
+        """Initialise the exception."""
+        message = f"Dimension {dim} has size {size}, expected {expected_size}."
+        logging.warning(message)
+        super().__init__(message)
+
+class AppendDimensionSizeError(Exception):
+    """Exception raised when an append dimension has incorrect size."""
+
+    def __init__(self, dim, size, expected_size):
+        """Initialise the exception."""
+        message = f"Append dimension {dim} has {size} overlapping values, expected {expected_size}."
+        logging.warning(message)
+        super().__init__(message)
+
+class ChunkSizeError(Exception):
+    """Exception raised when data chunks do not match zarr store chunks."""
+
+    def __init__(self, chunks, store_chunks):
+        """Initialise the exception."""
+        message = f"Specified chunks are {chunks}, expected {store_chunks}."
+        logging.warning(message)
+        super().__init__(message)