splitnc 0.0.0__py3-none-any.whl

splitnc/__init__.py

@@ -0,0 +1 @@
+from .splitnc import *

splitnc/esm1p6.py

@@ -0,0 +1,158 @@
+import logging
+import re
+
+
+def _build_model():
+    # Model is always access-esm1p6
+    return "access-esm1p6"
+
+
+def _build_component(ds):
+    # Component: either CICE5 or UM7.3
+    source = ds.attrs["source"]
+    if "Los Alamos Sea Ice Model (CICE) Version 5" in source:
+        return "cice5"
+    elif "Data from Met Office Unified Model" in source and \
+        ds.attrs['um_version'] == "7.3":
+        return "um7p3"
+    else:
+        raise ValueError(f"Unknown source, {source}")
+
+
+def _build_dimensions(ds, field_name):
+    # Dimensions: Don't count time when seeing if field is 2d or 3d
+    ndims = len([d for d in ds[field_name].dims if d!='time'])
+    if ndims == 2:
+        return "2d"
+    elif ndims == 3:
+        return "3d"
+    else:
+        raise ValueError(f"Unexpected number for dimensions, {ndims}")
+
+
+def _build_frequency(ds, field_name, input_filepath):
+    # Frequency: use fx if no time dim
+    if 'time' not in ds[field_name].dims:
+        return "fx"
+
+    # Attempt to parse from expected filenames
+    filename = input_filepath.name
+
+    # Define the expected ice filenames
+    # e.g. iceh-2hourly-mean_0272.nc, iceh-1yearly-mean_0272.nc
+    ice_regex = r"iceh-(?P<num>\d+)(?P<unit>yearly|monthly|daily|hourly)-"
+    ice_unit_mapping = {
+        "yearly": "yr",
+        "monthly": "mon",
+        "daily": "day",
+        "hourly": "hr"
+    }
+
+    if match:=re.match(ice_regex, filename):
+        # Extract the frequency number and units for ice files
+        return f"{match['num']}{ice_unit_mapping[match['unit']]}"
+    elif "_mon.nc" in filename:
+        # Match the monthly pattern for atmosphere files
+        return "1mon"
+    elif "_dai.nc" in filename:
+        # Match the daily pattern for atmosphere files
+        return "1day"
+    elif match:=re.match(r".+_(\d+hr).nc", filename):
+        # Get the frequency from the atmosphere regex match for Xhr
+        return match[1]
+    elif "aiihca.pc" in filename:
+        # Match another pattern for hourly atmosphere files
+        return "1hr"
+
+    # No sub-hourly frequency data expected
+    raise ValueError("Unable to deduce frequency from filename")
+
+
+def _build_cell_method(ds, field_name):
+    attrs = ds[field_name].attrs
+
+    try:
+        if attrs['time_rep'] == "instantaneous":
+            # ice files sometimes have time_rep = instantaneous but not
+            # cell_methods = time: point
+            return ".snap"
+    except KeyError:
+        # Continue if 'time_rep' not in attrs
+        pass
+
+    # Time cell_method: Should be able to deduce from the cell_method
+    cell_method_regx = r"time: (\w+)"
+    try:    
+        if m:= re.search(cell_method_regx, attrs["cell_methods"]):
+            method = m[1]
+            if method == "point":
+                method = "snap"
+
+            # Since this element is optional add the . here
+            return "." + method
+    except KeyError:
+        # Continue if 'cell_methods' not in attrs
+        pass
+
+    # If there's time but no time_bnds and no time cell_method then assume snap
+    # This case is intended to catch instantaneous atmospheric fields from um2nc
+    if "time" in ds and "bounds" not in ds["time"].attrs:
+        return ".snap"
+
+    # Otherwise omit this element from the filename
+    return ""
+
+
+def _build_datestamp(ds, field_name, file_freq):
+    if 'time' not in ds[field_name].dims:
+        # No datetime for fixed files
+        return ""
+
+    # Truncate average time val by output file frequency
+    # datetimes do not correctly zero-pad so need to use %4Y
+    if re.match(r'\d+(yr|dec)', file_freq):
+        fmt = '%4Y'
+    elif re.match(r'\d+mon', file_freq):
+        fmt = '%4Y-%m'
+    elif re.match(r'\d+day', file_freq):
+        fmt = '%4Y-%m-%d'
+    else:
+        fmt = '%4Y-%m-%dT%H:%M:%S'
+
+    # Get the appropriately truncated datetime for the average time
+    try:
+        # Try the time bounds
+        time_arr = ds[ds['time'].attrs["bounds"]]
+        logging.debug("Using time bounds to calculate filename timestamp")
+    except KeyError:
+        # If there are no time bounds just use time
+        logging.debug("Unable to find time bounds, using time to calculate filename timestamp")
+        time_arr = ds['time']
+
+    # Calculate the middle point
+    first, last = time_arr.min(), time_arr.max()
+    datestamp_dt = (first + (last - first) / 2).dt
+
+    return "." + datestamp_dt.strftime(fmt).data.flatten()[0]
+
+
+def build_esm1p6_filename(ds, field_name, input_filepath, esm1p6_filename=False, file_freq="1yr"):
+    template = "{model}.{component}.{dimensions}.{field}.{freq}{time_cell_method}{datestamp}.nc"
+
+    # Model is always access-esm1p6
+    try:
+        d = {
+            "model": _build_model(),
+            "component": _build_component(ds),
+            "dimensions": _build_dimensions(ds, field_name),
+            "field": field_name,
+            "freq": _build_frequency(ds, field_name, input_filepath),
+            "time_cell_method": _build_cell_method(ds, field_name),
+            "datestamp": _build_datestamp(ds, field_name, file_freq),
+        }
+    except ValueError as e:
+        # Reraise the exception with some extra information
+        e.args = (*e.args, f"While building output filename for field {field_name} and {input_filepath}")
+        raise
+
+    return template.format(**d)

splitnc/splitnc.py

@@ -0,0 +1,580 @@
+import argparse
+from collections import Counter
+from datetime import datetime, timezone
+from glob import glob
+import logging
+from pathlib import Path
+from platform import python_version
+import re
+import sys
+
+import xarray as xr
+
+from splitnc.esm1p6 import build_esm1p6_filename
+
+
+def determine_field_vars(ds):
+    """
+    Attempt to determine which variables in the xarray dataset are fields
+
+    If a variable is not depended on by any other variables it is likely to be a
+    field. E.g.
+        F(x, y, z), x, y, z
+            F depends on x, y, and z so x, y, and z are not fields
+            Nothing depends on F, so F is likely to be a field
+
+    Need to check dimensions, bounds, and coordinates
+    """
+    reference_counts = Counter()
+
+    for varname in ds.variables:
+        # Any dims that are not variables will be ignored
+        reference_counts.update(ds[varname].dims)
+
+        try:
+            reference_counts.update(ds[varname].encoding["coordinates"].split())
+        except KeyError:
+            pass
+
+        try:
+            reference_counts.update([ds[varname].attrs["bounds"]])
+        except KeyError:
+            pass
+
+    return sorted(
+        [varname for varname in ds.variables if reference_counts[varname] == 0]
+    )
+
+
+def get_dependent_vars(ds, varname, curr_vars=None):
+    """
+    Get a list of variables that the given variable depends on.
+
+    Check dimensions, bounds, and coordinates
+
+    Recurse on each NEW dependent to get other dependents.
+
+    By only recursing on new dependents infinite recursion in the case of
+    circular dependencies is avoided.
+    """
+    logging.debug(f"Determining dependent variables for {varname}")
+    if curr_vars is None:
+        curr_vars = set()
+
+    # Get any dims that are also variables
+    new_vars = {d for d in ds[varname].dims if d in ds.variables}
+
+    # Get any coords
+    if (
+        "coordinates" in ds[varname].encoding
+        and ds[varname].encoding["coordinates"] is not None
+    ):
+        new_vars.update(ds[varname].encoding["coordinates"].split())
+
+    # Add bounds if the variable has them
+    if "bounds" in ds[varname].attrs:
+        bounds = ds[varname].attrs["bounds"]
+        new_vars.update([bounds])
+
+    # Get the set of vars that are actually new (to avoid infinite recursion)
+    diff_vars = new_vars.difference(curr_vars)
+
+    all_vars = curr_vars | new_vars
+
+    # Recurse on each new var
+    additional_vars = set()
+    for new_v in diff_vars:
+        additional_vars |= get_dependent_vars(ds, new_v, all_vars)
+
+    return diff_vars | additional_vars
+
+
+def get_vars_in_order(ds, varname):
+    """
+    Get the variables in order
+
+    - Start with the field for this dataset,
+    - Followed by the dimensions of the field
+      - each dim followed by their bounds if they exist
+    - Finish with anything remaining in alphabetical order
+    """
+    # Order the variables
+    vars_to_order = list(ds.variables)
+
+    # Start with the field
+    vars_in_order = [varname]
+    vars_to_order.remove(varname)
+
+    # Then the field's dimension and their bnds in order
+    for dim_name in ds[varname].dims:
+        if dim_name not in vars_to_order:
+            continue
+
+        vars_in_order.append(dim_name)
+        vars_to_order.remove(dim_name)
+        if "bounds" in ds[dim_name].attrs:
+            dim_bnd_name = ds[dim_name].attrs["bounds"]
+            if dim_bnd_name in vars_to_order:
+                vars_in_order.append(dim_bnd_name)
+                vars_to_order.remove(dim_bnd_name)
+
+    # Then the remaining variables in alphabetical order
+    vars_in_order += sorted(vars_to_order)
+
+    return vars_in_order
+
+
+def rename_variable(ds, oldname, newname):
+    """
+    Rename a variable, xarray handles most of the rename.
+
+    If the variable has a bounds variable, also rename the matching portion of
+    the bound's name. I.e. latitude -> lat therefore latitude_bnds -> lat_bnds
+    """
+    logging.debug(f"Renaming {oldname} to {newname}")
+    ds_new = ds.rename({oldname: newname})
+
+    for v in ds.variables:
+        # Update cell_methods
+        try:
+            old_cell_methods = ds_new[v].attrs['cell_methods']
+            if old_cell_methods and oldname in old_cell_methods:
+                new_cell_methods = old_cell_methods.replace(oldname, newname)
+                logging.debug(f"Renaming {oldname} to {newname} in {v}'s cell_methods - {old_cell_methods} to {new_cell_methods}")
+                ds_new[v].attrs['cell_methods'] = new_cell_methods
+        except KeyError:
+            # Do nothing if there's no cell_methods
+            pass
+
+        # Update coordinates
+        try:
+            old_coords = ds_new[v].encoding['coordinates']
+            if old_coords and oldname in old_coords:
+                new_coords = old_coords.replace(oldname, newname)
+                logging.debug(f"Renaming {oldname} to {newname} in {v}'s coordinates - {old_coords} to {new_coords}")
+                ds_new[v].encoding['coordinates'] = new_coords
+        except KeyError:
+            # Do nothing if there's no coords
+            pass
+
+    # Update bounds
+    try:
+        old_bnd_name = ds_new[newname].attrs["bounds"]
+        new_bnd_name = old_bnd_name.replace(oldname, newname)
+
+        logging.debug(f"Renaming {old_bnd_name} to {new_bnd_name}")
+        ds_new = rename_variable(ds_new, old_bnd_name, new_bnd_name)
+
+        # Update the attr on the original variable
+        logging.debug(f'Updating "bounds" attr on {newname} to {new_bnd_name}')
+        ds_new[newname].attrs["bounds"] = new_bnd_name
+    except KeyError:
+        # This variable doesn't have bounds
+        pass
+
+    return ds_new
+
+
+def match_regex_list(regex_list, string_list):
+    """
+    Return strings in the given list that match the supplied regex
+    """
+    compiled_regex = [re.compile(regex) for regex in regex_list]
+    return [s for s in string_list if any(r.fullmatch(s) for r in compiled_regex)]
+
+
+def build_rename_dict(ds, rename_regex):
+    """
+    Use the supplied regex to build a dictionary of {"oldname": "newname"} to
+    pass to xarray's rename.
+
+    "newname" should be supplied as a named capture group in the regex.
+
+    E.g. to rename "time_0", "time_1" or "height_0" to "time" or "height", one
+    could use the regex "(?P<newname>.+)_\\d+".
+    """
+    logging.debug("Building rename dict")
+    rename_dict = {}
+    for coord in ds.coords:
+        m = re.fullmatch(rename_regex, str(coord))
+
+        if m:
+            try:
+                newname = m["newname"]
+            except IndexError as e:
+                logging.error(
+                    f"{coord} matched regex for renaming, {rename_regex}, "
+                    'but no "newname" capture group found'
+                )
+                raise e
+
+            logging.debug(f"{coord} will be renamed to {newname}")
+
+            rename_dict[coord] = newname
+
+    return rename_dict
+
+
+def build_history():
+    time_stamp = datetime.now(timezone.utc).isoformat(timespec='seconds')
+    python_exe = f"python{python_version()}"
+
+    # The list of files given on the commandline is not needed in the history
+    args = " ".join(sys.argv)
+  
+    return f"{time_stamp} : splitnc (https://github.com/ACCESS-NRI/esm1.6-scripts) : {python_exe} {args}"
+
+
+def update_history_attr(ds, new_history):
+    if "history" in ds.attrs:
+        old_history = ds.attrs["history"] + "\n"
+    else:
+        old_history = ""
+
+    ds.attrs["history"] = old_history + new_history
+
+
+def fix_cell_methods(ds, varname):
+    """
+    Fix missing cell_methods for instantaneous variables from um2nc.
+
+    If variable has 'time' but no time 'bounds' and there are no other 'time'
+    cell_methods then add 'time: point' to the cell_methods.
+    """
+    if "time" in ds and "bounds" not in ds["time"].attrs:
+        try:
+            cell_methods = ds[varname].attrs["cell_methods"]
+        except KeyError:
+            cell_methods = ""
+
+        if "time" not in cell_methods:
+            new_cell_methods = f"{cell_methods} time: point".strip()
+            logging.debug(f"Updating cell_methods for {varname} to {new_cell_methods}")
+            ds[varname].attrs["cell_methods"] = new_cell_methods
+
+
+def build_filename(ds, field_name, input_filepath, esm1p6_filename=False, file_freq="1yr"):
+    """
+    Build the filename used for the output.
+
+    If esm1p6_filename=False then <field_name>_<orginal_file_name> will be used.
+
+    Otherwise a filename that follows the ESM1.6 naming scheme will be used:
+    {model}.{component}.{dimensions}.{field}.{freq}.{time_cell_method}.{datestamp}.nc
+    More info here: https://access-om3-configs.access-hive.org.au/configurations/Ocean_diagnostics/
+    Elements of this schema will be deduced from the Dataset, the original filename,
+    and the given output file frequency.
+    """
+    if esm1p6_filename:
+        return build_esm1p6_filename(ds, field_name, input_filepath,
+            esm1p6_filename=esm1p6_filename, file_freq=file_freq)
+    else:
+        return f"{field_name}_{input_filepath.name}"
+
+
+def process_file(filepath, **kwargs):
+    # Define default kwargs and update them with kwargs
+    kwargs = {
+        "excluded_vars": [],
+        "shared_vars": [],
+        "field_vars": None,
+        "rename_regex": None,
+        "update_history": True,
+        "fix_cell_methods": False,
+        "output_dir": False,
+        "use_esm1p6_filenames": False,
+        "file_freq": "1yr",
+        "overwrite": False,
+    } | kwargs
+
+    logging.debug(f"Processing {filepath}")
+    filepath = Path(filepath)
+
+    # Use cftime to suppress warnings
+    decoder = xr.coders.CFDatetimeCoder(time_unit='us')
+    with xr.open_dataset(filepath, decode_times=decoder) as ds:
+        # Resolve any regex in the excluded_vars list
+        if excluded_vars:=kwargs["excluded_vars"]:
+            excluded_vars = match_regex_list(excluded_vars, ds.variables)
+        logging.debug(f"List of defined excluded variables is: {excluded_vars}")
+
+        # Resolve any regex in the shared_vars list
+        if shared_vars:=kwargs["shared_vars"]:
+            shared_vars = match_regex_list(shared_vars, ds.variables)
+
+            # shared_vars should not be in excluded vars
+            shared_vars = [v for v in shared_vars if v not in excluded_vars]
+        logging.debug(f"List of defined shared variables is: {shared_vars}")
+
+        # Determine the field vars
+        if field_vars:=kwargs["field_vars"]:
+            # There may be regex to process
+            field_vars = match_regex_list(field_vars, ds.variables)
+        else:
+            logging.debug("Automatically determining field variables")
+            field_vars = determine_field_vars(ds)
+
+        # Shared and excluded vars shouldn't be field_vars
+        logging.debug("Removing shared variables from list of field variables")
+        field_vars = [v for v in field_vars if v not in shared_vars and v not in excluded_vars]
+        logging.debug(f"List of field vars is: {field_vars}")
+
+        # Build the mapping dict for renaming, e.g. {"time_0: "time"}
+        if rename_regex:=kwargs["rename_regex"]:
+            rename_dict = build_rename_dict(ds, rename_regex)
+        else:
+            rename_dict = {}
+        logging.debug(f"Rename dict is {rename_dict}")
+
+        for v in field_vars:
+            # Get the list of vars to keep for this field
+            logging.debug(f"Determining dependent variables for field variable {v}")
+            dependent_vars = get_dependent_vars(ds, v)
+            full_var_list = [v] + list(dependent_vars) + shared_vars
+
+            # Drop any vars not in the list
+            drop_vars_list = [v for v in ds.variables if v not in full_var_list]
+            ds_v = ds.drop_vars(drop_vars_list)
+
+            # Rename anything in the rename dict
+            if rename_dict:
+                for old_name, new_name in rename_dict.items():
+                    if (
+                        old_name in ds_v.variables
+                        or old_name in ds_v.dims
+                        or old_name in ds_v.coords
+                    ):
+                        ds_v = rename_variable(ds_v, old_name, new_name)
+
+            # Coordinates shouldn't have _FillValues
+            for coord in list(ds_v.coords):
+                if coord in ds_v.variables:
+                    logging.debug(f'Setting "_FillValue" to None for {coord}')
+                    ds_v[coord].encoding["_FillValue"] = None
+
+            # Bounds shouldn't have coordinates or _FillValues
+            bnds_set = {
+                ds_v[bnd_v].attrs["bounds"]
+                for bnd_v in ds_v.variables
+                if "bounds" in ds_v[bnd_v].attrs
+            }
+            logging.debug(f"Bounds variables are {bnds_set}")
+            for bnd in bnds_set:
+                logging.debug(
+                    f'Setting "coordinates" and "_FillValue" to None for {bnd}'
+                )
+                ds_v[bnd].encoding["coordinates"] = None
+                ds_v[bnd].encoding["_FillValue"] = None
+
+            # Order the variables
+            vars_in_order = get_vars_in_order(ds_v, v)
+            logging.debug(f"Ordering variable as {vars_in_order}")
+            ds_v = ds_v[vars_in_order]
+
+            # Update the history attribute
+            if kwargs["update_history"]:
+                new_history = build_history()
+                logging.debug(f"Updating history attribute with: {new_history}")
+                update_history_attr(ds_v, new_history)
+
+            # Fix cell_methods
+            if kwargs["fix_cell_methods"]:
+                fix_cell_methods(ds_v, v)
+
+            if output_dir:=kwargs["output_dir"]:
+                output_dir = Path(output_dir)
+            else:
+                output_dir = filepath.parent
+
+            # Build the output filepath
+            filename = build_filename(
+                ds=ds_v,
+                field_name=v,
+                input_filepath=filepath,
+                esm1p6_filename=kwargs["use_esm1p6_filenames"],
+                file_freq=kwargs["file_freq"],
+            )
+            output_filepath = output_dir / filename
+            logging.debug(f"Output filepath is {output_filepath}")
+
+            # Write to file
+            if not kwargs["overwrite"] and output_filepath.exists():
+                logging.error(f"Output file already exists - {output_filepath}")
+                logging.error("Use --overwrite to overwrite existing files")
+
+                raise FileExistsError(f"{output_filepath} already exists")
+
+            logging.debug("Creating parent directory and writing to output file")
+            output_filepath.parent.mkdir(parents=True, exist_ok=True)
+            ds_v.to_netcdf(output_filepath)
+
+
+#### Main
+def arg_parse(cmdline_args=None):
+    # If -c/--command-line-file is being used then all other args are ignored
+    # This affects which are "required" (or nargs for filepaths)
+    args = sys.argv if cmdline_args is None else cmdline_args
+    cmd_file_arg_present = "-c" in args or "--command-line-file" in args
+
+    parser = argparse.ArgumentParser(
+        prog="splitnc",
+        description="Splits a multi-field netCDF file into separate one-field files",
+    )
+
+    # Create a custom type for comma separated strings as lists
+    def comma_separated_string_type(s):
+        return s.split(",")
+
+    # Open the named file and parse it as a command line split it around the
+    # whitespaces (including newlines)
+    def command_line_file(filepath):
+        with open(filepath, "r") as f:
+            file_str = f.read()
+
+        return file_str.split()
+
+    # Filepath wildcards won't be expanded if supplied via a command line file
+    # I.e. *.nc won't be expanded by the shell to [file1.nc, file2.nc]
+    def globbable_string_list(string_list):
+        return glob(string_list)
+
+    # Let filepaths be optional (i.e. nargs=* instead of +) so that it isn't
+    # required and --cmd-line-file can be used on it's own
+    parser.add_argument(
+        "filepaths",
+        nargs="*" if cmd_file_arg_present else "+",
+        default=[],
+        type=globbable_string_list,
+        help="One or more filepaths to process",
+    )
+    parser.add_argument(
+        "--field-vars",
+        type=comma_separated_string_type,
+        default=[],
+        metavar="FIELD_VAR1,FIELD_VAR2,...",
+        help="Specify the names of the field variables to split into separate "
+        "files - dimensions, bounds, and coordinates of these fields will "
+        "be included in each file. Disables automatic field variable "
+        "identification. Regex patterns can be used here.",
+    )
+    parser.add_argument(
+        "--shared-vars",
+        type=comma_separated_string_type,
+        default=[],
+        metavar="SHARED_VAR1,SHARED_VAR2,...",
+        help="Specify the names of variables that should be shared across "
+        "files that cannot be automatically identified, as a comma "
+        "separated list. Regex patterns can be used here.",
+    )
+    parser.add_argument(
+        "--excluded-vars",
+        type=comma_separated_string_type,
+        default=[],
+        metavar="EXCLUDED_VAR1,EXCLUDED_VAR2,...",
+        help="Specify the names of variables that should be excluded from "
+        "files. This option can be used with automatic identification of field "
+        "variables. Regex patterns can be used here.",
+    )
+    parser.add_argument(
+        "--rename-regex",
+        metavar="REGEX",
+        help="Look for duplicated coordinate names that match the given regex "
+        'and rename them to the first "newname" capture group in the '
+        'regex. E.g. "(?P<newname>.*)_\\d+" will match "time_0" and '
+        'rename it to "time".',
+    )
+    parser.add_argument(
+        "--use-esm1p6-filenames",
+        action="store_true",
+        help="Use the ESM1.6 filename pattern for the output files: "
+        "access-esm1p6.{component}.{dimensions}.{field}.{freq}.{time_cell_method}.{datestamp}.nc"
+        " splitnc will attempt to deduce all the components of the filename. "
+        "If this option is not given {field}_{original_filename} will be used."
+    )
+    parser.add_argument(
+        "--fix-cell-methods",
+        action="store_true",
+        help="Correct cell_methods by adding 'time: point' to cell_methods "
+        "for variables that have 'time' but not 'time_bnds' and no other "
+        "'time' cell_methods."
+    )
+    parser.add_argument(
+        "--file-freq",
+        default="1yr",
+        help="Specify the frequency of the files (not the data), e.g. if each "
+        "file contains a month of data then the file-frequency is '1mon'. Used "
+        "to determine the resolution of the timestamp for ESM1.6 filenames. "
+        "Follows the ACCESS frequency vocabulary (e.g. '1yr', '1mon', '1day', "
+        "'1hr'), any unrecognised frequency will use the full timestamp. "
+        "Defaults to '1yr'."
+    )
+    parser.add_argument(
+        "--output-dir",
+        help="Output directory for the processed files. If not given output "
+        "files will be placed in the same directory as the original file.",
+    )
+    parser.add_argument(
+        "--overwrite", action="store_true", help="Overwrite existing files"
+    )
+    # By default update the history attr
+    # To avoid passing around a negative store_false and rename this arg
+    parser.add_argument(
+        "--dont-update-history",
+        action="store_false",
+        dest="update_history",
+        help="Disable automatic update of history attribute"
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-c",
+        "--command-line-file",
+        type=command_line_file,
+        help="A file containing a list of command-line arguments. Newlines in "
+        "this file will be ignored. If supplied all other command line "
+        "arguments will be ignored.",
+    )
+
+    args = parser.parse_args(args=cmdline_args)
+
+    # File paths may need flattened since glob was used
+    args.filepaths = [
+        filepath for glob_list in args.filepaths for filepath in glob_list
+    ]
+
+    # If the command line yaml was supplied use the contents instead of argv
+    if args.command_line_file:
+        return arg_parse(args.command_line_file)
+    else:
+        return args
+
+
+def setup_logging(verbose=False):
+    logging.basicConfig(
+        level=logging.DEBUG if verbose else logging.WARNING,
+        format="{asctime} - {levelname} - {message}",
+        style="{",
+        datefmt="%Y-%m-%d %H:%M",
+    )
+
+
+def main():
+    args = arg_parse()
+
+    setup_logging(args.verbose)
+
+    logging.debug(f"Command line args are: {args}")
+
+    if len(args.filepaths) == 0:
+        logging.error("No files to process.")
+        raise ValueError("No files to process.")
+
+    for f in args.filepaths:
+        process_file(f, **vars(args))
+
+
+if __name__ == "__main__":
+    main()

splitnc-0.0.0.dist-info/METADATA

@@ -0,0 +1,144 @@
+Metadata-Version: 2.4
+Name: splitnc
+Version: 0.0.0
+Summary: Split multi-field ESM1.6 files into single-field files
+Author-email: Joshua Torrance <joshua.torrance@anu.edu.au>, Spencer Wong <spencer.wong@anu.edu.au>
+Maintainer-email: Joshua Torrance <joshua.torrance@anu.edu.au>
+License-Expression: Apache-2.0
+Project-URL: Repository, https://github.com/ACCESS-NRI/splitnc
+Keywords: ACCESS-NRI,netcdf
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: netcdf4
+Requires-Dist: xarray
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: netCDF4; extra == "test"
+Requires-Dist: xarray>2025.1.2; extra == "test"
+Dynamic: license-file
+
+# splitnc
+This script splits multi-field netCDF files into single-field files.
+It is designed to work on ESM1.6's atmosphere and ice files.
+
+## Automatic Field Identification
+By default `splitnc` will attempt to identify the fields for a multi-field netCDF files by looking for variables that no other variables depend on.
+A variable that no others depend on is likely to be a field.
+E.g. many variables depend on `time`, but none depend on `sea_surface_temperature`.
+
+Alternatively the fields to separate to individual files can be specified as a comma separated list with the `--field-vars` command line option.
+`--field-vars` interprets each item as regex, e.g. one could use `--field-vars fld_.+` to match all variable names that start with the string `fld_`.
+
+## "Ancillary" Variables
+
+Some variables with no dependents should not be separated into individual files, these variables must be manually identified with the `--shared-vars` command line option.
+These variables will then be present in every output file.
+Regex is also supported for this option.
+
+If there are ancillary fields that should only be present in only some of the output field files then multiple invocations of `splitnc` using `--field-vars` and `--shared-vars` will be required.
+
+Example of these variables are the `latitude_longitude` found in atmosphere files or the `uarea`, `tmask`, `tarea`, `VGRDb`, `VGRDi`, `VGRDs` variables from ice files.
+
+## Config File
+
+The `-c`/`--command-line-file` option can be used to supply a filepath to a file that contains command line options.
+If this option is used, all other options supplied on the command line will be ignored.
+Newline characters in the file will be treated as whitespace, i.e. newlines can be used as well as spaces to separate command line arguments.
+
+For example to replicate this command line,
+```
+splitnc --verbose --overwrite --output-dir /output/directory --shared-vars latitude_longitude --rename-regex "(?P<newname>.+)_\d+" /input/directory/*.nc
+```
+the following file could be used;
+```
+--verbose
+--overwrite
+--output-dir /output/directory
+--shared-vars latitude_longitude
+--rename-regex "(?P<newname>.+)_\d+"
+/input/directory/*.nc
+```
+
+## Command Line Options
+
+```quote
+usage: splitnc [-h] [--field-vars FIELD_VAR1,FIELD_VAR2,...] [--shared-vars SHARED_VAR1,SHARED_VAR2,...]
+               [--output-name-pattern OUTPUT_NAME_PATTERN] [--rename-regex REGEX] [--output-dir OUTPUT_DIR] [--overwrite] [-v]
+               [-c COMMAND_LINE_FILE]
+               [filepaths ...]
+
+Splits a multi-field netCDF file into separate one-field files
+
+positional arguments:
+  filepaths             One or more filepaths to process
+
+options:
+  -h, --help            show this help message and exit
+  --field-vars FIELD_VAR1,FIELD_VAR2,...
+                        Specify the names of the field variables to split into separate files - dimensions, bounds, and
+                        coordinates of these fields will be included in each file. Disables automatic field variable
+                        identification. Regex patterns can be used here.
+  --shared-vars SHARED_VAR1,SHARED_VAR2,...
+                        Specify the names of variables that should be shared across files that cannot be automatically
+                        identified, as a comma separated list. Regex patterns can be used here.
+  --excluded-vars EXCLUDED_VAR1,EXCLUDED_VAR2,...
+                        Specify the names of variables that should be excluded from files. This option can be used with
+                        automatic identification of field variables. Regex patterns can be used here.
+  --rename-regex REGEX  Look for duplicated coordinate names that match the given regex and rename them to the first
+                        "newname" capture group in the regex. E.g. "(?P<newname>.*)_\d+" will match "time_0" and rename
+                        it to "time".
+  --use-esm1p6-filenames
+                        Use the ESM1.6 filename pattern for the output files:
+                        access-esm1p6.{component}.{dimensions}.{field}.{freq}.{time_cell_method}.{datestamp}.nc
+                        splitnc will attempt to deduce all the components of the filename. If this option is not given
+                        {field}_{original_filename} will be used.
+  --fix-cell-methods    Correct cell_methods by adding 'time: point' to cell_methods for variables that have 'time' but
+                        not 'time_bnds' and no other 'time' cell_methods.
+  --file-freq FILE_FREQ
+                        Specify the frequency of the files (not the data), e.g. if each file contains a month of data
+                        then the file-frequency is '1mon'. Used to determine the resolution of the timestamp for ESM1.6
+                        filenames. Follows the ACCESS frequency vocabulary (e.g. '1yr', '1mon', '1day', '1hr'), any
+                        unrecognised frequency will use the full timestamp. Defaults to '1yr'.
+  --output-dir OUTPUT_DIR
+                        Output directory for the processed files. If not given output files will be placed in the same
+                        directory as the original file.
+  --overwrite           Overwrite existing files
+  --dont-update-history
+                        Disable automatic update of history attribute
+  -v, --verbose
+  -c COMMAND_LINE_FILE, --command-line-file COMMAND_LINE_FILE
+                        A file containing a list of command-line arguments. Newlines in this file will be ignored. If
+                        supplied all other command line arguments will be ignored.
+```
+
+## Example Usage
+
+`splitnc` just needs the `xarray` and `netCDF4` python modules.
+On Gadi use load any module with `xarray`, such as `conda/analysis3`.
+Alternatively create a new python environment and install `xarray` and `netCDF4`.
+
+### Atmosphere
+To use this script for split multi-field atmosphere files from ACCESS-ESM1.6:
+```bash
+splitnc --shared-vars latitude_longitude  --rename-regex "(?P<newname>.+)_\\d+" $INPUT_DIR/*.nc
+```
+
+`splitnc` will automatically determine which variables are fields by looking at which variables depend on other variables.
+Variables with nothing depending on them are deemed to be fields.
+Alternatively one could use `--field-vars fld_.+` to match the variable names in these files.
+
+The `--rename-regex` option with the supplied regex will rename variables like
+`time_0` or `pseudo_level_0` are renamed to `time` or `pseudo_level`.
+
+The `--shared-vars` option will ensure that the variable `latitude_longitude` is
+included in all files even though none of the field variable depend on it.
+
+### Ice
+To use this script for split multi-field ice files from ACCESS-ESM1.6:
+```bash
+splitnc --shared-vars uarea,tmask,tarea --excluded-vars VGRD. $INPUT_DIR/*.nc
+```
+
+In comparison to the atmosphere files, ice files have different shared-vars and there are no duplicated variables that require renaming.
+The variables `VGRDb`, `VGRDi`, and `VGRDs` are not required and can thus be excluded from the output.

splitnc-0.0.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+splitnc/__init__.py,sha256=YoWizDzFzj52Hb_0Lb8qzhWUL-HxOGv1exV-v1dSEW8,23
+splitnc/esm1p6.py,sha256=WsOcUoLFuQmS2Z5QM663JmdnjEb6cEnk6IaFT6YPaM4,5285
+splitnc/splitnc.py,sha256=Uckg9i0ATtRJ87k-7VlXwa7i_jjXPKPbz2PtBSKWNTg,20822
+splitnc-0.0.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+splitnc-0.0.0.dist-info/METADATA,sha256=N8lLH7ms4uxmD8F2miGkz8VGu0GwLEmhsjFMA_ni8H8,7720
+splitnc-0.0.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+splitnc-0.0.0.dist-info/entry_points.txt,sha256=QbOAcAYWj8A_WygPT88igXxc-ke9hypfaqn2ZvFATvA,41
+splitnc-0.0.0.dist-info/top_level.txt,sha256=lif5g27mG8Nw83f9OBwNwlfacXkin5Ud8xJPCYrXmlA,8
+splitnc-0.0.0.dist-info/RECORD,,

splitnc-0.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

splitnc-0.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+splitnc = splitnc:main

splitnc-0.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

splitnc-0.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+splitnc