datacube-ows 1.9.1__py3-none-any.whl

datacube_ows/__init__.py

@@ -0,0 +1,12 @@
+# This file is part of datacube-ows, part of the Open Data Cube project.
+# See https://opendatacube.org for more information.
+#
+# Copyright (c) 2017-2024 OWS Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+try:
+    from ._version import version as __version__
+except ImportError:
+    # Default version number.
+    # Will only be used when running datacube-ows direct from source code (not properly installed)
+    __version__ = "1.9.1?"

datacube_ows/_version.py

@@ -0,0 +1,21 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '1.9.1'
+__version_tuple__ = version_tuple = (1, 9, 1)

datacube_ows/band_utils.py

@@ -0,0 +1,241 @@
+# This file is part of datacube-ows, part of the Open Data Cube project.
+# See https://opendatacube.org for more information.
+#
+# Copyright (c) 2017-2024 OWS Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+import numpy
+
+# Style index functions
+
+
+def scale_data(imgband_data, scale_from, scale_to):
+    sc_min, sc_max = scale_from
+    tc_min, tc_max = scale_to
+    clipped = imgband_data.clip(sc_min, sc_max)
+    normalised = (clipped - sc_min) / (sc_max - sc_min)
+    scaled = normalised * (tc_max - tc_min)
+    return scaled + tc_min
+
+
+def scalable(undecorated):
+    def decorated(*args, **kwargs):
+        scale_from = kwargs.pop("scale_from", None)
+        scale_to = kwargs.pop("scale_to", None)
+        if scale_from is not None and scale_to is None:
+            scale_to = [0, 255]
+        unscaled = undecorated(*args, **kwargs)
+        if scale_from:
+            return scale_data(unscaled, scale_from, scale_to)
+        return unscaled
+
+    return decorated
+
+
+def band_modulator(undecorated):
+    def decorated(data, *args, **kwargs):
+        band_mapper = kwargs.get("band_mapper", None)
+        mult_band = kwargs.pop("mult_band", None)
+        raw_data = undecorated(data, *args, **kwargs)
+        if mult_band and band_mapper:
+            mult_band = band_mapper(mult_band)
+        if mult_band:
+            return data[mult_band] * raw_data
+        return raw_data
+    return decorated
+
+def pre_scaled_band(data, band, scale, offset):
+    # Pre-scale a band as `data[band] * scale + offset`
+    return data[band] * scale + offset
+
+def sum_bands(data, band1, band2, band_mapper=None):
+    if band_mapper:
+        band1 = band_mapper(band1)
+        band2 = band_mapper(band2)
+    return data[band1] + data[band2]
+
+def pre_scaled_sum_bands(
+    data, band1, band2,
+    scale1=1.0, offset1=0.0,
+    scale2=1.0, offset2=0.0,
+    band_mapper=None):
+    # Calculate the sum of two bands, after pre-scaling them with a scale and offset
+    if band_mapper:
+        band1 = band_mapper(band1)
+        band2 = band_mapper(band2)
+    # Pre-scaled bands data
+    data1 = pre_scaled_band(data, band1, scale1, offset1)
+    data2 = pre_scaled_band(data, band2, scale2, offset2)
+    return data1 + data2
+
+
+def delta_bands(data, band1, band2, band_mapper=None):
+    if band_mapper:
+        band1 = band_mapper(band1)
+        band2 = band_mapper(band2)
+    typ1 = data[band1].dtype
+    typ2 = data[band2].dtype
+    if typ1.name.startswith('uint'):
+        nodata = data[band1].nodata
+        data[band1] = data[band1].astype('int32')
+        data[band1].attrs["nodata"] = nodata
+    if typ2.name.startswith('uint'):
+        nodata = data[band2].nodata
+        data[band2] = data[band2].astype('int32')
+        data[band2].attrs["nodata"] = nodata
+    # if typ1.name.startswith('uint') or typ2.name.startswith('uint'):
+        # data = data.astype('int32', copy=False)
+    return data[band1] - data[band2]
+
+def pre_scaled_delta_bands(
+    data, band1, band2,
+    scale1=1.0, offset1=0.0,
+    scale2=1.0, offset2=0.0,
+    band_mapper=None):
+    # Calculate the difference between two bands, after pre-scaling them with a scale
+    # and offset
+    if band_mapper:
+        band1 = band_mapper(band1)
+        band2 = band_mapper(band2)
+    typ1 = data[band1].dtype
+    typ2 = data[band2].dtype
+    if typ1.name.startswith('uint'):
+        nodata = data[band1].nodata
+        data[band1] = data[band1].astype('int32')
+        data[band1].attrs["nodata"] = nodata
+    if typ2.name.startswith('uint'):
+        nodata = data[band2].nodata
+        data[band2] = data[band2].astype('int32')
+        data[band2].attrs["nodata"] = nodata
+    # if typ1.name.startswith('uint') or typ2.name.startswith('uint'):
+        # data = data.astype('int32', copy=False)
+        # Pre-scaled bands data
+    data1 = pre_scaled_band(data, band1, scale1, offset1)
+    data2 = pre_scaled_band(data, band2, scale2, offset2)
+    return data1 - data2
+
+# N.B. Modifying scale_to would be dangerous - don't do it.
+# pylint: disable=dangerous-default-value
+@scalable
+def norm_diff(data, band1, band2, band_mapper=None):
+    # Calculate a normalised difference index.
+    return delta_bands(data, band1, band2, band_mapper) / sum_bands(data, band1, band2, band_mapper)
+
+@scalable
+def pre_scaled_norm_diff(data, band1, band2,
+                         scale1=1.0, offset1=0.0,
+                         scale2=1.0, offset2=0.0,
+                         band_mapper=None,):
+    # Calculate a normalised difference index, after scaling the input bands with a
+    # scale and offset
+    return (pre_scaled_delta_bands(
+        data, band1, band2, scale1, offset1, scale2, offset2, band_mapper) /
+        pre_scaled_sum_bands(
+            data, band1, band2, scale1, offset1, scale2, offset2, band_mapper)
+    )
+@scalable
+def constant(data, band, const, band_mapper=None):
+    # Covert an xarray for a flat constant.
+    # Useful for displaying mask extents as a flat colour and debugging.
+    # params is assumed to be a tuple containing a constant value and a band name/alias.
+
+    if band_mapper:
+        band = band_mapper(band)
+    return data[band] * 0.0 + const
+
+
+@scalable
+def single_band(data, band, band_mapper=None):
+    # Use the raw value of a band directly as the index function.
+
+    if band_mapper:
+        band = band_mapper(band)
+    return data[band]
+
+
+@scalable
+def band_quotient(data, band1, band2, band_mapper=None):
+    if band_mapper:
+        band1 = band_mapper(band1)
+        band2 = band_mapper(band2)
+    return data[band1] / data[band2]
+
+
+@scalable
+def band_quotient_sum(data, band1a, band1b, band2a, band2b, band_mapper=None):
+    return band_quotient(data, band1a, band1b, band_mapper) + band_quotient(data, band2a, band2b, band_mapper)
+
+
+@scalable
+def sentinel2_ndci(data, b_red_edge, b_red, b_green, b_swir, band_mapper=None):
+    red_delta = delta_bands(data, b_red_edge, b_red, band_mapper)
+    red_sum = sum_bands(data, b_red_edge, b_red, band_mapper)
+    mndwi = norm_diff(data, b_green, b_swir, band_mapper)
+
+    return red_delta / red_sum.where(mndwi > 0.1)
+
+
+def multi_date_delta(data, time_direction=-1):
+    data1, data2 = (data.sel(time=dt) for dt in data.coords["time"].values)
+
+#    data1, data2 = data.values.item(0), data.values.item(1)
+    if time_direction >= 0:
+        return data1 - data2
+    else:
+        return data2 - data1
+
+def multi_date_pass(data):
+    return data
+
+def multi_date_raw_example(data, band1, band2, band_mapper=None):
+    if band_mapper:
+        band1 = band_mapper(band1)
+        band2 = band_mapper(band2)
+    data1, data2 = (data.sel(time=dt) for dt in data.coords["time"].values)
+    return (
+            (data2[band1] - data1[band1]) - (data2[band2] - data1[band2])
+    ) / (
+            data1[band1] + data1[band2] + data2[band1] + data2[band2]
+    )
+
+
+@band_modulator
+@scalable
+def single_band_log(data, band, scale_factor, exponent, band_mapper=None):
+    if band_mapper:
+        band = band_mapper(band)
+    d = data[band]
+    return scale_factor * ((d ** exponent) - 1.0)
+
+
+@band_modulator
+@scalable
+def single_band_arcsec(data, band, band_mapper=None):
+    if band_mapper:
+        band = band_mapper(band)
+    d = data[band]
+    return numpy.arccos(1 / (d + 1))
+
+
+@band_modulator
+@scalable
+def single_band_offset_log(data, band, scale=1.0, offset=None, band_mapper=None):
+    if band_mapper:
+        band = band_mapper(band)
+    d = data[band]
+    if offset is not None:
+        d = data[band] * scale + offset
+        unscaled = numpy.log(d)
+    else:
+        unscaled = numpy.log1p(d * scale)
+    return unscaled
+
+
+@scalable
+def radar_vegetation_index(data, band_hv, band_hh, band_mapper=None):
+    if band_mapper:
+        band_hv = band_mapper(band_hv)
+        band_hh = band_mapper(band_hh)
+    hv_sq = data[band_hv] * data[band_hv]
+    hh_sq = data[band_hh] * data[band_hh]
+    return (hv_sq * 4.0) / (hh_sq + hv_sq)

datacube_ows/cfg_parser_impl.py

@@ -0,0 +1,388 @@
+#!/usr/bin/env python3
+# This file is part of datacube-ows, part of the Open Data Cube project.
+# See https://opendatacube.org for more information.
+#
+# Copyright (c) 2017-2024 OWS Contributors
+# SPDX-License-Identifier: Apache-2.0
+
+
+import json
+import os
+import sys
+from typing import cast
+
+import click
+from babel.messages.pofile import write_po
+from datacube import Datacube
+from deepdiff import DeepDiff
+
+from datacube_ows import __version__
+from datacube_ows.config_utils import ConfigException
+from datacube_ows.ows_configuration import (OWSConfig, OWSFolder, OWSLayer,
+                                            OWSNamedLayer, read_config)
+
+
+@click.group(invoke_without_command=True)
+@click.option(
+    "--version", is_flag=True, default=False, help="Show OWS version number and exit"
+)
+def main(version: bool) -> int:
+    # --version
+    if version:
+        click.echo(f"Open Data Cube Open Web Services (datacube-ows) version {__version__}")
+    return 0
+
+
+@main.command()
+@click.argument("paths", nargs=-1)
+@click.option(
+    "-p",
+    "--parse-only",
+    is_flag=True,
+    default=False,
+    help="Only parse the syntax of the config file - do not validate against database",
+)
+@click.option(
+    "-f",
+    "--folders",
+    is_flag=True,
+    default=False,
+    help="Print the folder/layer heirarchy(ies) to stdout.",
+)
+@click.option(
+    "-s",
+    "--styles",
+    is_flag=True,
+    default=False,
+    help="Print the styles for each layer to stdout (format depends on --folders flag).",
+)
+@click.option(
+    "-i",
+    "--input-file",
+    help="Provide a file path for the input inventory json file to be compared with config file",
+)
+@click.option(
+    "-o",
+    "--output-file",
+    help="Provide an output inventory file name with extension .json",
+)
+def check(parse_only: bool, folders: bool, styles: bool, input_file: str, output_file: str, paths: list[str]) -> int:
+    """Check configuration files
+
+    Takes a list of configuration specifications which are each loaded and validated in turn,
+    with each specification being interpreted as per the $DATACUBE_OWS_CFG environment variable.
+
+    If no specification is provided, the $DATACUBE_OWS_CFG environment variable is used.
+    """
+    if parse_only and (folders or styles):
+        click.echo(
+            "The --folders (-f) and --styles (-s) flags cannot be used in conjunction with the --parser-only (-p) flag."
+        )
+        sys.exit(1)
+    all_ok = True
+    if not paths:
+        if parse_path(None, parse_only, folders, styles, input_file, output_file):
+            return 0
+        else:
+            sys.exit(1)
+    for path in paths:
+        if not parse_path(path, parse_only, folders, styles, input_file, output_file):
+            all_ok = False
+
+    if not all_ok:
+        sys.exit(1)
+    return 0
+
+
+def parse_path(path: str | None, parse_only: bool, folders: bool, styles: bool, input_file: str, output_file: str) -> bool:
+    try:
+        raw_cfg = read_config(path)
+        cfg = OWSConfig(refresh=True, cfg=raw_cfg)
+        if not parse_only:
+            with Datacube() as dc:
+                cfg.make_ready(dc)
+    except ConfigException as e:
+        click.echo(f"Config exception for path {str(e)}")
+        return False
+    click.echo("Configuration parsed OK")
+    click.echo(f"Configured message file location: {cfg.msg_file_name}")
+    click.echo(f"Configured translations directory location: {cfg.translations_dir}")
+    if folders:
+        click.echo()
+        click.echo("Folder/Layer Hierarchy")
+        click.echo("======================")
+        print_layers(cfg.layers, styles, depth=0)
+        click.echo()
+    elif styles:
+        click.echo()
+        click.echo("Layers and Styles")
+        click.echo("=================")
+        for lyr in cfg.layer_index.values():
+            click.echo(f"{lyr.name} [{','.join(lyr.product_names)}]")
+            print_styles(lyr)
+        click.echo()
+    if input_file or output_file:
+        layers_report(cfg.layer_index, input_file, output_file)
+    return True
+
+
+@main.command()
+@click.option(
+    "-c",
+    "--cfg-only",
+    is_flag=True,
+    default=False,
+    help="Read metadata from config only - ignore configured metadata message file.",
+)
+@click.option(
+    "-m",
+    "--msg-file",
+    default="messages.po",
+    help="Write to a message file with the translatable metadata from the configuration. (Defaults to 'messages.po')"
+)
+@click.argument("path", nargs=1, required=False)
+def extract(path: str, cfg_only: bool, msg_file: str) -> int:
+    """Extract metadata from existing configuration into a message file template.
+
+    Takes a configuration specification which is loaded as per the $DATACUBE_OWS_CFG environment variable.
+
+    If no specification is provided, the $DATACUBE_OWS_CFG environment variable is used.
+    """
+    try:
+        raw_cfg = read_config(path)
+        cfg = OWSConfig(refresh=True, cfg=raw_cfg, ignore_msgfile=cfg_only)
+        with Datacube() as dc:
+            cfg.make_ready(dc)
+    except ConfigException as e:
+        click.echo(f"Config exception for path {str(e)}")
+        return False
+    click.echo("Configuration parsed OK")
+    click.echo(f"Configured message file location: {cfg.msg_file_name}")
+    click.echo(f"Configured translations directory location: {cfg.translations_dir}")
+    write_msg_file(msg_file, cfg)
+    click.echo(f"Message file {msg_file} written")
+    return 0
+
+
+@main.command()
+@click.option(
+    "-n",
+    "--new",
+    is_flag=True,
+    default=False,
+    help="Create a new translation template. (Default is to update an existing one.)"
+)
+@click.option(
+    "-m",
+    "--msg-file",
+    default=None,
+    help="Use this message file as the template for translation files. (defaults to message filename from configuration)"
+)
+@click.option(
+    "-d",
+    "--translations-dir",
+    default=None,
+    help="Path to the output translations directory. Defaults to value from configuration"
+)
+@click.option(
+    "-D",
+    "--domain",
+    default=None,
+    help="The domain of the translation files. Defaults to value from configuration"
+)
+@click.option(
+    "-c",
+    "--cfg",
+    default=None,
+    help="Configuration specification to use to determine translations directory and domain (defaults to environment $DATACUBE_OWS_CFG)"
+)
+@click.argument("languages", nargs=-1)
+def translation(languages: list[str], msg_file: str | None, new: bool,
+                domain: str | None, translations_dir: str | None, cfg: str | None) -> int:
+    """Generate a new translations catalog based on the specified message file.
+
+    Takes a list of languages to generate catalogs for. "all" can be included as a shorthand
+    for all languages listed as supported in the configuration.
+    """
+    if len(languages) == 0:
+        click.echo("No language(s) specified.")
+        sys.exit(1)
+    if msg_file is None or domain is None or translations_dir is None or "all" in languages:
+        try:
+            raw_cfg = read_config(cfg)
+            config = OWSConfig(refresh=True, cfg=raw_cfg)
+        except ConfigException as e:
+            click.echo(f"Config exception for path: {str(e)}")
+            sys.exit(1)
+        if domain is None:
+            click.echo(f"Using message domain '{config.message_domain}' from configuration")
+            domain = config.message_domain
+        if translations_dir is None and config.translations_dir is None:
+            click.echo("No translations directory was supplied or is configured")
+            sys.exit(1)
+        elif translations_dir is None:
+            click.echo(f"Using translations directory '{config.translations_dir}' from configuration")
+            translations_dir = config.translations_dir
+        if msg_file is None and config.msg_file_name is None:
+            click.echo("No message file name was supplied or is configured")
+            sys.exit(1)
+        elif msg_file is None:
+            click.echo(f"Using message file location '{config.msg_file_name}' from configuration")
+            msg_file = config.msg_file_name
+        all_langs = config.locales
+    else:
+        all_langs = []
+    assert msg_file is not None  # For type checker
+    assert domain is not None  # For type checker
+    assert translations_dir is not None  # For type checker
+    try:
+        fp = open(msg_file, "rb")
+        fp.close()
+    except OSError:
+        click.echo("Message file {msg_file} does not exist or cannot be read.")
+        sys.exit(1)
+    for language in languages:
+        if language == "all":
+            for supp_lang in all_langs:
+                if new:
+                    create_translation(msg_file, translations_dir, domain, supp_lang)
+                else:
+                    update_translation(msg_file, translations_dir, domain, supp_lang)
+        else:
+            if new:
+                create_translation(msg_file, translations_dir, domain, language)
+            else:
+                update_translation(msg_file, translations_dir, domain, language)
+    click.echo("Language templates created.")
+    return 0
+
+
+def create_translation(msg_file: str, translations_dir: str, domain: str, locale: str) -> bool:
+    click.echo(f"Creating template for language: {locale}")
+    os.system(f"pybabel init -i {msg_file} -d {translations_dir} -D {domain} -l {locale}")
+    return True
+
+
+def update_translation(msg_file: str, translations_dir: str, domain: str, locale: str) -> bool:
+    click.echo(f"Updating template for language: {locale}")
+    os.system(f"pybabel update --no-fuzzy-matching --ignore-obsolete "
+              f"-i {msg_file} -d {translations_dir} -D {domain} -l {locale}")
+    return True
+
+
+@main.command(name="compile")
+@click.option(
+    "-d",
+    "--translations-dir",
+    default=None,
+    help="Path to the output translations directory. Defaults to value from configuration"
+)
+@click.option(
+    "-D",
+    "--domain",
+    default=None,
+    help="The domain of the translation files. Defaults to value from configuration"
+)
+@click.option(
+    "-c",
+    "--cfg",
+    default=None,
+    help="Configuration specification to use to determine translations directory and domain (defaults to environment $DATACUBE_OWS_CFG)"
+)
+@click.argument("languages", nargs=-1)
+def compile_cmd(languages: list[str], domain: str | None, translations_dir: str | None, cfg: str | None) -> int:
+    """Compile completed translation files.
+
+    Takes a list of languages to generate catalogs for. "all" can be included as a shorthand
+    for all languages listed as supported in the configuration.
+    """
+    if len(languages) == 0:
+        click.echo("No language(s) specified.")
+        sys.exit(1)
+    if domain is None or translations_dir is None or "all" in languages:
+        try:
+            raw_cfg = read_config(cfg)
+            config = OWSConfig(refresh=True, cfg=raw_cfg)
+        except ConfigException as e:
+            click.echo(f"Config exception for path: {str(e)}")
+            sys.exit(1)
+        if domain is None:
+            click.echo(f"Using message domain '{config.message_domain}' from configuration")
+            domain = config.message_domain
+        if translations_dir is None and config.translations_dir is None:
+            click.echo("No translations directory was supplied or is configured")
+            sys.exit(1)
+        elif translations_dir is None:
+            click.echo(f"Using translations directory '{config.translations_dir}' from configuration")
+            translations_dir = config.translations_dir
+        all_langs = config.locales
+    else:
+        all_langs = []
+    assert translations_dir is not None
+    for language in languages:
+        if language == "all":
+            for supp_lang in all_langs:
+                compile_translation(translations_dir, domain, supp_lang)
+        else:
+            compile_translation(translations_dir, domain, language)
+    click.echo("Language templates created.")
+    return 0
+
+
+def compile_translation(translations_dir: str, domain: str, language: str) -> bool:
+    click.echo(f"Compiling template for language: {language}")
+    os.system(f"pybabel compile -d {translations_dir} -D {domain} -l {language}")
+    return True
+
+
+def write_msg_file(msg_file: str, cfg: OWSConfig) -> None:
+    with open(msg_file, "wb") as fp:
+        write_po(fp, cfg.export_metadata())
+
+
+def layers_report(config_values: dict[str, OWSNamedLayer], input_file: str, output_file: str) -> None:
+    report = {"total_layers_count": len(config_values.values()), "layers": []}
+    for lyr in config_values.values():
+        layer = {
+            "layer": lyr.name,
+            "product": list(lyr.product_names),
+            "styles_count": len(lyr.styles),
+            "styles_list": [styl.name for styl in lyr.styles],
+        }
+        cast(list[dict], report["layers"]).append(layer)
+    if input_file:
+        with open(input_file) as f:
+            input_file_data = json.load(f)
+        ddiff = DeepDiff(input_file_data, report, ignore_order=True)
+        if len(ddiff) != 0:
+            click.echo(ddiff)
+            sys.exit(1)
+    elif output_file:
+        with open(output_file, 'w') as reportfile:
+            json.dump(report, reportfile, indent=4)
+
+
+def print_layers(layers: list[OWSLayer], styles: bool, depth: int) -> None:
+    for lyr in layers:
+        if isinstance(lyr, OWSFolder):
+            indent(depth)
+            click.echo(f"* {lyr.title}")
+            click.echo(f"{lyr.child_layers} {styles} {depth + 1}")
+        else:
+            indent(depth)
+            click.echo(f"{lyr.name} [{','.join(lyr.product_names)}]")
+            if styles:
+                click.echo(f"{lyr} {depth}")
+
+
+def print_styles(lyr: OWSNamedLayer, depth: int = 0) -> None:
+    for styl in lyr.styles:
+        indent(0, for_styles=True)
+        print(f". {styl.name}")
+
+
+def indent(depth: int, for_styles: bool = False) -> None:
+    for i in range(depth):
+        click.echo("  ", nl=False)
+    if for_styles:
+        click.echo("      ", nl=False)