hestia-earth-models 0.61.6__py3-none-any.whl → 0.61.8__py3-none-any.whl

hestia_earth/models/utils/array_builders.py

@@ -0,0 +1,578 @@
+"""
+Based on code by Cool Farm Tool:
+https://gitlab.com/MethodsCFT/coolfarm-soc/-/blob/main/src/cfasoc/builders.py
+"""
+import hashlib
+from numpy import array, concatenate, cumsum, full, hstack, random, mean, prod, vstack
+from numpy.typing import NDArray, DTypeLike
+from typing import Union
+
+from .descriptive_stats import calc_z_critical
+
+
+def repeat_single(shape: tuple, value: float, dtype: DTypeLike = None) -> NDArray:
+    """
+    Repeat a single value to form an array of a defined shape.
+
+    Parameters
+    ----------
+    shape : tuple
+        Shape (rows, columns).
+    value : float
+        Value to be repeated.
+    dtype : DTypeLike, optional
+        The desired data-type for the array.
+
+    Returns
+    -------
+    NDArray
+        Array with repeated value.
+    """
+    return full(shape=shape, fill_value=value, dtype=dtype)
+
+
+def repeat_array_as_columns(n_iterations: int, array: NDArray) -> NDArray:
+    """
+    Repeat a numpy array horizontally as columns.
+
+    Parameters
+    ----------
+    n_iterations : int
+        Number of times the columns should be repeated.
+    array : NDArray
+        Array to repeat.
+
+    Returns
+    -------
+    NDArray
+        Repeated array.
+    """
+    return hstack([array for _ in range(n_iterations)])
+
+
+def repeat_array_as_rows(n_iterations: int, array: NDArray) -> NDArray:
+    """
+    Repeat a numpy array vertically as rows.
+
+    Parameters
+    ----------
+    n_iterations : int
+        Number of times the rows should be repeated.
+    array : NDArray
+        Array to repeat.
+
+    Returns
+    -------
+    NDArray
+        Repeated array.
+    """
+    return vstack([array for _ in range(n_iterations)])
+
+
+def repeat_1d_array_as_columns(n_columns: int, column: NDArray) -> NDArray:
+    """
+    Repeat a column (NDArray) to form an array of a defined shape
+
+    Parameters
+    ----------
+    n_columns : int
+        How many times the column (NDArray) should be repeated.
+    column : NDArray
+        The column (NDArray) to be repeated.
+
+    Returns
+    -------
+    NDArray
+        Repeated array.
+    """
+    return vstack([column for _ in range(n_columns)]).transpose()
+
+
+def discrete_uniform_1d(
+    shape: tuple, low: float, high: float, seed: Union[int, random.Generator, None] = None
+) -> NDArray:
+    """
+    Sample from a discrete uniform distribution and produce an array of a specified shape.
+    All rows in a specified column will have the same sample value, but each column will be different (1 dimensional
+    variability).
+
+    Parameters
+    ----------
+    shape : tuple
+        Shape (rows, columns).
+    low : float
+        Lower bound of the discrete uniform distribution to be sampled.
+    high : float
+        Upper bound of the discrete uniform distribution to be sampled.
+    seed : int | Generator | None, optional
+        A seed to initialize the BitGenerator. If passed a Generator, it will be returned unaltered. If `None`, then
+        fresh, unpredictable entropy will be pulled from the OS.
+
+    Returns
+    -------
+    NDArray
+        Array of samples with 1 dimensional variability.
+    """
+    n_rows, n_columns = shape
+    rng = random.default_rng(seed)
+    return repeat_array_as_rows(
+        n_rows,
+        rng.uniform(low=low, high=high, size=n_columns)
+    )
+
+
+def discrete_uniform_2d(
+    shape: tuple, low: float, high: float, seed: Union[int, random.Generator, None] = None
+) -> NDArray:
+    """
+    Sample from a discrete uniform distribution and produce an array of a specified shape.
+    All rows and columns contain different sample values (2 dimensional variability).
+
+    Parameters
+    ----------
+    shape : tuple
+        Shape (rows, columns).
+    low : float
+        Lower bound of the discrete uniform distribution to be sampled.
+    high : float
+        Upper bound of the discrete uniform distribution to be sampled.
+    seed : int | Generator | None, optional
+        A seed to initialize the BitGenerator. If passed a Generator, it will be returned unaltered. If `None`, then
+        fresh, unpredictable entropy will be pulled from the OS.
+
+    Returns
+    -------
+    NDArray
+        Array of samples with 2 dimensional variability.
+    """
+    rng = random.default_rng(seed)
+    return rng.uniform(low=low, high=high, size=shape)
+
+
+def triangular_1d(
+    shape: tuple, low: float, high: float, mode: float, seed: Union[int, random.Generator, None] = None
+) -> NDArray:
+    """
+    Sample from a triangular distribution and produce an array of a specified shape.
+    All rows in a specified column will have the same sample value, but each column will be different (1 dimensional
+    variability).
+
+    Parameters
+    ----------
+    shape : tuple
+        Shape (rows, columns).
+    low : float
+        Lower bound of the triangular distribution to be sampled.
+    high : float
+        Upper bound of the triangular distribution to be sampled.
+    mode : float
+        Mode of the triangular distribution to be sampled.
+    seed : int | Generator | None, optional
+        A seed to initialize the BitGenerator. If passed a Generator, it will be returned unaltered. If `None`, then
+        fresh, unpredictable entropy will be pulled from the OS.
+
+    Returns
+    -------
+    NDArray
+        Array of samples with 1 dimensional variability.
+    """
+    n_rows, n_columns = shape
+    rng = random.default_rng(seed)
+    return repeat_array_as_rows(
+        n_rows,
+        rng.triangular(left=low, mode=mode, right=high, size=n_columns)
+    )
+
+
+def triangular_2d(
+    shape: tuple, low: float, high: float, mode: float, seed: Union[int, random.Generator, None] = None
+) -> NDArray:
+    """
+    Sample from a triangular distribution and produce an array of a specified shape.
+    All rows and columns contain different sample values (2 dimensional variability).
+
+    Parameters
+    ----------
+    shape : tuple
+        Shape (rows, columns).
+    low : float
+        Lower bound of the triangular distribution to be sampled.
+    high : float
+        Upper bound of the triangular distribution to be sampled.
+    mode : float
+        Mode of the triangular distribution to be sampled.
+    seed : int | Generator | None, optional
+        A seed to initialize the BitGenerator. If passed a Generator, it will be returned unaltered. If `None`, then
+        fresh, unpredictable entropy will be pulled from the OS.
+
+    Returns
+    -------
+    NDArray
+        Array of samples with 2 dimensional variability.
+    """
+    rng = random.default_rng(seed)
+    return rng.triangular(left=low, mode=mode, right=high, size=shape)
+
+
+def normal_1d(
+    shape: tuple, mu: float, sigma: float, seed: Union[int, random.Generator, None] = None
+) -> NDArray:
+    """
+    Sample from a normal distribution and produce an array of a specified shape.
+    All rows in a specified column will have the same sample value, but each column will be different (1 dimensional
+    variability).
+
+    Parameters
+    ----------
+    shape : tuple
+        Shape (rows, columns).
+    mu : float
+        Mean of the normal distribution to be sampled.
+    sigma : float
+        Standard deviation of the normal distribution to be sampled.
+    seed : int | Generator | None, optional
+        A seed to initialize the BitGenerator. If passed a Generator, it will be returned unaltered. If `None`, then
+        fresh, unpredictable entropy will be pulled from the OS.
+
+    Returns
+    -------
+    NDArray
+        Array of samples with 1 dimensional variability.
+    """
+    n_rows, n_columns = shape
+    rng = random.default_rng(seed)
+    return repeat_array_as_rows(
+        n_rows,
+        rng.normal(loc=mu, scale=sigma, size=n_columns)
+    )
+
+
+def normal_2d(
+    shape: tuple, mu: float, sigma: float, seed: Union[int, random.Generator, None] = None
+) -> NDArray:
+    """
+    Sample from a normal distribution and produce an array of a specified shape.
+    All rows and columns contain different sample values (2 dimensional variability).
+
+    Parameters
+    ----------
+    shape : tuple
+        Shape (rows, columns).
+    mu : float
+        Mean of the normal distribution to be sampled.
+    sigma : float
+        Standard deviation of the normal distribution to be sampled.
+    seed : int | Generator | None, optional
+        A seed to initialize the BitGenerator. If passed a Generator, it will be returned unaltered. If `None`, then
+        fresh, unpredictable entropy will be pulled from the OS.
+
+    Returns
+    -------
+    NDArray
+        Array of samples with 2 dimensional variability.
+    """
+    rng = random.default_rng(seed)
+    return rng.normal(loc=mu, scale=sigma, size=shape)
+
+
+def truncated_normal_1d(
+    shape: tuple, mu: float, sigma: float, low: float, high: float, seed: Union[int, random.Generator, None] = None
+) -> NDArray:
+    """
+    Sample from a truncated normal distribution and produce an array of a specified shape.
+    All rows in a specified column will have the same sample value, but each column will be different (1 dimensional
+    variability).
+
+    Parameters
+    ----------
+    shape : tuple
+        Shape (rows, columns).
+    mu : float
+        Mean of the normal distribution to be sampled.
+    sigma : float
+        Standard deviation of the normal distribution to be sampled.
+    low : float
+        Lower bound of the normal distribution to be sampled.
+    high : float
+        Upper bound of the normal distribution to be sampled.
+    seed : int | Generator | None, optional
+        A seed to initialize the BitGenerator. If passed a Generator, it will be returned unaltered. If `None`, then
+        fresh, unpredictable entropy will be pulled from the OS.
+
+    Returns
+    -------
+    NDArray
+        Array of samples with 1 dimensional variability.
+    """
+    n_rows, n_columns = shape
+    return repeat_array_as_rows(
+        n_rows,
+        _truncnorm_rvs(a=low, b=high, loc=mu, scale=sigma, shape=n_columns, seed=seed)
+    )
+
+
+def truncated_normal_2d(
+    shape: tuple, mu: float, sigma: float, low: float, high: float, seed: Union[int, random.Generator, None] = None
+) -> NDArray:
+    """
+    Sample from a truncated normal distribution and produce an array of a specified shape.
+    All rows and columns contain different sample values (2 dimensional variability).
+
+    Parameters
+    ----------
+    shape : tuple
+        Shape (rows, columns).
+    mu : float
+        Mean of the normal distribution to be sampled.
+    sigma : float
+        Standard deviation of the normal distribution to be sampled.
+    low : float
+        Lower bound of the normal distribution to be sampled.
+    high : float
+        Upper bound of the normal distribution to be sampled.
+    seed : int | Generator | None, optional
+        A seed to initialize the BitGenerator. If passed a Generator, it will be returned unaltered. If `None`, then
+        fresh, unpredictable entropy will be pulled from the OS.
+
+    Returns
+    -------
+    NDArray
+        Array of samples with 2 dimensional variability.
+    """
+    return _truncnorm_rvs(a=low, b=high, loc=mu, scale=sigma, shape=shape, seed=seed)
+
+
+def _truncnorm_rvs(
+    a: float,
+    b: float,
+    loc: float,
+    scale: float,
+    shape: Union[int, tuple[int, ...]],
+    seed: Union[int, random.Generator, None] = None
+) -> NDArray:
+    """
+    Generate random samples from a truncated normal distribution. Unlike the `scipy` equivalent, the `a` and `b` values
+    are the abscissae at which we wish to truncate the distribution (as opposed to the number of standard deviations
+    from `loc`).
+
+    Parameters
+    ----------
+    loc : float
+        Mean ("centre") of the distribution.
+    scale : float
+        Standard deviation (spread or "width") of the distribution. Must be non-negative.
+    size : int | tuple[int, ...]
+        Output shape. If the given shape is, e.g., (m, n, k), then m * n * k samples are drawn.
+    seed : int | Generator | None, optional
+        A seed to initialize the BitGenerator. If passed a Generator, it will be returned unaltered. If `None`, then
+        fresh, unpredictable entropy will be pulled from the OS.
+
+    Returns
+    -------
+    NDArray
+        Array of samples.
+    """
+    size = prod(shape)
+    samples = array([])
+    rng = random.default_rng(seed)
+
+    while samples.size < size:
+        samples_temp = rng.normal(loc, scale, (size - samples.size) * 2)
+        valid_samples = samples_temp[(a <= samples_temp) & (samples_temp <= b)]
+        samples = concatenate([samples, valid_samples])
+
+    return samples[:size].reshape(shape)
+
+
+def plus_minus_uncertainty_to_normal_1d(
+    shape: tuple,
+    value: float,
+    uncertainty: float,
+    confidence_interval: float = 95,
+    seed: Union[int, random.Generator, None] = None
+) -> NDArray:
+    """
+    Return a normally distributed sample given a value and uncertainty expressed as +/- a percentage.
+
+    All rows in a specified column will have the same sample value, but each column will be different (1 dimensional
+    variability).
+
+    This function has been written to serve Table 5.5b on Page 5.32, Tier 2 Steady State Method for Mineral Soils,
+    Chapter 5 Cropland, 2019 Refinement to the 2006 IPCC Guidelines for National Greenhouse Gas Inventories. Table 5.5b
+    notes:
+
+        "Uncertainty is assumed to be ±75% for the N content estimates and ±50% for the lignin content estimates,
+        expressed as a 95% confidence intervals."
+
+    This function also serves Table 11.2 on Page 11.19, Tier 2 Steady State Method for Mineral Soils, Chapter 11 N2O
+    Emissions from Managed Soils, and CO2 Emissions from Lime and Urea Application, 2019 Refinement to the 2006 IPCC
+    Guidelines for National Greenhouse Gas Inventories.
+
+    Parameters
+    ----------
+    shape : tuple
+        Shape (rows, columns).
+    value : float
+        Reported value.
+    uncertainty : float
+        Uncertainty expressed as +/- a percentage.
+    confidence_interval : float
+        Confidence interval the uncertainty represents.
+    seed : int | Generator | None, optional
+        A seed to initialize the BitGenerator. If passed a Generator, it will be returned unaltered. If `None`, then
+        fresh, unpredictable entropy will be pulled from the OS.
+
+    Returns
+    -------
+    NDArray
+        Array of samples with 1 dimensional variability.
+    """
+    n_rows, n_columns = shape
+    n_sds = calc_z_critical(confidence_interval)
+    sigma = (value * (uncertainty / 100)) / n_sds
+    return repeat_array_as_rows(
+        n_rows,
+        normal_1d(shape=(1, n_columns), mu=value, sigma=sigma, seed=seed)
+    )
+
+
+def plus_minus_uncertainty_to_normal_2d(
+    shape: tuple,
+    value: float,
+    uncertainty: float,
+    confidence_interval: float = 95,
+    seed: Union[int, random.Generator, None] = None
+) -> NDArray:
+    """
+    Return a normally distributed sample given a value and uncertainty expressed as +/- a percentage.
+
+    All rows and columns contain different sample values (2 dimensional variability).
+
+    This function has been written to serve Table 5.5b on Page 5.32, Tier 2 Steady State Method for Mineral Soils,
+    Chapter 5 Cropland, 2019 Refinement to the 2006 IPCC Guidelines for National Greenhouse Gas Inventories. Table 5.5b
+    notes:
+
+        "Uncertainty is assumed to be ±75% for the N content estimates and ±50% for the lignin content estimates,
+        expressed as a 95% confidence intervals."
+
+    This function also serves Table 11.2 on Page 11.19, Tier 2 Steady State Method for Mineral Soils, Chapter 11 N2O
+    Emissions from Managed Soils, and CO2 Emissions from Lime and Urea Application, 2019 Refinement to the 2006 IPCC
+    Guidelines for National Greenhouse Gas Inventories.
+
+    Parameters
+    ----------
+    shape : tuple
+        Shape (rows, columns).
+    value : float
+        Reported value.
+    uncertainty : float
+        Uncertainty expressed as +/- a percentage.
+    confidence_interval : float
+        Confidence interval the uncertainty represents.
+    seed : int | Generator | None, optional
+        A seed to initialize the BitGenerator. If passed a Generator, it will be returned unaltered. If `None`, then
+        fresh, unpredictable entropy will be pulled from the OS.
+
+    Returns
+    -------
+    NDArray
+        Array of samples with 2 dimensional variability.
+    """
+    n_sds = calc_z_critical(confidence_interval)
+    sigma = (value * (uncertainty / 100)) / n_sds
+    return normal_2d(shape=shape, mu=value, sigma=sigma, seed=seed)
+
+
+def grouped_avg(arr: NDArray, n: int = 12) -> NDArray:
+    """ Row-wise averaging of numpy arrays. For example:
+    1   2   3
+    4   5   6
+    7   8   9
+    10  11  12
+    13  14  15
+    16  17  18
+
+    if n = 6, becomes:
+    8.5 9.5 10.5
+
+    because:
+    (1 + 4 + 7 + 10 + 13 + 16) / 6 = 8.5
+    (2 + 5 + 8 + 11 + 14 + 17) / 6 = 9.5
+    etc.
+
+    if n = 2, becomes:
+    2.5  3.5  4.5
+    8.5  9.5  10.5
+    14.5 15.5 16.5
+
+    because:
+    (in column 0) (1 + 4) / 2 = 2.5, (7 + 10) / 2 = 8.5, (13 + 16) / 2 = 14.5
+    (in column 1) (2 + 5) / 2 = 3.5, (8 + 11) / 2 = 9.5, (14 + 17) / 2 = 15.5
+
+    Source: https://stackoverflow.com/questions/30379311/fast-way-to-take-average-of-every-n-rows-in-a-npy-array
+
+    Parameters
+    ----------
+    arr : NDArray
+        Input array.
+    n : int, optional
+        Number of rows to average. Defaults to 12.
+
+    Returns
+    -------
+    NDArray
+        Output array
+    """
+    result = cumsum(arr, 0)[n-1::n] / float(n)
+    result[1:] = result[1:] - result[:-1]
+    return result
+
+
+def avg_run_in_columnwise(arr: NDArray, n: int):
+    """
+    Reduce the first `n` elements of each column in an array by averaging them, while leaving the rest of the array
+    modified.
+
+    Parameters
+    ----------
+    arr : NDArray
+        Input array.
+    n : int
+        The number of run-in elements to average.
+
+    Returns
+    -------
+    NDArray
+        The new array where the first element in each column is an average of the run in elements.
+    """
+    run_in: NDArray = mean(arr[:n], 0)
+    return vstack([run_in, arr[n:]])
+
+
+def avg_run_in_rowwise(arr: NDArray, n: int):
+    """
+    Reduce the first `n` elements of each row in an array by averaging them, while leaving the rest of the array
+    modified.
+
+    Parameters
+    ----------
+    arr : NDArray
+        Input array.
+    n : int
+        The number of run-in elements to average.
+
+    Returns
+    -------
+    NDArray
+        The new array where the first element in each row is an average of the run in elements.
+    """
+    return avg_run_in_columnwise(arr.transpose(), n).transpose()
+
+
+def gen_seed(node: dict) -> int:
+    """
+    Generate a seed based on a node's `@id` so that rng is the same each time the model is re-run.
+    """
+    node_id = node.get("@id", "")
+    hashed = hashlib.shake_128(node_id.encode(), usedforsecurity=False).hexdigest(4)
+    return abs(int(hashed, 16))

hestia_earth/models/utils/blank_node.py

@@ -26,7 +26,7 @@ from hestia_earth.utils.tools import (
 )
 
 from ..log import debugValues, log_as_table
-from . import is_from_model, _filter_list_term_unit
+from . import is_from_model, _filter_list_term_unit, is_iterable
 from .constant import Units
 from .property import get_node_property, get_node_property_value
 from .lookup import (
@@ -463,8 +463,7 @@ def _convert_to_set(
     set
         A set containing the elements of the input variable.
     """
-    is_iterable = isinstance(variable, Iterable) and not isinstance(variable, (str, bytes))
-    return set(variable) if is_iterable else {variable}
+    return set(variable) if is_iterable(variable) else {variable}
 
 
 def node_term_match(

hestia_earth/models/utils/crop.py

@@ -1,8 +1,9 @@
-from hestia_earth.schema import TermTermType
+from hestia_earth.schema import TermTermType, SiteSiteType
 from hestia_earth.utils.model import find_primary_product
 from hestia_earth.utils.tools import safe_parse_float
 
 from .term import get_lookup_value
+from .site import valid_site_type as site_valid_site_type
 
 FAO_LOOKUP_COLUMN = 'cropGroupingFAO'
 FAOSTAT_AREA_LOOKUP_COLUMN = 'cropGroupingFaostatArea'
@@ -34,3 +35,25 @@ def get_N2ON_fertiliser_coeff_from_primary_product(model: str, log_id: str, cycl
 
 def is_plantation(model: str, log_id: str, term_id: str):
     return get_crop_lookup_value(model, log_id, term_id, 'isPlantation')
+
+
+def valid_site_type(cycle: dict, include_permanent_pasture=False):
+    """
+    Check if the `site.siteType` of the cycle is `cropland`.
+
+    Parameters
+    ----------
+    cycle : dict
+        The `Cycle`.
+    include_permanent_pasture : bool
+        If set to `True`, `permanent pasture` is also allowed. Defaults to `False`.
+
+    Returns
+    -------
+    bool
+        `True` if `siteType` matches the allowed values, `False` otherwise.
+    """
+    site_types = [SiteSiteType.CROPLAND.value] + (
+        [SiteSiteType.PERMANENT_PASTURE.value] if include_permanent_pasture else []
+    )
+    return site_valid_site_type(cycle.get('site', {}), site_types)

hestia_earth/models/utils/cycle.py

@@ -9,7 +9,6 @@ from .property import get_node_property
 from .completeness import _is_term_type_complete
 from .blank_node import get_N_total, get_P2O5_total
 from .measurement import most_relevant_measurement_value
-from .site import valid_site_type as site_valid_site_type
 from .crop import is_plantation
 from .currency import DEFAULT_CURRENCY
 from .inorganicFertiliser import get_cycle_inputs as get_inorganicFertiliser_inputs
@@ -330,28 +329,6 @@ def land_occupation_per_kg(model: str, term_id: str, cycle: dict, site: dict, pr
     )
 
 
-def valid_site_type(cycle: dict, include_permanent_pasture=False):
-    """
-    Check if the `site.siteType` of the cycle is `cropland`.
-
-    Parameters
-    ----------
-    cycle : dict
-        The `Cycle`.
-    include_permanent_pasture : bool
-        If set to `True`, `permanent pasture` is also allowed. Defaults to `False`.
-
-    Returns
-    -------
-    bool
-        `True` if `siteType` matches the allowed values, `False` otherwise.
-    """
-    site_types = [SiteSiteType.CROPLAND.value] + (
-        [SiteSiteType.PERMANENT_PASTURE.value] if include_permanent_pasture else []
-    )
-    return site_valid_site_type(cycle.get('site', {}), site_types)
-
-
 def is_organic(cycle: dict):
     """
     Check if the `Cycle` is organic, i.e. if it contains an organic standard label `Practice`.