hestia-earth-models 0.64.4__py3-none-any.whl → 0.64.5__py3-none-any.whl

hestia_earth/models/utils/descriptive_stats.py

@@ -1,245 +1,10 @@
 from collections.abc import Iterable
 from enum import Enum
-from functools import reduce
-from numpy import abs, around, exp, float64, inf, pi, sign, sqrt
+from numpy import around
 from numpy.typing import NDArray
 from typing import Optional, Union
 
 
-def calc_z_critical(
-    confidence_interval: float,
-    n_sided: int = 2
-) -> float64:
-    """
-    Calculate the z-critical value from the confidence interval.
-
-    Parameters
-    ----------
-    confidence_interval : float
-        The confidence interval as a percentage between 0 and 100%.
-    n_sided : int, optional
-        The number of tails (default value = `2`).
-
-    Returns
-    -------
-    float64
-        The z-critical value as a floating point between 0 and infinity.
-    """
-    alpha = 1 - confidence_interval / 100
-    return _normal_ppf(1 - alpha / n_sided)
-
-
-def _normal_ppf(q: float64, tol: float64 = 1e-10) -> float64:
-    """
-    Calculates the percent point function (PPF), also known as the inverse cumulative distribution function (CDF), of a
-    standard normal distribution using the Newton-Raphson method.
-
-    Parameters
-    ----------
-    q : float64
-        The quantile at which to evaluate the PPF.
-    tol : float64, optional
-        The tolerance for the Newton-Raphson method. Defaults to 1e-10.
-
-    Returns
-    -------
-    float64
-        The PPF value at the given quantile.
-    """
-    INITIAL_GUESS = 0
-    MAX_ITER = 100
-
-    def step(x):
-        """Perform one step of the Newton-Raphson method."""
-        x_new = x - (_normal_cdf(x) - q) / _normal_pdf(x)
-        return x_new if abs(x_new - x) >= tol else x
-
-    return (
-        inf if q == 1 else
-        -inf if q == 0 else
-        reduce(lambda x, _: step(x), range(MAX_ITER), INITIAL_GUESS)
-    )
-
-
-def _normal_cdf(x: float64) -> float64:
-    """
-    Calculates the cumulative distribution function (CDF) of a standard normal distribution for a single value using a
-    custom error function (erf).
-
-    Parameters
-    ----------
-    x : float64
-        The point at which to evaluate the CDF.
-
-    Returns
-    -------
-    float64
-        The CDF value at the given point.
-    """
-    return 0.5 * (1 + _erf(x / sqrt(2)))
-
-
-def _erf(x: float64) -> float64:
-    """
-    Approximates the error function of a standard normal distribution using a numerical approximation based on
-    Abramowitz and Stegun formula 7.1.26.
-
-    Parameters
-    ----------
-    x : float64
-        The input value.
-
-    Returns
-    -------
-    float64
-        The approximated value of the error function.
-    """
-    # constants
-    A_1 = 0.254829592
-    A_2 = -0.284496736
-    A_3 = 1.421413741
-    A_4 = -1.453152027
-    A_5 = 1.061405429
-    P = 0.3275911
-
-    # Save the sign of x
-    sign_ = sign(x)
-    x_ = abs(x)
-
-    # A&S formula 7.1.26
-    t = 1.0 / (1.0 + P * x_)
-    y = 1.0 - (((((A_5 * t + A_4) * t) + A_3) * t + A_2) * t + A_1) * t * exp(-x_ * x_)
-
-    return sign_ * y
-
-
-def _normal_pdf(x: float64) -> float64:
-    """
-    Calculates the probability density function (PDF) of a standard normal distribution for a single value.
-
-    Parameters
-    ----------
-    x : float64
-        The point at which to evaluate the PDF.
-
-    Returns
-    -------
-    float64
-        The PDF value at the given point.
-    """
-    return 1 / sqrt(2 * pi) * exp(-0.5 * x**2)
-
-
-def _calc_confidence_level(
-    z_critical: float64,
-    n_sided: int = 2
-) -> float64:
-    """
-    Calculate the confidence interval from the z-critical value.
-
-    Parameters
-    ----------
-    z_critical_value : np.float64
-        The confidence interval as a floating point number between 0 and infinity.
-    n_sided : int, optional
-        The number of tails (default value = `2`).
-
-    Returns
-    -------
-    np.float64
-        The confidence interval as a percentage between 0 and 100%.
-    """
-    alpha = (1 - _normal_cdf(z_critical)) * n_sided
-    return (1 - alpha) * 100
-
-
-def calc_required_iterations_monte_carlo(
-    confidence_level: float,
-    precision: float,
-    sd: float
-) -> int:
-    """
-    Calculate the number of iterations required for a Monte Carlo simulation to have a desired precision, subject to a
-    given confidence level.
-
-    Parameters
-    ----------
-    confidence_level : float
-        The confidence level, as a percentage out of 100, that the precision should be subject too (i.e., we are x%
-        sure that the sample mean deviates from the true populatation mean by less than the desired precision).
-    precision : float
-        The desired precision as a floating point value (i.e., if the Monte Carlo simulation will be used to estimate
-        `organicCarbonPerHa` to a precision of 100 kg C ha-1 this value should be 100).
-    sd : float
-        The standard deviation of the sample. This can be estimated by running the model 500 times (a number that does
-        not take too much time to run but is large enough for the sample standard deviation to converge reasonably
-        well).
-
-    Returns
-    -------
-    int
-        The required number of iterations.
-    """
-    z_critical_value = calc_z_critical(confidence_level)
-    return round(((sd * z_critical_value) / precision) ** 2)
-
-
-def calc_confidence_level_monte_carlo(
-    n_iterations: int,
-    precision: float,
-    sd: float
-) -> float:
-    """
-    Calculate the confidence level that the sample mean calculated by the Monte Carlo simulation deviates from the
-    true population mean by less than the desired precision.
-
-    Parameters
-    ----------
-    n_iterations : int
-        The number of iterations that the Monte Carlo simulation was run for.
-    precision : float
-        The desired precision as a floating point value (i.e., if the Monte Carlo simulation will be used to estimate
-        `organicCarbonPerHa` to a precision of 100 kg C ha-1 this value should be 100).
-    sd : float
-        The standard deviation of the sample.
-
-    Returns
-    -------
-    float
-        The confidence level, as a percentage out of 100, that the precision should be subject too (i.e., we are x%
-        sure that the sample mean deviates from the true populatation mean by less than the desired precision).
-    """
-    return _calc_confidence_level(precision*sqrt(n_iterations)/sd)
-
-
-def calc_precision_monte_carlo(
-    confidence_level: float,
-    n_iterations: int,
-    sd: float
-) -> float:
-    """
-    Calculate the +/- precision of a Monte Carlo simulation for a desired confidence level.
-
-    Parameters
-    ----------
-    confidence_level : float
-        The confidence level, as a percentage out of 100, that the precision should be subject too (i.e., we are x%
-        sure that the sample mean deviates from the true populatation mean by less than the desired precision).
-    n_iterations : int
-        The number of iterations that the Monte Carlo simulation was run for.
-    sd : float
-        The standard deviation of the sample.
-
-    Returns
-    -------
-    float
-        The precision of the sample mean estimated by the Monte Carlo model as a floating point value with the same
-        units as the estimated mean.
-    """
-    z_critical = calc_z_critical(confidence_level)
-    return (sd*z_critical)/sqrt(n_iterations)
-
-
 def calc_descriptive_stats(
     arr: NDArray,
     stats_definition: Union[Enum, str],
@@ -268,10 +33,9 @@ def calc_descriptive_stats(
     min_ = around(arr.min(axis=axis), decimals)
     max_ = around(arr.max(axis=axis), decimals)
 
-    rows, columns = arr.shape
     observations = (
-        [rows] * columns if axis == 0
-        else [columns] * rows if axis == 1
+        [arr.shape[0]] * arr.shape[1] if axis == 0
+        else [arr.shape[1]] * arr.shape[0] if axis == 1
         else [arr.size]
     )
 

hestia_earth/models/utils/feedipedia.py

@@ -4,6 +4,7 @@ from hestia_earth.utils.tools import non_empty_list, safe_parse_float
 
 from hestia_earth.models.log import logShouldRun
 from .property import _new_property
+from .blank_node import merge_blank_nodes
 
 DRY_MATTER_TERM_ID = 'dryMatter'
 
@@ -17,6 +18,12 @@ def get_feedipedia_properties():
     return term_ids
 
 
+def _should_rescale_by_dm(property_id: str):
+    lookup = download_lookup('property.csv')
+    value = get_table_value(lookup, 'termid', property_id, column_name('feedipediaConversionEnum'))
+    return 'dm' in value
+
+
 def _dm_property(term_id: str, property_values: dict, dm_property_values: dict, dry_matter_property: dict):
     blank_node = _new_property(term_id)
     blank_node_data = {}
@@ -25,7 +32,11 @@ def _dm_property(term_id: str, property_values: dict, dm_property_values: dict,
         old_dm_value = safe_parse_float(dm_property_values.get(property_key))
         old_property_value = safe_parse_float(property_values.get(property_key))
         if all([new_dm_value, old_dm_value, old_property_value]):
-            blank_node_data[property_key] = round(old_property_value / old_dm_value * new_dm_value, 2)
+            new_value = round(
+                old_property_value / old_dm_value * new_dm_value,
+                2
+            ) if _should_rescale_by_dm(term_id) else old_property_value
+            blank_node_data[property_key] = new_value
     return (blank_node | blank_node_data) if blank_node_data else None
 
 
@@ -68,6 +79,8 @@ def rescale_properties_from_dryMatter(model: str, node: dict, blank_nodes: list)
         ])
         for prop in new_properties:
             logShouldRun(node, model, term_id, True, property=prop.get('term', {}).get('@id'))
-        return {**blank_node, 'properties': all_properties + new_properties} if new_properties else blank_node
+        return (
+            blank_node | {'properties': merge_blank_nodes(all_properties, new_properties)}
+        ) if new_properties else blank_node
 
     return list(map(exec, blank_nodes))

hestia_earth/models/utils/landCover.py

@@ -0,0 +1,9 @@
+from hestia_earth.schema import TermTermType
+
+from .term import get_lookup_value
+
+
+def get_pef_grouping(term_id: str):
+    term = {'@id': term_id, 'termType': TermTermType.LANDCOVER.value}
+    grouping = get_lookup_value(term, column='pefTermGrouping')
+    return f"{grouping}" if grouping else None

hestia_earth/models/utils/lookup.py

@@ -1,5 +1,5 @@
-from typing import Optional
-
+from typing import Optional, List
+from numpy import recarray
 from hestia_earth.schema import SchemaType
 from hestia_earth.utils.lookup import (
     download_lookup, get_table_value, column_name, extract_grouped_data, _get_single_table_value
@@ -155,3 +155,14 @@ def is_input_id_allowed(data: dict, term: dict):
         _ALLOW_ALL in allowed_values,
         len(values) == 0
     ]) else any([value in allowed_values for value in values])
+
+
+def fallback_country(country_id: str, lookup_arrays: List[recarray]) -> str:
+    """
+    Given a site dict with 'country_id' location term, and lookup table,
+    checks if a location can be used in lookup file
+    else fallback to the default "region-world"
+    """
+    is_in_lookup = lambda v: all(v in array['termid'] for array in lookup_arrays)  # noqa: E731
+    fallback_id = 'region-world'
+    return country_id if is_in_lookup(country_id) else fallback_id if is_in_lookup(fallback_id) else None

hestia_earth/models/utils/measurement.py

@@ -1,9 +1,8 @@
 from collections import defaultdict
 from collections.abc import Iterable
 from functools import reduce
-from dateutil import parser
 from statistics import mode, mean
-from typing import Any, Optional,  Union
+from typing import Any, Optional, Union
 
 from hestia_earth.schema import MeasurementMethodClassification, SchemaType
 from hestia_earth.utils.api import download_hestia
@@ -11,12 +10,12 @@ from hestia_earth.utils.model import linked_node
 from hestia_earth.utils.tools import non_empty_list, flatten, safe_parse_float
 from hestia_earth.utils.date import diff_in_days
 
+from hestia_earth.models.utils.blank_node import most_relevant_blank_node_by_id
 from . import _term_id, _include_method, flatten_args
 from .term import get_lookup_value
 
 
 # TODO: verify those values
-MAX_DEPTH = 1000
 OLDEST_DATE = '1800'
 SOIL_TEXTURE_IDS = ['sandContent', 'siltContent', 'clayContent']
 MEASUREMENT_REDUCE = {
@@ -42,32 +41,8 @@ def measurement_value(measurement: dict, is_larger_unit: bool = False) -> float:
     return MEASUREMENT_REDUCE.get(reducer, lambda v: v[0])(value) if is_value_valid else 0
 
 
-def _measurement_date(measurement: dict): return parser.isoparse(measurement.get('endDate', OLDEST_DATE))
-
-
-def _distance(measurement: dict, date): return abs((_measurement_date(measurement) - date).days)
-
-
-def _most_recent_measurements(measurements: list, date: str) -> list:
-    closest_date = parser.isoparse(date)
-    min_distance = min([_distance(m, closest_date) for m in measurements])
-    return list(filter(lambda m: _distance(m, closest_date) == min_distance, measurements))
-
-
-def _shallowest_measurement(measurements: list) -> dict:
-    min_depth = min([m.get('depthUpper', MAX_DEPTH) for m in measurements])
-    return next((m for m in measurements if m.get('depthUpper', MAX_DEPTH) == min_depth), {})
-
-
-def most_relevant_measurement(measurements: list, term_id: str, date: str):
-    filtered_measurements = [m for m in measurements if m.get('term', {}).get('@id') == term_id]
-    return {} if len(filtered_measurements) == 0 \
-        else _shallowest_measurement(_most_recent_measurements(filtered_measurements, date)) \
-        if date and len(filtered_measurements) > 1 else filtered_measurements[0]
-
-
 def most_relevant_measurement_value(measurements: list, term_id: str, date: str, default=None):
-    measurement = most_relevant_measurement(measurements, term_id, date)
+    measurement = most_relevant_blank_node_by_id(measurements, term_id, date)
     return measurement_value(measurement) if measurement else default