hestia-earth-models 0.62.0__py3-none-any.whl → 0.62.2__py3-none-any.whl

hestia_earth/models/utils/blank_node.py

@@ -1,11 +1,13 @@
+import calendar
 from calendar import monthrange
 from collections import defaultdict
 from collections.abc import Iterable
-from datetime import datetime
+from datetime import datetime, timedelta
+from uuid import uuid4
+
 from dateutil.relativedelta import relativedelta
 from enum import Enum
 from functools import reduce
-from statistics import mode, mean
 from typing import (
     Any,
     List,
@@ -24,9 +26,10 @@ from hestia_earth.utils.tools import (
     safe_parse_float,
     non_empty_list
 )
+from hestia_earth.utils.blank_node import ArrayTreatment, get_node_value
 
 from ..log import debugValues, log_as_table
-from . import is_from_model, _filter_list_term_unit, is_iterable
+from . import is_from_model, _filter_list_term_unit, is_iterable, _omit
 from .constant import Units
 from .property import get_node_property, get_node_property_value
 from .lookup import (
@@ -38,6 +41,44 @@ from .lookup import (
 from .term import get_lookup_value
 
 
+def lookups_logs(model: str, blank_nodes: list, lookups_per_termType: dict, **log_args):
+    def mapper(blank_node: dict):
+        term = blank_node.get('term', {})
+        term_id = term.get('@id')
+        term_type = term.get('termType')
+        lookups = lookups_per_termType.get(term_type, [])
+        lookups = lookups if isinstance(lookups, list) else [lookups]
+
+        def _reduce_lookups_logs(logs: dict, column: str):
+            lookup_value = get_lookup_value(term, column, model=model, **log_args)
+            return logs | {column: lookup_value}
+
+        return reduce(_reduce_lookups_logs, lookups, {'id': term_id})
+
+    logs = list(map(mapper, blank_nodes))
+
+    return log_as_table(logs)
+
+
+def properties_logs(blank_nodes: list, properties: Union[dict, list]):
+    def mapper(blank_node: dict):
+        term = blank_node.get('term', {})
+        term_id = term.get('@id')
+        term_type = term.get('termType')
+        props = properties.get(term_type, []) if isinstance(properties, dict) else properties
+        props = props if isinstance(props, list) else [props]
+
+        def _reduce_properties_logs(logs: dict, prop: str):
+            value = get_node_property(term, prop).get('value')
+            return logs | {prop: value}
+
+        return reduce(_reduce_properties_logs, properties, {'id': term_id})
+
+    logs = list(map(mapper, blank_nodes))
+
+    return log_as_table(logs)
+
+
 def group_by_keys(group_keys: list = ['term']):
     def run(group: dict, node: dict):
         group_key = '-'.join(non_empty_list(map(lambda v: node.get(v, {}).get('@id'), group_keys)))
@@ -46,7 +87,9 @@ def group_by_keys(group_keys: list = ['term']):
     return run
 
 
-def _module_term_id(term_id: str, module): return getattr(module, 'TERM_ID', term_id).split(',')[0]
+def _module_term_id(term_id: str, module):
+    term_id_str = term_id.split('.')[-1] if '.' in term_id else term_id
+    return getattr(module, 'TERM_ID', term_id_str).split(',')[0]
 
 
 def _run_model_required(model: str, term: dict, data: dict):
@@ -342,111 +385,6 @@ def convert_to_carbon(node: dict, model: str, term_id: str, blank_nodes: list, *
     ]) if len(missing_carbon_property) == 0 else None
 
 
-class ArrayTreatment(Enum):
-    """
-    Enum representing different treatments for arrays of values.
-    """
-    MEAN = 'mean'
-    MODE = 'mode'
-    SUM = 'sum'
-    FIRST = 'first'
-    LAST = 'last'
-
-
-def _should_run_array_treatment(value):
-    return isinstance(value, Iterable) and len(value) > 0
-
-
-DEFAULT_ARRAY_TREATMENT = ArrayTreatment.MEAN
-ARRAY_TREATMENT_TO_REDUCER = {
-    ArrayTreatment.MEAN: lambda value: mean(value) if _should_run_array_treatment(value) else 0,
-    ArrayTreatment.MODE: lambda value: mode(value) if _should_run_array_treatment(value) else 0,
-    ArrayTreatment.SUM: lambda value: sum(value) if _should_run_array_treatment(value) else 0,
-    ArrayTreatment.FIRST: lambda value: value[0] if _should_run_array_treatment(value) else 0,
-    ArrayTreatment.LAST: lambda value: value[-1] if _should_run_array_treatment(value) else 0
-}
-"""
-A dictionary mapping ArrayTreatment enums to corresponding reducer functions.
-"""
-
-
-def _retrieve_array_treatment(
-    node: dict,
-    is_larger_unit: bool = False,
-    default: ArrayTreatment = ArrayTreatment.FIRST
-) -> ArrayTreatment:
-    """
-    Retrieves the array treatment for a given node.
-
-    Array treatments are used to reduce an array's list of values into
-    a single value. The array treatment is retrieved from a lookup on
-    the node's term.
-
-    Parameters
-    ----------
-    node : dict
-        The dictionary representing the node.
-    is_larger_unit : bool, optional
-        Flag indicating whether to use the larger unit lookup, by default `False`.
-    default : ArrayTreatment, optional
-        Default value to return if the lookup fails, by default `ArrayTreatment.FIRST`.
-
-    Returns
-    -------
-    ArrayTreatment
-        The retrieved array treatment.
-
-    """
-    ARRAY_TREATMENT_LOOKUPS = [
-        'arrayTreatmentLargerUnitOfTime',
-        'arrayTreatment'
-    ]
-    lookup = ARRAY_TREATMENT_LOOKUPS[0] if is_larger_unit else ARRAY_TREATMENT_LOOKUPS[1]
-
-    term = node.get('term', {})
-    lookup_value = get_lookup_value(term, lookup, skip_debug=True)
-
-    return next(
-        (treatment for treatment in ArrayTreatment if treatment.value == lookup_value),
-        default
-    )
-
-
-def get_node_value(
-    node: dict,
-    is_larger_unit: bool = False,
-    array_treatment: Optional[ArrayTreatment] = None,
-    default: Any = 0
-) -> Union[float, bool]:
-    """
-    Get the value from the dictionary representing the node,
-    applying optional array treatment if the value is a list.
-
-    Parameters
-    ----------
-    node : dict
-        The dictionary representing the node.
-    is_larger_unit : bool, optional
-        A flag indicating whether the unit of time is larger, by default `False`.
-    array_treatment : ArrayTreatment, optional
-        An optional override for the treatment to be applied to an array value, if `None` the array treatment in the
-        node's term's lookup is used (which defaults to `FIRST` if no array treatment is specified), by default `None`.
-
-    Returns
-    -------
-    float | bool
-        The extracted value from the node.
-
-    """
-    value = node.get("value", 0)
-
-    reducer = ARRAY_TREATMENT_TO_REDUCER[(
-        array_treatment or _retrieve_array_treatment(node, is_larger_unit=is_larger_unit)
-    )] if isinstance(value, list) and len(value) > 0 else None
-
-    return reducer(value) if reducer else value if isinstance(value, bool) else value or default
-
-
 def _convert_to_set(
     variable: Union[Iterable[Any], Any]
 ) -> set:
@@ -560,7 +498,7 @@ def cumulative_nodes_match(
     """
     values = [
         get_node_value(
-            node, is_larger_unit, array_treatment
+            node, 'value', is_larger_unit, array_treatment
         ) or default_node_value for node in nodes if function(node)
     ]
 
@@ -1274,3 +1212,99 @@ def get_inputs_from_properties(input: dict, term_types: Union[TermTermType, List
         } for p in (properties or []) if all([p.get('key'), p.get('value')])
     ]) if input_value > 0 else []
     return filter_list_term_type(inputs, term_types)
+
+
+def _get_condensed_nodes(nodes: list) -> tuple[list, bool]:
+    """Only considers nodes which already match on non-date criteria."""
+    CONDENSABLE_UNITS = [Units.BOOLEAN.value, Units.PERCENTAGE_AREA.value]
+    condensed_nodes = []
+    matched_uuids = set()
+    nodes_by_start_date = {_full_date_str(date_str=n["startDate"], is_end=True): n for n in nodes if "startDate" in n}
+    if len(nodes_by_start_date) != len(nodes):
+        return nodes, False
+
+    for node in nodes:
+        search_date = _offset_date(date_str=node.get("endDate", ""))
+        if node["uuid"] in matched_uuids:
+            continue
+        if (search_date in nodes_by_start_date and nodes_by_start_date[search_date]["uuid"] not in matched_uuids
+                and node.get("term", {}).get("units") in CONDENSABLE_UNITS):
+            new_node = node.copy()
+            new_node["endDate"] = nodes_by_start_date[search_date]["endDate"]
+            condensed_nodes.append(new_node)
+            matched_uuids.add(nodes_by_start_date[search_date]["uuid"])
+        elif node["uuid"] not in matched_uuids:
+            condensed_nodes.append(node)
+
+    return condensed_nodes, len(matched_uuids) > 0
+
+
+def condense_nodes(nodes: list) -> list:
+    grouped_nodes = _group_nodes_by_term_and_value(nodes)
+    condensed_nodes = dict()
+    any_changes_made = False
+
+    for key, node_group in grouped_nodes.items():
+        condensed_nodes[key] = node_group
+        while len(condensed_nodes[key]) > 1:
+            condensed_nodes[key], changes_made = _get_condensed_nodes(condensed_nodes[key])
+            if not changes_made:
+                break
+            any_changes_made = True
+
+    if not any_changes_made:
+        return [_omit(values=n, keys=["uuid"]) for n in nodes]
+
+    return sorted(
+        flatten([_omit(values=n, keys=["uuid"]) for nodes in condensed_nodes.values() for n in nodes]),
+        key=lambda x: x["startDate"]
+    )
+
+
+DATE_FORMAT = "%Y-%m-%d"
+
+
+def _variable_length_str_to_date(date_str: str, is_end: bool) -> datetime:
+    """Converts to date, adding start or end of year to YYYY strings as indicated by is_end."""
+    return datetime.strptime(_full_date_str(date_str, is_end=is_end), DATE_FORMAT)
+
+
+def _full_date_str(date_str: str, is_end: bool) -> str:
+    suffix = ""
+    if len(date_str) == 4:
+        # Format YYYY
+        suffix = "-12-31" if is_end else "-01-01"
+    elif len(date_str) == 7:
+        # Format YYYY-MM
+        suffix = f"-{calendar.monthrange(int(date_str[:4]), int(date_str[5:7]))[1]}" if is_end else "-01"
+
+    return date_str + suffix
+
+
+def _with_full_dates(node: dict) -> dict:
+    output_node = node.copy()
+    if "startDate" in output_node:
+        output_node["startDate"] = _full_date_str(output_node["startDate"], is_end=False)
+    if "endDate" in output_node:
+        output_node["endDate"] = _full_date_str(output_node["endDate"], is_end=True)
+
+    return output_node
+
+
+def _offset_date(date_str: str, days: int = 1, is_end: bool = True) -> str:
+    return (
+        _variable_length_str_to_date(date_str=date_str, is_end=is_end) + timedelta(days=days)
+    ).strftime(DATE_FORMAT)
+
+
+def _group_nodes_by_term_and_value(nodes: list) -> dict:
+    grouped_nodes = defaultdict(list)
+
+    for node in nodes:
+        term_id = node.get("term", {}).get("@id", "")
+        value = "-".join([str(v) for v in node.get("value")]) if isinstance(node.get("value"), list) \
+            else node.get("value")
+        node["uuid"] = uuid4()
+        grouped_nodes[(term_id, value)].append(_with_full_dates(node))
+
+    return grouped_nodes

hestia_earth/models/utils/constant.py

@@ -3,6 +3,7 @@ from hestia_earth.utils.tools import list_sum
 
 
 class Units(Enum):
+    BOOLEAN = 'boolean'
     HEAD = 'head'
     NUMBER = 'number'
     KG = 'kg'
@@ -31,6 +32,7 @@ class Units(Enum):
     KG_COLD_CARCASS_WEIGHT = 'kg cold carcass weight'
     KG_COLD_DRESSED_CARCASS_WEIGHT = 'kg cold dressed carcass weight'
     KG_READY_TO_COOK_WEIGHT = 'kg ready-to-cook weight'
+    PERCENTAGE_AREA = '% area'
     TO_C = '-C'
     TO_N = '-N'
 

hestia_earth/models/utils/lookup.py

@@ -76,6 +76,17 @@ _ALLOW_ALL = 'all'
 def _is_site(site: dict): return site.get('@type', site.get('type')) == SchemaType.SITE.value
 
 
+def _get_sites(node: dict):
+    site = node.get('site', node.get('cycle', {}).get('site'))
+    other_sites = node.get('otherSites', node.get('cycle', {}).get('otherSites', []))
+    return non_empty_list([site] + other_sites)
+
+
+def _get_site_types(node: dict):
+    sites = [node] if _is_site(node) else _get_sites(node)
+    return non_empty_list([site.get('siteType') for site in sites])
+
+
 def _model_lookup_values(model: str, term: dict, restriction: str):
     lookup = download_lookup(f"{term.get('termType')}-model-{restriction}.csv")
     values = get_table_value(lookup, 'termid', term.get('@id'), column_name(model))
@@ -83,10 +94,11 @@ def _model_lookup_values(model: str, term: dict, restriction: str):
 
 
 def is_model_siteType_allowed(model: str, term: dict, data: dict):
-    site = data if _is_site(data) else data.get('site', data.get('cycle', {}).get('site')) or {}
-    site_type = site.get('siteType')
+    site_types = _get_site_types(data)
     allowed_values = _model_lookup_values(model, term, 'siteTypesAllowed')
-    return True if _ALLOW_ALL in allowed_values or not site_type else site_type in allowed_values
+    return True if _ALLOW_ALL in allowed_values or not site_types else any([
+        (site_type in allowed_values) for site_type in site_types
+    ])
 
 
 def _lookup_values(term: dict, column: str):
@@ -96,10 +108,11 @@ def _lookup_values(term: dict, column: str):
 
 
 def is_siteType_allowed(data: dict, term: dict):
-    site = data if _is_site(data) else data.get('site', data.get('cycle', {}).get('site')) or {}
-    site_type = site.get('siteType')
+    site_types = _get_site_types(data)
     allowed_values = _lookup_values(term, 'siteTypesAllowed')
-    return True if _ALLOW_ALL in allowed_values or not site_type else site_type in allowed_values
+    return True if _ALLOW_ALL in allowed_values or not site_types else any([
+        (site_type in allowed_values) for site_type in site_types
+    ])
 
 
 def is_product_termType_allowed(data: dict, term: dict):

hestia_earth/models/utils/property.py

@@ -87,17 +87,17 @@ def node_has_property(term_id: str):
     return lambda product: find_term_match(product.get('properties', []), term_id, None) is not None
 
 
-def node_property_lookup_value(model: str, term: dict, prop_id: str, default=None, **log_args):
+def node_property_lookup_value(model: str, node_term: dict, prop_id: str, default=None, **log_args):
     # as the lookup table might not exist, we are making sure we return `0` in thise case
     try:
-        lookup_name = f"{term.get('termType')}-property.csv"
+        lookup_name = f"{node_term.get('termType')}-property.csv"
         lookup = download_lookup(lookup_name)
-        term_id = term.get('@id')
+        term_id = node_term.get('@id')
         lookup_value = get_table_value(lookup, 'termid', term_id, column_name(prop_id))
         value = extract_grouped_data(lookup_value, 'Avg') if (
             isinstance(lookup_value, str) and 'Avg' in lookup_value
         ) else lookup_value
-        debugMissingLookup(lookup_name, 'termid', term_id, prop_id, value, model=model, term=term_id, **log_args)
+        debugMissingLookup(lookup_name, 'termid', term_id, prop_id, value, model=model, **log_args)
         return safe_parse_float(value, default=None)
     except Exception:
         return default
@@ -111,9 +111,9 @@ def get_node_property_value(model: str, node: dict, prop_id: str, default=None,
     return default if value is None else (value / 100 if units == '%' else value)
 
 
-def get_node_property_value_converted(model: str, node: dict, prop_id: str, default=None):
+def get_node_property_value_converted(model: str, node: dict, prop_id: str, default=None, **log_args):
     node_value = list_sum(node.get('value', []))
-    prop_value = get_node_property_value(model, node, prop_id)
+    prop_value = get_node_property_value(model, node, prop_id, **log_args)
     return default if prop_value is None else node_value * prop_value
 
 

hestia_earth/models/utils/site.py

@@ -5,6 +5,7 @@ from hestia_earth.utils.tools import non_empty_list, flatten, safe_parse_date
 
 from hestia_earth.models.log import debugMissingLookup
 from . import cached_value, _load_calculated_node
+from .term import get_land_cover_siteTypes
 
 CACHE_YEARS_KEY = 'years'
 WATER_TYPES = [
@@ -120,3 +121,9 @@ def region_factor(model: str, region_id: str, term_id: str, termType: TermTermTy
     value = get_table_value(download_lookup(lookup_name), 'termid', region_id, column_name(term_id))
     debugMissingLookup(lookup_name, 'termid', region_id, term_id, value, model=model, term=term_id)
     return value
+
+
+def get_land_cover_term_id(site_type: str):
+    land_cover_terms = get_land_cover_siteTypes()
+    term = next((term for term in land_cover_terms if term["name"].lower() == site_type.lower()), {})
+    return term.get('@id')

hestia_earth/models/utils/source.py

@@ -18,7 +18,7 @@ def _find_source(biblio_title: str = None):
 
 
 def get_source(node: dict, biblio_title: str = None):
-    source = cached_value(node, CACHE_SOURCES_KEY, {}).get(biblio_title, _find_source(biblio_title))
+    source = cached_value(node, CACHE_SOURCES_KEY, {}).get(biblio_title) or _find_source(biblio_title)
     return {'source': source} if source else {}
 
 

hestia_earth/models/utils/term.py

@@ -1,4 +1,4 @@
-from hestia_earth.schema import SchemaType, TermTermType
+from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
 from hestia_earth.utils.api import find_node, search
 
@@ -625,3 +625,23 @@ def get_electricity_grid_mix_terms():
             ],
         }
     }, limit=LIMIT, fields=['@type', '@id', 'name', 'termType', 'units'])
+
+
+def get_land_cover_siteTypes():
+    """
+    Find all `Land Cover` terms with siteTypes
+
+    Returns
+    -------
+        List of landCover terms with associated siteTypes.
+    """
+    return search({
+        "bool": {
+            "must": [
+                {"match": {"@type": "Term"}},
+                {"match": {"termType": "landCover"}}
+            ],
+            "should": [{"match": {"name": siteType.value}} for siteType in SiteSiteType],
+            "minimum_should_match": 1
+        },
+    })

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.62.0'
+VERSION = '0.62.2'

{hestia_earth_models-0.62.0.dist-info → hestia_earth_models-0.62.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.62.0
+Version: 0.62.2
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hestia-earth.schema ==29.*
-Requires-Dist: hestia-earth.utils >=0.13.0
+Requires-Dist: hestia-earth.utils >=0.13.2
 Requires-Dist: python-dateutil >=2.8.1
 Requires-Dist: CurrencyConverter ==0.16.8
 Requires-Dist: haversine >=2.7.0