hestia-earth-models 0.64.14__py3-none-any.whl → 0.65.1__py3-none-any.whl

hestia_earth/models/site/management.py

@@ -20,10 +20,11 @@ from hestia_earth.utils.tools import safe_parse_float, flatten
 from hestia_earth.utils.blank_node import get_node_value
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
-from hestia_earth.models.utils import _include
+from hestia_earth.models.utils import _include, _omit
 from hestia_earth.models.utils.management import _new_management
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.blank_node import condense_nodes
+from hestia_earth.models.utils.crop import get_landCover_term_id
 from hestia_earth.models.utils.site import (
     related_cycles, get_land_cover_term_id as get_landCover_term_id_from_site_type
 )
@@ -39,8 +40,7 @@ REQUIREMENTS = {
                 "products": [
                     {
                         "@type": "Product",
-                        "term.termType": ["crop", "forage", "landCover"],
-                        "units": ["% area", "boolean"]
+                        "term.termType": ["crop", "forage", "landCover"]
                     }
                 ],
                 "practices": [
@@ -52,7 +52,6 @@ REQUIREMENTS = {
                             "landUseManagement",
                             "system"
                         ],
-                        "units": ["% area", "boolean"],
                         "value": ""
                     }
                 ],
@@ -87,11 +86,11 @@ LOOKUPS = {
     "inorganicFertiliser": "nitrogenContent",
     "organicFertiliser": "ANIMAL_MANURE",
     "soilAmendment": "PRACTICE_INCREASING_C_INPUT",
-    "landUseManagement": "GAP_FILL_TO_MANAGEMENT"
+    "landUseManagement": "GAP_FILL_TO_MANAGEMENT",
+    "property": "GAP_FILL_TO_MANAGEMENT"
 }
 MODEL_KEY = 'management'
 
-_LAND_COVER_KEY = LOOKUPS['crop'][0]
 _ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
 _INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID = "inorganicNitrogenFertiliserUsed"
 _ORGANIC_FERTILISER_USED_TERM_ID = "organicFertiliserUsed"
@@ -140,6 +139,16 @@ def management(data: dict):
     return node
 
 
+def _should_gap_fill(term: dict):
+    value = get_lookup_value(lookup_term=term, column='GAP_FILL_TO_MANAGEMENT')
+    return bool(value)
+
+
+def _filter_properties(blank_node: dict):
+    properties = list(filter(lambda p: _should_gap_fill(p.get('term', {})), blank_node.get('properties', [])))
+    return _omit(blank_node, ['properties']) | ({'properties': properties} if properties else {})
+
+
 def _map_to_value(value: dict):
     return {
         'id': value.get('term', {}).get('@id'),
@@ -154,37 +163,23 @@ def _extract_node_value(node: dict) -> dict:
     return node | {'value': get_node_value(node)}
 
 
-def _default_dates(cycle: dict, values: list):
-    return [(_include(cycle, ["startDate", "endDate"]) | v) for v in values]
-
-
-def _dates_from_current_cycle(cycle: dict, values: list) -> list:
-    """Always uses the dates from the cycle."""
-    return [v | _include(cycle, ["startDate", "endDate"]) for v in values]
-
-
 def _copy_item_if_exists(source: dict, keys: list[str] = None, dest: dict = None) -> dict:
-    keys = keys or []
-    dest = dest or {}
-    return reduce(lambda p, c: p | ({c: source[c]} if c in source else {}), keys, dest)
+    return reduce(lambda p, c: p | ({c: source[c]} if source.get(c) else {}), keys or [], dest or {})
 
 
 def _get_landCover_term_id(product: dict) -> str:
     term = product.get('term', {})
-    value = get_lookup_value(term, _LAND_COVER_KEY, model=MODEL, term=term.get('@id'), model_key=MODEL_KEY)
-    return value.split(';')[0] if value else None
+    return get_landCover_term_id(term, model=MODEL, term=term.get('@id'), model_key=MODEL_KEY)
 
 
-def _get_relevant_items(
-    cycle: dict, item_name: str, relevant_terms: list, date_fill: callable = _default_dates
-):
+def _get_relevant_items(cycle: dict, item_name: str, relevant_terms: list):
     """
     Get items from the list of cycles with any of the relevant terms.
-    Also adds dates if missing.
+    Also adds dates from Cycle.
     """
     return [
-        item
-        for item in date_fill(cycle=cycle, values=filter_list_term_type(cycle.get(item_name, []), relevant_terms))
+        _include(cycle, ["startDate", "endDate"]) | item
+        for item in filter_list_term_type(cycle.get(item_name, []), relevant_terms)
     ]
 
 
@@ -226,54 +221,74 @@ def _run_from_siteType(site: dict, cycle: dict):
     }] if should_run else []
 
 
-def _run_from_landCover(cycle: dict):
-    products = _get_relevant_items(
-        cycle=cycle,
-        item_name="products",
-        relevant_terms=[TermTermType.LANDCOVER]
-    )
+def _run_products(cycle: dict, products: list, total_products: int = None, use_cycle_dates: bool = False):
+    default_dates = _include(cycle, ["startDate", "endDate"])
+    return [
+        _map_to_value(default_dates | _copy_item_if_exists(
+            source=product,
+            keys=['properties', 'startDate', 'endDate'],
+            dest={
+                "term": {'@id': _get_landCover_term_id(product)},
+                "value": round(100 / (total_products or len(products)), 2)
+            }
+        ) | (
+            default_dates if use_cycle_dates else {}
+        ))
+        for product in products
+    ]
+
+
+def _run_from_landCover(cycle: dict, crop_forage_products: list):
+    """
+    Copy landCover items, and include crop/forage landCover items with properties to count in ratio.
+    """
     products = [
         _map_to_value(_extract_node_value(
             _include(
                 value=product,
                 keys=["term", "value", "startDate", "endDate", "properties"]
             )
-        )) for product in products
+        )) for product in _get_relevant_items(
+            cycle=cycle,
+            item_name="products",
+            relevant_terms=[TermTermType.LANDCOVER]
+        )
     ]
-    return products
+    return products + _run_products(
+        cycle,
+        crop_forage_products,
+        total_products=len(crop_forage_products) + len(products),
+        use_cycle_dates=True
+    )
 
 
 def _run_from_crop_forage(cycle: dict):
     products = _get_relevant_items(
         cycle=cycle,
         item_name="products",
-        relevant_terms=[TermTermType.CROP, TermTermType.FORAGE],
-        date_fill=_dates_from_current_cycle
+        relevant_terms=[TermTermType.CROP, TermTermType.FORAGE]
     )
+    # only take products with a matching landCover term
     products = list(filter(_get_landCover_term_id, products))
-    products = [
-        _map_to_value(_copy_item_if_exists(
-            source=product,
-            keys=["startDate", "endDate", "properties"],
-            dest={
-                "term": {'@id': _get_landCover_term_id(product)},
-                "value": round(100 / len(products), 2)
-            }
-        ))
-        for product in products
-    ]
-    return products
+    # remove any properties that should not get gap-filled
+    products = list(map(_filter_properties, products))
 
+    # split products with properties and those without
+    products_with_gap_filled_props = [p for p in products if p.get('properties')]
+    products_without_gap_filled_props = [p for p in products if not p.get('properties')]
 
-def _has_gap_fill_to_management_set(practice: dict):
+    return _run_from_landCover(
+        cycle=cycle,
+        crop_forage_products=products_with_gap_filled_props
+    ) + _run_products(cycle, products_without_gap_filled_props, use_cycle_dates=False)
+
+
+def _should_run_practice(practice: dict):
     """
     Include only landUseManagement practices where GAP_FILL_TO_MANAGEMENT = True
     """
     term = practice.get('term', {})
-    return (
-        term.get('termType') != TermTermType.LANDUSEMANAGEMENT.value or
-        get_lookup_value(lookup_term=term, column=LOOKUPS["landUseManagement"])
-    )
+    return term.get('termType') != TermTermType.LANDUSEMANAGEMENT.value or _should_gap_fill(term)
 
 
 def _run_from_practices(cycle: dict):
@@ -295,13 +310,13 @@ def _run_from_practices(cycle: dict):
             ]
         )
     ]
-    practices = list(map(_map_to_value, filter(_has_gap_fill_to_management_set, practices)))
+    practices = list(map(_map_to_value, filter(_should_run_practice, practices)))
     return practices
 
 
 def _run_cycle(site: dict, cycle: dict):
     inputs = _run_from_inputs(site, cycle)
-    products = _run_from_landCover(cycle) + _run_from_crop_forage(cycle)
+    products = _run_from_crop_forage(cycle)
     site_types = _run_from_siteType(site, cycle)
     practices = _run_from_practices(cycle)
     return [

hestia_earth/models/site/pre_checks/cache_sources.py

@@ -3,9 +3,7 @@ Pre Checks Cache Sources
 
 This model caches the sources of all Site models.
 """
-from hestia_earth.models.log import debugValues
-from hestia_earth.models.utils import CACHE_KEY, cached_value
-from hestia_earth.models.utils.source import CACHE_SOURCES_KEY, find_sources
+from hestia_earth.models.utils.cache_sources import cache_sources
 
 REQUIREMENTS = {
     "Site": {}
@@ -15,20 +13,4 @@ RETURNS = {
 }
 
 
-def _run(site: dict):
-    sources = find_sources()
-    debugValues(site, sources=';'.join([str(title) for title in sources.keys()]))
-    return sources
-
-
-def _should_run(site: dict):
-    has_cache = cached_value(site, CACHE_SOURCES_KEY)
-    return not bool(has_cache)
-
-
-def run(site: dict):
-    should_run = _should_run(site)
-    return {
-        **site,
-        CACHE_KEY: cached_value(site) | {CACHE_SOURCES_KEY: _run(site)}
-    } if should_run else site
+def run(site: dict): return cache_sources(site)

hestia_earth/models/utils/__init__.py

@@ -7,7 +7,7 @@ import sys
 import datetime
 from functools import reduce
 import operator
-from typing import Any, Union
+from typing import Any, Callable, Union
 from hestia_earth.schema import SchemaType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import linked_node
@@ -198,3 +198,14 @@ def hectar_to_square_meter(value): return value * 10000
 
 
 def square_meter_to_hectare(value): return value / 10000
+
+
+def split_on_condition(iterable: Iterable, condition: Callable):
+    """
+    Split an iterable into two iterables of the same type based on whether or not each item satisfies a condition.
+    """
+    def construct(iterator: Generator) -> Iterable:
+        return ''.join(iterable) if isinstance(iterable, str) else type(iterable)(iterator)
+
+    cond_true, cond_false = tee((condition(item), item) for item in iterable)
+    return construct(i for p, i in cond_true if p), construct(i for p, i in cond_false if not p)

hestia_earth/models/utils/aggregated.py

@@ -107,10 +107,10 @@ def should_link_input_to_impact(cycle: dict):
     def should_run(input: dict):
         term = input.get('term', {})
         return all([
-            not input.get('impactAssessment'),
             _should_aggregate_input(term),
             # make sure Input is not a Product as well or we might double-count emissions
             find_term_match(cycle.get('products', []), term.get('@id'), None) is None,
+            not input.get('impactAssessment'),
             # ignore inputs which are flagged as Product of the Cycle
             not input.get('fromCycle', False),
             not input.get('producedInCycle', False)

hestia_earth/models/utils/blank_node.py

@@ -377,7 +377,7 @@ def get_P2O5_total(nodes: list) -> list:
     return get_total_value(kg_P_nodes) + get_total_value_converted(kg_N_nodes + kg_nodes, 'phosphateContentAsP2O5')
 
 
-def convert_to_nitrogen(node: dict, model: str, term_id: str, blank_nodes: list, **log_args):
+def convert_to_nitrogen(node: dict, model: str, blank_nodes: list, **log_args):
     def prop_value(input: dict):
         value = get_node_property_value(model, input, 'nitrogenContent', default=None, **log_args)
         return value or get_node_property_value(model, input, 'crudeProteinContent', default=0, **log_args) / 6.25
@@ -385,7 +385,7 @@ def convert_to_nitrogen(node: dict, model: str, term_id: str, blank_nodes: list,
     values = [(i, prop_value(i)) for i in blank_nodes]
     missing_nitrogen_property = [i.get('term', {}).get('@id') for i, p_value in values if not p_value]
 
-    debugValues(node, model=model, term=term_id,
+    debugValues(node, model=model,
                 missing_nitrogen_property=';'.join(set(missing_nitrogen_property)),
                 **log_args)
 
@@ -394,7 +394,7 @@ def convert_to_nitrogen(node: dict, model: str, term_id: str, blank_nodes: list,
     ]) if len(missing_nitrogen_property) == 0 else None
 
 
-def convert_to_carbon(node: dict, model: str, term_id: str, blank_nodes: list, **log_args):
+def convert_to_carbon(node: dict, model: str, blank_nodes: list, **log_args):
     def prop_value(input: dict):
         value = get_node_property_value(model, input, 'carbonContent', default=None, **log_args)
         return value or \
@@ -403,7 +403,7 @@ def convert_to_carbon(node: dict, model: str, term_id: str, blank_nodes: list, *
     values = [(i, prop_value(i)) for i in blank_nodes]
     missing_carbon_property = [i.get('term', {}).get('@id') for i, p_value in values if not p_value]
 
-    debugValues(node, model=model, term=term_id,
+    debugValues(node, model=model,
                 missing_carbon_property=';'.join(missing_carbon_property),
                 **log_args)
 
@@ -1248,6 +1248,12 @@ def get_inputs_from_properties(input: dict, term_types: Union[TermTermType, List
     return filter_list_term_type(inputs, term_types)
 
 
+def _has_unique_key(nodes: list, key: str): return len(set([n.get(key) for n in nodes])) == 1
+
+
+def _same_dates(nodes: list): return all([_has_unique_key(nodes, 'startDate'), _has_unique_key(nodes, 'endDate')])
+
+
 def _should_group_node(node: dict): return node.get('startDate') and node.get('endDate')
 
 
@@ -1281,19 +1287,21 @@ def _group_nodes_by_consecutive_dates(nodes: list):
     if group:
         groups.append(group)
 
-    return groups
+    # if all nodes have a single startDate and endDate, then they are all the same
+    return [nodes] if _same_dates(nodes) else groups
+
+
+def _sum_nodes_value(nodes: list):
+    values = flatten([n.get('value', []) for n in nodes])
+    is_boolean = all([isinstance(v, bool) for v in values])
+    return values[0] if is_boolean else [list_sum(values)] if isinstance(nodes[0]['value'], list) else list_sum(values)
 
 
 def _node_from_group(nodes: list):
     # `nodes` contain list with consecutive dates
-    # if all nodes have the same dates, sum up the values
-    same_startDate = len(set([n.get('startDate') for n in nodes])) == 1
-    same_endDate = len(set([n.get('endDate') for n in nodes])) == 1
-    total_value = list_sum(flatten([n.get('value', []) for n in nodes]))
     return nodes[0] if len(nodes) == 1 else (
-        nodes[0] | {
-            'value': [total_value] if isinstance(nodes[0]['value'], list) else total_value
-        } if all([same_startDate, same_endDate])
+        # if all nodes have the same dates, sum up the values
+        nodes[0] | {'value': _sum_nodes_value(nodes)} if _same_dates(nodes)
         else nodes[0] | {
             'startDate': min(n.get('startDate') for n in nodes),
             'endDate': max(n.get('endDate') for n in nodes)

hestia_earth/models/utils/cache_sources.py

@@ -0,0 +1,15 @@
+from . import CACHE_KEY, cached_value
+from .source import CACHE_SOURCES_KEY, find_sources
+
+
+def _run(): return {CACHE_SOURCES_KEY: find_sources()}
+
+
+def _has_value(node: dict): return bool(cached_value(node, CACHE_SOURCES_KEY))
+
+
+def cache_sources(node: dict):
+    return {
+        **node,
+        CACHE_KEY: cached_value(node) | _run()
+    } if not _has_value(node) else node

hestia_earth/models/utils/crop.py

@@ -61,3 +61,8 @@ def valid_site_type(cycle: dict, include_permanent_pasture=False):
         [SiteSiteType.PERMANENT_PASTURE.value] if include_permanent_pasture else []
     )
     return site_valid_site_type(cycle.get('site', {}), site_types)
+
+
+def get_landCover_term_id(lookup_term: dict, **log_args) -> str:
+    value = get_lookup_value(lookup_term, 'landCoverTermId', **log_args)
+    return value.split(';')[0] if value else None

hestia_earth/models/utils/indicator.py

@@ -5,9 +5,11 @@ from hestia_earth.utils.model import linked_node
 from . import _term_id, _include_methodModel
 
 
-def _new_indicator(term, model=None, land_cover_id=None):
+def _new_indicator(term, model=None, land_cover_id: str = None, previous_land_cover_id: str = None):
     node = {'@type': SchemaType.INDICATOR.value}
     node['term'] = linked_node(term if isinstance(term, dict) else download_hestia(_term_id(term)))
     if land_cover_id:
         node['landCover'] = linked_node(download_hestia(land_cover_id))
+    if previous_land_cover_id:
+        node['previousLandCover'] = linked_node(download_hestia(previous_land_cover_id))
     return _include_methodModel(node, model)

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.64.14'
+VERSION = '0.65.1'

{hestia_earth_models-0.64.14.dist-info → hestia_earth_models-0.65.1.dist-info}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019-2024 Harmonised Environmental Storage and Tracking of the Impacts of Agriculture (HESTIA) Project
+Copyright (c) 2019-2025 Harmonised Environmental Storage and Tracking of the Impacts of Agriculture (HESTIA) Project
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{hestia_earth_models-0.64.14.dist-info → hestia_earth_models-0.65.1.dist-info}/METADATA

@@ -1,18 +1,18 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.64.14
+Version: 0.65.1
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team
 Author-email: guillaumeroyer.mail@gmail.com
-License: GPL
+License: MIT
 Platform: UNKNOWN
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Programming Language :: Python :: 3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hestia-earth-schema==30.*
-Requires-Dist: hestia-earth-utils>=0.13.11
+Requires-Dist: hestia-earth-utils>=0.13.12
 Requires-Dist: python-dateutil>=2.8.1
 Requires-Dist: CurrencyConverter==0.16.8
 Requires-Dist: haversine>=2.7.0