hestia-earth-models 0.70.2__tar.gz → 0.70.3__tar.gz

{hestia_earth_models-0.70.2 → hestia_earth_models-0.70.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia_earth_models
-Version: 0.70.2
+Version: 0.70.3
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team

{hestia_earth_models-0.70.2 → hestia_earth_models-0.70.3}/hestia_earth/models/config/Cycle.json

@@ -182,6 +182,14 @@
       "mergeStrategy": "list",
       "stage": 1
     },
+    {
+      "key": "practices",
+      "model": "cycle",
+      "value": "practice.landCover",
+      "runStrategy": "always",
+      "mergeStrategy": "list",
+      "stage": 1
+    },
     {
       "key": "products",
       "model": "hestia",

hestia_earth_models-0.70.3/hestia_earth/models/cycle/practice/landCover.py

@@ -0,0 +1,181 @@
+from functools import reduce
+from hestia_earth.schema import TermTermType, SiteSiteType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import non_empty_list, flatten, list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import _omit, _include
+from hestia_earth.models.utils.practice import _new_practice
+from hestia_earth.models.utils.term import get_lookup_value
+from hestia_earth.models.utils.blank_node import condense_nodes
+from hestia_earth.models.utils.crop import get_landCover_term_id
+from .. import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "endDate": "",
+        "products": [
+            {
+                "@type": "Product",
+                "term.termType": ["crop", "forage"],
+                "optional": {
+                    "startDate": "",
+                    "endDate": ""
+                }
+            }
+        ],
+        "site": {
+            "@type": "Site",
+            "siteType": "cropland"
+        },
+        "none": {
+            "practices": [{"@type": "Practice", "term.termType": "landCover"}]
+        },
+        "optional": {
+            "startDate": ""
+        }
+    }
+}
+RETURNS = {
+    "Practice": [{
+        "term.termType": "landCover",
+        "value": "",
+        "endDate": "",
+        "startDate": ""
+    }]
+}
+LOOKUPS = {
+    "crop": ["landCoverTermId", "maximumCycleDuration"],
+    "forage": ["landCoverTermId"],
+    "property": ["GAP_FILL_TO_MANAGEMENT", "CALCULATE_TOTAL_LAND_COVER_SHARE_SEPARATELY"]
+}
+MODEL_KEY = 'landCover'
+
+
+def practice(data: dict):
+    node = _new_practice(data.get('id'))
+    node['value'] = [data['value']]
+    node['endDate'] = data['endDate']
+    if data.get('startDate'):
+        node['startDate'] = data['startDate']
+    if data.get('properties'):
+        node['properties'] = data['properties']
+    return node
+
+
+def _should_gap_fill(term: dict):
+    value = get_lookup_value(lookup_term=term, column='GAP_FILL_TO_MANAGEMENT')
+    return bool(value)
+
+
+def _filter_properties(blank_node: dict):
+    properties = list(filter(lambda p: _should_gap_fill(p.get('term', {})), blank_node.get('properties', [])))
+    return _omit(blank_node, ['properties']) | ({'properties': properties} if properties else {})
+
+
+def _map_to_value(value: dict):
+    return {
+        'id': value.get('term', {}).get('@id'),
+        'value': value.get('value'),
+        'startDate': value.get('startDate'),
+        'endDate': value.get('endDate'),
+        'properties': value.get('properties')
+    }
+
+
+def _copy_item_if_exists(source: dict, keys: list[str] = None, dest: dict = None) -> dict:
+    return reduce(lambda p, c: p | ({c: source[c]} if source.get(c) else {}), keys or [], dest or {})
+
+
+def _run(cycle: dict, products: list, total: float):
+    # remove any properties that should not get gap-filled
+    products = list(map(_filter_properties, products))
+
+    nodes = [
+        _map_to_value(_include(cycle, ["startDate", "endDate"]) | _copy_item_if_exists(
+            source=product,
+            keys=['properties', 'startDate', 'endDate'],
+            dest={
+                "term": {'@id': product.get('land-cover-id')},
+                "value": round((100 - total) / len(products), 2)
+            }
+        ))
+        for product in products
+    ]
+
+    return condense_nodes(list(map(practice, nodes)))
+
+
+def _should_group_landCover(term: dict):
+    value = get_lookup_value(lookup_term=term, column='CALCULATE_TOTAL_LAND_COVER_SHARE_SEPARATELY')
+    return bool(value)
+
+
+def _has_prop_grouped_with_landCover(product: dict):
+    return bool(
+        next((
+            p
+            for p in product.get('properties', [])
+            if _should_group_landCover(p.get('term', {}))
+        ), None)
+    )
+
+
+def _product_wit_landCover_id(product: dict):
+    landCover_id = get_landCover_term_id(product.get('term', {}))
+    return product | {'land-cover-id': landCover_id} if landCover_id else None
+
+
+def _should_run(cycle: dict):
+    is_cropland = cycle.get('site', {}).get('siteType') == SiteSiteType.CROPLAND.value
+
+    practices = filter_list_term_type(cycle.get('practices', []), TermTermType.LANDCOVER)
+    # split practices with properties that group with landCover
+    practices_max_100 = [
+        p for p in practices if _has_prop_grouped_with_landCover(p)
+    ]
+    total_practices_max_100 = list_sum([
+        list_sum(p.get('value', []))
+        for p in practices_max_100
+    ])
+    practices_without_grouped_props = [
+        p for p in practices if not _has_prop_grouped_with_landCover(p)
+    ]
+
+    products = filter_list_term_type(cycle.get('products', []), [TermTermType.CROP, TermTermType.FORAGE])
+    # only take products with a matching landCover term
+    products = non_empty_list(map(_product_wit_landCover_id, products))
+
+    # Products that can sum up to 100% => run if total is below 100%
+    products_max_100 = [
+        p for p in products
+        if _has_prop_grouped_with_landCover(p)
+    ] if total_practices_max_100 < 100 else []
+
+    # Products that must sum up to 100% => can not run practices already exist as already 100%
+    products_is_100 = [
+        p for p in products
+        if not _has_prop_grouped_with_landCover(p)
+    ] if not practices_without_grouped_props else []
+
+    has_crop_forage_products = bool(products_max_100 + products_is_100)
+
+    logRequirements(cycle, model=MODEL, model_key=MODEL_KEY,
+                    is_cropland=is_cropland,
+                    has_crop_forage_products=has_crop_forage_products)
+
+    should_run = all([is_cropland, has_crop_forage_products])
+    logShouldRun(cycle, MODEL, None, should_run, model_key=MODEL_KEY)
+
+    return should_run, [
+        (products_max_100, total_practices_max_100),
+        (products_is_100, 0)
+    ]
+
+
+def run(cycle: dict):
+    should_run, products_list = _should_run(cycle)
+    return flatten([
+        _run(cycle, products, total)
+        for products, total in products_list if products
+    ]) if should_run else []

{hestia_earth_models-0.70.2 → hestia_earth_models-0.70.3}/hestia_earth/models/emepEea2019/nh3ToAirExcreta.py

@@ -46,7 +46,7 @@ def _run(excreta_EF_input: float):
 def _should_run(cycle: dict):
     excreta_complete = _is_term_type_complete(cycle, TermTermType.EXCRETA)
     excreta_EF_input = get_excreta_inputs_with_factor(
-        cycle, f"{list(LOOKUPS.keys())[0]}.csv", excreta_convertion_func=total_excreta_tan, model=MODEL, term=TERM_ID
+        cycle, f"{list(LOOKUPS.keys())[0]}.csv", excreta_conversion_func=total_excreta_tan, model=MODEL, term=TERM_ID
     )
 
     logRequirements(cycle, model=MODEL, term=TERM_ID,

{hestia_earth_models-0.70.2 → hestia_earth_models-0.70.3}/hestia_earth/models/hestia/management.py

@@ -1,6 +1,5 @@
 from typing import List
 from datetime import timedelta, datetime
-from functools import reduce
 from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType, COMPLETENESS_MAPPING
 from hestia_earth.utils.lookup import column_name, get_table_value, download_lookup
 from hestia_earth.utils.model import filter_list_term_type
@@ -8,7 +7,7 @@ from hestia_earth.utils.tools import safe_parse_float, flatten
 from hestia_earth.utils.blank_node import get_node_value
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
-from hestia_earth.models.utils import _include, _omit, group_by
+from hestia_earth.models.utils import _include, group_by
 from hestia_earth.models.utils.management import _new_management
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.blank_node import condense_nodes, DatestrFormat, _gapfill_datestr, DatestrGapfillMode
@@ -24,20 +23,16 @@ REQUIREMENTS = {
             "Cycle": [{
                 "@type": "Cycle",
                 "endDate": "",
-                "products": [
-                    {
-                        "@type": "Product",
-                        "term.termType": ["crop", "forage", "landCover"]
-                    }
-                ],
                 "practices": [
                     {
+                        "@type": "Practice",
                         "term.termType": [
                             "waterRegime",
                             "tillage",
                             "cropResidueManagement",
                             "landUseManagement",
-                            "system"
+                            "system",
+                            "landCover"
                         ],
                         "value": ""
                     }
@@ -62,7 +57,6 @@ REQUIREMENTS = {
 }
 RETURNS = {
     "Management": [{
-        "@type": "Management",
         "term.termType": [
             "landCover", "waterRegime", "tillage", "cropResidueManagement", "landUseManagement", "system"
         ],
@@ -78,7 +72,7 @@ LOOKUPS = {
     "organicFertiliser": "ANIMAL_MANURE",
     "soilAmendment": "PRACTICE_INCREASING_C_INPUT",
     "landUseManagement": "GAP_FILL_TO_MANAGEMENT",
-    "property": ["GAP_FILL_TO_MANAGEMENT", "CALCULATE_TOTAL_LAND_COVER_SHARE_SEPARATELY"]
+    "property": "GAP_FILL_TO_MANAGEMENT"
 }
 MODEL_KEY = 'management'
 
@@ -87,7 +81,8 @@ _PRACTICES_TERM_TYPES = [
     TermTermType.TILLAGE,
     TermTermType.CROPRESIDUEMANAGEMENT,
     TermTermType.LANDUSEMANAGEMENT,
-    TermTermType.SYSTEM
+    TermTermType.SYSTEM,
+    TermTermType.LANDCOVER
 ]
 _PRACTICES_COMPLETENESS_MAPPING = COMPLETENESS_MAPPING.get(SchemaType.PRACTICE.value, {})
 _ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
@@ -125,7 +120,6 @@ _INPUT_RULES = {
 _SKIP_LAND_COVER_SITE_TYPES = [
     SiteSiteType.CROPLAND.value
 ]
-_CYCLE_DATE_TERM_TYPES = {TermTermType.CROP.value, TermTermType.FORAGE.value}
 
 
 def management(data: dict):
@@ -188,11 +182,6 @@ def _should_gap_fill(term: dict):
     return bool(value)
 
 
-def _filter_properties(blank_node: dict):
-    properties = list(filter(lambda p: _should_gap_fill(p.get('term', {})), blank_node.get('properties', [])))
-    return _omit(blank_node, ['properties']) | ({'properties': properties} if properties else {})
-
-
 def _map_to_value(value: dict):
     return {
         'id': value.get('term', {}).get('@id'),
@@ -207,10 +196,6 @@ def _extract_node_value(node: dict) -> dict:
     return node | {'value': get_node_value(node)}
 
 
-def _copy_item_if_exists(source: dict, keys: list[str] = None, dest: dict = None) -> dict:
-    return reduce(lambda p, c: p | ({c: source[c]} if source.get(c) else {}), keys or [], dest or {})
-
-
 def _get_relevant_items(cycle: dict, item_name: str, term_types: List[TermTermType], completeness_mapping: dict = {}):
     """
     Get items from the list of cycles with any of the relevant terms.
@@ -278,84 +263,6 @@ def _run_from_siteType(site: dict, cycle: dict):
     }] if should_run else []
 
 
-def _run_products(cycle: dict, products: list, total_products: int = None, use_cycle_dates: bool = False):
-    default_dates = _include_with_date_gap_fill(cycle, ["startDate", "endDate"])
-    return [
-        _map_to_value(default_dates | _copy_item_if_exists(
-            source=product,
-            keys=['properties', 'startDate', 'endDate'],
-            dest={
-                "term": {'@id': get_landCover_term_id(product.get('term', {}))},
-                "value": round(100 / (total_products or len(products)), 2)
-            }
-        ) | (
-            default_dates if use_cycle_dates or product.get("term", {}).get("termType") in _CYCLE_DATE_TERM_TYPES
-            else {}
-        ))
-        for product in products
-    ]
-
-
-def _run_from_landCover(cycle: dict, crop_forage_products: list):
-    """
-    Copy landCover items, and include crop/forage landCover items with properties to count in ratio.
-    """
-    land_cover_products = [
-        _map_to_value(_extract_node_value(
-            _include_with_date_gap_fill(
-                value=product,
-                keys=["term", "value", "startDate", "endDate", "properties"]
-            )
-        )) for product in _get_relevant_items(
-            cycle=cycle,
-            item_name="products",
-            term_types=[TermTermType.LANDCOVER]
-        )
-    ]
-    return land_cover_products + _run_products(
-        cycle,
-        crop_forage_products,
-        total_products=len(crop_forage_products) + len(land_cover_products),
-        use_cycle_dates=True
-    )
-
-
-def _should_group_landCover(term: dict):
-    value = get_lookup_value(lookup_term=term, column='CALCULATE_TOTAL_LAND_COVER_SHARE_SEPARATELY')
-    return bool(value)
-
-
-def _has_prop_grouped_with_landCover(product: dict):
-    return bool(
-        next((
-            p
-            for p in product.get('properties', [])
-            if _should_group_landCover(p.get('term', {}))
-        ), None)
-    )
-
-
-def _run_from_crop_forage(cycle: dict, site: dict):
-    products = _get_relevant_items(
-        cycle=cycle,
-        item_name="products",
-        term_types=[TermTermType.CROP, TermTermType.FORAGE]
-    ) if site.get("siteType", "") == SiteSiteType.CROPLAND.value else []
-    # only take products with a matching landCover term
-    products = [p for p in products if get_landCover_term_id(p.get('term', {}))]
-    # remove any properties that should not get gap-filled
-    products = list(map(_filter_properties, products))
-
-    # split products with properties that group with landCover
-    products_with_gap_filled_props = [p for p in products if _has_prop_grouped_with_landCover(p)]
-    products_without_gap_filled_props = [p for p in products if not _has_prop_grouped_with_landCover(p)]
-
-    return _run_from_landCover(
-        cycle=cycle,
-        crop_forage_products=products_with_gap_filled_props
-    ) + _run_products(cycle, products_without_gap_filled_props, use_cycle_dates=False)
-
-
 def _should_run_practice(practice: dict):
     """
     Include only landUseManagement practices where GAP_FILL_TO_MANAGEMENT = True
@@ -369,7 +276,7 @@ def _run_from_practices(cycle: dict):
         _extract_node_value(
             _include_with_date_gap_fill(
                 value=practice,
-                keys=["term", "value", "startDate", "endDate"]
+                keys=["term", "value", "startDate", "endDate", "properties"]
             )
         ) for practice in _get_relevant_items(
             cycle=cycle,
@@ -378,18 +285,16 @@ def _run_from_practices(cycle: dict):
             completeness_mapping=_PRACTICES_COMPLETENESS_MAPPING
         )
     ]
-    practices = list(map(_map_to_value, filter(_should_run_practice, practices)))
-    return practices
+    return list(map(_map_to_value, filter(_should_run_practice, practices)))
 
 
 def _run_cycle(site: dict, cycle: dict):
     inputs = _run_from_inputs(site, cycle)
-    products = _run_from_crop_forage(site=site, cycle=cycle)
     site_types = _run_from_siteType(site=site, cycle=cycle)
     practices = _run_from_practices(cycle)
     return [
         node | {'cycle-id': cycle.get('@id')}
-        for node in inputs + products + site_types + practices
+        for node in inputs + site_types + practices
     ]
 
 
@@ -409,5 +314,4 @@ def run(site: dict):
         )
         logShouldRun(site, MODEL, id, True, model_key=MODEL_KEY)
 
-    management_nodes = condense_nodes(list(map(management, nodes)))
-    return management_nodes
+    return condense_nodes(list(map(management, nodes)))

{hestia_earth_models-0.70.2 → hestia_earth_models-0.70.3}/hestia_earth/models/ipcc2019/no3ToGroundwaterExcreta.py

@@ -98,7 +98,7 @@ def _should_run(cycle: dict):
 
     use_excreta_management = _should_run_with_excreta_management(cycle)
     excreta_EF_input = get_excreta_inputs_with_factor(
-        cycle, f"{list(LOOKUPS.keys())[0]}.csv", excreta_convertion_func=total_excreta, model=MODEL, term=TERM_ID
+        cycle, f"{list(LOOKUPS.keys())[0]}.csv", excreta_conversion_func=total_excreta, model=MODEL, term=TERM_ID
     ) if use_excreta_management else None
 
     logRequirements(cycle, model=MODEL, term=TERM_ID,

{hestia_earth_models-0.70.2 → hestia_earth_models-0.70.3}/hestia_earth/models/ipcc2019/organicCarbonPerHa_tier_2.py

@@ -204,11 +204,19 @@ _CARBON_INPUT_PROPERTY_TERM_IDS = [
     _DRY_MATTER_TERM_ID
 ]
 
-_CARBON_SOURCE_TERM_TYPES = [
+_INPUT_CARBON_SOURCE_TERM_TYPES = [
     TermTermType.ORGANICFERTILISER.value,
     TermTermType.SOILAMENDMENT.value
 ]
 
+_PRACTICE_CARBON_SOURCE_TERM_TYPES = [
+    TermTermType.LANDCOVER
+]
+
+_PRODUCT_CARBON_SOURCE_TERM_TYPES = [
+    TermTermType.CROPRESIDUE
+]
+
 _VALID_SITE_TYPES = [
     SiteSiteType.CROPLAND.value
 ]
@@ -1447,7 +1455,10 @@ def _get_carbon_sources(cycle: dict) -> list[CarbonSource]:
     list[CarbonSource]
         A formatted list of `CarbonSource`s.
     """
-    inputs_and_products = cycle.get("inputs", []) + cycle.get("products", [])
+    carbon_source_nodes = filter_list_term_type(
+        cycle.get("inputs", []) + cycle.get("practices", []) + cycle.get("products", []),
+        _INPUT_CARBON_SOURCE_TERM_TYPES + _PRACTICE_CARBON_SOURCE_TERM_TYPES + _PRODUCT_CARBON_SOURCE_TERM_TYPES
+    )
 
     group_fac = cycle.get('fraction_of_group_duration')
     node_fac = cycle.get('fraction_of_node_duration')
@@ -1466,7 +1477,7 @@ def _get_carbon_sources(cycle: dict) -> list[CarbonSource]:
                 if validator(node)
             ),
             None
-        ) for node in inputs_and_products
+        ) for node in carbon_source_nodes
     ])
 
 
@@ -1603,7 +1614,7 @@ def _should_run_carbon_source(node: dict) -> bool:
     """
     return any([
         node.get("term", {}).get("@id") in _CARBON_SOURCE_TERM_IDS,
-        node.get("term", {}).get("termType") in _CARBON_SOURCE_TERM_TYPES
+        node.get("term", {}).get("termType") in _INPUT_CARBON_SOURCE_TERM_TYPES
     ])