hestia-earth-models 0.70.1__py3-none-any.whl → 0.70.3__py3-none-any.whl

hestia_earth/models/cml2001Baseline/resourceUseMineralsAndMetalsDuringCycle.py

@@ -6,7 +6,8 @@ from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum
 
 from hestia_earth.models.log import logRequirements, logShouldRun
-from hestia_earth.models.utils import _filter_list_term_unit, Units
+from hestia_earth.models.utils import _filter_list_term_unit
+from hestia_earth.models.utils.constant import Units
 from hestia_earth.models.utils.blank_node import _sum_nodes_value
 from hestia_earth.models.utils.indicator import _new_indicator
 from . import MODEL

hestia_earth/models/config/Cycle.json

@@ -182,6 +182,14 @@
       "mergeStrategy": "list",
       "stage": 1
     },
+    {
+      "key": "practices",
+      "model": "cycle",
+      "value": "practice.landCover",
+      "runStrategy": "always",
+      "mergeStrategy": "list",
+      "stage": 1
+    },
     {
       "key": "products",
       "model": "hestia",
@@ -1759,6 +1767,51 @@
         },
         "stage": 2
       },
+      {
+        "key": "emissions",
+        "model": "ipcc2019",
+        "value": "ch4ToAirOrganicSoilCultivation",
+        "runStrategy": "add_blank_node_if_missing",
+        "runArgs": {
+          "runNonMeasured": true,
+          "runNonAddedTerm": true
+        },
+        "mergeStrategy": "list",
+        "mergeArgs": {
+          "replaceThreshold": ["value", 0.01]
+        },
+        "stage": 2
+      },
+      {
+        "key": "emissions",
+        "model": "ipcc2019",
+        "value": "co2ToAirOrganicSoilCultivation",
+        "runStrategy": "add_blank_node_if_missing",
+        "runArgs": {
+          "runNonMeasured": true,
+          "runNonAddedTerm": true
+        },
+        "mergeStrategy": "list",
+        "mergeArgs": {
+          "replaceThreshold": ["value", 0.01]
+        },
+        "stage": 2
+      },
+      {
+        "key": "emissions",
+        "model": "ipcc2019",
+        "value": "n2OToAirOrganicSoilCultivationDirect",
+        "runStrategy": "add_blank_node_if_missing",
+        "runArgs": {
+          "runNonMeasured": true,
+          "runNonAddedTerm": true
+        },
+        "mergeStrategy": "list",
+        "mergeArgs": {
+          "replaceThreshold": ["value", 0.01]
+        },
+        "stage": 2
+      },
       {
         "key": "emissions",
         "model": "ipcc2019",
@@ -2178,6 +2231,21 @@
           "replaceThreshold": ["value", 0.01]
         },
         "stage": 2
+      },
+      {
+        "key": "emissions",
+        "model": "hestia",
+        "value": "pToSurfaceWaterAquacultureSystems",
+        "runStrategy": "add_blank_node_if_missing",
+        "runArgs": {
+          "runNonMeasured": true,
+          "runNonAddedTerm": true
+        },
+        "mergeStrategy": "list",
+        "mergeArgs": {
+          "replaceThreshold": ["value", 0.01]
+        },
+        "stage": 2
       }
     ],
     {

hestia_earth/models/config/Site.json

@@ -400,6 +400,14 @@
       "mergeStrategy": "list",
       "stage": 1
     },
+    {
+      "key": "measurements",
+      "model": "site",
+      "value": "grouped_measurement",
+      "runStrategy": "always",
+      "mergeStrategy": "list",
+      "stage": 1
+    },
     {
       "key": "measurements",
       "model": "hestia",

hestia_earth/models/cycle/practice/landCover.py

@@ -0,0 +1,181 @@
+from functools import reduce
+from hestia_earth.schema import TermTermType, SiteSiteType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import non_empty_list, flatten, list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import _omit, _include
+from hestia_earth.models.utils.practice import _new_practice
+from hestia_earth.models.utils.term import get_lookup_value
+from hestia_earth.models.utils.blank_node import condense_nodes
+from hestia_earth.models.utils.crop import get_landCover_term_id
+from .. import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "endDate": "",
+        "products": [
+            {
+                "@type": "Product",
+                "term.termType": ["crop", "forage"],
+                "optional": {
+                    "startDate": "",
+                    "endDate": ""
+                }
+            }
+        ],
+        "site": {
+            "@type": "Site",
+            "siteType": "cropland"
+        },
+        "none": {
+            "practices": [{"@type": "Practice", "term.termType": "landCover"}]
+        },
+        "optional": {
+            "startDate": ""
+        }
+    }
+}
+RETURNS = {
+    "Practice": [{
+        "term.termType": "landCover",
+        "value": "",
+        "endDate": "",
+        "startDate": ""
+    }]
+}
+LOOKUPS = {
+    "crop": ["landCoverTermId", "maximumCycleDuration"],
+    "forage": ["landCoverTermId"],
+    "property": ["GAP_FILL_TO_MANAGEMENT", "CALCULATE_TOTAL_LAND_COVER_SHARE_SEPARATELY"]
+}
+MODEL_KEY = 'landCover'
+
+
+def practice(data: dict):
+    node = _new_practice(data.get('id'))
+    node['value'] = [data['value']]
+    node['endDate'] = data['endDate']
+    if data.get('startDate'):
+        node['startDate'] = data['startDate']
+    if data.get('properties'):
+        node['properties'] = data['properties']
+    return node
+
+
+def _should_gap_fill(term: dict):
+    value = get_lookup_value(lookup_term=term, column='GAP_FILL_TO_MANAGEMENT')
+    return bool(value)
+
+
+def _filter_properties(blank_node: dict):
+    properties = list(filter(lambda p: _should_gap_fill(p.get('term', {})), blank_node.get('properties', [])))
+    return _omit(blank_node, ['properties']) | ({'properties': properties} if properties else {})
+
+
+def _map_to_value(value: dict):
+    return {
+        'id': value.get('term', {}).get('@id'),
+        'value': value.get('value'),
+        'startDate': value.get('startDate'),
+        'endDate': value.get('endDate'),
+        'properties': value.get('properties')
+    }
+
+
+def _copy_item_if_exists(source: dict, keys: list[str] = None, dest: dict = None) -> dict:
+    return reduce(lambda p, c: p | ({c: source[c]} if source.get(c) else {}), keys or [], dest or {})
+
+
+def _run(cycle: dict, products: list, total: float):
+    # remove any properties that should not get gap-filled
+    products = list(map(_filter_properties, products))
+
+    nodes = [
+        _map_to_value(_include(cycle, ["startDate", "endDate"]) | _copy_item_if_exists(
+            source=product,
+            keys=['properties', 'startDate', 'endDate'],
+            dest={
+                "term": {'@id': product.get('land-cover-id')},
+                "value": round((100 - total) / len(products), 2)
+            }
+        ))
+        for product in products
+    ]
+
+    return condense_nodes(list(map(practice, nodes)))
+
+
+def _should_group_landCover(term: dict):
+    value = get_lookup_value(lookup_term=term, column='CALCULATE_TOTAL_LAND_COVER_SHARE_SEPARATELY')
+    return bool(value)
+
+
+def _has_prop_grouped_with_landCover(product: dict):
+    return bool(
+        next((
+            p
+            for p in product.get('properties', [])
+            if _should_group_landCover(p.get('term', {}))
+        ), None)
+    )
+
+
+def _product_wit_landCover_id(product: dict):
+    landCover_id = get_landCover_term_id(product.get('term', {}))
+    return product | {'land-cover-id': landCover_id} if landCover_id else None
+
+
+def _should_run(cycle: dict):
+    is_cropland = cycle.get('site', {}).get('siteType') == SiteSiteType.CROPLAND.value
+
+    practices = filter_list_term_type(cycle.get('practices', []), TermTermType.LANDCOVER)
+    # split practices with properties that group with landCover
+    practices_max_100 = [
+        p for p in practices if _has_prop_grouped_with_landCover(p)
+    ]
+    total_practices_max_100 = list_sum([
+        list_sum(p.get('value', []))
+        for p in practices_max_100
+    ])
+    practices_without_grouped_props = [
+        p for p in practices if not _has_prop_grouped_with_landCover(p)
+    ]
+
+    products = filter_list_term_type(cycle.get('products', []), [TermTermType.CROP, TermTermType.FORAGE])
+    # only take products with a matching landCover term
+    products = non_empty_list(map(_product_wit_landCover_id, products))
+
+    # Products that can sum up to 100% => run if total is below 100%
+    products_max_100 = [
+        p for p in products
+        if _has_prop_grouped_with_landCover(p)
+    ] if total_practices_max_100 < 100 else []
+
+    # Products that must sum up to 100% => can not run practices already exist as already 100%
+    products_is_100 = [
+        p for p in products
+        if not _has_prop_grouped_with_landCover(p)
+    ] if not practices_without_grouped_props else []
+
+    has_crop_forage_products = bool(products_max_100 + products_is_100)
+
+    logRequirements(cycle, model=MODEL, model_key=MODEL_KEY,
+                    is_cropland=is_cropland,
+                    has_crop_forage_products=has_crop_forage_products)
+
+    should_run = all([is_cropland, has_crop_forage_products])
+    logShouldRun(cycle, MODEL, None, should_run, model_key=MODEL_KEY)
+
+    return should_run, [
+        (products_max_100, total_practices_max_100),
+        (products_is_100, 0)
+    ]
+
+
+def run(cycle: dict):
+    should_run, products_list = _should_run(cycle)
+    return flatten([
+        _run(cycle, products, total)
+        for products, total in products_list if products
+    ]) if should_run else []

hestia_earth/models/emepEea2019/nh3ToAirExcreta.py

@@ -46,7 +46,7 @@ def _run(excreta_EF_input: float):
 def _should_run(cycle: dict):
     excreta_complete = _is_term_type_complete(cycle, TermTermType.EXCRETA)
     excreta_EF_input = get_excreta_inputs_with_factor(
-        cycle, f"{list(LOOKUPS.keys())[0]}.csv", excreta_convertion_func=total_excreta_tan, model=MODEL, term=TERM_ID
+        cycle, f"{list(LOOKUPS.keys())[0]}.csv", excreta_conversion_func=total_excreta_tan, model=MODEL, term=TERM_ID
     )
 
     logRequirements(cycle, model=MODEL, term=TERM_ID,

hestia_earth/models/hestia/excretaKgMass.py

@@ -70,7 +70,7 @@ def _run_product(cycle: dict, product_term_id: str):
     # convert to 1kg first, then apply ratio to current value
     term = download_term(product_term_id, TermTermType.EXCRETA)
     product = {
-        'term': term,
+        'term': term or {},
         'value': [1]
     }
 

hestia_earth/models/hestia/management.py

@@ -1,6 +1,5 @@
 from typing import List
 from datetime import timedelta, datetime
-from functools import reduce
 from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType, COMPLETENESS_MAPPING
 from hestia_earth.utils.lookup import column_name, get_table_value, download_lookup
 from hestia_earth.utils.model import filter_list_term_type
@@ -8,7 +7,7 @@ from hestia_earth.utils.tools import safe_parse_float, flatten
 from hestia_earth.utils.blank_node import get_node_value
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
-from hestia_earth.models.utils import _include, _omit, group_by
+from hestia_earth.models.utils import _include, group_by
 from hestia_earth.models.utils.management import _new_management
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.blank_node import condense_nodes, DatestrFormat, _gapfill_datestr, DatestrGapfillMode
@@ -24,20 +23,16 @@ REQUIREMENTS = {
             "Cycle": [{
                 "@type": "Cycle",
                 "endDate": "",
-                "products": [
-                    {
-                        "@type": "Product",
-                        "term.termType": ["crop", "forage", "landCover"]
-                    }
-                ],
                 "practices": [
                     {
+                        "@type": "Practice",
                         "term.termType": [
                             "waterRegime",
                             "tillage",
                             "cropResidueManagement",
                             "landUseManagement",
-                            "system"
+                            "system",
+                            "landCover"
                         ],
                         "value": ""
                     }
@@ -62,7 +57,6 @@ REQUIREMENTS = {
 }
 RETURNS = {
     "Management": [{
-        "@type": "Management",
         "term.termType": [
             "landCover", "waterRegime", "tillage", "cropResidueManagement", "landUseManagement", "system"
         ],
@@ -78,7 +72,7 @@ LOOKUPS = {
     "organicFertiliser": "ANIMAL_MANURE",
     "soilAmendment": "PRACTICE_INCREASING_C_INPUT",
     "landUseManagement": "GAP_FILL_TO_MANAGEMENT",
-    "property": ["GAP_FILL_TO_MANAGEMENT", "CALCULATE_TOTAL_LAND_COVER_SHARE_SEPARATELY"]
+    "property": "GAP_FILL_TO_MANAGEMENT"
 }
 MODEL_KEY = 'management'
 
@@ -87,7 +81,8 @@ _PRACTICES_TERM_TYPES = [
     TermTermType.TILLAGE,
     TermTermType.CROPRESIDUEMANAGEMENT,
     TermTermType.LANDUSEMANAGEMENT,
-    TermTermType.SYSTEM
+    TermTermType.SYSTEM,
+    TermTermType.LANDCOVER
 ]
 _PRACTICES_COMPLETENESS_MAPPING = COMPLETENESS_MAPPING.get(SchemaType.PRACTICE.value, {})
 _ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
@@ -125,7 +120,6 @@ _INPUT_RULES = {
 _SKIP_LAND_COVER_SITE_TYPES = [
     SiteSiteType.CROPLAND.value
 ]
-_CYCLE_DATE_TERM_TYPES = {TermTermType.CROP.value, TermTermType.FORAGE.value}
 
 
 def management(data: dict):
@@ -147,7 +141,7 @@ def _get_cycle_duration(cycle: dict, land_cover_id: str):
         land_cover_id,
         column_name('maximumCycleDuration')
     ), default=None)
-    return cycle_duration or (lookup_value - 1 if lookup_value else None)
+    return cycle_duration or lookup_value
 
 
 def _gap_filled_date_only_str(date_str: str, mode: str = DatestrGapfillMode.END) -> str:
@@ -165,11 +159,9 @@ def _gap_filled_start_date(land_cover_id: str, end_date: str, cycle: dict) -> di
     """If possible, gap-fill the startDate based on the endDate - cycleDuration"""
     cycle_duration = _get_cycle_duration(cycle, land_cover_id)
     return {
-        "startDate": max(
-            _gap_filled_date_obj(end_date) - timedelta(days=cycle_duration)
-            if cycle_duration else datetime.fromtimestamp(0),
-            _gap_filled_date_obj(cycle.get("startDate"), mode=DatestrGapfillMode.START)
-            if cycle.get("startDate") else datetime.fromtimestamp(0)
+        "startDate": (
+            _gap_filled_date_obj(cycle.get("startDate"), mode=DatestrGapfillMode.START) if cycle.get("startDate") else
+            _gap_filled_date_obj(end_date) - timedelta(days=cycle_duration - 1)
         )
     } if any([cycle_duration, cycle.get("startDate")]) else {}
 
@@ -190,11 +182,6 @@ def _should_gap_fill(term: dict):
     return bool(value)
 
 
-def _filter_properties(blank_node: dict):
-    properties = list(filter(lambda p: _should_gap_fill(p.get('term', {})), blank_node.get('properties', [])))
-    return _omit(blank_node, ['properties']) | ({'properties': properties} if properties else {})
-
-
 def _map_to_value(value: dict):
     return {
         'id': value.get('term', {}).get('@id'),
@@ -209,10 +196,6 @@ def _extract_node_value(node: dict) -> dict:
     return node | {'value': get_node_value(node)}
 
 
-def _copy_item_if_exists(source: dict, keys: list[str] = None, dest: dict = None) -> dict:
-    return reduce(lambda p, c: p | ({c: source[c]} if source.get(c) else {}), keys or [], dest or {})
-
-
 def _get_relevant_items(cycle: dict, item_name: str, term_types: List[TermTermType], completeness_mapping: dict = {}):
     """
     Get items from the list of cycles with any of the relevant terms.
@@ -280,84 +263,6 @@ def _run_from_siteType(site: dict, cycle: dict):
     }] if should_run else []
 
 
-def _run_products(cycle: dict, products: list, total_products: int = None, use_cycle_dates: bool = False):
-    default_dates = _include_with_date_gap_fill(cycle, ["startDate", "endDate"])
-    return [
-        _map_to_value(default_dates | _copy_item_if_exists(
-            source=product,
-            keys=['properties', 'startDate', 'endDate'],
-            dest={
-                "term": {'@id': get_landCover_term_id(product.get('term', {}))},
-                "value": round(100 / (total_products or len(products)), 2)
-            }
-        ) | (
-            default_dates if use_cycle_dates or product.get("term", {}).get("termType") in _CYCLE_DATE_TERM_TYPES
-            else {}
-        ))
-        for product in products
-    ]
-
-
-def _run_from_landCover(cycle: dict, crop_forage_products: list):
-    """
-    Copy landCover items, and include crop/forage landCover items with properties to count in ratio.
-    """
-    land_cover_products = [
-        _map_to_value(_extract_node_value(
-            _include_with_date_gap_fill(
-                value=product,
-                keys=["term", "value", "startDate", "endDate", "properties"]
-            )
-        )) for product in _get_relevant_items(
-            cycle=cycle,
-            item_name="products",
-            term_types=[TermTermType.LANDCOVER]
-        )
-    ]
-    return land_cover_products + _run_products(
-        cycle,
-        crop_forage_products,
-        total_products=len(crop_forage_products) + len(land_cover_products),
-        use_cycle_dates=True
-    )
-
-
-def _should_group_landCover(term: dict):
-    value = get_lookup_value(lookup_term=term, column='CALCULATE_TOTAL_LAND_COVER_SHARE_SEPARATELY')
-    return bool(value)
-
-
-def _has_prop_grouped_with_landCover(product: dict):
-    return bool(
-        next((
-            p
-            for p in product.get('properties', [])
-            if _should_group_landCover(p.get('term', {}))
-        ), None)
-    )
-
-
-def _run_from_crop_forage(cycle: dict, site: dict):
-    products = _get_relevant_items(
-        cycle=cycle,
-        item_name="products",
-        term_types=[TermTermType.CROP, TermTermType.FORAGE]
-    ) if site.get("siteType", "") == SiteSiteType.CROPLAND.value else []
-    # only take products with a matching landCover term
-    products = [p for p in products if get_landCover_term_id(p.get('term', {}))]
-    # remove any properties that should not get gap-filled
-    products = list(map(_filter_properties, products))
-
-    # split products with properties that group with landCover
-    products_with_gap_filled_props = [p for p in products if _has_prop_grouped_with_landCover(p)]
-    products_without_gap_filled_props = [p for p in products if not _has_prop_grouped_with_landCover(p)]
-
-    return _run_from_landCover(
-        cycle=cycle,
-        crop_forage_products=products_with_gap_filled_props
-    ) + _run_products(cycle, products_without_gap_filled_props, use_cycle_dates=False)
-
-
 def _should_run_practice(practice: dict):
     """
     Include only landUseManagement practices where GAP_FILL_TO_MANAGEMENT = True
@@ -371,7 +276,7 @@ def _run_from_practices(cycle: dict):
         _extract_node_value(
             _include_with_date_gap_fill(
                 value=practice,
-                keys=["term", "value", "startDate", "endDate"]
+                keys=["term", "value", "startDate", "endDate", "properties"]
             )
         ) for practice in _get_relevant_items(
             cycle=cycle,
@@ -380,18 +285,16 @@ def _run_from_practices(cycle: dict):
             completeness_mapping=_PRACTICES_COMPLETENESS_MAPPING
         )
     ]
-    practices = list(map(_map_to_value, filter(_should_run_practice, practices)))
-    return practices
+    return list(map(_map_to_value, filter(_should_run_practice, practices)))
 
 
 def _run_cycle(site: dict, cycle: dict):
     inputs = _run_from_inputs(site, cycle)
-    products = _run_from_crop_forage(site=site, cycle=cycle)
     site_types = _run_from_siteType(site=site, cycle=cycle)
     practices = _run_from_practices(cycle)
     return [
         node | {'cycle-id': cycle.get('@id')}
-        for node in inputs + products + site_types + practices
+        for node in inputs + site_types + practices
     ]
 
 
@@ -411,5 +314,4 @@ def run(site: dict):
         )
         logShouldRun(site, MODEL, id, True, model_key=MODEL_KEY)
 
-    management_nodes = condense_nodes(list(map(management, nodes)))
-    return management_nodes
+    return condense_nodes(list(map(management, nodes)))