hestia-earth-models 0.64.12__py3-none-any.whl → 0.64.14__py3-none-any.whl

hestia_earth/models/site/management.py

@@ -14,18 +14,19 @@ When nodes are chronologically consecutive with "% area" or "boolean" units and
 condensed into a single node to aid readability.
 """
 from functools import reduce
-
-from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType
-from hestia_earth.utils.api import download_hestia
-from hestia_earth.utils.model import filter_list_term_type, linked_node
-from hestia_earth.utils.tools import safe_parse_float, flatten, non_empty_list
+from hestia_earth.schema import TermTermType, SiteSiteType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import safe_parse_float, flatten
 from hestia_earth.utils.blank_node import get_node_value
 
-from hestia_earth.models.log import logRequirements, logShouldRun, log_blank_nodes_id
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils import _include
+from hestia_earth.models.utils.management import _new_management
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.blank_node import condense_nodes
-from hestia_earth.models.utils.site import related_cycles
-from hestia_earth.models.utils.site import get_land_cover_term_id as get_landCover_term_id_from_site_type
+from hestia_earth.models.utils.site import (
+    related_cycles, get_land_cover_term_id as get_landCover_term_id_from_site_type
+)
 from . import MODEL
 
 REQUIREMENTS = {
@@ -89,36 +90,37 @@ LOOKUPS = {
     "landUseManagement": "GAP_FILL_TO_MANAGEMENT"
 }
 MODEL_KEY = 'management'
-LAND_COVER_KEY = LOOKUPS['crop'][0]
-ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
-INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID = "inorganicNitrogenFertiliserUsed"
-ORGANIC_FERTILISER_USED_TERM_ID = "organicFertiliserUsed"
-AMENDMENT_INCREASING_C_USED_TERM_ID = "amendmentIncreasingSoilCarbonUsed"
-INPUT_RULES = {
+
+_LAND_COVER_KEY = LOOKUPS['crop'][0]
+_ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
+_INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID = "inorganicNitrogenFertiliserUsed"
+_ORGANIC_FERTILISER_USED_TERM_ID = "organicFertiliserUsed"
+_AMENDMENT_INCREASING_C_USED_TERM_ID = "amendmentIncreasingSoilCarbonUsed"
+_INPUT_RULES = {
     TermTermType.INORGANICFERTILISER.value: (
         (
             TermTermType.INORGANICFERTILISER.value,  # Lookup column
             lambda x: safe_parse_float(x) > 0,  # Condition
-            INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID  # New term.
+            _INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID  # New term.
         ),
     ),
     TermTermType.SOILAMENDMENT.value: (
         (
             TermTermType.SOILAMENDMENT.value,
-            lambda x: x is True,
-            AMENDMENT_INCREASING_C_USED_TERM_ID
+            lambda x: bool(x) is True,
+            _AMENDMENT_INCREASING_C_USED_TERM_ID
         ),
     ),
     TermTermType.ORGANICFERTILISER.value: (
         (
             TermTermType.SOILAMENDMENT.value,
-            lambda x: x is True,
-            ORGANIC_FERTILISER_USED_TERM_ID
+            lambda x: bool(x) is True,
+            _ORGANIC_FERTILISER_USED_TERM_ID
         ),
         (
             TermTermType.ORGANICFERTILISER.value,
-            lambda x: x is True,
-            ANIMAL_MANURE_USED_TERM_ID
+            lambda x: bool(x) is True,
+            _ANIMAL_MANURE_USED_TERM_ID
         )
     )
 }
@@ -128,15 +130,28 @@ _SKIP_LAND_COVER_SITE_TYPES = [
 
 
 def management(data: dict):
-    node = {'@type': SchemaType.MANAGEMENT.value}
-    return node | data
+    node = _new_management(data.get('id'))
+    node['value'] = data['value']
+    node['endDate'] = data['endDate']
+    if data.get('startDate'):
+        node['startDate'] = data['startDate']
+    if data.get('properties'):
+        node['properties'] = data['properties']
+    return node
 
 
-def _extract_node_value(node: dict) -> dict:
-    return node | {'value': get_node_value(node)}
+def _map_to_value(value: dict):
+    return {
+        'id': value.get('term', {}).get('@id'),
+        'value': value.get('value'),
+        'startDate': value.get('startDate'),
+        'endDate': value.get('endDate'),
+        'properties': value.get('properties')
+    }
 
 
-def _include(value: dict, keys: list) -> dict: return {k: v for k, v in value.items() if k in keys}
+def _extract_node_value(node: dict) -> dict:
+    return node | {'value': get_node_value(node)}
 
 
 def _default_dates(cycle: dict, values: list):
@@ -155,148 +170,121 @@ def _copy_item_if_exists(source: dict, keys: list[str] = None, dest: dict = None
 
 
 def _get_landCover_term_id(product: dict) -> str:
-    value = get_lookup_value(product.get('term', {}), LAND_COVER_KEY, model=MODEL, model_key=LAND_COVER_KEY)
+    term = product.get('term', {})
+    value = get_lookup_value(term, _LAND_COVER_KEY, model=MODEL, term=term.get('@id'), model_key=MODEL_KEY)
     return value.split(';')[0] if value else None
 
 
 def _get_relevant_items(
-    cycles: list[dict], item_name: str, relevant_terms: list, date_fill: callable = _default_dates
+    cycle: dict, item_name: str, relevant_terms: list, date_fill: callable = _default_dates
 ):
     """
     Get items from the list of cycles with any of the relevant terms.
     Also adds dates if missing.
     """
     return [
-        [
-            item
-            for item in date_fill(cycle=cycle, values=filter_list_term_type(cycle.get(item_name, []), relevant_terms))
-        ]
-        for cycle in cycles
+        item
+        for item in date_fill(cycle=cycle, values=filter_list_term_type(cycle.get(item_name, []), relevant_terms))
     ]
 
 
-def _get_lookup_with_debug(term: dict, column: str) -> any:
-    get_lookup_value(term, column, model_key=MODEL_KEY, land_cover_key=LAND_COVER_KEY)
-
-
-def _data_from_input(cycle: dict, term_id: str) -> dict:
-    return {
-        "term": {
-            "@type": "Term",
-            "@id": term_id,
-            "termType": "landUseManagement"
-        },
-        "value": True,
-        "startDate": cycle["startDate"],
-        "endDate": cycle["endDate"]
-    }
-
-
-def _process_rule(cycle, term, term_type) -> list:
+def _process_rule(node: dict, term: dict) -> list:
     relevant_terms = []
-    for column, condition, new_term in INPUT_RULES[term_type]:
-        lookup_result = _get_lookup_with_debug(term, LOOKUPS[column])
+    for column, condition, new_term in _INPUT_RULES[term.get('termType')]:
+        lookup_result = get_lookup_value(term, LOOKUPS[column], model=MODEL, term=term.get('@id'), model_key=MODEL_KEY)
 
         if condition(lookup_result):
-            relevant_terms.append(_data_from_input(cycle=cycle, term_id=new_term))
+            relevant_terms.append(node | {'id': new_term})
 
     return relevant_terms
 
 
-def _get_relevant_inputs(cycles: list[dict]) -> list:
-    relevant_inputs = []
-    for cycle in [c for c in cycles if "inputs" in c]:
-        for i in cycle["inputs"]:
-            if i.get("term", {}).get("termType", "") in INPUT_RULES:
-                relevant_inputs.extend(
-                    _process_rule(
-                        cycle=cycle,
-                        term=i.get("term", {}),
-                        term_type=i.get("term", {}).get("termType", "")
-                    )
-                )
-
-    return relevant_inputs
-
-
-def _has_gap_fill_to_management_set(practices: list) -> list:
-    """
-    Include only landUseManagement practices where GAP_FILL_TO_MANAGEMENT = True
-    """
-    result = [
-        p for p in practices
-        if p.get("term", {}).get("termType", {}) != TermTermType.LANDUSEMANAGEMENT.value
-        or get_lookup_value(lookup_term=p.get("term", {}), column=LOOKUPS["landUseManagement"])
-    ]
-    return result
-
-
-def _should_run_all_products(cycles: list, site_type: str):
-    products_land_cover = flatten(_get_relevant_items(
-        cycles=cycles,
+def _run_from_inputs(site: dict, cycle: dict) -> list:
+    inputs = flatten([
+        _process_rule(node={
+            'value': True,
+            'startDate': cycle.get('startDate'),
+            'endDate': cycle.get('endDate')
+        }, term=input.get('term'))
+        for input in cycle.get('inputs', [])
+        if input.get('term', {}).get('termType') in _INPUT_RULES
+    ])
+    return inputs
+
+
+def _run_from_siteType(site: dict, cycle: dict):
+    site_type = site.get('siteType')
+    site_type_id = get_landCover_term_id_from_site_type(site_type) if site_type not in _SKIP_LAND_COVER_SITE_TYPES \
+        else None
+
+    should_run = all([site_type_id])
+    return [{
+        'id': site_type_id,
+        'value': 100,
+        'startDate': cycle.get('startDate'),
+        'endDate': cycle.get('endDate')
+    }] if should_run else []
+
+
+def _run_from_landCover(cycle: dict):
+    products = _get_relevant_items(
+        cycle=cycle,
         item_name="products",
         relevant_terms=[TermTermType.LANDCOVER]
-    )) if site_type else []
-    products_land_cover = [
-        _extract_node_value(
+    )
+    products = [
+        _map_to_value(_extract_node_value(
             _include(
                 value=product,
                 keys=["term", "value", "startDate", "endDate", "properties"]
             )
-        ) for product in products_land_cover
+        )) for product in products
     ]
+    return products
+
 
-    products_crop_forage = _get_relevant_items(
-        cycles=cycles,
+def _run_from_crop_forage(cycle: dict):
+    products = _get_relevant_items(
+        cycle=cycle,
         item_name="products",
         relevant_terms=[TermTermType.CROP, TermTermType.FORAGE],
         date_fill=_dates_from_current_cycle
     )
-    products_crop_forage = [
-        _copy_item_if_exists(
+    products = list(filter(_get_landCover_term_id, products))
+    products = [
+        _map_to_value(_copy_item_if_exists(
             source=product,
             keys=["startDate", "endDate", "properties"],
             dest={
-                "term": linked_node(download_hestia(_get_landCover_term_id(product))),
-                "value": round(100 / len(_products), 2)
+                "term": {'@id': _get_landCover_term_id(product)},
+                "value": round(100 / len(products), 2)
             }
-        )
-        for _products in products_crop_forage
-        for product in list(filter(_get_landCover_term_id, _products))
-    ] if site_type else []
-    dates = sorted(list(set(
-        non_empty_list(flatten([[cycle.get('startDate'), cycle.get('endDate')] for cycle in cycles]))
-    ))) if site_type not in _SKIP_LAND_COVER_SITE_TYPES else []
-    site_type_id = get_landCover_term_id_from_site_type(site_type) if site_type else None
-    site_type_term = download_hestia(site_type_id) if all([len(dates) >= 2, site_type_id]) else None
-    products_site_type = [{
-        "term": linked_node(site_type_term),
-        "value": 100,
-        "startDate": dates[0],
-        "endDate": dates[-1]
-    }] if site_type_term else []
-
-    return products_site_type, products_crop_forage, products_land_cover
-
-
-def _should_run(site: dict):
-    cycles = related_cycles(site)
+        ))
+        for product in products
+    ]
+    return products
+
 
-    products_animal, products_crop_forage, products_land_cover = _should_run_all_products(
-        cycles=cycles,
-        site_type=site.get("siteType")
+def _has_gap_fill_to_management_set(practice: dict):
+    """
+    Include only landUseManagement practices where GAP_FILL_TO_MANAGEMENT = True
+    """
+    term = practice.get('term', {})
+    return (
+        term.get('termType') != TermTermType.LANDUSEMANAGEMENT.value or
+        get_lookup_value(lookup_term=term, column=LOOKUPS["landUseManagement"])
     )
-    all_products = products_land_cover + products_crop_forage + products_animal
-    all_products = condense_nodes(all_products)
 
+
+def _run_from_practices(cycle: dict):
     practices = [
         _extract_node_value(
             _include(
                 value=practice,
                 keys=["term", "value", "startDate", "endDate"]
             )
-        ) for practice in flatten(_get_relevant_items(
-            cycles=cycles,
+        ) for practice in _get_relevant_items(
+            cycle=cycle,
             item_name="practices",
             relevant_terms=[
                 TermTermType.WATERREGIME,
@@ -305,28 +293,38 @@ def _should_run(site: dict):
                 TermTermType.LANDUSEMANAGEMENT,
                 TermTermType.SYSTEM
             ]
-        ))
+        )
+    ]
+    practices = list(map(_map_to_value, filter(_has_gap_fill_to_management_set, practices)))
+    return practices
+
+
+def _run_cycle(site: dict, cycle: dict):
+    inputs = _run_from_inputs(site, cycle)
+    products = _run_from_landCover(cycle) + _run_from_crop_forage(cycle)
+    site_types = _run_from_siteType(site, cycle)
+    practices = _run_from_practices(cycle)
+    return [
+        node | {'cycle-id': cycle.get('@id')}
+        for node in inputs + products + site_types + practices
     ]
-    practices = _has_gap_fill_to_management_set(practices)
-    practices = condense_nodes(practices)
-
-    relevant_inputs = _get_relevant_inputs(cycles)
-    logRequirements(
-        site,
-        model=MODEL,
-        term=None,
-        model_key=MODEL_KEY,
-        products_crop_forage_ids=log_blank_nodes_id(products_crop_forage),
-        products_land_cover_ids=log_blank_nodes_id(products_land_cover),
-        products_animal=log_blank_nodes_id(products_animal),
-        practice_ids=log_blank_nodes_id(practices),
-        inputs=log_blank_nodes_id(relevant_inputs)
-    )
-    should_run = any(all_products + practices + relevant_inputs)
-    logShouldRun(site, MODEL, None, should_run=should_run, model_key=MODEL_KEY)
-    return should_run, all_products, practices, relevant_inputs
 
 
 def run(site: dict):
-    should_run, products, practices, inputs = _should_run(site)
-    return list(map(management, products + practices + inputs)) if should_run else []
+    cycles = related_cycles(site)
+    nodes = flatten([_run_cycle(site, cycle) for cycle in cycles])
+
+    # group nodes with same `id` to display as a single log per node
+    grouped_nodes = reduce(lambda p, c: p | {c['id']: p.get(c['id'], []) + [c]}, nodes, {})
+    for id, values in grouped_nodes.items():
+        logRequirements(
+            site,
+            model=MODEL,
+            term=id,
+            model_key=MODEL_KEY,
+            details=log_as_table(values, ignore_keys=['id', 'properties']),
+        )
+        logShouldRun(site, MODEL, id, True, model_key=MODEL_KEY)
+
+    management_nodes = condense_nodes(list(map(management, nodes)))
+    return management_nodes

hestia_earth/models/stehfestBouwman2006/n2OToAirCropResidueDecompositionDirect.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_crop_residue_decomposition_N_total
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
-from .n2OToAirSoilFlux import _should_run, _get_value
+from .n2OToAirSoilFlux_utils import _should_run, _get_value
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/stehfestBouwman2006/n2OToAirExcretaDirect.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_excreta_N_total
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
-from .n2OToAirSoilFlux import _get_value, _should_run
+from .n2OToAirSoilFlux_utils import _get_value, _should_run
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/stehfestBouwman2006/n2OToAirInorganicFertiliserDirect.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.emission import _new_emission
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.cycle import get_inorganic_fertiliser_N_total
-from .n2OToAirSoilFlux import _get_value, _should_run
+from .n2OToAirSoilFlux_utils import _get_value, _should_run
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/stehfestBouwman2006/n2OToAirOrganicFertiliserDirect.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.emission import _new_emission
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.cycle import get_organic_fertiliser_N_total
-from .n2OToAirSoilFlux import _get_value, _should_run
+from .n2OToAirSoilFlux_utils import _get_value, _should_run
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/stehfestBouwman2006/noxToAirCropResidueDecomposition.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_crop_residue_decomposition_N_total
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
-from .noxToAirSoilFlux import _should_run, _get_value
+from .noxToAirSoilFlux_utils import _should_run, _get_value
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/stehfestBouwman2006/noxToAirExcreta.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_excreta_N_total
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
-from .noxToAirSoilFlux import _should_run, _get_value
+from .noxToAirSoilFlux_utils import _should_run, _get_value
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/stehfestBouwman2006/noxToAirInorganicFertiliser.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_inorganic_fertiliser_N_total
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
-from .noxToAirSoilFlux import _should_run, _get_value
+from .noxToAirSoilFlux_utils import _should_run, _get_value
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/stehfestBouwman2006/noxToAirOrganicFertiliser.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_organic_fertiliser_N_total
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
-from .noxToAirSoilFlux import _should_run, _get_value
+from .noxToAirSoilFlux_utils import _should_run, _get_value
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/stehfestBouwman2006GisImplementation/noxToAirCropResidueDecomposition.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_crop_residue_decomposition_N_total
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
-from .noxToAirSoilFlux import _should_run, _get_value
+from .noxToAirSoilFlux_utils import _should_run, _get_value
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/stehfestBouwman2006GisImplementation/noxToAirExcreta.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_excreta_N_total
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
-from .noxToAirSoilFlux import _should_run, _get_value
+from .noxToAirSoilFlux_utils import _should_run, _get_value
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/stehfestBouwman2006GisImplementation/noxToAirInorganicFertiliser.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_inorganic_fertiliser_N_total
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
-from .noxToAirSoilFlux import _should_run, _get_value
+from .noxToAirSoilFlux_utils import _should_run, _get_value
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/stehfestBouwman2006GisImplementation/noxToAirOrganicFertiliser.py

@@ -4,7 +4,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_organic_fertiliser_N_total
 from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
-from .noxToAirSoilFlux import _should_run, _get_value
+from .noxToAirSoilFlux_utils import _should_run, _get_value
 from . import MODEL
 
 REQUIREMENTS = {

hestia_earth/models/usetoxV2/freshwaterEcotoxicityPotentialCtue.py

@@ -1,5 +1,5 @@
 from hestia_earth.models.utils.indicator import _new_indicator
-from hestia_earth.models.utils.pesticideAI import impact_lookup_value, impact_value_set_none
+from hestia_earth.models.utils.pesticideAI import impact_lookup_value
 from . import MODEL
 
 REQUIREMENTS = {
@@ -30,5 +30,4 @@ def _indicator(value: float):
 
 def run(impact_assessment: dict):
     value = impact_lookup_value(MODEL, TERM_ID, impact_assessment, LOOKUPS['pesticideAI'])
-    should_run = any([value is not None, impact_value_set_none(impact_assessment)])
-    return _indicator(value) if should_run else None
+    return _indicator(value) if value is not None else None

hestia_earth/models/utils/__init__.py

@@ -29,7 +29,10 @@ def cached_value(node: dict, key: str = None, default=None):
 def _term_id(term): return term.get('@id') if isinstance(term, dict) else term
 
 
-def _omit(values: dict, keys: list): return {k: v for k, v in values.items() if k not in keys}
+def _omit(values: dict, keys: list) -> dict: return {k: v for k, v in values.items() if k not in keys}
+
+
+def _include(value: dict, keys: list) -> dict: return {k: v for k, v in value.items() if k in keys}
 
 
 def _include_model(node: dict, term_id: str):

hestia_earth/models/utils/blank_node.py

@@ -280,9 +280,9 @@ def get_total_value_converted_with_min_ratio(
         ) for blank_node in blank_nodes
     ]
     value_logs = log_as_table([{
-        'id': term_id,
-        'value': value,
-        prop_id: prop_value
+        'node-id': term_id,
+        'node-value': value,
+        f"{prop_id}-value": prop_value
     } for term_id, value, prop_value in values])
 
     total_value = list_sum([value for term_id, value, prop_value in values])
@@ -746,7 +746,7 @@ DATESTR_GAPFILL_MODE_TO_GAPFILL_FUNCTION = {
 
 def _gapfill_datestr(datestr: str, mode: DatestrGapfillMode = DatestrGapfillMode.START) -> str:
     """
-    Gapfill incomplete datestrs and returns them in the format `YYYY-MM-DDTHH:MM_SS`.
+    Gapfill incomplete datestrs and returns them in the format `YYYY-MM-DDTHH:mm:ss`.
     """
     VALID_DATE_FORMATS = {
         DatestrFormat.YEAR, DatestrFormat.YEAR_MONTH, DatestrFormat.YEAR_MONTH_DAY
@@ -768,7 +768,7 @@ def _datetime_range_duration(range: DatetimeRange, add_second=False) -> float:
     """
     Determine the length of a `DatetimeRange` in seconds.
 
-    Option to `add_second` to account for 1 second between 23:59:59 and 00:00:00)
+    Option to `add_second` to account for 1 second between 23:59:59 and 00:00:00
     """
     return (range.end - range.start).total_seconds() + int(add_second)
 
@@ -779,7 +779,7 @@ def _calc_datetime_range_intersection_duration(
     """
     Determine the length of a `DatetimeRange` in seconds.
 
-    Option to `add_second` to account for 1 second between 23:59:59 and 00:00:00)
+    Option to `add_second` to account for 1 second between 23:59:59 and 00:00:00
     """
     latest_start = max(range_a.start, range_b.start)
     earliest_end = min(range_a.end, range_b.end)
@@ -793,7 +793,7 @@ def _calc_datetime_range_intersection_duration(
 
     # if less than 0 the ranges do not intersect, so return 0.
     return (
-        _datetime_range_duration(intersection_range) + int(add_second)
+        _datetime_range_duration(intersection_range, add_second=add_second)
         if duration > 0 else 0
     )
 
@@ -829,7 +829,10 @@ def _should_run_node_by_end_date(node: dict) -> bool:
     """
     Validate nodes for `group_nodes_by_year` using the "startDate" and "endDate" fields.
     """
-    return _get_datestr_format(node.get("endDate")) in VALID_DATE_FORMATS_GROUP_NODES_BY_YEAR
+    return (
+        _get_datestr_format(node.get("endDate")) in VALID_DATE_FORMATS_GROUP_NODES_BY_YEAR
+        and validate_start_date_end_date(node)
+    )
 
 
 def _should_run_node_by_dates(node: dict) -> bool:
@@ -927,8 +930,8 @@ def _build_time_fraction_dict(
         node_datetime_range, group_datetime_range, add_second=True
     )
 
-    fraction_of_group_duration = intersection_duration / group_duration
-    fraction_of_node_duration = intersection_duration / node_duration
+    fraction_of_group_duration = intersection_duration / group_duration if group_duration > 0 else 0
+    fraction_of_node_duration = intersection_duration / node_duration if node_duration > 0 else 0
 
     return {
         "fraction_of_group_duration": fraction_of_group_duration,
@@ -1283,10 +1286,19 @@ def _group_nodes_by_consecutive_dates(nodes: list):
 
 def _node_from_group(nodes: list):
     # `nodes` contain list with consecutive dates
-    return nodes[0] if len(nodes) == 1 else nodes[0] | {
-        'startDate': min(n.get('startDate') for n in nodes),
-        'endDate': max(n.get('endDate') for n in nodes)
-    }
+    # if all nodes have the same dates, sum up the values
+    same_startDate = len(set([n.get('startDate') for n in nodes])) == 1
+    same_endDate = len(set([n.get('endDate') for n in nodes])) == 1
+    total_value = list_sum(flatten([n.get('value', []) for n in nodes]))
+    return nodes[0] if len(nodes) == 1 else (
+        nodes[0] | {
+            'value': [total_value] if isinstance(nodes[0]['value'], list) else total_value
+        } if all([same_startDate, same_endDate])
+        else nodes[0] | {
+            'startDate': min(n.get('startDate') for n in nodes),
+            'endDate': max(n.get('endDate') for n in nodes)
+        }
+    )
 
 
 def _condense_nodes(nodes: list):
@@ -1457,3 +1469,11 @@ def convert_unit_properties(node_value: Union[int, float], node: dict, dest_unit
         lambda value, conversion_property_field: _convert_via_property(node, value, conversion_property_field),
         conversions, node_value
     ) if conversions else None
+
+
+def validate_start_date_end_date(node: dict) -> bool:
+    """Return `True` if `node.startDate` is before `node.endDate`, `False` if otherwise."""
+    start_date = _gapfill_datestr(node.get("startDate", OLDEST_DATE))
+    end_date = _gapfill_datestr(node.get("endDate", OLDEST_DATE), DatestrGapfillMode.END)
+
+    return safe_parse_date(start_date) < safe_parse_date(end_date)

hestia_earth/models/utils/constant.py

@@ -2,6 +2,9 @@ from enum import Enum
 
 from hestia_earth.utils.tools import list_sum
 
+DAYS_IN_YEAR = 365.25
+DAYS_PER_MONTH = DAYS_IN_YEAR/12
+
 
 class Units(Enum):
     BOOLEAN = 'boolean'