hestia-earth-models 0.62.0__py3-none-any.whl → 0.62.1__py3-none-any.whl

hestia_earth/models/site/pre_checks/cache_sources.py

@@ -1,7 +1,7 @@
 """
 Pre Checks Cache Sources
 
-This model caches the sources of all models.
+This model caches the sources of all Site models.
 """
 from hestia_earth.models.log import debugValues
 from hestia_earth.models.utils import CACHE_KEY, cached_value
@@ -15,24 +15,20 @@ RETURNS = {
 }
 
 
-def _should_run(site: dict):
+def _run(site: dict):
     sources = find_sources()
-    has_cache = cached_value(site, CACHE_SOURCES_KEY) is not None
+    debugValues(site, sources=';'.join([str(title) for title in sources.keys()]))
+    return sources
 
-    debugValues(site,
-                sources=';'.join([str(title) for title in sources.keys()]),
-                has_cache=has_cache)
 
-    should_run = all([
-        not has_cache,
-        len(sources.keys()) > 0
-    ])
-    return should_run, sources
+def _should_run(site: dict):
+    has_cache = cached_value(site, CACHE_SOURCES_KEY)
+    return not bool(has_cache)
 
 
 def run(site: dict):
-    should_run, sources = _should_run(site)
+    should_run = _should_run(site)
     return {
         **site,
-        CACHE_KEY: cached_value(site) | {CACHE_SOURCES_KEY: sources}
+        CACHE_KEY: cached_value(site) | {CACHE_SOURCES_KEY: _run(site)}
     } if should_run else site

hestia_earth/models/site/salineWater.py

@@ -23,7 +23,7 @@ TERM_ID = 'salineWater'
 
 def _measurement():
     data = _new_measurement(TERM_ID)
-    data['value'] = [100]
+    data['value'] = [True]
     data['methodClassification'] = MeasurementMethodClassification.MODELLED_USING_OTHER_MEASUREMENTS.value
     return data
 

hestia_earth/models/stehfestBouwman2006/n2OToAirCropResidueDecompositionDirect.py

@@ -1,6 +1,8 @@
-from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.schema import EmissionMethodTier, TermTermType
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_crop_residue_decomposition_N_total
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
 from .n2OToAirSoilFlux import _should_run, _get_value
 from . import MODEL
@@ -59,5 +61,13 @@ def _run(cycle: dict, content_list_of_items: list, N_total: float):
 
 
 def run(cycle: dict):
-    should_run, N_total, content_list_of_items = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, N_total, content_list_of_items = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, TermTermType.CROPRESIDUE)
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_cropResidue_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, content_list_of_items, N_total) if should_run else []

hestia_earth/models/stehfestBouwman2006/n2OToAirExcretaDirect.py

@@ -1,6 +1,8 @@
-from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.schema import EmissionMethodTier, TermTermType
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_excreta_N_total
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
 from .n2OToAirSoilFlux import _get_value, _should_run
 from . import MODEL
@@ -66,5 +68,13 @@ def _run(cycle: dict, content_list_of_items: list, N_total: float):
 
 
 def run(cycle: dict):
-    should_run, N_total, content_list_of_items = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, N_total, content_list_of_items = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, TermTermType.EXCRETA)
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_excreta_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, content_list_of_items, N_total) if should_run else []

hestia_earth/models/stehfestBouwman2006/n2OToAirInorganicFertiliserDirect.py

@@ -1,6 +1,8 @@
 from hestia_earth.schema import EmissionMethodTier
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.emission import _new_emission
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.cycle import get_inorganic_fertiliser_N_total
 from .n2OToAirSoilFlux import _get_value, _should_run
 from . import MODEL
@@ -69,5 +71,13 @@ def _run(cycle: dict, content_list_of_items: list, N_total: float):
 
 
 def run(cycle: dict):
-    should_run, N_total, content_list_of_items = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, N_total, content_list_of_items = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, 'fertiliser')
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_fertiliser_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, content_list_of_items, N_total) if should_run else []

hestia_earth/models/stehfestBouwman2006/n2OToAirOrganicFertiliserDirect.py

@@ -1,6 +1,8 @@
 from hestia_earth.schema import EmissionMethodTier
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.emission import _new_emission
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.cycle import get_organic_fertiliser_N_total
 from .n2OToAirSoilFlux import _get_value, _should_run
 from . import MODEL
@@ -69,5 +71,13 @@ def _run(cycle: dict, content_list_of_items: list, N_total: float):
 
 
 def run(cycle: dict):
-    should_run, N_total, content_list_of_items = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, N_total, content_list_of_items = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, 'fertiliser')
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_fertiliser_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, content_list_of_items, N_total) if should_run else []

hestia_earth/models/stehfestBouwman2006/noxToAirCropResidueDecomposition.py

@@ -1,6 +1,8 @@
-from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.schema import EmissionMethodTier, TermTermType
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_crop_residue_decomposition_N_total
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
 from .noxToAirSoilFlux import _should_run, _get_value
 from . import MODEL
@@ -54,5 +56,13 @@ def _run(cycle: dict, ecoClimateZone: str, nitrogenContent: float, N_total: floa
 
 
 def run(cycle: dict):
-    should_run, ecoClimateZone, nitrogenContent, N_total = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, ecoClimateZone, nitrogenContent, N_total = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, TermTermType.CROPRESIDUE)
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_cropResidue_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, ecoClimateZone, nitrogenContent, N_total) if should_run else []

hestia_earth/models/stehfestBouwman2006/noxToAirExcreta.py

@@ -1,6 +1,8 @@
-from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.schema import EmissionMethodTier, TermTermType
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_excreta_N_total
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
 from .noxToAirSoilFlux import _should_run, _get_value
 from . import MODEL
@@ -61,5 +63,13 @@ def _run(cycle: dict, ecoClimateZone: str, nitrogenContent: float, N_total: floa
 
 
 def run(cycle: dict):
-    should_run, ecoClimateZone, nitrogenContent, N_total = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, ecoClimateZone, nitrogenContent, N_total = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, TermTermType.EXCRETA)
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_excreta_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, ecoClimateZone, nitrogenContent, N_total) if should_run else []

hestia_earth/models/stehfestBouwman2006/noxToAirInorganicFertiliser.py

@@ -1,6 +1,8 @@
 from hestia_earth.schema import EmissionMethodTier
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_inorganic_fertiliser_N_total
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
 from .noxToAirSoilFlux import _should_run, _get_value
 from . import MODEL
@@ -64,5 +66,13 @@ def _run(cycle: dict, ecoClimateZone: str, nitrogenContent: float, N_total: floa
 
 
 def run(cycle: dict):
-    should_run, ecoClimateZone, nitrogenContent, N_total = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, ecoClimateZone, nitrogenContent, N_total = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, 'fertiliser')
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_fertiliser_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, ecoClimateZone, nitrogenContent, N_total) if should_run else []

hestia_earth/models/stehfestBouwman2006/noxToAirOrganicFertiliser.py

@@ -1,6 +1,8 @@
 from hestia_earth.schema import EmissionMethodTier
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_organic_fertiliser_N_total
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
 from .noxToAirSoilFlux import _should_run, _get_value
 from . import MODEL
@@ -64,5 +66,13 @@ def _run(cycle: dict, ecoClimateZone: str, nitrogenContent: float, N_total: floa
 
 
 def run(cycle: dict):
-    should_run, ecoClimateZone, nitrogenContent, N_total = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, ecoClimateZone, nitrogenContent, N_total = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, 'fertiliser')
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_fertiliser_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, ecoClimateZone, nitrogenContent, N_total) if should_run else []

hestia_earth/models/stehfestBouwman2006GisImplementation/noxToAirCropResidueDecomposition.py

@@ -1,6 +1,8 @@
-from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.schema import EmissionMethodTier, TermTermType
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_crop_residue_decomposition_N_total
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
 from .noxToAirSoilFlux import _should_run, _get_value
 from . import MODEL
@@ -51,5 +53,13 @@ def _run(cycle: dict, country_id: str, N_total: float):
 
 
 def run(cycle: dict):
-    should_run, country_id, N_total = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, country_id, N_total, *args = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, TermTermType.CROPRESIDUE)
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_cropResidue_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, country_id, N_total) if should_run else []

hestia_earth/models/stehfestBouwman2006GisImplementation/noxToAirExcreta.py

@@ -1,6 +1,8 @@
-from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.schema import EmissionMethodTier, TermTermType
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_excreta_N_total
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
 from .noxToAirSoilFlux import _should_run, _get_value
 from . import MODEL
@@ -58,5 +60,13 @@ def _run(cycle: dict, country_id: str, N_total: float):
 
 
 def run(cycle: dict):
-    should_run, country_id, N_total = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, country_id, N_total, *args = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, TermTermType.EXCRETA)
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_excreta_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, country_id, N_total) if should_run else []

hestia_earth/models/stehfestBouwman2006GisImplementation/noxToAirInorganicFertiliser.py

@@ -1,6 +1,8 @@
 from hestia_earth.schema import EmissionMethodTier
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_inorganic_fertiliser_N_total
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
 from .noxToAirSoilFlux import _should_run, _get_value
 from . import MODEL
@@ -61,5 +63,13 @@ def _run(cycle: dict, country_id: str, N_total: float):
 
 
 def run(cycle: dict):
-    should_run, country_id, N_total, *args = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, country_id, N_total, *args = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, 'fertiliser')
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_fertiliser_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, country_id, N_total) if should_run else []

hestia_earth/models/stehfestBouwman2006GisImplementation/noxToAirOrganicFertiliser.py

@@ -1,6 +1,8 @@
 from hestia_earth.schema import EmissionMethodTier
 
+from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.cycle import get_organic_fertiliser_N_total
+from hestia_earth.models.utils.completeness import _is_term_type_complete
 from hestia_earth.models.utils.emission import _new_emission
 from .noxToAirSoilFlux import _should_run, _get_value
 from . import MODEL
@@ -61,5 +63,13 @@ def _run(cycle: dict, country_id: str, N_total: float):
 
 
 def run(cycle: dict):
-    should_run, country_id, N_total, *args = _should_run(cycle, TERM_ID, TIER)
+    default_should_run, country_id, N_total, *args = _should_run(cycle, TERM_ID, TIER)
+    term_type_complete = _is_term_type_complete(cycle, 'fertiliser')
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_fertiliser_complete=term_type_complete)
+
+    should_run = all([default_should_run, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+
     return _run(cycle, country_id, N_total) if should_run else []

hestia_earth/models/utils/blank_node.py

@@ -1,11 +1,13 @@
+import calendar
 from calendar import monthrange
 from collections import defaultdict
 from collections.abc import Iterable
-from datetime import datetime
+from datetime import datetime, timedelta
+from uuid import uuid4
+
 from dateutil.relativedelta import relativedelta
 from enum import Enum
 from functools import reduce
-from statistics import mode, mean
 from typing import (
     Any,
     List,
@@ -24,9 +26,10 @@ from hestia_earth.utils.tools import (
     safe_parse_float,
     non_empty_list
 )
+from hestia_earth.utils.blank_node import ArrayTreatment, get_node_value
 
 from ..log import debugValues, log_as_table
-from . import is_from_model, _filter_list_term_unit, is_iterable
+from . import is_from_model, _filter_list_term_unit, is_iterable, _omit
 from .constant import Units
 from .property import get_node_property, get_node_property_value
 from .lookup import (
@@ -46,7 +49,9 @@ def group_by_keys(group_keys: list = ['term']):
     return run
 
 
-def _module_term_id(term_id: str, module): return getattr(module, 'TERM_ID', term_id).split(',')[0]
+def _module_term_id(term_id: str, module):
+    term_id_str = term_id.split('.')[-1] if '.' in term_id else term_id
+    return getattr(module, 'TERM_ID', term_id_str).split(',')[0]
 
 
 def _run_model_required(model: str, term: dict, data: dict):
@@ -342,111 +347,6 @@ def convert_to_carbon(node: dict, model: str, term_id: str, blank_nodes: list, *
     ]) if len(missing_carbon_property) == 0 else None
 
 
-class ArrayTreatment(Enum):
-    """
-    Enum representing different treatments for arrays of values.
-    """
-    MEAN = 'mean'
-    MODE = 'mode'
-    SUM = 'sum'
-    FIRST = 'first'
-    LAST = 'last'
-
-
-def _should_run_array_treatment(value):
-    return isinstance(value, Iterable) and len(value) > 0
-
-
-DEFAULT_ARRAY_TREATMENT = ArrayTreatment.MEAN
-ARRAY_TREATMENT_TO_REDUCER = {
-    ArrayTreatment.MEAN: lambda value: mean(value) if _should_run_array_treatment(value) else 0,
-    ArrayTreatment.MODE: lambda value: mode(value) if _should_run_array_treatment(value) else 0,
-    ArrayTreatment.SUM: lambda value: sum(value) if _should_run_array_treatment(value) else 0,
-    ArrayTreatment.FIRST: lambda value: value[0] if _should_run_array_treatment(value) else 0,
-    ArrayTreatment.LAST: lambda value: value[-1] if _should_run_array_treatment(value) else 0
-}
-"""
-A dictionary mapping ArrayTreatment enums to corresponding reducer functions.
-"""
-
-
-def _retrieve_array_treatment(
-    node: dict,
-    is_larger_unit: bool = False,
-    default: ArrayTreatment = ArrayTreatment.FIRST
-) -> ArrayTreatment:
-    """
-    Retrieves the array treatment for a given node.
-
-    Array treatments are used to reduce an array's list of values into
-    a single value. The array treatment is retrieved from a lookup on
-    the node's term.
-
-    Parameters
-    ----------
-    node : dict
-        The dictionary representing the node.
-    is_larger_unit : bool, optional
-        Flag indicating whether to use the larger unit lookup, by default `False`.
-    default : ArrayTreatment, optional
-        Default value to return if the lookup fails, by default `ArrayTreatment.FIRST`.
-
-    Returns
-    -------
-    ArrayTreatment
-        The retrieved array treatment.
-
-    """
-    ARRAY_TREATMENT_LOOKUPS = [
-        'arrayTreatmentLargerUnitOfTime',
-        'arrayTreatment'
-    ]
-    lookup = ARRAY_TREATMENT_LOOKUPS[0] if is_larger_unit else ARRAY_TREATMENT_LOOKUPS[1]
-
-    term = node.get('term', {})
-    lookup_value = get_lookup_value(term, lookup, skip_debug=True)
-
-    return next(
-        (treatment for treatment in ArrayTreatment if treatment.value == lookup_value),
-        default
-    )
-
-
-def get_node_value(
-    node: dict,
-    is_larger_unit: bool = False,
-    array_treatment: Optional[ArrayTreatment] = None,
-    default: Any = 0
-) -> Union[float, bool]:
-    """
-    Get the value from the dictionary representing the node,
-    applying optional array treatment if the value is a list.
-
-    Parameters
-    ----------
-    node : dict
-        The dictionary representing the node.
-    is_larger_unit : bool, optional
-        A flag indicating whether the unit of time is larger, by default `False`.
-    array_treatment : ArrayTreatment, optional
-        An optional override for the treatment to be applied to an array value, if `None` the array treatment in the
-        node's term's lookup is used (which defaults to `FIRST` if no array treatment is specified), by default `None`.
-
-    Returns
-    -------
-    float | bool
-        The extracted value from the node.
-
-    """
-    value = node.get("value", 0)
-
-    reducer = ARRAY_TREATMENT_TO_REDUCER[(
-        array_treatment or _retrieve_array_treatment(node, is_larger_unit=is_larger_unit)
-    )] if isinstance(value, list) and len(value) > 0 else None
-
-    return reducer(value) if reducer else value if isinstance(value, bool) else value or default
-
-
 def _convert_to_set(
     variable: Union[Iterable[Any], Any]
 ) -> set:
@@ -560,7 +460,7 @@ def cumulative_nodes_match(
     """
     values = [
         get_node_value(
-            node, is_larger_unit, array_treatment
+            node, 'value', is_larger_unit, array_treatment
         ) or default_node_value for node in nodes if function(node)
     ]
 
@@ -1274,3 +1174,99 @@ def get_inputs_from_properties(input: dict, term_types: Union[TermTermType, List
         } for p in (properties or []) if all([p.get('key'), p.get('value')])
     ]) if input_value > 0 else []
     return filter_list_term_type(inputs, term_types)
+
+
+def _get_condensed_nodes(nodes: list) -> tuple[list, bool]:
+    """Only considers nodes which already match on non-date criteria."""
+    CONDENSABLE_UNITS = [Units.BOOLEAN.value, Units.PERCENTAGE_AREA.value]
+    condensed_nodes = []
+    matched_uuids = set()
+    nodes_by_start_date = {_full_date_str(date_str=n["startDate"], is_end=True): n for n in nodes if "startDate" in n}
+    if len(nodes_by_start_date) != len(nodes):
+        return nodes, False
+
+    for node in nodes:
+        search_date = _offset_date(date_str=node.get("endDate", ""))
+        if node["uuid"] in matched_uuids:
+            continue
+        if (search_date in nodes_by_start_date and nodes_by_start_date[search_date]["uuid"] not in matched_uuids
+                and node.get("term", {}).get("units") in CONDENSABLE_UNITS):
+            new_node = node.copy()
+            new_node["endDate"] = nodes_by_start_date[search_date]["endDate"]
+            condensed_nodes.append(new_node)
+            matched_uuids.add(nodes_by_start_date[search_date]["uuid"])
+        elif node["uuid"] not in matched_uuids:
+            condensed_nodes.append(node)
+
+    return condensed_nodes, len(matched_uuids) > 0
+
+
+def condense_nodes(nodes: list) -> list:
+    grouped_nodes = _group_nodes_by_term_and_value(nodes)
+    condensed_nodes = dict()
+    any_changes_made = False
+
+    for key, node_group in grouped_nodes.items():
+        condensed_nodes[key] = node_group
+        while len(condensed_nodes[key]) > 1:
+            condensed_nodes[key], changes_made = _get_condensed_nodes(condensed_nodes[key])
+            if not changes_made:
+                break
+            any_changes_made = True
+
+    if not any_changes_made:
+        return [_omit(values=n, keys=["uuid"]) for n in nodes]
+
+    return sorted(
+        flatten([_omit(values=n, keys=["uuid"]) for nodes in condensed_nodes.values() for n in nodes]),
+        key=lambda x: x["startDate"]
+    )
+
+
+DATE_FORMAT = "%Y-%m-%d"
+
+
+def _variable_length_str_to_date(date_str: str, is_end: bool) -> datetime:
+    """Converts to date, adding start or end of year to YYYY strings as indicated by is_end."""
+    return datetime.strptime(_full_date_str(date_str, is_end=is_end), DATE_FORMAT)
+
+
+def _full_date_str(date_str: str, is_end: bool) -> str:
+    suffix = ""
+    if len(date_str) == 4:
+        # Format YYYY
+        suffix = "-12-31" if is_end else "-01-01"
+    elif len(date_str) == 7:
+        # Format YYYY-MM
+        suffix = f"-{calendar.monthrange(int(date_str[:4]), int(date_str[5:7]))[1]}" if is_end else "-01"
+
+    return date_str + suffix
+
+
+def _with_full_dates(node: dict) -> dict:
+    output_node = node.copy()
+    if "startDate" in output_node:
+        output_node["startDate"] = _full_date_str(output_node["startDate"], is_end=False)
+    if "endDate" in output_node:
+        output_node["endDate"] = _full_date_str(output_node["endDate"], is_end=True)
+
+    return output_node
+
+
+def _offset_date(date_str: str, days: int = 1, is_end: bool = True) -> str:
+    return (
+        _variable_length_str_to_date(date_str=date_str, is_end=is_end) + timedelta(days=days)
+    ).strftime(DATE_FORMAT)
+
+
+def _group_nodes_by_term_and_value(nodes: list) -> dict:
+    grouped_nodes = defaultdict(list)
+
+    for node in nodes:
+        term_id = node.get("term", {}).get("@id", "")
+        value = "-".join([str(v) for v in node.get("value")]) if isinstance(node.get("value"), list) \
+            else node.get("value")
+        node["uuid"] = uuid4()
+        grouped_nodes[(term_id, value)].append(_with_full_dates(node))
+
+    return grouped_nodes

hestia_earth/models/utils/constant.py

@@ -3,6 +3,7 @@ from hestia_earth.utils.tools import list_sum
 
 
 class Units(Enum):
+    BOOLEAN = 'boolean'
     HEAD = 'head'
     NUMBER = 'number'
     KG = 'kg'
@@ -31,6 +32,7 @@ class Units(Enum):
     KG_COLD_CARCASS_WEIGHT = 'kg cold carcass weight'
     KG_COLD_DRESSED_CARCASS_WEIGHT = 'kg cold dressed carcass weight'
     KG_READY_TO_COOK_WEIGHT = 'kg ready-to-cook weight'
+    PERCENTAGE_AREA = '% area'
     TO_C = '-C'
     TO_N = '-N'
 

hestia_earth/models/utils/lookup.py

@@ -76,6 +76,17 @@ _ALLOW_ALL = 'all'
 def _is_site(site: dict): return site.get('@type', site.get('type')) == SchemaType.SITE.value
 
 
+def _get_sites(node: dict):
+    site = node.get('site', node.get('cycle', {}).get('site'))
+    other_sites = node.get('otherSites', node.get('cycle', {}).get('otherSites', []))
+    return non_empty_list([site] + other_sites)
+
+
+def _get_site_types(node: dict):
+    sites = [node] if _is_site(node) else _get_sites(node)
+    return non_empty_list([site.get('siteType') for site in sites])
+
+
 def _model_lookup_values(model: str, term: dict, restriction: str):
     lookup = download_lookup(f"{term.get('termType')}-model-{restriction}.csv")
     values = get_table_value(lookup, 'termid', term.get('@id'), column_name(model))
@@ -83,10 +94,11 @@ def _model_lookup_values(model: str, term: dict, restriction: str):
 
 
 def is_model_siteType_allowed(model: str, term: dict, data: dict):
-    site = data if _is_site(data) else data.get('site', data.get('cycle', {}).get('site')) or {}
-    site_type = site.get('siteType')
+    site_types = _get_site_types(data)
     allowed_values = _model_lookup_values(model, term, 'siteTypesAllowed')
-    return True if _ALLOW_ALL in allowed_values or not site_type else site_type in allowed_values
+    return True if _ALLOW_ALL in allowed_values or not site_types else any([
+        (site_type in allowed_values) for site_type in site_types
+    ])
 
 
 def _lookup_values(term: dict, column: str):
@@ -96,10 +108,11 @@ def _lookup_values(term: dict, column: str):
 
 
 def is_siteType_allowed(data: dict, term: dict):
-    site = data if _is_site(data) else data.get('site', data.get('cycle', {}).get('site')) or {}
-    site_type = site.get('siteType')
+    site_types = _get_site_types(data)
     allowed_values = _lookup_values(term, 'siteTypesAllowed')
-    return True if _ALLOW_ALL in allowed_values or not site_type else site_type in allowed_values
+    return True if _ALLOW_ALL in allowed_values or not site_types else any([
+        (site_type in allowed_values) for site_type in site_types
+    ])
 
 
 def is_product_termType_allowed(data: dict, term: dict):