hestia-earth-models 0.67.1__py3-none-any.whl → 0.68.0__py3-none-any.whl

hestia_earth/models/site/management.py

@@ -185,7 +185,7 @@ def _gap_filled_start_date(land_cover_id: str, end_date: str, cycle: dict) -> di
     cycle_duration = _get_cycle_duration(cycle, land_cover_id)
     return {
         "startDate": max(
-            _gap_filled_date_obj(end_date) - timedelta(days=cycle_duration)
+            _gap_filled_date_obj(end_date) - timedelta(days=cycle_duration - 1)
             if cycle_duration else datetime.fromtimestamp(0),
             _gap_filled_date_obj(cycle.get("startDate"), mode=DatestrGapfillMode.START)
             if cycle.get("startDate") else datetime.fromtimestamp(0)

hestia_earth/models/site/post_checks/__init__.py

@@ -2,13 +2,14 @@ from os.path import dirname, abspath
 import sys
 
 from hestia_earth.models.utils import _run_in_serie
-from . import cache
+from . import cache, country
 
 CURRENT_DIR = dirname(abspath(__file__)) + '/'
 sys.path.append(CURRENT_DIR)
 
 MODELS = [
-    cache.run
+    cache.run,
+    country.run
 ]
 
 

hestia_earth/models/site/post_checks/country.py

@@ -0,0 +1,9 @@
+"""
+Post Checks Country
+
+Stores a simpler version of the country as originally set on the Site.
+"""
+from hestia_earth.utils.model import linked_node
+
+
+def run(site: dict): return site | {'country': linked_node(site.get('country'))}

hestia_earth/models/site/pre_checks/__init__.py

@@ -2,7 +2,7 @@ from os.path import dirname, abspath
 import sys
 
 from hestia_earth.models.utils import _run_in_serie
-from . import cache_years, cache_geospatialDatabase, cache_sources
+from . import cache_years, cache_geospatialDatabase, cache_sources, country
 
 CURRENT_DIR = dirname(abspath(__file__)) + '/'
 sys.path.append(CURRENT_DIR)
@@ -10,7 +10,8 @@ sys.path.append(CURRENT_DIR)
 MODELS = [
     cache_years.run,
     cache_geospatialDatabase.run,
-    cache_sources.run
+    cache_sources.run,
+    country.run
 ]
 
 

hestia_earth/models/site/pre_checks/country.py

@@ -0,0 +1,9 @@
+"""
+Pre Checks Country
+
+Load the complete country data from HESTIA to be able to use `subClassOf` for example.
+"""
+from hestia_earth.utils.api import download_hestia
+
+
+def run(site: dict): return site | {'country': download_hestia(site.get('country', {}).get('@id'))}

hestia_earth/models/utils/__init__.py

@@ -8,10 +8,9 @@ import datetime
 from functools import reduce
 import operator
 from pydash.objects import get
-from typing import Any, Callable, Union
+from typing import Any, Callable
 from hestia_earth.schema import SchemaType
 from hestia_earth.utils.api import download_hestia
-from hestia_earth.utils.model import linked_node
 from hestia_earth.utils.tools import flatten, non_empty_list
 from hestia_earth.utils.date import is_in_days, is_in_months
 
@@ -44,20 +43,6 @@ def _omit(values: dict, keys: list) -> dict: return {k: v for k, v in values.ite
 def _include(value: dict, keys: list) -> dict: return {k: v for k, v in value.items() if k in keys}
 
 
-def _include_model(node: dict, term_id: str):
-    term = download_hestia(term_id) or {}
-    return {**node, **({} if term.get('@id') is None else {'model': linked_node(term)})}
-
-
-def _include_method(node: dict, term_id: Union[None, str, dict], key='method'):
-    term = (download_hestia(term_id) or {}) if isinstance(term_id, str) else term_id
-    return node | ({} if term is None or term.get('@id') is None else {key: linked_node(term)})
-
-
-def _include_methodModel(node: dict, term_id: str):
-    return _include_method(node, term_id=term_id, key='methodModel')
-
-
 def _run_in_serie(data: dict, models: list): return reduce(lambda prev, model: model(prev), models, data)
 
 

hestia_earth/models/utils/blank_node.py

@@ -15,7 +15,6 @@ from typing import (
 from dateutil import parser
 from dateutil.relativedelta import relativedelta
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.blank_node import ArrayTreatment, get_node_value
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import (
@@ -26,18 +25,21 @@ from hestia_earth.utils.tools import (
     safe_parse_float,
     non_empty_list
 )
-
-from . import is_from_model, _filter_list_term_unit, is_iterable, full_date_str
-from .constant import Units, get_atomic_conversion
-from .lookup import (
+from hestia_earth.utils.lookup_utils import (
     is_model_siteType_allowed,
     is_siteType_allowed,
-    is_product_id_allowed, is_product_termType_allowed,
-    is_input_id_allowed, is_input_termType_allowed, _node_value
+    is_product_id_allowed,
+    is_product_termType_allowed,
+    is_input_id_allowed,
+    is_input_termType_allowed
 )
+
+from hestia_earth.models.log import debugValues, log_as_table
+from . import is_from_model, _filter_list_term_unit, is_iterable, full_date_str
+from .constant import Units, get_atomic_conversion
+from .lookup import _node_value
 from .property import get_node_property, get_node_property_value
-from .term import get_lookup_value
-from ..log import debugValues, log_as_table
+from .term import get_lookup_value, download_term
 
 # TODO: verify those values
 MAX_DEPTH = 1000
@@ -122,29 +124,29 @@ def _module_term_id(term_id: str, module):
     return getattr(module, 'TERM_ID', term_id_str).split(',')[0]
 
 
-def _run_model_required(model: str, term: dict, data: dict):
-    siteType_allowed = is_model_siteType_allowed(model, term, data)
+def _run_model_required(model: str, term_id: str, data: dict):
+    siteType_allowed = is_model_siteType_allowed(model, term_id, data)
 
     run_required = all([siteType_allowed])
-    debugValues(data, model=model, term=term.get('@id'),
+    debugValues(data, model=model, term=term_id,
                 run_required=run_required,
                 siteType_allowed=siteType_allowed)
     return run_required
 
 
-def _run_required(model: str, term: dict, data: dict):
-    siteType_allowed = is_siteType_allowed(data, term)
-    product_id_allowed = is_product_id_allowed(data, term)
-    product_termType_allowed = is_product_termType_allowed(data, term)
-    input_id_allowed = is_input_id_allowed(data, term)
-    input_termType_allowed = is_input_termType_allowed(data, term)
+def _run_required(model: str, term_id: str, data: dict):
+    siteType_allowed = is_siteType_allowed(data, term_id)
+    product_id_allowed = is_product_id_allowed(data, term_id)
+    product_termType_allowed = is_product_termType_allowed(data, term_id)
+    input_id_allowed = is_input_id_allowed(data, term_id)
+    input_termType_allowed = is_input_termType_allowed(data, term_id)
 
     run_required = all([
         siteType_allowed, product_id_allowed, product_termType_allowed, input_id_allowed, input_termType_allowed
     ])
     # model is only used for logs here, skip logs if model not provided
     if model:
-        debugValues(data, model=model, term=term.get('@id'),
+        debugValues(data, model=model, term=term_id,
                     siteType_allowed=siteType_allowed,
                     product_id_allowed=product_id_allowed,
                     product_termType_allowed=product_termType_allowed,
@@ -152,7 +154,7 @@ def _run_required(model: str, term: dict, data: dict):
                     input_termType_allowed=input_termType_allowed)
         # logging this for the model would cause issues parsing statuses
         if model != 'emissionNotRelevant':
-            debugValues(data, model=model, term=term.get('@id'), run_required=run_required)
+            debugValues(data, model=model, term=term_id, run_required=run_required)
     return run_required
 
 
@@ -174,10 +176,9 @@ def is_run_required(model: str, term_id: str, node: dict):
     bool
         True if the model is required to run.
     """
-    term = download_hestia(term_id)
     return (
-        (_run_model_required(model, term, node) if model else True) and _run_required(model, term, node)
-    ) if term else True
+        (_run_model_required(model, term_id, node) if model else True) and _run_required(model, term_id, node)
+    ) if term_id else True
 
 
 def run_if_required(model: str, term_id: str, data: dict, module):
@@ -1279,7 +1280,7 @@ def get_inputs_from_properties(input: dict, term_types: Union[TermTermType, List
     properties = (
         input.get('properties') or
         input.get('term', {}).get('defaultProperties') or
-        download_hestia(input.get('term', {}).get('@id')).get('defaultProperties')
+        download_term(input.get('term', {})).get('defaultProperties')
     )
     inputs = non_empty_list([
         {
@@ -1559,7 +1560,6 @@ def convert_unit_properties(node_value: Union[int, float], node: dict, dest_unit
     Convert a number `node_value` belonging to a term `node`, to unit `to_units` by chaining multiple unit conversions
     together.
     Uses terms properties for the conversion.
-    Uses cached calls to download_hestia() internally for speedup
     Returns None if no conversion possible.
     """
     src_unit = node.get("units") or node.get('term', {}).get('units', "")

hestia_earth/models/utils/completeness.py

@@ -1,6 +1,7 @@
 from typing import Union
 from hestia_earth.schema import Completeness, TermTermType
-from hestia_earth.utils.api import download_hestia
+
+from .term import download_term
 
 completeness_fields = Completeness().required
 
@@ -12,7 +13,7 @@ def _completeness_term_type(cycle: dict, term: Union[str, dict, TermTermType]):
 
 
 def _get_term_type_completeness(cycle: dict, term: Union[str, dict]):
-    term = download_hestia(term) if isinstance(term, str) else term
+    term = download_term(term) if isinstance(term, str) else term
     term_type = term.get('termType') if term else None
     return cycle.get('completeness', {}).get(term_type, False)
 

hestia_earth/models/utils/cycle.py

@@ -1,9 +1,10 @@
 from hestia_earth.schema import CycleFunctionalUnit, SiteSiteType, TermTermType, AnimalReferencePeriod
 from hestia_earth.utils.model import filter_list_term_type, find_term_match, find_primary_product
 from hestia_earth.utils.tools import list_sum, safe_parse_float, safe_parse_date, non_empty_list
+from hestia_earth.utils.lookup_utils import is_siteType_allowed
 
 from ..log import logRequirements, debugValues
-from .lookup import all_factor_value, is_siteType_allowed
+from .lookup import all_factor_value
 from .term import get_lookup_value
 from .property import get_node_property
 from .completeness import _is_term_type_complete
@@ -344,7 +345,7 @@ def is_organic(cycle: dict):
         `True` if the `Cycle` is organic, `False` otherwise.
     """
     practices = filter_list_term_type(cycle.get('practices', []), TermTermType.STANDARDSLABELS)
-    return any([get_lookup_value(p.get('term', {}), 'isOrganic') == 'organic' for p in practices])
+    return next((get_lookup_value(p.get('term', {}), 'isOrganic') == 'organic' for p in practices), False)
 
 
 def is_irrigated(cycle: dict, **log_ars):
@@ -463,9 +464,9 @@ def get_animals_by_period(cycle: dict, period: AnimalReferencePeriod = AnimalRef
     ]
 
 
-def get_allowed_sites(model: str, term_id: str, termType: TermTermType, cycle: dict):
+def get_allowed_sites(model: str, term_id: str, cycle: dict):
     sites = non_empty_list([cycle.get('site', None)]) + cycle.get('otherSites', [])
-    allowed_sites = [s for s in sites if is_siteType_allowed(s, {'@id': term_id, 'termType': termType.value})]
+    allowed_sites = [s for s in sites if is_siteType_allowed(s, term_id)]
     allowed_site_ids = non_empty_list([s.get('@id', s.get('id')) for s in allowed_sites])
     debugValues(cycle, model=model, term=term_id, site_ids=';'.join(allowed_site_ids))
     return allowed_sites

hestia_earth/models/utils/emission.py

@@ -1,22 +1,22 @@
 from collections.abc import Iterable
 from typing import Optional, Union
 from hestia_earth.schema import EmissionMethodTier, SchemaType, TermTermType
-from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import linked_node
 
 
-from . import _term_id, _include_methodModel, flatten_args
+from . import flatten_args
+from .term import download_term
 from .blank_node import find_terms_value
+from .method import include_methodModel
 from .constant import Units, get_atomic_conversion
 
-
 EMISSION_METHOD_TIERS = [e.value for e in EmissionMethodTier]
 
 
 def _new_emission(term, model=None):
     node = {'@type': SchemaType.EMISSION.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_hestia(_term_id(term)))
-    return _include_methodModel(node, model)
+    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term, TermTermType.EMISSION))
+    return include_methodModel(node, model)
 
 
 def get_nh3_no3_nox_to_n(cycle: dict, nh3_term_id: str, no3_term_id: str, nox_term_id: str, allow_none: bool = False):

hestia_earth/models/utils/feedipedia.py

@@ -1,5 +1,5 @@
 from hestia_earth.utils.model import find_term_match
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
+from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, lookup_term_ids
 from hestia_earth.utils.tools import non_empty_list, safe_parse_float
 
 from hestia_earth.models.log import logShouldRun
@@ -11,9 +11,9 @@ DRY_MATTER_TERM_ID = 'dryMatter'
 
 def get_feedipedia_properties():
     lookup = download_lookup('property.csv')
-    term_ids = list(lookup.termid)
     term_ids = [
-        term_id for term_id in term_ids if get_table_value(lookup, 'termid', term_id, column_name('feedipediaName'))
+        term_id for term_id in lookup_term_ids(lookup)
+        if get_table_value(lookup, 'termid', term_id, column_name('feedipediaName'))
     ]
     return term_ids
 
@@ -25,7 +25,6 @@ def _should_rescale_by_dm(property_id: str):
 
 
 def _dm_property(term_id: str, property_values: dict, dm_property_values: dict, dry_matter_property: dict):
-    blank_node = _new_property(term_id)
     blank_node_data = {}
     for property_key in property_values.keys():
         new_dm_value = safe_parse_float(dry_matter_property.get(property_key))
@@ -37,7 +36,7 @@ def _dm_property(term_id: str, property_values: dict, dm_property_values: dict,
                 2
             ) if _should_rescale_by_dm(term_id) else old_property_value
             blank_node_data[property_key] = new_value
-    return (blank_node | blank_node_data) if blank_node_data else None
+    return (_new_property(term_id) | blank_node_data) if blank_node_data else None
 
 
 def _map_properties(lookup, term_id: str, column_prefix: str):
@@ -74,7 +73,8 @@ def rescale_properties_from_dryMatter(model: str, node: dict, blank_nodes: list)
         new_properties = non_empty_list([
             exec_property(blank_node, p, dry_matter_property) for p in properties if all([
                 not find_term_match(all_properties, p),
-                p != DRY_MATTER_TERM_ID
+                p != DRY_MATTER_TERM_ID,
+                dry_matter_property
             ])
         ])
         for prop in new_properties:

hestia_earth/models/utils/impact_assessment.py

@@ -1,6 +1,5 @@
 from typing import Optional
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.lookup import download_lookup
 from hestia_earth.utils.model import find_term_match, filter_list_term_type
 from hestia_earth.utils.tools import list_sum, safe_parse_date
 
@@ -161,7 +160,7 @@ def impact_country_value(
     nodes = filter_list_term_type(impact.get('emissionsResourceUse', []), term_type)
 
     country_id = get_country_id(impact)
-    country_id = fallback_country(country_id, [download_lookup(lookup)]) if country_fallback else country_id
+    country_id = fallback_country(country_id, [lookup]) if country_fallback else country_id
 
     values = list(map(_term_factor_value(model, term_id, lookup, country_id, group_key), nodes))
     debugValues(impact, model=model, term=term_id,

hestia_earth/models/utils/indicator.py

@@ -1,15 +1,17 @@
-from hestia_earth.schema import SchemaType
-from hestia_earth.utils.api import download_hestia
+from hestia_earth.schema import SchemaType, TermTermType
 from hestia_earth.utils.model import linked_node
 
-from . import _term_id, _include_methodModel
+from .method import include_methodModel
+from .term import download_term
 
 
 def _new_indicator(term, model=None, land_cover_id: str = None, previous_land_cover_id: str = None):
     node = {'@type': SchemaType.INDICATOR.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_hestia(_term_id(term)))
+    node['term'] = linked_node(term if isinstance(term, dict) else download_term(
+        term, TermTermType.CHARACTERISEDINDICATOR)
+    )
     if land_cover_id:
-        node['landCover'] = linked_node(download_hestia(land_cover_id))
+        node['landCover'] = linked_node(download_term(land_cover_id, TermTermType.LANDCOVER))
     if previous_land_cover_id:
-        node['previousLandCover'] = linked_node(download_hestia(previous_land_cover_id))
-    return _include_methodModel(node, model)
+        node['previousLandCover'] = linked_node(download_term(previous_land_cover_id, TermTermType.LANDCOVER))
+    return include_methodModel(node, model)

hestia_earth/models/utils/inorganicFertiliser.py

@@ -1,18 +1,18 @@
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.lookup import column_name, download_lookup, get_table_value, extract_grouped_data
+from hestia_earth.utils.lookup import download_lookup, extract_grouped_data, lookup_term_ids
 from hestia_earth.utils.tools import safe_parse_float
 from hestia_earth.utils.model import filter_list_term_type
 
-from ..log import debugMissingLookup
 from .term import get_lookup_value
 from .fertiliser import get_fertilisers_from_inputs
+from .lookup import get_region_lookup_value
 
 BREAKDOWN_LOOKUP = 'region-inorganicFertiliser-fertGroupingNitrogen-breakdown.csv'
 
 
 def get_terms():
     lookup = download_lookup('inorganicFertiliser.csv', True)
-    return list(lookup.termid)
+    return lookup_term_ids(lookup)
 
 
 def get_term_lookup(term_id: str, col_name: str):
@@ -35,9 +35,7 @@ def get_NH3_emission_factor(term_id: str, soilPh: float, temperature: float):
 
 
 def get_country_breakdown(model: str, term_id: str, country_id: str, col_name: str):
-    lookup = download_lookup(BREAKDOWN_LOOKUP)
-    value = get_table_value(lookup, 'termid', country_id, column_name(col_name))
-    debugMissingLookup(BREAKDOWN_LOOKUP, 'termid', country_id, col_name, value, model=model, term=term_id)
+    value = get_region_lookup_value(BREAKDOWN_LOOKUP, country_id, col_name, model=model, term=term_id)
     return safe_parse_float(value, None)
 
 

hestia_earth/models/utils/input.py

@@ -1,19 +1,20 @@
 from hestia_earth.schema import SchemaType, TermTermType
-from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import find_term_match, linked_node, filter_list_term_type
 from hestia_earth.utils.tools import list_sum, non_empty_list, list_average, flatten
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
 
-from ..log import logger
-from . import _term_id, _include_model, _filter_list_term_unit, _load_calculated_node
+from hestia_earth.models.log import logger
+from . import _filter_list_term_unit, _load_calculated_node
 from .constant import Units
 from .blank_node import get_total_value, get_total_value_converted, get_lookup_value
+from .term import download_term
+from .method import include_model
 
 
 def _new_input(term, model=None):
     node = {'@type': SchemaType.INPUT.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_hestia(_term_id(term)))
-    return _include_model(node, model)
+    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term))
+    return include_model(node, model)
 
 
 def load_impacts(inputs: list):

hestia_earth/models/utils/lookup.py

@@ -1,10 +1,13 @@
 from typing import Optional, List
-from numpy import recarray
-from hestia_earth.schema import SchemaType
 from hestia_earth.utils.lookup import (
-    download_lookup, get_table_value, column_name, extract_grouped_data, _get_single_table_value
+    download_lookup,
+    get_table_value,
+    column_name,
+    extract_grouped_data,
+    _get_single_table_value,
+    lookup_term_ids
 )
-from hestia_earth.utils.tools import list_sum, safe_parse_float, non_empty_list
+from hestia_earth.utils.tools import list_sum, safe_parse_float
 
 from ..log import debugValues, log_as_table, debugMissingLookup
 
@@ -15,13 +18,11 @@ def _node_value(node):
 
 
 def _factor_value(model: str, term_id: str, lookup_name: str, lookup_col: str, grouped_key: Optional[str] = None):
-    lookup = download_lookup(lookup_name)
-
     def get_value(data: dict):
         node_term_id = data.get('term', {}).get('@id')
         grouped_data_key = grouped_key or data.get('methodModel', {}).get('@id')
         value = _node_value(data)
-        coefficient = get_table_value(lookup, 'termid', node_term_id, column_name(lookup_col))
+        coefficient = get_region_lookup_value(lookup_name, node_term_id, lookup_col, model=model, term=term_id)
         # value is either a number or matching between a model and a value (restrict value to specific model only)
         coefficient = safe_parse_float(
             extract_grouped_data(coefficient, grouped_data_key), None
@@ -69,12 +70,10 @@ def all_factor_value(
 
 
 def _term_factor_value(model: str, term_id: str, lookup_name: str, lookup_term_id: str, group_key: str = None):
-    lookup = download_lookup(lookup_name, False)  # avoid saving in memory as there could be many different files used
-
     def get_value(data: dict):
         node_term_id = data.get('term', {}).get('@id')
         value = _node_value(data)
-        coefficient = get_table_value(lookup, 'termid', lookup_term_id, column_name(node_term_id))
+        coefficient = get_region_lookup_value(lookup_name, lookup_term_id, node_term_id, model=model, term=term_id)
         coefficient = safe_parse_float(extract_grouped_data(coefficient, group_key) if group_key else coefficient)
         if value is not None and coefficient is not None:
             debugValues(data, model=model, term=term_id,
@@ -107,98 +106,31 @@ def _aware_factor_value(model: str, term_id: str, lookup_name: str, aware_id: st
     return get_value
 
 
-_ALLOW_ALL = 'all'
-
-
-def _is_site(site: dict):
-    return site.get('@type', site.get('type')) == SchemaType.SITE.value if site else None
-
-
-def _get_sites(node: dict):
-    site = node.get('site', node.get('cycle', {}).get('site'))
-    other_sites = node.get('otherSites', node.get('cycle', {}).get('otherSites', []))
-    return non_empty_list([site] + other_sites)
-
-
-def _get_site_types(node: dict):
-    sites = [node] if _is_site(node) else _get_sites(node)
-    return non_empty_list([site.get('siteType') for site in sites])
-
-
-def _model_lookup_values(model: str, term: dict, restriction: str):
-    lookup = download_lookup(f"{term.get('termType')}-model-{restriction}.csv")
-    values = get_table_value(lookup, 'termid', term.get('@id'), column_name(model))
-    return (values or _ALLOW_ALL).split(';') if isinstance(values, str) else _ALLOW_ALL
-
-
-def is_model_siteType_allowed(model: str, term: dict, data: dict):
-    site_types = _get_site_types(data)
-    allowed_values = _model_lookup_values(model, term, 'siteTypesAllowed')
-    return True if _ALLOW_ALL in allowed_values or not site_types else any([
-        (site_type in allowed_values) for site_type in site_types
-    ])
-
-
-def _lookup_values(term: dict, column: str):
-    lookup = download_lookup(f"{term.get('termType')}.csv")
-    values = get_table_value(lookup, 'termid', term.get('@id'), column_name(column))
-    return (values or _ALLOW_ALL).split(';') if isinstance(values, str) else _ALLOW_ALL
-
-
-def is_siteType_allowed(data: dict, term: dict):
-    site_types = _get_site_types(data)
-    allowed_values = _lookup_values(term, 'siteTypesAllowed')
-    return True if _ALLOW_ALL in allowed_values or not site_types else any([
-        (site_type in allowed_values) for site_type in site_types
-    ])
-
-
-def is_product_termType_allowed(data: dict, term: dict):
-    products = data.get('products', [])
-    values = non_empty_list([p.get('term', {}).get('termType') for p in products])
-    allowed_values = _lookup_values(term, 'productTermTypesAllowed')
-    return True if any([
-        _ALLOW_ALL in allowed_values,
-        len(values) == 0
-    ]) else any([value in allowed_values for value in values])
-
-
-def is_product_id_allowed(data: dict, term: dict):
-    products = data.get('products', [])
-    values = non_empty_list([p.get('term', {}).get('@id') for p in products])
-    allowed_values = _lookup_values(term, 'productTermIdsAllowed')
-    return True if any([
-        _ALLOW_ALL in allowed_values,
-        len(values) == 0
-    ]) else any([value in allowed_values for value in values])
-
-
-def is_input_termType_allowed(data: dict, term: dict):
-    inputs = data.get('inputs', [])
-    values = non_empty_list([p.get('term', {}).get('termType') for p in inputs])
-    allowed_values = _lookup_values(term, 'inputTermTypesAllowed')
-    return True if any([
-        _ALLOW_ALL in allowed_values,
-        len(values) == 0
-    ]) else any([value in allowed_values for value in values])
-
-
-def is_input_id_allowed(data: dict, term: dict):
-    inputs = data.get('inputs', [])
-    values = non_empty_list([p.get('term', {}).get('@id') for p in inputs])
-    allowed_values = _lookup_values(term, 'inputTermIdsAllowed')
-    return True if any([
-        _ALLOW_ALL in allowed_values,
-        len(values) == 0
-    ]) else any([value in allowed_values for value in values])
+def _country_in_lookup(country_id: str):
+    def in_lookup(lookup_name: str):
+        return (
+            download_lookup(lookup_name.replace('region', country_id)) is not None or
+            country_id in lookup_term_ids(download_lookup(lookup_name))
+        )
+    return in_lookup
 
 
-def fallback_country(country_id: str, lookup_arrays: List[recarray]) -> str:
+def fallback_country(country_id: str, lookups: List[str]) -> str:
     """
-    Given a site dict with 'country_id' location term, and lookup table,
-    checks if a location can be used in lookup file
-    else fallback to the default "region-world"
+    Given a country `@id`, and lookup tables, checks if a location can be used in lookup file
+    else fallback to the default "region-world".
     """
-    is_in_lookup = lambda v: all(v in array['termid'] for array in lookup_arrays)  # noqa: E731
+    is_in_lookup = lambda v: all(map(_country_in_lookup(v), lookups))  # noqa: E731
     fallback_id = 'region-world'
-    return country_id if is_in_lookup(country_id) else fallback_id if is_in_lookup(fallback_id) else None
+    return country_id if country_id and is_in_lookup(country_id) else fallback_id if is_in_lookup(fallback_id) else None
+
+
+def get_region_lookup_value(lookup_name: str, term_id: str, column: str, **log_args):
+    # for performance, try to load the region specific lookup if exists
+    lookup = (
+        download_lookup(lookup_name.replace('region-', f"{term_id}-"))
+        if lookup_name and lookup_name.startswith('region-') else None
+    ) or download_lookup(lookup_name)
+    value = get_table_value(lookup, 'termid', term_id, column_name(column))
+    debugMissingLookup(lookup_name, 'termid', term_id, column, value, **log_args)
+    return value

hestia_earth/models/utils/management.py

@@ -1,11 +1,11 @@
 from hestia_earth.schema import SchemaType
 from hestia_earth.utils.model import linked_node
-from hestia_earth.utils.api import download_hestia
 
-from . import _term_id, _include_model
+from .term import download_term
+from .method import include_model
 
 
 def _new_management(term, model=None):
     node = {'@type': SchemaType.MANAGEMENT.value}
-    node['term'] = linked_node(term if isinstance(term, dict) else download_hestia(_term_id(term)))
-    return _include_model(node, model)
+    node['term'] = linked_node(term if isinstance(term, dict) else download_term(term))
+    return include_model(node, model)