hestia-earth-models 0.74.3__py3-none-any.whl → 0.74.5__py3-none-any.whl

hestia_earth/models/pooreNemecek2018/landOccupationDuringCycle.py

@@ -8,18 +8,17 @@ from . import MODEL
 
 REQUIREMENTS = {
     "ImpactAssessment": {
-        "product": {"@type": "Term"},
+        "product": {
+            "@type": "Term",
+            "economicValueShare": "> 0",
+            "value": "> 0"
+        },
         "cycle": {
             "@type": "Cycle",
             "or": [
                 {
                     "@doc": "if the [cycle.functionalUnit](https://hestia.earth/schema/Cycle#functionalUnit) = 1 ha, additional properties are required",  # noqa: E501
                     "cycleDuration": "",
-                    "products": [{
-                        "@type": "Product",
-                        "value": "> 0",
-                        "economicValueShare": "> 0"
-                    }],
                     "practices": [{"@type": "Practice", "value": "", "term.@id": "longFallowRatio"}]
                 },
                 {
@@ -54,6 +53,7 @@ def _indicator(term_id: str, value: float, land_cover_term_id: str):
 
 def _should_run(impact_assessment: dict):
     product = get_product(impact_assessment)
+    product_evs = product.get('economicValueShare')
     cycle = impact_assessment.get('cycle', {})
     site = get_site(impact_assessment)
     land_cover_from_product = get_landCover_term_id(product.get('term', {}))
@@ -62,11 +62,12 @@ def _should_run(impact_assessment: dict):
     land_occupation_m2_kg = land_occupation_per_kg(MODEL, TERM_ID, cycle, site, product)
 
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
+                    economicValueShare=product_evs,
                     land_occupation_kg=land_occupation_m2_kg,
                     land_cover_from_product=land_cover_from_product,
                     land_cover_from_site=land_cover_from_site)
 
-    should_run = all([land_cover_term_id, land_occupation_m2_kg is not None])
+    should_run = all([product_evs, land_cover_term_id, land_occupation_m2_kg is not None])
     logShouldRun(impact_assessment, MODEL, TERM_ID, should_run)
     return should_run, land_occupation_m2_kg, land_cover_term_id
 

hestia_earth/models/poschEtAl2008/terrestrialAcidificationPotentialAccumulatedExceedance.py

@@ -36,7 +36,7 @@ def _indicator(value: float):
 
 def run(impact_assessment: dict):
     value = impact_country_value(MODEL, TERM_ID, impact_assessment, f"{list(LOOKUPS.keys())[0]}.csv",
-                                 country_fallback=True)
+                                 default_world_value=True)
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     value=value)
     logShouldRun(impact_assessment, MODEL, TERM_ID, value is not None)

hestia_earth/models/poschEtAl2008/terrestrialEutrophicationPotentialAccumulatedExceedance.py

@@ -36,7 +36,7 @@ def _indicator(value: float):
 
 def run(impact_assessment: dict):
     value = impact_country_value(MODEL, TERM_ID, impact_assessment, f"{list(LOOKUPS.keys())[0]}.csv",
-                                 country_fallback=True)
+                                 default_world_value=True)
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
                     value=value)
     logShouldRun(impact_assessment, MODEL, TERM_ID, value is not None)

hestia_earth/models/preload_requests.py

@@ -7,12 +7,13 @@ from hestia_earth.utils.storage import _load_from_storage
 
 from .log import logger
 from .mocking.build_mock_search import create_search_results
-from .mocking import RESULTS_PATH, enable_mock as _mock
+from .mocking.mock_search import unmock
+from .mocking import RESULTS_PATH, enable_mock
 
 
 def _write_results(filepath: str, data: dict):
     with open(filepath, 'w') as f:
-        f.write(json.dumps(data, indent=2, ensure_ascii=False))
+        f.write(json.dumps(data, ensure_ascii=False))
 
 
 def _load_data_from_glossary():
@@ -26,7 +27,8 @@ def enable_preload(
     filepath: str = RESULTS_PATH,
     node: dict = None,
     overwrite_existing: bool = True,
-    use_glossary: bool = False
+    use_glossary: bool = False,
+    keep_in_memory: bool = False
 ):
     """
     Prefetch calls to HESTIA API in a local file.
@@ -43,6 +45,9 @@ def enable_preload(
     use_glossary : bool
         Optional - Try to fetch search results from the glossary.
         Only available with access to HESTIA infrastructure.
+    keep_in_memory : bool
+        Optional - keep the results file in memory, to avoid reading it from disk every request.
+        This increases the memory usage of the function, but reduces latency reading the file.
     """
     should_generate = overwrite_existing or not os.path.exists(filepath)
 
@@ -56,4 +61,13 @@ def enable_preload(
         _write_results(filepath, data)
 
     # enable mock search results from file
-    _mock(filepath=filepath, node=node)
+    enable_mock(filepath=filepath, node=node, keep_in_memory=keep_in_memory)
+
+
+def disable_preload():
+    """
+    Disable preloading API calls done with the `enable_preload` function.
+    All API calls will be done as usual after this function is called.
+    """
+    logger.debug('Removing mock of search results')
+    unmock()

hestia_earth/models/schmidt2007/utils.py

@@ -10,9 +10,9 @@ from . import MODEL
 def get_waste_values(term_id: str, cycle: dict, lookup_col: str):
     products = filter_list_term_type(cycle.get('products', []), TermTermType.WASTE)
     value = all_factor_value(
-        logs_model=MODEL,
-        logs_term_id=term_id,
-        node=cycle,
+        log_model=MODEL,
+        log_term_id=term_id,
+        log_node=cycle,
         lookup_name=f"{TermTermType.WASTE.value}.csv",
         lookup_col=lookup_col,
         blank_nodes=products,

hestia_earth/models/utils/__init__.py

@@ -23,7 +23,10 @@ CACHE_KEY = '_cache'
 
 def cached_value(node: dict, key: str = None, default=None):
     cache = node.get(CACHE_KEY, {})
-    return cache.get(key, default) if key else cache
+    return cache.get(key, default) if all([
+        key,
+        cache is not None
+    ]) else cache
 
 
 def group_by(values: list, keys: list):

hestia_earth/models/utils/aggregated.py

@@ -1,13 +1,12 @@
 import math
-from hestia_earth.schema import SchemaType
-from hestia_earth.utils.api import search
+from hestia_earth.schema import NodeType
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
 from hestia_earth.utils.model import find_term_match, linked_node
 from hestia_earth.utils.tools import safe_parse_date, non_empty_list
 
 from hestia_earth.models.log import debugValues, logShouldRun
+from hestia_earth.models.data.hestiaAggregatedData import DEFAULT_COUNTRY_ID, find_closest_impact_id
 from . import current_year
-from .cycle import is_organic
 
 MODEL_KEY = 'impactAssessment'
 
@@ -17,84 +16,38 @@ def aggregated_end_date(end_date: str):
     return min([round(math.floor(year / 10) * 10) + 9, current_year()])
 
 
-def _match_country_query(name: str = 'World', boost: int = 1):
-    return {'match': {'country.name.keyword': {'query': name, 'boost': boost}}}
-
-
-def _match_country(country: dict):
-    country_name = country.get('name') if country else None
-    return {
-        'bool': {
-            # either get with exact country, or default to global
-            'should': non_empty_list([
-                _match_country_query(name=country_name, boost=1000) if country_name else None,
-                _match_country_query()
-            ]),
-            'minimum_should_match': 1
-        }
-    }
-
-
-def find_closest_impact(cycle: dict, end_date: str, term: dict, country: dict, must_queries=[]):
-    query = {
-        'bool': {
-            'must': non_empty_list([
-                {'match': {'@type': SchemaType.IMPACTASSESSMENT.value}},
-                {'match': {'aggregated': 'true'}},
-                {'match': {'aggregatedDataValidated': 'true'}},
-                {
-                    'bool': {
-                        # handle old ImpactAssessment data
-                        'should': [
-                            {'match': {'product.term.name.keyword': term.get('name')}},
-                            {'match': {'product.name.keyword': term.get('name')}}
-                        ],
-                        'minimum_should_match': 1
-                    }
-                } if term else None,
-                _match_country(country),
-                {'range': {'endDate': {'lte': f"{end_date}-12-31"}}}
-            ]) + must_queries,
-            'should': [
-                # if the Cycle is organic, we can try to match organic aggregate first
-                {'match': {'name': {'query': 'Organic' if is_organic(cycle) else 'Conventional', 'boost': 1000}}}
-            ]
-        }
-    }
-    results = search(query, fields=['@type', '@id', 'name', 'endDate']) if term else []
-    # sort by distance to date and score and take min
-    results = sorted(
-        results,
-        key=lambda v: abs(int(end_date) - int(v.get('endDate', '0'))) * v.get('_score', 0),
-    )
-    return results[0] if len(results) > 0 else None
-
-
-def _link_input_to_impact(model: str, cycle: dict, date: int):
+def _link_input_to_impact(model: str, cycle: dict, date: int, **log_args):
     def run(input: dict):
         term = input.get('term', {})
         term_id = term.get('@id')
         country = input.get('country')
-        impact = find_closest_impact(cycle, date, term, country)
-
-        search_by_country_id = (country or {}).get('@id') or 'region-world'
-        debugValues(cycle, model=model, term=term_id, key=MODEL_KEY,
+        country_id = (country or {}).get('@id')
+        impact_id = (
+            find_closest_impact_id(product_id=term_id, country_id=country_id, year=date) or
+            find_closest_impact_id(product_id=term_id, country_id=DEFAULT_COUNTRY_ID, year=date)
+        )
+
+        search_by_country_id = country_id or DEFAULT_COUNTRY_ID
+        debugValues(cycle, model=model, term=term_id, key=MODEL_KEY, **log_args,
                     search_by_input_term_id=term_id,
                     search_by_country_id=search_by_country_id,
                     search_by_end_date=str(date),
-                    impact_assessment_id_found=(impact or {}).get('@id'))
+                    impact_assessment_id_found=impact_id)
 
-        should_run = all([impact is not None])
-        logShouldRun(cycle, model, term_id, should_run)
-        logShouldRun(cycle, model, term_id, should_run, key=MODEL_KEY)  # show specifically under Input
+        should_run = all([impact_id])
+        logShouldRun(cycle, model, term_id, should_run, **log_args)
+        logShouldRun(cycle, model, term_id, should_run, key=MODEL_KEY, **log_args)  # show specifically under Input
 
-        return input | {MODEL_KEY: linked_node(impact), 'impactAssessmentIsProxy': True} if impact else None
+        return input | {
+            MODEL_KEY: linked_node({'@type': NodeType.IMPACTASSESSMENT.value, '@id': impact_id}),
+            'impactAssessmentIsProxy': True
+        } if impact_id else None
     return run
 
 
-def link_inputs_to_impact(model: str, cycle: dict, inputs: list):
+def link_inputs_to_impact(model: str, cycle: dict, inputs: list, **log_args):
     date = aggregated_end_date(cycle.get('endDate'))
-    return non_empty_list(map(_link_input_to_impact(model, cycle, date), inputs))
+    return non_empty_list(map(_link_input_to_impact(model, cycle, date, **log_args), inputs))
 
 
 def _should_aggregate_input(term: dict):

hestia_earth/models/utils/cycle.py

@@ -409,9 +409,9 @@ def impact_lookup_value(model: str, term_id: str, cycle: dict, blank_nodes: list
     """
     term_type = blank_nodes[0].get('term', {}).get('termType') if len(blank_nodes) > 0 else None
     return all_factor_value(
-        logs_model=model,
-        logs_term_id=term_id,
-        node=cycle,
+        log_model=model,
+        log_term_id=term_id,
+        log_node=cycle,
         lookup_name=f"{term_type}.csv",
         lookup_col=lookup_col,
         blank_nodes=blank_nodes

hestia_earth/models/utils/impact_assessment.py

@@ -3,8 +3,7 @@ from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum, safe_parse_date
 
-from hestia_earth.models.log import debugValues
-from .lookup import all_factor_value, region_factor_value, aware_factor_value, fallback_country
+from .lookup import all_factor_value, region_factor_value, aware_factor_value
 from .product import find_by_product
 from .site import region_level_1_id
 
@@ -106,8 +105,12 @@ def get_country_id(impact_assessment: dict, blank_node: dict = None) -> str:
     return term_id if term_id else None
 
 
+def get_emissionsResourceUse(impact_assessment: dict):
+    return impact_assessment.get('cache_emissionsResourceUse', []) or impact_assessment.get('emissionsResourceUse', [])
+
+
 def impact_emission_lookup_value(
-    model: str, term_id: str, impact: dict, lookup_col: str, grouped_key: Optional[str] = None
+    model: str, term_id: str, impact: dict, lookup_col: str, group_key: Optional[str] = None
 ) -> float:
     """
     Calculate the value of the impact based on lookup factors and emissions value.
@@ -122,7 +125,7 @@ def impact_emission_lookup_value(
         The `ImpactAssessment`.
     lookup_col : str
         The lookup column to fetch the factors from.
-    grouped_key : str
+    group_key : str
         key of grouped data to extract in a lookup table
 
     Returns
@@ -130,25 +133,29 @@ def impact_emission_lookup_value(
     int
         The impact total value.
     """
+    # use cache version which is grouped
+    blank_nodes = get_emissionsResourceUse(impact)
+    blank_nodes = filter_list_term_type(blank_nodes, TermTermType.EMISSION)
+
     return all_factor_value(
-        logs_model=model,
-        logs_term_id=term_id,
-        node=impact,
+        log_model=model,
+        log_term_id=term_id,
+        log_node=impact,
         lookup_name='emission.csv',
         lookup_col=lookup_col,
-        blank_nodes=filter_list_term_type(impact.get('emissionsResourceUse', []), TermTermType.EMISSION),
-        grouped_key=grouped_key,
+        blank_nodes=blank_nodes,
+        group_key=group_key,
         default_no_values=None
     )
 
 
 def impact_country_value(
-    logs_model: str,
-    logs_term_id: str,
+    log_model: str,
+    log_term_id: str,
     impact: dict,
     lookup: str,
     group_key: str = None,
-    country_fallback: bool = False,
+    default_world_value: bool = False,
     default_no_values=None
 ) -> float:
     """
@@ -156,9 +163,9 @@ def impact_country_value(
 
     Parameters
     ----------
-    logs_model : str
+    log_model : str
         The model to display in the logs only.
-    logs_term_id : str
+    log_term_id : str
         The term to display in the logs only.
     impact : dict
         The `ImpactAssessment`.
@@ -166,31 +173,32 @@ def impact_country_value(
         The name of the lookup to fetch the factors from.
     group_key : str
         Optional: key to use if the data is a group of values.
-    country_fallback : bool
-        Optional: if True fallback to default `region-world` country_id if country_id in `ImpactAssessment` not found in
-        lookup file containing factors.
+    default_world_value : bool
+        Optional: when True, if the value is not found for the country, try using World value instead.
 
     Returns
     -------
     int
         The impact total value.
     """
+    # use cache version which is grouped
+    blank_nodes = get_emissionsResourceUse(impact)
     term_type = TermTermType.RESOURCEUSE.value if 'resourceUse' in lookup else TermTermType.EMISSION.value
-    blank_nodes = filter_list_term_type(impact.get('emissionsResourceUse', []), term_type)
+    blank_nodes = filter_list_term_type(blank_nodes, term_type)
 
     country_id = get_country_id(impact)
-    country_id = fallback_country(country_id, [lookup]) if country_fallback else country_id
 
     return all_factor_value(
-        logs_model=logs_model,
-        logs_term_id=logs_term_id,
-        node=impact,
+        log_model=log_model,
+        log_term_id=log_term_id,
+        log_node=impact,
         lookup_name=lookup,
         lookup_col=country_id,
         blank_nodes=blank_nodes,
-        grouped_key=group_key,
+        group_key=group_key,
         default_no_values=default_no_values,
-        factor_value_func=region_factor_value
+        factor_value_func=region_factor_value,
+        default_world_value=default_world_value
     )
 
 
@@ -216,18 +224,18 @@ def impact_aware_value(model: str, term_id: str, impact: dict, lookup: str, grou
     int
         The impact total value.
     """
-    blank_nodes = impact.get('emissionsResourceUse', [])
-    site = get_site(impact)
-    aware_id = site.get('awareWaterBasinId')
+    # use cache version which is grouped
+    blank_nodes = get_emissionsResourceUse(impact)
+    aware_id = get_site(impact).get('awareWaterBasinId')
 
     return None if aware_id is None else all_factor_value(
-        logs_model=model,
-        logs_term_id=term_id,
-        node=impact,
+        log_model=model,
+        log_term_id=term_id,
+        log_node=impact,
         lookup_name=lookup,
         lookup_col=aware_id,
         blank_nodes=blank_nodes,
-        grouped_key=group_key,
+        group_key=group_key,
         default_no_values=None,
         factor_value_func=aware_factor_value
     )
@@ -253,17 +261,18 @@ def impact_endpoint_value(model: str, term_id: str, impact: dict, lookup_col: st
     int
         The impact total value.
     """
-    nodes = [i for i in impact.get('impacts', []) if (
+    blank_nodes = impact.get('impacts', [])
+    blank_nodes = [i for i in blank_nodes if (
         i.get('methodModel').get('@id') == model or
         not i.get('methodModel').get('@id').startswith(model[0:6])  # allow other non-related models to be accounted for
     )]
     return all_factor_value(
-        logs_model=model,
-        logs_term_id=term_id,
-        node=impact,
+        log_model=model,
+        log_term_id=term_id,
+        log_node=impact,
         lookup_name='characterisedIndicator.csv',
         lookup_col=lookup_col,
-        blank_nodes=nodes,
+        blank_nodes=blank_nodes,
         default_no_values=None
     )
 
@@ -273,11 +282,6 @@ def convert_value_from_cycle(
 ):
     pyield = list_sum(product.get('value', [])) if product else 0
     economic_value = product.get('economicValueShare') if product else 0
-
-    debugValues(log_node, model=model, term=term_id,
-                product_yield=pyield,
-                economicValueShare=economic_value)
-
     return (value / pyield) * economic_value / 100 if all([
         value is not None, pyield > 0, economic_value
     ]) else default