hestia-earth-models 0.74.17__py3-none-any.whl → 0.75.1__py3-none-any.whl

hestia_earth/models/requirements.py

@@ -4,7 +4,7 @@ from functools import reduce
 from importlib import import_module
 from pydash.objects import merge, merge_with, omit
 from hestia_earth.schema import is_schema_type, is_type_valid, NodeType
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
+from hestia_earth.utils.lookup import download_lookup, get_table_value
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.tools import flatten, non_empty_list
 
@@ -87,7 +87,7 @@ ALL_MODELS = _models(CURRENT_DIR)
 def _lookup_allowed(model_term_id: str, column: str, value: str):
     term = download_hestia(model_term_id)
     lookup = download_lookup(f"{term.get('termType')}.csv")
-    values = get_table_value(lookup, 'termid', model_term_id, column_name(column))
+    values = get_table_value(lookup, 'term.id', model_term_id, column)
     allowed_values = (values or 'all').split(';')
     return 'all' in allowed_values or value in allowed_values
 

hestia_earth/models/utils/aggregated.py

@@ -1,6 +1,6 @@
 import math
 from hestia_earth.schema import NodeType
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
+from hestia_earth.utils.lookup import download_lookup, get_table_value
 from hestia_earth.utils.model import find_term_match, linked_node
 from hestia_earth.utils.tools import safe_parse_date, non_empty_list
 
@@ -52,7 +52,7 @@ def link_inputs_to_impact(model: str, cycle: dict, inputs: list, **log_args):
 
 def _should_not_skip_input(term: dict):
     lookup = download_lookup(f"{term.get('termType')}.csv", True)
-    value = get_table_value(lookup, 'termid', term.get('@id'), column_name('skipLinkedImpactAssessment'))
+    value = get_table_value(lookup, 'term.id', term.get('@id'), 'skipLinkedImpactAssessment')
     return True if value is None or value == '' else not value
 
 

hestia_earth/models/utils/background_emissions.py

@@ -1,7 +1,7 @@
 from functools import reduce
 from typing import Callable, Tuple
 from hestia_earth.schema import TermTermType, EmissionMethodTier
-from hestia_earth.utils.lookup import _is_missing_value, lookup_columns
+from hestia_earth.utils.lookup import is_missing_value, lookup_columns
 from hestia_earth.utils.model import find_term_match, filter_list_term_type
 from hestia_earth.utils.tools import flatten, non_empty_list, safe_parse_float, omit
 from hestia_earth.utils.emission import cycle_emissions_in_system_boundary
@@ -121,19 +121,20 @@ def _values_from_column(index_column: str, column: str, value: str):
         column != index_column,
         not column.startswith('ecoinvent'),
         not column.startswith('ecoalim'),
-        not _is_missing_value(value)
+        not is_missing_value(value)
     ]) else {}
 
 
 def convert_background_lookup(lookup, index_column: str):
     columns = lookup_columns(lookup)
+    indexed_df = lookup.set_index(index_column, drop=False).copy()
     return {
-        row[index_column]: reduce(
-            lambda prev, curr: prev | _values_from_column(index_column, curr, row[curr]),
+        index_key: reduce(
+            lambda prev, curr: prev | _values_from_column(index_column, curr, row_data[curr]),
             columns,
             {}
         )
-        for row in lookup
+        for index_key, row_data in indexed_df.to_dict('index').items()
     }
 
 

hestia_earth/models/utils/ecoClimateZone.py

@@ -1,11 +1,9 @@
 from enum import Enum
 from functools import reduce
 from typing import Any, Optional, Union
-
 from hestia_earth.schema import NodeType
-
 from hestia_earth.utils.blank_node import get_node_value
-from hestia_earth.utils.lookup import download_lookup, _get_single_table_value, column_name, extract_grouped_data
+from hestia_earth.utils.lookup import download_lookup, get_table_value, extract_grouped_data
 from hestia_earth.utils.model import find_term_match
 from hestia_earth.utils.tools import safe_parse_float
 
@@ -84,10 +82,11 @@ def get_ecoClimateZone_lookup_value(eco_climate_zone: str, col_name: str, group_
     """
     try:
         lookup = download_lookup('ecoClimateZone.csv')
-        code = int(str(eco_climate_zone))
-        data = _get_single_table_value(lookup, column_name('ecoClimateZone'), code, column_name(col_name))
+        code = int(eco_climate_zone)
+        data = get_table_value(lookup, 'ecoClimateZone', code, col_name)
         return safe_parse_float(
-            data if group_name is None else extract_grouped_data(data, group_name), default=None
+            data if group_name is None else extract_grouped_data(data, group_name),
+            default=None
         )
     except Exception:
         return 0
@@ -115,8 +114,8 @@ def get_ecoClimateZone_lookup_grouped_value(
     """
     try:
         lookup = download_lookup('ecoClimateZone.csv')
-        code = int(str(eco_climate_zone))
-        data = _get_single_table_value(lookup, column_name('ecoClimateZone'), code, column_name(col_name))
+        code = int(eco_climate_zone)
+        data = get_table_value(lookup, 'ecoClimateZone', code, col_name)
         grouped_data = reduce(
             lambda prev, curr: prev | {curr.split(':')[0]: safe_parse_float(curr.split(':')[1], default=None)},
             data.split(';'),

hestia_earth/models/utils/excretaManagement.py

@@ -1,6 +1,6 @@
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import filter_list_term_type
-from hestia_earth.utils.lookup import column_name, download_lookup, get_table_value
+from hestia_earth.utils.lookup import download_lookup, get_table_value
 from hestia_earth.utils.tools import safe_parse_float, list_sum
 
 from hestia_earth.models.log import debugValues, debugMissingLookup, log_as_table
@@ -17,8 +17,8 @@ def get_lookup_factor(practices: list, lookup_col: str):
 
 def _get_nh3_factor(lookup_name: str, term_id: str, input: dict, **log_args):
     input_term_id = input.get('term', {}).get('@id')
-    value = get_table_value(download_lookup(lookup_name), 'termid', term_id, column_name(input_term_id))
-    debugMissingLookup(lookup_name, 'termid', term_id, input_term_id, value, **log_args)
+    value = get_table_value(download_lookup(lookup_name), 'term.id', term_id, input_term_id)
+    debugMissingLookup(lookup_name, 'term.id', term_id, input_term_id, value, **log_args)
     return safe_parse_float(value, default=None)
 
 

hestia_earth/models/utils/feedipedia.py

@@ -1,5 +1,5 @@
 from hestia_earth.utils.model import find_term_match
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, lookup_term_ids
+from hestia_earth.utils.lookup import download_lookup, get_table_value, lookup_term_ids
 from hestia_earth.utils.tools import non_empty_list, safe_parse_float
 
 from hestia_earth.models.log import logShouldRun
@@ -13,14 +13,14 @@ def get_feedipedia_properties():
     lookup = download_lookup('property.csv')
     term_ids = [
         term_id for term_id in lookup_term_ids(lookup)
-        if get_table_value(lookup, 'termid', term_id, column_name('feedipediaName'))
+        if get_table_value(lookup, 'term.id', term_id, 'feedipediaName')
     ]
     return term_ids
 
 
 def _should_rescale_by_dm(property_id: str):
     lookup = download_lookup('property.csv')
-    value = get_table_value(lookup, 'termid', property_id, column_name('feedipediaConversionEnum'))
+    value = get_table_value(lookup, 'term.id', property_id, 'feedipediaConversionEnum')
     return 'dm' in value
 
 
@@ -40,10 +40,10 @@ def _dm_property(term_id: str, property_values: dict, dm_property_values: dict,
 
 
 def _map_properties(lookup, term_id: str, column_prefix: str):
-    value = get_table_value(lookup, 'termid', term_id, column_name(column_prefix))
-    sd = get_table_value(lookup, 'termid', term_id, column_name(f"{column_prefix}-sd"))
-    min = get_table_value(lookup, 'termid', term_id, column_name(f"{column_prefix}-min"))
-    max = get_table_value(lookup, 'termid', term_id, column_name(f"{column_prefix}-max"))
+    value = get_table_value(lookup, 'term.id', term_id, column_prefix)
+    sd = get_table_value(lookup, 'term.id', term_id, f"{column_prefix}-sd")
+    min = get_table_value(lookup, 'term.id', term_id, f"{column_prefix}-min")
+    max = get_table_value(lookup, 'term.id', term_id, f"{column_prefix}-max")
     return {'value': value, 'sd': sd, 'min': min, 'max': max}
 
 

hestia_earth/models/utils/input.py

@@ -2,7 +2,7 @@ from typing import Union, Optional, List
 from hestia_earth.schema import SchemaType, TermTermType
 from hestia_earth.utils.model import find_term_match, linked_node, filter_list_term_type
 from hestia_earth.utils.tools import list_sum, non_empty_list, list_average, flatten
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
+from hestia_earth.utils.lookup import download_lookup, get_table_value
 from hestia_earth.utils.term import download_term
 
 from hestia_earth.models.log import logger
@@ -106,7 +106,7 @@ def match_lookup_value(input: dict, col_name: str, col_value):
     term_type = input.get('term', {}).get('termType')
     lookup = download_lookup(f"{term_type}.csv")
     term_id = input.get('term', {}).get('@id')
-    return get_table_value(lookup, 'termid', term_id, column_name(col_name)) == col_value
+    return get_table_value(lookup, 'term.id', term_id, col_name) == col_value
 
 
 def get_feed_inputs(cycle: dict):

hestia_earth/models/utils/liveAnimal.py

@@ -1,6 +1,6 @@
 from statistics import mean
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, extract_grouped_data
+from hestia_earth.utils.lookup import download_lookup, get_table_value, extract_grouped_data
 from hestia_earth.utils.model import find_primary_product, filter_list_term_type
 from hestia_earth.utils.tools import safe_parse_float
 
@@ -30,9 +30,9 @@ def get_default_digestibility(model: str, term_id: str, cycle: dict):
 
     for system in systems:
         system_id = system.get('term', {}).get('@id')
-        lookup_col = column_name(product_id)
-        value = get_table_value(lookup, 'termid', system_id, lookup_col)
-        debugMissingLookup(lookup_name, 'termid', term_id, lookup_col, value, model=model, term=term_id)
+        lookup_col = product_id
+        value = get_table_value(lookup, 'term.id', system_id, lookup_col)
+        debugMissingLookup(lookup_name, 'term.id', term_id, lookup_col, value, model=model, term=term_id)
         min = safe_parse_float(extract_grouped_data(value, 'min'), default=None)
         max = safe_parse_float(extract_grouped_data(value, 'max'), default=None)
         if min and max:

hestia_earth/models/utils/lookup.py

@@ -1,12 +1,6 @@
 from functools import lru_cache
 from typing import Optional, List
-from hestia_earth.utils.lookup import (
-    download_lookup,
-    get_table_value,
-    column_name,
-    extract_grouped_data,
-    _get_single_table_value
-)
+from hestia_earth.utils.lookup import download_lookup, get_table_value, extract_grouped_data, is_missing_value
 from hestia_earth.utils.tools import list_sum, safe_parse_float
 
 from ..log import debugValues, log_as_table, debugMissingLookup
@@ -42,9 +36,9 @@ def _factor_value(
         )
         # value is either a number or matching between a model and a value (restrict value to specific model only)
         return safe_parse_float(
-            extract_grouped_data(coefficient, grouped_data_key),
+            extract_grouped_data(coefficient, grouped_data_key) if ':' in str(coefficient) else coefficient,
             default=None
-        ) if ':' in str(coefficient) else safe_parse_float(coefficient, default=None)
+        )
 
     def get_value(blank_node: dict):
         node_term_id = blank_node.get('term', {}).get('@id')
@@ -111,15 +105,15 @@ def aware_factor_value(
     default_world_value: Optional[bool] = False
 ):
     lookup = download_lookup(lookup_name, False)  # avoid saving in memory as there could be many different files used
-    lookup_col = column_name('awareWaterBasinId')
+    lookup_col = 'awareWaterBasinId'
 
     @lru_cache()
     def get_coefficient(node_term_id: str):
-        coefficient = _get_single_table_value(lookup, lookup_col, int(aware_id), column_name(node_term_id))
+        coefficient = get_table_value(lookup, lookup_col, int(aware_id), node_term_id)
         return safe_parse_float(
-            extract_grouped_data(coefficient, group_key),
+            extract_grouped_data(coefficient, group_key) if group_key else coefficient,
             default=None
-        ) if group_key else coefficient
+        )
 
     def get_value(blank_node: dict):
         node_term_id = blank_node.get('term', {}).get('@id')
@@ -167,7 +161,7 @@ def all_factor_value(
         term_id, region_id = missing_value
         debugMissingLookup(
             lookup_name=lookup_name,
-            row='termid',
+            row='term.id',
             row_value=region_id or term_id,
             col=term_id if region_id else lookup_col,
             value=None,
@@ -195,18 +189,19 @@ def all_factor_value(
 
 def get_region_lookup(lookup_name: str, term_id: str):
     # for performance, try to load the region specific lookup if exists
-    return (
-        download_lookup(lookup_name.replace('region-', f"{term_id}-"), build_index=True)
+    lookup = (
+        download_lookup(lookup_name.replace('region-', f"{term_id}-"))
         if lookup_name and lookup_name.startswith('region-') else None
-    ) or download_lookup(lookup_name, build_index=True)
+    )
+    return lookup if lookup is not None and not lookup.empty else download_lookup(lookup_name)
 
 
 @lru_cache()
 def get_region_lookup_value(lookup_name: str, term_id: str, column: str, fallback_world: bool = False, **log_args):
     # for performance, try to load the region specific lookup if exists
     lookup = get_region_lookup(lookup_name, term_id)
-    value = get_table_value(lookup, 'termid', term_id, column_name(column))
-    if value is None and fallback_world:
+    value = get_table_value(lookup, 'term.id', term_id, column)
+    if is_missing_value(value) and fallback_world:
         return get_region_lookup_value(lookup_name, 'region-world', column, **log_args)
-    debugMissingLookup(lookup_name, 'termid', term_id, column, value, **log_args)
+    debugMissingLookup(lookup_name, 'term.id', term_id, column, value, **log_args)
     return value

hestia_earth/models/utils/property.py

@@ -1,6 +1,6 @@
 from functools import lru_cache
 from hestia_earth.schema import SchemaType, TermTermType
-from hestia_earth.utils.lookup import download_lookup, extract_grouped_data, get_table_value, column_name
+from hestia_earth.utils.lookup import download_lookup, extract_grouped_data, get_table_value
 from hestia_earth.utils.model import find_term_match, linked_node
 from hestia_earth.utils.tools import list_sum, safe_parse_float
 from hestia_earth.utils.term import download_term
@@ -104,11 +104,11 @@ def node_has_property(term_id: str):
 def node_property_lookup_value(model: str, term_id: str, term_type: str, prop_id: str, default=None, **log_args):
     try:
         lookup_name = f"{term_type}-property.csv"
-        lookup_value = get_table_value(download_lookup(lookup_name), 'termid', term_id, column_name(prop_id))
+        lookup_value = get_table_value(download_lookup(lookup_name), 'term.id', term_id, prop_id)
         value = extract_grouped_data(lookup_value, 'Avg') if (
             isinstance(lookup_value, str) and 'Avg' in lookup_value
         ) else lookup_value
-        debugMissingLookup(lookup_name, 'termid', term_id, prop_id, value, model=model, **log_args)
+        debugMissingLookup(lookup_name, 'term.id', term_id, prop_id, value, model=model, **log_args)
         return safe_parse_float(value, default=None)
     except Exception:
         value = get_lookup_value({'@id': term_id, 'termType': term_type}, prop_id, skip_debug=True)

hestia_earth/models/utils/term.py

@@ -1,6 +1,6 @@
 from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType
 from hestia_earth.utils.api import find_node, search
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
+from hestia_earth.utils.lookup import download_lookup, get_table_value
 
 from .constant import Units
 from ..log import debugMissingLookup
@@ -9,15 +9,15 @@ from ..log import debugMissingLookup
 LIMIT = 9999
 
 
-def get_lookup_value(lookup_term: dict, column: str, skip_debug: bool = False, **log_args):
+def get_lookup_value(lookup_term: dict, column: str, skip_debug: bool = False, default_value='', **log_args):
     table_name = f"{lookup_term.get('termType')}.csv" if lookup_term else None
     value = get_table_value(
-        download_lookup(table_name), 'termid', lookup_term.get('@id'), column_name(column)
+        download_lookup(table_name), 'term.id', lookup_term.get('@id'), column
     ) if table_name else None
     debugMissingLookup(
-        table_name, 'termid', lookup_term.get('@id'), column, value, **log_args
+        table_name, 'term.id', lookup_term.get('@id'), column, value, **log_args
     ) if lookup_term and not skip_debug else None
-    return value
+    return default_value if value is None else value
 
 
 def get_liquid_fuel_terms():

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.74.17'
+VERSION = '0.75.1'

hestia_earth/orchestrator/__init__.py

@@ -1,12 +1,9 @@
-from pkgutil import extend_path
 from typing import Union, List
 from hestia_earth.utils.tools import current_time_ms
 
 from .log import logger
 from .models import run as run_models
 
-__path__ = extend_path(__path__, __name__)
-
 
 def _required(message): raise Exception(message)
 

hestia_earth/orchestrator/models/transformations.py

@@ -1,7 +1,7 @@
 from copy import deepcopy
 from functools import reduce
 from hestia_earth.schema import CompletenessJSONLD
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
+from hestia_earth.utils.lookup import download_lookup, get_table_value
 from hestia_earth.models.transformation.input.utils import replace_input_field
 from hestia_earth.models.utils.transformation import previous_transformation
 from hestia_earth.models.utils.product import find_by_product
@@ -22,7 +22,7 @@ def _include_practice(practice: dict):
     term_type = term.get('termType')
     term_id = term.get('@id')
     lookup = download_lookup(f"{term_type}.csv")
-    value = get_table_value(lookup, 'termid', term_id, column_name('includeForTransformation'))
+    value = get_table_value(lookup, 'term.id', term_id, 'includeForTransformation')
     return False if value is None or value == '' or not value else True
 
 

{hestia_earth_models-0.74.17.dist-info → hestia_earth_models-0.75.1.dist-info}/METADATA

@@ -1,23 +1,23 @@
 Metadata-Version: 2.4
 Name: hestia_earth_models
-Version: 0.74.17
+Version: 0.75.1
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team
 Author-email: guillaumeroyer.mail@gmail.com
 License: MIT
-Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hestia-earth-schema<34.0.0,>=33.5.0
-Requires-Dist: hestia-earth-utils>=0.15.16
+Requires-Dist: hestia-earth-schema<36.0.0,>=35.0.1
+Requires-Dist: hestia-earth-utils>=0.16.7
 Requires-Dist: CurrencyConverter==0.16.8
 Requires-Dist: haversine>=2.7.0
 Requires-Dist: pydash
 Requires-Dist: area
 Provides-Extra: spatial
-Requires-Dist: hestia-earth-earth-engine>=0.5.0; extra == "spatial"
+Requires-Dist: hestia-earth-earth-engine>=0.6.0; extra == "spatial"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -28,6 +28,7 @@ Dynamic: license
 Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
+Dynamic: requires-python
 Dynamic: summary
 
 # HESTIA Engine Models
@@ -39,20 +40,15 @@ HESTIA's set of models for running calculations or retrieving data using externa
 
 ## Documentation
 
-Documentation for every model can be found in the [HESTIA API Documentation](https://hestia.earth/docs/#hestia-calculation-models).
+Documentation for every model can be found in the [HESTIA Guide](https://hestia.earth/guide/models).
 
 ## Install
 
-1. Install python `3` (we recommend using python `3.6` minimum)
+1. Install python `3.12` minimum
 2. Install the module:
 ```bash
 pip install hestia_earth.models
 ```
-3. Set the following environment variables:
-```
-API_URL=https://api.hestia.earth
-WEB_URL=https://hestia.earth
-```
 
 ### Usage
 
@@ -95,7 +91,7 @@ This will display the Cycle recalculated with all HESTIA default models running.
 #### Using Spatial Models
 
 We have models that can gap-fill geographical information on a `Site`. If you want to use these models:
-1. Install the library: `pip install hestia_earth.earth_engine`
+1. Install the library: `pip install hestia-earth-earth-engine`
 2. Follow the [Getting Started instructions](https://gitlab.com/hestia-earth/hestia-earth-engine#getting-started).
 
 #### Using the ecoinventV3 model