hestia-earth-models 0.62.2__py3-none-any.whl → 0.62.3__py3-none-any.whl

hestia_earth/models/requirements.py

@@ -353,7 +353,7 @@ def get_single_returns(model: str, key: str):
     Returns
     -------
     list
-        The returned data following Hestia's schema.
+        The returned data following HESTIA's schema.
     """
     returns = _model_returns({'model': model, 'key': key})
     return_type = list(returns.keys())[0] if returns else None
@@ -368,7 +368,7 @@ def list_models(
     termType: str = None, tier: str = None, productTermId: str = None, productTermType: str = None, siteType: str = None
 ) -> list:
     """
-    Return list of models present in Hestia.
+    Return list of models present in HESTIA.
 
     Parameters
     ----------
@@ -410,7 +410,7 @@ def get_models(termId: str):
     Returns
     -------
     list
-        The list of modelscurrently in Hestia that are matching this Term (as `{"model": model, "key": key}`).
+        The list of modelscurrently in HESTIA that are matching this Term (as `{"model": model, "key": key}`).
     """
     return list(filter(lambda m: m.get('key') == termId, ALL_MODELS))
 
@@ -420,7 +420,7 @@ def get_all(
     termType: str = None, tier: str = None, productTermId: str = None, productTermType: str = None, siteType: str = None
 ) -> list:
     """
-    Get the requirements to run all the models in Hestia.
+    Get the requirements to run all the models in HESTIA.
 
     Parameters
     ----------
@@ -441,7 +441,7 @@ def get_all(
     Returns
     -------
     list
-        The data requirements following Hestia's schema as multiple nodes.
+        The data requirements following HESTIA's schema as multiple nodes.
     """
     models = list_models(
         termType=termType, tier=tier, productTermId=productTermId, productTermType=productTermType, siteType=siteType
@@ -469,7 +469,7 @@ def get_single(model: str, key: str) -> list:
     Returns
     -------
     list
-        The data requirements following Hestia's schema as multiple nodes.
+        The data requirements following HESTIA's schema as multiple nodes.
     """
     model = {'model': model, 'key': key}
     linked_models = _recursive_linked_models(model)

hestia_earth/models/site/management.py

@@ -18,14 +18,13 @@ from functools import reduce
 from hestia_earth.schema import SchemaType, TermTermType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import filter_list_term_type, linked_node
-from hestia_earth.utils.tools import safe_parse_float
+from hestia_earth.utils.tools import flatten, safe_parse_float
 from hestia_earth.utils.blank_node import get_node_value
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_blank_nodes_id
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.blank_node import condense_nodes
 from hestia_earth.models.utils.site import related_cycles
-from hestia_earth.models.utils.site import get_land_cover_term_id as get_landCover_term_id_from_site_type
 from . import MODEL
 
 REQUIREMENTS = {
@@ -131,16 +130,14 @@ def _extract_node_value(node: dict) -> dict:
     return node | {'value': get_node_value(node)}
 
 
-def _include(value: dict, keys: list) -> dict: return {k: v for k, v in value.items() if k in keys}
+def _include(value: dict, keys: list): return {k: v for k, v in value.items() if k in keys}
 
 
-def _default_dates(cycle: dict, values: list) -> list:
-    """Only uses the dates from the cycle if they are missing from the item."""
+def _default_dates(cycle: dict, values: list):
     return [(_include(cycle, ["startDate", "endDate"]) | v) for v in values]
 
 
 def _overwrite_dates(cycle: dict, values: list) -> list:
-    """Always uses the dates from the cycle."""
     return [v | _include(cycle, ["startDate", "endDate"]) for v in values]
 
 
@@ -152,21 +149,22 @@ def _copy_item_if_exists(source: dict, keys: list[str] = None, dest: dict = None
 
 def _get_landCover_term_id(product: dict) -> str:
     value = get_lookup_value(product.get('term', {}), LAND_COVER_KEY, model=MODEL, model_key=LAND_COVER_KEY)
+    # TODO: what should happen when there are multiple values?
     return value.split(';')[0] if value else None
 
 
-def _get_relevant_items(
-    cycles: list[dict], item_name: str, relevant_terms: list, date_fill: callable = _default_dates
-) -> list[tuple]:
-    """
-    Get items (optionally in tuples with site_type) from the list of cycles with any of the relevant terms.
-    Also adds dates if missing.
-    """
-    return [
-        (item, cycle.get("site", {}).get("siteType", ""))
-        for cycle in cycles
-        for item in date_fill(cycle=cycle, values=filter_list_term_type(cycle.get(item_name, []), relevant_terms))
-    ]
+def _get_items_with_relevant_term_type(
+        cycles: list[dict], item_name: str, relevant_values: list, date_fill: callable = _default_dates
+):
+    """Get items from the list of cycles with any of the relevant values. Also adds dates if missing."""
+    return flatten(
+        [
+            date_fill(
+                cycle=cycle,
+                values=filter_list_term_type(cycle.get(item_name, []), relevant_values)
+            ) for cycle in cycles
+        ]
+    )
 
 
 def _get_lookup_with_debug(term: dict, column: str) -> any:
@@ -213,23 +211,27 @@ def _get_relevant_inputs(cycles: list[dict]) -> list:
     return relevant_inputs
 
 
-def _should_run_all_products(cycles):
+def _should_run(site: dict):
+    # Only get related cycles once.
+    cycles = related_cycles(site)
+
     products_land_cover = [
         _extract_node_value(
             _include(
                 value=product,
                 keys=["term", "value", "startDate", "endDate", "properties"]
             )
-        ) for product, _ in _get_relevant_items(
+        ) for product in _get_items_with_relevant_term_type(
             cycles=cycles,
             item_name="products",
-            relevant_terms=[TermTermType.LANDCOVER]
+            relevant_values=[TermTermType.LANDCOVER]
         )
     ]
-    products_crop_forage = _get_relevant_items(
+
+    products_crop_forage = _get_items_with_relevant_term_type(
         cycles=cycles,
         item_name="products",
-        relevant_terms=[TermTermType.CROP, TermTermType.FORAGE],
+        relevant_values=[TermTermType.CROP, TermTermType.FORAGE],
         date_fill=_overwrite_dates
     )
     products_crop_forage = [
@@ -241,33 +243,9 @@ def _should_run_all_products(cycles):
                 "value": 100
             }
         )
-        for product in list(filter(_get_landCover_term_id, [i[0] for i in products_crop_forage]))
+        for product in list(filter(_get_landCover_term_id, products_crop_forage))
     ]
-    products_animal_with_site_type = _get_relevant_items(
-        cycles=cycles,
-        item_name="products",
-        relevant_terms=[TermTermType.LIVEANIMAL, TermTermType.ANIMALPRODUCT]
-    )
-    products_animal = [
-        _copy_item_if_exists(
-            source=product,
-            keys=["startDate", "endDate", "properties"],
-            dest={
-                "term": linked_node(download_hestia(get_landCover_term_id_from_site_type(site_type))),
-                "value": 100
-            }
-        )
-        for product, site_type in products_animal_with_site_type if product
-    ]
-
-    return products_animal, products_crop_forage, products_land_cover
-
-
-def _should_run(site: dict):
-    cycles = related_cycles(site)
-
-    products_animal, products_crop_forage, products_land_cover = _should_run_all_products(cycles)
-    all_products = products_land_cover + products_crop_forage + products_animal
+    all_products = products_land_cover + products_crop_forage
     all_products = condense_nodes(all_products)
 
     practices = [
@@ -276,10 +254,10 @@ def _should_run(site: dict):
                 value=practice,
                 keys=["term", "value", "startDate", "endDate"]
             )
-        ) for practice, _ in _get_relevant_items(
+        ) for practice in _get_items_with_relevant_term_type(
             cycles=cycles,
             item_name="practices",
-            relevant_terms=[
+            relevant_values=[
                 TermTermType.WATERREGIME,
                 TermTermType.TILLAGE,
                 TermTermType.CROPRESIDUEMANAGEMENT,
@@ -297,7 +275,6 @@ def _should_run(site: dict):
         model_key=MODEL_KEY,
         products_crop_forage_ids=log_blank_nodes_id(products_crop_forage),
         products_land_cover_ids=log_blank_nodes_id(products_land_cover),
-        products_animal=log_blank_nodes_id(products_animal),
         practice_ids=log_blank_nodes_id(practices),
         inputs=log_blank_nodes_id(relevant_inputs)
     )

hestia_earth/models/site/organicCarbonPerHa.py

@@ -278,7 +278,7 @@ def _rescale_soc_value(
     """
     Rescale an SOC measurement value from a source depth interval to a target depth interval.
 
-    Depths are converted from centimetres (Hestia schema) to metres for use in `cdf` function.
+    Depths are converted from centimetres (HESTIA schema) to metres for use in `cdf` function.
 
     Parameters
     ----------

hestia_earth/models/utils/__init__.py

@@ -61,7 +61,7 @@ def _filter_list_term_unit(values: list, unit: Any):
 
 def is_from_model(node: dict) -> bool:
     """
-    Check if the Blank Node came from one of the Hestia Models.
+    Check if the Blank Node came from one of the HESTIA Models.
 
     Parameters
     ----------

hestia_earth/models/utils/blank_node.py

@@ -267,12 +267,17 @@ def get_total_value_converted_with_min_ratio(
     total_value_with_property = list_sum([value for term_id, value, prop_value in values if prop_value])
     total_value_ratio = total_value_with_property / total_value if total_value > 0 else 0
 
-    debugValues(node, model=model, term=term, property_id=prop_id,
-                total_value=total_value,
-                total_value_with_property=total_value_with_property,
-                total_value_ratio=total_value_ratio,
-                min_value_ratio=min_ratio,
-                conversion_details=value_logs)
+    logs = {
+        f"{prop_id}-term-id": prop_id,
+        f"{prop_id}-total-value": total_value,
+        f"{prop_id}-total-value-with-property": total_value_with_property,
+        f"{prop_id}-total-value-with-ratio": total_value_ratio,
+        f"{prop_id}-min-value-ratio": min_ratio,
+        f"{prop_id}-values": value_logs
+    }
+
+    debugValues(node, model=model, term=term,
+                **logs)
 
     return list_sum([
         value * prop_value for term_id, value, prop_value in values if all([value, prop_value])
@@ -612,7 +617,7 @@ def cumulative_nodes_lookup_match(
 
 class DatestrFormat(Enum):
     """
-    Enum representing ISO date formats permitted by Hestia.
+    Enum representing ISO date formats permitted by HESTIA.
 
     See: https://en.wikipedia.org/wiki/ISO_8601
     """
@@ -677,7 +682,7 @@ def _check_datestr_format(datestr: str, format: DatestrFormat) -> bool:
 
 def _get_datestr_format(datestr: str, default: Optional[Any] = None) -> Union[DatestrFormat, Any, None]:
     """
-    Check a datestr against each ISO format permitted by the Hestia schema and
+    Check a datestr against each ISO format permitted by the HESTIA schema and
     return the matching format.
     """
     return next(
@@ -1170,7 +1175,7 @@ def group_nodes_by_last_date(nodes: list) -> dict[str, list[dict]]:
     Parameters
     ----------
     nodes : list[dict]
-        A list of Hestia format nodes.
+        A list of HESTIA format nodes.
 
     Return
     ------

hestia_earth/models/utils/cycle.py

@@ -22,7 +22,7 @@ def unique_currencies(cycle: dict) -> list:
     Parameters
     ----------
     cycle : dict
-        The `Cycle` as defined in the Hestia Schema.
+        The `Cycle` as defined in the HESTIA Schema.
 
     Returns
     -------
@@ -41,7 +41,7 @@ def default_currency(cycle: dict) -> str:
     Parameters
     ----------
     cycle : dict
-        The `Cycle` as defined in the Hestia Schema.
+        The `Cycle` as defined in the HESTIA Schema.
 
     Returns
     -------
@@ -59,7 +59,7 @@ def get_crop_residue_decomposition_N_total(cycle: dict) -> float:
     Parameters
     ----------
     cycle : dict
-        The `Cycle` as defined in the Hestia Schema.
+        The `Cycle` as defined in the HESTIA Schema.
 
     Returns
     -------
@@ -87,7 +87,7 @@ def get_excreta_N_total(cycle: dict) -> float:
     Parameters
     ----------
     cycle : dict
-        The `Cycle` as defined in the Hestia Schema.
+        The `Cycle` as defined in the HESTIA Schema.
 
     Returns
     -------
@@ -115,7 +115,7 @@ def get_organic_fertiliser_N_total(cycle: dict) -> float:
     Parameters
     ----------
     cycle : dict
-        The `Cycle` as defined in the Hestia Schema.
+        The `Cycle` as defined in the HESTIA Schema.
 
     Returns
     -------
@@ -141,7 +141,7 @@ def get_organic_fertiliser_P_total(cycle: dict) -> float:
     Parameters
     ----------
     cycle : dict
-        The `Cycle` as defined in the Hestia Schema.
+        The `Cycle` as defined in the HESTIA Schema.
 
     Returns
     -------
@@ -165,7 +165,7 @@ def get_inorganic_fertiliser_N_total(cycle: dict) -> float:
     Parameters
     ----------
     cycle : dict
-        The `Cycle` as defined in the Hestia Schema.
+        The `Cycle` as defined in the HESTIA Schema.
 
     Returns
     -------
@@ -189,7 +189,7 @@ def get_inorganic_fertiliser_P_total(cycle: dict) -> float:
     Parameters
     ----------
     cycle : dict
-        The `Cycle` as defined in the Hestia Schema.
+        The `Cycle` as defined in the HESTIA Schema.
 
     Returns
     -------
@@ -260,7 +260,7 @@ def land_occupation_per_ha(model: str, term_id: str, cycle: dict):
     term_id : str
         The name of the term running this function. For debugging purpose only.
     cycle : dict
-        The `Cycle` as defined in the Hestia Schema.
+        The `Cycle` as defined in the HESTIA Schema.
 
     Returns
     -------
@@ -302,9 +302,9 @@ def land_occupation_per_kg(model: str, term_id: str, cycle: dict, site: dict, pr
     term_id : str
         The name of the term running this function. For debugging purpose only.
     cycle : dict
-        The `Cycle` as defined in the Hestia Schema.
+        The `Cycle` as defined in the HESTIA Schema.
     site : dict
-        The `Site` as defined in the Hestia Schema.
+        The `Site` as defined in the HESTIA Schema.
     primary_product : dict
         The primary `Product` of the `Cycle`.
 
@@ -447,7 +447,7 @@ def check_cycle_site_ids_identical(cycles: list[dict]) -> bool:
     Parameters
     ----------
     cycles : list[dict]
-        A list of Hestia `Cycle` nodes, see: https://www.hestia.earth/schema/Cycle.
+        A list of HESTIA `Cycle` nodes, see: https://www.hestia.earth/schema/Cycle.
 
     Returns
     -------

hestia_earth/models/utils/measurement.py

@@ -111,7 +111,7 @@ def group_measurement_values_by_year(
     Parameters
     ----------
     measurement : dict
-        A Hestia `Measurement` node, see: https://www.hestia.earth/schema/Measurement.
+        A HESTIA `Measurement` node, see: https://www.hestia.earth/schema/Measurement.
     inner_key: Any | None
         An optional inner dictionary key for the outputted annualised groups (can be used to merge annualised
         dictionaries together), default value: `None`.
@@ -165,9 +165,9 @@ def most_relevant_measurement_value_by_depth_and_date(
     Parameters
     ----------
     measurements list[dict]
-        A list of Hestia `Measurement` nodes, see: https://www.hestia.earth/schema/Measurement.
+        A list of HESTIA `Measurement` nodes, see: https://www.hestia.earth/schema/Measurement.
     term_id : str
-        The `@id` of a Hestia `Term`. Example: `"sandContent"`
+        The `@id` of a HESTIA `Term`. Example: `"sandContent"`
     date : str
         The target date in ISO 8601 string format (`"YYYY-MM-DD"`), see: https://en.wikipedia.org/wiki/ISO_8601.
     depth_upper : int

hestia_earth/models/utils/site.py

@@ -5,7 +5,6 @@ from hestia_earth.utils.tools import non_empty_list, flatten, safe_parse_date
 
 from hestia_earth.models.log import debugMissingLookup
 from . import cached_value, _load_calculated_node
-from .term import get_land_cover_siteTypes
 
 CACHE_YEARS_KEY = 'years'
 WATER_TYPES = [
@@ -121,9 +120,3 @@ def region_factor(model: str, region_id: str, term_id: str, termType: TermTermTy
     value = get_table_value(download_lookup(lookup_name), 'termid', region_id, column_name(term_id))
     debugMissingLookup(lookup_name, 'termid', region_id, term_id, value, model=model, term=term_id)
     return value
-
-
-def get_land_cover_term_id(site_type: str):
-    land_cover_terms = get_land_cover_siteTypes()
-    term = next((term for term in land_cover_terms if term["name"].lower() == site_type.lower()), {})
-    return term.get('@id')

hestia_earth/models/utils/term.py

@@ -1,4 +1,4 @@
-from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType
+from hestia_earth.schema import SchemaType, TermTermType
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
 from hestia_earth.utils.api import find_node, search
 
@@ -615,7 +615,7 @@ def get_electricity_grid_mix_terms():
     list
         List of matching `Term` as dict.
     """
-    return search({
+    terms = search({
         "bool": {
             "must": [
                 {"match": {"@type": SchemaType.TERM.value}},
@@ -625,23 +625,4 @@ def get_electricity_grid_mix_terms():
             ],
         }
     }, limit=LIMIT, fields=['@type', '@id', 'name', 'termType', 'units'])
-
-
-def get_land_cover_siteTypes():
-    """
-    Find all `Land Cover` terms with siteTypes
-
-    Returns
-    -------
-        List of landCover terms with associated siteTypes.
-    """
-    return search({
-        "bool": {
-            "must": [
-                {"match": {"@type": "Term"}},
-                {"match": {"termType": "landCover"}}
-            ],
-            "should": [{"match": {"name": siteType.value}} for siteType in SiteSiteType],
-            "minimum_should_match": 1
-        },
-    })
+    return list(map(lambda n: n["@id"], terms))

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.62.2'
+VERSION = '0.62.3'

{hestia_earth_models-0.62.2.dist-info → hestia_earth_models-0.62.3.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.62.2
-Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
+Version: 0.62.3
+Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
-Author: Hestia Team
+Author: HESTIA Team
 Author-email: guillaumeroyer.mail@gmail.com
 License: GPL
 Platform: UNKNOWN
@@ -11,25 +11,25 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Programming Language :: Python :: 3.6
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hestia-earth.schema ==29.*
-Requires-Dist: hestia-earth.utils >=0.13.2
-Requires-Dist: python-dateutil >=2.8.1
-Requires-Dist: CurrencyConverter ==0.16.8
-Requires-Dist: haversine >=2.7.0
+Requires-Dist: hestia-earth.schema==29.*
+Requires-Dist: hestia-earth.utils>=0.13.2
+Requires-Dist: python-dateutil>=2.8.1
+Requires-Dist: CurrencyConverter==0.16.8
+Requires-Dist: haversine>=2.7.0
 Requires-Dist: pydash
 Provides-Extra: spatial
-Requires-Dist: hestia-earth.earth-engine >=0.4.7 ; extra == 'spatial'
+Requires-Dist: hestia-earth.earth-engine>=0.4.7; extra == "spatial"
 
-# Hestia Engine Models
+# HESTIA Engine Models
 
 [![Pipeline Status](https://gitlab.com/hestia-earth/hestia-engine-models/badges/master/pipeline.svg)](https://gitlab.com/hestia-earth/hestia-engine-models/commits/master)
 [![Coverage Report](https://gitlab.com/hestia-earth/hestia-engine-models/badges/master/coverage.svg)](https://gitlab.com/hestia-earth/hestia-engine-models/commits/master)
 
-Hestia's set of models for running calculations or retrieving data using external datasets and internal lookups.
+HESTIA's set of models for running calculations or retrieving data using external datasets and internal lookups.
 
 ## Documentation
 
-Documentation for every model can be found in the [Hestia API Documentation](https://hestia.earth/docs/#hestia-calculation-models).
+Documentation for every model can be found in the [HESTIA API Documentation](https://hestia.earth/docs/#hestia-calculation-models).
 
 ## Install