hestia-earth-models 0.62.3__tar.gz → 0.62.4__tar.gz

{hestia-earth-models-0.62.3/hestia_earth_models.egg-info → hestia-earth-models-0.62.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.62.3
+Version: 0.62.4
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team

{hestia-earth-models-0.62.3 → hestia-earth-models-0.62.4}/hestia_earth/models/faostat2018/liveweightPerHead.py

@@ -77,7 +77,7 @@ def _run_liveAnimal(cycle: dict, product: dict, year: int, country_id: str):
     product_id = product.get('term', {}).get('@id')
     animal_product_id = get_liveAnimal_to_animalProduct_id(product_id, LOOKUPS['liveAnimal'][0], term=TERM_ID)
 
-    animal_product_term = download_hestia(animal_product_id)
+    animal_product_term = download_hestia(animal_product_id) if animal_product_id else {}
     kg_liveweight, groupingFAO = _product_value({**product, 'term': animal_product_term}, year, country_id)
 
     logRequirements(cycle, model=MODEL, term=product_id, property=TERM_ID,

{hestia-earth-models-0.62.3 → hestia-earth-models-0.62.4}/hestia_earth/models/log.py

@@ -63,7 +63,7 @@ def logShouldRun(log_node: dict, model: str, term: Union[str, None], should_run:
 
 
 def debugMissingLookup(lookup_name: str, row: str, row_value: str, col: str, value, **kwargs):
-    if value is None:
+    if value is None or value == '':
         extra = (', ' + _join_args(**kwargs)) if len(kwargs.keys()) > 0 else ''
         logger.warn('Missing lookup=%s, %s=%s, column=%s' + extra, lookup_name, row, row_value, col)
 

{hestia-earth-models-0.62.3 → hestia-earth-models-0.62.4}/hestia_earth/models/mocking/search-results.json

@@ -1636,6 +1636,150 @@
       }
     ]
   },
+  {
+    "name": "get_land_cover_siteTypes",
+    "query": {
+      "bool": {
+        "must": [
+          {
+            "match": {
+              "@type": "Term"
+            }
+          },
+          {
+            "match": {
+              "termType": "landCover"
+            }
+          }
+        ],
+        "should": [
+          {
+            "match": {
+              "name": "agri-food processor"
+            }
+          },
+          {
+            "match": {
+              "name": "animal housing"
+            }
+          },
+          {
+            "match": {
+              "name": "cropland"
+            }
+          },
+          {
+            "match": {
+              "name": "food retailer"
+            }
+          },
+          {
+            "match": {
+              "name": "forest"
+            }
+          },
+          {
+            "match": {
+              "name": "glass or high accessible cover"
+            }
+          },
+          {
+            "match": {
+              "name": "lake"
+            }
+          },
+          {
+            "match": {
+              "name": "other natural vegetation"
+            }
+          },
+          {
+            "match": {
+              "name": "permanent pasture"
+            }
+          },
+          {
+            "match": {
+              "name": "pond"
+            }
+          },
+          {
+            "match": {
+              "name": "river or stream"
+            }
+          },
+          {
+            "match": {
+              "name": "sea or ocean"
+            }
+          }
+        ],
+        "minimum_should_match": 1
+      }
+    },
+    "results": [
+      {
+        "@type": "Term",
+        "name": "Glass or high accessible cover",
+        "@id": "glassOrHighAccessibleCover",
+        "_score": 60.694187
+      },
+      {
+        "@type": "Term",
+        "name": "River or stream",
+        "@id": "riverOrStream",
+        "_score": 50.06671
+      },
+      {
+        "@type": "Term",
+        "name": "Other natural vegetation",
+        "@id": "otherNaturalVegetation",
+        "_score": 40.722
+      },
+      {
+        "@type": "Term",
+        "name": "Natural forest",
+        "@id": "naturalForest",
+        "_score": 31.252472
+      },
+      {
+        "@type": "Term",
+        "name": "Permanent pasture",
+        "@id": "permanentPasture",
+        "_score": 27.857792
+      },
+      {
+        "@type": "Term",
+        "name": "Animal housing",
+        "@id": "animalHousing",
+        "_score": 26.769783
+      },
+      {
+        "@type": "Term",
+        "name": "Root or tuber crop plant",
+        "@id": "rootOrTuberCropPlant",
+        "_score": 24.781918
+      },
+      {
+        "@type": "Term",
+        "name": "High intensity grazing pasture",
+        "@id": "highIntensityGrazingPasture",
+        "_score": 23.502655
+      },
+      {
+        "@type": "Term",
+        "name": "Permanent cropland",
+        "@id": "permanentCropland",
+        "_score": 20.467493
+      },
+      {
+        "@type": "Term",
+        "name": "Forest",
+        "@id": "forest",
+        "_score": 19.942879
+      }
+    ]
+  },
   {
     "name": "get_liquid_fuel_terms",
     "query": {

{hestia-earth-models-0.62.3 → hestia-earth-models-0.62.4}/hestia_earth/models/site/management.py

@@ -15,16 +15,17 @@ condensed into a single node to aid readability.
 """
 from functools import reduce
 
-from hestia_earth.schema import SchemaType, TermTermType
+from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import filter_list_term_type, linked_node
-from hestia_earth.utils.tools import flatten, safe_parse_float
+from hestia_earth.utils.tools import safe_parse_float, flatten
 from hestia_earth.utils.blank_node import get_node_value
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_blank_nodes_id
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.blank_node import condense_nodes
 from hestia_earth.models.utils.site import related_cycles
+from hestia_earth.models.utils.site import get_land_cover_term_id as get_landCover_term_id_from_site_type
 from . import MODEL
 
 REQUIREMENTS = {
@@ -119,6 +120,9 @@ INPUT_RULES = {
         )
     )
 }
+_SKIP_LAND_COVER_SITE_TYPES = [
+    SiteSiteType.CROPLAND.value
+]
 
 
 def management(data: dict):
@@ -130,14 +134,15 @@ def _extract_node_value(node: dict) -> dict:
     return node | {'value': get_node_value(node)}
 
 
-def _include(value: dict, keys: list): return {k: v for k, v in value.items() if k in keys}
+def _include(value: dict, keys: list) -> dict: return {k: v for k, v in value.items() if k in keys}
 
 
 def _default_dates(cycle: dict, values: list):
     return [(_include(cycle, ["startDate", "endDate"]) | v) for v in values]
 
 
-def _overwrite_dates(cycle: dict, values: list) -> list:
+def _dates_from_current_cycle(cycle: dict, values: list) -> list:
+    """Always uses the dates from the cycle."""
     return [v | _include(cycle, ["startDate", "endDate"]) for v in values]
 
 
@@ -149,22 +154,21 @@ def _copy_item_if_exists(source: dict, keys: list[str] = None, dest: dict = None
 
 def _get_landCover_term_id(product: dict) -> str:
     value = get_lookup_value(product.get('term', {}), LAND_COVER_KEY, model=MODEL, model_key=LAND_COVER_KEY)
-    # TODO: what should happen when there are multiple values?
     return value.split(';')[0] if value else None
 
 
-def _get_items_with_relevant_term_type(
-        cycles: list[dict], item_name: str, relevant_values: list, date_fill: callable = _default_dates
-):
-    """Get items from the list of cycles with any of the relevant values. Also adds dates if missing."""
-    return flatten(
-        [
-            date_fill(
-                cycle=cycle,
-                values=filter_list_term_type(cycle.get(item_name, []), relevant_values)
-            ) for cycle in cycles
-        ]
-    )
+def _get_relevant_items(
+        cycles: list[dict], item_name: str, relevant_terms: list, date_fill: callable = _default_dates
+) -> list:
+    """
+    Get items from the list of cycles with any of the relevant terms.
+    Also adds dates if missing.
+    """
+    return [
+        item
+        for cycle in cycles
+        for item in date_fill(cycle=cycle, values=filter_list_term_type(cycle.get(item_name, []), relevant_terms))
+    ]
 
 
 def _get_lookup_with_debug(term: dict, column: str) -> any:
@@ -211,28 +215,25 @@ def _get_relevant_inputs(cycles: list[dict]) -> list:
     return relevant_inputs
 
 
-def _should_run(site: dict):
-    # Only get related cycles once.
-    cycles = related_cycles(site)
-
+def _should_run_all_products(cycles: list, site_type: str):
     products_land_cover = [
         _extract_node_value(
             _include(
                 value=product,
                 keys=["term", "value", "startDate", "endDate", "properties"]
             )
-        ) for product in _get_items_with_relevant_term_type(
+        ) for product in _get_relevant_items(
             cycles=cycles,
             item_name="products",
-            relevant_values=[TermTermType.LANDCOVER]
+            relevant_terms=[TermTermType.LANDCOVER]
         )
-    ]
+    ] if site_type else []
 
-    products_crop_forage = _get_items_with_relevant_term_type(
+    products_crop_forage = _get_relevant_items(
         cycles=cycles,
         item_name="products",
-        relevant_values=[TermTermType.CROP, TermTermType.FORAGE],
-        date_fill=_overwrite_dates
+        relevant_terms=[TermTermType.CROP, TermTermType.FORAGE],
+        date_fill=_dates_from_current_cycle
     )
     products_crop_forage = [
         _copy_item_if_exists(
@@ -243,9 +244,33 @@ def _should_run(site: dict):
                 "value": 100
             }
         )
-        for product in list(filter(_get_landCover_term_id, products_crop_forage))
-    ]
-    all_products = products_land_cover + products_crop_forage
+        for product in list(filter(_get_landCover_term_id, [i for i in products_crop_forage]))
+    ] if site_type else []
+    dates = sorted(list(set(
+        flatten([[cycle.get('startDate'), cycle.get('endDate')] for cycle in cycles])
+    ))) if site_type not in _SKIP_LAND_COVER_SITE_TYPES else []
+    site_type_term = download_hestia(get_landCover_term_id_from_site_type(site_type)) if all([
+        len(dates) >= 2,
+        site_type
+    ]) else None
+    products_site_type = [{
+        "term": linked_node(site_type_term),
+        "value": 100,
+        "startDate": dates[0],
+        "endDate": dates[-1]
+    }] if site_type_term else []
+
+    return products_site_type, products_crop_forage, products_land_cover
+
+
+def _should_run(site: dict):
+    cycles = related_cycles(site)
+
+    products_animal, products_crop_forage, products_land_cover = _should_run_all_products(
+        cycles=cycles,
+        site_type=site.get("siteType", "")
+    )
+    all_products = products_land_cover + products_crop_forage + products_animal
     all_products = condense_nodes(all_products)
 
     practices = [
@@ -254,10 +279,10 @@ def _should_run(site: dict):
                 value=practice,
                 keys=["term", "value", "startDate", "endDate"]
             )
-        ) for practice in _get_items_with_relevant_term_type(
+        ) for practice in _get_relevant_items(
             cycles=cycles,
             item_name="practices",
-            relevant_values=[
+            relevant_terms=[
                 TermTermType.WATERREGIME,
                 TermTermType.TILLAGE,
                 TermTermType.CROPRESIDUEMANAGEMENT,
@@ -275,6 +300,7 @@ def _should_run(site: dict):
         model_key=MODEL_KEY,
         products_crop_forage_ids=log_blank_nodes_id(products_crop_forage),
         products_land_cover_ids=log_blank_nodes_id(products_land_cover),
+        products_animal=log_blank_nodes_id(products_animal),
         practice_ids=log_blank_nodes_id(practices),
         inputs=log_blank_nodes_id(relevant_inputs)
     )

{hestia-earth-models-0.62.3 → hestia-earth-models-0.62.4}/hestia_earth/models/utils/site.py

@@ -5,6 +5,7 @@ from hestia_earth.utils.tools import non_empty_list, flatten, safe_parse_date
 
 from hestia_earth.models.log import debugMissingLookup
 from . import cached_value, _load_calculated_node
+from .term import get_land_cover_siteTypes
 
 CACHE_YEARS_KEY = 'years'
 WATER_TYPES = [
@@ -120,3 +121,9 @@ def region_factor(model: str, region_id: str, term_id: str, termType: TermTermTy
     value = get_table_value(download_lookup(lookup_name), 'termid', region_id, column_name(term_id))
     debugMissingLookup(lookup_name, 'termid', region_id, term_id, value, model=model, term=term_id)
     return value
+
+
+def get_land_cover_term_id(site_type: str):
+    land_cover_terms = get_land_cover_siteTypes()
+    term = next((term for term in land_cover_terms if term["name"].lower() == site_type.lower()), {})
+    return term.get("@id", "")

{hestia-earth-models-0.62.3 → hestia-earth-models-0.62.4}/hestia_earth/models/utils/term.py

@@ -1,4 +1,4 @@
-from hestia_earth.schema import SchemaType, TermTermType
+from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
 from hestia_earth.utils.api import find_node, search
 
@@ -626,3 +626,23 @@ def get_electricity_grid_mix_terms():
         }
     }, limit=LIMIT, fields=['@type', '@id', 'name', 'termType', 'units'])
     return list(map(lambda n: n["@id"], terms))
+
+
+def get_land_cover_siteTypes():
+    """
+    Find all `Land Cover` terms with siteTypes
+
+    Returns
+    -------
+        List of landCover terms with associated siteTypes.
+    """
+    return search({
+        "bool": {
+            "must": [
+                {"match": {"@type": "Term"}},
+                {"match": {"termType": "landCover"}}
+            ],
+            "should": [{"match": {"name": siteType.value}} for siteType in SiteSiteType],
+            "minimum_should_match": 1
+        },
+    })

hestia-earth-models-0.62.4/hestia_earth/models/version.py

@@ -0,0 +1 @@
+VERSION = '0.62.4'

{hestia-earth-models-0.62.3 → hestia-earth-models-0.62.4/hestia_earth_models.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.62.3
+Version: 0.62.4
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team

{hestia-earth-models-0.62.3 → hestia-earth-models-0.62.4}/tests/models/faostat2018/test_liveweightPerHead.py

@@ -123,3 +123,12 @@ def test_run_liveAnimal_sheepLamb(mock_download_hestia, *args):
     }
     value = run(cycle)
     assert value == expected
+
+
+@patch(f"{class_path}._new_property", side_effect=fake_new_property)
+def test_run_liveAninal_missing_term_id(*args):
+    with open(f"{fixtures_folder}/liveAnimal/missing-fao-term-id/cycle.jsonld", encoding='utf-8') as f:
+        cycle = json.load(f)
+
+    value = run(cycle)
+    assert value == []

{hestia-earth-models-0.62.3 → hestia-earth-models-0.62.4}/tests/models/site/test_management.py

@@ -2,7 +2,7 @@ import json
 from unittest.mock import patch
 
 import pytest
-from hestia_earth.schema import TermTermType
+from hestia_earth.schema import TermTermType, SiteSiteType
 
 from hestia_earth.models.site.management import MODEL, MODEL_KEY, run, _should_run
 from hestia_earth.models.utils.blank_node import condense_nodes
@@ -17,8 +17,17 @@ TERM_BY_ID = {
     'oatPlant': {'@type': 'Term', '@id': 'oatPlant', 'termType': TermTermType.LANDCOVER.value},
     'agatiTree': {'@type': 'Term', '@id': 'agatiTree', 'termType': TermTermType.LANDCOVER.value},
     'wildGarlicPlant': {'@type': 'Term', '@id': 'wildGarlicPlant', 'termType': TermTermType.LANDCOVER.value},
+    'animalHousing': {
+        "@type": "Term",
+        "@id": "animalHousing",
+        "name": "Animal housing",
+        "termType": TermTermType.LANDCOVER.value,
+        "units": "% area"
+    }
 }
 
+LAND_COVER_TERM_BY_SITE_TYPE = {SiteSiteType.ANIMAL_HOUSING.value: "animalHousing"}
+
 
 def lookup_side_effect(*args, **kwargs):
     # Values taken from real lookups.
@@ -233,7 +242,7 @@ def test_should_run(mock_related_cycles, *args):
     should_run, *args = _should_run({})
     assert should_run is True
 
-    # with relevant product => run
+    # with relevant product, blank site_type => no run
     mock_related_cycles.return_value = [
         {
             "products": [
@@ -253,6 +262,28 @@ def test_should_run(mock_related_cycles, *args):
         }
     ]
     should_run, *args = _should_run({})
+    assert should_run is False
+
+    # with relevant product and site_type => run
+    mock_related_cycles.return_value = [
+        {
+            "products": [
+                {
+                    "term": {
+                        "termType": TermTermType.CROP.value,
+                        "@id": "genericCropProduct"
+                    },
+                    "value": [51],
+                    "startDate": "2001",
+                    "endDate": "2002",
+                    "properties": {"test": "properties"}
+                }
+            ],
+            "startDate": "2021",
+            "endDate": "2022"
+        }
+    ]
+    should_run, *args = _should_run({"siteType": "cropland"})
     assert should_run is True
     assert args[0] == [{
         'term': TERM_BY_ID['genericCropPlant'],
@@ -270,7 +301,7 @@ def test_should_run(mock_related_cycles, *args):
         ("Example 2", f"{fixtures_folder}/inputs/example2"),
         ("Example 3", f"{fixtures_folder}/inputs/example3"),
         ("Example 4", f"{fixtures_folder}/inputs/example4"),
-        ("Condense Nodes", f"{fixtures_folder}/inputs/condense_nodes")
+        ("Condense Nodes", f"{fixtures_folder}/inputs/condense_nodes"),
         # Expected:
         #   - appleTree (81) x 3 condenses 2020-03-01 to 2021-02-15
         #   - animalManureUsed (true) x 2 condenses 2001-04-01 to 2001-12-31
@@ -282,18 +313,34 @@ def test_should_run(mock_related_cycles, *args):
         #   - sassafrasTree (86) x 2 condenses 2001-01-01 to 2004-12-31
         #   - bananaPlant (87) does not condense [non-consecutive years]
         #   - durianTree (89) does not condense [dates overwritten See 808]
+        ("Site Type", f"{fixtures_folder}/inputs/site_type")
     ]
 )
-@patch(f"{CLASS_PATH}.download_hestia", side_effect=lambda id, *args: TERM_BY_ID[id])
+@patch(
+    f"{CLASS_PATH}.get_landCover_term_id_from_site_type",
+    side_effect=lambda site_type: LAND_COVER_TERM_BY_SITE_TYPE[site_type]
+)
+@patch(f"{CLASS_PATH}.download_hestia", side_effect=lambda term_id, *args: TERM_BY_ID[term_id])
 @patch(f"{CLASS_PATH}.related_cycles")
 @patch(f"{CLASS_PATH}._get_lookup_with_debug", side_effect=lookup_side_effect)
-def test_run(mock_get_lookup_with_debug, mock_related_cycles, mock_download, test_name, fixture_path):
+def test_run(
+        mock_get_lookup_with_debug,
+        mock_related_cycles,
+        mock_download,
+        mock_land_cover_lookup,
+        test_name,
+        fixture_path
+):
     with open(f"{fixture_path}/cycles.jsonld", encoding='utf-8') as f:
         cycles = json.load(f)
     mock_related_cycles.return_value = cycles
 
-    with open(f"{fixtures_folder}/site.jsonld", encoding='utf-8') as f:
-        site = json.load(f)
+    try:
+        with open(f"{fixture_path}/site.jsonld", encoding='utf-8') as f:
+            site = json.load(f)
+    except FileNotFoundError:
+        with open(f"{fixtures_folder}/site.jsonld", encoding='utf-8') as f:
+            site = json.load(f)
 
     with open(f"{fixture_path}/result.jsonld", encoding='utf-8') as f:
         expected = json.load(f)

hestia-earth-models-0.62.3/hestia_earth/models/version.py

@@ -1 +0,0 @@
-VERSION = '0.62.3'