hestia-earth-models 0.62.4__py3-none-any.whl → 0.62.6__py3-none-any.whl

hestia_earth/models/geospatialDatabase/croppingIntensity.py

@@ -1,31 +1,40 @@
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.model import find_primary_product
+from hestia_earth.utils.model import find_primary_product, linked_node
+from hestia_earth.utils.tools import non_empty_list
 
-from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.log import logRequirements, logShouldRun, debugValues, log_as_table
 from hestia_earth.models.utils.practice import _new_practice
 from hestia_earth.models.utils.crop import is_plantation
+from hestia_earth.models.utils.cycle import get_allowed_sites
 from .utils import get_region_factor
 from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "products": [{
-            "@type": "Product",
-            "primary": "True",
-            "term.termType": "crop"
-        }],
         "site": {
             "@type": "Site",
             "or": [
                 {"region": {"@type": "Term"}},
                 {"country": {"@type": "Term"}}
             ]
+        },
+        "optional": {
+            "otherSites": [
+                {
+                    "@type": "Site",
+                    "or": [
+                        {"region": {"@type": "Term"}},
+                        {"country": {"@type": "Term"}}
+                    ]
+                }
+            ]
         }
     }
 }
 RETURNS = {
     "Practice": [{
-        "value": ""
+        "value": "",
+        "site": ""
     }]
 }
 LOOKUPS = {
@@ -35,25 +44,14 @@ LOOKUPS = {
 TERM_ID = 'croppingIntensity'
 
 
-def _practice(value: float):
+def _practice(site: dict, value: float):
     practice = _new_practice(TERM_ID)
+    practice['site'] = linked_node(site)
     # force conversion to float from numpy, avoiding errors when reading it in other models
     practice['value'] = [float(round(value, 7))]
     return practice
 
 
-def _run(cycle: dict):
-    site = cycle.get('site')
-    region_factor = get_region_factor(TERM_ID, site, TermTermType.LANDUSEMANAGEMENT)
-
-    logRequirements(site, model=MODEL, term=TERM_ID,
-                    region_factor=region_factor)
-
-    should_run = all([region_factor])
-    logShouldRun(site, MODEL, TERM_ID, should_run)
-    return [_practice(region_factor)] if should_run else []
-
-
 def _should_run(cycle: dict):
     product = find_primary_product(cycle) or {}
     not_plantation = not is_plantation(MODEL, TERM_ID, product.get('term', {}).get('@id'))
@@ -66,4 +64,16 @@ def _should_run(cycle: dict):
     return should_run
 
 
-def run(cycle: dict): return _run(cycle) if _should_run(cycle) else []
+def run(cycle: dict):
+    sites = get_allowed_sites(MODEL, TERM_ID, TermTermType.LANDUSEMANAGEMENT, cycle) if _should_run(cycle) else []
+    sites = [
+        (site, get_region_factor(TERM_ID, site, TermTermType.LANDUSEMANAGEMENT)) for site in sites
+    ]
+
+    debugValues(cycle, model=MODEL, term=TERM_ID,
+                site_factors=log_as_table([{
+                    'site-id': site.get('@id', site.get('id')),
+                    'factor': factor
+                } for site, factor in sites]))
+
+    return non_empty_list([_practice(site, factor) for site, factor in sites if bool(factor)])

hestia_earth/models/geospatialDatabase/longFallowRatio.py

@@ -1,31 +1,40 @@
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.model import find_primary_product
+from hestia_earth.utils.model import find_primary_product, linked_node
+from hestia_earth.utils.tools import non_empty_list
 
-from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.log import logRequirements, logShouldRun, debugValues, log_as_table
 from hestia_earth.models.utils.practice import _new_practice
 from hestia_earth.models.utils.crop import is_plantation
+from hestia_earth.models.utils.cycle import get_allowed_sites
 from .utils import get_region_factor
 from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "products": [{
-            "@type": "Product",
-            "primary": "True",
-            "term.termType": "crop"
-        }],
         "site": {
             "@type": "Site",
             "or": [
                 {"region": {"@type": "Term"}},
                 {"country": {"@type": "Term"}}
             ]
+        },
+        "optional": {
+            "otherSites": [
+                {
+                    "@type": "Site",
+                    "or": [
+                        {"region": {"@type": "Term"}},
+                        {"country": {"@type": "Term"}}
+                    ]
+                }
+            ]
         }
     }
 }
 RETURNS = {
     "Practice": [{
-        "value": ""
+        "value": "",
+        "site": ""
     }]
 }
 LOOKUPS = {
@@ -35,24 +44,14 @@ LOOKUPS = {
 TERM_ID = 'longFallowRatio'
 
 
-def _practice(value: float):
+def _practice(site: dict, value: float):
     practice = _new_practice(TERM_ID)
-    practice['value'] = [round(value, 7)]
+    practice['site'] = linked_node(site)
+    # force conversion to float from numpy, avoiding errors when reading it in other models
+    practice['value'] = [float(round(value, 7))]
     return practice
 
 
-def _run(cycle: dict):
-    site = cycle.get('site')
-    region_factor = get_region_factor(TERM_ID, site, TermTermType.LANDUSEMANAGEMENT)
-
-    logRequirements(site, model=MODEL, term=TERM_ID,
-                    region_factor=region_factor)
-
-    should_run = all([region_factor])
-    logShouldRun(site, MODEL, TERM_ID, should_run)
-    return [_practice(region_factor)] if should_run else []
-
-
 def _should_run(cycle: dict):
     product = find_primary_product(cycle) or {}
     not_plantation = not is_plantation(MODEL, TERM_ID, product.get('term', {}).get('@id'))
@@ -65,4 +64,16 @@ def _should_run(cycle: dict):
     return should_run
 
 
-def run(cycle: dict): return _run(cycle) if _should_run(cycle) else []
+def run(cycle: dict):
+    sites = get_allowed_sites(MODEL, TERM_ID, TermTermType.LANDUSEMANAGEMENT, cycle) if _should_run(cycle) else []
+    sites = [
+        (site, get_region_factor(TERM_ID, site, TermTermType.LANDUSEMANAGEMENT)) for site in sites
+    ]
+
+    debugValues(cycle, model=MODEL, term=TERM_ID,
+                site_factors=log_as_table([{
+                    'site-id': site.get('@id', site.get('id')),
+                    'factor': factor
+                } for site, factor in sites]))
+
+    return non_empty_list([_practice(site, factor) for site, factor in sites if bool(factor)])

hestia_earth/models/ipcc2019/animal/pastureGrass.py

@@ -18,6 +18,7 @@ from hestia_earth.models.utils.input import _new_input
 from hestia_earth.models.utils.term import get_wool_terms, get_lookup_value
 from hestia_earth.models.utils.completeness import _is_term_type_complete, _is_term_type_incomplete
 from hestia_earth.models.utils.property import get_node_property
+from hestia_earth.models.utils.cycle import get_animals_by_period
 from .. import MODEL
 from ..pastureGrass_utils import (
     practice_input_id,
@@ -29,7 +30,6 @@ from ..pastureGrass_utils import (
     calculate_NEfeed,
     calculate_GE,
     product_wool_energy,
-    get_animals,
     get_animal_values
 )
 
@@ -308,7 +308,7 @@ def _should_run(cycle: dict, animals: list, practices: dict):
 
 
 def run(cycle: dict):
-    animals = get_animals(cycle)
+    animals = get_animals_by_period(cycle)
     practices = list(filter(should_run_practice(cycle), cycle.get('practices', [])))
     should_run, meanDE, meanECHHV, system = _should_run(cycle, animals, practices)
     return list(map(_run_animal(cycle, meanDE, meanECHHV, system, practices), animals)) if should_run else []

hestia_earth/models/ipcc2019/pastureGrass.py

@@ -17,6 +17,7 @@ from hestia_earth.models.utils.blank_node import lookups_logs, properties_logs
 from hestia_earth.models.utils.input import _new_input
 from hestia_earth.models.utils.term import get_wool_terms
 from hestia_earth.models.utils.completeness import _is_term_type_complete, _is_term_type_incomplete
+from hestia_earth.models.utils.cycle import get_animals_by_period
 from . import MODEL
 from .pastureGrass_utils import (
     practice_input_id,
@@ -28,7 +29,6 @@ from .pastureGrass_utils import (
     calculate_NEfeed,
     calculate_GE,
     product_wool_energy,
-    get_animals,
     get_animal_values
 )
 
@@ -164,7 +164,7 @@ def calculate_NEwool(cycle: dict) -> float:
 
 
 def _run_practice(cycle: dict, meanDE: float, meanECHHV: float, system: dict):
-    animals = get_animals(cycle)
+    animals = get_animals_by_period(cycle)
     REM = calculate_REM(meanDE)
     REG = calculate_REG(meanDE)
     NEwool = calculate_NEwool(cycle)

hestia_earth/models/ipcc2019/pastureGrass_utils.py

@@ -1,4 +1,4 @@
-from hestia_earth.schema import TermTermType, AnimalReferencePeriod
+from hestia_earth.schema import TermTermType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, extract_grouped_data
 from hestia_earth.utils.tools import list_sum, safe_parse_float
@@ -341,12 +341,3 @@ def should_run_practice(cycle: dict):
         return all([len(value) > 0, term_id == MODEL_KEY, key_term_type in KEY_TERM_TYPES])
 
     return should_run
-
-
-def get_animals(cycle: dict):
-    return [
-        a for a in cycle.get('animals', []) if all([
-            a.get('value'),
-            a.get('referencePeriod') == AnimalReferencePeriod.AVERAGE.value
-        ])
-    ]

hestia_earth/models/mocking/mock_search.py

@@ -52,20 +52,26 @@ def _load_results(filepath: str):
 
 def _find_search_result(filepath: str, query: dict):
     search_results = _load_results(filepath)
-    res = next((n for n in search_results if n['query'] == query), {})
-    return res.get('results', [])
+    res = next((n for n in search_results if n['query'] == query), None)
+    return None if res is None else res.get('results', [])
 
 
-def _fake_search(filepath: str):
-    def mock(query: dict, *_a, **_b): return _find_search_result(filepath, query)
+def _mocked_search(original_func, filepath: str):
+    def mock(query: dict, **kwargs):
+        result = _find_search_result(filepath, query)
+        return original_func(query, **kwargs) if result is None else result
     return mock
 
 
-def _fake_find_node(filepath: str):
-    def mock(_n, query: dict, *_a, **_b): return _find_search_result(filepath, query)
+def _mocked_find_node(original_func, filepath: str):
+    def mock(node_type: str, query: dict, **kwargs):
+        result = _find_search_result(filepath, query)
+        return original_func(node_type, query, **kwargs) if result is None else result
     return mock
 
 
 def mock(filepath: str):
-    term.search = _fake_search(filepath)
-    term.find_node = _fake_find_node(filepath)
+    original_search = term.search
+    term.search = _mocked_search(original_search, filepath)
+    original_find_node = term.find_node
+    term.find_node = _mocked_find_node(original_find_node, filepath)

hestia_earth/models/mocking/search-results.json

@@ -1054,11 +1054,11 @@
     "results": [
       {
         "@type": "Term",
-        "@id": "residueIncorporatedLessThan30DaysBeforeCultivation"
+        "@id": "residueIncorporated"
       },
       {
         "@type": "Term",
-        "@id": "residueIncorporated"
+        "@id": "residueIncorporatedMoreThan30DaysBeforeCultivation"
       },
       {
         "@type": "Term",
@@ -1066,7 +1066,7 @@
       },
       {
         "@type": "Term",
-        "@id": "residueIncorporatedMoreThan30DaysBeforeCultivation"
+        "@id": "residueIncorporatedLessThan30DaysBeforeCultivation"
       },
       {
         "@type": "Term",
@@ -1110,27 +1110,27 @@
       },
       {
         "@type": "Term",
-        "@id": "aboveGroundCropResidueBurnt"
+        "@id": "discardedCropRemoved"
       },
       {
         "@type": "Term",
-        "@id": "belowGroundCropResidue"
+        "@id": "aboveGroundCropResidueTotal"
       },
       {
         "@type": "Term",
-        "@id": "aboveGroundCropResidueLeftOnField"
+        "@id": "aboveGroundCropResidueIncorporated"
       },
       {
         "@type": "Term",
-        "@id": "discardedCropRemoved"
+        "@id": "aboveGroundCropResidueBurnt"
       },
       {
         "@type": "Term",
-        "@id": "aboveGroundCropResidueTotal"
+        "@id": "belowGroundCropResidue"
       },
       {
         "@type": "Term",
-        "@id": "aboveGroundCropResidueIncorporated"
+        "@id": "aboveGroundCropResidueLeftOnField"
       }
     ]
   },
@@ -1486,7 +1486,7 @@
         "@type": "Term",
         "name": "Generic crop, seed",
         "@id": "genericCropSeed",
-        "_score": 25.968466
+        "_score": 26.344206
       }
     ]
   },
@@ -1534,10 +1534,6 @@
         "@type": "Term",
         "@id": "rainfedDeepWaterWaterDepth100Cm"
       },
-      {
-        "@type": "Term",
-        "@id": "irrigatedTypeUnspecified"
-      },
       {
         "@type": "Term",
         "@id": "irrigatedLocalizedIrrigation"
@@ -1550,6 +1546,10 @@
         "@type": "Term",
         "@id": "irrigatedSubIrrigation"
       },
+      {
+        "@type": "Term",
+        "@id": "irrigatedTypeUnspecified"
+      },
       {
         "@type": "Term",
         "@id": "irrigatedDripIrrigation"
@@ -1722,61 +1722,61 @@
         "@type": "Term",
         "name": "Glass or high accessible cover",
         "@id": "glassOrHighAccessibleCover",
-        "_score": 60.694187
+        "_score": 59.78251
       },
       {
         "@type": "Term",
         "name": "River or stream",
         "@id": "riverOrStream",
-        "_score": 50.06671
+        "_score": 51.118263
       },
       {
         "@type": "Term",
         "name": "Other natural vegetation",
         "@id": "otherNaturalVegetation",
-        "_score": 40.722
+        "_score": 41.25843
       },
       {
         "@type": "Term",
         "name": "Natural forest",
         "@id": "naturalForest",
-        "_score": 31.252472
+        "_score": 31.747753
       },
       {
         "@type": "Term",
         "name": "Permanent pasture",
         "@id": "permanentPasture",
-        "_score": 27.857792
+        "_score": 28.454437
       },
       {
         "@type": "Term",
         "name": "Animal housing",
         "@id": "animalHousing",
-        "_score": 26.769783
+        "_score": 27.424242
       },
       {
         "@type": "Term",
         "name": "Root or tuber crop plant",
         "@id": "rootOrTuberCropPlant",
-        "_score": 24.781918
+        "_score": 25.260658
       },
       {
         "@type": "Term",
         "name": "High intensity grazing pasture",
         "@id": "highIntensityGrazingPasture",
-        "_score": 23.502655
+        "_score": 24.038225
       },
       {
         "@type": "Term",
         "name": "Permanent cropland",
         "@id": "permanentCropland",
-        "_score": 20.467493
+        "_score": 20.689457
       },
       {
         "@type": "Term",
         "name": "Forest",
         "@id": "forest",
-        "_score": 19.942879
+        "_score": 20.366226
       }
     ]
   },
@@ -2024,15 +2024,15 @@
     "results": [
       {
         "@type": "Term",
-        "@id": "verticalTillage"
+        "@id": "noTillage"
       },
       {
         "@type": "Term",
-        "@id": "noTillage"
+        "@id": "ridgeTillage"
       },
       {
         "@type": "Term",
-        "@id": "ridgeTillage"
+        "@id": "verticalTillage"
       },
       {
         "@type": "Term",

hestia_earth/models/pooreNemecek2018/longFallowPeriod.py

@@ -7,8 +7,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}],
-        "site": {"@type": "Site", "siteType": "cropland"}
+        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}]
     }
 }
 LOOKUPS = {

hestia_earth/models/pooreNemecek2018/nurseryDensity.py

@@ -7,8 +7,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}],
-        "site": {"@type": "Site", "siteType": "cropland"}
+        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}]
     }
 }
 LOOKUPS = {

hestia_earth/models/pooreNemecek2018/nurseryDuration.py

@@ -7,8 +7,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}],
-        "site": {"@type": "Site", "siteType": "cropland"}
+        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}]
     }
 }
 LOOKUPS = {

hestia_earth/models/pooreNemecek2018/plantationDensity.py

@@ -7,8 +7,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}],
-        "site": {"@type": "Site", "siteType": "cropland"}
+        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}]
     }
 }
 LOOKUPS = {

hestia_earth/models/pooreNemecek2018/plantationLifespan.py

@@ -7,8 +7,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}],
-        "site": {"@type": "Site", "siteType": "cropland"}
+        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}]
     }
 }
 LOOKUPS = {

hestia_earth/models/pooreNemecek2018/plantationProductiveLifespan.py

@@ -9,8 +9,7 @@ from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}],
-        "site": {"@type": "Site", "siteType": "cropland"}
+        "products": [{"@type": "Product", "value": "", "term.termType": "crop"}]
     }
 }
 LOOKUPS = {

hestia_earth/models/preload_requests.py

@@ -2,13 +2,14 @@
 Preload all search requests to avoid making the same searches many times while running models.
 """
 import json
+import os
 
 from .log import logger
 from .mocking.mock_search import create_search_results
 from .mocking import RESULTS_PATH, enable_mock as _mock
 
 
-def enable_preload(filepath: str = RESULTS_PATH, node: dict = None):
+def enable_preload(filepath: str = RESULTS_PATH, node: dict = None, overwrite_existing: bool = True):
     """
     Prefetch calls to HESTIA API in a local file.
 
@@ -18,15 +19,21 @@ def enable_preload(filepath: str = RESULTS_PATH, node: dict = None):
         The path of the file containing the search results. Defaults to current library folder.
     node : dict
         Optional - The node used to run calculations. This is especially useful when running calculations on a Site.
+    overwrite_existing : bool
+        Optional - If the file already exists, the file can be used instead of generating it again.
+        Will overwrite by default.
     """
-    logger.debug('Preloading search results and storing in %s', filepath)
+    should_generate = overwrite_existing or not os.path.exists(filepath)
 
-    # build the search results
-    data = create_search_results()
+    if should_generate:
+        logger.debug('Preloading search results and storing in %s', filepath)
 
-    # store in file
-    with open(filepath, 'w') as f:
-        f.write(json.dumps(data, indent=2, ensure_ascii=False))
+        # build the search results
+        data = create_search_results()
+
+        # store in file
+        with open(filepath, 'w') as f:
+            f.write(json.dumps(data, indent=2, ensure_ascii=False))
 
     # enable mock search results from file
     _mock(filepath=filepath, node=node)

hestia_earth/models/site/management.py

@@ -18,7 +18,7 @@ from functools import reduce
 from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import filter_list_term_type, linked_node
-from hestia_earth.utils.tools import safe_parse_float, flatten
+from hestia_earth.utils.tools import safe_parse_float, flatten, non_empty_list
 from hestia_earth.utils.blank_node import get_node_value
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_blank_nodes_id
@@ -48,7 +48,8 @@ REQUIREMENTS = {
                             "waterRegime",
                             "tillage",
                             "cropResidueManagement",
-                            "landUseManagement"
+                            "landUseManagement",
+                            "system"
                         ],
                         "units": ["% area", "boolean"],
                         "value": ""
@@ -72,7 +73,7 @@ RETURNS = {
     "Management": [{
         "@type": "Management",
         "term.termType": [
-            "landCover", "waterRegime", "tillage", "cropResidueManagement", "landUseManagement"
+            "landCover", "waterRegime", "tillage", "cropResidueManagement", "landUseManagement", "system"
         ],
         "value": "",
         "endDate": "",
@@ -247,7 +248,7 @@ def _should_run_all_products(cycles: list, site_type: str):
         for product in list(filter(_get_landCover_term_id, [i for i in products_crop_forage]))
     ] if site_type else []
     dates = sorted(list(set(
-        flatten([[cycle.get('startDate'), cycle.get('endDate')] for cycle in cycles])
+        non_empty_list(flatten([[cycle.get('startDate'), cycle.get('endDate')] for cycle in cycles]))
     ))) if site_type not in _SKIP_LAND_COVER_SITE_TYPES else []
     site_type_term = download_hestia(get_landCover_term_id_from_site_type(site_type)) if all([
         len(dates) >= 2,
@@ -286,7 +287,8 @@ def _should_run(site: dict):
                 TermTermType.WATERREGIME,
                 TermTermType.TILLAGE,
                 TermTermType.CROPRESIDUEMANAGEMENT,
-                TermTermType.LANDUSEMANAGEMENT
+                TermTermType.LANDUSEMANAGEMENT,
+                TermTermType.SYSTEM
             ]
         )
     ]

hestia_earth/models/utils/__init__.py

@@ -122,15 +122,6 @@ def get_kg_term_units(term_id: str, units: str):
     }.get(units, lambda x: None)(term_id)
 
 
-def _get_by_key(x, y):
-    return x if x is None else (
-        x.get(y) if isinstance(x, dict) else list(map(lambda v: get_dict_key(v, y), x))
-    )
-
-
-def get_dict_key(value: dict, key: str): return reduce(lambda x, y: _get_by_key(x, y), key.split('.'), value)
-
-
 def first_day_of_month(year: int, month: int):
     return datetime.date(int(year), int(month), 1)