hestia-earth-models 0.63.0__tar.gz → 0.64.0__tar.gz

{hestia-earth-models-0.63.0/hestia_earth_models.egg-info → hestia-earth-models-0.64.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.63.0
+Version: 0.64.0
 Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: HESTIA Team

hestia-earth-models-0.64.0/hestia_earth/models/cycle/unknownPreSeasonWaterRegime.py

@@ -0,0 +1,51 @@
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.term import get_flooded_pre_season_terms
+from hestia_earth.models.utils.practice import _new_practice
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "none": {
+            "practices": [{
+                "@type": "Practice",
+                "term.termType": "landUseManagement",
+                "term.@id": [
+                    "nonFloodedPreSeasonLessThan180Days",
+                    "nonFloodedPreSeasonMoreThan180Days",
+                    "nonFloodedPreSeasonMoreThan365Days",
+                    "floodedPreSeasonMoreThan30Days"
+                ]
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Practice": [{
+        "value": "100"
+    }]
+}
+TERM_ID = 'unknownPreSeasonWaterRegime'
+
+
+def _practice():
+    practice = _new_practice(TERM_ID)
+    practice['value'] = [100]
+    return practice
+
+
+def _should_run(cycle: dict):
+    practices = cycle.get('practices', [])
+    flooded_terms = get_flooded_pre_season_terms()
+    print(flooded_terms)
+    existing_practice = next((p for p in practices if p.get('term', {}).get('@id') in flooded_terms), None)
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    existing_practice=existing_practice)
+
+    should_run = all([not existing_practice])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+    return should_run
+
+
+def run(cycle: dict):
+    return [_practice()] if _should_run(cycle) else []

{hestia-earth-models-0.63.0 → hestia-earth-models-0.64.0}/hestia_earth/models/ecoinventV3/__init__.py

@@ -19,6 +19,7 @@ from hestia_earth.utils.tools import flatten, list_sum
 
 from hestia_earth.models.log import debugValues, logShouldRun, logRequirements
 from hestia_earth.models.data.ecoinventV3 import ecoinventV3_emissions
+from hestia_earth.models.utils import is_from_model
 from hestia_earth.models.utils.emission import _new_emission
 from hestia_earth.models.utils.blank_node import group_by_keys
 from hestia_earth.models.utils.pesticideAI import get_pesticides_from_inputs
@@ -109,18 +110,33 @@ def _add_emission(cycle: dict, input: dict):
 
 
 def _run_input(cycle: dict):
+    emissions = cycle.get('emissions', [])
+
     def run(inputs: list):
         input = inputs[0]
         input_term_id = input.get('term', {}).get('@id')
+        operation_term_id = input.get('operation', {}).get('@id')
+        animal_term_id = input.get('animal', {}).get('@id')
         input_value = list_sum(flatten(input.get('value', []) for input in inputs))
         mappings = get_input_mappings(MODEL, cycle, input)
         has_mappings = len(mappings) > 0
+        # skip input that has background emissions we have already gap-filled (model run before)
+        has_no_gap_filled_background_emissions = not any([
+            is_from_model(e)
+            for e in emissions
+            if all([
+                any([i.get('@id') == input_term_id for i in e.get('inputs', [])]),
+                e.get('operation', {}).get('@id') == operation_term_id,
+                e.get('animal', {}).get('@id') == animal_term_id
+            ])
+        ])
 
         logRequirements(cycle, model=MODEL, term=input_term_id,
                         has_ecoinvent_mappings=has_mappings,
-                        ecoinvent_mappings=';'.join([v[0] for v in mappings]))
+                        ecoinvent_mappings=';'.join([v[0] for v in mappings]),
+                        has_no_gap_filled_background_emissions=has_no_gap_filled_background_emissions)
 
-        should_run = all([has_mappings])
+        should_run = all([has_mappings, has_no_gap_filled_background_emissions])
         logShouldRun(cycle, MODEL, input_term_id, should_run, methodTier=TIER)
         grouped_emissions = reduce(_add_emission(cycle, input), mappings, {}) if should_run else {}
         return [

{hestia-earth-models-0.63.0 → hestia-earth-models-0.64.0}/hestia_earth/models/faostat2018/landTransformationFromCropland100YearAverage.py

@@ -16,7 +16,7 @@ REQUIREMENTS = {
     }
 }
 LOOKUPS = {
-    "region-faostatCroplandArea": ""
+    "region-faostatArea": ""
 }
 RETURNS = {
     "Indicator": [{

{hestia-earth-models-0.63.0 → hestia-earth-models-0.64.0}/hestia_earth/models/faostat2018/landTransformationFromCropland20YearAverage.py

@@ -16,7 +16,7 @@ REQUIREMENTS = {
     }
 }
 LOOKUPS = {
-    "region-faostatCroplandArea": ""
+    "region-faostatArea": ""
 }
 RETURNS = {
     "Indicator": [{

{hestia-earth-models-0.63.0 → hestia-earth-models-0.64.0}/hestia_earth/models/faostat2018/utils.py

@@ -62,14 +62,14 @@ def _cropland_split_delta(table_value: str, start_year: int, end_year: int):
 
 
 def get_cropland_ratio(country: str, start_year: int, end_year: int):
-    lookup = download_lookup('region-faostatCroplandArea.csv')
+    lookup = download_lookup('region-faostatArea.csv')
     total_delta = _cropland_split_delta(
         get_table_value(lookup, 'termid', country, column_name('Cropland')), start_year, end_year
     )
 
     # get both values and only return result if we have both
     permanent_delta = _cropland_split_delta(
-        get_table_value(lookup, 'termid', country, column_name('Land under permanent crops')), start_year, end_year
+        get_table_value(lookup, 'termid', country, column_name('Permanent crops')), start_year, end_year
     )
     temporary_delta = _cropland_split_delta(
         get_table_value(lookup, 'termid', country, column_name('Arable land')), start_year, end_year

{hestia-earth-models-0.63.0 → hestia-earth-models-0.64.0}/hestia_earth/models/geospatialDatabase/clayContent.py

@@ -1,4 +1,5 @@
 from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.blank_node import has_original_by_ids
@@ -29,37 +30,63 @@ REQUIREMENTS = {
 RETURNS = {
     "Measurement": [{
         "value": "",
-        "depthUpper": "0",
-        "depthLower": "30",
+        "depthUpper": "",
+        "depthLower": "",
         "methodClassification": "geospatial dataset"
     }]
 }
 TERM_ID = 'clayContent'
-EE_PARAMS = {
-    'collection': 'T_CLAY',
-    'ee_type': 'raster',
-    'reducer': 'mean'
-}
-BIBLIO_TITLE = 'Harmonized World Soil Database Version 1.2. Food and Agriculture Organization of the United Nations (FAO).'  # noqa: E501
+EE_PARAMS = [
+    {
+        'collection': 'T_CLAY_v2_depth_1',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 0,
+        'depthLower': 20
+    },
+    {
+        'collection': 'T_CLAY_v2_depth_2',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 20,
+        'depthLower': 40
+    },
+    {
+        'collection': 'T_CLAY_v2_depth_3',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 40,
+        'depthLower': 60
+    },
+    {
+        'collection': 'T_CLAY_v2_depth_4',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 60,
+        'depthLower': 80
+    }
+]
+BIBLIO_TITLE = 'Harmonized World Soil Database Version 2.0.'
 
 
-def _measurement(site: dict, value: int):
-    measurement = _new_measurement(TERM_ID, None)
+def _measurement(site: dict, value: int, depthUpper: int, depthLower: int):
+    measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
-    measurement['depthUpper'] = 0
-    measurement['depthLower'] = 30
+    measurement['depthUpper'] = depthUpper
+    measurement['depthLower'] = depthLower
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
-def _download(site: dict):
-    value = download(TERM_ID, site, EE_PARAMS)
-    return None if value is None else round(value, 2)
+def _run_depths(site: dict, params: dict):
+    value = download(TERM_ID, site, params)
+    return None if value is None else (
+        _measurement(site, round(value, 2), params.get('depthUpper'), params.get('depthLower'))
+    )
 
 
 def _run(site: dict):
-    value = _download(site)
-    return [_measurement(site, value)] if value is not None else []
+    return non_empty_list([_run_depths(site, params) for params in EE_PARAMS])
 
 
 def _should_run(site: dict):

{hestia-earth-models-0.63.0 → hestia-earth-models-0.64.0}/hestia_earth/models/geospatialDatabase/organicCarbonPerKgSoil.py

@@ -1,5 +1,5 @@
 from hestia_earth.schema import MeasurementMethodClassification
-from hestia_earth.utils.tools import safe_parse_float
+from hestia_earth.utils.tools import safe_parse_float, non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.measurement import _new_measurement
@@ -19,37 +19,63 @@ REQUIREMENTS = {
 RETURNS = {
     "Measurement": [{
         "value": "",
-        "depthUpper": "0",
-        "depthLower": "30",
+        "depthUpper": "",
+        "depthLower": "",
         "methodClassification": "geospatial dataset"
     }]
 }
 TERM_ID = 'organicCarbonPerKgSoil'
-EE_PARAMS = {
-    'collection': 'T_OC',
-    'ee_type': 'raster',
-    'reducer': 'mean'
-}
-BIBLIO_TITLE = 'Harmonized World Soil Database Version 1.2. Food and Agriculture Organization of the United Nations (FAO).'  # noqa: E501
+EE_PARAMS = [
+    {
+        'collection': 'T_OC_v2_depth_1',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 0,
+        'depthLower': 20
+    },
+    {
+        'collection': 'T_OC_v2_depth_2',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 20,
+        'depthLower': 40
+    },
+    {
+        'collection': 'T_OC_v2_depth_3',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 40,
+        'depthLower': 60
+    },
+    {
+        'collection': 'T_OC_v2_depth_4',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 60,
+        'depthLower': 80
+    }
+]
+BIBLIO_TITLE = 'Harmonized World Soil Database Version 2.0.'
 
 
-def _measurement(site: dict, value: float):
+def _measurement(site: dict, value: int, depthUpper: int, depthLower: int):
     measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
-    measurement['depthUpper'] = 0
-    measurement['depthLower'] = 30
+    measurement['depthUpper'] = depthUpper
+    measurement['depthLower'] = depthLower
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
-def _download(site: dict):
-    value = download(TERM_ID, site, EE_PARAMS)
-    return safe_parse_float(value) * 10 if value is not None else None
+def _run_depths(site: dict, params: dict):
+    value = download(TERM_ID, site, params)
+    return None if value is None else (
+        _measurement(site, safe_parse_float(value) * 10, params.get('depthUpper'), params.get('depthLower'))
+    )
 
 
 def _run(site: dict):
-    value = _download(site)
-    return [_measurement(site, value)] if value is not None else []
+    return non_empty_list([_run_depths(site, params) for params in EE_PARAMS])
 
 
 def _should_run(site: dict):

{hestia-earth-models-0.63.0 → hestia-earth-models-0.64.0}/hestia_earth/models/geospatialDatabase/sandContent.py

@@ -1,4 +1,5 @@
 from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.blank_node import has_original_by_ids
@@ -29,37 +30,63 @@ REQUIREMENTS = {
 RETURNS = {
     "Measurement": [{
         "value": "",
-        "depthUpper": "0",
-        "depthLower": "30",
+        "depthUpper": "",
+        "depthLower": "",
         "methodClassification": "geospatial dataset"
     }]
 }
 TERM_ID = 'sandContent'
-EE_PARAMS = {
-    'collection': 'T_SAND',
-    'ee_type': 'raster',
-    'reducer': 'mean'
-}
-BIBLIO_TITLE = 'Harmonized World Soil Database Version 1.2. Food and Agriculture Organization of the United Nations (FAO).'  # noqa: E501
+EE_PARAMS = [
+    {
+        'collection': 'T_SAND_v2_depth_1',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 0,
+        'depthLower': 20
+    },
+    {
+        'collection': 'T_SAND_v2_depth_2',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 20,
+        'depthLower': 40
+    },
+    {
+        'collection': 'T_SAND_v2_depth_3',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 40,
+        'depthLower': 60
+    },
+    {
+        'collection': 'T_SAND_v2_depth_4',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 60,
+        'depthLower': 80
+    }
+]
+BIBLIO_TITLE = 'Harmonized World Soil Database Version 2.0.'
 
 
-def _measurement(site: dict, value: int):
+def _measurement(site: dict, value: int, depthUpper: int, depthLower: int):
     measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
-    measurement['depthUpper'] = 0
-    measurement['depthLower'] = 30
+    measurement['depthUpper'] = depthUpper
+    measurement['depthLower'] = depthLower
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
-def _download(site: dict):
-    value = download(TERM_ID, site, EE_PARAMS)
-    return None if value is None else round(value, 2)
+def _run_depths(site: dict, params: dict):
+    value = download(TERM_ID, site, params)
+    return None if value is None else (
+        _measurement(site, round(value, 2), params.get('depthUpper'), params.get('depthLower'))
+    )
 
 
 def _run(site: dict):
-    value = _download(site)
-    return [_measurement(site, value)] if value is not None else []
+    return non_empty_list([_run_depths(site, params) for params in EE_PARAMS])
 
 
 def _should_run(site: dict):

hestia-earth-models-0.64.0/hestia_earth/models/geospatialDatabase/siltContent.py

@@ -0,0 +1,105 @@
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.model import find_term_match
+from hestia_earth.utils.tools import non_empty_list, flatten
+
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils.measurement import _new_measurement, measurement_value, group_measurements_by_depth
+from hestia_earth.models.utils.source import get_source
+from . import MODEL, clayContent, sandContent
+
+REQUIREMENTS = {
+    "Site": {
+        "measurements": [
+            {
+                "@type": "Measurement",
+                "term.@id": "clayContent",
+                "value": ">= 0",
+                "depthUpper": ">= 0",
+                "depthLower": ">= 0"
+            },
+            {
+                "@type": "Measurement",
+                "term.@id": "sandContent",
+                "value": ">= 0",
+                "depthUpper": ">= 0",
+                "depthLower": ">= 0"
+            }
+        ],
+        "or": [
+            {"latitude": "", "longitude": ""},
+            {"boundary": {}},
+            {"region": {"@type": "Term", "termType": "region"}}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "depthUpper": "",
+        "depthLower": "",
+        "methodClassification": "geospatial dataset"
+    }]
+}
+TERM_ID = 'siltContent'
+OTHER_TERM_IDS = [
+    clayContent.TERM_ID,
+    sandContent.TERM_ID
+]
+BIBLIO_TITLE = 'Harmonized World Soil Database Version 2.0.'
+
+
+def _measurement(site: dict, value: int, depthUpper: int, depthLower: int):
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = [value]
+    measurement['depthUpper'] = depthUpper
+    measurement['depthLower'] = depthLower
+    measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
+    return measurement | get_source(site, BIBLIO_TITLE)
+
+
+def _run(site: dict, measurements: list):
+    value = 100 - sum([measurement_value(m) for m in measurements])
+    return [_measurement(site, value, measurements[0].get('depthUpper'), measurements[0].get('depthLower'))]
+
+
+def _should_run(site: dict):
+    grouped_measurements = group_measurements_by_depth(site.get('measurements', []), include_dates=False)
+    relevant_measurements = [
+        [
+            m for m in measurements if all([
+                m.get('term', {}).get('@id') in OTHER_TERM_IDS,
+                len(m.get('value', [])) > 0,
+                m.get('depthUpper', -1) >= 0,
+                m.get('depthLower', -1) >= 0
+            ])
+        ]
+        for measurements in grouped_measurements.values()
+    ]
+    valid_measurements = [
+        values for values in relevant_measurements if len(values) == len(OTHER_TERM_IDS)
+    ]
+
+    has_valid_measurements = len(valid_measurements) > 0
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    has_valid_measurements=has_valid_measurements,
+                    measurements=log_as_table([
+                        {
+                            'depths': '-'.join(non_empty_list([
+                                str(values[0].get('depthUpper', '')),
+                                str(values[0].get('depthLower', ''))
+                            ]))
+                        } | {
+                            term_id: measurement_value(find_term_match(values, term_id))
+                            for term_id in OTHER_TERM_IDS
+                        } for values in valid_measurements
+                    ]))
+
+    should_run = all([has_valid_measurements])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+    return should_run, valid_measurements
+
+
+def run(site: dict):
+    should_run, measurements = _should_run(site)
+    return flatten([_run(site, values) for values in measurements]) if should_run else []

{hestia-earth-models-0.63.0 → hestia-earth-models-0.64.0}/hestia_earth/models/geospatialDatabase/soilPh.py

@@ -1,4 +1,5 @@
 from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.measurement import _new_measurement
@@ -18,37 +19,63 @@ REQUIREMENTS = {
 RETURNS = {
     "Measurement": [{
         "value": "",
-        "depthUpper": "0",
-        "depthLower": "30",
+        "depthUpper": "",
+        "depthLower": "",
         "methodClassification": "geospatial dataset"
     }]
 }
 TERM_ID = 'soilPh'
-EE_PARAMS = {
-    'collection': 'T_PH_H2O',
-    'ee_type': 'raster',
-    'reducer': 'mean'
-}
-BIBLIO_TITLE = 'Harmonized World Soil Database Version 1.2. Food and Agriculture Organization of the United Nations (FAO).'  # noqa: E501
+EE_PARAMS = [
+    {
+        'collection': 'T_PH_H2O_v2_depth_1',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 0,
+        'depthLower': 20
+    },
+    {
+        'collection': 'T_PH_H2O_v2_depth_2',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 20,
+        'depthLower': 40
+    },
+    {
+        'collection': 'T_PH_H2O_v2_depth_3',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 40,
+        'depthLower': 60
+    },
+    {
+        'collection': 'T_PH_H2O_v2_depth_4',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 60,
+        'depthLower': 80
+    }
+]
+BIBLIO_TITLE = 'Harmonized World Soil Database Version 2.0.'
 
 
-def _measurement(site: dict, value: float):
+def _measurement(site: dict, value: int, depthUpper: int, depthLower: int):
     measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
-    measurement['depthUpper'] = 0
-    measurement['depthLower'] = 30
+    measurement['depthUpper'] = depthUpper
+    measurement['depthLower'] = depthLower
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
-def _download(site: dict):
-    value = download(TERM_ID, site, EE_PARAMS)
-    return None if value is None else round(value, 7)
+def _run_depths(site: dict, params: dict):
+    value = download(TERM_ID, site, params)
+    return None if value is None else (
+        _measurement(site, round(value, 7), params.get('depthUpper'), params.get('depthLower'))
+    )
 
 
 def _run(site: dict):
-    value = _download(site)
-    return [] if value is None else [_measurement(site, value)]
+    return non_empty_list([_run_depths(site, params) for params in EE_PARAMS])
 
 
 def _should_run(site: dict):