hestia-earth-models 0.63.0__py3-none-any.whl → 0.64.0__py3-none-any.whl

hestia_earth/models/cycle/unknownPreSeasonWaterRegime.py

@@ -0,0 +1,51 @@
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.term import get_flooded_pre_season_terms
+from hestia_earth.models.utils.practice import _new_practice
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "none": {
+            "practices": [{
+                "@type": "Practice",
+                "term.termType": "landUseManagement",
+                "term.@id": [
+                    "nonFloodedPreSeasonLessThan180Days",
+                    "nonFloodedPreSeasonMoreThan180Days",
+                    "nonFloodedPreSeasonMoreThan365Days",
+                    "floodedPreSeasonMoreThan30Days"
+                ]
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Practice": [{
+        "value": "100"
+    }]
+}
+TERM_ID = 'unknownPreSeasonWaterRegime'
+
+
+def _practice():
+    practice = _new_practice(TERM_ID)
+    practice['value'] = [100]
+    return practice
+
+
+def _should_run(cycle: dict):
+    practices = cycle.get('practices', [])
+    flooded_terms = get_flooded_pre_season_terms()
+    print(flooded_terms)
+    existing_practice = next((p for p in practices if p.get('term', {}).get('@id') in flooded_terms), None)
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    existing_practice=existing_practice)
+
+    should_run = all([not existing_practice])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+    return should_run
+
+
+def run(cycle: dict):
+    return [_practice()] if _should_run(cycle) else []

hestia_earth/models/ecoinventV3/__init__.py

@@ -19,6 +19,7 @@ from hestia_earth.utils.tools import flatten, list_sum
 
 from hestia_earth.models.log import debugValues, logShouldRun, logRequirements
 from hestia_earth.models.data.ecoinventV3 import ecoinventV3_emissions
+from hestia_earth.models.utils import is_from_model
 from hestia_earth.models.utils.emission import _new_emission
 from hestia_earth.models.utils.blank_node import group_by_keys
 from hestia_earth.models.utils.pesticideAI import get_pesticides_from_inputs
@@ -109,18 +110,33 @@ def _add_emission(cycle: dict, input: dict):
 
 
 def _run_input(cycle: dict):
+    emissions = cycle.get('emissions', [])
+
     def run(inputs: list):
         input = inputs[0]
         input_term_id = input.get('term', {}).get('@id')
+        operation_term_id = input.get('operation', {}).get('@id')
+        animal_term_id = input.get('animal', {}).get('@id')
         input_value = list_sum(flatten(input.get('value', []) for input in inputs))
         mappings = get_input_mappings(MODEL, cycle, input)
         has_mappings = len(mappings) > 0
+        # skip input that has background emissions we have already gap-filled (model run before)
+        has_no_gap_filled_background_emissions = not any([
+            is_from_model(e)
+            for e in emissions
+            if all([
+                any([i.get('@id') == input_term_id for i in e.get('inputs', [])]),
+                e.get('operation', {}).get('@id') == operation_term_id,
+                e.get('animal', {}).get('@id') == animal_term_id
+            ])
+        ])
 
         logRequirements(cycle, model=MODEL, term=input_term_id,
                         has_ecoinvent_mappings=has_mappings,
-                        ecoinvent_mappings=';'.join([v[0] for v in mappings]))
+                        ecoinvent_mappings=';'.join([v[0] for v in mappings]),
+                        has_no_gap_filled_background_emissions=has_no_gap_filled_background_emissions)
 
-        should_run = all([has_mappings])
+        should_run = all([has_mappings, has_no_gap_filled_background_emissions])
         logShouldRun(cycle, MODEL, input_term_id, should_run, methodTier=TIER)
         grouped_emissions = reduce(_add_emission(cycle, input), mappings, {}) if should_run else {}
         return [

hestia_earth/models/faostat2018/landTransformationFromCropland100YearAverage.py

@@ -16,7 +16,7 @@ REQUIREMENTS = {
     }
 }
 LOOKUPS = {
-    "region-faostatCroplandArea": ""
+    "region-faostatArea": ""
 }
 RETURNS = {
     "Indicator": [{

hestia_earth/models/faostat2018/landTransformationFromCropland20YearAverage.py

@@ -16,7 +16,7 @@ REQUIREMENTS = {
     }
 }
 LOOKUPS = {
-    "region-faostatCroplandArea": ""
+    "region-faostatArea": ""
 }
 RETURNS = {
     "Indicator": [{

hestia_earth/models/faostat2018/utils.py

@@ -62,14 +62,14 @@ def _cropland_split_delta(table_value: str, start_year: int, end_year: int):
 
 
 def get_cropland_ratio(country: str, start_year: int, end_year: int):
-    lookup = download_lookup('region-faostatCroplandArea.csv')
+    lookup = download_lookup('region-faostatArea.csv')
     total_delta = _cropland_split_delta(
         get_table_value(lookup, 'termid', country, column_name('Cropland')), start_year, end_year
     )
 
     # get both values and only return result if we have both
     permanent_delta = _cropland_split_delta(
-        get_table_value(lookup, 'termid', country, column_name('Land under permanent crops')), start_year, end_year
+        get_table_value(lookup, 'termid', country, column_name('Permanent crops')), start_year, end_year
     )
     temporary_delta = _cropland_split_delta(
         get_table_value(lookup, 'termid', country, column_name('Arable land')), start_year, end_year

hestia_earth/models/geospatialDatabase/clayContent.py

@@ -1,4 +1,5 @@
 from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.blank_node import has_original_by_ids
@@ -29,37 +30,63 @@ REQUIREMENTS = {
 RETURNS = {
     "Measurement": [{
         "value": "",
-        "depthUpper": "0",
-        "depthLower": "30",
+        "depthUpper": "",
+        "depthLower": "",
         "methodClassification": "geospatial dataset"
     }]
 }
 TERM_ID = 'clayContent'
-EE_PARAMS = {
-    'collection': 'T_CLAY',
-    'ee_type': 'raster',
-    'reducer': 'mean'
-}
-BIBLIO_TITLE = 'Harmonized World Soil Database Version 1.2. Food and Agriculture Organization of the United Nations (FAO).'  # noqa: E501
+EE_PARAMS = [
+    {
+        'collection': 'T_CLAY_v2_depth_1',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 0,
+        'depthLower': 20
+    },
+    {
+        'collection': 'T_CLAY_v2_depth_2',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 20,
+        'depthLower': 40
+    },
+    {
+        'collection': 'T_CLAY_v2_depth_3',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 40,
+        'depthLower': 60
+    },
+    {
+        'collection': 'T_CLAY_v2_depth_4',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 60,
+        'depthLower': 80
+    }
+]
+BIBLIO_TITLE = 'Harmonized World Soil Database Version 2.0.'
 
 
-def _measurement(site: dict, value: int):
-    measurement = _new_measurement(TERM_ID, None)
+def _measurement(site: dict, value: int, depthUpper: int, depthLower: int):
+    measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
-    measurement['depthUpper'] = 0
-    measurement['depthLower'] = 30
+    measurement['depthUpper'] = depthUpper
+    measurement['depthLower'] = depthLower
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
-def _download(site: dict):
-    value = download(TERM_ID, site, EE_PARAMS)
-    return None if value is None else round(value, 2)
+def _run_depths(site: dict, params: dict):
+    value = download(TERM_ID, site, params)
+    return None if value is None else (
+        _measurement(site, round(value, 2), params.get('depthUpper'), params.get('depthLower'))
+    )
 
 
 def _run(site: dict):
-    value = _download(site)
-    return [_measurement(site, value)] if value is not None else []
+    return non_empty_list([_run_depths(site, params) for params in EE_PARAMS])
 
 
 def _should_run(site: dict):

hestia_earth/models/geospatialDatabase/organicCarbonPerKgSoil.py

@@ -1,5 +1,5 @@
 from hestia_earth.schema import MeasurementMethodClassification
-from hestia_earth.utils.tools import safe_parse_float
+from hestia_earth.utils.tools import safe_parse_float, non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.measurement import _new_measurement
@@ -19,37 +19,63 @@ REQUIREMENTS = {
 RETURNS = {
     "Measurement": [{
         "value": "",
-        "depthUpper": "0",
-        "depthLower": "30",
+        "depthUpper": "",
+        "depthLower": "",
         "methodClassification": "geospatial dataset"
     }]
 }
 TERM_ID = 'organicCarbonPerKgSoil'
-EE_PARAMS = {
-    'collection': 'T_OC',
-    'ee_type': 'raster',
-    'reducer': 'mean'
-}
-BIBLIO_TITLE = 'Harmonized World Soil Database Version 1.2. Food and Agriculture Organization of the United Nations (FAO).'  # noqa: E501
+EE_PARAMS = [
+    {
+        'collection': 'T_OC_v2_depth_1',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 0,
+        'depthLower': 20
+    },
+    {
+        'collection': 'T_OC_v2_depth_2',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 20,
+        'depthLower': 40
+    },
+    {
+        'collection': 'T_OC_v2_depth_3',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 40,
+        'depthLower': 60
+    },
+    {
+        'collection': 'T_OC_v2_depth_4',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 60,
+        'depthLower': 80
+    }
+]
+BIBLIO_TITLE = 'Harmonized World Soil Database Version 2.0.'
 
 
-def _measurement(site: dict, value: float):
+def _measurement(site: dict, value: int, depthUpper: int, depthLower: int):
     measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
-    measurement['depthUpper'] = 0
-    measurement['depthLower'] = 30
+    measurement['depthUpper'] = depthUpper
+    measurement['depthLower'] = depthLower
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
-def _download(site: dict):
-    value = download(TERM_ID, site, EE_PARAMS)
-    return safe_parse_float(value) * 10 if value is not None else None
+def _run_depths(site: dict, params: dict):
+    value = download(TERM_ID, site, params)
+    return None if value is None else (
+        _measurement(site, safe_parse_float(value) * 10, params.get('depthUpper'), params.get('depthLower'))
+    )
 
 
 def _run(site: dict):
-    value = _download(site)
-    return [_measurement(site, value)] if value is not None else []
+    return non_empty_list([_run_depths(site, params) for params in EE_PARAMS])
 
 
 def _should_run(site: dict):

hestia_earth/models/geospatialDatabase/sandContent.py

@@ -1,4 +1,5 @@
 from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.blank_node import has_original_by_ids
@@ -29,37 +30,63 @@ REQUIREMENTS = {
 RETURNS = {
     "Measurement": [{
         "value": "",
-        "depthUpper": "0",
-        "depthLower": "30",
+        "depthUpper": "",
+        "depthLower": "",
         "methodClassification": "geospatial dataset"
     }]
 }
 TERM_ID = 'sandContent'
-EE_PARAMS = {
-    'collection': 'T_SAND',
-    'ee_type': 'raster',
-    'reducer': 'mean'
-}
-BIBLIO_TITLE = 'Harmonized World Soil Database Version 1.2. Food and Agriculture Organization of the United Nations (FAO).'  # noqa: E501
+EE_PARAMS = [
+    {
+        'collection': 'T_SAND_v2_depth_1',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 0,
+        'depthLower': 20
+    },
+    {
+        'collection': 'T_SAND_v2_depth_2',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 20,
+        'depthLower': 40
+    },
+    {
+        'collection': 'T_SAND_v2_depth_3',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 40,
+        'depthLower': 60
+    },
+    {
+        'collection': 'T_SAND_v2_depth_4',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 60,
+        'depthLower': 80
+    }
+]
+BIBLIO_TITLE = 'Harmonized World Soil Database Version 2.0.'
 
 
-def _measurement(site: dict, value: int):
+def _measurement(site: dict, value: int, depthUpper: int, depthLower: int):
     measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
-    measurement['depthUpper'] = 0
-    measurement['depthLower'] = 30
+    measurement['depthUpper'] = depthUpper
+    measurement['depthLower'] = depthLower
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
-def _download(site: dict):
-    value = download(TERM_ID, site, EE_PARAMS)
-    return None if value is None else round(value, 2)
+def _run_depths(site: dict, params: dict):
+    value = download(TERM_ID, site, params)
+    return None if value is None else (
+        _measurement(site, round(value, 2), params.get('depthUpper'), params.get('depthLower'))
+    )
 
 
 def _run(site: dict):
-    value = _download(site)
-    return [_measurement(site, value)] if value is not None else []
+    return non_empty_list([_run_depths(site, params) for params in EE_PARAMS])
 
 
 def _should_run(site: dict):

hestia_earth/models/geospatialDatabase/siltContent.py

@@ -1,12 +1,30 @@
 from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.model import find_term_match
+from hestia_earth.utils.tools import non_empty_list, flatten
 
-from hestia_earth.models.log import logRequirements, logShouldRun, log_blank_nodes_id
-from hestia_earth.models.utils.measurement import _new_measurement, measurement_value
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils.measurement import _new_measurement, measurement_value, group_measurements_by_depth
 from hestia_earth.models.utils.source import get_source
 from . import MODEL, clayContent, sandContent
 
 REQUIREMENTS = {
     "Site": {
+        "measurements": [
+            {
+                "@type": "Measurement",
+                "term.@id": "clayContent",
+                "value": ">= 0",
+                "depthUpper": ">= 0",
+                "depthLower": ">= 0"
+            },
+            {
+                "@type": "Measurement",
+                "term.@id": "sandContent",
+                "value": ">= 0",
+                "depthUpper": ">= 0",
+                "depthLower": ">= 0"
+            }
+        ],
         "or": [
             {"latitude": "", "longitude": ""},
             {"boundary": {}},
@@ -17,8 +35,8 @@ REQUIREMENTS = {
 RETURNS = {
     "Measurement": [{
         "value": "",
-        "depthUpper": "0",
-        "depthLower": "30",
+        "depthUpper": "",
+        "depthLower": "",
         "methodClassification": "geospatial dataset"
     }]
 }
@@ -27,39 +45,61 @@ OTHER_TERM_IDS = [
     clayContent.TERM_ID,
     sandContent.TERM_ID
 ]
-BIBLIO_TITLE = 'Harmonized World Soil Database Version 1.2. Food and Agriculture Organization of the United Nations (FAO).'  # noqa: E501
+BIBLIO_TITLE = 'Harmonized World Soil Database Version 2.0.'
 
 
-def _measurement(site: dict, value: int):
+def _measurement(site: dict, value: int, depthUpper: int, depthLower: int):
     measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
-    measurement['depthUpper'] = 0
-    measurement['depthLower'] = 30
+    measurement['depthUpper'] = depthUpper
+    measurement['depthLower'] = depthLower
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
 def _run(site: dict, measurements: list):
     value = 100 - sum([measurement_value(m) for m in measurements])
-    return [_measurement(site, value)]
+    return [_measurement(site, value, measurements[0].get('depthUpper'), measurements[0].get('depthLower'))]
 
 
 def _should_run(site: dict):
-    measurements = [
-        m for m in site.get('measurements', [])
-        if m.get('term', {}).get('@id') in OTHER_TERM_IDS and len(m.get('value', [])) > 0
+    grouped_measurements = group_measurements_by_depth(site.get('measurements', []), include_dates=False)
+    relevant_measurements = [
+        [
+            m for m in measurements if all([
+                m.get('term', {}).get('@id') in OTHER_TERM_IDS,
+                len(m.get('value', [])) > 0,
+                m.get('depthUpper', -1) >= 0,
+                m.get('depthLower', -1) >= 0
+            ])
+        ]
+        for measurements in grouped_measurements.values()
+    ]
+    valid_measurements = [
+        values for values in relevant_measurements if len(values) == len(OTHER_TERM_IDS)
     ]
-    has_all_measurements = len(measurements) == len(OTHER_TERM_IDS)
+
+    has_valid_measurements = len(valid_measurements) > 0
 
     logRequirements(site, model=MODEL, term=TERM_ID,
-                    has_all_measurements=has_all_measurements,
-                    measurement_ids=log_blank_nodes_id(measurements))
+                    has_valid_measurements=has_valid_measurements,
+                    measurements=log_as_table([
+                        {
+                            'depths': '-'.join(non_empty_list([
+                                str(values[0].get('depthUpper', '')),
+                                str(values[0].get('depthLower', ''))
+                            ]))
+                        } | {
+                            term_id: measurement_value(find_term_match(values, term_id))
+                            for term_id in OTHER_TERM_IDS
+                        } for values in valid_measurements
+                    ]))
 
-    should_run = all([has_all_measurements])
+    should_run = all([has_valid_measurements])
     logShouldRun(site, MODEL, TERM_ID, should_run)
-    return should_run, measurements
+    return should_run, valid_measurements
 
 
 def run(site: dict):
     should_run, measurements = _should_run(site)
-    return _run(site, measurements) if should_run else []
+    return flatten([_run(site, values) for values in measurements]) if should_run else []

hestia_earth/models/geospatialDatabase/soilPh.py

@@ -1,4 +1,5 @@
 from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.measurement import _new_measurement
@@ -18,37 +19,63 @@ REQUIREMENTS = {
 RETURNS = {
     "Measurement": [{
         "value": "",
-        "depthUpper": "0",
-        "depthLower": "30",
+        "depthUpper": "",
+        "depthLower": "",
         "methodClassification": "geospatial dataset"
     }]
 }
 TERM_ID = 'soilPh'
-EE_PARAMS = {
-    'collection': 'T_PH_H2O',
-    'ee_type': 'raster',
-    'reducer': 'mean'
-}
-BIBLIO_TITLE = 'Harmonized World Soil Database Version 1.2. Food and Agriculture Organization of the United Nations (FAO).'  # noqa: E501
+EE_PARAMS = [
+    {
+        'collection': 'T_PH_H2O_v2_depth_1',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 0,
+        'depthLower': 20
+    },
+    {
+        'collection': 'T_PH_H2O_v2_depth_2',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 20,
+        'depthLower': 40
+    },
+    {
+        'collection': 'T_PH_H2O_v2_depth_3',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 40,
+        'depthLower': 60
+    },
+    {
+        'collection': 'T_PH_H2O_v2_depth_4',
+        'ee_type': 'raster',
+        'reducer': 'mean',
+        'depthUpper': 60,
+        'depthLower': 80
+    }
+]
+BIBLIO_TITLE = 'Harmonized World Soil Database Version 2.0.'
 
 
-def _measurement(site: dict, value: float):
+def _measurement(site: dict, value: int, depthUpper: int, depthLower: int):
     measurement = _new_measurement(TERM_ID)
     measurement['value'] = [value]
-    measurement['depthUpper'] = 0
-    measurement['depthLower'] = 30
+    measurement['depthUpper'] = depthUpper
+    measurement['depthLower'] = depthLower
     measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
-def _download(site: dict):
-    value = download(TERM_ID, site, EE_PARAMS)
-    return None if value is None else round(value, 7)
+def _run_depths(site: dict, params: dict):
+    value = download(TERM_ID, site, params)
+    return None if value is None else (
+        _measurement(site, round(value, 7), params.get('depthUpper'), params.get('depthLower'))
+    )
 
 
 def _run(site: dict):
-    value = _download(site)
-    return [] if value is None else [_measurement(site, value)]
+    return non_empty_list([_run_depths(site, params) for params in EE_PARAMS])
 
 
 def _should_run(site: dict):