hestia-earth-models 0.59.2__tar.gz → 0.59.4__tar.gz

{hestia-earth-models-0.59.2/hestia_earth_models.egg-info → hestia-earth-models-0.59.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.59.2
+Version: 0.59.4
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team

hestia-earth-models-0.59.4/hestia_earth/models/geospatialDatabase/potentialEvapotranspirationMonthly.py

@@ -0,0 +1,98 @@
+"""
+Must be associated with at least 1 `Cycle` that has an
+[endDate](https://hestia.earth/schema/Cycle#endDate) after `1979-01-01` and before `2020-06-01`.
+"""
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import first_day_of_month, last_day_of_month
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.source import get_source
+from hestia_earth.models.utils.site import related_years
+from .utils import KELVIN_0, download, has_geospatial_data, should_download
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "or": [
+            {"latitude": "", "longitude": ""},
+            {"boundary": {}},
+            {"region": {"@type": "Term", "termType": "region"}}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "dates": "",
+        "methodClassification": "geospatial dataset"
+    }]
+}
+TERM_ID = 'potentialEvapotranspirationMonthly'
+EE_PARAMS = {
+    'collection': 'IDAHO_EPSCOR/TERRACLIMATE',
+    'band_name': 'pet',
+    'ee_type': 'raster',
+    'reducer': 'mean',
+    'reducer_annual': 'sum'
+}
+BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
+
+
+def _measurement(site: dict, value: list, dates: list):
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = value
+    measurement['dates'] = dates
+    measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
+    return measurement | get_source(site, BIBLIO_TITLE)
+
+
+def _to_celcius(kelvin_value: int): return kelvin_value - KELVIN_0 if kelvin_value else None
+
+
+def _download(site: dict,  start_date: str, end_date: str):
+    return download(
+        TERM_ID,
+        site,
+        {
+            **EE_PARAMS,
+            'start_date': start_date,
+            'end_date': end_date
+        }
+    )
+
+
+def _run(site: dict, years: list):
+    # fetch from first year to last
+    years = range(years[0], years[-1] + 1) if len(years) > 1 else years
+
+    dates = flatten([
+        [
+            (first_day_of_month(year, month).strftime('%Y-%m-%d'), last_day_of_month(year, month).strftime('%Y-%m-%d'))
+            for month in range(1, 13)
+        ] for year in years
+    ])
+    values = non_empty_list([
+        (_to_celcius(_download(site, start_date, end_date)), start_date[0:7]) for start_date, end_date in dates
+    ])
+    return _measurement(site, [v for v, d in values], [d for v, d in values])
+
+
+def run(site: dict):
+    contains_geospatial_data = has_geospatial_data(site)
+    below_max_area_size = should_download(TERM_ID, site)
+
+    years = related_years(site)
+    has_years = len(years) > 0
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    contains_geospatial_data=contains_geospatial_data,
+                    below_max_area_size=below_max_area_size,
+                    has_years=has_years,
+                    years=';'.join(map(lambda y: str(y), years)))
+
+    should_run = all([contains_geospatial_data, below_max_area_size, has_years])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+
+    return _run(site, years) if should_run else []

hestia-earth-models-0.59.4/hestia_earth/models/geospatialDatabase/precipitationMonthly.py

@@ -0,0 +1,98 @@
+"""
+Must be associated with at least 1 `Cycle` that has an
+[endDate](https://hestia.earth/schema/Cycle#endDate) after `1979-01-01` and before `2020-06-01`.
+"""
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import first_day_of_month, last_day_of_month
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.source import get_source
+from hestia_earth.models.utils.site import related_years
+from .utils import KELVIN_0, download, has_geospatial_data, should_download
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "or": [
+            {"latitude": "", "longitude": ""},
+            {"boundary": {}},
+            {"region": {"@type": "Term", "termType": "region"}}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "dates": "",
+        "methodClassification": "geospatial dataset"
+    }]
+}
+TERM_ID = 'precipitationMonthly'
+EE_PARAMS = {
+    'collection': 'ECMWF/ERA5/MONTHLY',
+    'band_name': 'total_precipitation',
+    'ee_type': 'raster',
+    'reducer': 'mean',
+    'reducer_annual': 'sum'
+}
+BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
+
+
+def _measurement(site: dict, value: list, dates: list):
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = value
+    measurement['dates'] = dates
+    measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
+    return measurement | get_source(site, BIBLIO_TITLE)
+
+
+def _to_celcius(kelvin_value: int): return kelvin_value - KELVIN_0 if kelvin_value else None
+
+
+def _download(site: dict,  start_date: str, end_date: str):
+    return download(
+        TERM_ID,
+        site,
+        {
+            **EE_PARAMS,
+            'start_date': start_date,
+            'end_date': end_date
+        }
+    )
+
+
+def _run(site: dict, years: list):
+    # fetch from first year to last
+    years = range(years[0], years[-1] + 1) if len(years) > 1 else years
+
+    dates = flatten([
+        [
+            (first_day_of_month(year, month).strftime('%Y-%m-%d'), last_day_of_month(year, month).strftime('%Y-%m-%d'))
+            for month in range(1, 13)
+        ] for year in years
+    ])
+    values = non_empty_list([
+        (_to_celcius(_download(site, start_date, end_date)), start_date[0:7]) for start_date, end_date in dates
+    ])
+    return _measurement(site, [v for v, d in values], [d for v, d in values])
+
+
+def run(site: dict):
+    contains_geospatial_data = has_geospatial_data(site)
+    below_max_area_size = should_download(TERM_ID, site)
+
+    years = related_years(site)
+    has_years = len(years) > 0
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    contains_geospatial_data=contains_geospatial_data,
+                    below_max_area_size=below_max_area_size,
+                    has_years=has_years,
+                    years=';'.join(map(lambda y: str(y), years)))
+
+    should_run = all([contains_geospatial_data, below_max_area_size, has_years])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+
+    return _run(site, years) if should_run else []

{hestia-earth-models-0.59.2 → hestia-earth-models-0.59.4}/hestia_earth/models/geospatialDatabase/temperatureAnnual.py

@@ -9,7 +9,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.source import get_source
 from hestia_earth.models.utils.site import related_years
-from .utils import download, has_geospatial_data, should_download
+from .utils import KELVIN_0, download, has_geospatial_data, should_download
 from . import MODEL
 
 REQUIREMENTS = {
@@ -38,7 +38,6 @@ EE_PARAMS = {
     'reducer_annual': 'mean'
 }
 BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
-KELVIN_0 = 273.15
 
 
 def _measurement(site: dict, value: float, year: int):

hestia-earth-models-0.59.4/hestia_earth/models/geospatialDatabase/temperatureMonthly.py

@@ -0,0 +1,98 @@
+"""
+Must be associated with at least 1 `Cycle` that has an
+[endDate](https://hestia.earth/schema/Cycle#endDate) after `1979-01-01` and before `2020-06-01`.
+"""
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import first_day_of_month, last_day_of_month
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.source import get_source
+from hestia_earth.models.utils.site import related_years
+from .utils import KELVIN_0, download, has_geospatial_data, should_download
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "or": [
+            {"latitude": "", "longitude": ""},
+            {"boundary": {}},
+            {"region": {"@type": "Term", "termType": "region"}}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "dates": "",
+        "methodClassification": "geospatial dataset"
+    }]
+}
+TERM_ID = 'temperatureMonthly'
+EE_PARAMS = {
+    'collection': 'ECMWF/ERA5/MONTHLY',
+    'band_name': 'mean_2m_air_temperature',
+    'ee_type': 'raster',
+    'reducer': 'mean',
+    'reducer_annual': 'mean'
+}
+BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
+
+
+def _measurement(site: dict, value: list, dates: list):
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = value
+    measurement['dates'] = dates
+    measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
+    return measurement | get_source(site, BIBLIO_TITLE)
+
+
+def _to_celcius(kelvin_value: int): return kelvin_value - KELVIN_0 if kelvin_value else None
+
+
+def _download(site: dict,  start_date: str, end_date: str):
+    return download(
+        TERM_ID,
+        site,
+        {
+            **EE_PARAMS,
+            'start_date': start_date,
+            'end_date': end_date
+        }
+    )
+
+
+def _run(site: dict, years: list):
+    # fetch from first year to last
+    years = range(years[0], years[-1] + 1) if len(years) > 1 else years
+
+    dates = flatten([
+        [
+            (first_day_of_month(year, month).strftime('%Y-%m-%d'), last_day_of_month(year, month).strftime('%Y-%m-%d'))
+            for month in range(1, 13)
+        ] for year in years
+    ])
+    values = non_empty_list([
+        (_to_celcius(_download(site, start_date, end_date)), start_date[0:7]) for start_date, end_date in dates
+    ])
+    return _measurement(site, [v for v, d in values], [d for v, d in values])
+
+
+def run(site: dict):
+    contains_geospatial_data = has_geospatial_data(site)
+    below_max_area_size = should_download(TERM_ID, site)
+
+    years = related_years(site)
+    has_years = len(years) > 0
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    contains_geospatial_data=contains_geospatial_data,
+                    below_max_area_size=below_max_area_size,
+                    has_years=has_years,
+                    years=';'.join(map(lambda y: str(y), years)))
+
+    should_run = all([contains_geospatial_data, below_max_area_size, has_years])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+
+    return _run(site, years) if should_run else []

{hestia-earth-models-0.59.2 → hestia-earth-models-0.59.4}/hestia_earth/models/geospatialDatabase/utils.py

@@ -1,6 +1,7 @@
 import os
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.api import download_hestia
+from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import debugValues, logErrorRun, logRequirements
 from hestia_earth.models.utils.site import cached_value, region_factor, region_level_1_id
@@ -19,6 +20,7 @@ GEOPANDAS_COLLECTION_NAME = {
     'gadm36_4': 'gadm/gadm36_4',
     'gadm36_5': 'gadm/gadm36_5'
 }
+KELVIN_0 = 273.15
 
 
 def use_geopandas(): return os.getenv('HEE_USE_GEOPANDAS', 'false') == 'true'
@@ -131,8 +133,13 @@ def _parse_run_query(term: str, query: dict):
 
 def _get_cached_data(term: str, site: dict, data: dict):
     cache = _cached_value(site, term)
+    cache_sub_key = '-'.join(non_empty_list([
+        data.get('year'),
+        data.get('start_date'),
+        data.get('end_date')
+    ]))
     # data can be grouped by year when required
-    value = cache.get(data.get('year')) if data.get('year') and cache is not None else cache
+    value = cache.get(cache_sub_key) if cache_sub_key and cache is not None else cache
     if value is not None:
         debugValues(site, model=MODEL, term=term, value_from_cache=value)
     return value

hestia-earth-models-0.59.4/hestia_earth/models/ipcc2006/n2OToAirCropResidueDecompositionDirect.py

@@ -0,0 +1,83 @@
+from hestia_earth.schema import EmissionMethodTier, EmissionStatsDefinition, TermTermType
+
+from hestia_earth.models.log import logRequirements, logShouldRun, debugValues, log_as_table
+from hestia_earth.models.utils.constant import Units, get_atomic_conversion
+from hestia_earth.models.utils.completeness import _is_term_type_complete
+from hestia_earth.models.utils.emission import _new_emission
+from hestia_earth.models.utils.cycle import get_crop_residue_decomposition_N_total
+from hestia_earth.models.utils.product import has_flooded_rice
+from .utils import N2O_FACTORS
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "completeness.cropResidue": "True",
+        "products": [{
+            "@type": "Product",
+            "value": "",
+            "term.termType": "cropResidue",
+            "properties": [{"@type": "Property", "value": "", "term.@id": "nitrogenContent"}]
+        }],
+        "optional": {
+            "products": [{"@type": "Product", "term.@id": "riceGrainInHuskFlooded"}]
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "min": "",
+        "max": "",
+        "methodTier": "tier 1",
+        "statsDefinition": "modelled"
+    }]
+}
+LOOKUPS = {
+    "cropResidue": "decomposesOnField"
+}
+TERM_ID = 'n2OToAirCropResidueDecompositionDirect'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def _emission(value: float, min: float, max: float):
+    emission = _new_emission(TERM_ID, MODEL)
+    emission['value'] = [value]
+    emission['min'] = [min]
+    emission['max'] = [max]
+    emission['methodTier'] = TIER
+    emission['statsDefinition'] = EmissionStatsDefinition.MODELLED.value
+    return emission
+
+
+def _run(cycle: dict, N_total: float):
+    flooded_rice = has_flooded_rice(cycle.get('products', []))
+
+    converted_N_total = N_total * get_atomic_conversion(Units.KG_N2O, Units.TO_N)
+    factors = N2O_FACTORS['flooded_rice'] if flooded_rice else N2O_FACTORS['default']
+
+    debugValues(cycle, model=MODEL, term=TERM_ID,
+                has_flooded_rice=flooded_rice,
+                factors_used=log_as_table(factors))
+
+    value = converted_N_total * factors['value']
+    min = converted_N_total * factors['min']
+    max = converted_N_total * factors['max']
+    return [_emission(value, min, max)]
+
+
+def _should_run(cycle: dict):
+    term_type_complete = _is_term_type_complete(cycle, TermTermType.CROPRESIDUE)
+    N_crop_residue = get_crop_residue_decomposition_N_total(cycle)
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_cropResidue_complete=term_type_complete,
+                    N_crop_residue=N_crop_residue)
+
+    should_run = all([N_crop_residue is not None, term_type_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
+    return should_run, N_crop_residue
+
+
+def run(cycle: dict):
+    should_run, N_total = _should_run(cycle)
+    return _run(cycle, N_total) if should_run else []

{hestia-earth-models-0.59.2 → hestia-earth-models-0.59.4}/hestia_earth/models/ipcc2006/utils.py

@@ -2,6 +2,18 @@ from hestia_earth.utils.model import find_primary_product
 
 COEFF_NH3NOX_N2O = 0.01
 COEFF_NO3_N2O = 0.0075
+N2O_FACTORS = {
+    'default': {
+        'value': 0.01,
+        'min': 0.003,
+        'max': 0.03
+    },
+    'flooded_rice': {
+        'value': 0.003,
+        'min': 0,
+        'max': 0.006
+    }
+}
 
 
 def get_N_N2O_excreta_coeff_from_primary_product(cycle: dict):

hestia-earth-models-0.59.4/hestia_earth/models/ipcc2019/nh3ToAirInorganicFertiliser.py

@@ -0,0 +1,112 @@
+from hestia_earth.schema import EmissionMethodTier, EmissionStatsDefinition, TermTermType
+from hestia_earth.utils.model import filter_list_term_type, find_term_match
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun, debugValues, log_as_table
+from hestia_earth.models.utils.blank_node import get_N_total
+from hestia_earth.models.utils.constant import Units, get_atomic_conversion
+from hestia_earth.models.utils.completeness import _is_term_type_complete
+from hestia_earth.models.utils.emission import _new_emission
+from hestia_earth.models.utils.cycle import get_inorganic_fertiliser_N_total
+from hestia_earth.models.utils.term import get_lookup_value
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "completeness.fertiliser": "True",
+        "inputs": [
+            {
+                "@type": "Input",
+                "value": "",
+                "term.termType": "inorganicFertiliser",
+                "optional": {
+                    "properties": [{"@type": "Property", "value": "", "term.@id": "nitrogenContent"}]
+                }
+            }
+        ]
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "sd": "",
+        "min": "",
+        "max": "",
+        "methodTier": "tier 1",
+        "statsDefinition": "modelled",
+        "methodModelDescription": ["Aggregated version", "Disaggragated version"]
+    }]
+}
+LOOKUPS = {
+    "inorganicFertiliser": ["IPCC_2019_FRACGASF_NH3-N", "IPCC_2019_FRACGASF_NH3-N-min", "IPCC_2019_FRACGASF_NH3-N-max"]
+}
+TERM_ID = 'nh3ToAirInorganicFertiliser'
+TIER = EmissionMethodTier.TIER_1.value
+TERM_TYPE = TermTermType.INORGANICFERTILISER
+UNSPECIFIED_TERM_ID = 'inorganicNitrogenFertiliserUnspecifiedKgN'
+
+
+def _emission(value: float, min: float, max: float, aggregated: bool = False):
+    emission = _new_emission(TERM_ID, MODEL)
+    emission['value'] = [value]
+    emission['min'] = [min]
+    emission['max'] = [max]
+    emission['methodTier'] = TIER
+    emission['statsDefinition'] = EmissionStatsDefinition.MODELLED.value
+    emission['methodModelDescription'] = 'Aggregated version' if aggregated else 'Disaggregated version'
+    return emission
+
+
+def _input_values(input: dict):
+    N_total = list_sum(get_N_total([input]))
+    lookups = LOOKUPS[TERM_TYPE.value]
+    return {
+        'id': input.get('term', {}).get('@id'),
+        'N': N_total,
+        'value': get_lookup_value(input.get('term', {}), lookups[0]),
+        'min': get_lookup_value(input.get('term', {}), lookups[1]),
+        'max': get_lookup_value(input.get('term', {}), lookups[2])
+    }
+
+
+def _filter_input_values(values: list, key: str): return [value for value in values if value.get(key)]
+
+
+def _run(cycle: dict):
+    inputs = filter_list_term_type(cycle.get('inputs', []), TERM_TYPE)
+    input_values = list(map(_input_values, inputs))
+
+    debugValues(cycle, model=MODEL, term=TERM_ID,
+                input_values=log_as_table(input_values))
+
+    value = list_sum([
+        v.get('N', 0) * v.get('value', 0) for v in _filter_input_values(input_values, 'value')
+    ]) * get_atomic_conversion(Units.KG_NH3, Units.TO_N)
+
+    min = list_sum([
+        v.get('N', 0) * v.get('min', 0) for v in _filter_input_values(input_values, 'min')
+    ]) * get_atomic_conversion(Units.KG_NH3, Units.TO_N)
+
+    max = list_sum([
+        v.get('N', 0) * v.get('max', 0) for v in _filter_input_values(input_values, 'max')
+    ]) * get_atomic_conversion(Units.KG_NH3, Units.TO_N)
+
+    aggregated = list_sum(find_term_match(inputs, UNSPECIFIED_TERM_ID).get('value', [0])) > 0
+
+    return [_emission(value, min, max, aggregated=aggregated)]
+
+
+def _should_run(cycle: dict):
+    N_inorganic_fertiliser = get_inorganic_fertiliser_N_total(cycle)
+    fertiliser_complete = _is_term_type_complete(cycle, 'fertiliser')
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    N_inorganic_fertiliser=N_inorganic_fertiliser,
+                    term_type_fertiliser_complete=fertiliser_complete)
+
+    should_run = all([N_inorganic_fertiliser is not None, fertiliser_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+    return should_run
+
+
+def run(cycle: dict): return _run(cycle) if _should_run(cycle) else []

{hestia-earth-models-0.59.2 → hestia-earth-models-0.59.4}/hestia_earth/models/ipcc2019/nh3ToAirOrganicFertiliser.py

@@ -66,6 +66,9 @@ def _input_values(input: dict):
     }
 
 
+def _filter_input_values(values: list, key: str): return [value for value in values if value.get(key)]
+
+
 def _run(cycle: dict):
     inputs = filter_list_term_type(cycle.get('inputs', []), TermTermType.ORGANICFERTILISER)
     input_values = list(map(_input_values, inputs))
@@ -74,15 +77,15 @@ def _run(cycle: dict):
                 input_values=log_as_table(input_values))
 
     value = list_sum([
-        v.get('N', 0) * v.get('value', 0) for v in input_values
+        v.get('N', 0) * v.get('value', 0) for v in _filter_input_values(input_values, 'value')
     ]) * get_atomic_conversion(Units.KG_NH3, Units.TO_N)
 
     min = list_sum([
-        v.get('N', 0) * v.get('min', 0) for v in input_values
+        v.get('N', 0) * v.get('min', 0) for v in _filter_input_values(input_values, 'min')
     ]) * get_atomic_conversion(Units.KG_NH3, Units.TO_N)
 
     max = list_sum([
-        v.get('N', 0) * v.get('max', 0) for v in input_values
+        v.get('N', 0) * v.get('max', 0) for v in _filter_input_values(input_values, 'max')
     ]) * get_atomic_conversion(Units.KG_NH3, Units.TO_N)
 
     return [_emission(value, min, max)]