hestia-earth-models 0.59.3__tar.gz → 0.59.5__tar.gz

{hestia-earth-models-0.59.3/hestia_earth_models.egg-info → hestia-earth-models-0.59.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.59.3
+Version: 0.59.5
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team

{hestia-earth-models-0.59.3 → hestia-earth-models-0.59.5}/hestia_earth/models/cycle/liveAnimal.py

@@ -43,6 +43,9 @@ RETURNS = {
         "value": ""
     }]
 }
+LOOKUPS = {
+    "animalProduct": "liveAnimalTermId"
+}
 MODEL_KEY = 'liveAnimal'
 VALID_SITE_TYPES = [SiteSiteType.ANIMAL_HOUSING.value, SiteSiteType.PERMANENT_PASTURE.value]
 

{hestia-earth-models-0.59.3 → hestia-earth-models-0.59.5}/hestia_earth/models/cycle/milkYield.py

@@ -48,7 +48,7 @@ RETURNS = {
     }]
 }
 LOOKUPS = {
-    "animalProduct": ["liveAnimal", "milkYieldPracticeId"]
+    "animalProduct": ["liveAnimalTermId", "milkYieldPracticeId"]
 }
 
 MODEL_KEY = 'milkYield'

{hestia-earth-models-0.59.3 → hestia-earth-models-0.59.5}/hestia_earth/models/cycle/utils.py

@@ -6,7 +6,7 @@ from . import MODEL
 
 
 def _get_liveAnimal_term_id(product: dict, **log_ars):
-    term_id = get_lookup_value(product.get('term', {}), 'liveAnimal', model=MODEL, **log_ars)
+    term_id = get_lookup_value(product.get('term', {}), 'liveAnimalTermId', model=MODEL, **log_ars)
     return term_id.split(';')[0] if term_id else None
 
 

{hestia-earth-models-0.59.3 → hestia-earth-models-0.59.5}/hestia_earth/models/geospatialDatabase/potentialEvapotranspirationLongTermAnnualMean.py

@@ -47,9 +47,9 @@ def _measurement(value: float):
 
 
 def _download(site: dict):
-    scale = 10
+    factor = 0.1
     value = download(TERM_ID, site, EE_PARAMS)
-    return value / scale if value else None
+    return value * factor if value else None
 
 
 def _run(site: dict):

hestia-earth-models-0.59.5/hestia_earth/models/geospatialDatabase/potentialEvapotranspirationMonthly.py

@@ -0,0 +1,99 @@
+"""
+Must be associated with at least 1 `Cycle` that has an
+[endDate](https://hestia.earth/schema/Cycle#endDate) after `1979-01-01` and before `2020-06-01`.
+"""
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import first_day_of_month, last_day_of_month
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.source import get_source
+from hestia_earth.models.utils.site import related_years
+from .utils import download, has_geospatial_data, should_download
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "or": [
+            {"latitude": "", "longitude": ""},
+            {"boundary": {}},
+            {"region": {"@type": "Term", "termType": "region"}}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "dates": "",
+        "methodClassification": "geospatial dataset"
+    }]
+}
+TERM_ID = 'potentialEvapotranspirationMonthly'
+EE_PARAMS = {
+    'collection': 'IDAHO_EPSCOR/TERRACLIMATE',
+    'band_name': 'pet',
+    'ee_type': 'raster',
+    'reducer': 'mean',
+    'reducer_annual': 'sum'
+}
+BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
+
+
+def _measurement(site: dict, value: list, dates: list):
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = value
+    measurement['dates'] = dates
+    measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
+    return measurement | get_source(site, BIBLIO_TITLE)
+
+
+def _download(site: dict,  start_date: str, end_date: str):
+    return download(
+        TERM_ID,
+        site,
+        {
+            **EE_PARAMS,
+            'start_date': start_date,
+            'end_date': end_date
+        }
+    )
+
+
+def _value_at(site: dict, start_date: str, end_date: str):
+    factor = 0.1
+    value = _download(site, start_date, end_date)
+    return (value * factor, start_date[0:7]) if value is not None else None
+
+
+def _run(site: dict, years: list):
+    # fetch from first year to last
+    years = range(years[0], years[-1] + 1) if len(years) > 1 else years
+
+    dates = flatten([
+        [
+            (first_day_of_month(year, month).strftime('%Y-%m-%d'), last_day_of_month(year, month).strftime('%Y-%m-%d'))
+            for month in range(1, 13)
+        ] for year in years
+    ])
+    values = non_empty_list([_value_at(site, start_date, end_date) for start_date, end_date in dates])
+    return [_measurement(site, [v for v, d in values], [d for v, d in values])] if values else []
+
+
+def run(site: dict):
+    contains_geospatial_data = has_geospatial_data(site)
+    below_max_area_size = should_download(TERM_ID, site)
+
+    years = related_years(site)
+    has_years = len(years) > 0
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    contains_geospatial_data=contains_geospatial_data,
+                    below_max_area_size=below_max_area_size,
+                    has_years=has_years,
+                    years=';'.join(map(lambda y: str(y), years)))
+
+    should_run = all([contains_geospatial_data, below_max_area_size, has_years])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+
+    return _run(site, years) if should_run else []

hestia-earth-models-0.59.5/hestia_earth/models/geospatialDatabase/precipitationMonthly.py

@@ -0,0 +1,100 @@
+"""
+Must be associated with at least 1 `Cycle` that has an
+[endDate](https://hestia.earth/schema/Cycle#endDate) after `1979-01-01` and before `2020-06-01`.
+"""
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import first_day_of_month, last_day_of_month
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.source import get_source
+from hestia_earth.models.utils.site import related_years
+from .utils import download, has_geospatial_data, should_download
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "or": [
+            {"latitude": "", "longitude": ""},
+            {"boundary": {}},
+            {"region": {"@type": "Term", "termType": "region"}}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "dates": "",
+        "methodClassification": "geospatial dataset"
+    }]
+}
+TERM_ID = 'precipitationMonthly'
+EE_PARAMS = {
+    'collection': 'ECMWF/ERA5/MONTHLY',
+    'band_name': 'total_precipitation',
+    'ee_type': 'raster',
+    'reducer': 'mean',
+    'reducer_annual': 'sum'
+}
+BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
+
+
+def _measurement(site: dict, value: list, dates: list):
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = value
+    measurement['dates'] = dates
+    measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
+    return measurement | get_source(site, BIBLIO_TITLE)
+
+
+def _download(site: dict,  start_date: str, end_date: str):
+    return download(
+        TERM_ID,
+        site,
+        {
+            **EE_PARAMS,
+            'start_date': start_date,
+            'end_date': end_date
+        }
+    )
+
+
+def _value_at(site: dict, start_date: str, end_date: str):
+    # collection is in meters, convert to millimeters
+    factor = 1000
+    value = _download(site, start_date, end_date)
+    return (value * factor, start_date[0:7]) if value is not None else None
+
+
+def _run(site: dict, years: list):
+    # fetch from first year to last
+    years = range(years[0], years[-1] + 1) if len(years) > 1 else years
+
+    dates = flatten([
+        [
+            (first_day_of_month(year, month).strftime('%Y-%m-%d'), last_day_of_month(year, month).strftime('%Y-%m-%d'))
+            for month in range(1, 13)
+        ] for year in years
+    ])
+    values = non_empty_list([_value_at(site, start_date, end_date) for start_date, end_date in dates])
+    return [_measurement(site, [v for v, d in values], [d for v, d in values])] if values else []
+
+
+def run(site: dict):
+    contains_geospatial_data = has_geospatial_data(site)
+    below_max_area_size = should_download(TERM_ID, site)
+
+    years = related_years(site)
+    has_years = len(years) > 0
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    contains_geospatial_data=contains_geospatial_data,
+                    below_max_area_size=below_max_area_size,
+                    has_years=has_years,
+                    years=';'.join(map(lambda y: str(y), years)))
+
+    should_run = all([contains_geospatial_data, below_max_area_size, has_years])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+
+    return _run(site, years) if should_run else []

{hestia-earth-models-0.59.3 → hestia-earth-models-0.59.5}/hestia_earth/models/geospatialDatabase/temperatureAnnual.py

@@ -9,7 +9,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.source import get_source
 from hestia_earth.models.utils.site import related_years
-from .utils import download, has_geospatial_data, should_download
+from .utils import to_celcius, download, has_geospatial_data, should_download
 from . import MODEL
 
 REQUIREMENTS = {
@@ -38,7 +38,6 @@ EE_PARAMS = {
     'reducer_annual': 'mean'
 }
 BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
-KELVIN_0 = 273.15
 
 
 def _measurement(site: dict, value: float, year: int):
@@ -50,9 +49,6 @@ def _measurement(site: dict, value: float, year: int):
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
-def _to_celcius(kelvin_value: int): return kelvin_value - KELVIN_0 if kelvin_value else None
-
-
 def _download(site: dict, year: int):
     return download(
         TERM_ID,
@@ -65,7 +61,7 @@ def _download(site: dict, year: int):
 
 
 def _run(site: dict, year: int):
-    value = _to_celcius(_download(site, year))
+    value = to_celcius(_download(site, year))
     return _measurement(site, value, year) if value else None
 
 

{hestia-earth-models-0.59.3 → hestia-earth-models-0.59.5}/hestia_earth/models/geospatialDatabase/temperatureLongTermAnnualMean.py

@@ -3,9 +3,8 @@ from hestia_earth.schema import MeasurementMethodClassification
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.source import get_source
-from .utils import download, has_geospatial_data, should_download
+from .utils import to_celcius, download, has_geospatial_data, should_download
 from . import MODEL
-from .temperatureAnnual import _to_celcius
 
 REQUIREMENTS = {
     "Site": {
@@ -50,7 +49,7 @@ def _measurement(site: dict, value: float):
 
 
 def _run(site: dict):
-    value = _to_celcius(download(TERM_ID, site, EE_PARAMS))
+    value = to_celcius(download(TERM_ID, site, EE_PARAMS))
     return [_measurement(site, value)] if value is not None else []
 
 

hestia-earth-models-0.59.5/hestia_earth/models/geospatialDatabase/temperatureMonthly.py

@@ -0,0 +1,98 @@
+"""
+Must be associated with at least 1 `Cycle` that has an
+[endDate](https://hestia.earth/schema/Cycle#endDate) after `1979-01-01` and before `2020-06-01`.
+"""
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import first_day_of_month, last_day_of_month
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.source import get_source
+from hestia_earth.models.utils.site import related_years
+from .utils import to_celcius, download, has_geospatial_data, should_download
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "or": [
+            {"latitude": "", "longitude": ""},
+            {"boundary": {}},
+            {"region": {"@type": "Term", "termType": "region"}}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "dates": "",
+        "methodClassification": "geospatial dataset"
+    }]
+}
+TERM_ID = 'temperatureMonthly'
+EE_PARAMS = {
+    'collection': 'ECMWF/ERA5/MONTHLY',
+    'band_name': 'mean_2m_air_temperature',
+    'ee_type': 'raster',
+    'reducer': 'mean',
+    'reducer_annual': 'mean'
+}
+BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
+
+
+def _measurement(site: dict, value: list, dates: list):
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = value
+    measurement['dates'] = dates
+    measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
+    return measurement | get_source(site, BIBLIO_TITLE)
+
+
+def _download(site: dict,  start_date: str, end_date: str):
+    return download(
+        TERM_ID,
+        site,
+        {
+            **EE_PARAMS,
+            'start_date': start_date,
+            'end_date': end_date
+        }
+    )
+
+
+def _value_at(site: dict, start_date: str, end_date: str):
+    value = _download(site, start_date, end_date)
+    return (to_celcius(value), start_date[0:7]) if value is not None else None
+
+
+def _run(site: dict, years: list):
+    # fetch from first year to last
+    years = range(years[0], years[-1] + 1) if len(years) > 1 else years
+
+    dates = flatten([
+        [
+            (first_day_of_month(year, month).strftime('%Y-%m-%d'), last_day_of_month(year, month).strftime('%Y-%m-%d'))
+            for month in range(1, 13)
+        ] for year in years
+    ])
+    values = non_empty_list([_value_at(site, start_date, end_date) for start_date, end_date in dates])
+    return [_measurement(site, [v for v, d in values], [d for v, d in values])] if values else []
+
+
+def run(site: dict):
+    contains_geospatial_data = has_geospatial_data(site)
+    below_max_area_size = should_download(TERM_ID, site)
+
+    years = related_years(site)
+    has_years = len(years) > 0
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    contains_geospatial_data=contains_geospatial_data,
+                    below_max_area_size=below_max_area_size,
+                    has_years=has_years,
+                    years=';'.join(map(lambda y: str(y), years)))
+
+    should_run = all([contains_geospatial_data, below_max_area_size, has_years])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+
+    return _run(site, years) if should_run else []

{hestia-earth-models-0.59.3 → hestia-earth-models-0.59.5}/hestia_earth/models/geospatialDatabase/utils.py

@@ -1,6 +1,7 @@
 import os
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.api import download_hestia
+from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import debugValues, logErrorRun, logRequirements
 from hestia_earth.models.utils.site import cached_value, region_factor, region_level_1_id
@@ -19,6 +20,10 @@ GEOPANDAS_COLLECTION_NAME = {
     'gadm36_4': 'gadm/gadm36_4',
     'gadm36_5': 'gadm/gadm36_5'
 }
+KELVIN_0 = 273.15
+
+
+def to_celcius(kelvin_value: int): return kelvin_value - KELVIN_0 if kelvin_value else None
 
 
 def use_geopandas(): return os.getenv('HEE_USE_GEOPANDAS', 'false') == 'true'
@@ -131,8 +136,15 @@ def _parse_run_query(term: str, query: dict):
 
 def _get_cached_data(term: str, site: dict, data: dict):
     cache = _cached_value(site, term)
+    cache_sub_key = '-'.join(non_empty_list([
+        data.get('year'),
+        data.get('start_date'),
+        data.get('end_date')
+    ]))
     # data can be grouped by year when required
-    value = cache.get(data.get('year')) if data.get('year') and cache is not None else cache
+    value = cache.get(cache_sub_key) if (
+        isinstance(cache, dict) and cache_sub_key in cache and cache is not None
+     ) else cache
     if value is not None:
         debugValues(site, model=MODEL, term=term, value_from_cache=value)
     return value