hestia-earth-models 0.59.3__py3-none-any.whl → 0.59.5__py3-none-any.whl

hestia_earth/models/cycle/liveAnimal.py

@@ -43,6 +43,9 @@ RETURNS = {
         "value": ""
     }]
 }
+LOOKUPS = {
+    "animalProduct": "liveAnimalTermId"
+}
 MODEL_KEY = 'liveAnimal'
 VALID_SITE_TYPES = [SiteSiteType.ANIMAL_HOUSING.value, SiteSiteType.PERMANENT_PASTURE.value]
 

hestia_earth/models/cycle/milkYield.py

@@ -48,7 +48,7 @@ RETURNS = {
     }]
 }
 LOOKUPS = {
-    "animalProduct": ["liveAnimal", "milkYieldPracticeId"]
+    "animalProduct": ["liveAnimalTermId", "milkYieldPracticeId"]
 }
 
 MODEL_KEY = 'milkYield'

hestia_earth/models/cycle/utils.py

@@ -6,7 +6,7 @@ from . import MODEL
 
 
 def _get_liveAnimal_term_id(product: dict, **log_ars):
-    term_id = get_lookup_value(product.get('term', {}), 'liveAnimal', model=MODEL, **log_ars)
+    term_id = get_lookup_value(product.get('term', {}), 'liveAnimalTermId', model=MODEL, **log_ars)
     return term_id.split(';')[0] if term_id else None
 
 

hestia_earth/models/geospatialDatabase/potentialEvapotranspirationLongTermAnnualMean.py

@@ -47,9 +47,9 @@ def _measurement(value: float):
 
 
 def _download(site: dict):
-    scale = 10
+    factor = 0.1
     value = download(TERM_ID, site, EE_PARAMS)
-    return value / scale if value else None
+    return value * factor if value else None
 
 
 def _run(site: dict):

hestia_earth/models/geospatialDatabase/potentialEvapotranspirationMonthly.py

@@ -0,0 +1,99 @@
+"""
+Must be associated with at least 1 `Cycle` that has an
+[endDate](https://hestia.earth/schema/Cycle#endDate) after `1979-01-01` and before `2020-06-01`.
+"""
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import first_day_of_month, last_day_of_month
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.source import get_source
+from hestia_earth.models.utils.site import related_years
+from .utils import download, has_geospatial_data, should_download
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "or": [
+            {"latitude": "", "longitude": ""},
+            {"boundary": {}},
+            {"region": {"@type": "Term", "termType": "region"}}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "dates": "",
+        "methodClassification": "geospatial dataset"
+    }]
+}
+TERM_ID = 'potentialEvapotranspirationMonthly'
+EE_PARAMS = {
+    'collection': 'IDAHO_EPSCOR/TERRACLIMATE',
+    'band_name': 'pet',
+    'ee_type': 'raster',
+    'reducer': 'mean',
+    'reducer_annual': 'sum'
+}
+BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
+
+
+def _measurement(site: dict, value: list, dates: list):
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = value
+    measurement['dates'] = dates
+    measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
+    return measurement | get_source(site, BIBLIO_TITLE)
+
+
+def _download(site: dict,  start_date: str, end_date: str):
+    return download(
+        TERM_ID,
+        site,
+        {
+            **EE_PARAMS,
+            'start_date': start_date,
+            'end_date': end_date
+        }
+    )
+
+
+def _value_at(site: dict, start_date: str, end_date: str):
+    factor = 0.1
+    value = _download(site, start_date, end_date)
+    return (value * factor, start_date[0:7]) if value is not None else None
+
+
+def _run(site: dict, years: list):
+    # fetch from first year to last
+    years = range(years[0], years[-1] + 1) if len(years) > 1 else years
+
+    dates = flatten([
+        [
+            (first_day_of_month(year, month).strftime('%Y-%m-%d'), last_day_of_month(year, month).strftime('%Y-%m-%d'))
+            for month in range(1, 13)
+        ] for year in years
+    ])
+    values = non_empty_list([_value_at(site, start_date, end_date) for start_date, end_date in dates])
+    return [_measurement(site, [v for v, d in values], [d for v, d in values])] if values else []
+
+
+def run(site: dict):
+    contains_geospatial_data = has_geospatial_data(site)
+    below_max_area_size = should_download(TERM_ID, site)
+
+    years = related_years(site)
+    has_years = len(years) > 0
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    contains_geospatial_data=contains_geospatial_data,
+                    below_max_area_size=below_max_area_size,
+                    has_years=has_years,
+                    years=';'.join(map(lambda y: str(y), years)))
+
+    should_run = all([contains_geospatial_data, below_max_area_size, has_years])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+
+    return _run(site, years) if should_run else []

hestia_earth/models/geospatialDatabase/precipitationMonthly.py

@@ -0,0 +1,100 @@
+"""
+Must be associated with at least 1 `Cycle` that has an
+[endDate](https://hestia.earth/schema/Cycle#endDate) after `1979-01-01` and before `2020-06-01`.
+"""
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import first_day_of_month, last_day_of_month
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.source import get_source
+from hestia_earth.models.utils.site import related_years
+from .utils import download, has_geospatial_data, should_download
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "or": [
+            {"latitude": "", "longitude": ""},
+            {"boundary": {}},
+            {"region": {"@type": "Term", "termType": "region"}}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "dates": "",
+        "methodClassification": "geospatial dataset"
+    }]
+}
+TERM_ID = 'precipitationMonthly'
+EE_PARAMS = {
+    'collection': 'ECMWF/ERA5/MONTHLY',
+    'band_name': 'total_precipitation',
+    'ee_type': 'raster',
+    'reducer': 'mean',
+    'reducer_annual': 'sum'
+}
+BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
+
+
+def _measurement(site: dict, value: list, dates: list):
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = value
+    measurement['dates'] = dates
+    measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
+    return measurement | get_source(site, BIBLIO_TITLE)
+
+
+def _download(site: dict,  start_date: str, end_date: str):
+    return download(
+        TERM_ID,
+        site,
+        {
+            **EE_PARAMS,
+            'start_date': start_date,
+            'end_date': end_date
+        }
+    )
+
+
+def _value_at(site: dict, start_date: str, end_date: str):
+    # collection is in meters, convert to millimeters
+    factor = 1000
+    value = _download(site, start_date, end_date)
+    return (value * factor, start_date[0:7]) if value is not None else None
+
+
+def _run(site: dict, years: list):
+    # fetch from first year to last
+    years = range(years[0], years[-1] + 1) if len(years) > 1 else years
+
+    dates = flatten([
+        [
+            (first_day_of_month(year, month).strftime('%Y-%m-%d'), last_day_of_month(year, month).strftime('%Y-%m-%d'))
+            for month in range(1, 13)
+        ] for year in years
+    ])
+    values = non_empty_list([_value_at(site, start_date, end_date) for start_date, end_date in dates])
+    return [_measurement(site, [v for v, d in values], [d for v, d in values])] if values else []
+
+
+def run(site: dict):
+    contains_geospatial_data = has_geospatial_data(site)
+    below_max_area_size = should_download(TERM_ID, site)
+
+    years = related_years(site)
+    has_years = len(years) > 0
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    contains_geospatial_data=contains_geospatial_data,
+                    below_max_area_size=below_max_area_size,
+                    has_years=has_years,
+                    years=';'.join(map(lambda y: str(y), years)))
+
+    should_run = all([contains_geospatial_data, below_max_area_size, has_years])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+
+    return _run(site, years) if should_run else []

hestia_earth/models/geospatialDatabase/temperatureAnnual.py

@@ -9,7 +9,7 @@ from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.source import get_source
 from hestia_earth.models.utils.site import related_years
-from .utils import download, has_geospatial_data, should_download
+from .utils import to_celcius, download, has_geospatial_data, should_download
 from . import MODEL
 
 REQUIREMENTS = {
@@ -38,7 +38,6 @@ EE_PARAMS = {
     'reducer_annual': 'mean'
 }
 BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
-KELVIN_0 = 273.15
 
 
 def _measurement(site: dict, value: float, year: int):
@@ -50,9 +49,6 @@ def _measurement(site: dict, value: float, year: int):
     return measurement | get_source(site, BIBLIO_TITLE)
 
 
-def _to_celcius(kelvin_value: int): return kelvin_value - KELVIN_0 if kelvin_value else None
-
-
 def _download(site: dict, year: int):
     return download(
         TERM_ID,
@@ -65,7 +61,7 @@ def _download(site: dict, year: int):
 
 
 def _run(site: dict, year: int):
-    value = _to_celcius(_download(site, year))
+    value = to_celcius(_download(site, year))
     return _measurement(site, value, year) if value else None
 
 

hestia_earth/models/geospatialDatabase/temperatureLongTermAnnualMean.py

@@ -3,9 +3,8 @@ from hestia_earth.schema import MeasurementMethodClassification
 from hestia_earth.models.log import logRequirements, logShouldRun
 from hestia_earth.models.utils.measurement import _new_measurement
 from hestia_earth.models.utils.source import get_source
-from .utils import download, has_geospatial_data, should_download
+from .utils import to_celcius, download, has_geospatial_data, should_download
 from . import MODEL
-from .temperatureAnnual import _to_celcius
 
 REQUIREMENTS = {
     "Site": {
@@ -50,7 +49,7 @@ def _measurement(site: dict, value: float):
 
 
 def _run(site: dict):
-    value = _to_celcius(download(TERM_ID, site, EE_PARAMS))
+    value = to_celcius(download(TERM_ID, site, EE_PARAMS))
     return [_measurement(site, value)] if value is not None else []
 
 

hestia_earth/models/geospatialDatabase/temperatureMonthly.py

@@ -0,0 +1,98 @@
+"""
+Must be associated with at least 1 `Cycle` that has an
+[endDate](https://hestia.earth/schema/Cycle#endDate) after `1979-01-01` and before `2020-06-01`.
+"""
+from hestia_earth.schema import MeasurementMethodClassification
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils import first_day_of_month, last_day_of_month
+from hestia_earth.models.utils.measurement import _new_measurement
+from hestia_earth.models.utils.source import get_source
+from hestia_earth.models.utils.site import related_years
+from .utils import to_celcius, download, has_geospatial_data, should_download
+from . import MODEL
+
+REQUIREMENTS = {
+    "Site": {
+        "or": [
+            {"latitude": "", "longitude": ""},
+            {"boundary": {}},
+            {"region": {"@type": "Term", "termType": "region"}}
+        ]
+    }
+}
+RETURNS = {
+    "Measurement": [{
+        "value": "",
+        "dates": "",
+        "methodClassification": "geospatial dataset"
+    }]
+}
+TERM_ID = 'temperatureMonthly'
+EE_PARAMS = {
+    'collection': 'ECMWF/ERA5/MONTHLY',
+    'band_name': 'mean_2m_air_temperature',
+    'ee_type': 'raster',
+    'reducer': 'mean',
+    'reducer_annual': 'mean'
+}
+BIBLIO_TITLE = 'ERA5: Fifth generation of ECMWF atmospheric reanalyses of the global climate'
+
+
+def _measurement(site: dict, value: list, dates: list):
+    measurement = _new_measurement(TERM_ID)
+    measurement['value'] = value
+    measurement['dates'] = dates
+    measurement['methodClassification'] = MeasurementMethodClassification.GEOSPATIAL_DATASET.value
+    return measurement | get_source(site, BIBLIO_TITLE)
+
+
+def _download(site: dict,  start_date: str, end_date: str):
+    return download(
+        TERM_ID,
+        site,
+        {
+            **EE_PARAMS,
+            'start_date': start_date,
+            'end_date': end_date
+        }
+    )
+
+
+def _value_at(site: dict, start_date: str, end_date: str):
+    value = _download(site, start_date, end_date)
+    return (to_celcius(value), start_date[0:7]) if value is not None else None
+
+
+def _run(site: dict, years: list):
+    # fetch from first year to last
+    years = range(years[0], years[-1] + 1) if len(years) > 1 else years
+
+    dates = flatten([
+        [
+            (first_day_of_month(year, month).strftime('%Y-%m-%d'), last_day_of_month(year, month).strftime('%Y-%m-%d'))
+            for month in range(1, 13)
+        ] for year in years
+    ])
+    values = non_empty_list([_value_at(site, start_date, end_date) for start_date, end_date in dates])
+    return [_measurement(site, [v for v, d in values], [d for v, d in values])] if values else []
+
+
+def run(site: dict):
+    contains_geospatial_data = has_geospatial_data(site)
+    below_max_area_size = should_download(TERM_ID, site)
+
+    years = related_years(site)
+    has_years = len(years) > 0
+
+    logRequirements(site, model=MODEL, term=TERM_ID,
+                    contains_geospatial_data=contains_geospatial_data,
+                    below_max_area_size=below_max_area_size,
+                    has_years=has_years,
+                    years=';'.join(map(lambda y: str(y), years)))
+
+    should_run = all([contains_geospatial_data, below_max_area_size, has_years])
+    logShouldRun(site, MODEL, TERM_ID, should_run)
+
+    return _run(site, years) if should_run else []

hestia_earth/models/geospatialDatabase/utils.py

@@ -1,6 +1,7 @@
 import os
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.api import download_hestia
+from hestia_earth.utils.tools import non_empty_list
 
 from hestia_earth.models.log import debugValues, logErrorRun, logRequirements
 from hestia_earth.models.utils.site import cached_value, region_factor, region_level_1_id
@@ -19,6 +20,10 @@ GEOPANDAS_COLLECTION_NAME = {
     'gadm36_4': 'gadm/gadm36_4',
     'gadm36_5': 'gadm/gadm36_5'
 }
+KELVIN_0 = 273.15
+
+
+def to_celcius(kelvin_value: int): return kelvin_value - KELVIN_0 if kelvin_value else None
 
 
 def use_geopandas(): return os.getenv('HEE_USE_GEOPANDAS', 'false') == 'true'
@@ -131,8 +136,15 @@ def _parse_run_query(term: str, query: dict):
 
 def _get_cached_data(term: str, site: dict, data: dict):
     cache = _cached_value(site, term)
+    cache_sub_key = '-'.join(non_empty_list([
+        data.get('year'),
+        data.get('start_date'),
+        data.get('end_date')
+    ]))
     # data can be grouped by year when required
-    value = cache.get(data.get('year')) if data.get('year') and cache is not None else cache
+    value = cache.get(cache_sub_key) if (
+        isinstance(cache, dict) and cache_sub_key in cache and cache is not None
+     ) else cache
     if value is not None:
         debugValues(site, model=MODEL, term=term, value_from_cache=value)
     return value