hestia-earth-models 0.60.0__py3-none-any.whl → 0.61.0__py3-none-any.whl

hestia_earth/models/cache_sites.py

@@ -1,5 +1,6 @@
 from functools import reduce
 from enum import Enum
+from pydash.objects import merge
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.tools import flatten
 
@@ -64,12 +65,13 @@ def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors:
             **_cache_results(raster_results, rasters, index),
             **_cache_results(vector_results, vectors, index)
         } | ({CACHE_AREA_SIZE: area_size} if area_size is not None else {})
-        return {
-            **site,
-            CACHE_KEY: (
-                cached_value(site) | {CACHE_GEOSPATIAL_KEY: cached_data}
-            ) | ({CACHE_YEARS_KEY: years} if years else {})
-        }
+        cached_data = merge(cached_value(site, CACHE_GEOSPATIAL_KEY, {}), cached_data)
+        site_cache = merge(
+            site.get(CACHE_KEY, {}),
+            {CACHE_GEOSPATIAL_KEY: cached_data},
+            ({CACHE_YEARS_KEY: cached_value(site, CACHE_YEARS_KEY, []) + years} if years else {})
+        )
+        return merge(site, {CACHE_KEY: site_cache})
 
     return reduce(lambda prev, curr: prev + [_process_site(curr)], sites, [])
 
@@ -82,7 +84,7 @@ def _preload_regions_area_size(sites: dict):
     return {term_id: download_hestia(term_id).get('area') for term_id in region_ids}
 
 
-def _group_sites(sites: dict):
+def _group_sites(sites: dict, check_has_cache: bool = True):
     # preload area size for all regions
     regions_area_size = _preload_regions_area_size(sites)
 
@@ -90,7 +92,7 @@ def _group_sites(sites: dict):
         return regions_area_size.get(_site_gadm_id(site)) if _should_preload_region_area_size(site) else None
 
     sites = [
-        (n, ) + (_should_run(n, get_region_area_size(n))) for n in sites
+        (n, ) + (_should_run(n, area_size=get_region_area_size(n), check_has_cache=check_has_cache)) for n in sites
     ]
     # restrict sites based on should_cache result
     sites = [(site, area_size) for site, should_cache, area_size in sites if should_cache]
@@ -112,7 +114,25 @@ def _group_sites(sites: dict):
     }
 
 
-def run(sites: list, years: list = None, include_region: bool = False, years_only: bool = False):
+def _run(sites: list, years: list, include_region: bool, years_only: bool = False):
+    rasters, vectors = list_collections(years, include_region, years_only)
+    filtered_data = _group_sites(sites, not years_only)
+    return flatten([
+        _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
+        for param_type in [e for e in ParamType] if len(filtered_data.get(param_type)) > 0
+    ])
+
+
+def _group_years(years: list, years_range: int):
+    batches = sorted(list(set(list(range(years[0], years[-1] + 1, years_range)) + [years[0], years[-1]])))
+    grouped_batches = [batches[i:i+2] for i in range(0, len(batches))]
+    return [
+        # make sure we don't overlap
+        [v[0] + (0 if v[0] == years[0] else 1), v[1]] for v in grouped_batches if len(v) == 2
+    ]
+
+
+def run(sites: list, years: list = None, include_region: bool = False):
     """
     Run all queries at once for the list of provided Sites.
     Note: Earth Engine needs to be initiliased with `init_gee()` before running this function.
@@ -126,14 +146,16 @@ def run(sites: list, years: list = None, include_region: bool = False, years_onl
     include_region : bool
         Prefecth region IDs.
         This will cache region-level data and will make the request slower. Only use if needed.
-    years_only : bool
-        Run only the collections that depend on the years (if provided).
     """
-    rasters, vectors = list_collections(years, include_region, years_only)
-
-    filtered_data = _group_sites(sites)
-
-    return flatten([
-        _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
-        for param_type in [e for e in ParamType] if len(filtered_data.get(param_type)) > 0
-    ])
+    try:
+        return _run(sites, years, include_region)
+    except Exception as e:
+        # when querying with multiple years, we can reach a compute memory limit, so run the years separately
+        if str(e) == 'User memory limit exceeded.' and years:
+            sites = _run(sites, [], include_region)
+            # query for subranges
+            for sub_years in _group_years(years, years_range=5):
+                sites = _run(sites, sub_years, include_region, years_only=True)
+            return sites
+
+    return []

hestia_earth/models/cycle/cycleDuration.py

@@ -1,50 +1,89 @@
 """
 Cycle duration
 
-This model calculates the cycle duration using the cropping intensity for a single year.
+This model calculates the cycle duration using:
+* the `endDate` and the `startDate` if both are provided with a day precision;
+* for temporary crops, using the `croppingIntensity` if provided;
+* for permanent crops, the duration is set to `365`.
 """
-from hestia_earth.utils.model import find_term_match
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import find_term_match, find_primary_product
+from hestia_earth.utils.date import diff_in_days
 
 from hestia_earth.models.log import logRequirements, logShouldRun
-from hestia_earth.models.utils.cycle import valid_site_type
+from hestia_earth.models.utils.crop import get_crop_grouping_fao
 from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "cycleDuration": "365",
         "endDate": "",
-        "practices": [{"@type": "Practice", "value": "", "term.@id": "croppingIntensity"}],
-        "site": {
-            "@type": "Site",
-            "siteType": "cropland"
+        "optional": {
+            "startDate": "",
+            "products": {
+                "@type": "Product",
+                "primary": "True",
+                "term.termType": "crop"
+            },
+            "practices": [{"@type": "Practice", "value": "", "term.@id": "croppingIntensity"}]
         }
     }
 }
 RETURNS = {
     "a `number` or `None` if requirements are not met": ""
 }
+LOOKUPS = {
+    "crop": "cropGroupingFAO"
+}
 MODEL_KEY = 'cycleDuration'
 DEFAULT_DURATION = 365
 
 
-def _run(croppingIntensity: float): return croppingIntensity * DEFAULT_DURATION
+def _run_by_dates(cycle: dict):
+    start_date = cycle.get('startDate')
+    end_date = cycle.get('endDate')
+    return diff_in_days(start_date, end_date)
+
+
+def _should_run_by_dates(cycle: dict):
+    start_date = cycle.get('startDate', '')
+    start_date_has_day = len(start_date) == 10
+    end_date = cycle.get('endDate', '')
+    end_date_has_day = len(start_date) == 10
+
+    logRequirements(cycle, model=MODEL, key=MODEL_KEY, by='dates',
+                    start_date_has_day=start_date_has_day,
+                    start_date=start_date,
+                    end_date_has_day=end_date_has_day,
+                    end_date=end_date)
+
+    should_run = all([start_date_has_day, end_date_has_day])
+    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY, by='dates')
+    return should_run
+
+
+def _run_by_crop(cycle: dict):
+    product = find_primary_product(cycle)
+    grouping = get_crop_grouping_fao(MODEL, MODEL_KEY, product.get('term', {}))
+    is_permanent_crop = grouping == 'Permanent crops'
+    croppingIntensity = find_term_match(cycle.get('practices', []), 'croppingIntensity').get('value', [1])[0]
+    return DEFAULT_DURATION * (1 if is_permanent_crop else croppingIntensity)
 
 
-def _should_run(cycle: dict):
-    croppingIntensity = find_term_match(cycle.get('practices', []), 'croppingIntensity').get('value', [None])[0]
-    cycleDuration = cycle.get('cycleDuration')
-    site_type_valid = valid_site_type(cycle)
+def _should_run_by_crop(cycle: dict):
+    product = find_primary_product(cycle) or {}
+    product_term_type = product.get('term', {}).get('termType')
+    primary_product_is_crop = product_term_type == TermTermType.CROP.value
 
-    logRequirements(cycle, model=MODEL, key=MODEL_KEY,
-                    cycleDuration=cycleDuration,
-                    croppingIntensity=croppingIntensity,
-                    site_type_valid=site_type_valid)
+    logRequirements(cycle, model=MODEL, key=MODEL_KEY, by='product',
+                    primary_product_is_crop=primary_product_is_crop)
 
-    should_run = all([site_type_valid, cycleDuration == 365, croppingIntensity])
-    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
-    return should_run, croppingIntensity
+    should_run = all([primary_product_is_crop])
+    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY, by='product')
+    return should_run
 
 
 def run(cycle: dict):
-    should_run, croppingIntensity = _should_run(cycle)
-    return _run(croppingIntensity) if should_run else None
+    return _run_by_dates(cycle) if _should_run_by_dates(cycle) else (
+        _run_by_crop(cycle) if _should_run_by_crop(cycle)
+        else None
+    )

hestia_earth/models/cycle/startDate.py

@@ -1,18 +1,20 @@
 """
 Start Date
 
-This model updates the [Cycle startDate](https://hestia.earth/schema/Cycle#startDate) to be in the following format:
-`YYYY-MM-DD`.
+This model sets the [Cycle startDate](https://hestia.earth/schema/Cycle#startDate) based on the `endDate` and the
+`cycleDuration`. This only works when the `endDate` has been provided to a day precision (`2000-01-01`).
 """
+from datetime import timedelta
 from hestia_earth.utils.date import is_in_days
-from hestia_earth.utils.tools import non_empty_list
+from hestia_earth.utils.tools import safe_parse_date
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "startDate": ""
+        "endDate": "to day precision",
+        "cycleDuration": ""
     }
 }
 RETURNS = {
@@ -22,19 +24,22 @@ MODEL_KEY = 'startDate'
 
 
 def _run(cycle: dict):
-    value = cycle.get('startDate')
-    return '-'.join(non_empty_list([value, '01' if len(value) == 4 else '', '01']))
+    endDate = safe_parse_date(cycle.get('endDate'))
+    cycleDuration = cycle.get('cycleDuration')
+    return (endDate - timedelta(days=cycleDuration)).strftime('%Y-%m-%d')
 
 
 def _should_run(cycle: dict):
-    has_startDate = cycle.get('startDate') is not None
-    has_incorrect_format = has_startDate and not is_in_days(cycle.get('startDate'))
+    has_endDate = cycle.get('endDate') is not None
+    has_day_precision = has_endDate and is_in_days(cycle.get('endDate'))
+    has_cycleDuration = cycle.get('cycleDuration') is not None
 
     logRequirements(cycle, model=MODEL, key=MODEL_KEY,
-                    has_startDate=has_startDate,
-                    has_incorrect_format=has_incorrect_format)
+                    has_endDate=has_endDate,
+                    has_day_precision=has_day_precision,
+                    has_cycleDuration=has_cycleDuration)
 
-    should_run = all([has_startDate, has_incorrect_format])
+    should_run = all([has_endDate, has_day_precision, has_cycleDuration])
     logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
     return should_run
 

hestia_earth/models/cycle/startDateDefinition.py

@@ -0,0 +1,69 @@
+"""
+Start Date Definition
+
+This model sets the [Cycle startDateDefinition](https://hestia.earth/schema/Cycle#startDateDefinition)
+based on the result of the `cycleDuration` gap-filling model.
+"""
+from hestia_earth.schema import TermTermType, CycleStartDateDefinition
+from hestia_earth.utils.model import find_primary_product
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.crop import get_crop_grouping_fao
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "cycleDuration": "",
+        "products": {
+            "@type": "Product",
+            "primary": "True",
+            "term.termType": "crop"
+        },
+        "optional": {
+            "endDate": ""
+        }
+    }
+}
+RETURNS = {
+    "The startDateDefinition as a string": ""
+}
+MODEL_KEY = 'startDateDefinition'
+
+
+def _is_last_day_of_month(date: str):
+    date_parts = date.split('-')
+    return len(date_parts) > 1 and date_parts[1] == '12' and any([
+        len(date_parts) == 3 and date_parts[2] == '31',
+        len(date_parts) == 2
+    ])
+
+
+def _run(cycle: dict):
+    product = find_primary_product(cycle)
+    grouping = get_crop_grouping_fao(MODEL, MODEL_KEY, product.get('term', {}))
+    is_permanent_crop = grouping == 'Permanent crops'
+    print(is_permanent_crop)
+    return (
+        CycleStartDateDefinition.START_OF_YEAR.value if _is_last_day_of_month(cycle.get('endDate'))
+        else CycleStartDateDefinition.ONE_YEAR_PRIOR.value
+    ) if is_permanent_crop else (
+        CycleStartDateDefinition.HARVEST_OF_PREVIOUS_CROP.value
+    )
+
+
+def _should_run(cycle: dict):
+    cycleDuration_added = 'cycleDuration' in cycle.get('added', [])
+    product = find_primary_product(cycle) or {}
+    product_term_type = product.get('term', {}).get('termType')
+    primary_product_is_crop = product_term_type == TermTermType.CROP.value
+
+    logRequirements(cycle, model=MODEL, key=MODEL_KEY,
+                    cycleDuration_added=cycleDuration_added,
+                    primary_product_is_crop=primary_product_is_crop)
+
+    should_run = all([cycleDuration_added, primary_product_is_crop])
+    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
+    return should_run
+
+
+def run(cycle: dict): return _run(cycle) if _should_run(cycle) else None

hestia_earth/models/impact_assessment/emissions.py

@@ -42,11 +42,9 @@ def _indicator(product: dict):
         term_id = emission.get('term', {}).get('@id')
         value = convert_value_from_cycle(product, list_sum(emission.get('value', [0])), model=MODEL, term_id=term_id)
 
-        indicator = _new_indicator(emission.get('term', {}))
+        indicator = _new_indicator(emission.get('term', {}), emission.get('methodModel'))
         indicator['value'] = value
 
-        if 'methodModel' in emission:
-            indicator['methodModel'] = emission['methodModel']
         if len(emission.get('inputs', [])):
             indicator['inputs'] = emission['inputs']
         if emission.get('operation'):

hestia_earth/models/ipcc2019/pastureGrass.py

@@ -147,7 +147,7 @@ def _input(term_id: str, value: float):
     return node
 
 
-def _sum_values(values: list, index=0): return sum([v[index] for v in values])
+def _sum_values(values: list, index=0): return list_sum([v[index] for v in values])
 
 
 def calculate_NEwool(cycle: dict) -> float:

hestia_earth/models/linkedImpactAssessment/__init__.py

@@ -1,167 +1,13 @@
-"""
-Linked Impact Assessment
+from os.path import dirname, abspath
+import sys
+from importlib import import_module
 
-A model which takes recalculated data from an Impact Assessment linked to an Input in a Cycle.
-"""
-from functools import reduce
-from hestia_earth.schema import EmissionMethodTier
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
-from hestia_earth.utils.tools import flatten, list_sum
+from hestia_earth.models.utils.blank_node import run_if_required
 
-from hestia_earth.models.log import debugValues, logRequirements, logShouldRun, log_as_table
-from hestia_earth.models.utils.emission import _new_emission
-from hestia_earth.models.utils.input import load_impacts
-from hestia_earth.models.utils.blank_node import group_by_keys
-
-REQUIREMENTS = {
-    "Cycle": {
-        "inputs": [{
-            "@type": "Input",
-            "value": "> 0",
-            "impactAssessment": {
-                "@type": "ImpactAssessment",
-                "emissionsResourceUse": [{"@type": "Indicator", "value": ""}]
-            }
-        }],
-        "optional": {
-            "animals": [{
-                "@type": "Animal",
-                "inputs": [{
-                    "@type": "Input",
-                    "value": "> 0",
-                    "impactAssessment": {
-                        "@type": "ImpactAssessment",
-                        "emissionsResourceUse": [{"@type": "Indicator", "value": ""}]
-                    }
-                }]
-            }]
-        }
-    }
-}
-RETURNS = {
-    "Emission": [{
-        "value": "",
-        "methodTier": "background",
-        "inputs": "",
-        "operation": "",
-        "animals": ""
-    }]
-}
+CURRENT_DIR = dirname(abspath(__file__)) + '/'
+sys.path.append(CURRENT_DIR)
 MODEL = 'linkedImpactAssessment'
-MODEL_AGGREGATED = 'hestiaAggregatedData'
-TIER = EmissionMethodTier.BACKGROUND.value
-
-
-def _emission(model: str, term_id: str, value: float, input: dict, operation={}, animal={}):
-    emission = _new_emission(term_id, model)
-    emission['value'] = [value]
-    emission['methodTier'] = TIER
-    emission['inputs'] = [input]
-    if operation:
-        emission['operation'] = operation
-    if animal:
-        emission['animals'] = [animal]
-    return emission
-
-
-def _run_emission(cycle: dict, term_id: str, data: dict):
-    def run_input(values: dict):
-        value = values.get('value', 0)
-        term = values.get('term', {})
-        operation = values.get('operation', {})
-        animal = values.get('animal', {})
-        is_aggregated = any(values.get('aggregated', []))
-        model = MODEL_AGGREGATED if is_aggregated else MODEL
-
-        details = values.get('details', {})
-        logRequirements(cycle, model=model, term=term_id,
-                        values=log_as_table([{'impact-assessment-id': key} | value for key, value in details.items()]))
-
-        logShouldRun(cycle, model, term_id, True, methodTier=TIER,
-                     input=term.get('@id'),
-                     operation=operation.get('@id'),
-                     animal=animal.get('@id'))
-
-        return _emission(model, term_id, value, input=term, operation=operation, animal=animal)
-
-    return list(map(run_input, data.values()))
-
-
-def _emission_group(term_id: str):
-    lookup = download_lookup('emission.csv', True)
-    return get_table_value(lookup, 'termid', term_id, column_name('inputProductionGroupId'))
-
-
-def _group_emissions(impact: dict):
-    def _group_by(group: dict, emission: dict):
-        term_id = emission.get('term', {}).get('@id')
-        grouping = _emission_group(term_id)
-        value = emission.get('value') or 0
-        if grouping:
-            group[grouping] = group.get(grouping, 0) + value
-        return group
-
-    emissions = impact.get('emissionsResourceUse', [])
-    return reduce(_group_by, emissions, {})
-
-
-def _animal_inputs(animal: dict):
-    inputs = load_impacts(animal.get('inputs', []))
-    return [(input | {'animal': animal.get('term', {})}) for input in inputs]
-
-
-def _group_input_emissions(input: dict):
-    impact = input.get('impactAssessment')
-    emissions = _group_emissions(impact)
-    return input | {'emissions': emissions}
-
-
-def _group_inputs(group: dict, values: tuple):
-    # input_group_key = 'group-id'
-    # inputs = [{'term': {}, 'value':[], 'impactAssessment': {}, 'emissions': {'co2ToAirInputsProduction': 10}}]
-    input_group_key, inputs = values
-    for input in inputs:
-        input_value = list_sum(input.get('value'))
-        emissions = input.get('emissions', {})
-        for emission_term_id, emission_value in emissions.items():
-            group[emission_term_id] = group.get(emission_term_id, {})
-
-            grouped_inputs = group[emission_term_id].get(input_group_key, {
-                'term': input.get('term', {}),
-                'operation': input.get('operation', {}),
-                'animal': input.get('animal', {}),
-                'value': 0,
-                'aggregated': [],
-                'details': {}
-            })
-            grouped_inputs['aggregated'].append(input.get('impactAssessment', {}).get('agregated', False))
-            grouped_inputs['value'] = grouped_inputs['value'] + (emission_value * input_value)
-            # for logging
-            grouped_inputs['details'][input.get('impactAssessment', {}).get('@id')] = {
-                'emission-value': emission_value,
-                'input-value': input_value
-            }
-            group[emission_term_id][input_group_key] = grouped_inputs
-    return group
-
-
-def run(_, cycle: dict):
-    inputs = flatten(
-        load_impacts(cycle.get('inputs', [])) +
-        list(map(_animal_inputs, cycle.get('animals', [])))
-    )
-    inputs = [i for i in inputs if list_sum(i.get('value', [])) > 0]
-
-    # group inputs with same term/operation/animal to avoid adding emissions twice
-    # inputs = {'group-id': [{'term': {},'value':[10],'impactAssessment': {}}]}
-    inputs = reduce(group_by_keys(['term', 'operation', 'animal']), inputs, {})
-    inputs = {key: list(map(_group_input_emissions, value)) for key, value in inputs.items()}
-
-    debugValues(cycle, model=MODEL,
-                nb_inputs=len(inputs))
+PKG = '.'.join(['hestia_earth', 'models', MODEL])
 
-    # finally group everything by emission so we can log inputs together
-    # emissions = {'co2ToAirInputsProduct': {'group-id':{'term':{},'value':10,'details':{}}}}
-    emissions = reduce(_group_inputs, inputs.items(), {})
 
-    return flatten([_run_emission(cycle, term_id, data) for term_id, data in emissions.items()])
+def run(model: str, data): return run_if_required(MODEL, model, data, import_module(f".{model}", package=PKG))