hestia-earth-models 0.60.1__py3-none-any.whl → 0.61.0__py3-none-any.whl

hestia_earth/models/cycle/cycleDuration.py

@@ -1,50 +1,89 @@
 """
 Cycle duration
 
-This model calculates the cycle duration using the cropping intensity for a single year.
+This model calculates the cycle duration using:
+* the `endDate` and the `startDate` if both are provided with a day precision;
+* for temporary crops, using the `croppingIntensity` if provided;
+* for permanent crops, the duration is set to `365`.
 """
-from hestia_earth.utils.model import find_term_match
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import find_term_match, find_primary_product
+from hestia_earth.utils.date import diff_in_days
 
 from hestia_earth.models.log import logRequirements, logShouldRun
-from hestia_earth.models.utils.cycle import valid_site_type
+from hestia_earth.models.utils.crop import get_crop_grouping_fao
 from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "cycleDuration": "365",
         "endDate": "",
-        "practices": [{"@type": "Practice", "value": "", "term.@id": "croppingIntensity"}],
-        "site": {
-            "@type": "Site",
-            "siteType": "cropland"
+        "optional": {
+            "startDate": "",
+            "products": {
+                "@type": "Product",
+                "primary": "True",
+                "term.termType": "crop"
+            },
+            "practices": [{"@type": "Practice", "value": "", "term.@id": "croppingIntensity"}]
         }
     }
 }
 RETURNS = {
     "a `number` or `None` if requirements are not met": ""
 }
+LOOKUPS = {
+    "crop": "cropGroupingFAO"
+}
 MODEL_KEY = 'cycleDuration'
 DEFAULT_DURATION = 365
 
 
-def _run(croppingIntensity: float): return croppingIntensity * DEFAULT_DURATION
+def _run_by_dates(cycle: dict):
+    start_date = cycle.get('startDate')
+    end_date = cycle.get('endDate')
+    return diff_in_days(start_date, end_date)
+
+
+def _should_run_by_dates(cycle: dict):
+    start_date = cycle.get('startDate', '')
+    start_date_has_day = len(start_date) == 10
+    end_date = cycle.get('endDate', '')
+    end_date_has_day = len(start_date) == 10
+
+    logRequirements(cycle, model=MODEL, key=MODEL_KEY, by='dates',
+                    start_date_has_day=start_date_has_day,
+                    start_date=start_date,
+                    end_date_has_day=end_date_has_day,
+                    end_date=end_date)
+
+    should_run = all([start_date_has_day, end_date_has_day])
+    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY, by='dates')
+    return should_run
+
+
+def _run_by_crop(cycle: dict):
+    product = find_primary_product(cycle)
+    grouping = get_crop_grouping_fao(MODEL, MODEL_KEY, product.get('term', {}))
+    is_permanent_crop = grouping == 'Permanent crops'
+    croppingIntensity = find_term_match(cycle.get('practices', []), 'croppingIntensity').get('value', [1])[0]
+    return DEFAULT_DURATION * (1 if is_permanent_crop else croppingIntensity)
 
 
-def _should_run(cycle: dict):
-    croppingIntensity = find_term_match(cycle.get('practices', []), 'croppingIntensity').get('value', [None])[0]
-    cycleDuration = cycle.get('cycleDuration')
-    site_type_valid = valid_site_type(cycle)
+def _should_run_by_crop(cycle: dict):
+    product = find_primary_product(cycle) or {}
+    product_term_type = product.get('term', {}).get('termType')
+    primary_product_is_crop = product_term_type == TermTermType.CROP.value
 
-    logRequirements(cycle, model=MODEL, key=MODEL_KEY,
-                    cycleDuration=cycleDuration,
-                    croppingIntensity=croppingIntensity,
-                    site_type_valid=site_type_valid)
+    logRequirements(cycle, model=MODEL, key=MODEL_KEY, by='product',
+                    primary_product_is_crop=primary_product_is_crop)
 
-    should_run = all([site_type_valid, cycleDuration == 365, croppingIntensity])
-    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
-    return should_run, croppingIntensity
+    should_run = all([primary_product_is_crop])
+    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY, by='product')
+    return should_run
 
 
 def run(cycle: dict):
-    should_run, croppingIntensity = _should_run(cycle)
-    return _run(croppingIntensity) if should_run else None
+    return _run_by_dates(cycle) if _should_run_by_dates(cycle) else (
+        _run_by_crop(cycle) if _should_run_by_crop(cycle)
+        else None
+    )

hestia_earth/models/cycle/startDate.py

@@ -1,18 +1,20 @@
 """
 Start Date
 
-This model updates the [Cycle startDate](https://hestia.earth/schema/Cycle#startDate) to be in the following format:
-`YYYY-MM-DD`.
+This model sets the [Cycle startDate](https://hestia.earth/schema/Cycle#startDate) based on the `endDate` and the
+`cycleDuration`. This only works when the `endDate` has been provided to a day precision (`2000-01-01`).
 """
+from datetime import timedelta
 from hestia_earth.utils.date import is_in_days
-from hestia_earth.utils.tools import non_empty_list
+from hestia_earth.utils.tools import safe_parse_date
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from . import MODEL
 
 REQUIREMENTS = {
     "Cycle": {
-        "startDate": ""
+        "endDate": "to day precision",
+        "cycleDuration": ""
     }
 }
 RETURNS = {
@@ -22,19 +24,22 @@ MODEL_KEY = 'startDate'
 
 
 def _run(cycle: dict):
-    value = cycle.get('startDate')
-    return '-'.join(non_empty_list([value, '01' if len(value) == 4 else '', '01']))
+    endDate = safe_parse_date(cycle.get('endDate'))
+    cycleDuration = cycle.get('cycleDuration')
+    return (endDate - timedelta(days=cycleDuration)).strftime('%Y-%m-%d')
 
 
 def _should_run(cycle: dict):
-    has_startDate = cycle.get('startDate') is not None
-    has_incorrect_format = has_startDate and not is_in_days(cycle.get('startDate'))
+    has_endDate = cycle.get('endDate') is not None
+    has_day_precision = has_endDate and is_in_days(cycle.get('endDate'))
+    has_cycleDuration = cycle.get('cycleDuration') is not None
 
     logRequirements(cycle, model=MODEL, key=MODEL_KEY,
-                    has_startDate=has_startDate,
-                    has_incorrect_format=has_incorrect_format)
+                    has_endDate=has_endDate,
+                    has_day_precision=has_day_precision,
+                    has_cycleDuration=has_cycleDuration)
 
-    should_run = all([has_startDate, has_incorrect_format])
+    should_run = all([has_endDate, has_day_precision, has_cycleDuration])
     logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
     return should_run
 

hestia_earth/models/cycle/startDateDefinition.py

@@ -0,0 +1,69 @@
+"""
+Start Date Definition
+
+This model sets the [Cycle startDateDefinition](https://hestia.earth/schema/Cycle#startDateDefinition)
+based on the result of the `cycleDuration` gap-filling model.
+"""
+from hestia_earth.schema import TermTermType, CycleStartDateDefinition
+from hestia_earth.utils.model import find_primary_product
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.crop import get_crop_grouping_fao
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "cycleDuration": "",
+        "products": {
+            "@type": "Product",
+            "primary": "True",
+            "term.termType": "crop"
+        },
+        "optional": {
+            "endDate": ""
+        }
+    }
+}
+RETURNS = {
+    "The startDateDefinition as a string": ""
+}
+MODEL_KEY = 'startDateDefinition'
+
+
+def _is_last_day_of_month(date: str):
+    date_parts = date.split('-')
+    return len(date_parts) > 1 and date_parts[1] == '12' and any([
+        len(date_parts) == 3 and date_parts[2] == '31',
+        len(date_parts) == 2
+    ])
+
+
+def _run(cycle: dict):
+    product = find_primary_product(cycle)
+    grouping = get_crop_grouping_fao(MODEL, MODEL_KEY, product.get('term', {}))
+    is_permanent_crop = grouping == 'Permanent crops'
+    print(is_permanent_crop)
+    return (
+        CycleStartDateDefinition.START_OF_YEAR.value if _is_last_day_of_month(cycle.get('endDate'))
+        else CycleStartDateDefinition.ONE_YEAR_PRIOR.value
+    ) if is_permanent_crop else (
+        CycleStartDateDefinition.HARVEST_OF_PREVIOUS_CROP.value
+    )
+
+
+def _should_run(cycle: dict):
+    cycleDuration_added = 'cycleDuration' in cycle.get('added', [])
+    product = find_primary_product(cycle) or {}
+    product_term_type = product.get('term', {}).get('termType')
+    primary_product_is_crop = product_term_type == TermTermType.CROP.value
+
+    logRequirements(cycle, model=MODEL, key=MODEL_KEY,
+                    cycleDuration_added=cycleDuration_added,
+                    primary_product_is_crop=primary_product_is_crop)
+
+    should_run = all([cycleDuration_added, primary_product_is_crop])
+    logShouldRun(cycle, MODEL, None, should_run, key=MODEL_KEY)
+    return should_run
+
+
+def run(cycle: dict): return _run(cycle) if _should_run(cycle) else None

hestia_earth/models/impact_assessment/emissions.py

@@ -42,11 +42,9 @@ def _indicator(product: dict):
         term_id = emission.get('term', {}).get('@id')
         value = convert_value_from_cycle(product, list_sum(emission.get('value', [0])), model=MODEL, term_id=term_id)
 
-        indicator = _new_indicator(emission.get('term', {}))
+        indicator = _new_indicator(emission.get('term', {}), emission.get('methodModel'))
         indicator['value'] = value
 
-        if 'methodModel' in emission:
-            indicator['methodModel'] = emission['methodModel']
         if len(emission.get('inputs', [])):
             indicator['inputs'] = emission['inputs']
         if emission.get('operation'):

hestia_earth/models/ipcc2019/pastureGrass.py

@@ -147,7 +147,7 @@ def _input(term_id: str, value: float):
     return node
 
 
-def _sum_values(values: list, index=0): return sum([v[index] for v in values])
+def _sum_values(values: list, index=0): return list_sum([v[index] for v in values])
 
 
 def calculate_NEwool(cycle: dict) -> float:

hestia_earth/models/linkedImpactAssessment/__init__.py

@@ -1,167 +1,13 @@
-"""
-Linked Impact Assessment
+from os.path import dirname, abspath
+import sys
+from importlib import import_module
 
-A model which takes recalculated data from an Impact Assessment linked to an Input in a Cycle.
-"""
-from functools import reduce
-from hestia_earth.schema import EmissionMethodTier
-from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
-from hestia_earth.utils.tools import flatten, list_sum
+from hestia_earth.models.utils.blank_node import run_if_required
 
-from hestia_earth.models.log import debugValues, logRequirements, logShouldRun, log_as_table
-from hestia_earth.models.utils.emission import _new_emission
-from hestia_earth.models.utils.input import load_impacts
-from hestia_earth.models.utils.blank_node import group_by_keys
-
-REQUIREMENTS = {
-    "Cycle": {
-        "inputs": [{
-            "@type": "Input",
-            "value": "> 0",
-            "impactAssessment": {
-                "@type": "ImpactAssessment",
-                "emissionsResourceUse": [{"@type": "Indicator", "value": ""}]
-            }
-        }],
-        "optional": {
-            "animals": [{
-                "@type": "Animal",
-                "inputs": [{
-                    "@type": "Input",
-                    "value": "> 0",
-                    "impactAssessment": {
-                        "@type": "ImpactAssessment",
-                        "emissionsResourceUse": [{"@type": "Indicator", "value": ""}]
-                    }
-                }]
-            }]
-        }
-    }
-}
-RETURNS = {
-    "Emission": [{
-        "value": "",
-        "methodTier": "background",
-        "inputs": "",
-        "operation": "",
-        "animals": ""
-    }]
-}
+CURRENT_DIR = dirname(abspath(__file__)) + '/'
+sys.path.append(CURRENT_DIR)
 MODEL = 'linkedImpactAssessment'
-MODEL_AGGREGATED = 'hestiaAggregatedData'
-TIER = EmissionMethodTier.BACKGROUND.value
-
-
-def _emission(model: str, term_id: str, value: float, input: dict, operation={}, animal={}):
-    emission = _new_emission(term_id, model)
-    emission['value'] = [value]
-    emission['methodTier'] = TIER
-    emission['inputs'] = [input]
-    if operation:
-        emission['operation'] = operation
-    if animal:
-        emission['animals'] = [animal]
-    return emission
-
-
-def _run_emission(cycle: dict, term_id: str, data: dict):
-    def run_input(values: dict):
-        value = values.get('value', 0)
-        term = values.get('term', {})
-        operation = values.get('operation', {})
-        animal = values.get('animal', {})
-        is_aggregated = any(values.get('aggregated', []))
-        model = MODEL_AGGREGATED if is_aggregated else MODEL
-
-        details = values.get('details', {})
-        logRequirements(cycle, model=model, term=term_id,
-                        values=log_as_table([{'impact-assessment-id': key} | value for key, value in details.items()]))
-
-        logShouldRun(cycle, model, term_id, True, methodTier=TIER,
-                     input=term.get('@id'),
-                     operation=operation.get('@id'),
-                     animal=animal.get('@id'))
-
-        return _emission(model, term_id, value, input=term, operation=operation, animal=animal)
-
-    return list(map(run_input, data.values()))
-
-
-def _emission_group(term_id: str):
-    lookup = download_lookup('emission.csv', True)
-    return get_table_value(lookup, 'termid', term_id, column_name('inputProductionGroupId'))
-
-
-def _group_emissions(impact: dict):
-    def _group_by(group: dict, emission: dict):
-        term_id = emission.get('term', {}).get('@id')
-        grouping = _emission_group(term_id)
-        value = emission.get('value') or 0
-        if grouping:
-            group[grouping] = group.get(grouping, 0) + value
-        return group
-
-    emissions = impact.get('emissionsResourceUse', [])
-    return reduce(_group_by, emissions, {})
-
-
-def _animal_inputs(animal: dict):
-    inputs = load_impacts(animal.get('inputs', []))
-    return [(input | {'animal': animal.get('term', {})}) for input in inputs]
-
-
-def _group_input_emissions(input: dict):
-    impact = input.get('impactAssessment')
-    emissions = _group_emissions(impact)
-    return input | {'emissions': emissions}
-
-
-def _group_inputs(group: dict, values: tuple):
-    # input_group_key = 'group-id'
-    # inputs = [{'term': {}, 'value':[], 'impactAssessment': {}, 'emissions': {'co2ToAirInputsProduction': 10}}]
-    input_group_key, inputs = values
-    for input in inputs:
-        input_value = list_sum(input.get('value'))
-        emissions = input.get('emissions', {})
-        for emission_term_id, emission_value in emissions.items():
-            group[emission_term_id] = group.get(emission_term_id, {})
-
-            grouped_inputs = group[emission_term_id].get(input_group_key, {
-                'term': input.get('term', {}),
-                'operation': input.get('operation', {}),
-                'animal': input.get('animal', {}),
-                'value': 0,
-                'aggregated': [],
-                'details': {}
-            })
-            grouped_inputs['aggregated'].append(input.get('impactAssessment', {}).get('agregated', False))
-            grouped_inputs['value'] = grouped_inputs['value'] + (emission_value * input_value)
-            # for logging
-            grouped_inputs['details'][input.get('impactAssessment', {}).get('@id')] = {
-                'emission-value': emission_value,
-                'input-value': input_value
-            }
-            group[emission_term_id][input_group_key] = grouped_inputs
-    return group
-
-
-def run(_, cycle: dict):
-    inputs = flatten(
-        load_impacts(cycle.get('inputs', [])) +
-        list(map(_animal_inputs, cycle.get('animals', [])))
-    )
-    inputs = [i for i in inputs if list_sum(i.get('value', [])) > 0]
-
-    # group inputs with same term/operation/animal to avoid adding emissions twice
-    # inputs = {'group-id': [{'term': {},'value':[10],'impactAssessment': {}}]}
-    inputs = reduce(group_by_keys(['term', 'operation', 'animal']), inputs, {})
-    inputs = {key: list(map(_group_input_emissions, value)) for key, value in inputs.items()}
-
-    debugValues(cycle, model=MODEL,
-                nb_inputs=len(inputs))
+PKG = '.'.join(['hestia_earth', 'models', MODEL])
 
-    # finally group everything by emission so we can log inputs together
-    # emissions = {'co2ToAirInputsProduct': {'group-id':{'term':{},'value':10,'details':{}}}}
-    emissions = reduce(_group_inputs, inputs.items(), {})
 
-    return flatten([_run_emission(cycle, term_id, data) for term_id, data in emissions.items()])
+def run(model: str, data): return run_if_required(MODEL, model, data, import_module(f".{model}", package=PKG))

hestia_earth/models/linkedImpactAssessment/emissions.py

@@ -0,0 +1,168 @@
+"""
+Emissions
+
+A model which takes recalculated data from an Impact Assessment linked to an Input in a Cycle.
+"""
+from functools import reduce
+from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
+from hestia_earth.utils.tools import flatten, list_sum
+
+from hestia_earth.models.log import debugValues, logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils.emission import _new_emission
+from hestia_earth.models.utils.input import load_impacts
+from hestia_earth.models.utils.blank_node import group_by_keys
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "inputs": [{
+            "@type": "Input",
+            "value": "> 0",
+            "impactAssessment": {
+                "@type": "ImpactAssessment",
+                "emissionsResourceUse": [{"@type": "Indicator", "value": ""}]
+            }
+        }],
+        "optional": {
+            "animals": [{
+                "@type": "Animal",
+                "inputs": [{
+                    "@type": "Input",
+                    "value": "> 0",
+                    "impactAssessment": {
+                        "@type": "ImpactAssessment",
+                        "emissionsResourceUse": [{"@type": "Indicator", "value": ""}]
+                    }
+                }]
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "methodTier": "background",
+        "inputs": "",
+        "operation": "",
+        "animals": ""
+    }]
+}
+MODEL_KEY = 'emissions'
+MODEL_AGGREGATED = 'hestiaAggregatedData'
+TIER = EmissionMethodTier.BACKGROUND.value
+
+
+def _emission(model: str, term_id: str, value: float, input: dict, operation={}, animal={}):
+    emission = _new_emission(term_id, model)
+    emission['value'] = [value]
+    emission['methodTier'] = TIER
+    emission['inputs'] = [input]
+    if operation:
+        emission['operation'] = operation
+    if animal:
+        emission['animals'] = [animal]
+    return emission
+
+
+def _run_emission(cycle: dict, term_id: str, data: dict):
+    def run_input(values: dict):
+        value = values.get('value', 0)
+        term = values.get('term', {})
+        operation = values.get('operation', {})
+        animal = values.get('animal', {})
+        is_aggregated = any(values.get('aggregated', []))
+        model = MODEL_AGGREGATED if is_aggregated else MODEL
+
+        details = values.get('details', {})
+        logRequirements(cycle, model=model, term=term_id, key=MODEL_KEY,
+                        values=log_as_table([{'impact-assessment-id': key} | value for key, value in details.items()]))
+
+        logShouldRun(cycle, model, term_id, True, methodTier=TIER, key=MODEL_KEY,
+                     input=term.get('@id'),
+                     operation=operation.get('@id'),
+                     animal=animal.get('@id'))
+
+        return _emission(model, term_id, value, input=term, operation=operation, animal=animal)
+
+    return list(map(run_input, data.values()))
+
+
+def _emission_group(term_id: str):
+    lookup = download_lookup('emission.csv', True)
+    return get_table_value(lookup, 'termid', term_id, column_name('inputProductionGroupId'))
+
+
+def _group_emissions(impact: dict):
+    def _group_by(group: dict, emission: dict):
+        term_id = emission.get('term', {}).get('@id')
+        grouping = _emission_group(term_id)
+        value = emission.get('value') or 0
+        if grouping:
+            group[grouping] = group.get(grouping, 0) + value
+        return group
+
+    emissions = impact.get('emissionsResourceUse', [])
+    return reduce(_group_by, emissions, {})
+
+
+def _animal_inputs(animal: dict):
+    inputs = load_impacts(animal.get('inputs', []))
+    return [(input | {'animal': animal.get('term', {})}) for input in inputs]
+
+
+def _group_input_emissions(input: dict):
+    impact = input.get('impactAssessment')
+    emissions = _group_emissions(impact)
+    return input | {'emissions': emissions}
+
+
+def _group_inputs(group: dict, values: tuple):
+    # input_group_key = 'group-id'
+    # inputs = [{'term': {}, 'value':[], 'impactAssessment': {}, 'emissions': {'co2ToAirInputsProduction': 10}}]
+    input_group_key, inputs = values
+    for input in inputs:
+        input_value = list_sum(input.get('value'))
+        emissions = input.get('emissions', {})
+        for emission_term_id, emission_value in emissions.items():
+            group[emission_term_id] = group.get(emission_term_id, {})
+
+            grouped_inputs = group[emission_term_id].get(input_group_key, {
+                'term': input.get('term', {}),
+                'operation': input.get('operation', {}),
+                'animal': input.get('animal', {}),
+                'value': 0,
+                'aggregated': [],
+                'details': {}
+            })
+            grouped_inputs['aggregated'].append(input.get('impactAssessment', {}).get('agregated', False))
+            grouped_inputs['value'] = grouped_inputs['value'] + (emission_value * input_value)
+            # for logging
+            grouped_inputs['details'][input.get('impactAssessment', {}).get('@id')] = {
+                'emission-value': emission_value,
+                'input-value': input_value
+            }
+            group[emission_term_id][input_group_key] = grouped_inputs
+    return group
+
+
+def run(cycle: dict):
+    inputs = flatten(
+        load_impacts(cycle.get('inputs', [])) +
+        list(map(_animal_inputs, cycle.get('animals', [])))
+    )
+    inputs = [i for i in inputs if list_sum(i.get('value', [])) > 0]
+
+    # group inputs with same term/operation/animal to avoid adding emissions twice
+    # inputs = {'group-id': [{'term': {},'value':[10],'impactAssessment': {}}]}
+    inputs = reduce(group_by_keys(['term', 'operation', 'animal']), inputs, {})
+    inputs = {key: list(map(_group_input_emissions, value)) for key, value in inputs.items()}
+
+    debugValues(cycle, model=MODEL, key=MODEL_KEY,
+                nb_inputs=len(inputs))
+
+    # finally group everything by emission so we can log inputs together
+    # emissions = {'co2ToAirInputsProduct': {'group-id':{'term':{},'value':10,'details':{}}}}
+    emissions = reduce(_group_inputs, inputs.items(), {})
+
+    return flatten([_run_emission(cycle, term_id, data) for term_id, data in emissions.items()])

hestia_earth/models/{impact_assessment → linkedImpactAssessment}/utils.py

@@ -9,7 +9,7 @@ from . import MODEL
 
 
 def _indicator(term_id: str, value: float):
-    indicator = _new_indicator(term_id)
+    indicator = _new_indicator(term_id, MODEL)
     indicator['value'] = value
     return indicator