hestia-earth-models 0.59.4__py3-none-any.whl → 0.59.5__py3-none-any.whl

hestia_earth/models/linkedImpactAssessment/__init__.py

@@ -8,7 +8,7 @@ from hestia_earth.schema import EmissionMethodTier
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
 from hestia_earth.utils.tools import flatten, list_sum
 
-from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
+from hestia_earth.models.log import debugValues, logRequirements, logShouldRun, log_as_table
 from hestia_earth.models.utils.emission import _new_emission
 from hestia_earth.models.utils.input import load_impacts
 from hestia_earth.models.utils.blank_node import group_by_keys
@@ -52,71 +52,99 @@ MODEL_AGGREGATED = 'hestiaAggregatedData'
 TIER = EmissionMethodTier.BACKGROUND.value
 
 
-def _emission(cycle: dict, term_id: str, value: float, input: dict, model: str):
-    # log run on each emission so we know it did run
-    input_term_id = input.get('term', {}).get('@id')
-    operation_term_id = input.get('operation', {}).get('@id')
-    animal_term_id = input.get('animal', {}).get('@id')
-
-    logShouldRun(cycle, model, term_id, True, methodTier=TIER,
-                 input=input_term_id,
-                 operation=operation_term_id,
-                 animal=animal_term_id)
-
+def _emission(model: str, term_id: str, value: float, input: dict, operation={}, animal={}):
     emission = _new_emission(term_id, model)
     emission['value'] = [value]
     emission['methodTier'] = TIER
-    emission['inputs'] = [input.get('term')]
-    if input.get('operation'):
-        emission['operation'] = input.get('operation')
-    if input.get('animal'):
-        emission['animals'] = [input.get('animal')]
+    emission['inputs'] = [input]
+    if operation:
+        emission['operation'] = operation
+    if animal:
+        emission['animals'] = [animal]
     return emission
 
 
+def _run_emission(cycle: dict, term_id: str, data: dict):
+    def run_input(values: dict):
+        value = values.get('value', 0)
+        term = values.get('term', {})
+        operation = values.get('operation', {})
+        animal = values.get('animal', {})
+        is_aggregated = any(values.get('aggregated', []))
+        model = MODEL_AGGREGATED if is_aggregated else MODEL
+
+        details = values.get('details', {})
+        logRequirements(cycle, model=model, term=term_id,
+                        values=log_as_table([{'impact_assessment_id': key} | value for key, value in details.items()]))
+
+        logShouldRun(cycle, model, term_id, True, methodTier=TIER,
+                     input=term.get('@id'),
+                     operation=operation.get('@id'),
+                     animal=animal.get('@id'))
+
+        return _emission(model, term_id, value, input=term, operation=operation, animal=animal)
+
+    return list(map(run_input, data.values()))
+
+
 def _emission_group(term_id: str):
     lookup = download_lookup('emission.csv', True)
     return get_table_value(lookup, 'termid', term_id, column_name('inputProductionGroupId'))
 
 
 def _group_emissions(impact: dict):
-    def _group_by(prev: dict, emission: dict):
+    def _group_by(group: dict, emission: dict):
         term_id = emission.get('term', {}).get('@id')
         grouping = _emission_group(term_id)
         value = emission.get('value') or 0
         if grouping:
-            prev[grouping] = prev.get(grouping, 0) + value
-        return prev
+            group[grouping] = group.get(grouping, 0) + value
+        return group
 
     emissions = impact.get('emissionsResourceUse', [])
     return reduce(_group_by, emissions, {})
 
 
-def _run_input(cycle: dict):
-    def run(inputs: list):
-        input = inputs[0]
-        input_value = list_sum(flatten(input.get('value', []) for input in inputs))
-        term_id = input.get('term', {}).get('@id')
-        impact = input.get('impactAssessment')
-        model = MODEL_AGGREGATED if impact.get('aggregated', False) else MODEL
-        emissions = _group_emissions(impact)
-
-        logRequirements(cycle, model=model, term=term_id,
-                        impact_assessment_id=input.get('impactAssessment', {}).get('@id'),
-                        input_value=input_value)
-        logShouldRun(cycle, model, term_id, True, methodTier=TIER)
-
-        return [
-            _emission(cycle, id, input_value * value, input, model) for id, value in emissions.items()
-        ]
-    return run
-
-
 def _animal_inputs(animal: dict):
     inputs = load_impacts(animal.get('inputs', []))
     return [(input | {'animal': animal.get('term', {})}) for input in inputs]
 
 
+def _group_input_emissions(input: dict):
+    impact = input.get('impactAssessment')
+    emissions = _group_emissions(impact)
+    return input | {'emissions': emissions}
+
+
+def _group_inputs(group: dict, values: tuple):
+    # input_group_key = 'group-id'
+    # inputs = [{'term': {}, 'value':[], 'impactAssessment': {}, 'emissions': {'co2ToAirInputsProduction': 10}}]
+    input_group_key, inputs = values
+    for input in inputs:
+        input_value = list_sum(input.get('value'))
+        emissions = input.get('emissions', {})
+        for emission_term_id, emission_value in emissions.items():
+            group[emission_term_id] = group.get(emission_term_id, {})
+
+            grouped_inputs = group[emission_term_id].get(input_group_key, {
+                'term': input.get('term', {}),
+                'operation': input.get('operation', {}),
+                'animal': input.get('animal', {}),
+                'value': 0,
+                'aggregated': [],
+                'details': {}
+            })
+            grouped_inputs['aggregated'].append(input.get('impactAssessment', {}).get('agregated', False))
+            grouped_inputs['value'] = grouped_inputs['value'] + (emission_value * input_value)
+            # for logging
+            grouped_inputs['details'][input.get('impactAssessment', {}).get('@id')] = {
+                'emission_value': emission_value,
+                'input_value': input_value
+            }
+            group[emission_term_id][input_group_key] = grouped_inputs
+    return group
+
+
 def run(_, cycle: dict):
     inputs = flatten(
         load_impacts(cycle.get('inputs', [])) +
@@ -124,9 +152,16 @@ def run(_, cycle: dict):
     )
     inputs = [i for i in inputs if list_sum(i.get('value', [])) > 0]
 
+    # group inputs with same term/operation/animal to avoid adding emissions twice
+    # inputs = {'group-id': [{'term': {},'value':[10],'impactAssessment': {}}]}
+    inputs = reduce(group_by_keys(['term', 'operation', 'animal']), inputs, {})
+    inputs = {key: list(map(_group_input_emissions, value)) for key, value in inputs.items()}
+
     debugValues(cycle, model=MODEL,
                 nb_inputs=len(inputs))
 
-    # group inputs with same id/operation to avoid adding emissions twice
-    inputs = reduce(group_by_keys(['term', 'operation', 'animal']), inputs, {})
-    return flatten(map(_run_input(cycle), inputs.values()))
+    # finally group everything by emission so we can log inputs together
+    # emissions = {'co2ToAirInputsProduct': {'group-id':{'term':{},'value':10,'details':{}}}}
+    emissions = reduce(_group_inputs, inputs.items(), {})
+
+    return flatten([_run_emission(cycle, term_id, data) for term_id, data in emissions.items()])

hestia_earth/models/mocking/search-results.json

@@ -1532,45 +1532,6 @@
       }
     ]
   },
-  {
-    "name": "get_long_fallow_land_cover_terms",
-    "query": {
-      "bool": {
-        "must": [
-          {
-            "match": {
-              "@type": "Term"
-            }
-          },
-          {
-            "match": {
-              "termType.keyword": "landCover"
-            }
-          },
-          {
-            "match_phrase_prefix": {
-              "name": "long"
-            }
-          },
-          {
-            "match": {
-              "name": "fallow"
-            }
-          }
-        ]
-      }
-    },
-    "results": [
-      {
-        "@type": "Term",
-        "@id": "longFallow"
-      },
-      {
-        "@type": "Term",
-        "@id": "longBareFallow"
-      }
-    ]
-  },
   {
     "name": "get_milkYield_terms",
     "query": {

hestia_earth/models/schmidt2007/n2OToAirWasteTreatmentDirect.py

@@ -0,0 +1,58 @@
+from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.emission import _new_emission
+from .utils import get_waste_values
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "product": [
+                {"@type": "Product", "value": "", "term.termType": "waste"}
+            ],
+            "completeness.waste": ""
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "methodTier": "tier 1"
+    }]
+}
+LOOKUPS = {
+    "waste": "n2oEfSchmidt2007"
+}
+TERM_ID = 'n2OToAirWasteTreatmentDirect'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def _emission(value: float):
+    emission = _new_emission(TERM_ID, MODEL)
+    emission['value'] = [value]
+    emission['methodTier'] = TIER
+    return emission
+
+
+def _run(waste_values: list):
+    value = list_sum(waste_values)
+    return [_emission(value)]
+
+
+def _should_run(cycle: dict):
+    waste_values = get_waste_values(TERM_ID, cycle, LOOKUPS['waste'])
+    has_waste = len(waste_values) > 0
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    has_waste=has_waste)
+
+    should_run = any([has_waste])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
+    return should_run, waste_values
+
+
+def run(cycle: dict):
+    should_run, waste_values = _should_run(cycle)
+    return _run(waste_values) if should_run else []

hestia_earth/models/schmidt2007/nh3ToAirWasteTreatment.py

@@ -0,0 +1,58 @@
+from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.emission import _new_emission
+from .utils import get_waste_values
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "product": [
+                {"@type": "Product", "value": "", "term.termType": "waste"}
+            ],
+            "completeness.waste": ""
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "methodTier": "tier 1"
+    }]
+}
+LOOKUPS = {
+    "waste": "nh3EfSchmidt2007"
+}
+TERM_ID = 'nh3ToAirWasteTreatment'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def _emission(value: float):
+    emission = _new_emission(TERM_ID, MODEL)
+    emission['value'] = [value]
+    emission['methodTier'] = TIER
+    return emission
+
+
+def _run(waste_values: list):
+    value = list_sum(waste_values)
+    return [_emission(value)]
+
+
+def _should_run(cycle: dict):
+    waste_values = get_waste_values(TERM_ID, cycle, LOOKUPS['waste'])
+    has_waste = len(waste_values) > 0
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    has_waste=has_waste)
+
+    should_run = any([has_waste])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
+    return should_run, waste_values
+
+
+def run(cycle: dict):
+    should_run, waste_values = _should_run(cycle)
+    return _run(waste_values) if should_run else []

hestia_earth/models/site/management.py

@@ -1,12 +1,12 @@
 """
 Management node with data gap-filled data from cycles.
 """
-from typing import List
+from typing import List, Any
 from functools import reduce
 from hestia_earth.schema import SchemaType, TermTermType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import filter_list_term_type, linked_node
-from hestia_earth.utils.tools import flatten
+from hestia_earth.utils.tools import flatten, safe_parse_float
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_blank_nodes_id
 from hestia_earth.models.utils.term import get_lookup_value
@@ -37,6 +37,16 @@ REQUIREMENTS = {
                         ],
                         "value": ""
                     }
+                ],
+                "inputs": [
+                    {
+                        "@type": "Input",
+                        "term.termType": [
+                            "inorganicFertiliser",
+                            "organicFertiliser",
+                            "soilAmendment"
+                        ]
+                    }
                 ]
             }]
         }
@@ -53,9 +63,45 @@ RETURNS = {
         "startDate": ""
     }]
 }
-
+LOOKUPS = {
+    "inorganicFertiliser": "nitrogenContent",
+    "organicFertiliser": "ANIMAL_MANURE",
+    "soilAmendment": "PRACTICE_INCREASING_C_INPUT"
+}
 MODEL_KEY = 'management'
 LAND_COVER_KEY = 'landCoverId'
+ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
+INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID = "inorganicNitrogenFertiliserUsed"
+ORGANIC_FERTILISER_USED_TERM_ID = "organicFertiliserUsed"
+AMENDMENT_INCREASING_C_USED_TERM_ID = "amendmentIncreasingSoilCarbonUsed"
+INPUT_RULES = {
+    TermTermType.INORGANICFERTILISER.value: (
+        (
+            TermTermType.INORGANICFERTILISER.value,  # Lookup column
+            lambda x: safe_parse_float(x) > 0,  # Condition
+            INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID  # New term.
+        ),
+    ),
+    TermTermType.SOILAMENDMENT.value: (
+        (
+            TermTermType.SOILAMENDMENT.value,
+            lambda x: x is True,
+            AMENDMENT_INCREASING_C_USED_TERM_ID
+        ),
+    ),
+    TermTermType.ORGANICFERTILISER.value: (
+        (
+            TermTermType.SOILAMENDMENT.value,
+            lambda x: x is True,
+            ORGANIC_FERTILISER_USED_TERM_ID
+        ),
+        (
+            TermTermType.ORGANICFERTILISER.value,
+            lambda x: x is True,
+            ANIMAL_MANURE_USED_TERM_ID
+        )
+    )
+}
 
 
 def management(data: dict):
@@ -74,7 +120,9 @@ def _include_start_end(cycle: dict, values: list):
     return [(_include(cycle, ['startDate', 'endDate']) | v) for v in values]
 
 
-def _copy_item_if_exists(source: dict, keys: List[str] = [], dest: dict = {}) -> dict:
+def _copy_item_if_exists(source: dict, keys: List[str] = None, dest: dict = None) -> dict:
+    keys = keys or []
+    dest = dest or {}
     return reduce(lambda p, c: p | ({c: source[c]} if c in source else {}), keys, dest)
 
 
@@ -96,6 +144,50 @@ def _get_items_with_relevant_term_type(cycles: List[dict], item_name: str, relev
     )
 
 
+def _get_lookup_with_debug(term: dict, column: str) -> Any:
+    get_lookup_value(term, column, model_key=MODEL_KEY, land_cover_key=LAND_COVER_KEY)
+
+
+def _data_from_input(cycle: dict, term_id: str) -> dict:
+    return {
+        "term": {
+            "@type": "Term",
+            "@id": term_id,
+            "termType": "landUseManagement"
+        },
+        "value": True,
+        "startDate": cycle["startDate"],
+        "endDate": cycle["endDate"]
+    }
+
+
+def _process_rule(cycle, term, term_type) -> List:
+    relevant_terms = []
+    for column, condition, new_term in INPUT_RULES[term_type]:
+        lookup_result = _get_lookup_with_debug(term, LOOKUPS[column])
+
+        if condition(lookup_result):
+            relevant_terms.append(_data_from_input(cycle=cycle, term_id=new_term))
+
+    return relevant_terms
+
+
+def _get_relevant_inputs(cycles: List[dict]) -> List:
+    relevant_inputs = []
+    for cycle in [c for c in cycles if "inputs" in c]:
+        for i in cycle["inputs"]:
+            if i.get("term", {}).get("termType", "") in INPUT_RULES:
+                relevant_inputs.extend(
+                    _process_rule(
+                        cycle=cycle,
+                        term=i.get("term", {}),
+                        term_type=i.get("term", {}).get("termType", "")
+                    )
+                )
+
+    return relevant_inputs
+
+
 def _should_run(site: dict):
     # Only get related cycles once.
     cycles = related_cycles(site.get("@id"))
@@ -148,6 +240,7 @@ def _should_run(site: dict):
         )
     ]
 
+    relevant_inputs = _get_relevant_inputs(cycles)
     logRequirements(
         site,
         model=MODEL,
@@ -155,14 +248,14 @@ def _should_run(site: dict):
         model_key=MODEL_KEY,
         products_crop_forage_ids=log_blank_nodes_id(products_crop_forage),
         products_land_cover_ids=log_blank_nodes_id(products_land_cover),
-        practice_ids=log_blank_nodes_id(practices)
+        practice_ids=log_blank_nodes_id(practices),
+        inputs=log_blank_nodes_id(relevant_inputs)
     )
-
-    should_run = any(products_crop_forage + products_land_cover + practices)
-    logShouldRun(site, MODEL, None, should_run=_should_run, model_key=MODEL_KEY)
-    return should_run, products_crop_forage + products_land_cover, practices
+    should_run = any(products_crop_forage + products_land_cover + practices + relevant_inputs)
+    logShouldRun(site, MODEL, None, should_run=should_run, model_key=MODEL_KEY)
+    return should_run, products_crop_forage + products_land_cover, practices, relevant_inputs
 
 
 def run(site: dict):
-    should_run, products, practices = _should_run(site)
-    return list(map(management, products + practices)) if should_run else []
+    should_run, products, practices, inputs = _should_run(site)
+    return list(map(management, products + practices + inputs)) if should_run else []

hestia_earth/models/site/soilMeasurement.py

@@ -16,7 +16,7 @@ from . import MODEL
 REQUIREMENTS = {
     "Site": {
         "measurements": [
-            {"@type": "Measurement", "depthUpper": "", "depthLower": ""}
+            {"@type": "Measurement", "depthUpper": "", "depthLower": "", "value": ""}
         ]
     }
 }
@@ -159,21 +159,21 @@ def _get_needed_depths(site: dict) -> list:
 
 def _should_run(site: dict, model_key: str):
     # we only work with measurements with depths
-    measurements = [
-        m for m in site.get("measurements", [])
-        if get_lookup_value(m.get("term", {}), LOOKUPS["measurement"][0], model=MODEL, model_key=model_key)
-    ]
+    measurements = [m for m in site.get("measurements", []) if all([
+        get_lookup_value(m.get("term", {}), LOOKUPS["measurement"][0], model=MODEL, model_key=model_key),
+        m.get('value', [])
+    ])]
 
     measurements_with_depths = [m for m in measurements if all([
-        "depthUpper" in m.keys(),
-        "depthLower" in m.keys(),
+        "depthUpper" in m,
+        "depthLower" in m,
         (int(m.get("depthUpper", 0)), int(m.get("depthLower", 0))) not in STANDARD_DEPTHS
     ])]
     has_measurements_with_depths = len(measurements_with_depths) > 0
 
     measurements_missing_depth_recommended = [m for m in measurements if all([
-        "depthUpper" not in m.keys(),
-        "depthLower" not in m.keys(),
+        "depthUpper" not in m,
+        "depthLower" not in m,
         not get_lookup_value(m.get("term", {}), LOOKUPS["measurement"][1], model=MODEL, model_key=model_key)
     ])]
 

hestia_earth/models/utils/blank_node.py

@@ -37,9 +37,9 @@ from .term import get_lookup_value
 
 
 def group_by_keys(group_keys: list = ['term']):
-    def run(group: dict, input: dict):
-        group_key = '-'.join(non_empty_list(map(lambda v: input.get(v, {}).get('@id'), group_keys)))
-        group[group_key] = group.get(group_key, []) + [input]
+    def run(group: dict, node: dict):
+        group_key = '-'.join(non_empty_list(map(lambda v: node.get(v, {}).get('@id'), group_keys)))
+        group[group_key] = group.get(group_key, []) + [node]
         return group
     return run
 

hestia_earth/models/utils/term.py

@@ -603,26 +603,3 @@ def get_pasture_system_terms():
         'name': 'pasture'
     }, limit=LIMIT)
     return list(map(lambda n: n["@id"], terms))
-
-
-def get_long_fallow_land_cover_terms():
-    """
-    Find all `landCover` terms with the name `long fallow`:
-    https://hestia.earth/glossary?termType=landCover&query=long%fallow
-
-    Returns
-    -------
-    list
-        List of matching term `@id` as `str`.
-    """
-    terms = search({
-        "bool": {
-            "must": [
-                {"match": {"@type": SchemaType.TERM.value}},
-                {"match": {"termType.keyword": TermTermType.LANDCOVER.value}},
-                {"match_phrase_prefix": {"name": "long"}},
-                {"match": {"name": "fallow"}}
-            ],
-        }
-    }, limit=LIMIT)
-    return list(map(lambda n: n["@id"], terms))

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.59.4'
+VERSION = '0.59.5'

{hestia_earth_models-0.59.4.dist-info → hestia_earth_models-0.59.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.59.4
+Version: 0.59.5
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team