hestia-earth-models 0.62.1__py3-none-any.whl → 0.62.2__py3-none-any.whl

hestia_earth/models/ipcc2019/pastureGrass.py

@@ -12,7 +12,8 @@ from hestia_earth.schema import TermTermType
 from hestia_earth.utils.model import filter_list_term_type
 from hestia_earth.utils.tools import list_sum
 
-from hestia_earth.models.log import logRequirements, logShouldRun, debugValues
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils.blank_node import lookups_logs, properties_logs
 from hestia_earth.models.utils.input import _new_input
 from hestia_earth.models.utils.term import get_wool_terms
 from hestia_earth.models.utils.completeness import _is_term_type_complete, _is_term_type_incomplete
@@ -25,6 +26,7 @@ from .pastureGrass_utils import (
     calculate_REM,
     calculate_REG,
     calculate_NEfeed,
+    calculate_GE,
     product_wool_energy,
     get_animals,
     get_animal_values
@@ -119,7 +121,8 @@ REQUIREMENTS = {
 }
 LOOKUPS = {
     "animalManagement": [
-        "mjKgEvMilkIpcc2019"
+        "mjKgEvMilkIpcc2019",
+        "defaultFatContentEvMilkIpcc2019"
     ],
     "animalProduct": ["mjKgEvWoolNetEnergyWoolIpcc2019"],
     "liveAnimal": [
@@ -127,7 +130,8 @@ LOOKUPS = {
         "mjDayKgCfiNetEnergyMaintenanceIpcc2019",
         "ratioCPregnancyNetEnergyPregnancyIpcc2019",
         "ratioCNetEnergyGrowthCattleBuffaloIpcc2019",
-        "mjKgABNetEnergyGrowthSheepGoatsIpcc2019"
+        "mjKgABNetEnergyGrowthSheepGoatsIpcc2019",
+        "isWoolProducingAnimal"
     ],
     "system-liveAnimal-activityCoefficient-ipcc2019": "using animal term @id",
     "crop-property": ["energyDigestibilityRuminants", "energyContentHigherHeatingValue"],
@@ -150,9 +154,6 @@ def _input(term_id: str, value: float):
     return node
 
 
-def _sum_values(values: list, index=0): return list_sum([v[index] for v in values])
-
-
 def calculate_NEwool(cycle: dict) -> float:
     term_ids = get_wool_terms()
     products = [p for p in cycle.get('products', []) if p.get('term', {}).get('@id') in term_ids]
@@ -162,39 +163,20 @@ def calculate_NEwool(cycle: dict) -> float:
     return sum([value * lookup_value for (value, lookup_value) in product_values])
 
 
-def _calculate_GE(
-    cycle: dict, animals: list, REM: float, REG: float, NEwool: float, NEm_feed: float, NEg_feed: float, system: dict
-) -> float:
-    values = [get_animal_values(cycle, animal, system, log_node=cycle) for animal in animals]
-    NEm = _sum_values(values, 0)
-    NEa = _sum_values(values, 1)
-    NEl = _sum_values(values, 2)
-    NEwork = _sum_values(values, 3)
-    NEp = _sum_values(values, 4)
-    NEg = _sum_values(values, 5)
-
-    debugValues(cycle, model=MODEL, term=MODEL_KEY, model_key=MODEL_KEY,
-                NEm=NEm,
-                NEa=NEa,
-                NEl=NEl,
-                NEwork=NEwork,
-                NEp=NEp,
-                NEg=NEg,
-                NEm_feed=NEm_feed,
-                NEg_feed=NEg_feed)
-
-    return (NEm + NEa + NEl + NEwork + NEp - NEm_feed)/REM + (NEg + NEwool - NEg_feed)/REG
-
-
 def _run_practice(cycle: dict, meanDE: float, meanECHHV: float, system: dict):
     animals = get_animals(cycle)
     REM = calculate_REM(meanDE)
     REG = calculate_REG(meanDE)
     NEwool = calculate_NEwool(cycle)
     NEm_feed, NEg_feed = calculate_NEfeed(cycle)
+
+    animal_values = [{
+        'animalId': animal.get('term', {}).get('@id')
+    } | get_animal_values(cycle, animal, system) for animal in animals]
+
     GE = (
-        _calculate_GE(cycle, animals, REM, REG, NEwool, NEm_feed, NEg_feed, system) / (meanDE/100)
-    ) if all([REM, REG]) else 0
+        calculate_GE(animal_values, REM, REG, NEwool, NEm_feed, NEg_feed) / (meanDE/100)
+    ) if meanDE else 0
 
     def run(practice: dict):
         key = practice.get('key', {})
@@ -202,14 +184,34 @@ def _run_practice(cycle: dict, meanDE: float, meanECHHV: float, system: dict):
         input_term_id = practice_input_id(practice)
         value = (GE / meanECHHV) * (list_sum(practice.get('value', [0])) / 100)
 
+        logs = log_as_table([v | {
+            'practiceKeyId': key_id,
+            'REM': REM,
+            'REG': REG,
+            'NEwool': NEwool,
+            'NEmFeed': NEm_feed,
+            'NEgFeed': NEg_feed,
+            'GE': GE,
+            'meanECHHV': meanECHHV,
+            'meanDE': meanDE
+        } for v in animal_values])
+        animal_lookups = lookups_logs(MODEL, animals, LOOKUPS, model_key=MODEL_KEY, term=input_term_id)
+        animal_properties = properties_logs(animals, properties=[
+            'liveweightPerHead',
+            'hoursWorkedPerDay',
+            'animalsPerBirth',
+            'pregnancyRateTotal',
+            'weightAtMaturity',
+            'liveweightGain',
+            'weightAtWeaning',
+            'weightAtOneYear',
+            'weightAtSlaughter'
+        ])
+
         logRequirements(cycle, model=MODEL, term=input_term_id, model_key=MODEL_KEY,
-                        REM=REM,
-                        REG=REG,
-                        NEwool=NEwool,
-                        NEm_feed=NEm_feed,
-                        NEg_feed=NEg_feed,
-                        GE=GE,
-                        practice_key_id=key_id)
+                        animal_logs=logs,
+                        animal_lookups=animal_lookups,
+                        animal_properties=animal_properties)
 
         logShouldRun(cycle, MODEL, input_term_id, True, model_key=MODEL_KEY)
 
@@ -249,8 +251,8 @@ def _should_run(cycle: dict, practices: dict):
                         term_type_freshForage_incomplete=freshForage_incomplete,
                         has_cycle_inputs_feed=has_cycle_inputs_feed,
                         all_animals_have_value=all_animals_have_value,
-                        meanDE=meanDE,
-                        meanECHHV=meanECHHV)
+                        meanDE=calculate_meanDE(practices, term=term_id),
+                        meanECHHV=calculate_meanECHHV(practices, term=term_id))
 
         logShouldRun(cycle, MODEL, term_id, should_run, model_key=MODEL_KEY)
 

hestia_earth/models/ipcc2019/pastureGrass_utils.py

@@ -25,22 +25,18 @@ def _get_grouping(animal: dict) -> str:
     return get_lookup_value(term, 'ipcc2019AnimalTypeGrouping', model=MODEL, model_key=MODEL_KEY)
 
 
-def _get_activityCoefficient(cycle: dict, animal: dict, system: dict, log_node: dict) -> float:
+def _get_activityCoefficient(cycle: dict, animal: dict, system: dict) -> float:
     term = animal.get('term', {})
     term_id = term.get('@id')
     system_id = system.get('term', {}).get('@id')
     lookup = download_lookup('system-liveAnimal-activityCoefficient-ipcc2019.csv')
     activityCoefficient = safe_parse_float(get_table_value(lookup, 'termid', system_id, column_name(term_id)), 0)
 
-    debugValues(log_node, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                activityCoefficient=activityCoefficient)
-
     return activityCoefficient
 
 
-def _calculate_NEm(cycle: dict, animal: dict, log_node: dict) -> float:
+def _calculate_NEm(cycle: dict, animal: dict) -> float:
     term = animal.get('term', {})
-    term_id = term.get('@id')
 
     mjDayKgCfiNetEnergyMaintenance = safe_parse_float(
         get_lookup_value(term, 'mjDayKgCfiNetEnergyMaintenanceIpcc2019', model=MODEL, model_key=MODEL_KEY), 0
@@ -50,43 +46,25 @@ def _calculate_NEm(cycle: dict, animal: dict, log_node: dict) -> float:
     cycleDuration = cycle.get('cycleDuration', 365)
     NEm = mjDayKgCfiNetEnergyMaintenance * pow(liveweightPerHead, 0.75) * animal_value * cycleDuration
 
-    debugValues(log_node, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                mjDayKgCfiNetEnergyMaintenance=mjDayKgCfiNetEnergyMaintenance,
-                liveweightPerHead=liveweightPerHead,
-                NEm=NEm)
-
     return NEm
 
 
-def _calculate_NEa_cattleAndBuffalo(cycle: dict, animal: dict, system: dict, NEm: float, log_node: dict) -> float:
-    term = animal.get('term', {})
-    term_id = term.get('@id')
-
-    activityCoefficient = _get_activityCoefficient(cycle, animal, system, log_node)
+def _calculate_NEa_cattleAndBuffalo(cycle: dict, animal: dict, system: dict, NEm: float) -> float:
+    activityCoefficient = _get_activityCoefficient(cycle, animal, system)
 
     NEa = activityCoefficient * NEm
 
-    debugValues(log_node, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                NEa=NEa)
-
     return NEa
 
 
-def _calculate_NEa_sheepAndGoat(cycle: dict, animal: dict, system: dict, _NEm: float, log_node: dict) -> float:
-    term = animal.get('term', {})
-    term_id = term.get('@id')
-
-    activityCoefficient = _get_activityCoefficient(cycle, animal, system, log_node)
+def _calculate_NEa_sheepAndGoat(cycle: dict, animal: dict, system: dict, _NEm: float) -> float:
+    activityCoefficient = _get_activityCoefficient(cycle, animal, system)
 
     liveweightPerHead = get_node_property(animal, 'liveweightPerHead', False).get('value', 0)
     animal_value = animal.get('value', 0)
     cycleDuration = cycle.get('cycleDuration', 365)
     NEa = activityCoefficient * liveweightPerHead * animal_value * cycleDuration
 
-    debugValues(log_node, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                liveweightPerHead=liveweightPerHead,
-                NEa=NEa)
-
     return NEa
 
 
@@ -96,15 +74,12 @@ _NEa_BY_GROUPING = {
 }
 
 
-def _calculate_NEa(cycle: dict, animal: dict, system: dict, NEm: float, log_node: dict) -> float:
+def _calculate_NEa(cycle: dict, animal: dict, system: dict, NEm: float) -> float:
     grouping = _get_grouping(animal)
-    return _NEa_BY_GROUPING.get(grouping, lambda *args: 0)(cycle, animal, system, NEm, log_node)
+    return _NEa_BY_GROUPING.get(grouping, lambda *args: 0)(cycle, animal, system, NEm)
 
 
-def _calculate_NEl_cattleAndBuffalo(cycle: dict, animal: dict, log_node: dict) -> float:
-    term = animal.get('term', {})
-    term_id = term.get('@id')
-
+def _calculate_NEl_cattleAndBuffalo(cycle: dict, animal: dict) -> float:
     milkYieldPractice = get_milkYield_practice(animal)
     milkYield = list_sum(milkYieldPractice.get('value', []))
     fatContent = get_node_property(milkYieldPractice, 'fatContent').get('value', 0)
@@ -112,18 +87,10 @@ def _calculate_NEl_cattleAndBuffalo(cycle: dict, animal: dict, log_node: dict) -
     cycleDuration = cycle.get('cycleDuration', 365)
     NEl = milkYield * (1.47 + (0.4 * fatContent)) * animal_value * cycleDuration
 
-    debugValues(log_node, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                milkYield=milkYield,
-                fatContent=fatContent,
-                NEl=NEl)
-
     return NEl
 
 
-def _calculate_NEl_sheepAndGoat(cycle: dict, animal: dict, log_node: dict) -> float:
-    term = animal.get('term', {})
-    term_id = term.get('@id')
-
+def _calculate_NEl_sheepAndGoat(cycle: dict, animal: dict) -> float:
     milkYieldPractice = get_milkYield_practice(animal)
     milkYield = list_sum(milkYieldPractice.get('value', []))
     EV_milk = safe_parse_float(
@@ -131,8 +98,9 @@ def _calculate_NEl_sheepAndGoat(cycle: dict, animal: dict, log_node: dict) -> fl
         0
     )
     default_fatContent = safe_parse_float(
-        get_lookup_value(milkYieldPractice.get('term', {}),
-                         'defaultFatContentEvMilkIpcc2019', model=MODEL, model_key=MODEL_KEY),
+        get_lookup_value(
+            milkYieldPractice.get('term', {}), 'defaultFatContentEvMilkIpcc2019', model=MODEL, model_key=MODEL_KEY
+        ),
         7
     )
     fatContent = get_node_property(milkYieldPractice, 'fatContent').get('value', 0)
@@ -140,11 +108,6 @@ def _calculate_NEl_sheepAndGoat(cycle: dict, animal: dict, log_node: dict) -> fl
     cycleDuration = cycle.get('cycleDuration', 365)
     NEl = milkYield * (EV_milk * fatContent/default_fatContent) * animal_value * cycleDuration
 
-    debugValues(log_node, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                milkYield=milkYield,
-                EV_milk=EV_milk,
-                NEl=NEl)
-
     return NEl
 
 
@@ -154,22 +117,15 @@ _NEl_BY_GROUPING = {
 }
 
 
-def _calculate_NEl(cycle: dict, animal: dict, log_node: dict) -> float:
+def _calculate_NEl(cycle: dict, animal: dict) -> float:
     grouping = _get_grouping(animal)
-    return _NEl_BY_GROUPING.get(grouping, lambda *args: 0)(cycle, animal, log_node)
-
+    return _NEl_BY_GROUPING.get(grouping, lambda *args: 0)(cycle, animal)
 
-def _calculate_NEwork(cycle: dict, animal: dict, NEm: float, log_node: dict) -> float:
-    term = animal.get('term', {})
-    term_id = term.get('@id')
 
+def _calculate_NEwork(cycle: dict, animal: dict, NEm: float) -> float:
     hoursWorkedPerDay = get_node_property(animal, 'hoursWorkedPerDay').get('value', 0)
     NEwork = 0.1 * NEm * hoursWorkedPerDay
 
-    debugValues(log_node, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                hoursWorkedPerDay=hoursWorkedPerDay,
-                NEwork=NEwork)
-
     return NEwork
 
 
@@ -193,25 +149,16 @@ def _get_pregnancy_ratio(animal: dict) -> float:
     return _get_pregnancy_ratio_per_birth(animal, value) if ';' in value else safe_parse_float(value, 0)
 
 
-def _calculate_NEp(cycle: dict, animal: dict, NEm: float, log_node: dict) -> float:
-    term = animal.get('term', {})
-    term_id = term.get('@id')
-
+def _calculate_NEp(cycle: dict, animal: dict, NEm: float) -> float:
     ratioCPregnancyNetEnergyPregnancy = _get_pregnancy_ratio(animal)
     pregnancyRateTotal = get_node_property(animal, 'pregnancyRateTotal').get('value', 0)
     NEp = ratioCPregnancyNetEnergyPregnancy * pregnancyRateTotal/100 * NEm
 
-    debugValues(log_node, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                ratioCPregnancyNetEnergyPregnancy=ratioCPregnancyNetEnergyPregnancy,
-                pregnancyRateTotal=pregnancyRateTotal,
-                NEp=NEp)
-
     return NEp
 
 
-def _calculate_NEg_cattleAndBuffalo(cycle: dict, animal: dict, log_node: dict) -> float:
+def _calculate_NEg_cattleAndBuffalo(cycle: dict, animal: dict) -> float:
     term = animal.get('term', {})
-    term_id = term.get('@id')
 
     ratioCNetEnergyGrowthCattleBuffalo = safe_parse_float(
         get_lookup_value(term, 'ratioCNetEnergyGrowthCattleBuffaloIpcc2019', model=MODEL, model_key=MODEL_KEY), 0
@@ -228,22 +175,14 @@ def _calculate_NEg_cattleAndBuffalo(cycle: dict, animal: dict, log_node: dict) -
             ratioCNetEnergyGrowthCattleBuffalo * weightAtMaturity > 0
         ]) else 0
 
-    debugValues(log_node, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                ratioCNetEnergyGrowthCattleBuffalo=ratioCNetEnergyGrowthCattleBuffalo,
-                liveweightPerHead=liveweightPerHead,
-                weightAtMaturity=weightAtMaturity,
-                liveweightGain=liveweightGain,
-                NEg=NEg)
-
     return NEg
 
 
-def _calculate_NEg_sheepAndGoat(cycle: dict, animal: dict, log_node: dict) -> float:
+def _calculate_NEg_sheepAndGoat(cycle: dict, animal: dict) -> float:
     term = animal.get('term', {})
-    term_id = term.get('@id')
 
     MjKgABNetEnergyGrowthSheepGoats = get_lookup_value(
-        term, 'MjKgABNetEnergyGrowthSheepGoatsIpcc2019', model=MODEL, model_key=MODEL_KEY)
+        term, 'mjKgABNetEnergyGrowthSheepGoatsIpcc2019', model=MODEL, model_key=MODEL_KEY)
     MjKg_a = safe_parse_float(extract_grouped_data(MjKgABNetEnergyGrowthSheepGoats, 'a'), 0)
     MjKg_b = safe_parse_float(extract_grouped_data(MjKgABNetEnergyGrowthSheepGoats, 'b'), 0)
     BWi = get_node_property(animal, 'weightAtWeaning').get('value', 0)
@@ -253,13 +192,6 @@ def _calculate_NEg_sheepAndGoat(cycle: dict, animal: dict, log_node: dict) -> fl
     cycleDuration = cycle.get('cycleDuration', 365)
     NEg = (BWf - BWi) * (MjKg_a + 0.5 * MjKg_b * (BWi + BWf)) / 365 * animal_value * cycleDuration
 
-    debugValues(log_node, model=MODEL, term=term_id, model_key=MODEL_KEY,
-                MjKg_a=MjKg_a,
-                MjKg_b=MjKg_b,
-                BWi=BWi,
-                BWf=BWf,
-                NEg=NEg)
-
     return NEg
 
 
@@ -269,18 +201,19 @@ _NEg_BY_GROUPING = {
 }
 
 
-def _calculate_NEg(cycle: dict, animal: dict, log_node: dict) -> float:
+def _calculate_NEg(cycle: dict, animal: dict) -> float:
     grouping = _get_grouping(animal)
-    return _NEg_BY_GROUPING.get(grouping, lambda *args: 0)(cycle, animal, log_node)
+    return _NEg_BY_GROUPING.get(grouping, lambda *args: 0)(cycle, animal)
 
 
-def _pastureGrass_key_property_value(practice: dict, column: dict):
-    term_id = practice_input_id(practice)
-    term = download_hestia(term_id)
-    term_type = term.get('termType')
-    value = list_sum(practice.get('value', [0]))
-    lookup_value = node_property_lookup_value(MODEL, {'@id': term_id, 'termType': term_type}, column, default=0)
-    return (lookup_value, value)
+def _pastureGrass_key_property_value(column: dict, **log_args):
+    def get_value(practice: dict):
+        term_id = practice_input_id(practice)
+        term = download_hestia(term_id)
+        value = list_sum(practice.get('value', [0]))
+        lookup_value = node_property_lookup_value(MODEL, term, column, default=0, **log_args)
+        return (lookup_value, value)
+    return get_value
 
 
 def calculate_REM(energy: float = 0) -> float:
@@ -346,27 +279,48 @@ def calculate_NEfeed(node: dict) -> tuple:
     return (NEfeed_m, NEfeed_g)
 
 
-def get_animal_values(cycle: dict, animal: dict, system: dict, log_node: dict):
-    NEm = _calculate_NEm(cycle, animal, log_node=log_node)
-    NEa = _calculate_NEa(cycle, animal, system, NEm, log_node=log_node)
-    NEl = _calculate_NEl(cycle, animal, log_node=log_node)
-    NEwork = _calculate_NEwork(cycle, animal, NEm, log_node=log_node)
-    NEp = _calculate_NEp(cycle, animal, NEm, log_node=log_node)
-    NEg = _calculate_NEg(cycle, animal, log_node=log_node)
+def get_animal_values(cycle: dict, animal: dict, system: dict) -> dict:
+    NEm = _calculate_NEm(cycle, animal)
+    NEa = _calculate_NEa(cycle, animal, system, NEm)
+    NEl = _calculate_NEl(cycle, animal)
+    NEwork = _calculate_NEwork(cycle, animal, NEm)
+    NEp = _calculate_NEp(cycle, animal, NEm)
+    NEg = _calculate_NEg(cycle, animal)
+
+    return {
+        'NEm': NEm,
+        'NEa': NEa,
+        'NEl': NEl,
+        'NEwork': NEwork,
+        'NEp': NEp,
+        'NEg': NEg
+    }
+
+
+def _sum_values(values: list, key: str): return list_sum([v.get(key) for v in values])
+
+
+def calculate_GE(values: list, REM: float, REG: float, NEwool: float, NEm_feed: float, NEg_feed: float) -> float:
+    NEm = _sum_values(values, 'NEm')
+    NEa = _sum_values(values, 'NEa')
+    NEl = _sum_values(values, 'NEl')
+    NEwork = _sum_values(values, 'NEwork')
+    NEp = _sum_values(values, 'NEp')
+    NEg = _sum_values(values, 'NEg')
 
-    return (NEm, NEa, NEl, NEwork, NEp, NEg)
+    return ((NEm + NEa + NEl + NEwork + NEp - NEm_feed)/REM + (NEg + NEwool - NEg_feed)/REG) if all([REM, REG]) else 0
 
 
-def calculate_meanECHHV(practices: list) -> float:
-    values = list(map(lambda p: _pastureGrass_key_property_value(p, 'energyContentHigherHeatingValue'), practices))
+def calculate_meanECHHV(practices: list, **log_args) -> float:
+    values = list(map(_pastureGrass_key_property_value('energyContentHigherHeatingValue', **log_args), practices))
     total_weight = sum([weight/100 for _value, weight in values])
     return sum([
         (value * weight/100 if all([value, weight]) else 0) for value, weight in values
     ]) / total_weight if total_weight > 0 else 0
 
 
-def calculate_meanDE(practices: list) -> float:
-    values = list(map(lambda p: _pastureGrass_key_property_value(p, 'energyDigestibilityRuminants'), practices))
+def calculate_meanDE(practices: list, **log_args) -> float:
+    values = list(map(_pastureGrass_key_property_value('energyDigestibilityRuminants', **log_args), practices))
     total_weight = sum([weight/100 for _value, weight in values])
     meanDE = sum([
         (value * weight/100 if all([value, weight]) else 0) for value, weight in values