hestia-earth-models 0.59.4__py3-none-any.whl → 0.59.6__py3-none-any.whl

hestia_earth/models/schmidt2007/h2SToAirWasteTreatment.py

@@ -0,0 +1,58 @@
+from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.emission import _new_emission
+from .utils import get_waste_values
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "product": [
+                {"@type": "Product", "value": "", "term.termType": "waste"}
+            ],
+            "completeness.waste": ""
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "methodTier": "tier 1"
+    }]
+}
+LOOKUPS = {
+    "waste": "h2SEfSchmidt2007"
+}
+TERM_ID = 'h2SToAirWasteTreatment'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def _emission(value: float):
+    emission = _new_emission(TERM_ID, MODEL)
+    emission['value'] = [value]
+    emission['methodTier'] = TIER
+    return emission
+
+
+def _run(waste_values: list):
+    value = list_sum(waste_values)
+    return [_emission(value)]
+
+
+def _should_run(cycle: dict):
+    waste_values = get_waste_values(TERM_ID, cycle, LOOKUPS['waste'])
+    has_waste = len(waste_values) > 0
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    has_waste=has_waste)
+
+    should_run = any([has_waste])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
+    return should_run, waste_values
+
+
+def run(cycle: dict):
+    should_run, waste_values = _should_run(cycle)
+    return _run(waste_values) if should_run else []

hestia_earth/models/schmidt2007/n2OToAirWasteTreatmentDirect.py

@@ -0,0 +1,58 @@
+from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.emission import _new_emission
+from .utils import get_waste_values
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "product": [
+                {"@type": "Product", "value": "", "term.termType": "waste"}
+            ],
+            "completeness.waste": ""
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "methodTier": "tier 1"
+    }]
+}
+LOOKUPS = {
+    "waste": "n2oEfSchmidt2007"
+}
+TERM_ID = 'n2OToAirWasteTreatmentDirect'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def _emission(value: float):
+    emission = _new_emission(TERM_ID, MODEL)
+    emission['value'] = [value]
+    emission['methodTier'] = TIER
+    return emission
+
+
+def _run(waste_values: list):
+    value = list_sum(waste_values)
+    return [_emission(value)]
+
+
+def _should_run(cycle: dict):
+    waste_values = get_waste_values(TERM_ID, cycle, LOOKUPS['waste'])
+    has_waste = len(waste_values) > 0
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    has_waste=has_waste)
+
+    should_run = any([has_waste])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
+    return should_run, waste_values
+
+
+def run(cycle: dict):
+    should_run, waste_values = _should_run(cycle)
+    return _run(waste_values) if should_run else []

hestia_earth/models/schmidt2007/nh3ToAirWasteTreatment.py

@@ -0,0 +1,58 @@
+from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.emission import _new_emission
+from .utils import get_waste_values
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "product": [
+                {"@type": "Product", "value": "", "term.termType": "waste"}
+            ],
+            "completeness.waste": ""
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "methodTier": "tier 1"
+    }]
+}
+LOOKUPS = {
+    "waste": "nh3EfSchmidt2007"
+}
+TERM_ID = 'nh3ToAirWasteTreatment'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def _emission(value: float):
+    emission = _new_emission(TERM_ID, MODEL)
+    emission['value'] = [value]
+    emission['methodTier'] = TIER
+    return emission
+
+
+def _run(waste_values: list):
+    value = list_sum(waste_values)
+    return [_emission(value)]
+
+
+def _should_run(cycle: dict):
+    waste_values = get_waste_values(TERM_ID, cycle, LOOKUPS['waste'])
+    has_waste = len(waste_values) > 0
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    has_waste=has_waste)
+
+    should_run = any([has_waste])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
+    return should_run, waste_values
+
+
+def run(cycle: dict):
+    should_run, waste_values = _should_run(cycle)
+    return _run(waste_values) if should_run else []

hestia_earth/models/site/management.py

@@ -1,12 +1,12 @@
 """
 Management node with data gap-filled data from cycles.
 """
-from typing import List
+from typing import List, Any
 from functools import reduce
 from hestia_earth.schema import SchemaType, TermTermType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import filter_list_term_type, linked_node
-from hestia_earth.utils.tools import flatten
+from hestia_earth.utils.tools import flatten, safe_parse_float
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_blank_nodes_id
 from hestia_earth.models.utils.term import get_lookup_value
@@ -37,6 +37,16 @@ REQUIREMENTS = {
                         ],
                         "value": ""
                     }
+                ],
+                "inputs": [
+                    {
+                        "@type": "Input",
+                        "term.termType": [
+                            "inorganicFertiliser",
+                            "organicFertiliser",
+                            "soilAmendment"
+                        ]
+                    }
                 ]
             }]
         }
@@ -53,9 +63,47 @@ RETURNS = {
         "startDate": ""
     }]
 }
-
+LOOKUPS = {
+    "crop": ["landCoverTermId"],
+    "forage": ["landCoverTermId"],
+    "inorganicFertiliser": "nitrogenContent",
+    "organicFertiliser": "ANIMAL_MANURE",
+    "soilAmendment": "PRACTICE_INCREASING_C_INPUT"
+}
 MODEL_KEY = 'management'
-LAND_COVER_KEY = 'landCoverId'
+LAND_COVER_KEY = LOOKUPS['crop'][0]
+ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
+INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID = "inorganicNitrogenFertiliserUsed"
+ORGANIC_FERTILISER_USED_TERM_ID = "organicFertiliserUsed"
+AMENDMENT_INCREASING_C_USED_TERM_ID = "amendmentIncreasingSoilCarbonUsed"
+INPUT_RULES = {
+    TermTermType.INORGANICFERTILISER.value: (
+        (
+            TermTermType.INORGANICFERTILISER.value,  # Lookup column
+            lambda x: safe_parse_float(x) > 0,  # Condition
+            INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID  # New term.
+        ),
+    ),
+    TermTermType.SOILAMENDMENT.value: (
+        (
+            TermTermType.SOILAMENDMENT.value,
+            lambda x: x is True,
+            AMENDMENT_INCREASING_C_USED_TERM_ID
+        ),
+    ),
+    TermTermType.ORGANICFERTILISER.value: (
+        (
+            TermTermType.SOILAMENDMENT.value,
+            lambda x: x is True,
+            ORGANIC_FERTILISER_USED_TERM_ID
+        ),
+        (
+            TermTermType.ORGANICFERTILISER.value,
+            lambda x: x is True,
+            ANIMAL_MANURE_USED_TERM_ID
+        )
+    )
+}
 
 
 def management(data: dict):
@@ -74,7 +122,9 @@ def _include_start_end(cycle: dict, values: list):
     return [(_include(cycle, ['startDate', 'endDate']) | v) for v in values]
 
 
-def _copy_item_if_exists(source: dict, keys: List[str] = [], dest: dict = {}) -> dict:
+def _copy_item_if_exists(source: dict, keys: List[str] = None, dest: dict = None) -> dict:
+    keys = keys or []
+    dest = dest or {}
     return reduce(lambda p, c: p | ({c: source[c]} if c in source else {}), keys, dest)
 
 
@@ -96,6 +146,50 @@ def _get_items_with_relevant_term_type(cycles: List[dict], item_name: str, relev
     )
 
 
+def _get_lookup_with_debug(term: dict, column: str) -> Any:
+    get_lookup_value(term, column, model_key=MODEL_KEY, land_cover_key=LAND_COVER_KEY)
+
+
+def _data_from_input(cycle: dict, term_id: str) -> dict:
+    return {
+        "term": {
+            "@type": "Term",
+            "@id": term_id,
+            "termType": "landUseManagement"
+        },
+        "value": True,
+        "startDate": cycle["startDate"],
+        "endDate": cycle["endDate"]
+    }
+
+
+def _process_rule(cycle, term, term_type) -> List:
+    relevant_terms = []
+    for column, condition, new_term in INPUT_RULES[term_type]:
+        lookup_result = _get_lookup_with_debug(term, LOOKUPS[column])
+
+        if condition(lookup_result):
+            relevant_terms.append(_data_from_input(cycle=cycle, term_id=new_term))
+
+    return relevant_terms
+
+
+def _get_relevant_inputs(cycles: List[dict]) -> List:
+    relevant_inputs = []
+    for cycle in [c for c in cycles if "inputs" in c]:
+        for i in cycle["inputs"]:
+            if i.get("term", {}).get("termType", "") in INPUT_RULES:
+                relevant_inputs.extend(
+                    _process_rule(
+                        cycle=cycle,
+                        term=i.get("term", {}),
+                        term_type=i.get("term", {}).get("termType", "")
+                    )
+                )
+
+    return relevant_inputs
+
+
 def _should_run(site: dict):
     # Only get related cycles once.
     cycles = related_cycles(site.get("@id"))
@@ -148,6 +242,7 @@ def _should_run(site: dict):
         )
     ]
 
+    relevant_inputs = _get_relevant_inputs(cycles)
     logRequirements(
         site,
         model=MODEL,
@@ -155,14 +250,14 @@ def _should_run(site: dict):
         model_key=MODEL_KEY,
         products_crop_forage_ids=log_blank_nodes_id(products_crop_forage),
         products_land_cover_ids=log_blank_nodes_id(products_land_cover),
-        practice_ids=log_blank_nodes_id(practices)
+        practice_ids=log_blank_nodes_id(practices),
+        inputs=log_blank_nodes_id(relevant_inputs)
     )
-
-    should_run = any(products_crop_forage + products_land_cover + practices)
-    logShouldRun(site, MODEL, None, should_run=_should_run, model_key=MODEL_KEY)
-    return should_run, products_crop_forage + products_land_cover, practices
+    should_run = any(products_crop_forage + products_land_cover + practices + relevant_inputs)
+    logShouldRun(site, MODEL, None, should_run=should_run, model_key=MODEL_KEY)
+    return should_run, products_crop_forage + products_land_cover, practices, relevant_inputs
 
 
 def run(site: dict):
-    should_run, products, practices = _should_run(site)
-    return list(map(management, products + practices)) if should_run else []
+    should_run, products, practices, inputs = _should_run(site)
+    return list(map(management, products + practices + inputs)) if should_run else []

hestia_earth/models/site/soilMeasurement.py

@@ -16,7 +16,7 @@ from . import MODEL
 REQUIREMENTS = {
     "Site": {
         "measurements": [
-            {"@type": "Measurement", "depthUpper": "", "depthLower": ""}
+            {"@type": "Measurement", "depthUpper": "", "depthLower": "", "value": ""}
         ]
     }
 }
@@ -159,21 +159,21 @@ def _get_needed_depths(site: dict) -> list:
 
 def _should_run(site: dict, model_key: str):
     # we only work with measurements with depths
-    measurements = [
-        m for m in site.get("measurements", [])
-        if get_lookup_value(m.get("term", {}), LOOKUPS["measurement"][0], model=MODEL, model_key=model_key)
-    ]
+    measurements = [m for m in site.get("measurements", []) if all([
+        get_lookup_value(m.get("term", {}), LOOKUPS["measurement"][0], model=MODEL, model_key=model_key),
+        m.get('value', [])
+    ])]
 
     measurements_with_depths = [m for m in measurements if all([
-        "depthUpper" in m.keys(),
-        "depthLower" in m.keys(),
+        "depthUpper" in m,
+        "depthLower" in m,
         (int(m.get("depthUpper", 0)), int(m.get("depthLower", 0))) not in STANDARD_DEPTHS
     ])]
     has_measurements_with_depths = len(measurements_with_depths) > 0
 
     measurements_missing_depth_recommended = [m for m in measurements if all([
-        "depthUpper" not in m.keys(),
-        "depthLower" not in m.keys(),
+        "depthUpper" not in m,
+        "depthLower" not in m,
         not get_lookup_value(m.get("term", {}), LOOKUPS["measurement"][1], model=MODEL, model_key=model_key)
     ])]
 

hestia_earth/models/utils/__init__.py

@@ -134,4 +134,7 @@ def first_day_of_month(year: int, month: int):
 
 
 def last_day_of_month(year: int, month: int):
-    return datetime.date(int(year) + int(int(month) / 12), (int(month) % 12) + 1, 1) - datetime.timedelta(days=1)
+    # handle special case month 12
+    return datetime.date(int(year), 12, 31) if month == 12 else (
+        datetime.date(int(year) + int(int(month) / 12), (int(month) % 12) + 1, 1) - datetime.timedelta(days=1)
+    )

hestia_earth/models/utils/animalProduct.py

@@ -7,10 +7,12 @@ FAO_EQUIVALENT_LOOKUP_COLUMN = 'animalProductGroupingFAOEquivalent'
 
 
 def get_animalProduct_lookup_value(model: str, term_id: str, column: str):
-    return get_lookup_value({'@id': term_id, 'termType': TermTermType.ANIMALPRODUCT.value}, column,
-                            model=model, term=term_id)
+    return get_lookup_value(
+        {'@id': term_id, 'termType': TermTermType.ANIMALPRODUCT.value}, column, model=model, term=term_id
+    ) if term_id else None
 
 
 def get_animalProduct_grouping_fao(model: str, term: dict):
-    fao_product_id = get_animalProduct_lookup_value(model, term.get('@id'), FAO_EQUIVALENT_LOOKUP_COLUMN)
-    return get_animalProduct_lookup_value(model, fao_product_id, FAO_LOOKUP_COLUMN)
+    term_id = term.get('@id')
+    fao_product_id = get_animalProduct_lookup_value(model, term_id, FAO_EQUIVALENT_LOOKUP_COLUMN)
+    return get_animalProduct_lookup_value(model, fao_product_id or term_id, FAO_LOOKUP_COLUMN)

hestia_earth/models/utils/blank_node.py

@@ -37,9 +37,9 @@ from .term import get_lookup_value
 
 
 def group_by_keys(group_keys: list = ['term']):
-    def run(group: dict, input: dict):
-        group_key = '-'.join(non_empty_list(map(lambda v: input.get(v, {}).get('@id'), group_keys)))
-        group[group_key] = group.get(group_key, []) + [input]
+    def run(group: dict, node: dict):
+        group_key = '-'.join(non_empty_list(map(lambda v: node.get(v, {}).get('@id'), group_keys)))
+        group[group_key] = group.get(group_key, []) + [node]
         return group
     return run
 
@@ -1161,9 +1161,10 @@ def group_nodes_by_year_and_month(
         for year in range(range_start, range_end):
             for month in range(1, 13):
 
+                datestr_incomplete = f"{year}-{month:02}"
                 group_datetime_range = DatetimeRange(
-                    start=safe_parse_date(_gapfill_datestr(f"{year}-{month:02}", DatestrGapfillMode.START)),
-                    end=safe_parse_date(_gapfill_datestr(f"{year}-{month}", DatestrGapfillMode.END))
+                    start=safe_parse_date(_gapfill_datestr(datestr_incomplete, DatestrGapfillMode.START)),
+                    end=safe_parse_date(_gapfill_datestr(datestr_incomplete, DatestrGapfillMode.END))
                 )
 
                 is_final_month = _datetime_within_range(node_datetime_range.end, group_datetime_range)

hestia_earth/models/utils/product.py

@@ -188,6 +188,12 @@ PRODUCT_UNITS_CONVERSIONS = {
         ]
     },
     Units.KG_LIVEWEIGHT.value: {
+        Units.NUMBER.value: [
+            ('liveweightPerHead', True)
+        ],
+        Units.HEAD.value: [
+            ('liveweightPerHead', True)
+        ],
         Units.KG_LIVEWEIGHT.value: [],
         Units.KG_COLD_CARCASS_WEIGHT.value: [
             ('processingConversionLiveweightToColdCarcassWeight', True)
@@ -224,7 +230,9 @@ PRODUCT_UNITS_CONVERSIONS = {
     },
     Units.KG_COLD_DRESSED_CARCASS_WEIGHT.value: {
         Units.KG_LIVEWEIGHT.value: [
-            ('processingConversionLiveweightToColdDressedCarcassWeight', False)
+            ('processingConversionLiveweightToColdDressedCarcassWeight', False),
+            # fallback when cold dressed carcass weight is not provided
+            ('processingConversionLiveweightToColdCarcassWeight', False)
         ],
         Units.KG_COLD_DRESSED_CARCASS_WEIGHT.value: [],
         Units.KG_COLD_CARCASS_WEIGHT.value: [],

hestia_earth/models/utils/term.py

@@ -603,26 +603,3 @@ def get_pasture_system_terms():
         'name': 'pasture'
     }, limit=LIMIT)
     return list(map(lambda n: n["@id"], terms))
-
-
-def get_long_fallow_land_cover_terms():
-    """
-    Find all `landCover` terms with the name `long fallow`:
-    https://hestia.earth/glossary?termType=landCover&query=long%fallow
-
-    Returns
-    -------
-    list
-        List of matching term `@id` as `str`.
-    """
-    terms = search({
-        "bool": {
-            "must": [
-                {"match": {"@type": SchemaType.TERM.value}},
-                {"match": {"termType.keyword": TermTermType.LANDCOVER.value}},
-                {"match_phrase_prefix": {"name": "long"}},
-                {"match": {"name": "fallow"}}
-            ],
-        }
-    }, limit=LIMIT)
-    return list(map(lambda n: n["@id"], terms))

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.59.4'
+VERSION = '0.59.6'

{hestia_earth_models-0.59.4.dist-info → hestia_earth_models-0.59.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.59.4
+Version: 0.59.6
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team