hestia-earth-models 0.59.3__py3-none-any.whl → 0.59.5__py3-none-any.whl

hestia_earth/models/schmidt2007/n2OToAirWasteTreatmentDirect.py

@@ -0,0 +1,58 @@
+from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.emission import _new_emission
+from .utils import get_waste_values
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "product": [
+                {"@type": "Product", "value": "", "term.termType": "waste"}
+            ],
+            "completeness.waste": ""
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "methodTier": "tier 1"
+    }]
+}
+LOOKUPS = {
+    "waste": "n2oEfSchmidt2007"
+}
+TERM_ID = 'n2OToAirWasteTreatmentDirect'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def _emission(value: float):
+    emission = _new_emission(TERM_ID, MODEL)
+    emission['value'] = [value]
+    emission['methodTier'] = TIER
+    return emission
+
+
+def _run(waste_values: list):
+    value = list_sum(waste_values)
+    return [_emission(value)]
+
+
+def _should_run(cycle: dict):
+    waste_values = get_waste_values(TERM_ID, cycle, LOOKUPS['waste'])
+    has_waste = len(waste_values) > 0
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    has_waste=has_waste)
+
+    should_run = any([has_waste])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
+    return should_run, waste_values
+
+
+def run(cycle: dict):
+    should_run, waste_values = _should_run(cycle)
+    return _run(waste_values) if should_run else []

hestia_earth/models/schmidt2007/nh3ToAirWasteTreatment.py

@@ -0,0 +1,58 @@
+from hestia_earth.schema import EmissionMethodTier
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.emission import _new_emission
+from .utils import get_waste_values
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "product": [
+                {"@type": "Product", "value": "", "term.termType": "waste"}
+            ],
+            "completeness.waste": ""
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "methodTier": "tier 1"
+    }]
+}
+LOOKUPS = {
+    "waste": "nh3EfSchmidt2007"
+}
+TERM_ID = 'nh3ToAirWasteTreatment'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def _emission(value: float):
+    emission = _new_emission(TERM_ID, MODEL)
+    emission['value'] = [value]
+    emission['methodTier'] = TIER
+    return emission
+
+
+def _run(waste_values: list):
+    value = list_sum(waste_values)
+    return [_emission(value)]
+
+
+def _should_run(cycle: dict):
+    waste_values = get_waste_values(TERM_ID, cycle, LOOKUPS['waste'])
+    has_waste = len(waste_values) > 0
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    has_waste=has_waste)
+
+    should_run = any([has_waste])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run, methodTier=TIER)
+    return should_run, waste_values
+
+
+def run(cycle: dict):
+    should_run, waste_values = _should_run(cycle)
+    return _run(waste_values) if should_run else []

hestia_earth/models/site/management.py

@@ -1,17 +1,17 @@
 """
 Management node with data gap-filled data from cycles.
 """
-from typing import List
+from typing import List, Any
 from functools import reduce
 from hestia_earth.schema import SchemaType, TermTermType
 from hestia_earth.utils.api import download_hestia
 from hestia_earth.utils.model import filter_list_term_type, linked_node
-from hestia_earth.utils.tools import flatten
+from hestia_earth.utils.tools import flatten, safe_parse_float
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_blank_nodes_id
-from hestia_earth.models.utils.site import related_cycles
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.blank_node import get_node_value
+from hestia_earth.models.utils.site import related_cycles
 from . import MODEL
 
 REQUIREMENTS = {
@@ -37,6 +37,16 @@ REQUIREMENTS = {
                         ],
                         "value": ""
                     }
+                ],
+                "inputs": [
+                    {
+                        "@type": "Input",
+                        "term.termType": [
+                            "inorganicFertiliser",
+                            "organicFertiliser",
+                            "soilAmendment"
+                        ]
+                    }
                 ]
             }]
         }
@@ -53,9 +63,45 @@ RETURNS = {
         "startDate": ""
     }]
 }
-
+LOOKUPS = {
+    "inorganicFertiliser": "nitrogenContent",
+    "organicFertiliser": "ANIMAL_MANURE",
+    "soilAmendment": "PRACTICE_INCREASING_C_INPUT"
+}
 MODEL_KEY = 'management'
 LAND_COVER_KEY = 'landCoverId'
+ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
+INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID = "inorganicNitrogenFertiliserUsed"
+ORGANIC_FERTILISER_USED_TERM_ID = "organicFertiliserUsed"
+AMENDMENT_INCREASING_C_USED_TERM_ID = "amendmentIncreasingSoilCarbonUsed"
+INPUT_RULES = {
+    TermTermType.INORGANICFERTILISER.value: (
+        (
+            TermTermType.INORGANICFERTILISER.value,  # Lookup column
+            lambda x: safe_parse_float(x) > 0,  # Condition
+            INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID  # New term.
+        ),
+    ),
+    TermTermType.SOILAMENDMENT.value: (
+        (
+            TermTermType.SOILAMENDMENT.value,
+            lambda x: x is True,
+            AMENDMENT_INCREASING_C_USED_TERM_ID
+        ),
+    ),
+    TermTermType.ORGANICFERTILISER.value: (
+        (
+            TermTermType.SOILAMENDMENT.value,
+            lambda x: x is True,
+            ORGANIC_FERTILISER_USED_TERM_ID
+        ),
+        (
+            TermTermType.ORGANICFERTILISER.value,
+            lambda x: x is True,
+            ANIMAL_MANURE_USED_TERM_ID
+        )
+    )
+}
 
 
 def management(data: dict):
@@ -74,7 +120,9 @@ def _include_start_end(cycle: dict, values: list):
     return [(_include(cycle, ['startDate', 'endDate']) | v) for v in values]
 
 
-def _copy_item_if_exists(source: dict, keys: List[str] = [], dest: dict = {}) -> dict:
+def _copy_item_if_exists(source: dict, keys: List[str] = None, dest: dict = None) -> dict:
+    keys = keys or []
+    dest = dest or {}
     return reduce(lambda p, c: p | ({c: source[c]} if c in source else {}), keys, dest)
 
 
@@ -96,7 +144,51 @@ def _get_items_with_relevant_term_type(cycles: List[dict], item_name: str, relev
     )
 
 
-def should_run(site: dict):
+def _get_lookup_with_debug(term: dict, column: str) -> Any:
+    get_lookup_value(term, column, model_key=MODEL_KEY, land_cover_key=LAND_COVER_KEY)
+
+
+def _data_from_input(cycle: dict, term_id: str) -> dict:
+    return {
+        "term": {
+            "@type": "Term",
+            "@id": term_id,
+            "termType": "landUseManagement"
+        },
+        "value": True,
+        "startDate": cycle["startDate"],
+        "endDate": cycle["endDate"]
+    }
+
+
+def _process_rule(cycle, term, term_type) -> List:
+    relevant_terms = []
+    for column, condition, new_term in INPUT_RULES[term_type]:
+        lookup_result = _get_lookup_with_debug(term, LOOKUPS[column])
+
+        if condition(lookup_result):
+            relevant_terms.append(_data_from_input(cycle=cycle, term_id=new_term))
+
+    return relevant_terms
+
+
+def _get_relevant_inputs(cycles: List[dict]) -> List:
+    relevant_inputs = []
+    for cycle in [c for c in cycles if "inputs" in c]:
+        for i in cycle["inputs"]:
+            if i.get("term", {}).get("termType", "") in INPUT_RULES:
+                relevant_inputs.extend(
+                    _process_rule(
+                        cycle=cycle,
+                        term=i.get("term", {}),
+                        term_type=i.get("term", {}).get("termType", "")
+                    )
+                )
+
+    return relevant_inputs
+
+
+def _should_run(site: dict):
     # Only get related cycles once.
     cycles = related_cycles(site.get("@id"))
 
@@ -148,6 +240,7 @@ def should_run(site: dict):
         )
     ]
 
+    relevant_inputs = _get_relevant_inputs(cycles)
     logRequirements(
         site,
         model=MODEL,
@@ -155,14 +248,14 @@ def should_run(site: dict):
         model_key=MODEL_KEY,
         products_crop_forage_ids=log_blank_nodes_id(products_crop_forage),
         products_land_cover_ids=log_blank_nodes_id(products_land_cover),
-        practice_ids=log_blank_nodes_id(practices)
+        practice_ids=log_blank_nodes_id(practices),
+        inputs=log_blank_nodes_id(relevant_inputs)
     )
-
-    _should_run = any(products_crop_forage + products_land_cover + practices)
-    logShouldRun(site, MODEL, None, should_run=_should_run, model_key=MODEL_KEY)
-    return _should_run, products_crop_forage + products_land_cover, practices
+    should_run = any(products_crop_forage + products_land_cover + practices + relevant_inputs)
+    logShouldRun(site, MODEL, None, should_run=should_run, model_key=MODEL_KEY)
+    return should_run, products_crop_forage + products_land_cover, practices, relevant_inputs
 
 
 def run(site: dict):
-    _should_run, products, practices = should_run(site)
-    return list(map(management, products + practices)) if _should_run else []
+    should_run, products, practices, inputs = _should_run(site)
+    return list(map(management, products + practices + inputs)) if should_run else []

hestia_earth/models/site/pre_checks/cache_geospatialDatabase.py

@@ -4,10 +4,10 @@ Pre Checks Cache Geospatial Database
 This model caches results from Geospatial Database.
 """
 from functools import reduce
-from hestia_earth.utils.tools import flatten
+from hestia_earth.utils.tools import flatten, non_empty_list
 
 from hestia_earth.models.log import debugValues
-from hestia_earth.models.utils import CACHE_KEY, cached_value
+from hestia_earth.models.utils import CACHE_KEY, cached_value, first_day_of_month, last_day_of_month
 from hestia_earth.models.utils.site import CACHE_YEARS_KEY
 from hestia_earth.models.geospatialDatabase.utils import (
     MAX_AREA_SIZE, CACHE_VALUE, CACHE_AREA_SIZE,
@@ -34,7 +34,12 @@ def cache_site_results(results: list, collections: list, area_size: int = None):
         collection = collections[index]
         name = collection.get('name')
         value = results[index]
-        data = (group.get(name, {}) | {collection.get('year'): value}) if 'year' in collection else value
+        cache_sub_key = '-'.join(non_empty_list([
+            collection.get('year'),
+            collection.get('start_date'),
+            collection.get('end_date')
+        ]))
+        data = (group.get(name, {}) | {cache_sub_key: value} if cache_sub_key else value)
         return group | {name: data}
 
     return reduce(_combine_result, range(0, len(results)), {}) | (
@@ -42,13 +47,28 @@ def cache_site_results(results: list, collections: list, area_size: int = None):
     )
 
 
+def _extend_collection_by_month(year: int):
+    return [{
+        'start_date': first_day_of_month(year, month).strftime('%Y-%m-%d'),
+        'end_date': last_day_of_month(year, month).strftime('%Y-%m-%d')
+    } for month in range(1, 13)]
+
+
 def _extend_collection(name: str, collection: dict, years: list = []):
     data = collection | {'name': name, 'collection': _collection_name(collection.get('collection'))}
+
+    year_params = [{'year': str(year)} for year in years]
+    # fetch from first year to last
+    month_years = range(years[0], years[-1] + 1) if len(years) > 1 else years
+    month_params = flatten(map(_extend_collection_by_month, month_years))
+
     return [
-        data | {
-            'year': str(year)
-        } for year in years
-    ] if 'reducer_annual' in collection and 'reducer_period' not in collection else [data]
+        (data | params) for params in year_params
+    ] if name.endswith('Annual') else [
+        (data | params) for params in month_params
+    ] if name.endswith('Monthly') else [
+        data
+    ]
 
 
 def _extend_collections(values: list, years: list = []):

hestia_earth/models/site/soilMeasurement.py

@@ -16,7 +16,7 @@ from . import MODEL
 REQUIREMENTS = {
     "Site": {
         "measurements": [
-            {"@type": "Measurement", "depthUpper": "", "depthLower": ""}
+            {"@type": "Measurement", "depthUpper": "", "depthLower": "", "value": ""}
         ]
     }
 }
@@ -159,21 +159,21 @@ def _get_needed_depths(site: dict) -> list:
 
 def _should_run(site: dict, model_key: str):
     # we only work with measurements with depths
-    measurements = [
-        m for m in site.get("measurements", [])
-        if get_lookup_value(m.get("term", {}), LOOKUPS["measurement"][0], model=MODEL, model_key=model_key)
-    ]
+    measurements = [m for m in site.get("measurements", []) if all([
+        get_lookup_value(m.get("term", {}), LOOKUPS["measurement"][0], model=MODEL, model_key=model_key),
+        m.get('value', [])
+    ])]
 
     measurements_with_depths = [m for m in measurements if all([
-        "depthUpper" in m.keys(),
-        "depthLower" in m.keys(),
+        "depthUpper" in m,
+        "depthLower" in m,
         (int(m.get("depthUpper", 0)), int(m.get("depthLower", 0))) not in STANDARD_DEPTHS
     ])]
     has_measurements_with_depths = len(measurements_with_depths) > 0
 
     measurements_missing_depth_recommended = [m for m in measurements if all([
-        "depthUpper" not in m.keys(),
-        "depthLower" not in m.keys(),
+        "depthUpper" not in m,
+        "depthLower" not in m,
         not get_lookup_value(m.get("term", {}), LOOKUPS["measurement"][1], model=MODEL, model_key=model_key)
     ])]
 

hestia_earth/models/site/utils.py

@@ -1,10 +1,9 @@
 from functools import reduce
-import datetime
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.tools import non_empty_list
 from hestia_earth.utils.date import DAY
 
-from hestia_earth.models.utils import _omit
+from hestia_earth.models.utils import _omit, first_day_of_month, last_day_of_month
 from hestia_earth.models.utils.measurement import _new_measurement, measurement_value, has_all_months
 
 
@@ -68,10 +67,7 @@ def _group_by_month(term_id: str, dates: list, values: list):
 
     def map_to_month(data: list, year: int, month: int):
         # make sure we got all the necessary days
-        first_day_of_month = datetime.date(year, month, 1)
-        last_day_of_month = datetime.date(year + int(month / 12), (month % 12) + 1, 1) - datetime.timedelta(days=1)
-
-        difference = last_day_of_month - first_day_of_month
+        difference = last_day_of_month(year, month) - first_day_of_month(year, month)
         days_in_month = round(difference.days + difference.seconds / DAY, 1) + 1
 
         return measurement_value({

hestia_earth/models/utils/__init__.py

@@ -1,5 +1,6 @@
 from os.path import dirname, abspath
 import sys
+import datetime
 from functools import reduce
 import operator
 from typing import Any
@@ -126,3 +127,11 @@ def _get_by_key(x, y):
 
 
 def get_dict_key(value: dict, key: str): return reduce(lambda x, y: _get_by_key(x, y), key.split('.'), value)
+
+
+def first_day_of_month(year: int, month: int):
+    return datetime.date(int(year), int(month), 1)
+
+
+def last_day_of_month(year: int, month: int):
+    return datetime.date(int(year) + int(int(month) / 12), (int(month) % 12) + 1, 1) - datetime.timedelta(days=1)

hestia_earth/models/utils/blank_node.py

@@ -37,9 +37,9 @@ from .term import get_lookup_value
 
 
 def group_by_keys(group_keys: list = ['term']):
-    def run(group: dict, input: dict):
-        group_key = '-'.join(non_empty_list(map(lambda v: input.get(v, {}).get('@id'), group_keys)))
-        group[group_key] = group.get(group_key, []) + [input]
+    def run(group: dict, node: dict):
+        group_key = '-'.join(non_empty_list(map(lambda v: node.get(v, {}).get('@id'), group_keys)))
+        group[group_key] = group.get(group_key, []) + [node]
         return group
     return run
 

hestia_earth/models/utils/site.py

@@ -1,10 +1,10 @@
 from hestia_earth.schema import SchemaType, SiteSiteType, TermTermType
-from hestia_earth.utils.api import download_hestia, find_related
+from hestia_earth.utils.api import find_related
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
 from hestia_earth.utils.tools import non_empty_list, safe_parse_date
 
 from hestia_earth.models.log import debugMissingLookup
-from . import cached_value
+from . import cached_value, _load_calculated_node
 
 CACHE_YEARS_KEY = 'years'
 WATER_TYPES = [
@@ -47,8 +47,7 @@ def is_site(site: dict): return site.get('@type', site.get('type')) == SchemaTyp
 def related_cycles(site_id: str):
     """
     Get the list of `Cycle` related to the `Site`.
-
-    In Hestia, a `Cycle` must have a link to a `Site`, therefore a `Site` can be related to many `Cycle`s.
+    Gets the `recalculated` data if available, else `original`.
 
     Parameters
     ----------
@@ -61,7 +60,7 @@ def related_cycles(site_id: str):
         The related `Cycle`s as `dict`.
     """
     nodes = find_related(SchemaType.SITE, site_id, SchemaType.CYCLE)
-    return non_empty_list(map(lambda node: download_hestia(node.get('@id'), SchemaType.CYCLE), nodes or []))
+    return non_empty_list(map(lambda node: _load_calculated_node(node, SchemaType.CYCLE), nodes or []))
 
 
 def _cycle_end_year(cycle: dict):
@@ -75,6 +74,10 @@ def related_years(site: dict):
     )
 
 
+def related_months(site: dict):
+    return cached_value(site)
+
+
 def valid_site_type(site: dict, site_types=[SiteSiteType.CROPLAND.value, SiteSiteType.PERMANENT_PASTURE.value]):
     """
     Check if the site `siteType` is allowed.

hestia_earth/models/utils/term.py

@@ -603,26 +603,3 @@ def get_pasture_system_terms():
         'name': 'pasture'
     }, limit=LIMIT)
     return list(map(lambda n: n["@id"], terms))
-
-
-def get_long_fallow_land_cover_terms():
-    """
-    Find all `landCover` terms with the name `long fallow`:
-    https://hestia.earth/glossary?termType=landCover&query=long%fallow
-
-    Returns
-    -------
-    list
-        List of matching term `@id` as `str`.
-    """
-    terms = search({
-        "bool": {
-            "must": [
-                {"match": {"@type": SchemaType.TERM.value}},
-                {"match": {"termType.keyword": TermTermType.LANDCOVER.value}},
-                {"match_phrase_prefix": {"name": "long"}},
-                {"match": {"name": "fallow"}}
-            ],
-        }
-    }, limit=LIMIT)
-    return list(map(lambda n: n["@id"], terms))

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.59.3'
+VERSION = '0.59.5'

{hestia_earth_models-0.59.3.dist-info → hestia_earth_models-0.59.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.59.3
+Version: 0.59.5
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team
@@ -18,7 +18,7 @@ Requires-Dist: CurrencyConverter ==0.16.8
 Requires-Dist: haversine >=2.7.0
 Requires-Dist: pydash
 Provides-Extra: spatial
-Requires-Dist: hestia-earth.earth-engine >=0.4.2 ; extra == 'spatial'
+Requires-Dist: hestia-earth.earth-engine >=0.4.7 ; extra == 'spatial'
 
 # Hestia Engine Models