hestia-earth-models 0.59.2__py3-none-any.whl → 0.59.4__py3-none-any.whl

hestia_earth/models/ipcc2019/noxToAirInorganicFertiliser.py

@@ -0,0 +1,112 @@
+from hestia_earth.schema import EmissionMethodTier, EmissionStatsDefinition, TermTermType
+from hestia_earth.utils.model import filter_list_term_type, find_term_match
+from hestia_earth.utils.tools import list_sum
+
+from hestia_earth.models.log import logRequirements, logShouldRun, debugValues, log_as_table
+from hestia_earth.models.utils.blank_node import get_N_total
+from hestia_earth.models.utils.constant import Units, get_atomic_conversion
+from hestia_earth.models.utils.completeness import _is_term_type_complete
+from hestia_earth.models.utils.emission import _new_emission
+from hestia_earth.models.utils.cycle import get_inorganic_fertiliser_N_total
+from hestia_earth.models.utils.term import get_lookup_value
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "completeness.fertiliser": "True",
+        "inputs": [
+            {
+                "@type": "Input",
+                "value": "",
+                "term.termType": "inorganicFertiliser",
+                "optional": {
+                    "properties": [{"@type": "Property", "value": "", "term.@id": "nitrogenContent"}]
+                }
+            }
+        ]
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "sd": "",
+        "min": "",
+        "max": "",
+        "methodTier": "tier 1",
+        "statsDefinition": "modelled",
+        "methodModelDescription": ["Aggregated version", "Disaggragated version"]
+    }]
+}
+LOOKUPS = {
+    "inorganicFertiliser": ["IPCC_2019_FRACGASF_NOx-N", "IPCC_2019_FRACGASF_NOx-N-min", "IPCC_2019_FRACGASF_NOx-N-max"]
+}
+TERM_ID = 'noxToAirInorganicFertiliser'
+TIER = EmissionMethodTier.TIER_1.value
+TERM_TYPE = TermTermType.INORGANICFERTILISER
+UNSPECIFIED_TERM_ID = 'inorganicNitrogenFertiliserUnspecifiedKgN'
+
+
+def _emission(value: float, min: float, max: float, aggregated: bool = False):
+    emission = _new_emission(TERM_ID, MODEL)
+    emission['value'] = [value]
+    emission['min'] = [min]
+    emission['max'] = [max]
+    emission['methodTier'] = TIER
+    emission['statsDefinition'] = EmissionStatsDefinition.MODELLED.value
+    emission['methodModelDescription'] = 'Aggregated version' if aggregated else 'Disaggregated version'
+    return emission
+
+
+def _input_values(input: dict):
+    N_total = list_sum(get_N_total([input]))
+    lookups = LOOKUPS[TERM_TYPE.value]
+    return {
+        'id': input.get('term', {}).get('@id'),
+        'N': N_total,
+        'value': get_lookup_value(input.get('term', {}), lookups[0]),
+        'min': get_lookup_value(input.get('term', {}), lookups[1]),
+        'max': get_lookup_value(input.get('term', {}), lookups[2])
+    }
+
+
+def _filter_input_values(values: list, key: str): return [value for value in values if value.get(key)]
+
+
+def _run(cycle: dict):
+    inputs = filter_list_term_type(cycle.get('inputs', []), TERM_TYPE)
+    input_values = list(map(_input_values, inputs))
+
+    debugValues(cycle, model=MODEL, term=TERM_ID,
+                input_values=log_as_table(input_values))
+
+    value = list_sum([
+        v.get('N', 0) * v.get('value', 0) for v in _filter_input_values(input_values, 'value')
+    ]) * get_atomic_conversion(Units.KG_NOX, Units.TO_N)
+
+    min = list_sum([
+        v.get('N', 0) * v.get('min', 0) for v in _filter_input_values(input_values, 'min')
+    ]) * get_atomic_conversion(Units.KG_NOX, Units.TO_N)
+
+    max = list_sum([
+        v.get('N', 0) * v.get('max', 0) for v in _filter_input_values(input_values, 'max')
+    ]) * get_atomic_conversion(Units.KG_NOX, Units.TO_N)
+
+    aggregated = list_sum(find_term_match(inputs, UNSPECIFIED_TERM_ID).get('value', [0])) > 0
+
+    return [_emission(value, min, max, aggregated=aggregated)]
+
+
+def _should_run(cycle: dict):
+    N_inorganic_fertiliser = get_inorganic_fertiliser_N_total(cycle)
+    fertiliser_complete = _is_term_type_complete(cycle, 'fertiliser')
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    N_inorganic_fertiliser=N_inorganic_fertiliser,
+                    term_type_fertiliser_complete=fertiliser_complete)
+
+    should_run = all([N_inorganic_fertiliser is not None, fertiliser_complete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+    return should_run
+
+
+def run(cycle: dict): return _run(cycle) if _should_run(cycle) else []

hestia_earth/models/ipcc2019/noxToAirOrganicFertiliser.py

@@ -66,6 +66,9 @@ def _input_values(input: dict):
     }
 
 
+def _filter_input_values(values: list, key: str): return [value for value in values if value.get(key)]
+
+
 def _run(cycle: dict):
     inputs = filter_list_term_type(cycle.get('inputs', []), TermTermType.ORGANICFERTILISER)
     input_values = list(map(_input_values, inputs))
@@ -74,15 +77,15 @@ def _run(cycle: dict):
                 input_values=log_as_table(input_values))
 
     value = list_sum([
-        v.get('N', 0) * v.get('value', 0) for v in input_values
+        v.get('N', 0) * v.get('value', 0) for v in _filter_input_values(input_values, 'value')
     ]) * get_atomic_conversion(Units.KG_NOX, Units.TO_N)
 
     min = list_sum([
-        v.get('N', 0) * v.get('min', 0) for v in input_values
+        v.get('N', 0) * v.get('min', 0) for v in _filter_input_values(input_values, 'min')
     ]) * get_atomic_conversion(Units.KG_NOX, Units.TO_N)
 
     max = list_sum([
-        v.get('N', 0) * v.get('max', 0) for v in input_values
+        v.get('N', 0) * v.get('max', 0) for v in _filter_input_values(input_values, 'max')
     ]) * get_atomic_conversion(Units.KG_NOX, Units.TO_N)
 
     return [_emission(value, min, max)]

hestia_earth/models/ipcc2019/organicCarbonPerHa.py

@@ -18,15 +18,12 @@ from typing import (
 from hestia_earth.schema import (
     CycleFunctionalUnit,
     MeasurementMethodClassification,
-    SchemaType,
     SiteSiteType,
     TermTermType,
 )
-from hestia_earth.utils.api import find_related
 from hestia_earth.utils.model import find_term_match, filter_list_term_type
 from hestia_earth.utils.tools import flatten, list_sum, non_empty_list
 
-from hestia_earth.models.utils import _load_calculated_node
 from hestia_earth.models.log import log_as_table, logRequirements, logShouldRun
 from hestia_earth.models.utils.blank_node import (
     cumulative_nodes_match,
@@ -54,7 +51,7 @@ from hestia_earth.models.utils.term import (
     get_upland_rice_crop_terms,
     get_upland_rice_land_cover_terms
 )
-
+from hestia_earth.models.utils.site import related_cycles
 from .utils import check_consecutive
 from . import MODEL
 
@@ -107,7 +104,14 @@ REQUIREMENTS = {
                     "value": "",
                     "startDate": "",
                     "endDate": "",
-                    "term.@id": "organicFertiliserOrSoilCarbonIncreasingAmendmentUsed"
+                    "term.@id": "organicFertiliserUsed"
+                },
+                {
+                    "@type": "Management",
+                    "value": "",
+                    "startDate": "",
+                    "endDate": "",
+                    "term.@id": "amendmentIncreasingSoilCarbonUsed"
                 },
                 {"@type": "Management", "value": "", "startDate": "", "endDate": "", "term.@id": "shortBareFallow"}
             ]
@@ -238,7 +242,8 @@ IMPROVED_PASTURE_TERM_ID = "improvedPasture"
 SHORT_BARE_FALLOW_TERM_ID = "shortBareFallow"
 ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
 INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID = "inorganicNitrogenFertiliserUsed"
-ORGANIC_FERTILISER_USED_TERM_ID = "organicFertiliserOrSoilCarbonIncreasingAmendmentUsed"
+ORGANIC_FERTILISER_USED_TERM_ID = "organicFertiliserUsed"
+SOIL_AMENDMENT_USED_TERM_ID = "amendmentIncreasingSoilCarbonUsed"
 
 CLAY_CONTENT_MAX = 8
 SAND_CONTENT_MIN = 70
@@ -3301,7 +3306,7 @@ def _get_carbon_input_kwargs(
 
     has_organic_fertiliser_or_soil_amendment_used = any(
         get_node_value(node) for node in land_use_management_nodes
-        if node_term_match(node, ORGANIC_FERTILISER_USED_TERM_ID)
+        if node_term_match(node, [ORGANIC_FERTILISER_USED_TERM_ID, SOIL_AMENDMENT_USED_TERM_ID])
     )
 
     has_practice_increasing_c_input = cumulative_nodes_match(
@@ -3546,7 +3551,7 @@ def _should_run(site: dict) -> tuple[bool, dict]:
     site_type = site.get("siteType", "")
     management_nodes = site.get("management", [])
     measurement_nodes = site.get("measurements", [])
-    cycles = _calculated_cycles(site.get("@id"))
+    cycles = related_cycles(site.get("@id"))
 
     has_management = len(management_nodes) > 0
     has_measurements = len(measurement_nodes) > 0
@@ -3603,26 +3608,6 @@ def _should_run(site: dict) -> tuple[bool, dict]:
     return should_run_tier_1, should_run_tier_2, inventory, kwargs
 
 
-def _calculated_cycles(site_id: str):
-    """
-    Get the list of `Cycle`s related to the `Site`. Gets the `recalculated` data if available, else `original`.
-
-    Parameters
-    ----------
-    site_id : str
-        The `@id` of the `Site`.
-
-    Returns
-    -------
-    list[dict]
-        The related `Cycle`s as `dict`.
-    """
-    nodes = find_related(SchemaType.SITE, site_id, SchemaType.CYCLE)
-    return list(
-        map(lambda node: _load_calculated_node(node, SchemaType.CYCLE), nodes or [])
-    )
-
-
 def _should_run_tier_1(
     inventory: dict,
     *,

hestia_earth/models/mocking/search-results.json

@@ -918,11 +918,11 @@
       },
       {
         "@type": "Term",
-        "@id": "residueIncorporated"
+        "@id": "residueIncorporatedMoreThan30DaysBeforeCultivation"
       },
       {
         "@type": "Term",
-        "@id": "residueIncorporatedMoreThan30DaysBeforeCultivation"
+        "@id": "residueIncorporated"
       },
       {
         "@type": "Term",
@@ -1299,7 +1299,7 @@
         "@type": "Term",
         "name": "Generic crop, seed",
         "@id": "genericCropSeed",
-        "_score": 23.9092
+        "_score": 23.910892
       }
     ]
   },
@@ -1405,11 +1405,11 @@
     "results": [
       {
         "@type": "Term",
-        "@id": "waterMarine"
+        "@id": "waterDrainageCanal"
       },
       {
         "@type": "Term",
-        "@id": "waterDrainageCanal"
+        "@id": "waterMarine"
       },
       {
         "@type": "Term",
@@ -1740,15 +1740,15 @@
       },
       {
         "@type": "Term",
-        "@id": "fullInversionTillage"
+        "@id": "fullTillage"
       },
       {
         "@type": "Term",
-        "@id": "fullTillage"
+        "@id": "minimumTillage"
       },
       {
         "@type": "Term",
-        "@id": "minimumTillage"
+        "@id": "fullInversionTillage"
       },
       {
         "@type": "Term",

hestia_earth/models/site/management.py

@@ -9,9 +9,9 @@ from hestia_earth.utils.model import filter_list_term_type, linked_node
 from hestia_earth.utils.tools import flatten
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_blank_nodes_id
-from hestia_earth.models.utils.site import related_cycles
 from hestia_earth.models.utils.term import get_lookup_value
 from hestia_earth.models.utils.blank_node import get_node_value
+from hestia_earth.models.utils.site import related_cycles
 from . import MODEL
 
 REQUIREMENTS = {
@@ -96,7 +96,7 @@ def _get_items_with_relevant_term_type(cycles: List[dict], item_name: str, relev
     )
 
 
-def should_run(site: dict):
+def _should_run(site: dict):
     # Only get related cycles once.
     cycles = related_cycles(site.get("@id"))
 
@@ -158,11 +158,11 @@ def should_run(site: dict):
         practice_ids=log_blank_nodes_id(practices)
     )
 
-    _should_run = any(products_crop_forage + products_land_cover + practices)
+    should_run = any(products_crop_forage + products_land_cover + practices)
     logShouldRun(site, MODEL, None, should_run=_should_run, model_key=MODEL_KEY)
-    return _should_run, products_crop_forage + products_land_cover, practices
+    return should_run, products_crop_forage + products_land_cover, practices
 
 
 def run(site: dict):
-    _should_run, products, practices = should_run(site)
-    return list(map(management, products + practices)) if _should_run else []
+    should_run, products, practices = _should_run(site)
+    return list(map(management, products + practices)) if should_run else []

hestia_earth/models/site/pre_checks/cache_geospatialDatabase.py

@@ -4,10 +4,10 @@ Pre Checks Cache Geospatial Database
 This model caches results from Geospatial Database.
 """
 from functools import reduce
-from hestia_earth.utils.tools import flatten
+from hestia_earth.utils.tools import flatten, non_empty_list
 
 from hestia_earth.models.log import debugValues
-from hestia_earth.models.utils import CACHE_KEY, cached_value
+from hestia_earth.models.utils import CACHE_KEY, cached_value, first_day_of_month, last_day_of_month
 from hestia_earth.models.utils.site import CACHE_YEARS_KEY
 from hestia_earth.models.geospatialDatabase.utils import (
     MAX_AREA_SIZE, CACHE_VALUE, CACHE_AREA_SIZE,
@@ -34,7 +34,12 @@ def cache_site_results(results: list, collections: list, area_size: int = None):
         collection = collections[index]
         name = collection.get('name')
         value = results[index]
-        data = (group.get(name, {}) | {collection.get('year'): value}) if 'year' in collection else value
+        cache_sub_key = '-'.join(non_empty_list([
+            collection.get('year'),
+            collection.get('start_date'),
+            collection.get('end_date')
+        ]))
+        data = (group.get(name, {}) | {cache_sub_key: value} if cache_sub_key else value)
         return group | {name: data}
 
     return reduce(_combine_result, range(0, len(results)), {}) | (
@@ -42,13 +47,28 @@ def cache_site_results(results: list, collections: list, area_size: int = None):
     )
 
 
+def _extend_collection_by_month(year: int):
+    return [{
+        'start_date': first_day_of_month(year, month).strftime('%Y-%m-%d'),
+        'end_date': last_day_of_month(year, month).strftime('%Y-%m-%d')
+    } for month in range(1, 13)]
+
+
 def _extend_collection(name: str, collection: dict, years: list = []):
     data = collection | {'name': name, 'collection': _collection_name(collection.get('collection'))}
+
+    year_params = [{'year': str(year)} for year in years]
+    # fetch from first year to last
+    month_years = range(years[0], years[-1] + 1) if len(years) > 1 else years
+    month_params = flatten(map(_extend_collection_by_month, month_years))
+
     return [
-        data | {
-            'year': str(year)
-        } for year in years
-    ] if 'reducer_annual' in collection and 'reducer_period' not in collection else [data]
+        (data | params) for params in year_params
+    ] if name.endswith('Annual') else [
+        (data | params) for params in month_params
+    ] if name.endswith('Monthly') else [
+        data
+    ]
 
 
 def _extend_collections(values: list, years: list = []):

hestia_earth/models/site/utils.py

@@ -1,10 +1,9 @@
 from functools import reduce
-import datetime
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.tools import non_empty_list
 from hestia_earth.utils.date import DAY
 
-from hestia_earth.models.utils import _omit
+from hestia_earth.models.utils import _omit, first_day_of_month, last_day_of_month
 from hestia_earth.models.utils.measurement import _new_measurement, measurement_value, has_all_months
 
 
@@ -68,10 +67,7 @@ def _group_by_month(term_id: str, dates: list, values: list):
 
     def map_to_month(data: list, year: int, month: int):
         # make sure we got all the necessary days
-        first_day_of_month = datetime.date(year, month, 1)
-        last_day_of_month = datetime.date(year + int(month / 12), (month % 12) + 1, 1) - datetime.timedelta(days=1)
-
-        difference = last_day_of_month - first_day_of_month
+        difference = last_day_of_month(year, month) - first_day_of_month(year, month)
         days_in_month = round(difference.days + difference.seconds / DAY, 1) + 1
 
         return measurement_value({

hestia_earth/models/utils/__init__.py

@@ -1,5 +1,6 @@
 from os.path import dirname, abspath
 import sys
+import datetime
 from functools import reduce
 import operator
 from typing import Any
@@ -126,3 +127,11 @@ def _get_by_key(x, y):
 
 
 def get_dict_key(value: dict, key: str): return reduce(lambda x, y: _get_by_key(x, y), key.split('.'), value)
+
+
+def first_day_of_month(year: int, month: int):
+    return datetime.date(int(year), int(month), 1)
+
+
+def last_day_of_month(year: int, month: int):
+    return datetime.date(int(year) + int(int(month) / 12), (int(month) % 12) + 1, 1) - datetime.timedelta(days=1)

hestia_earth/models/utils/site.py

@@ -1,10 +1,10 @@
 from hestia_earth.schema import SchemaType, SiteSiteType, TermTermType
-from hestia_earth.utils.api import download_hestia, find_related
+from hestia_earth.utils.api import find_related
 from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name
 from hestia_earth.utils.tools import non_empty_list, safe_parse_date
 
 from hestia_earth.models.log import debugMissingLookup
-from . import cached_value
+from . import cached_value, _load_calculated_node
 
 CACHE_YEARS_KEY = 'years'
 WATER_TYPES = [
@@ -47,8 +47,7 @@ def is_site(site: dict): return site.get('@type', site.get('type')) == SchemaTyp
 def related_cycles(site_id: str):
     """
     Get the list of `Cycle` related to the `Site`.
-
-    In Hestia, a `Cycle` must have a link to a `Site`, therefore a `Site` can be related to many `Cycle`s.
+    Gets the `recalculated` data if available, else `original`.
 
     Parameters
     ----------
@@ -61,7 +60,7 @@ def related_cycles(site_id: str):
         The related `Cycle`s as `dict`.
     """
     nodes = find_related(SchemaType.SITE, site_id, SchemaType.CYCLE)
-    return non_empty_list(map(lambda node: download_hestia(node.get('@id'), SchemaType.CYCLE), nodes or []))
+    return non_empty_list(map(lambda node: _load_calculated_node(node, SchemaType.CYCLE), nodes or []))
 
 
 def _cycle_end_year(cycle: dict):
@@ -75,6 +74,10 @@ def related_years(site: dict):
     )
 
 
+def related_months(site: dict):
+    return cached_value(site)
+
+
 def valid_site_type(site: dict, site_types=[SiteSiteType.CROPLAND.value, SiteSiteType.PERMANENT_PASTURE.value]):
     """
     Check if the site `siteType` is allowed.

hestia_earth/models/version.py

@@ -1 +1 @@
-VERSION = '0.59.2'
+VERSION = '0.59.4'

{hestia_earth_models-0.59.2.dist-info → hestia_earth_models-0.59.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hestia-earth-models
-Version: 0.59.2
+Version: 0.59.4
 Summary: Hestia's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
 Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
 Author: Hestia Team
@@ -18,7 +18,7 @@ Requires-Dist: CurrencyConverter ==0.16.8
 Requires-Dist: haversine >=2.7.0
 Requires-Dist: pydash
 Provides-Extra: spatial
-Requires-Dist: hestia-earth.earth-engine >=0.4.2 ; extra == 'spatial'
+Requires-Dist: hestia-earth.earth-engine >=0.4.7 ; extra == 'spatial'
 
 # Hestia Engine Models