hestia-earth-models 0.72.2__tar.gz → 0.73.1__tar.gz

hestia_earth_models-0.73.1/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.1
+Name: hestia_earth_models
+Version: 0.73.1
+Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
+Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
+Author: HESTIA Team
+Author-email: guillaumeroyer.mail@gmail.com
+License: MIT
+Platform: UNKNOWN
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Programming Language :: Python :: 3.6
+Description-Content-Type: text/markdown
+Provides-Extra: spatial
+License-File: LICENSE
+
+# HESTIA Engine Models
+
+[![Pipeline Status](https://gitlab.com/hestia-earth/hestia-engine-models/badges/master/pipeline.svg)](https://gitlab.com/hestia-earth/hestia-engine-models/commits/master)
+[![Coverage Report](https://gitlab.com/hestia-earth/hestia-engine-models/badges/master/coverage.svg)](https://gitlab.com/hestia-earth/hestia-engine-models/commits/master)
+
+HESTIA's set of models for running calculations or retrieving data using external datasets and internal lookups.
+
+## Documentation
+
+Documentation for every model can be found in the [HESTIA API Documentation](https://hestia.earth/docs/#hestia-calculation-models).
+
+## Install
+
+1. Install python `3` (we recommend using python `3.6` minimum)
+2. Install the module:
+```bash
+pip install hestia_earth.models
+```
+3. Set the following environment variables:
+```
+API_URL=https://api.hestia.earth
+WEB_URL=https://hestia.earth
+```
+
+### Usage
+
+```python
+from hestia_earth.models.pooreNemecek2018 import run
+
+cycle_data = {"@type": "Cycle", ...}
+# cycle is a JSONLD node Cycle
+result = run('no3ToGroundwaterSoilFlux', cycle_data)
+print(result)
+```
+
+This will display only the result of the `no3ToGroundwaterSoilFlux` model (Emission).
+
+Additionally, to reduce the number of queries to the HESTIA API and run the models faster, prefetching can be enabled:
+```python
+from hestia_earth.models.preload_requests import enable_preload
+
+enable_preload()
+```
+
+#### Using the orchestrator
+
+The models come with an "orchestrator", which allows you to run a pre-configured set of models instead of a single one.
+
+The configuration for each Node (Cycle, Site or ImpactAssessment) can be found in the [config](./config) folder.
+
+Usage:
+```python
+from hestia_earth.orchestrator import run
+from hestia_earth.models.config import load_config
+
+cycle_data = {"@type": "Cycle", ...}
+result = run(cycle, load_config(cycle))
+print(result)
+```
+
+This will display the Cycle recalculated with all HESTIA default models running.
+
+#### Using Spatial Models
+
+We have models that can gap-fill geographical information on a `Site`. If you want to use these models:
+1. Install the library: `pip install hestia_earth.earth_engine`
+2. Follow the [Getting Started instructions](https://gitlab.com/hestia-earth/hestia-earth-engine#getting-started).
+
+#### Using the ecoinventV3 model
+
+ecoinvent is a consistent, transparent, and well validated life cycle inventory database.
+We use ecoinvent data to ascertain the environmental impacts of activities that occur outside of our system boundary, for example data on the environmental impacts of extracting oil and producing diesel, or the impacts of manufacturing plastics.
+
+The `ecoinventV3` model requires a valid [license](https://ecoinvent.org/offerings/licences/) to run. We are currently working on a way to enable users of this code with a valid ecoinvent licence to run these models themselves, but for now, these models are only available on the public platform.
+
+

hestia_earth_models-0.73.1/hestia_earth/models/akagiEtAl2011/__init__.py

@@ -0,0 +1,13 @@
+from os.path import dirname, abspath
+import sys
+from importlib import import_module
+
+from hestia_earth.models.utils.blank_node import run_if_required
+
+CURRENT_DIR = dirname(abspath(__file__)) + '/'
+sys.path.append(CURRENT_DIR)
+MODEL = 'akagiEtAl2011'
+PKG = '.'.join(['hestia_earth', 'models', MODEL])
+
+
+def run(model: str, data): return run_if_required(MODEL, model, data, import_module(f".{model}", package=PKG))

hestia_earth_models-0.73.1/hestia_earth/models/akagiEtAl2011/ch4ToAirCropResidueBurning.py

@@ -0,0 +1,32 @@
+from hestia_earth.schema import EmissionMethodTier
+
+from .utils import run_emission
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "products": [{
+                "@type": "Product",
+                "term.@id": ["aboveGroundCropResidueBurnt", "discardedCropBurnt"],
+                "value": ""
+            }],
+            "completeness.cropResidue": "True"
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "sd": "",
+        "methodTier": "tier 1",
+        "statsDefinition": "modelled"
+    }]
+}
+LOOKUPS = {
+    "emission": ["akagiEtAl2011CropResidueBurningFactor", "akagiEtAl2011CropResidueBurningFactor-sd"]
+}
+TERM_ID = 'ch4ToAirCropResidueBurning'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def run(cycle: dict): return run_emission(TERM_ID, cycle)

hestia_earth_models-0.73.1/hestia_earth/models/akagiEtAl2011/nh3ToAirCropResidueBurning.py

@@ -0,0 +1,32 @@
+from hestia_earth.schema import EmissionMethodTier
+
+from .utils import run_emission
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "products": [{
+                "@type": "Product",
+                "term.@id": ["aboveGroundCropResidueBurnt", "discardedCropBurnt"],
+                "value": ""
+            }],
+            "completeness.cropResidue": "True"
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "sd": "",
+        "methodTier": "tier 1",
+        "statsDefinition": "modelled"
+    }]
+}
+LOOKUPS = {
+    "emission": ["akagiEtAl2011CropResidueBurningFactor", "akagiEtAl2011CropResidueBurningFactor-sd"]
+}
+TERM_ID = 'nh3ToAirCropResidueBurning'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def run(cycle: dict): return run_emission(TERM_ID, cycle)

hestia_earth_models-0.73.1/hestia_earth/models/akagiEtAl2011/noxToAirCropResidueBurning.py

@@ -0,0 +1,32 @@
+from hestia_earth.schema import EmissionMethodTier
+
+from .utils import run_emission
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "products": [{
+                "@type": "Product",
+                "term.@id": ["aboveGroundCropResidueBurnt", "discardedCropBurnt"],
+                "value": ""
+            }],
+            "completeness.cropResidue": "True"
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "sd": "",
+        "methodTier": "tier 1",
+        "statsDefinition": "modelled"
+    }]
+}
+LOOKUPS = {
+    "emission": ["akagiEtAl2011CropResidueBurningFactor", "akagiEtAl2011CropResidueBurningFactor-sd"]
+}
+TERM_ID = 'noxToAirCropResidueBurning'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def run(cycle: dict): return run_emission(TERM_ID, cycle)

hestia_earth_models-0.73.1/hestia_earth/models/akagiEtAl2011/pm25ToAirCropResidueBurning.py

@@ -0,0 +1,32 @@
+from hestia_earth.schema import EmissionMethodTier
+
+from .utils import run_emission
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "products": [{
+                "@type": "Product",
+                "term.@id": ["aboveGroundCropResidueBurnt", "discardedCropBurnt"],
+                "value": ""
+            }],
+            "completeness.cropResidue": "True"
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "sd": "",
+        "methodTier": "tier 1",
+        "statsDefinition": "modelled"
+    }]
+}
+LOOKUPS = {
+    "emission": ["akagiEtAl2011CropResidueBurningFactor", "akagiEtAl2011CropResidueBurningFactor-sd"]
+}
+TERM_ID = 'pm25ToAirCropResidueBurning'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def run(cycle: dict): return run_emission(TERM_ID, cycle)

hestia_earth_models-0.73.1/hestia_earth/models/akagiEtAl2011/so2ToAirCropResidueBurning.py

@@ -0,0 +1,32 @@
+from hestia_earth.schema import EmissionMethodTier
+
+from .utils import run_emission
+
+REQUIREMENTS = {
+    "Cycle": {
+        "or": {
+            "products": [{
+                "@type": "Product",
+                "term.@id": ["aboveGroundCropResidueBurnt", "discardedCropBurnt"],
+                "value": ""
+            }],
+            "completeness.cropResidue": "True"
+        }
+    }
+}
+RETURNS = {
+    "Emission": [{
+        "value": "",
+        "sd": "",
+        "methodTier": "tier 1",
+        "statsDefinition": "modelled"
+    }]
+}
+LOOKUPS = {
+    "emission": ["akagiEtAl2011CropResidueBurningFactor", "akagiEtAl2011CropResidueBurningFactor-sd"]
+}
+TERM_ID = 'so2ToAirCropResidueBurning'
+TIER = EmissionMethodTier.TIER_1.value
+
+
+def run(cycle: dict): return run_emission(TERM_ID, cycle)

hestia_earth_models-0.73.1/hestia_earth/models/akagiEtAl2011/utils.py

@@ -0,0 +1,45 @@
+from hestia_earth.schema import EmissionMethodTier, EmissionStatsDefinition, TermTermType
+
+from hestia_earth.models.log import logRequirements, logShouldRun
+from hestia_earth.models.utils.emission import _new_emission
+from hestia_earth.models.utils.term import get_lookup_value
+from hestia_earth.models.utils.cropResidue import get_crop_residue_burnt_value
+from . import MODEL
+
+TIER = EmissionMethodTier.TIER_1.value
+LOOKUP_NAME = 'akagiEtAl2011CropResidueBurningFactor'
+
+
+def _emission(term_id: str, value: float, sd: float = None):
+    emission = _new_emission(term_id, MODEL)
+    emission['value'] = [value]
+    emission['methodTier'] = TIER
+    if sd is not None:
+        emission['sd'] = [sd]
+        emission['statsDefinition'] = EmissionStatsDefinition.MODELLED.value
+    return emission
+
+
+def _run(term_id: str, product_value: list):
+    value = sum(product_value)
+    term = {'termType': TermTermType.EMISSION.value, '@id': term_id}
+    factor = get_lookup_value(term, LOOKUP_NAME)
+    factor_sd = get_lookup_value(term, LOOKUP_NAME + '-sd')
+    return [_emission(term_id, value * factor, value * factor_sd)]
+
+
+def _should_run(term_id: str, cycle: dict):
+    crop_residue_burnt_value = get_crop_residue_burnt_value(cycle)
+    has_crop_residue_burnt = len(crop_residue_burnt_value) > 0
+
+    logRequirements(cycle, model=MODEL, term=term_id,
+                    has_crop_residue_burnt=has_crop_residue_burnt)
+
+    should_run = all([has_crop_residue_burnt])
+    logShouldRun(cycle, MODEL, term_id, should_run, methodTier=TIER)
+    return should_run, crop_residue_burnt_value
+
+
+def run_emission(term_id: str, cycle: dict):
+    should_run, crop_residue_burnt_value = _should_run(term_id, cycle)
+    return _run(term_id, crop_residue_burnt_value) if should_run else []

hestia_earth_models-0.73.1/hestia_earth/models/aware/scarcityWeightedWaterUse.py

@@ -0,0 +1,105 @@
+from hestia_earth.schema import SiteSiteType
+from hestia_earth.utils.model import find_term_match
+from hestia_earth.utils.lookup import download_lookup, _get_single_table_value, column_name
+from hestia_earth.utils.tools import safe_parse_float
+
+from hestia_earth.models.log import logRequirements, debugMissingLookup, logShouldRun
+from hestia_earth.models.utils import sum_values, multiply_values
+from hestia_earth.models.utils.indicator import _new_indicator
+from hestia_earth.models.utils.impact_assessment import (
+    convert_value_from_cycle, get_product, get_site, get_region_id
+)
+from hestia_earth.models.utils.input import sum_input_impacts
+from hestia_earth.models.utils.lookup import get_region_lookup_value
+from . import MODEL
+
+REQUIREMENTS = {
+    "ImpactAssessment": {
+        "site": {
+            "@type": "Site",
+            "or": {
+                "awareWaterBasinId": "",
+                "country": {"@type": "Term", "termType": "region"}
+            }
+        },
+        "optional": {
+            "emissionsResourceUse": [{
+                "@type": "Indicator",
+                "term.@id": "freshwaterWithdrawalsDuringCycle",
+                "value": ""
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Indicator": [{
+        "value": ""
+    }]
+}
+LOOKUPS = {
+    "@doc": "Different lookup files are used depending on the situation",
+    "awareWaterBasinId": "",
+    "region-aware-factors": ""
+}
+TERM_ID = 'scarcityWeightedWaterUse'
+AWARE_KEY = 'awareWaterBasinId'
+IRRIGATED_SITE_TYPES = [
+    SiteSiteType.CROPLAND.value,
+    SiteSiteType.GLASS_OR_HIGH_ACCESSIBLE_COVER.value,
+    SiteSiteType.PERMANENT_PASTURE.value
+]
+
+
+def _indicator(value: float):
+    indicator = _new_indicator(TERM_ID, MODEL)
+    indicator['value'] = value
+    return indicator
+
+
+def _get_factor_from_basinId(site: dict, aware_id: str):
+    lookup_col = 'YR_IRRI' if site.get('siteType') in IRRIGATED_SITE_TYPES else 'YR_NONIRRI'
+    value = _get_single_table_value(
+        download_lookup(f"{AWARE_KEY}.csv"), column_name(AWARE_KEY), int(aware_id), column_name(lookup_col)
+    )
+    debugMissingLookup(f"{AWARE_KEY}.csv", AWARE_KEY, aware_id, lookup_col, value, model=MODEL, term=TERM_ID)
+    return safe_parse_float(value, default=None)
+
+
+def _get_factor_from_region(impact_assessment: dict, fresh_water: dict, site: dict):
+    region_id = get_region_id(impact_assessment, fresh_water)
+    site_type = site.get('siteType')
+    lookup_name = 'region-aware-factors.csv'
+    lookup_suffix = 'unspecified' if not site_type else ('irri' if site_type in IRRIGATED_SITE_TYPES else 'non_irri')
+    column = f"Agg_CF_{lookup_suffix}"
+    value = get_region_lookup_value(lookup_name, region_id, column, model=MODEL, term=TERM_ID)
+    return safe_parse_float(value, default=None)
+
+
+def run(impact_assessment: dict):
+    cycle = impact_assessment.get('cycle', {})
+    product = get_product(impact_assessment)
+    fresh_water = find_term_match(impact_assessment.get('emissionsResourceUse', []), 'freshwaterWithdrawalsDuringCycle')
+    site = get_site(impact_assessment)
+    aware_id = site.get(AWARE_KEY)
+    factor = (
+        _get_factor_from_basinId(site, aware_id) if aware_id else None
+    ) or _get_factor_from_region(impact_assessment, fresh_water, site)
+    inputs_value = convert_value_from_cycle(
+        impact_assessment, product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
+    )
+
+    value = sum_values([
+        multiply_values([fresh_water.get('value'), factor]),
+        inputs_value
+    ])
+
+    logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
+                    fresh_water=fresh_water.get('value'),
+                    aware_id=aware_id,
+                    factor=factor,
+                    inputs_value=inputs_value)
+
+    should_run = all([value is not None])
+    logShouldRun(impact_assessment, MODEL, TERM_ID, should_run)
+
+    return [_indicator(value)] if should_run else []

hestia_earth_models-0.73.1/hestia_earth/models/cache_sites.py

@@ -0,0 +1,178 @@
+from functools import reduce
+from enum import Enum
+from pydash.objects import merge
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from .log import logger
+from .utils import CACHE_KEY, cached_value
+from .utils.site import CACHE_YEARS_KEY
+from .utils.term import download_term
+from .site.pre_checks.cache_geospatialDatabase import (
+    list_vectors, list_rasters, cache_site_results, _should_run
+)
+from .geospatialDatabase.utils import (
+    CACHE_VALUE as CACHE_GEOSPATIAL_KEY, CACHE_AREA_SIZE,
+    has_boundary, has_coordinates,
+    _run_query, _site_gadm_id
+)
+
+
+class ParamType(Enum):
+    BOUNDARIES = 'boundaries'
+    COORDINATES = 'coordinates'
+    GADM_IDS = 'gadm-ids'
+
+
+_VALUE_AS_PARAM = {
+    ParamType.COORDINATES: lambda data: {'latitude': data[0].get('latitude'), 'longitude': data[0].get('longitude')},
+    ParamType.BOUNDARIES: lambda data: data[0].get('boundary'),
+    ParamType.GADM_IDS: lambda data: data[0].get('region', data[0].get('country', {})).get('@id')
+}
+
+
+def _value_as_param(param_type: ParamType = None): return _VALUE_AS_PARAM.get(param_type, lambda *args: None)
+
+
+def _cache_results(results: list, collections: list, index: int):
+    start = index * len(collections)
+    end = start + len(collections)
+    return cache_site_results(results[start:end], collections)
+
+
+def _run_values(
+    sites: list,
+    param_type: ParamType = None,
+    rasters: list = [],
+    vectors: list = [],
+    years: list = None
+):
+    get_param_values = _value_as_param(param_type)
+    param_values = non_empty_list(map(get_param_values, sites))
+    # unique list
+    param_values = list(set(param_values)) if param_type == ParamType.GADM_IDS else list({
+        str(v): v for v in param_values
+    }.values())
+
+    raster_results = _run_query({
+        'ee_type': 'raster',
+        'collections': rasters,
+        param_type.value: param_values
+    }) if rasters else []
+
+    vector_results = _run_query({
+        'ee_type': 'vector',
+        'collections': vectors,
+        param_type.value: param_values
+    }) if vectors else []
+
+    def _process_site(site_values: tuple):
+        site, area_size = site_values
+
+        # get real index in values to handle duplicates
+        param_value = get_param_values([site])
+        index = param_values.index(param_value) if param_value is not None else None
+
+        cached_data = ({
+            **_cache_results(raster_results, rasters, index),
+            **_cache_results(vector_results, vectors, index)
+        } if index is not None else {}) | ({CACHE_AREA_SIZE: area_size} if area_size is not None else {})
+        cached_data = merge(cached_value(site, CACHE_GEOSPATIAL_KEY, {}), cached_data)
+        site_cache = merge(
+            site.get(CACHE_KEY, {}),
+            {CACHE_GEOSPATIAL_KEY: cached_data},
+            ({CACHE_YEARS_KEY: sorted(list(set(cached_value(site, CACHE_YEARS_KEY, []) + years)))} if years else {})
+        )
+        return merge(site, {CACHE_KEY: site_cache})
+
+    return reduce(lambda prev, curr: prev + [_process_site(curr)], sites, [])
+
+
+def _should_preload_region_area_size(site: dict): return not has_coordinates(site) and not has_boundary(site)
+
+
+def _preload_regions_area_size(sites: dict):
+    region_ids = set(map(_site_gadm_id, filter(_should_preload_region_area_size, sites)))
+    return {term_id: download_term(term_id, TermTermType.REGION).get('area') for term_id in region_ids}
+
+
+def _group_sites(sites: dict, check_has_cache: bool = True):
+    # preload area size for all regions
+    regions_area_size = _preload_regions_area_size(sites)
+
+    def get_region_area_size(site: dict):
+        return regions_area_size.get(_site_gadm_id(site)) if _should_preload_region_area_size(site) else None
+
+    sites = [
+        (n, ) + (_should_run(n, area_size=get_region_area_size(n), check_has_cache=check_has_cache)) for n in sites
+    ]
+    # restrict sites based on should_cache result
+    sites_run = [(site, area_size) for site, should_cache, area_size in sites if should_cache]
+    # will only cache area and years
+    sites_no_run = [(site, area_size) for site, should_cache, area_size in sites if not should_cache]
+
+    with_coordinates = [
+        (site, area_size) for site, area_size in sites_run if has_coordinates(site)
+    ]
+    with_boundaries = [
+        (site, area_size) for site, area_size in sites_run if not has_coordinates(site) and has_boundary(site)
+    ]
+    with_gadm_ids = [
+        (site, area_size) for site, area_size in sites_run if not has_coordinates(site) and not has_boundary(site)
+    ]
+
+    return {
+        ParamType.COORDINATES: with_coordinates,
+        ParamType.BOUNDARIES: with_boundaries,
+        ParamType.GADM_IDS: with_gadm_ids
+    }, sites_no_run
+
+
+def _run(sites: list, years: list = [], years_only: bool = False):
+    rasters = list_rasters(years=years, years_only=years_only)
+    vectors = [] if years_only else list_vectors(sites)
+    filtered_data, sites_no_run = _group_sites(sites, not years_only)
+    return flatten(
+        [
+            _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
+            for param_type in [e for e in ParamType] if len(filtered_data.get(param_type)) > 0
+        ] + (
+            _run_values(sites_no_run, years=years) if sites_no_run else []
+        )
+    )
+
+
+def _run_by_years(sites: list, years: list, batch_size: int):
+    batches = range(0, len(years), batch_size)
+
+    for batch_index in batches:
+        logger.info(f"Processing sites in batch {batch_index + 1} of {len(batches)}...")
+        sub_years = years[batch_index:batch_index + batch_size]
+        sites = _run(sites, sub_years, years_only=True)
+
+    return sites
+
+
+def run(sites: list, years: list = None):
+    """
+    Run all queries at once for the list of provided Sites.
+    Note: Earth Engine needs to be initiliased with `init_gee()` before running this function.
+
+    Parameters
+    ----------
+    sites : list[dict]
+        List of Site as dict.
+    years : list[int]
+        List of related years to fetch annual data.
+    """
+    sites = _run(sites)
+
+    # avoid memory limit errors by running only a few years at a time
+    unique_years = sorted(list(set(years)))
+
+    try:
+        return _run_by_years(sites, unique_years, batch_size=5)
+    except Exception as e:
+        if 'exceeded' in str(e):
+            return _run_by_years(sites, unique_years, batch_size=1)
+        raise e