hestia-earth-models 0.68.0__tar.gz → 0.69.0__tar.gz

hestia_earth_models-0.69.0/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.1
+Name: hestia_earth_models
+Version: 0.69.0
+Summary: HESTIA's set of modules for filling gaps in the activity data using external datasets (e.g. populating soil properties with a geospatial dataset using provided coordinates) and internal lookups (e.g. populating machinery use from fuel use). Includes rules for when gaps should be filled versus not (e.g. never gap fill yield, gap fill crop residue if yield provided etc.).
+Home-page: https://gitlab.com/hestia-earth/hestia-engine-models
+Author: HESTIA Team
+Author-email: guillaumeroyer.mail@gmail.com
+License: MIT
+Platform: UNKNOWN
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Programming Language :: Python :: 3.6
+Description-Content-Type: text/markdown
+Provides-Extra: spatial
+License-File: LICENSE
+
+# HESTIA Engine Models
+
+[![Pipeline Status](https://gitlab.com/hestia-earth/hestia-engine-models/badges/master/pipeline.svg)](https://gitlab.com/hestia-earth/hestia-engine-models/commits/master)
+[![Coverage Report](https://gitlab.com/hestia-earth/hestia-engine-models/badges/master/coverage.svg)](https://gitlab.com/hestia-earth/hestia-engine-models/commits/master)
+
+HESTIA's set of models for running calculations or retrieving data using external datasets and internal lookups.
+
+## Documentation
+
+Documentation for every model can be found in the [HESTIA API Documentation](https://hestia.earth/docs/#hestia-calculation-models).
+
+## Install
+
+1. Install python `3` (we recommend using python `3.6` minimum)
+2. Install the module:
+```bash
+pip install hestia_earth.models
+```
+3. Set the following environment variables:
+```
+API_URL=https://api.hestia.earth
+WEB_URL=https://hestia.earth
+```
+
+### Usage
+
+```python
+from hestia_earth.models.pooreNemecek2018 import run
+
+cycle_data = {"@type": "Cycle", ...}
+# cycle is a JSONLD node Cycle
+result = run('no3ToGroundwaterSoilFlux', cycle_data)
+print(result)
+```
+
+This will display only the result of the `no3ToGroundwaterSoilFlux` model (Emission).
+
+Additionally, to reduce the number of queries to the HESTIA API and run the models faster, prefetching can be enabled:
+```python
+from hestia_earth.models.preload_requests import enable_preload
+
+enable_preload()
+```
+
+#### Using the orchestrator
+
+The models come with an "orchestrator", which allows you to run a pre-configured set of models instead of a single one.
+
+The configuration for each Node (Cycle, Site or ImpactAssessment) can be found in the [config](./config) folder.
+
+Usage:
+```python
+from hestia_earth.orchestrator import run
+from hestia_earth.models.config import load_config
+
+cycle_data = {"@type": "Cycle", ...}
+result = run(cycle, load_config(cycle))
+print(result)
+```
+
+This will display the Cycle recalculated with all HESTIA default models running.
+
+#### Using Spatial Models
+
+We have models that can gap-fill geographical information on a `Site`. If you want to use these models:
+1. Install the library: `pip install hestia_earth.earth_engine`
+2. Follow the [Getting Started instructions](https://gitlab.com/hestia-earth/hestia-earth-engine#getting-started).
+
+#### Using the ecoinventV3 model
+
+ecoinvent is a consistent, transparent, and well validated life cycle inventory database.
+We use ecoinvent data to ascertain the environmental impacts of activities that occur outside of our system boundary, for example data on the environmental impacts of extracting oil and producing diesel, or the impacts of manufacturing plastics.
+
+The `ecoinventV3` model requires a valid [license](https://ecoinvent.org/offerings/licences/) to run. We are currently working on a way to enable users of this code with a valid ecoinvent licence to run these models themselves, but for now, these models are only available on the public platform.
+
+

hestia_earth_models-0.69.0/hestia_earth/models/agribalyse2016/fuelElectricity.py

@@ -0,0 +1,125 @@
+from hestia_earth.schema import TermTermType
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import flatten, list_sum, non_empty_list
+
+from hestia_earth.models.log import debugValues, logRequirements, logShouldRun, log_as_table
+from hestia_earth.models.utils import group_by
+from hestia_earth.models.utils.term import get_lookup_value
+from hestia_earth.models.utils.input import _new_input
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "completeness.electricityFuel": "False",
+        "practices": [{
+            "@type": "Practice",
+            "term.termType": "operation",
+            "value": "> 0"
+        }]
+    }
+}
+LOOKUPS = {
+    "operation": "fuelUse"
+}
+RETURNS = {
+    "Input": [{
+        "term.termType": "fuel",
+        "value": "",
+        "operation": ""
+    }]
+}
+MODEL_KEY = 'fuelElectricity'
+
+
+def _input(term_id: str, value: float, operation: dict):
+    input = _new_input(term_id, MODEL)
+    input['value'] = [value]
+    input['operation'] = operation
+    return input
+
+
+def _operation_input(operation: dict):
+    input = operation.get('input', {})
+    return _input(input.get('id'), input.get('value') * operation.get('value'), operation.get('term', {}))
+
+
+def _run_operation(cycle: dict):
+    def exec(operations: list):
+        input_term_id = operations[0].get('input').get('id')
+        values_logs = log_as_table([
+            {
+                'id': p.get('term').get('@id'),
+                'value': p.get('value'),
+                'coefficient': p.get('input').get('value')
+            } for p in operations
+        ])
+
+        debugValues(cycle, model=MODEL, term=input_term_id,
+                    values=values_logs)
+
+        logShouldRun(cycle, MODEL, input_term_id, True, model_key=MODEL_KEY)
+
+        return list(map(_operation_input, operations))
+    return exec
+
+
+def _operation_data(practice: dict):
+    term = practice.get('term', {})
+    values = practice.get('value', [])
+    value = list_sum(values) if all([not isinstance(v, str) for v in values]) else None  # str allowed for Practice
+
+    coeffs = get_lookup_value(term, LOOKUPS['operation'], model=MODEL, model_key=MODEL_KEY)
+    values = non_empty_list(coeffs.split(';')) if coeffs else []
+    inputs = [{'id': c.split(':')[0], 'value': float(c.split(':')[1])} for c in values]
+
+    return [{
+        'term': term,
+        'value': value,
+        'input': input,
+        'dates': ';'.join(practice.get('dates', []))
+    } for input in inputs]
+
+
+def _should_run(cycle: dict):
+    is_incomplete = not cycle.get('completeness', {}).get('electricityFuel', False)
+    operations = filter_list_term_type(cycle.get('practices', []), TermTermType.OPERATION)
+
+    operations = flatten(map(_operation_data, operations))
+    term_ids = list(set(non_empty_list(map(lambda v: v.get('term', {}).get('@id'), operations))))
+
+    valid_operations = [v for v in operations if (v.get('value') or 0) > 0]
+    has_operations = len(valid_operations) > 0
+
+    # group operations by term to show in logs
+    grouped_operations = group_by(operations, ['term.@id'])
+
+    for term_id, operations in grouped_operations.items():
+        logs = [
+            {
+                'value': operation.get('value'),
+                'dates': operation.get('dates'),
+                'input-id': operation.get('input', {}).get('@id'),
+            }
+            for operation in operations
+        ]
+        logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
+                        details=log_as_table(logs))
+
+        should_run = any([(v.get('value') or 0) > 0 for v in operations])
+        logShouldRun(cycle, MODEL, term_id, should_run, model_key=MODEL_KEY)
+
+    logRequirements(cycle, model=MODEL, model_key=MODEL_KEY,
+                    is_term_type_electricityFuel_incomplete=is_incomplete,
+                    has_operations=has_operations,
+                    operations=';'.join(term_ids))
+
+    should_run = all([is_incomplete, has_operations])
+    logShouldRun(cycle, MODEL, None, should_run, model_key=MODEL_KEY)
+    return should_run, valid_operations
+
+
+def run(cycle: dict):
+    should_run, operations = _should_run(cycle)
+    # group operations by input to show logs as table
+    grouped_operations = group_by(operations, ['input.id'])
+    return flatten(map(_run_operation(cycle), grouped_operations.values())) if should_run else []

hestia_earth_models-0.69.0/hestia_earth/models/agribalyse2016/machineryInfrastructureDepreciatedAmountPerCycle.py

@@ -0,0 +1,80 @@
+from hestia_earth.utils.tools import list_sum, flatten
+
+from hestia_earth.models.log import logRequirements, logShouldRun, debugValues, log_as_table
+from hestia_earth.models.utils.productivity import get_productivity, PRODUCTIVITY
+from hestia_earth.models.utils.input import _new_input
+from hestia_earth.models.utils.completeness import _is_term_type_incomplete
+from hestia_earth.models.utils.term import get_liquid_fuel_terms
+from . import MODEL
+
+REQUIREMENTS = {
+    "Cycle": {
+        "inputs": [{
+            "@type": "Input",
+            "term.termType": "fuel",
+            "value": ""
+        }],
+        "completeness.material": "False",
+        "site": {
+            "@type": "Site",
+            "country": {"@type": "Term", "termType": "region"}
+        }
+    }
+}
+LOOKUPS = {
+    "region": "HDI"
+}
+RETURNS = {
+    "Input": [{
+        "value": ""
+    }]
+}
+TERM_ID = 'machineryInfrastructureDepreciatedAmountPerCycle'
+
+
+def _input(value: float):
+    input = _new_input(TERM_ID, MODEL)
+    input['value'] = [value]
+    return input
+
+
+def _get_input_value_from_term(inputs: list, term_id: str):
+    values = flatten([
+        input.get('value', []) for input in inputs if input.get('term', {}).get('@id') == term_id
+    ])
+    return list_sum(values, 0) if len(values) > 0 else None
+
+
+def get_value(country: dict, cycle: dict):
+    liquid_fuels = get_liquid_fuel_terms()
+    productivity_key = get_productivity(country)
+    machinery_usage = 11.5 if productivity_key == PRODUCTIVITY.HIGH else 23
+    values = [(term_id, _get_input_value_from_term(cycle.get('inputs', []), term_id)) for term_id in liquid_fuels]
+    value_logs = log_as_table([{'id': term_id, 'value': value} for term_id, value in values])
+    values = [value for term_id, value in values if value is not None]
+    fuel_use = list_sum(values, 0)
+    debugValues(cycle, model=MODEL, term=TERM_ID,
+                productivity_key=productivity_key.value,
+                fuel_use_details=value_logs,
+                fuel_use=fuel_use)
+    return fuel_use/machinery_usage if fuel_use > 0 else None
+
+
+def _run(cycle: dict):
+    country = cycle.get('site', {}).get('country', {})
+    value = get_value(country, cycle)
+    return [_input(value)] if value is not None else []
+
+
+def _should_run(cycle: dict):
+    term_type_incomplete = _is_term_type_incomplete(cycle, TERM_ID)
+
+    logRequirements(cycle, model=MODEL, term=TERM_ID,
+                    term_type_material_incomplete=term_type_incomplete)
+
+    should_run = all([term_type_incomplete])
+    logShouldRun(cycle, MODEL, TERM_ID, should_run)
+    return should_run
+
+
+def run(cycle: dict): return _run(cycle) if _should_run(cycle) else []

hestia_earth_models-0.69.0/hestia_earth/models/aware/scarcityWeightedWaterUse.py

@@ -0,0 +1,112 @@
+from hestia_earth.schema import SiteSiteType
+from hestia_earth.utils.lookup import download_lookup, _get_single_table_value, column_name
+from hestia_earth.utils.tools import safe_parse_float
+
+from hestia_earth.models.log import logRequirements, debugMissingLookup, logShouldRun
+from hestia_earth.models.utils import sum_values, multiply_values
+from hestia_earth.models.utils.indicator import _new_indicator
+from hestia_earth.models.utils.impact_assessment import (
+    convert_value_from_cycle, emission_value, get_product, get_site, get_region_id
+)
+from hestia_earth.models.utils.input import sum_input_impacts
+from hestia_earth.models.utils.lookup import get_region_lookup_value
+from . import MODEL
+
+REQUIREMENTS = {
+    "ImpactAssessment": {
+        "site": {
+            "@type": "Site",
+            "or": {
+                "awareWaterBasinId": "",
+                "country": {"@type": "Term", "termType": "region"}
+            }
+        },
+        "optional": {
+            "emissionsResourceUse": [{
+                "@type": "Indicator",
+                "term.@id": "freshwaterWithdrawalsDuringCycle",
+                "value": ""
+            }]
+        }
+    }
+}
+RETURNS = {
+    "Indicator": [{
+        "value": ""
+    }]
+}
+LOOKUPS = {
+    "@doc": "Different lookup files are used depending on the situation",
+    "awareWaterBasinId": "",
+    "region-aware-factors": ""
+}
+TERM_ID = 'scarcityWeightedWaterUse'
+AWARE_KEY = 'awareWaterBasinId'
+IRRIGATED_SITE_TYPES = [
+    SiteSiteType.CROPLAND.value,
+    SiteSiteType.GLASS_OR_HIGH_ACCESSIBLE_COVER.value,
+    SiteSiteType.PERMANENT_PASTURE.value
+]
+
+
+def _indicator(value: float):
+    indicator = _new_indicator(TERM_ID, MODEL)
+    indicator['value'] = value
+    return indicator
+
+
+def _get_factor_from_basinId(site: dict, aware_id: str):
+    lookup_col = 'YR_IRRI' if site.get('siteType') in IRRIGATED_SITE_TYPES else 'YR_NONIRRI'
+    value = _get_single_table_value(
+        download_lookup(f"{AWARE_KEY}.csv"), column_name(AWARE_KEY), int(aware_id), column_name(lookup_col)
+    )
+    debugMissingLookup(f"{AWARE_KEY}.csv", AWARE_KEY, aware_id, lookup_col, value, model=MODEL, term=TERM_ID)
+    return safe_parse_float(value, None)
+
+
+def _get_factor_from_region(impact_assessment: dict, site: dict):
+    region_id = get_region_id(impact_assessment)
+    site_type = site.get('siteType')
+    lookup_name = 'region-aware-factors.csv'
+    lookup_suffix = 'unspecified' if not site_type else ('irri' if site_type in IRRIGATED_SITE_TYPES else 'non_irri')
+    column = f"Agg_CF_{lookup_suffix}"
+    value = get_region_lookup_value(lookup_name, region_id, column, model=MODEL, term=TERM_ID)
+    return safe_parse_float(value, None)
+
+
+def _run(impact_assessment: dict):
+    cycle = impact_assessment.get('cycle', {})
+    product = get_product(impact_assessment)
+    fresh_water = emission_value(impact_assessment, 'freshwaterWithdrawalsDuringCycle')
+    site = get_site(impact_assessment)
+    aware_id = site.get(AWARE_KEY)
+    factor = (
+        _get_factor_from_basinId(site, aware_id) if aware_id else None
+    ) or _get_factor_from_region(impact_assessment, site)
+    inputs_value = convert_value_from_cycle(
+        impact_assessment, product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
+    )
+
+    logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
+                    fresh_water=fresh_water,
+                    aware_id=aware_id,
+                    factor=factor,
+                    inputs_value=inputs_value)
+
+    value = sum_values([
+        multiply_values([fresh_water, factor]),
+        inputs_value
+    ])
+    return _indicator(value) if value is not None else None
+
+
+def _should_run(impact_assessment: dict):
+    site = get_site(impact_assessment)
+    # does not run without a site as data is geospatial
+    should_run = all([site])
+    logShouldRun(impact_assessment, MODEL, TERM_ID, should_run)
+    return should_run
+
+
+def run(impact_assessment: dict):
+    return _run(impact_assessment) if _should_run(impact_assessment) else None

hestia_earth_models-0.69.0/hestia_earth/models/cache_sites.py

@@ -0,0 +1,173 @@
+from functools import reduce
+from enum import Enum
+from pydash.objects import merge
+from hestia_earth.utils.api import download_hestia
+from hestia_earth.utils.tools import flatten, non_empty_list
+
+from .log import logger
+from .utils import CACHE_KEY, cached_value
+from .utils.site import CACHE_YEARS_KEY
+from .site.pre_checks.cache_geospatialDatabase import (
+    list_vectors, list_rasters, cache_site_results, _should_run
+)
+from .geospatialDatabase.utils import (
+    CACHE_VALUE as CACHE_GEOSPATIAL_KEY, CACHE_AREA_SIZE,
+    has_boundary, has_coordinates,
+    _run_query, _site_gadm_id
+)
+
+
+class ParamType(Enum):
+    BOUNDARIES = 'boundaries'
+    COORDINATES = 'coordinates'
+    GADM_IDS = 'gadm-ids'
+
+
+_VALUE_AS_PARAM = {
+    ParamType.COORDINATES: lambda data: {'latitude': data[0].get('latitude'), 'longitude': data[0].get('longitude')},
+    ParamType.BOUNDARIES: lambda data: data[0].get('boundary'),
+    ParamType.GADM_IDS: lambda data: data[0].get('region', data[0].get('country', {})).get('@id')
+}
+
+
+def _value_as_param(param_type: ParamType = None): return _VALUE_AS_PARAM.get(param_type, lambda *args: None)
+
+
+def _cache_results(results: list, collections: list, index: int):
+    start = index * len(collections)
+    end = start + len(collections)
+    return cache_site_results(results[start:end], collections)
+
+
+def _run_values(
+    sites: list,
+    param_type: ParamType = None,
+    rasters: list = [],
+    vectors: list = [],
+    years: list = None
+):
+    get_param_values = _value_as_param(param_type)
+    param_values = non_empty_list(map(get_param_values, sites))
+    # unique list
+    param_values = list(set(param_values)) if param_type == ParamType.GADM_IDS else list({
+        str(v): v for v in param_values
+    }.values())
+
+    raster_results = _run_query({
+        'ee_type': 'raster',
+        'collections': rasters,
+        param_type.value: param_values
+    }) if rasters else []
+
+    vector_results = _run_query({
+        'ee_type': 'vector',
+        'collections': vectors,
+        param_type.value: param_values
+    }) if vectors else []
+
+    def _process_site(site_values: tuple):
+        site, area_size = site_values
+
+        # get real index in values to handle duplicates
+        param_value = get_param_values([site])
+        index = param_values.index(param_value) if param_value is not None else None
+
+        cached_data = ({
+            **_cache_results(raster_results, rasters, index),
+            **_cache_results(vector_results, vectors, index)
+        } if index is not None else {}) | ({CACHE_AREA_SIZE: area_size} if area_size is not None else {})
+        cached_data = merge(cached_value(site, CACHE_GEOSPATIAL_KEY, {}), cached_data)
+        site_cache = merge(
+            site.get(CACHE_KEY, {}),
+            {CACHE_GEOSPATIAL_KEY: cached_data},
+            ({CACHE_YEARS_KEY: list(set(cached_value(site, CACHE_YEARS_KEY, []) + years))} if years else {})
+        )
+        return merge(site, {CACHE_KEY: site_cache})
+
+    return reduce(lambda prev, curr: prev + [_process_site(curr)], sites, [])
+
+
+def _should_preload_region_area_size(site: dict): return not has_coordinates(site) and not has_boundary(site)
+
+
+def _preload_regions_area_size(sites: dict):
+    region_ids = set(map(_site_gadm_id, filter(_should_preload_region_area_size, sites)))
+    return {term_id: download_hestia(term_id).get('area') for term_id in region_ids}
+
+
+def _group_sites(sites: dict, check_has_cache: bool = True):
+    # preload area size for all regions
+    regions_area_size = _preload_regions_area_size(sites)
+
+    def get_region_area_size(site: dict):
+        return regions_area_size.get(_site_gadm_id(site)) if _should_preload_region_area_size(site) else None
+
+    sites = [
+        (n, ) + (_should_run(n, area_size=get_region_area_size(n), check_has_cache=check_has_cache)) for n in sites
+    ]
+    # restrict sites based on should_cache result
+    sites_run = [(site, area_size) for site, should_cache, area_size in sites if should_cache]
+    # will only cache area and years
+    sites_no_run = [(site, area_size) for site, should_cache, area_size in sites if not should_cache]
+
+    with_coordinates = [
+        (site, area_size) for site, area_size in sites_run if has_coordinates(site)
+    ]
+    with_boundaries = [
+        (site, area_size) for site, area_size in sites_run if not has_coordinates(site) and has_boundary(site)
+    ]
+    with_gadm_ids = [
+        (site, area_size) for site, area_size in sites_run if not has_coordinates(site) and not has_boundary(site)
+    ]
+
+    return {
+        ParamType.COORDINATES: with_coordinates,
+        ParamType.BOUNDARIES: with_boundaries,
+        ParamType.GADM_IDS: with_gadm_ids
+    }, sites_no_run
+
+
+def _run(sites: list, years: list = [], years_only: bool = False):
+    rasters = list_rasters(years=years, years_only=years_only)
+    vectors = [] if years_only else list_vectors(sites)
+    filtered_data, sites_no_run = _group_sites(sites, not years_only)
+    return flatten([
+        _run_values(filtered_data.get(param_type), param_type, rasters, vectors, years)
+        for param_type in [e for e in ParamType] if len(filtered_data.get(param_type)) > 0
+    ] + _run_values(sites_no_run, years=years))
+
+
+def _run_by_years(sites: list, years: list, batch_size: int):
+    batches = range(0, len(years), batch_size)
+
+    for batch_index in batches:
+        logger.info(f"Processing sites in batch {batch_index + 1} of {len(batches)}...")
+        sub_years = years[batch_index:batch_index + batch_size]
+        sites = _run(sites, sub_years, years_only=True)
+
+    return sites
+
+
+def run(sites: list, years: list = None):
+    """
+    Run all queries at once for the list of provided Sites.
+    Note: Earth Engine needs to be initiliased with `init_gee()` before running this function.
+
+    Parameters
+    ----------
+    sites : list[dict]
+        List of Site as dict.
+    years : list[int]
+        List of related years to fetch annual data.
+    """
+    sites = _run(sites)
+
+    # avoid memory limit errors by running only a few years at a time
+    unique_years = sorted(list(set(years)))
+
+    try:
+        return _run_by_years(sites, unique_years, batch_size=5)
+    except Exception as e:
+        if 'exceeded' in str(e):
+            return _run_by_years(sites, unique_years, batch_size=1)
+        raise e