hestia-earth-models 0.64.8__py3-none-any.whl → 0.64.10__py3-none-any.whl

hestia_earth/models/environmentalFootprintV3/soilQualityIndexLandTransformation.py

@@ -4,53 +4,53 @@ based on an updated [LANCA model (De Laurentiis et al. 2019)](
 http://publications.jrc.ec.europa.eu/repository/handle/JRC113865) and on the LANCA (Regionalised) Characterisation
 Factors version 2.5 (Horn and Meier, 2018).
 """
-from typing import List
+from typing import List, Tuple, Optional
 
 from hestia_earth.schema import TermTermType
 from hestia_earth.utils.lookup import download_lookup
 from hestia_earth.utils.model import filter_list_term_type
-from hestia_earth.utils.tools import list_sum, safe_parse_date, non_empty_list
+from hestia_earth.utils.tools import list_sum
 
 from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
-from hestia_earth.models.utils import hectar_to_square_meter
-from hestia_earth.models.utils import pairwise
 from . import MODEL
 from .utils import get_coefficient_factor
+from ..utils.impact_assessment import get_country_id
 from ..utils.indicator import _new_indicator
 from ..utils.landCover import get_pef_grouping
-from ..utils.lookup import fallback_country
+from ..utils.lookup import fallback_country, _node_value
+from ..utils.term import get_land_cover_terms
 
 REQUIREMENTS = {
     "ImpactAssessment": {
-        "cycle": {
-            "@type": "Cycle",
-            "site": {
-                "area": "> 0",
-                "@type": "Site",
-                "management": [{"@type": "Management", "term.termType": "landCover"}],
-                "optional": {"country": {"@type": "Term", "termType": "region"}}
-            },
-            "optional": {
-                "otherSitesArea": "> 0",
-                "otherSites": [{
-                    "@type": "Site",
-                    "management": [{"@type": "Management", "term.termType": "landCover"}],
-                    "optional": {"country": {"@type": "Term", "termType": "region"}}
-                }]
+        "emissionsResourceUse": [
+            {
+                "@type": "Indicator",
+                "term.units": "m2 / year",
+                "term.termType": "resourceUse",
+                "term.name": "Land transformation from",
+                "value": "> 0",
+                "landCover": {"@type": "Term", "term.termType": "landCover"}
             }
-        }
+        ],
+        "optional": {"country": {"@type": "Term", "termType": "region"}}
     }
 }
 
+# Note: CFs in `region-pefTermGrouping-landTransformation-from.csv` appear to be the opposite values as those in
+# `region-pefTermGrouping-landTransformation-to.csv` but can be different in some cases.
 LOOKUPS = {
-    "@doc": "Uses `landCover.csv` for column headers and region-pefTermGrouping-landTransformation-X.csv for to/from CFs. (CFs in `region-pefTermGrouping-landTransformation-from.csv` appear to be the opposite values as those in `region-pefTermGrouping-landTransformation-to.csv` but can be different in some cases)",  # noqa: E501
-    "region-pefTermGrouping-landTransformation-from": "",
-    "region-pefTermGrouping-landTransformation-to": "",
+    "region-pefTermGrouping-landTransformation-from": "using country and `pefTermGrouping` from `landCover`",
+    "region-pefTermGrouping-landTransformation-to": "using country and `pefTermGrouping` from `landCover`",
     "landCover": "pefTermGrouping"
 }
 
-from_lookup_file = f"{list(LOOKUPS.keys())[1]}.csv"
-to_lookup_file = f"{list(LOOKUPS.keys())[2]}.csv"
+from_lookup_file = f"{list(LOOKUPS.keys())[0]}.csv"
+to_lookup_file = f"{list(LOOKUPS.keys())[1]}.csv"
+
+LOOKUP = {
+    "from": from_lookup_file,
+    "to": to_lookup_file
+}
 
 RETURNS = {
     "Indicator": {
@@ -67,116 +67,102 @@ def _indicator(value: float):
     return indicator
 
 
-def _run(sites: List[dict]):
-    result = []
-    for site in sites:
-        values = [(transformation_from_factor + transformation_to_factor) * hectar_to_square_meter(site['area'])
-                  for transformation_from_factor, transformation_to_factor in site['transformation_factors']]
-        result.append(list_sum(values))
-    return _indicator(list_sum(result)) if result else None
+def _run(transformations: List[dict]):
+    values = [(transformation['factor-from'] + transformation['factor-to']) * transformation['area'] for transformation
+              in transformations]
+    return _indicator(list_sum(values))
 
 
-def _should_run(impact_assessment: dict):
-    cycle = impact_assessment.get('cycle', {})
+def _extract_land_cover_from_indicator_id(indicator: dict) -> Optional[str]:
+    """
+    Given a indicator with term type `resourceUse` return the equivalent `landCover` term
+    """
+    term_in_id = indicator.get('term', {}).get('@id', '') \
+        .removeprefix("landTransformationFrom") \
+        .removesuffix("20YearAverageInputsProduction") \
+        .removesuffix("20YearAverageDuringCycle")
+    term_in_id = term_in_id[0].lower() + term_in_id[1:] if term_in_id else None
+    return term_in_id
 
-    has_site = bool(cycle.get('site', {}))
-    site_area = cycle.get('site', {}).get("area", False)
-    has_area = site_area > 0
 
-    has_other_sites = bool(cycle.get('otherSites', []))
+def _is_valid_indicator(indicator: dict, land_cover_term_ids: list[str]) -> bool:
+    term_id = _extract_land_cover_from_indicator_id(indicator)
+    return term_id in land_cover_term_ids
 
-    all_sites = non_empty_list([cycle.get('site')] + cycle.get('otherSites', []))
-    site_areas = [cycle.get('site', {}).get('area')] + cycle.get('otherSitesArea', [])
 
-    sites = [
-        {
-            'site_id': site.get('@id', site.get('id')),
-            'transformation_pairs': list(pairwise(
-                sorted(filter_list_term_type(site.get("management", []), TermTermType.LANDCOVER.value),
-                       key=lambda d: safe_parse_date(d['endDate'])))),
-            'country_id_str': site.get('country', {}).get('@id', ''),
-            'area': site_areas[index] if len(site_areas) >= index + 1 else None,
-        }
-        for index, site in enumerate(all_sites)
-    ]
+def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
+    resource_uses = filter_list_term_type(impact_assessment.get('emissionsResourceUse', []), TermTermType.RESOURCEUSE)
+    land_cover_term_ids = get_land_cover_terms() if resource_uses else []
 
-    sites = [
-        site for site in sites
-        if all([
-            (site.get('area') or 0) > 0,
-            site.get('transformation_pairs', [])
-        ])
-    ]
+    land_transformation_indicators = [i for i in resource_uses if _is_valid_indicator(i, land_cover_term_ids)]
 
-    sites = [
-        site |
+    found_transformations = [
         {
-            'column_names': [(get_pef_grouping(from_transformation['term']['@id']),
-                              get_pef_grouping(to_transformation['term']['@id']))
-                             for from_transformation, to_transformation in site['transformation_pairs']],
-        } for site in sites
+            'area': _node_value(transformation_indicator) * 20,
+            'area-unit': transformation_indicator.get('term', {}).get("units"),
+            'land-cover-id-from': _extract_land_cover_from_indicator_id(transformation_indicator),
+            'land-cover-id-to': transformation_indicator.get('landCover', {}).get("@id"),
+            'indicator-id': transformation_indicator.get('term', {}).get('@id', ''),
+            'good-land-cover-term': transformation_indicator.get('landCover', {}).get('termType') == 'landCover',
+            'country-id': get_country_id(impact_assessment),
+            'area-is-valid': _node_value(transformation_indicator) is not None and _node_value(
+                transformation_indicator) > 0,
+            'area-unit-is-valid': transformation_indicator.get('term', {}).get("units") == "m2 / year",
+            'lookup-country': fallback_country(get_country_id(impact_assessment),
+                                               [download_lookup(from_lookup_file), download_lookup(to_lookup_file)]),
+        } for transformation_indicator in land_transformation_indicators
     ]
 
-    has_lookup_column = lambda s: s['column_names'] and all([all(pair) for pair in s['column_names']])  # noqa: E731
-    valid_sites = [site for site in sites if has_lookup_column(site)]
-
-    has_valid_sites = bool(valid_sites)
-
-    valid_sites = [
-        site |
-        {
-            'country_id': fallback_country(site['country_id_str'],
-                                           [download_lookup(from_lookup_file), download_lookup(to_lookup_file)]
-                                           )
-        } for site in valid_sites
+    found_transformations_with_coefficient = [
+        transformation | {
+            "using-fallback-country-region-world-CFs": transformation['lookup-country'] != transformation['country-id'],
+            'factor-from': get_coefficient_factor(
+                lookup_name=from_lookup_file,
+                country_id=transformation['lookup-country'],
+                term_id=TERM_ID,
+                occupation_type=get_pef_grouping(transformation['land-cover-id-from'])) if
+            transformation['land-cover-id-from'] else None,
+            'factor-to': get_coefficient_factor(
+                lookup_name=to_lookup_file,
+                country_id=transformation['lookup-country'],
+                term_id=TERM_ID,
+                occupation_type=get_pef_grouping(transformation['land-cover-id-to'])) if
+            transformation['land-cover-id-to'] else None
+        } for transformation in found_transformations
     ]
 
-    valid_sites = [
-        site |
-        {
-            'transformation_factors': [(get_coefficient_factor(lookup_name=from_lookup_file,
-                                                               country_id=site['country_id'], term_id=TERM_ID,
-                                                               occupation_type=from_transformation_header),
-                                        get_coefficient_factor(lookup_name=to_lookup_file,
-                                                               country_id=site['country_id'], term_id=TERM_ID,
-                                                               occupation_type=to_transformation_header))
-                                       for from_transformation_header, to_transformation_header in site['column_names']]
-        } for site in valid_sites
+    valid_transformations_with_coef = [
+        t for t in found_transformations_with_coefficient if all([
+            t['area-is-valid'],
+            t['area-unit-is-valid'],
+            t['factor-from'] is not None,
+            t['factor-to'] is not None
+        ])
     ]
 
-    log_equivalent_eu_pef_land_use_names = [
-        [{
-            'site-id': site['site_id'],
-            'hestia-term': hestia_term,
-            'corine-term': corine_term,
-        } for hestia_term, corine_term in site['column_names']
-        ] for site in valid_sites]
-
-    log_transformation_factors = [
-        [{
-            'site-id': site['site_id'],
-            'country-id-used-for-factors': site['country_id'],
-            'country-id-in-input': site['country_id_str'],
-            'factor-from': from_transformation_header,
-            'factor-to': to_transformation_header,
-        } for from_transformation_header, to_transformation_header in site['transformation_factors']
-        ] for site in valid_sites]
+    has_land_transformation_indicators = bool(land_transformation_indicators)
+
+    all_transformations_are_valid = all(
+        [
+            all([t['area-is-valid'], t['area-unit-is-valid'], t['good-land-cover-term']])
+            for t in found_transformations_with_coefficient
+        ]
+    ) if found_transformations_with_coefficient else False
 
     logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
-                    has_site=has_site,
-                    has_area=has_area,
-                    has_other_sites=has_other_sites,
-                    has_valid_sites=has_valid_sites,
-                    equivalent_EU_PEF_landUse_names=log_as_table(log_equivalent_eu_pef_land_use_names),
-                    transformation_factors=log_as_table(log_transformation_factors)
+                    has_land_occupation_indicators=has_land_transformation_indicators,
+                    all_transformations_are_valid=all_transformations_are_valid,
+                    has_valid_transformations_with_coef=bool(valid_transformations_with_coef),
+                    found_transformations=log_as_table(found_transformations_with_coefficient)
                     )
-    should_run = has_valid_sites
 
-    logShouldRun(impact_assessment, MODEL, TERM_ID, should_run)
+    should_run = has_land_transformation_indicators is False or all([has_land_transformation_indicators,
+                                                                     all_transformations_are_valid])
 
-    return should_run, valid_sites
+    logShouldRun(impact_assessment, MODEL, TERM_ID, should_run)
+    return should_run, valid_transformations_with_coef
 
 
 def run(impact_assessment: dict):
-    should_run, sites = _should_run(impact_assessment)
-    return _run(sites) if should_run else None
+    should_run, transformations = _should_run(impact_assessment)
+    return _run(transformations) if should_run else None

hestia_earth/models/environmentalFootprintV3/soilQualityIndexTotalLandUseEffects.py

@@ -5,6 +5,7 @@ http://publications.jrc.ec.europa.eu/repository/handle/JRC113865) and on the LAN
 Factors version 2.5 (Horn and Meier, 2018).
 """
 from hestia_earth.utils.model import find_term_match
+from hestia_earth.utils.tools import list_sum
 
 from hestia_earth.models.log import logRequirements, logShouldRun
 from . import MODEL
@@ -35,32 +36,42 @@ def _indicator(value: float):
     return indicator
 
 
-def _run(land_occupation_indicator, transformation_indicator):
-    value = land_occupation_indicator['value'] + transformation_indicator['value']
-    return _indicator(value) if value else None
+def _run(indicators: list):
+    values = [indicator['value'] for indicator in indicators]
+    return _indicator(list_sum(values))
 
 
-def _should_run(impactassessment: dict):
-    land_occupation_indicator = find_term_match(impactassessment.get('emissionsResourceUse', []),
-                                                "soilQualityIndexLandOccupation")
-    transformation_indicator = find_term_match(impactassessment.get('emissionsResourceUse', []),
-                                               "soilQualityIndexLandTransformation")
+def _should_run(impactassessment: dict) -> tuple[bool, list]:
+    land_indicators = [
+        i for i in impactassessment.get('emissionsResourceUse', []) if
+        i.get('term', {}).get('@id', '') in ['soilQualityIndexLandOccupation', 'soilQualityIndexLandTransformation']
+    ]
+    has_indicators = bool(land_indicators)
 
-    has_valid_values = all([isinstance(land_occupation_indicator.get('value', None), (int, float)),
-                            isinstance(transformation_indicator.get('value', None), (int, float))])
+    land_occupation_indicator = find_term_match(land_indicators, "soilQualityIndexLandOccupation",
+                                                default_val=None)
+    has_land_occupation_indicator = bool(land_occupation_indicator)
+
+    land_transformation_indicator = find_term_match(land_indicators, "soilQualityIndexLandTransformation",
+                                                    default_val=None)
+    has_land_transformation_indicator = bool(land_transformation_indicator)
+
+    has_valid_values = all([isinstance(indicator.get('value', None), (int, float)) for indicator in land_indicators])
 
     logRequirements(impactassessment, model=MODEL, term=TERM_ID,
-                    transformation_indicator=transformation_indicator,
-                    land_occupation_indicator=land_occupation_indicator,
+                    has_indicators=has_indicators,
+                    has_land_occupation_indicator=has_land_occupation_indicator,
+                    has_land_transformation_indicator=has_land_transformation_indicator,
                     has_valid_values=has_valid_values
                     )
 
-    should_run = all([transformation_indicator, land_occupation_indicator, has_valid_values])
+    should_run = all([has_indicators, has_valid_values,
+                      has_land_occupation_indicator, has_land_transformation_indicator])
 
     logShouldRun(impactassessment, MODEL, TERM_ID, should_run)
-    return should_run, land_occupation_indicator, transformation_indicator
+    return should_run, land_indicators
 
 
 def run(impactassessment: dict):
-    should_run, land_occupation_indicator, transformation_indicator = _should_run(impactassessment)
-    return _run(land_occupation_indicator, transformation_indicator) if should_run else None
+    should_run, indicators = _should_run(impactassessment)
+    return _run(indicators) if should_run else None

hestia_earth/models/faostat2018/landTransformationFromCropland100YearAverage.py

@@ -20,7 +20,8 @@ LOOKUPS = {
 }
 RETURNS = {
     "Indicator": [{
-        "value": ""
+        "value": "",
+        "landCover": ""
     }]
 }
 TERM_ID = 'landTransformationFromTemporaryCropland100YearAverageDuringCycle,landTransformationFromPermanentCropland100YearAverageDuringCycle'  # noqa: E501
@@ -30,7 +31,7 @@ PERMANENT_TERM_ID = 'landTransformationFromPermanentCropland100YearAverageDuring
 
 
 def _indicator(term_id: str, value: float):
-    indicator = _new_indicator(term_id, MODEL)
+    indicator = _new_indicator(term_id, MODEL, 'cropland')
     indicator['value'] = value
     return indicator
 

hestia_earth/models/faostat2018/landTransformationFromCropland20YearAverage.py

@@ -20,7 +20,8 @@ LOOKUPS = {
 }
 RETURNS = {
     "Indicator": [{
-        "value": ""
+        "value": "",
+        "landCover": ""
     }]
 }
 TERM_ID = 'landTransformationFromTemporaryCropland20YearAverageDuringCycle,landTransformationFromPermanentCropland20YearAverageDuringCycle'  # noqa: E501
@@ -30,7 +31,7 @@ PERMANENT_TERM_ID = 'landTransformationFromPermanentCropland20YearAverageDuringC
 
 
 def _indicator(term_id: str, value: float):
-    indicator = _new_indicator(term_id, MODEL)
+    indicator = _new_indicator(term_id, MODEL, 'cropland')
     indicator['value'] = value
     return indicator
 

hestia_earth/models/frischknechtEtAl2000/ionisingRadiationKbqU235Eq.py

@@ -1,11 +1,11 @@
-from functools import reduce
 from hestia_earth.schema import TermTermType
-from hestia_earth.utils.tools import list_sum, flatten, non_empty_list
+from hestia_earth.utils.lookup import get_table_value, download_lookup, column_name
+from hestia_earth.utils.model import filter_list_term_type
+from hestia_earth.utils.tools import flatten
 
-from hestia_earth.models.log import logRequirements, logShouldRun
-from hestia_earth.models.utils.emission import filter_emission_inputs
+from hestia_earth.models.log import logRequirements, logShouldRun, log_as_table
 from hestia_earth.models.utils.indicator import _new_indicator
-from hestia_earth.models.utils.lookup import factor_value
+from hestia_earth.models.utils.lookup import _node_value
 from . import MODEL
 
 REQUIREMENTS = {
@@ -18,7 +18,7 @@ REQUIREMENTS = {
                 "ionisingCompoundsToWaterInputsProduction",
                 "ionisingCompoundsToSaltwaterInputsProduction"
             ],
-            "inputs": [{"@type": "Input", "term.termType": "waste"}]
+            "inputs": {"@type": "Term", "term.termType": "waste", "term.units": "kg"}
         }]
     }
 }
@@ -30,15 +30,23 @@ LOOKUPS = {
     ]
 }
 RETURNS = {
-    "Indicator": {
+    "Indicator": [{
         "value": "",
         "inputs": ""
-    }
+    }]
 }
 
 TERM_ID = 'ionisingRadiationKbqU235Eq'
 
 
+def _valid_waste(input: dict) -> bool:
+    return input.get('units', '').startswith("kg") and input.get('termType', '') == TermTermType.WASTE.value
+
+
+def _valid_emission(emission: dict) -> bool:
+    return len(emission.get('inputs', [])) == 1 and isinstance(_node_value(emission), (int, float))
+
+
 def _indicator(value: float, input: dict):
     indicator = _new_indicator(TERM_ID, MODEL)
     indicator['value'] = value
@@ -46,45 +54,69 @@ def _indicator(value: float, input: dict):
     return indicator
 
 
-def _run(grouped_emissions_inputs: list):
-    input = grouped_emissions_inputs[0].get('input')
-    values = [
-        factor_value(
-            model=MODEL,
-            term_id=TERM_ID,
-            lookup_name=f"{TermTermType.WASTE.value}.csv",
-            lookup_col=i.get('emission').get('term', {}).get('@id')
-        )(data=i.get('emission') | {'term': i.get('input')})
-        for i in grouped_emissions_inputs
+def _run(emissions: list):
+    indicators = [
+        _indicator(value=emission['value'] * emission['coefficient'], input=emission['input'])
+        for emission in emissions
     ]
-    value = list_sum(values)
-    return _indicator(value, input) if value else None
+    return indicators
 
 
 def _should_run(impact_assessment: dict):
-    emissions = flatten([
-        ([
-            {'emission': emission, 'input': input}
-            for input in filter_emission_inputs(emission, TermTermType.WASTE)
+    emissions = [emission for emission in filter_list_term_type(impact_assessment.get('emissionsResourceUse', []),
+                                                                TermTermType.EMISSION)
+                 if emission.get('term', {}).get('@id', '') in LOOKUPS[TermTermType.WASTE.value]]
+
+    has_emissions = bool(emissions)
+
+    emissions_unpacked = flatten(
+        [
+            [
+                {
+                    "input-term-id": input.get('@id'),
+                    "input-term-type": input.get('termType'),
+                    "indicator-term-id": emission['term']['@id'],
+                    "indicator-is-valid": _valid_emission(emission),
+                    "input": input,
+                    "indicator-input-is-valid": _valid_waste(input),
+                    "value": _node_value(emission),
+                    "coefficient": get_table_value(array=download_lookup(filename="waste.csv"),
+                                                   col_match='termid',
+                                                   col_match_with=input.get('@id'),
+                                                   col_val=column_name(emission['term']['@id'])) if input else None
+                } for input in emission['inputs'] or [{}]]
+            for emission in emissions
+        ]
+    )
+
+    valid_emission_with_cf = [em for em in emissions_unpacked if em['coefficient'] is not None
+                              and em['indicator-is-valid'] is True
+                              and em['indicator-input-is-valid'] is True]
+
+    valid_input_requirements = all([
+        all([
+            em['indicator-is-valid'],
+            em['indicator-input-is-valid']
         ])
-        for emission in impact_assessment.get('emissionsResourceUse', [])
-        if emission.get('term', {}).get('@id') in LOOKUPS[TermTermType.WASTE.value]
+        for em in emissions_unpacked
     ])
-    emissions_per_input = reduce(
-        lambda p, c: p | {c.get('input').get('@id'): p.get(c.get('input').get('@id'), []) + [c]},
-        emissions,
-        {}
-    )
 
-    logRequirements(impact_assessment, model=MODEL,
-                    has_emissions=bool(emissions_per_input))
+    all_emissions_have_known_cf = all([em['coefficient'] is not None for em in emissions_unpacked]) and bool(
+        emissions_unpacked)
+
+    logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
+                    has_emissions=has_emissions,
+                    valid_input_requirements=valid_input_requirements,
+                    all_emissions_have_known_CF=all_emissions_have_known_cf,
+                    emissions=log_as_table(emissions_unpacked)
+                    )
 
-    should_run = all([emissions_per_input])
+    should_run = valid_input_requirements
 
     logShouldRun(impact_assessment, MODEL, TERM_ID, should_run)
-    return should_run, emissions_per_input
+    return should_run, valid_emission_with_cf
 
 
 def run(impact_assessment: dict):
-    should_run, emissions_per_input = _should_run(impact_assessment)
-    return non_empty_list(map(_run, emissions_per_input.values())) if should_run else []
+    should_run, emissions = _should_run(impact_assessment)
+    return _run(emissions) if should_run else []