hestia-earth-models 0.64.13__py3-none-any.whl → 0.65.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of hestia-earth-models might be problematic. Click here for more details.

Files changed (118) hide show
  1. hestia_earth/models/agribalyse2016/fuelElectricity.py +1 -1
  2. hestia_earth/models/cache_sites.py +15 -24
  3. hestia_earth/models/chaudharyBrooks2018/damageToTerrestrialEcosystemsLandTransformation.py +6 -9
  4. hestia_earth/models/cycle/input/hestiaAggregatedData.py +46 -22
  5. hestia_earth/models/cycle/materialAndSubstrate.py +158 -0
  6. hestia_earth/models/cycle/pre_checks/cache_sources.py +3 -25
  7. hestia_earth/models/cycle/product/economicValueShare.py +2 -2
  8. hestia_earth/models/environmentalFootprintV3/soilQualityIndexLandTransformation.py +11 -33
  9. hestia_earth/models/faostat2018/landTransformation100YearAverageDuringCycle.py +34 -0
  10. hestia_earth/models/faostat2018/landTransformation20YearAverageDuringCycle.py +34 -0
  11. hestia_earth/models/faostat2018/utils.py +47 -3
  12. hestia_earth/models/hestia/landCover.py +5 -5
  13. hestia_earth/models/hestia/seed_emissions.py +275 -0
  14. hestia_earth/models/ipcc2019/aboveGroundBiomass.py +2 -2
  15. hestia_earth/models/ipcc2019/belowGroundBiomass.py +8 -2
  16. hestia_earth/models/ipcc2019/biomass_utils.py +11 -4
  17. hestia_earth/models/ipcc2019/ch4ToAirEntericFermentation.py +21 -12
  18. hestia_earth/models/ipcc2019/ch4ToAirExcreta.py +1 -2
  19. hestia_earth/models/ipcc2019/co2ToAirAboveGroundBiomassStockChange.py +2 -1
  20. hestia_earth/models/ipcc2019/co2ToAirBelowGroundBiomassStockChange.py +2 -1
  21. hestia_earth/models/ipcc2019/co2ToAirCarbonStockChange_utils.py +8 -7
  22. hestia_earth/models/ipcc2019/co2ToAirSoilOrganicCarbonStockChange.py +2 -1
  23. hestia_earth/models/ipcc2019/organicCarbonPerHa_tier_1_utils.py +28 -34
  24. hestia_earth/models/ipcc2019/organicCarbonPerHa_tier_2_utils.py +8 -12
  25. hestia_earth/models/ipcc2019/organicCarbonPerHa_utils.py +13 -30
  26. hestia_earth/models/linkedImpactAssessment/{landTransformationFromCropland20YearAverageInputsProduction.py → landTransformation100YearAverageInputsProduction.py} +5 -2
  27. hestia_earth/models/linkedImpactAssessment/{landTransformationFromCropland100YearAverageInputsProduction.py → landTransformation20YearAverageInputsProduction.py} +5 -2
  28. hestia_earth/models/linkedImpactAssessment/utils.py +69 -12
  29. hestia_earth/models/mocking/__init__.py +1 -1
  30. hestia_earth/models/mocking/search-results.json +1026 -1026
  31. hestia_earth/models/pooreNemecek2018/excretaKgN.py +45 -41
  32. hestia_earth/models/pooreNemecek2018/excretaKgVs.py +89 -63
  33. hestia_earth/models/pooreNemecek2018/saplingsDepreciatedAmountPerCycle.py +8 -8
  34. hestia_earth/models/pooreNemecek2018/utils.py +60 -19
  35. hestia_earth/models/preload_requests.py +24 -4
  36. hestia_earth/models/schererPfister2015/nErosionSoilFlux.py +4 -3
  37. hestia_earth/models/schererPfister2015/pErosionSoilFlux.py +4 -3
  38. hestia_earth/models/schererPfister2015/utils.py +12 -9
  39. hestia_earth/models/site/management.py +70 -55
  40. hestia_earth/models/site/pre_checks/cache_sources.py +2 -20
  41. hestia_earth/models/utils/__init__.py +12 -1
  42. hestia_earth/models/utils/aggregated.py +1 -1
  43. hestia_earth/models/utils/blank_node.py +20 -12
  44. hestia_earth/models/utils/cache_sources.py +15 -0
  45. hestia_earth/models/utils/constant.py +3 -0
  46. hestia_earth/models/utils/crop.py +5 -0
  47. hestia_earth/models/utils/indicator.py +3 -1
  48. hestia_earth/models/version.py +1 -1
  49. {hestia_earth_models-0.64.13.dist-info → hestia_earth_models-0.65.0.dist-info}/METADATA +2 -2
  50. {hestia_earth_models-0.64.13.dist-info → hestia_earth_models-0.65.0.dist-info}/RECORD +81 -108
  51. tests/models/cml2001Baseline/test_abioticResourceDepletionMineralsAndMetals.py +1 -1
  52. tests/models/cycle/input/test_hestiaAggregatedData.py +5 -2
  53. tests/models/cycle/test_materialsAndSubstrate.py +49 -0
  54. tests/models/environmentalFootprintV3/test_soilQualityIndexLandTransformation.py +39 -28
  55. tests/models/{hyde32/test_landTransformationFromForest20YearAverageDuringCycle.py → faostat2018/test_landTransformation100YearAverageDuringCycle.py} +5 -5
  56. tests/models/{hyde32/test_landTransformationFromForest100YearAverageDuringCycle.py → faostat2018/test_landTransformation20YearAverageDuringCycle.py} +5 -5
  57. tests/models/faostat2018/test_utils.py +28 -0
  58. tests/models/hestia/test_landCover.py +2 -1
  59. tests/models/hestia/test_seed_emissions.py +27 -0
  60. tests/models/ipcc2019/test_aboveGroundBiomass.py +40 -4
  61. tests/models/ipcc2019/test_belowGroundBiomass.py +40 -4
  62. tests/models/ipcc2019/test_co2ToAirAboveGroundBiomassStockChange.py +52 -15
  63. tests/models/ipcc2019/test_co2ToAirBelowGroundBiomassStockChange.py +50 -14
  64. tests/models/ipcc2019/test_co2ToAirSoilOrganicCarbonStockChange.py +53 -32
  65. tests/models/ipcc2019/test_organicCarbonPerHa.py +91 -108
  66. tests/models/ipcc2019/test_organicCarbonPerHa_tier_1_utils.py +33 -50
  67. tests/models/ipcc2019/test_organicCarbonPerHa_tier_2_utils.py +0 -52
  68. tests/models/linkedImpactAssessment/test_freshwaterWithdrawalsInputsProduction.py +6 -4
  69. tests/models/linkedImpactAssessment/test_landOccupationInputsProduction.py +6 -4
  70. tests/models/linkedImpactAssessment/{test_landTransformationFromForest100YearAverageInputsProduction.py → test_landTransformation100YearAverageInputsProduction.py} +7 -5
  71. tests/models/linkedImpactAssessment/{test_landTransformationFromForest20YearAverageInputsProduction.py → test_landTransformation20YearAverageInputsProduction.py} +7 -5
  72. tests/models/pooreNemecek2018/test_excretaKgN.py +2 -2
  73. tests/models/pooreNemecek2018/test_excretaKgVs.py +1 -1
  74. tests/models/pooreNemecek2018/test_utils.py +26 -0
  75. tests/models/site/test_management.py +10 -27
  76. tests/models/test_cache_sites.py +40 -12
  77. tests/models/utils/test_blank_node.py +0 -8
  78. tests/models/utils/test_cache_sources.py +21 -0
  79. hestia_earth/models/blonkConsultants2016/landTransformationFromForest20YearAverageDuringCycle.py +0 -90
  80. hestia_earth/models/faostat2018/landTransformationFromCropland100YearAverage.py +0 -74
  81. hestia_earth/models/faostat2018/landTransformationFromCropland20YearAverage.py +0 -74
  82. hestia_earth/models/hyde32/__init__.py +0 -13
  83. hestia_earth/models/hyde32/landTransformationFromCropland100YearAverageDuringCycle.py +0 -60
  84. hestia_earth/models/hyde32/landTransformationFromCropland20YearAverageDuringCycle.py +0 -60
  85. hestia_earth/models/hyde32/landTransformationFromForest100YearAverageDuringCycle.py +0 -60
  86. hestia_earth/models/hyde32/landTransformationFromForest20YearAverageDuringCycle.py +0 -60
  87. hestia_earth/models/hyde32/landTransformationFromOtherNaturalVegetation100YearAverageDuringCycle.py +0 -61
  88. hestia_earth/models/hyde32/landTransformationFromOtherNaturalVegetation20YearAverageDuringCycle.py +0 -61
  89. hestia_earth/models/hyde32/landTransformationFromPermanentPasture100YearAverageDuringCycle.py +0 -61
  90. hestia_earth/models/hyde32/landTransformationFromPermanentPasture20YearAverageDuringCycle.py +0 -61
  91. hestia_earth/models/hyde32/utils.py +0 -72
  92. hestia_earth/models/linkedImpactAssessment/landTransformationFromForest100YearAverageInputsProduction.py +0 -36
  93. hestia_earth/models/linkedImpactAssessment/landTransformationFromForest20YearAverageInputsProduction.py +0 -36
  94. hestia_earth/models/linkedImpactAssessment/landTransformationFromOtherNaturalVegetation100YearAverageInputsProduction.py +0 -36
  95. hestia_earth/models/linkedImpactAssessment/landTransformationFromOtherNaturalVegetation20YearAverageInputsProduction.py +0 -36
  96. hestia_earth/models/linkedImpactAssessment/landTransformationFromPermanentPasture100YearAverageInputsProduction.py +0 -36
  97. hestia_earth/models/linkedImpactAssessment/landTransformationFromPermanentPasture20YearAverageInputsProduction.py +0 -36
  98. tests/models/blonkConsultants2016/test_landTransformationFromForest20YearAverageDuringCycle.py +0 -36
  99. tests/models/cycle/pre_checks/test_cache_sources.py +0 -25
  100. tests/models/faostat2018/test_landTransformationFromCropland100YearAverage.py +0 -40
  101. tests/models/faostat2018/test_landTransformationFromCropland20YearAverage.py +0 -40
  102. tests/models/hyde32/__init__.py +0 -0
  103. tests/models/hyde32/test_landTransformationFromCropland100YearAverageDuringCycle.py +0 -21
  104. tests/models/hyde32/test_landTransformationFromCropland20YearAverageDuringCycle.py +0 -21
  105. tests/models/hyde32/test_landTransformationFromOtherNaturalVegetation100YearAverageDuringCycle.py +0 -23
  106. tests/models/hyde32/test_landTransformationFromOtherNaturalVegetation20YearAverageDuringCycle.py +0 -21
  107. tests/models/hyde32/test_landTransformationFromPermanentPasture100YearAverageDuringCycle.py +0 -21
  108. tests/models/hyde32/test_landTransformationFromPermanentPasture20YearAverageDuringCycle.py +0 -21
  109. tests/models/linkedImpactAssessment/test_landTransformationFromCropland100YearAverageInputsProduction.py +0 -23
  110. tests/models/linkedImpactAssessment/test_landTransformationFromCropland20YearAverageInputsProduction.py +0 -23
  111. tests/models/linkedImpactAssessment/test_landTransformationFromOtherNaturalVegetation100YearAverageInputsProduction.py +0 -23
  112. tests/models/linkedImpactAssessment/test_landTransformationFromOtherNaturalVegetation20YearAverageInputsProduction.py +0 -23
  113. tests/models/linkedImpactAssessment/test_landTransformationFromPermanentPasture100YearAverageInputsProduction.py +0 -24
  114. tests/models/linkedImpactAssessment/test_landTransformationFromPermanentPasture20YearAverageInputsProduction.py +0 -24
  115. tests/models/site/pre_checks/test_cache_sources.py +0 -21
  116. {hestia_earth_models-0.64.13.dist-info → hestia_earth_models-0.65.0.dist-info}/LICENSE +0 -0
  117. {hestia_earth_models-0.64.13.dist-info → hestia_earth_models-0.65.0.dist-info}/WHEEL +0 -0
  118. {hestia_earth_models-0.64.13.dist-info → hestia_earth_models-0.65.0.dist-info}/top_level.txt +0 -0
@@ -108,7 +108,7 @@ def _should_run(cycle: dict):
108
108
  has_operations = len(operations) > 0
109
109
 
110
110
  logRequirements(cycle, model=MODEL, model_key=MODEL_KEY,
111
- is_incomplete=is_incomplete,
111
+ is_term_type_electricityFuel_incomplete=is_incomplete,
112
112
  has_operations=has_operations,
113
113
  operations=';'.join(non_empty_list(map(lambda v: v.get('term', {}).get('@id'), operations))))
114
114
 
@@ -46,13 +46,13 @@ def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors:
46
46
  'ee_type': 'raster',
47
47
  'collections': rasters,
48
48
  param_type.value: param_values
49
- })
49
+ }) if rasters else []
50
50
 
51
51
  vector_results = _run_query({
52
52
  'ee_type': 'vector',
53
53
  'collections': vectors,
54
54
  param_type.value: param_values
55
- })
55
+ }) if vectors else []
56
56
 
57
57
  def _process_site(site_values: tuple):
58
58
  site, area_size = site_values
@@ -114,7 +114,7 @@ def _group_sites(sites: dict, check_has_cache: bool = True):
114
114
  }
115
115
 
116
116
 
117
- def _run(sites: list, years: list, include_region: bool, years_only: bool = False):
117
+ def _run(sites: list, years: list = [], include_region: bool = False, years_only: bool = False):
118
118
  rasters, vectors = list_collections(years, include_region, years_only)
119
119
  filtered_data = _group_sites(sites, not years_only)
120
120
  return flatten([
@@ -123,15 +123,6 @@ def _run(sites: list, years: list, include_region: bool, years_only: bool = Fals
123
123
  ])
124
124
 
125
125
 
126
- def _group_years(years: list, years_range: int):
127
- batches = sorted(list(set(list(range(years[0], years[-1] + 1, years_range)) + [years[0], years[-1]])))
128
- grouped_batches = [batches[i:i+2] for i in range(0, len(batches))]
129
- return [
130
- # make sure we don't overlap
131
- [v[0] + (0 if v[0] == years[0] else 1), v[1]] for v in grouped_batches if len(v) == 2
132
- ]
133
-
134
-
135
126
  def run(sites: list, years: list = None, include_region: bool = False):
136
127
  """
137
128
  Run all queries at once for the list of provided Sites.
@@ -147,15 +138,15 @@ def run(sites: list, years: list = None, include_region: bool = False):
147
138
  Prefecth region IDs.
148
139
  This will cache region-level data and will make the request slower. Only use if needed.
149
140
  """
150
- try:
151
- return _run(sites, years, include_region)
152
- except Exception as e:
153
- # when querying with multiple years, we can reach a compute memory limit, so run the years separately
154
- if str(e) == 'User memory limit exceeded.' and years:
155
- sites = _run(sites, [], include_region)
156
- # query for subranges
157
- for sub_years in _group_years(years, years_range=5):
158
- sites = _run(sites, sub_years, include_region, years_only=True)
159
- return sites
160
-
161
- return []
141
+ sites = _run(sites, include_region=include_region)
142
+
143
+ # avoid memory limit errors by running only a few years at a time
144
+ unique_years = sorted(list(set(years)))
145
+ batch_size = 5
146
+ batches = range(0, len(unique_years), batch_size)
147
+
148
+ for batch_index in batches:
149
+ sub_years = unique_years[batch_index:batch_index + batch_size]
150
+ sites = _run(sites, sub_years, include_region, years_only=True)
151
+
152
+ return sites
@@ -29,8 +29,7 @@ REQUIREMENTS = {
29
29
  },
30
30
  "optional": {
31
31
  "emissionsResourceUse": [
32
- {"@type": "Indicator", "value": "", "term.@id": "landTransformationFromForest20YearAverageDuringCycle"},
33
- {"@type": "Indicator", "value": "", "term.@id": "landTransformationFromOtherNaturalVegetation20YearAverageDuringCycle"} # noqa: E501
32
+ {"@type": "Indicator", "value": "", "term.@id": "landTransformation20YearAverageDuringCycle"}
34
33
  ]
35
34
  }
36
35
  }
@@ -46,11 +45,9 @@ LOOKUPS = {
46
45
  "region-siteType-LandTransformationChaudaryBrooks2018CF": "using `country`"
47
46
  }
48
47
  TERM_ID = 'damageToTerrestrialEcosystemsLandTransformation'
49
- LOOKUP_SUFFIX = 'LandTransformationChaudaryBrooks2018CF'
50
- TRANSFORMATION_TERM_IDS = [
51
- 'landTransformationFromForest20YearAverageDuringCycle',
52
- 'landTransformationFromOtherNaturalVegetation20YearAverageDuringCycle'
53
- ]
48
+
49
+ _LOOKUP_SUFFIX = 'LandTransformationChaudaryBrooks2018CF'
50
+ _TRANSFORMATION_TERM_ID = 'landTransformation20YearAverageDuringCycle'
54
51
 
55
52
 
56
53
  def _indicator(value: float):
@@ -68,8 +65,8 @@ def _value(impact_assessment: dict, term_id: str):
68
65
  def _run(impact_assessment: dict):
69
66
  cycle = impact_assessment.get('cycle', {})
70
67
  product = get_product(impact_assessment)
71
- landTransformation = sum_values([_value(impact_assessment, term_id) for term_id in TRANSFORMATION_TERM_IDS])
72
- region_factor = get_region_factor(TERM_ID, impact_assessment, LOOKUP_SUFFIX, 'medium_intensity')
68
+ landTransformation = _value(impact_assessment, _TRANSFORMATION_TERM_ID)
69
+ region_factor = get_region_factor(TERM_ID, impact_assessment, _LOOKUP_SUFFIX, 'medium_intensity')
73
70
  inputs_value = convert_value_from_cycle(
74
71
  product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
75
72
  )
@@ -5,12 +5,20 @@ This model adds `impactAssessment` to inputs based on data which has been aggreg
5
5
  Note: to get more accurate impacts, we recommend setting the
6
6
  [input.impactAssessment](https://hestia.earth/schema/Input#impactAssessment)
7
7
  instead of the region-level averages using this model.
8
+
9
+ For `seed` inputs, we will match ImpactAssessment in the following order of priority:
10
+ - match using the value from the lookup `linkedImpactAssessmentTermId`;
11
+ - match using the primary crop product;
12
+ - match using the "generic" crop product term.
8
13
  """
9
14
  from hestia_earth.schema import TermTermType
10
- from hestia_earth.utils.model import find_primary_product, find_term_match, linked_node
15
+ from hestia_earth.utils.api import download_hestia
16
+ from hestia_earth.utils.model import find_primary_product, linked_node, filter_list_term_type
17
+ from hestia_earth.utils.tools import non_empty_list
11
18
 
12
19
  from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
13
20
  from hestia_earth.models.utils.crop import valid_site_type
21
+ from hestia_earth.models.utils.blank_node import get_lookup_value
14
22
  from hestia_earth.models.utils.term import get_generic_crop
15
23
  from hestia_earth.models.utils.aggregated import (
16
24
  should_link_input_to_impact, link_inputs_to_impact, find_closest_impact, aggregated_end_date
@@ -55,28 +63,35 @@ RETURNS = {
55
63
  "impactAssessmentIsProxy": "True"
56
64
  }]
57
65
  }
66
+ LOOKUPS = {
67
+ "seed": "linkedImpactAssessmentTermId"
68
+ }
58
69
  MODEL_ID = 'hestiaAggregatedData'
59
70
  MODEL_KEY = 'impactAssessment'
60
- SEED_TERM_ID = 'seed'
61
71
 
62
72
 
63
- def _run_seed(cycle: dict, primary_product: dict, seed_input: dict):
73
+ def _run_seed(cycle: dict, primary_product: dict, seed_input: dict, product_term_id: str):
74
+ product = download_hestia(product_term_id)
64
75
  region = seed_input.get('region')
65
76
  country = seed_input.get('country')
66
77
  # to avoid double counting seed => aggregated impact => seed, we need to get the impact of the previous decade
67
78
  # if the data does not exist, use the aggregated impact of generic crop instead
68
79
  date = aggregated_end_date(cycle.get('endDate'))
69
- impact = find_closest_impact(cycle, date, primary_product, region, country, [
80
+ match_end_date = [
70
81
  {'match': {'endDate': date - 10}}
71
- ]) or find_closest_impact(cycle, date, {'term': get_generic_crop()}, region, country)
82
+ ]
83
+
84
+ impact = find_closest_impact(cycle, date, {'term': product}, region, country, match_end_date) or \
85
+ find_closest_impact(cycle, date, primary_product, region, country, match_end_date) or \
86
+ find_closest_impact(cycle, date, {'term': get_generic_crop()}, region, country)
72
87
 
73
- debugValues(cycle, model=MODEL_ID, term=SEED_TERM_ID, key=MODEL_KEY,
88
+ debugValues(cycle, model=MODEL_ID, term=seed_input.get('term', {}).get('@id'), key=MODEL_KEY,
74
89
  input_region=(region or {}).get('@id'),
75
90
  input_country=(country or {}).get('@id'),
76
91
  date=date,
77
92
  impact=(impact or {}).get('@id'))
78
93
 
79
- return [{**seed_input, MODEL_KEY: linked_node(impact), 'impactAssessmentIsProxy': True}] if impact else []
94
+ return seed_input | {MODEL_KEY: linked_node(impact), 'impactAssessmentIsProxy': True} if impact else None
80
95
 
81
96
 
82
97
  def _should_run_seed(cycle: dict):
@@ -84,27 +99,33 @@ def _should_run_seed(cycle: dict):
84
99
  product_id = primary_product.get('term', {}).get('@id')
85
100
  term_type = primary_product.get('term', {}).get('termType')
86
101
  is_crop_product = term_type == TermTermType.CROP.value
87
- input = find_term_match(cycle.get('inputs', []), SEED_TERM_ID, None)
88
- has_input = input is not None
89
102
  site_type_valid = valid_site_type(cycle, True)
90
103
 
91
- should_run = all([site_type_valid, is_crop_product, has_input])
104
+ seed_inputs = filter_list_term_type(cycle.get('inputs', []), TermTermType.SEED)
105
+ seed_inputs = [
106
+ {
107
+ 'input': seed_input,
108
+ 'product': get_lookup_value(seed_input.get('term', {}), LOOKUPS['seed'], key=MODEL_KEY)
109
+ }
110
+ for seed_input in seed_inputs
111
+ ]
112
+
113
+ should_run = all([site_type_valid, is_crop_product, bool(seed_inputs)])
92
114
 
93
- # ignore logs if seed is not present
94
- if has_input:
95
- debugValues(cycle, model=MODEL_ID, term=SEED_TERM_ID, key=MODEL_KEY,
96
- primary_product_id=product_id,
97
- primary_product_term_type=term_type)
115
+ for seed_input in seed_inputs:
116
+ term_id = seed_input.get('input').get('term', {}).get('@id')
117
+ linked_product_id = seed_input.get('product')
98
118
 
99
- logRequirements(cycle, model=MODEL_ID, term=SEED_TERM_ID, key=MODEL_KEY,
119
+ logRequirements(cycle, model=MODEL_ID, term=term_id, key=MODEL_KEY,
100
120
  site_type_valid=site_type_valid,
101
121
  is_crop_product=is_crop_product,
102
- has_input=has_input)
122
+ primary_product_id=product_id,
123
+ linked_product_id=linked_product_id)
103
124
 
104
- logShouldRun(cycle, MODEL_ID, SEED_TERM_ID, should_run)
105
- logShouldRun(cycle, MODEL_ID, SEED_TERM_ID, should_run, key=MODEL_KEY) # show specifically under Input
125
+ logShouldRun(cycle, MODEL_ID, term_id, should_run)
126
+ logShouldRun(cycle, MODEL_ID, term_id, should_run, key=MODEL_KEY) # show specifically under Input
106
127
 
107
- return should_run, primary_product, input
128
+ return should_run, primary_product, seed_inputs
108
129
 
109
130
 
110
131
  def _should_run(cycle: dict):
@@ -124,9 +145,12 @@ def _should_run(cycle: dict):
124
145
 
125
146
  def run(cycle: dict):
126
147
  should_run, inputs = _should_run(cycle)
127
- should_run_seed, primary_product, seed_input = _should_run_seed(cycle)
148
+ should_run_seed, primary_product, seed_inputs = _should_run_seed(cycle)
128
149
  return (
129
150
  link_inputs_to_impact(MODEL_ID, cycle, inputs) if should_run else []
130
151
  ) + (
131
- _run_seed(cycle, primary_product, seed_input) if should_run_seed else []
152
+ non_empty_list([
153
+ _run_seed(cycle, primary_product, seed_input.get('input'), seed_input.get('product'))
154
+ for seed_input in seed_inputs
155
+ ]) if should_run_seed else []
132
156
  )
@@ -0,0 +1,158 @@
1
+ """
2
+ Material and Substrate
3
+
4
+ This model gap-fills depreciated amount per Cycle from Site Infrastructure node.
5
+ """
6
+ from typing import Union
7
+ from hestia_earth.schema import TermTermType
8
+ from hestia_earth.utils.lookup import download_lookup
9
+ from hestia_earth.utils.tools import to_precision, flatten, list_sum
10
+ from hestia_earth.utils.model import filter_list_term_type
11
+
12
+ from hestia_earth.models.log import logShouldRun, logRequirements
13
+ from hestia_earth.models.utils.constant import DAYS_IN_YEAR
14
+ from hestia_earth.models.utils.input import _new_input
15
+ from hestia_earth.models.utils.completeness import _is_term_type_incomplete
16
+ from .import MODEL
17
+
18
+ REQUIREMENTS = {
19
+ "Cycle": {
20
+ "completeness.material": "False",
21
+ "cycleDuration": "",
22
+ "site": {
23
+ "@type": "Site",
24
+ "infrastructure": [
25
+ {
26
+ "@type": "Infrastructure",
27
+ "defaultLifespan": "",
28
+ "inputs": [
29
+ {
30
+ "@type": "Input",
31
+ "term.termType": ["material", "substrate"],
32
+ "value": "",
33
+ "lifespan": ""
34
+ }
35
+ ]
36
+ }
37
+ ]
38
+ }
39
+ }
40
+ }
41
+ RETURNS = {
42
+ "Input": [{
43
+ "value": "",
44
+ "min": "",
45
+ "max": "",
46
+ "sd": "",
47
+ "statsDefinition": ""
48
+ }]
49
+ }
50
+ LOOKUPS = {
51
+ "material": ""
52
+ }
53
+ MODEL_KEY = 'materialAndSubstrate'
54
+
55
+ _ID_SUFFIX = "DepreciatedAmountPerCycle"
56
+ _OPTIONAL_VALUES = ["min", "max", "sd"]
57
+ _SIGNIFICANT_DIGITS = 5
58
+
59
+
60
+ def _input(term_id: str, value: float, stats: dict) -> dict:
61
+ node = _new_input(term_id + _ID_SUFFIX)
62
+ node['value'] = [value]
63
+ return node | stats
64
+
65
+
66
+ def _get_value(node: dict, field_name: str) -> float:
67
+ value = node.get(field_name)
68
+ return list_sum(value) if isinstance(value, list) else value
69
+
70
+
71
+ def calculate_value(input_node: dict, field_name: str, cycle_duration: float) -> Union[float, None]:
72
+ lifespan = input_node.get("lifespan")
73
+ value = _get_value(node=input_node, field_name=field_name)
74
+ return (
75
+ to_precision(number=(value / (lifespan * DAYS_IN_YEAR)) * cycle_duration, digits=_SIGNIFICANT_DIGITS)
76
+ if value else None
77
+ )
78
+
79
+
80
+ def _run_input(cycle: dict, input_node: dict) -> dict:
81
+ cycle_duration = cycle.get("cycleDuration")
82
+
83
+ value = calculate_value(
84
+ input_node=input_node,
85
+ field_name="value",
86
+ cycle_duration=cycle_duration
87
+ )
88
+
89
+ optional_gap_filled_values = {
90
+ field_name: [
91
+ calculate_value(input_node=input_node, field_name=field_name, cycle_duration=cycle_duration)
92
+ ]
93
+ for field_name in _OPTIONAL_VALUES if field_name in input_node
94
+ }
95
+ if "statsDefinition" in input_node:
96
+ optional_gap_filled_values["statsDefinition"] = input_node["statsDefinition"]
97
+
98
+ return _input(input_node.get('term', {}).get('@id'), value, optional_gap_filled_values)
99
+
100
+
101
+ def _has_depreciated_term(term: dict):
102
+ lookup = download_lookup(f"{term.get('termType')}.csv")
103
+ return term.get('@id') + _ID_SUFFIX in list(lookup.termid)
104
+
105
+
106
+ def _should_run_input(cycle: dict, input_node: dict) -> bool:
107
+ term = input_node.get('term', {})
108
+ term_id = term.get('@id')
109
+ has_lifespan = input_node.get('lifespan', 0) > 0
110
+ has_valid_value = _get_value(input_node, 'value') > 0
111
+ has_depreciated_term = _has_depreciated_term(term)
112
+
113
+ should_run = all([
114
+ has_depreciated_term,
115
+ has_valid_value,
116
+ has_lifespan
117
+ ])
118
+
119
+ logRequirements(cycle, model=MODEL, term=term_id, model_key=MODEL_KEY,
120
+ has_valid_value=has_valid_value,
121
+ has_lifespan=has_lifespan,
122
+ has_depreciated_term=has_depreciated_term)
123
+ logShouldRun(input_node, MODEL, term_id, should_run, model_key=MODEL_KEY)
124
+ return should_run
125
+
126
+
127
+ def _should_run_infrastructure(cycle: dict, infra_node: dict) -> tuple[bool, list]:
128
+ inputs = filter_list_term_type(infra_node.get('inputs', []), [TermTermType.MATERIAL, TermTermType.SUBSTRATE])
129
+ inputs = [
130
+ i | {
131
+ 'lifespan': i.get('lifespan') or infra_node.get('defaultLifespan')
132
+ }
133
+ for i in inputs
134
+ ]
135
+ return [i for i in inputs if _should_run_input(cycle, i)]
136
+
137
+
138
+ def _should_run(cycle: dict) -> tuple[bool, list]:
139
+ inputs = flatten([
140
+ _should_run_infrastructure(cycle, i)
141
+ for i in cycle.get('site', {}).get('infrastructure', [])
142
+ ])
143
+ has_material_inputs = len(inputs) > 0
144
+ cycle_duration = cycle.get('cycleDuration')
145
+ is_incomplete = _is_term_type_incomplete(cycle, TermTermType.MATERIAL)
146
+
147
+ logRequirements(cycle, model=MODEL, term=None, model_key=MODEL_KEY,
148
+ term_type_material_incomplete=is_incomplete,
149
+ has_material_inputs=has_material_inputs)
150
+
151
+ should_run = all([is_incomplete, has_material_inputs, cycle_duration])
152
+ logShouldRun(cycle, MODEL, None, should_run, model_key=MODEL_KEY)
153
+ return should_run, inputs
154
+
155
+
156
+ def run(cycle: dict):
157
+ should_run, inputs = _should_run(cycle)
158
+ return [_run_input(cycle, i) for i in inputs] if should_run else []
@@ -1,11 +1,9 @@
1
1
  """
2
2
  Pre Checks Cache Sources
3
3
 
4
- This model caches the sources of all models.
4
+ This model caches the sources of all Cycle models.
5
5
  """
6
- from hestia_earth.models.log import debugValues
7
- from hestia_earth.models.utils import CACHE_KEY, cached_value
8
- from hestia_earth.models.utils.source import CACHE_SOURCES_KEY, find_sources
6
+ from hestia_earth.models.utils.cache_sources import cache_sources
9
7
 
10
8
  REQUIREMENTS = {
11
9
  "Cycle": {}
@@ -15,24 +13,4 @@ RETURNS = {
15
13
  }
16
14
 
17
15
 
18
- def _should_run(site: dict):
19
- sources = find_sources()
20
- has_cache = cached_value(site, CACHE_SOURCES_KEY) is not None
21
-
22
- debugValues(site,
23
- sources=';'.join([str(title) for title in sources.keys()]),
24
- has_cache=has_cache)
25
-
26
- should_run = all([
27
- not has_cache,
28
- len(sources) > 0
29
- ])
30
- return should_run, sources
31
-
32
-
33
- def run(site: dict):
34
- should_run, sources = _should_run(site)
35
- return {
36
- **site,
37
- CACHE_KEY: cached_value(site) | {CACHE_SOURCES_KEY: sources}
38
- } if should_run else site
16
+ def run(cycle: dict): return cache_sources(cycle)
@@ -140,7 +140,7 @@ def _should_run_by_revenue(cycle: dict, products: list):
140
140
  for p in products:
141
141
  term_id = p.get('term', {}).get('@id')
142
142
  logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by=run_by,
143
- is_complete=is_complete,
143
+ is_term_type_product_complete=is_complete,
144
144
  total_economicValueShare=total_economicValueShare,
145
145
  below_threshold=below_threshold,
146
146
  all_with_revenue=all_with_revenue,
@@ -168,7 +168,7 @@ def _should_run_single_missing_evs(cycle: dict, products: list):
168
168
  term_id = missing_values[0].get('term', {}).get('@id') if single_missing_value else None
169
169
 
170
170
  logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by=run_by,
171
- is_complete=is_complete,
171
+ is_term_type_product_complete=is_complete,
172
172
  total_value=total_value,
173
173
  below_threshold=below_threshold,
174
174
  single_missing_value=single_missing_value)
@@ -4,7 +4,7 @@ based on an updated [LANCA model (De Laurentiis et al. 2019)](
4
4
  http://publications.jrc.ec.europa.eu/repository/handle/JRC113865) and on the LANCA (Regionalised) Characterisation
5
5
  Factors version 2.5 (Horn and Meier, 2018).
6
6
  """
7
- from typing import List, Tuple, Optional
7
+ from typing import List, Tuple
8
8
 
9
9
  from hestia_earth.schema import TermTermType
10
10
  from hestia_earth.utils.lookup import download_lookup
@@ -18,18 +18,17 @@ from ..utils.impact_assessment import get_country_id
18
18
  from ..utils.indicator import _new_indicator
19
19
  from ..utils.landCover import get_pef_grouping
20
20
  from ..utils.lookup import fallback_country, _node_value
21
- from ..utils.term import get_land_cover_terms
22
21
 
23
22
  REQUIREMENTS = {
24
23
  "ImpactAssessment": {
25
24
  "emissionsResourceUse": [
26
25
  {
27
26
  "@type": "Indicator",
28
- "term.units": "m2 / year",
29
27
  "term.termType": "resourceUse",
30
- "term.name": "Land transformation from",
28
+ "term.name": "Land transformation",
31
29
  "value": "> 0",
32
- "landCover": {"@type": "Term", "term.termType": "landCover"}
30
+ "landCover": {"@type": "Term", "term.termType": "landCover"},
31
+ "previousLandCover": {"@type": "Term", "term.termType": "landCover"}
33
32
  }
34
33
  ],
35
34
  "optional": {"country": {"@type": "Term", "termType": "region"}}
@@ -73,44 +72,21 @@ def _run(transformations: List[dict]):
73
72
  return _indicator(list_sum(values))
74
73
 
75
74
 
76
- def _extract_land_cover_from_indicator_id(indicator: dict) -> Optional[str]:
77
- """
78
- Given a indicator with term type `resourceUse` return the equivalent `landCover` term
79
- """
80
- term_in_id = indicator.get('term', {}).get('@id', '') \
81
- .removeprefix("landTransformationFrom") \
82
- .removesuffix("20YearAverageInputsProduction") \
83
- .removesuffix("20YearAverageDuringCycle")
84
- term_in_id = term_in_id[0].lower() + term_in_id[1:] if term_in_id else None
85
- return term_in_id
86
-
87
-
88
- def _is_valid_indicator(indicator: dict, land_cover_term_ids: list[str]) -> bool:
89
- term_id = _extract_land_cover_from_indicator_id(indicator)
90
- return term_id in land_cover_term_ids
91
-
92
-
93
75
  def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
94
76
  resource_uses = filter_list_term_type(impact_assessment.get('emissionsResourceUse', []), TermTermType.RESOURCEUSE)
95
- land_cover_term_ids = get_land_cover_terms() if resource_uses else []
96
-
97
- land_transformation_indicators = [i for i in resource_uses if _is_valid_indicator(i, land_cover_term_ids)]
98
-
99
77
  found_transformations = [
100
78
  {
101
79
  'area': _node_value(transformation_indicator) * 20,
102
- 'area-unit': transformation_indicator.get('term', {}).get("units"),
103
- 'land-cover-id-from': _extract_land_cover_from_indicator_id(transformation_indicator),
80
+ 'land-cover-id-from': transformation_indicator.get('previousLandCover', {}).get("@id"),
104
81
  'land-cover-id-to': transformation_indicator.get('landCover', {}).get("@id"),
105
82
  'indicator-id': transformation_indicator.get('term', {}).get('@id', ''),
106
83
  'good-land-cover-term': transformation_indicator.get('landCover', {}).get('termType') == 'landCover',
107
84
  'country-id': get_country_id(impact_assessment),
108
85
  'area-is-valid': _node_value(transformation_indicator) is not None and _node_value(
109
86
  transformation_indicator) > 0,
110
- 'area-unit-is-valid': transformation_indicator.get('term', {}).get("units") == "m2 / year",
111
87
  'lookup-country': fallback_country(get_country_id(impact_assessment),
112
88
  [download_lookup(from_lookup_file), download_lookup(to_lookup_file)]),
113
- } for transformation_indicator in land_transformation_indicators
89
+ } for transformation_indicator in resource_uses
114
90
  ]
115
91
 
116
92
  found_transformations_with_coefficient = [
@@ -134,17 +110,19 @@ def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
134
110
  valid_transformations_with_coef = [
135
111
  t for t in found_transformations_with_coefficient if all([
136
112
  t['area-is-valid'],
137
- t['area-unit-is-valid'],
138
113
  t['factor-from'] is not None,
139
114
  t['factor-to'] is not None
140
115
  ])
141
116
  ]
142
117
 
143
- has_land_transformation_indicators = bool(land_transformation_indicators)
118
+ has_land_transformation_indicators = any([
119
+ indicator.get('term', {}).get('@id').startswith('landTransformation')
120
+ for indicator in resource_uses
121
+ ])
144
122
 
145
123
  all_transformations_are_valid = all(
146
124
  [
147
- all([t['area-is-valid'], t['area-unit-is-valid'], t['good-land-cover-term']])
125
+ all([t['area-is-valid'], t['good-land-cover-term']])
148
126
  for t in found_transformations_with_coefficient
149
127
  ]
150
128
  ) if found_transformations_with_coefficient else False
@@ -0,0 +1,34 @@
1
+ from .utils import should_run_landTransformationFromCropland, run_landTransformationFromCropland
2
+
3
+ REQUIREMENTS = {
4
+ "ImpactAssessment": {
5
+ "endDate": "",
6
+ "country": {"@type": "Term", "termType": "region"},
7
+ "emissionsResourceUse": [{
8
+ "@type": "Indicator",
9
+ "term.@id": "landTransformation100YearAverageDuringCycle",
10
+ "value": "",
11
+ "previousLandCover": {
12
+ "@type": "Term",
13
+ "termType": "landCover",
14
+ "@id": "cropland"
15
+ }
16
+ }]
17
+ }
18
+ }
19
+ LOOKUPS = {
20
+ "region-faostatArea": ""
21
+ }
22
+ RETURNS = {
23
+ "Indicator": [{
24
+ "value": "",
25
+ "landCover": "",
26
+ "previousLandCover": ""
27
+ }]
28
+ }
29
+ TERM_ID = 'landTransformation100YearAverageDuringCycle'
30
+
31
+
32
+ def run(impact: dict):
33
+ should_run, indicators = should_run_landTransformationFromCropland(TERM_ID, impact)
34
+ return run_landTransformationFromCropland(TERM_ID, impact, indicators, 100) if should_run else []
@@ -0,0 +1,34 @@
1
+ from .utils import should_run_landTransformationFromCropland, run_landTransformationFromCropland
2
+
3
+ REQUIREMENTS = {
4
+ "ImpactAssessment": {
5
+ "endDate": "",
6
+ "country": {"@type": "Term", "termType": "region"},
7
+ "emissionsResourceUse": [{
8
+ "@type": "Indicator",
9
+ "term.@id": "landTransformation20YearAverageDuringCycle",
10
+ "value": "",
11
+ "previousLandCover": {
12
+ "@type": "Term",
13
+ "termType": "landCover",
14
+ "@id": "cropland"
15
+ }
16
+ }]
17
+ }
18
+ }
19
+ LOOKUPS = {
20
+ "region-faostatArea": ""
21
+ }
22
+ RETURNS = {
23
+ "Indicator": [{
24
+ "value": "",
25
+ "landCover": "",
26
+ "previousLandCover": ""
27
+ }]
28
+ }
29
+ TERM_ID = 'landTransformation20YearAverageDuringCycle'
30
+
31
+
32
+ def run(impact: dict):
33
+ should_run, indicators = should_run_landTransformationFromCropland(TERM_ID, impact)
34
+ return run_landTransformationFromCropland(TERM_ID, impact, indicators, 20) if should_run else []