hestia-earth-models 0.64.14__py3-none-any.whl → 0.65.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of hestia-earth-models might be problematic. Click here for more details.

Files changed (112) hide show
  1. hestia_earth/models/agribalyse2016/fuelElectricity.py +1 -1
  2. hestia_earth/models/cache_sites.py +15 -24
  3. hestia_earth/models/chaudharyBrooks2018/damageToTerrestrialEcosystemsLandTransformation.py +6 -9
  4. hestia_earth/models/cycle/input/hestiaAggregatedData.py +46 -22
  5. hestia_earth/models/cycle/pre_checks/cache_sources.py +3 -25
  6. hestia_earth/models/cycle/product/economicValueShare.py +2 -2
  7. hestia_earth/models/environmentalFootprintV3/soilQualityIndexLandTransformation.py +11 -33
  8. hestia_earth/models/faostat2018/landTransformation100YearAverageDuringCycle.py +34 -0
  9. hestia_earth/models/faostat2018/landTransformation20YearAverageDuringCycle.py +34 -0
  10. hestia_earth/models/faostat2018/utils.py +47 -3
  11. hestia_earth/models/hestia/landCover.py +5 -5
  12. hestia_earth/models/hestia/seed_emissions.py +275 -0
  13. hestia_earth/models/ipcc2019/aboveGroundBiomass.py +2 -2
  14. hestia_earth/models/ipcc2019/belowGroundBiomass.py +8 -2
  15. hestia_earth/models/ipcc2019/biomass_utils.py +11 -4
  16. hestia_earth/models/ipcc2019/ch4ToAirEntericFermentation.py +19 -10
  17. hestia_earth/models/ipcc2019/co2ToAirAboveGroundBiomassStockChange.py +2 -1
  18. hestia_earth/models/ipcc2019/co2ToAirBelowGroundBiomassStockChange.py +2 -1
  19. hestia_earth/models/ipcc2019/co2ToAirCarbonStockChange_utils.py +8 -7
  20. hestia_earth/models/ipcc2019/co2ToAirSoilOrganicCarbonStockChange.py +2 -1
  21. hestia_earth/models/ipcc2019/organicCarbonPerHa_tier_1_utils.py +28 -34
  22. hestia_earth/models/ipcc2019/organicCarbonPerHa_tier_2_utils.py +8 -12
  23. hestia_earth/models/ipcc2019/organicCarbonPerHa_utils.py +13 -30
  24. hestia_earth/models/linkedImpactAssessment/{landTransformationFromCropland20YearAverageInputsProduction.py → landTransformation100YearAverageInputsProduction.py} +5 -2
  25. hestia_earth/models/linkedImpactAssessment/{landTransformationFromCropland100YearAverageInputsProduction.py → landTransformation20YearAverageInputsProduction.py} +5 -2
  26. hestia_earth/models/linkedImpactAssessment/utils.py +69 -12
  27. hestia_earth/models/mocking/search-results.json +444 -444
  28. hestia_earth/models/pooreNemecek2018/excretaKgN.py +45 -41
  29. hestia_earth/models/pooreNemecek2018/excretaKgVs.py +89 -63
  30. hestia_earth/models/pooreNemecek2018/saplingsDepreciatedAmountPerCycle.py +8 -8
  31. hestia_earth/models/pooreNemecek2018/utils.py +60 -19
  32. hestia_earth/models/schererPfister2015/nErosionSoilFlux.py +4 -3
  33. hestia_earth/models/schererPfister2015/pErosionSoilFlux.py +4 -3
  34. hestia_earth/models/schererPfister2015/utils.py +12 -9
  35. hestia_earth/models/site/management.py +70 -55
  36. hestia_earth/models/site/pre_checks/cache_sources.py +2 -20
  37. hestia_earth/models/utils/__init__.py +12 -1
  38. hestia_earth/models/utils/aggregated.py +1 -1
  39. hestia_earth/models/utils/blank_node.py +20 -12
  40. hestia_earth/models/utils/cache_sources.py +15 -0
  41. hestia_earth/models/utils/crop.py +5 -0
  42. hestia_earth/models/utils/indicator.py +3 -1
  43. hestia_earth/models/version.py +1 -1
  44. {hestia_earth_models-0.64.14.dist-info → hestia_earth_models-0.65.0.dist-info}/METADATA +2 -2
  45. {hestia_earth_models-0.64.14.dist-info → hestia_earth_models-0.65.0.dist-info}/RECORD +75 -104
  46. tests/models/cml2001Baseline/test_abioticResourceDepletionMineralsAndMetals.py +1 -1
  47. tests/models/cycle/input/test_hestiaAggregatedData.py +5 -2
  48. tests/models/environmentalFootprintV3/test_soilQualityIndexLandTransformation.py +39 -28
  49. tests/models/{hyde32/test_landTransformationFromForest20YearAverageDuringCycle.py → faostat2018/test_landTransformation100YearAverageDuringCycle.py} +5 -5
  50. tests/models/{hyde32/test_landTransformationFromForest100YearAverageDuringCycle.py → faostat2018/test_landTransformation20YearAverageDuringCycle.py} +5 -5
  51. tests/models/faostat2018/test_utils.py +28 -0
  52. tests/models/hestia/test_landCover.py +2 -1
  53. tests/models/hestia/test_seed_emissions.py +27 -0
  54. tests/models/ipcc2019/test_aboveGroundBiomass.py +40 -4
  55. tests/models/ipcc2019/test_belowGroundBiomass.py +40 -4
  56. tests/models/ipcc2019/test_co2ToAirAboveGroundBiomassStockChange.py +52 -15
  57. tests/models/ipcc2019/test_co2ToAirBelowGroundBiomassStockChange.py +50 -14
  58. tests/models/ipcc2019/test_co2ToAirSoilOrganicCarbonStockChange.py +53 -32
  59. tests/models/ipcc2019/test_organicCarbonPerHa.py +91 -108
  60. tests/models/ipcc2019/test_organicCarbonPerHa_tier_1_utils.py +33 -50
  61. tests/models/ipcc2019/test_organicCarbonPerHa_tier_2_utils.py +0 -52
  62. tests/models/linkedImpactAssessment/test_freshwaterWithdrawalsInputsProduction.py +6 -4
  63. tests/models/linkedImpactAssessment/test_landOccupationInputsProduction.py +6 -4
  64. tests/models/linkedImpactAssessment/{test_landTransformationFromForest100YearAverageInputsProduction.py → test_landTransformation100YearAverageInputsProduction.py} +7 -5
  65. tests/models/linkedImpactAssessment/{test_landTransformationFromForest20YearAverageInputsProduction.py → test_landTransformation20YearAverageInputsProduction.py} +7 -5
  66. tests/models/pooreNemecek2018/test_excretaKgN.py +2 -2
  67. tests/models/pooreNemecek2018/test_excretaKgVs.py +1 -1
  68. tests/models/pooreNemecek2018/test_utils.py +26 -0
  69. tests/models/site/test_management.py +10 -27
  70. tests/models/test_cache_sites.py +40 -12
  71. tests/models/utils/test_blank_node.py +0 -8
  72. tests/models/utils/test_cache_sources.py +21 -0
  73. hestia_earth/models/blonkConsultants2016/landTransformationFromForest20YearAverageDuringCycle.py +0 -90
  74. hestia_earth/models/faostat2018/landTransformationFromCropland100YearAverage.py +0 -74
  75. hestia_earth/models/faostat2018/landTransformationFromCropland20YearAverage.py +0 -74
  76. hestia_earth/models/hyde32/__init__.py +0 -13
  77. hestia_earth/models/hyde32/landTransformationFromCropland100YearAverageDuringCycle.py +0 -60
  78. hestia_earth/models/hyde32/landTransformationFromCropland20YearAverageDuringCycle.py +0 -60
  79. hestia_earth/models/hyde32/landTransformationFromForest100YearAverageDuringCycle.py +0 -60
  80. hestia_earth/models/hyde32/landTransformationFromForest20YearAverageDuringCycle.py +0 -60
  81. hestia_earth/models/hyde32/landTransformationFromOtherNaturalVegetation100YearAverageDuringCycle.py +0 -61
  82. hestia_earth/models/hyde32/landTransformationFromOtherNaturalVegetation20YearAverageDuringCycle.py +0 -61
  83. hestia_earth/models/hyde32/landTransformationFromPermanentPasture100YearAverageDuringCycle.py +0 -61
  84. hestia_earth/models/hyde32/landTransformationFromPermanentPasture20YearAverageDuringCycle.py +0 -61
  85. hestia_earth/models/hyde32/utils.py +0 -72
  86. hestia_earth/models/linkedImpactAssessment/landTransformationFromForest100YearAverageInputsProduction.py +0 -36
  87. hestia_earth/models/linkedImpactAssessment/landTransformationFromForest20YearAverageInputsProduction.py +0 -36
  88. hestia_earth/models/linkedImpactAssessment/landTransformationFromOtherNaturalVegetation100YearAverageInputsProduction.py +0 -36
  89. hestia_earth/models/linkedImpactAssessment/landTransformationFromOtherNaturalVegetation20YearAverageInputsProduction.py +0 -36
  90. hestia_earth/models/linkedImpactAssessment/landTransformationFromPermanentPasture100YearAverageInputsProduction.py +0 -36
  91. hestia_earth/models/linkedImpactAssessment/landTransformationFromPermanentPasture20YearAverageInputsProduction.py +0 -36
  92. tests/models/blonkConsultants2016/test_landTransformationFromForest20YearAverageDuringCycle.py +0 -36
  93. tests/models/cycle/pre_checks/test_cache_sources.py +0 -25
  94. tests/models/faostat2018/test_landTransformationFromCropland100YearAverage.py +0 -40
  95. tests/models/faostat2018/test_landTransformationFromCropland20YearAverage.py +0 -40
  96. tests/models/hyde32/__init__.py +0 -0
  97. tests/models/hyde32/test_landTransformationFromCropland100YearAverageDuringCycle.py +0 -21
  98. tests/models/hyde32/test_landTransformationFromCropland20YearAverageDuringCycle.py +0 -21
  99. tests/models/hyde32/test_landTransformationFromOtherNaturalVegetation100YearAverageDuringCycle.py +0 -23
  100. tests/models/hyde32/test_landTransformationFromOtherNaturalVegetation20YearAverageDuringCycle.py +0 -21
  101. tests/models/hyde32/test_landTransformationFromPermanentPasture100YearAverageDuringCycle.py +0 -21
  102. tests/models/hyde32/test_landTransformationFromPermanentPasture20YearAverageDuringCycle.py +0 -21
  103. tests/models/linkedImpactAssessment/test_landTransformationFromCropland100YearAverageInputsProduction.py +0 -23
  104. tests/models/linkedImpactAssessment/test_landTransformationFromCropland20YearAverageInputsProduction.py +0 -23
  105. tests/models/linkedImpactAssessment/test_landTransformationFromOtherNaturalVegetation100YearAverageInputsProduction.py +0 -23
  106. tests/models/linkedImpactAssessment/test_landTransformationFromOtherNaturalVegetation20YearAverageInputsProduction.py +0 -23
  107. tests/models/linkedImpactAssessment/test_landTransformationFromPermanentPasture100YearAverageInputsProduction.py +0 -24
  108. tests/models/linkedImpactAssessment/test_landTransformationFromPermanentPasture20YearAverageInputsProduction.py +0 -24
  109. tests/models/site/pre_checks/test_cache_sources.py +0 -21
  110. {hestia_earth_models-0.64.14.dist-info → hestia_earth_models-0.65.0.dist-info}/LICENSE +0 -0
  111. {hestia_earth_models-0.64.14.dist-info → hestia_earth_models-0.65.0.dist-info}/WHEEL +0 -0
  112. {hestia_earth_models-0.64.14.dist-info → hestia_earth_models-0.65.0.dist-info}/top_level.txt +0 -0
@@ -108,7 +108,7 @@ def _should_run(cycle: dict):
108
108
  has_operations = len(operations) > 0
109
109
 
110
110
  logRequirements(cycle, model=MODEL, model_key=MODEL_KEY,
111
- is_incomplete=is_incomplete,
111
+ is_term_type_electricityFuel_incomplete=is_incomplete,
112
112
  has_operations=has_operations,
113
113
  operations=';'.join(non_empty_list(map(lambda v: v.get('term', {}).get('@id'), operations))))
114
114
 
@@ -46,13 +46,13 @@ def _run_values(sites: list, param_type: ParamType, rasters: list = [], vectors:
46
46
  'ee_type': 'raster',
47
47
  'collections': rasters,
48
48
  param_type.value: param_values
49
- })
49
+ }) if rasters else []
50
50
 
51
51
  vector_results = _run_query({
52
52
  'ee_type': 'vector',
53
53
  'collections': vectors,
54
54
  param_type.value: param_values
55
- })
55
+ }) if vectors else []
56
56
 
57
57
  def _process_site(site_values: tuple):
58
58
  site, area_size = site_values
@@ -114,7 +114,7 @@ def _group_sites(sites: dict, check_has_cache: bool = True):
114
114
  }
115
115
 
116
116
 
117
- def _run(sites: list, years: list, include_region: bool, years_only: bool = False):
117
+ def _run(sites: list, years: list = [], include_region: bool = False, years_only: bool = False):
118
118
  rasters, vectors = list_collections(years, include_region, years_only)
119
119
  filtered_data = _group_sites(sites, not years_only)
120
120
  return flatten([
@@ -123,15 +123,6 @@ def _run(sites: list, years: list, include_region: bool, years_only: bool = Fals
123
123
  ])
124
124
 
125
125
 
126
- def _group_years(years: list, years_range: int):
127
- batches = sorted(list(set(list(range(years[0], years[-1] + 1, years_range)) + [years[0], years[-1]])))
128
- grouped_batches = [batches[i:i+2] for i in range(0, len(batches))]
129
- return [
130
- # make sure we don't overlap
131
- [v[0] + (0 if v[0] == years[0] else 1), v[1]] for v in grouped_batches if len(v) == 2
132
- ]
133
-
134
-
135
126
  def run(sites: list, years: list = None, include_region: bool = False):
136
127
  """
137
128
  Run all queries at once for the list of provided Sites.
@@ -147,15 +138,15 @@ def run(sites: list, years: list = None, include_region: bool = False):
147
138
  Prefecth region IDs.
148
139
  This will cache region-level data and will make the request slower. Only use if needed.
149
140
  """
150
- try:
151
- return _run(sites, years, include_region)
152
- except Exception as e:
153
- # when querying with multiple years, we can reach a compute memory limit, so run the years separately
154
- if str(e) == 'User memory limit exceeded.' and years:
155
- sites = _run(sites, [], include_region)
156
- # query for subranges
157
- for sub_years in _group_years(years, years_range=5):
158
- sites = _run(sites, sub_years, include_region, years_only=True)
159
- return sites
160
-
161
- return []
141
+ sites = _run(sites, include_region=include_region)
142
+
143
+ # avoid memory limit errors by running only a few years at a time
144
+ unique_years = sorted(list(set(years)))
145
+ batch_size = 5
146
+ batches = range(0, len(unique_years), batch_size)
147
+
148
+ for batch_index in batches:
149
+ sub_years = unique_years[batch_index:batch_index + batch_size]
150
+ sites = _run(sites, sub_years, include_region, years_only=True)
151
+
152
+ return sites
@@ -29,8 +29,7 @@ REQUIREMENTS = {
29
29
  },
30
30
  "optional": {
31
31
  "emissionsResourceUse": [
32
- {"@type": "Indicator", "value": "", "term.@id": "landTransformationFromForest20YearAverageDuringCycle"},
33
- {"@type": "Indicator", "value": "", "term.@id": "landTransformationFromOtherNaturalVegetation20YearAverageDuringCycle"} # noqa: E501
32
+ {"@type": "Indicator", "value": "", "term.@id": "landTransformation20YearAverageDuringCycle"}
34
33
  ]
35
34
  }
36
35
  }
@@ -46,11 +45,9 @@ LOOKUPS = {
46
45
  "region-siteType-LandTransformationChaudaryBrooks2018CF": "using `country`"
47
46
  }
48
47
  TERM_ID = 'damageToTerrestrialEcosystemsLandTransformation'
49
- LOOKUP_SUFFIX = 'LandTransformationChaudaryBrooks2018CF'
50
- TRANSFORMATION_TERM_IDS = [
51
- 'landTransformationFromForest20YearAverageDuringCycle',
52
- 'landTransformationFromOtherNaturalVegetation20YearAverageDuringCycle'
53
- ]
48
+
49
+ _LOOKUP_SUFFIX = 'LandTransformationChaudaryBrooks2018CF'
50
+ _TRANSFORMATION_TERM_ID = 'landTransformation20YearAverageDuringCycle'
54
51
 
55
52
 
56
53
  def _indicator(value: float):
@@ -68,8 +65,8 @@ def _value(impact_assessment: dict, term_id: str):
68
65
  def _run(impact_assessment: dict):
69
66
  cycle = impact_assessment.get('cycle', {})
70
67
  product = get_product(impact_assessment)
71
- landTransformation = sum_values([_value(impact_assessment, term_id) for term_id in TRANSFORMATION_TERM_IDS])
72
- region_factor = get_region_factor(TERM_ID, impact_assessment, LOOKUP_SUFFIX, 'medium_intensity')
68
+ landTransformation = _value(impact_assessment, _TRANSFORMATION_TERM_ID)
69
+ region_factor = get_region_factor(TERM_ID, impact_assessment, _LOOKUP_SUFFIX, 'medium_intensity')
73
70
  inputs_value = convert_value_from_cycle(
74
71
  product, sum_input_impacts(cycle.get('inputs', []), TERM_ID), model=MODEL, term_id=TERM_ID
75
72
  )
@@ -5,12 +5,20 @@ This model adds `impactAssessment` to inputs based on data which has been aggreg
5
5
  Note: to get more accurate impacts, we recommend setting the
6
6
  [input.impactAssessment](https://hestia.earth/schema/Input#impactAssessment)
7
7
  instead of the region-level averages using this model.
8
+
9
+ For `seed` inputs, we will match ImpactAssessment in the following order of priority:
10
+ - match using the value from the lookup `linkedImpactAssessmentTermId`;
11
+ - match using the primary crop product;
12
+ - match using the "generic" crop product term.
8
13
  """
9
14
  from hestia_earth.schema import TermTermType
10
- from hestia_earth.utils.model import find_primary_product, find_term_match, linked_node
15
+ from hestia_earth.utils.api import download_hestia
16
+ from hestia_earth.utils.model import find_primary_product, linked_node, filter_list_term_type
17
+ from hestia_earth.utils.tools import non_empty_list
11
18
 
12
19
  from hestia_earth.models.log import debugValues, logRequirements, logShouldRun
13
20
  from hestia_earth.models.utils.crop import valid_site_type
21
+ from hestia_earth.models.utils.blank_node import get_lookup_value
14
22
  from hestia_earth.models.utils.term import get_generic_crop
15
23
  from hestia_earth.models.utils.aggregated import (
16
24
  should_link_input_to_impact, link_inputs_to_impact, find_closest_impact, aggregated_end_date
@@ -55,28 +63,35 @@ RETURNS = {
55
63
  "impactAssessmentIsProxy": "True"
56
64
  }]
57
65
  }
66
+ LOOKUPS = {
67
+ "seed": "linkedImpactAssessmentTermId"
68
+ }
58
69
  MODEL_ID = 'hestiaAggregatedData'
59
70
  MODEL_KEY = 'impactAssessment'
60
- SEED_TERM_ID = 'seed'
61
71
 
62
72
 
63
- def _run_seed(cycle: dict, primary_product: dict, seed_input: dict):
73
+ def _run_seed(cycle: dict, primary_product: dict, seed_input: dict, product_term_id: str):
74
+ product = download_hestia(product_term_id)
64
75
  region = seed_input.get('region')
65
76
  country = seed_input.get('country')
66
77
  # to avoid double counting seed => aggregated impact => seed, we need to get the impact of the previous decade
67
78
  # if the data does not exist, use the aggregated impact of generic crop instead
68
79
  date = aggregated_end_date(cycle.get('endDate'))
69
- impact = find_closest_impact(cycle, date, primary_product, region, country, [
80
+ match_end_date = [
70
81
  {'match': {'endDate': date - 10}}
71
- ]) or find_closest_impact(cycle, date, {'term': get_generic_crop()}, region, country)
82
+ ]
83
+
84
+ impact = find_closest_impact(cycle, date, {'term': product}, region, country, match_end_date) or \
85
+ find_closest_impact(cycle, date, primary_product, region, country, match_end_date) or \
86
+ find_closest_impact(cycle, date, {'term': get_generic_crop()}, region, country)
72
87
 
73
- debugValues(cycle, model=MODEL_ID, term=SEED_TERM_ID, key=MODEL_KEY,
88
+ debugValues(cycle, model=MODEL_ID, term=seed_input.get('term', {}).get('@id'), key=MODEL_KEY,
74
89
  input_region=(region or {}).get('@id'),
75
90
  input_country=(country or {}).get('@id'),
76
91
  date=date,
77
92
  impact=(impact or {}).get('@id'))
78
93
 
79
- return [{**seed_input, MODEL_KEY: linked_node(impact), 'impactAssessmentIsProxy': True}] if impact else []
94
+ return seed_input | {MODEL_KEY: linked_node(impact), 'impactAssessmentIsProxy': True} if impact else None
80
95
 
81
96
 
82
97
  def _should_run_seed(cycle: dict):
@@ -84,27 +99,33 @@ def _should_run_seed(cycle: dict):
84
99
  product_id = primary_product.get('term', {}).get('@id')
85
100
  term_type = primary_product.get('term', {}).get('termType')
86
101
  is_crop_product = term_type == TermTermType.CROP.value
87
- input = find_term_match(cycle.get('inputs', []), SEED_TERM_ID, None)
88
- has_input = input is not None
89
102
  site_type_valid = valid_site_type(cycle, True)
90
103
 
91
- should_run = all([site_type_valid, is_crop_product, has_input])
104
+ seed_inputs = filter_list_term_type(cycle.get('inputs', []), TermTermType.SEED)
105
+ seed_inputs = [
106
+ {
107
+ 'input': seed_input,
108
+ 'product': get_lookup_value(seed_input.get('term', {}), LOOKUPS['seed'], key=MODEL_KEY)
109
+ }
110
+ for seed_input in seed_inputs
111
+ ]
112
+
113
+ should_run = all([site_type_valid, is_crop_product, bool(seed_inputs)])
92
114
 
93
- # ignore logs if seed is not present
94
- if has_input:
95
- debugValues(cycle, model=MODEL_ID, term=SEED_TERM_ID, key=MODEL_KEY,
96
- primary_product_id=product_id,
97
- primary_product_term_type=term_type)
115
+ for seed_input in seed_inputs:
116
+ term_id = seed_input.get('input').get('term', {}).get('@id')
117
+ linked_product_id = seed_input.get('product')
98
118
 
99
- logRequirements(cycle, model=MODEL_ID, term=SEED_TERM_ID, key=MODEL_KEY,
119
+ logRequirements(cycle, model=MODEL_ID, term=term_id, key=MODEL_KEY,
100
120
  site_type_valid=site_type_valid,
101
121
  is_crop_product=is_crop_product,
102
- has_input=has_input)
122
+ primary_product_id=product_id,
123
+ linked_product_id=linked_product_id)
103
124
 
104
- logShouldRun(cycle, MODEL_ID, SEED_TERM_ID, should_run)
105
- logShouldRun(cycle, MODEL_ID, SEED_TERM_ID, should_run, key=MODEL_KEY) # show specifically under Input
125
+ logShouldRun(cycle, MODEL_ID, term_id, should_run)
126
+ logShouldRun(cycle, MODEL_ID, term_id, should_run, key=MODEL_KEY) # show specifically under Input
106
127
 
107
- return should_run, primary_product, input
128
+ return should_run, primary_product, seed_inputs
108
129
 
109
130
 
110
131
  def _should_run(cycle: dict):
@@ -124,9 +145,12 @@ def _should_run(cycle: dict):
124
145
 
125
146
  def run(cycle: dict):
126
147
  should_run, inputs = _should_run(cycle)
127
- should_run_seed, primary_product, seed_input = _should_run_seed(cycle)
148
+ should_run_seed, primary_product, seed_inputs = _should_run_seed(cycle)
128
149
  return (
129
150
  link_inputs_to_impact(MODEL_ID, cycle, inputs) if should_run else []
130
151
  ) + (
131
- _run_seed(cycle, primary_product, seed_input) if should_run_seed else []
152
+ non_empty_list([
153
+ _run_seed(cycle, primary_product, seed_input.get('input'), seed_input.get('product'))
154
+ for seed_input in seed_inputs
155
+ ]) if should_run_seed else []
132
156
  )
@@ -1,11 +1,9 @@
1
1
  """
2
2
  Pre Checks Cache Sources
3
3
 
4
- This model caches the sources of all models.
4
+ This model caches the sources of all Cycle models.
5
5
  """
6
- from hestia_earth.models.log import debugValues
7
- from hestia_earth.models.utils import CACHE_KEY, cached_value
8
- from hestia_earth.models.utils.source import CACHE_SOURCES_KEY, find_sources
6
+ from hestia_earth.models.utils.cache_sources import cache_sources
9
7
 
10
8
  REQUIREMENTS = {
11
9
  "Cycle": {}
@@ -15,24 +13,4 @@ RETURNS = {
15
13
  }
16
14
 
17
15
 
18
- def _should_run(site: dict):
19
- sources = find_sources()
20
- has_cache = cached_value(site, CACHE_SOURCES_KEY) is not None
21
-
22
- debugValues(site,
23
- sources=';'.join([str(title) for title in sources.keys()]),
24
- has_cache=has_cache)
25
-
26
- should_run = all([
27
- not has_cache,
28
- len(sources) > 0
29
- ])
30
- return should_run, sources
31
-
32
-
33
- def run(site: dict):
34
- should_run, sources = _should_run(site)
35
- return {
36
- **site,
37
- CACHE_KEY: cached_value(site) | {CACHE_SOURCES_KEY: sources}
38
- } if should_run else site
16
+ def run(cycle: dict): return cache_sources(cycle)
@@ -140,7 +140,7 @@ def _should_run_by_revenue(cycle: dict, products: list):
140
140
  for p in products:
141
141
  term_id = p.get('term', {}).get('@id')
142
142
  logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by=run_by,
143
- is_complete=is_complete,
143
+ is_term_type_product_complete=is_complete,
144
144
  total_economicValueShare=total_economicValueShare,
145
145
  below_threshold=below_threshold,
146
146
  all_with_revenue=all_with_revenue,
@@ -168,7 +168,7 @@ def _should_run_single_missing_evs(cycle: dict, products: list):
168
168
  term_id = missing_values[0].get('term', {}).get('@id') if single_missing_value else None
169
169
 
170
170
  logRequirements(cycle, model=MODEL, term=term_id, key=MODEL_KEY, by=run_by,
171
- is_complete=is_complete,
171
+ is_term_type_product_complete=is_complete,
172
172
  total_value=total_value,
173
173
  below_threshold=below_threshold,
174
174
  single_missing_value=single_missing_value)
@@ -4,7 +4,7 @@ based on an updated [LANCA model (De Laurentiis et al. 2019)](
4
4
  http://publications.jrc.ec.europa.eu/repository/handle/JRC113865) and on the LANCA (Regionalised) Characterisation
5
5
  Factors version 2.5 (Horn and Meier, 2018).
6
6
  """
7
- from typing import List, Tuple, Optional
7
+ from typing import List, Tuple
8
8
 
9
9
  from hestia_earth.schema import TermTermType
10
10
  from hestia_earth.utils.lookup import download_lookup
@@ -18,18 +18,17 @@ from ..utils.impact_assessment import get_country_id
18
18
  from ..utils.indicator import _new_indicator
19
19
  from ..utils.landCover import get_pef_grouping
20
20
  from ..utils.lookup import fallback_country, _node_value
21
- from ..utils.term import get_land_cover_terms
22
21
 
23
22
  REQUIREMENTS = {
24
23
  "ImpactAssessment": {
25
24
  "emissionsResourceUse": [
26
25
  {
27
26
  "@type": "Indicator",
28
- "term.units": "m2 / year",
29
27
  "term.termType": "resourceUse",
30
- "term.name": "Land transformation from",
28
+ "term.name": "Land transformation",
31
29
  "value": "> 0",
32
- "landCover": {"@type": "Term", "term.termType": "landCover"}
30
+ "landCover": {"@type": "Term", "term.termType": "landCover"},
31
+ "previousLandCover": {"@type": "Term", "term.termType": "landCover"}
33
32
  }
34
33
  ],
35
34
  "optional": {"country": {"@type": "Term", "termType": "region"}}
@@ -73,44 +72,21 @@ def _run(transformations: List[dict]):
73
72
  return _indicator(list_sum(values))
74
73
 
75
74
 
76
- def _extract_land_cover_from_indicator_id(indicator: dict) -> Optional[str]:
77
- """
78
- Given a indicator with term type `resourceUse` return the equivalent `landCover` term
79
- """
80
- term_in_id = indicator.get('term', {}).get('@id', '') \
81
- .removeprefix("landTransformationFrom") \
82
- .removesuffix("20YearAverageInputsProduction") \
83
- .removesuffix("20YearAverageDuringCycle")
84
- term_in_id = term_in_id[0].lower() + term_in_id[1:] if term_in_id else None
85
- return term_in_id
86
-
87
-
88
- def _is_valid_indicator(indicator: dict, land_cover_term_ids: list[str]) -> bool:
89
- term_id = _extract_land_cover_from_indicator_id(indicator)
90
- return term_id in land_cover_term_ids
91
-
92
-
93
75
  def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
94
76
  resource_uses = filter_list_term_type(impact_assessment.get('emissionsResourceUse', []), TermTermType.RESOURCEUSE)
95
- land_cover_term_ids = get_land_cover_terms() if resource_uses else []
96
-
97
- land_transformation_indicators = [i for i in resource_uses if _is_valid_indicator(i, land_cover_term_ids)]
98
-
99
77
  found_transformations = [
100
78
  {
101
79
  'area': _node_value(transformation_indicator) * 20,
102
- 'area-unit': transformation_indicator.get('term', {}).get("units"),
103
- 'land-cover-id-from': _extract_land_cover_from_indicator_id(transformation_indicator),
80
+ 'land-cover-id-from': transformation_indicator.get('previousLandCover', {}).get("@id"),
104
81
  'land-cover-id-to': transformation_indicator.get('landCover', {}).get("@id"),
105
82
  'indicator-id': transformation_indicator.get('term', {}).get('@id', ''),
106
83
  'good-land-cover-term': transformation_indicator.get('landCover', {}).get('termType') == 'landCover',
107
84
  'country-id': get_country_id(impact_assessment),
108
85
  'area-is-valid': _node_value(transformation_indicator) is not None and _node_value(
109
86
  transformation_indicator) > 0,
110
- 'area-unit-is-valid': transformation_indicator.get('term', {}).get("units") == "m2 / year",
111
87
  'lookup-country': fallback_country(get_country_id(impact_assessment),
112
88
  [download_lookup(from_lookup_file), download_lookup(to_lookup_file)]),
113
- } for transformation_indicator in land_transformation_indicators
89
+ } for transformation_indicator in resource_uses
114
90
  ]
115
91
 
116
92
  found_transformations_with_coefficient = [
@@ -134,17 +110,19 @@ def _should_run(impact_assessment: dict) -> Tuple[bool, list]:
134
110
  valid_transformations_with_coef = [
135
111
  t for t in found_transformations_with_coefficient if all([
136
112
  t['area-is-valid'],
137
- t['area-unit-is-valid'],
138
113
  t['factor-from'] is not None,
139
114
  t['factor-to'] is not None
140
115
  ])
141
116
  ]
142
117
 
143
- has_land_transformation_indicators = bool(land_transformation_indicators)
118
+ has_land_transformation_indicators = any([
119
+ indicator.get('term', {}).get('@id').startswith('landTransformation')
120
+ for indicator in resource_uses
121
+ ])
144
122
 
145
123
  all_transformations_are_valid = all(
146
124
  [
147
- all([t['area-is-valid'], t['area-unit-is-valid'], t['good-land-cover-term']])
125
+ all([t['area-is-valid'], t['good-land-cover-term']])
148
126
  for t in found_transformations_with_coefficient
149
127
  ]
150
128
  ) if found_transformations_with_coefficient else False
@@ -0,0 +1,34 @@
1
+ from .utils import should_run_landTransformationFromCropland, run_landTransformationFromCropland
2
+
3
+ REQUIREMENTS = {
4
+ "ImpactAssessment": {
5
+ "endDate": "",
6
+ "country": {"@type": "Term", "termType": "region"},
7
+ "emissionsResourceUse": [{
8
+ "@type": "Indicator",
9
+ "term.@id": "landTransformation100YearAverageDuringCycle",
10
+ "value": "",
11
+ "previousLandCover": {
12
+ "@type": "Term",
13
+ "termType": "landCover",
14
+ "@id": "cropland"
15
+ }
16
+ }]
17
+ }
18
+ }
19
+ LOOKUPS = {
20
+ "region-faostatArea": ""
21
+ }
22
+ RETURNS = {
23
+ "Indicator": [{
24
+ "value": "",
25
+ "landCover": "",
26
+ "previousLandCover": ""
27
+ }]
28
+ }
29
+ TERM_ID = 'landTransformation100YearAverageDuringCycle'
30
+
31
+
32
+ def run(impact: dict):
33
+ should_run, indicators = should_run_landTransformationFromCropland(TERM_ID, impact)
34
+ return run_landTransformationFromCropland(TERM_ID, impact, indicators, 100) if should_run else []
@@ -0,0 +1,34 @@
1
+ from .utils import should_run_landTransformationFromCropland, run_landTransformationFromCropland
2
+
3
+ REQUIREMENTS = {
4
+ "ImpactAssessment": {
5
+ "endDate": "",
6
+ "country": {"@type": "Term", "termType": "region"},
7
+ "emissionsResourceUse": [{
8
+ "@type": "Indicator",
9
+ "term.@id": "landTransformation20YearAverageDuringCycle",
10
+ "value": "",
11
+ "previousLandCover": {
12
+ "@type": "Term",
13
+ "termType": "landCover",
14
+ "@id": "cropland"
15
+ }
16
+ }]
17
+ }
18
+ }
19
+ LOOKUPS = {
20
+ "region-faostatArea": ""
21
+ }
22
+ RETURNS = {
23
+ "Indicator": [{
24
+ "value": "",
25
+ "landCover": "",
26
+ "previousLandCover": ""
27
+ }]
28
+ }
29
+ TERM_ID = 'landTransformation20YearAverageDuringCycle'
30
+
31
+
32
+ def run(impact: dict):
33
+ should_run, indicators = should_run_landTransformationFromCropland(TERM_ID, impact)
34
+ return run_landTransformationFromCropland(TERM_ID, impact, indicators, 20) if should_run else []
@@ -1,14 +1,16 @@
1
+ from numpy import recarray
1
2
  from hestia_earth.schema import TermTermType
2
3
  from hestia_earth.utils.api import download_hestia
3
4
  from hestia_earth.utils.lookup import download_lookup, get_table_value, column_name, extract_grouped_data_closest_date
4
- from hestia_earth.utils.tools import safe_parse_float
5
- from numpy import recarray
5
+ from hestia_earth.utils.tools import safe_parse_float, flatten
6
6
 
7
- from hestia_earth.models.log import logger, debugMissingLookup
7
+ from hestia_earth.models.log import logger, debugMissingLookup, logRequirements, logShouldRun, debugValues
8
8
  from hestia_earth.models.utils.animalProduct import (
9
9
  FAO_LOOKUP_COLUMN, FAO_EQUIVALENT_LOOKUP_COLUMN, get_animalProduct_lookup_value
10
10
  )
11
11
  from hestia_earth.models.utils.product import convert_product_to_unit
12
+ from hestia_earth.models.utils.impact_assessment import get_country_id, impact_end_year
13
+ from hestia_earth.models.utils.indicator import _new_indicator
12
14
  from . import MODEL
13
15
 
14
16
  LOOKUP_PREFIX = f"{TermTermType.REGION.value}-{TermTermType.ANIMALPRODUCT.value}-{FAO_LOOKUP_COLUMN}"
@@ -140,3 +142,45 @@ def get_change_in_harvested_area_for_crop(country_id: str, crop_name: str, start
140
142
  return _split_delta(
141
143
  get_table_value(lookup, 'termid', country_id, column_name(crop_name)), start_year, end_year
142
144
  )
145
+
146
+
147
+ def should_run_landTransformationFromCropland(term_id: str, impact: dict):
148
+ indicators = [
149
+ i for i in impact.get('emissionsResourceUse', [])
150
+ if all([
151
+ i.get('term', {}).get('@id') == term_id,
152
+ i.get('previousLandCover', {}).get('@id') == 'cropland',
153
+ i.get('value', -1) > 0
154
+ ])
155
+ ]
156
+ has_cropland = bool(indicators)
157
+
158
+ should_run = all([has_cropland])
159
+ logRequirements(impact, model=MODEL, term=term_id,
160
+ has_cropland_indicators=has_cropland)
161
+ logShouldRun(impact, MODEL, term_id, should_run)
162
+
163
+ return should_run, indicators
164
+
165
+
166
+ def run_landTransformationFromCropland(term_id: str, impact: dict, indicators: list, years: int):
167
+ country_id = get_country_id(impact)
168
+ end_year = impact_end_year(impact)
169
+ total, permanent, temporary = get_cropland_ratio(country_id, end_year - years, end_year)
170
+
171
+ debugValues(impact, model=MODEL, term_id=term_id,
172
+ diff_temporary_area=temporary,
173
+ diff_permanent_area=permanent,
174
+ diff_total_area=total)
175
+
176
+ return flatten([
177
+ [
178
+ _new_indicator(term_id, MODEL, indicator.get('landCover', {}).get('@id'), 'annualCropland') | {
179
+ 'value': indicator.get('value') * temporary / total
180
+ },
181
+ _new_indicator(term_id, MODEL, indicator.get('landCover', {}).get('@id'), 'permanentCropland') | {
182
+ 'value': indicator.get('value') * permanent / total
183
+ }
184
+ ]
185
+ for indicator in indicators
186
+ ]) if total is not None else []
@@ -198,14 +198,14 @@ def _estimate_maximum_forest_change(
198
198
  pasture_change=pasture_change,
199
199
  total_cropland_change=total_cropland_change
200
200
  ) if not positive_change else (
201
- total_agricultural_change
201
+ -total_agricultural_change
202
202
  if -min(forest_change, 0) > total_agricultural_change else
203
203
  min(forest_change, 0)
204
204
  )
205
205
 
206
206
 
207
207
  def _negative_agricultural_land_change(forest_change, pasture_change, total_cropland_change):
208
- return pasture_change if 0 < pasture_change < -min(forest_change, 0) \
208
+ return -pasture_change if 0 < pasture_change < -min(forest_change, 0) \
209
209
  else min(forest_change, 0) if pasture_change > 0 \
210
210
  else -total_cropland_change if 0 < total_cropland_change < -min(forest_change, 0) \
211
211
  else min(forest_change, 0) if 0 < total_cropland_change \
@@ -432,7 +432,7 @@ def _run_make_management_nodes(existing_nodes: list, percentage_transformed_from
432
432
  LAND_USE_TERMS_FOR_TRANSFORMATION[land_type], f"{start_year}-01-01", f"{start_year}-12-31"
433
433
  ),
434
434
  "land_type": land_type,
435
- "percentage": 0.0 if ratio == -0.0 else to_precision(
435
+ "percentage": 0 if ratio == -0.0 else to_precision(
436
436
  number=ratio * 100,
437
437
  digits=OUTPUT_SIGNIFICANT_DIGITS
438
438
  )
@@ -462,7 +462,7 @@ def get_ratio_of_expanded_area(country_id: str, fao_name: str, end_year: int) ->
462
462
  faostat_name=fao_name
463
463
  )
464
464
  return 0.0 if any([expansion is None, end_value is None]) else max(
465
- 0.0, _safe_divide(numerator=expansion, denominator=(end_value - expansion))
465
+ 0.0, _safe_divide(numerator=expansion, denominator=end_value)
466
466
  )
467
467
 
468
468
 
@@ -619,7 +619,7 @@ def _should_run_historical_land_use_change_single_crop(
619
619
  other_land_loss_to_pasture=other_land_loss_to[PERMANENT_PASTURE]
620
620
  )
621
621
 
622
- # BA to BD
622
+ # C14-G14
623
623
  shares_of_expansion = _get_shares_of_expansion(
624
624
  land_use_type=land_use_type,
625
625
  percent_annual_cropland_was=percent_annual_cropland_was,