hestia-earth-models 0.65.10__py3-none-any.whl → 0.66.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. hestia_earth/models/cache_sites.py +7 -9
  2. hestia_earth/models/config/Cycle.json +34 -16
  3. hestia_earth/models/config/ImpactAssessment.json +12 -0
  4. hestia_earth/models/config/Site.json +4 -1
  5. hestia_earth/models/cycle/completeness/freshForage.py +10 -2
  6. hestia_earth/models/cycle/cropResidueManagement.py +3 -1
  7. hestia_earth/models/ecoinventV3/__init__.py +2 -1
  8. hestia_earth/models/environmentalFootprintV3/environmentalFootprintSingleOverallScore.py +135 -0
  9. hestia_earth/models/environmentalFootprintV3/soilQualityIndexLandTransformation.py +17 -6
  10. hestia_earth/models/geospatialDatabase/{aware.py → awareWaterBasinId.py} +1 -1
  11. hestia_earth/models/hestia/landCover.py +57 -39
  12. hestia_earth/models/hestia/residueRemoved.py +80 -0
  13. hestia_earth/models/hestia/resourceUse_utils.py +64 -38
  14. hestia_earth/models/hestia/utils.py +1 -2
  15. hestia_earth/models/ipcc2019/aboveGroundBiomass.py +33 -12
  16. hestia_earth/models/ipcc2019/animal/pastureGrass.py +1 -1
  17. hestia_earth/models/ipcc2019/belowGroundBiomass.py +32 -11
  18. hestia_earth/models/ipcc2019/ch4ToAirEntericFermentation.py +17 -8
  19. hestia_earth/models/ipcc2019/co2ToAirCarbonStockChange_utils.py +5 -3
  20. hestia_earth/models/ipcc2019/organicCarbonPerHa_tier_2_utils.py +27 -17
  21. hestia_earth/models/ipcc2019/pastureGrass.py +1 -1
  22. hestia_earth/models/ipcc2019/pastureGrass_utils.py +8 -1
  23. hestia_earth/models/log.py +1 -1
  24. hestia_earth/models/mocking/search-results.json +34 -34
  25. hestia_earth/models/pooreNemecek2018/freshwaterWithdrawalsDuringCycle.py +0 -1
  26. hestia_earth/models/pooreNemecek2018/landOccupationDuringCycle.py +13 -10
  27. hestia_earth/models/site/defaultMethodClassification.py +9 -2
  28. hestia_earth/models/site/defaultMethodClassificationDescription.py +4 -2
  29. hestia_earth/models/site/management.py +49 -31
  30. hestia_earth/models/site/pre_checks/cache_geospatialDatabase.py +19 -14
  31. hestia_earth/models/utils/blank_node.py +10 -4
  32. hestia_earth/models/utils/crop.py +1 -1
  33. hestia_earth/models/utils/cycle.py +3 -3
  34. hestia_earth/models/utils/lookup.py +1 -1
  35. hestia_earth/models/version.py +1 -1
  36. hestia_earth/orchestrator/strategies/merge/merge_list.py +17 -6
  37. {hestia_earth_models-0.65.10.dist-info → hestia_earth_models-0.66.0.dist-info}/METADATA +1 -1
  38. {hestia_earth_models-0.65.10.dist-info → hestia_earth_models-0.66.0.dist-info}/RECORD +59 -54
  39. tests/models/environmentalFootprintV3/test_environmentalFootprintSingleOverallScore.py +92 -0
  40. tests/models/environmentalFootprintV3/test_soilQualityIndexLandTransformation.py +4 -19
  41. tests/models/faostat2018/product/test_price.py +1 -1
  42. tests/models/geospatialDatabase/{test_aware.py → test_awareWaterBasinId.py} +1 -1
  43. tests/models/hestia/test_landCover.py +4 -2
  44. tests/models/hestia/test_landTransformation20YearAverageDuringCycle.py +3 -1
  45. tests/models/hestia/test_residueRemoved.py +20 -0
  46. tests/models/ipcc2019/test_aboveGroundBiomass.py +3 -1
  47. tests/models/ipcc2019/test_belowGroundBiomass.py +4 -2
  48. tests/models/ipcc2019/test_organicCarbonPerHa.py +94 -1
  49. tests/models/pooreNemecek2018/test_landOccupationDuringCycle.py +1 -3
  50. tests/models/site/pre_checks/test_cache_geospatialDatabase.py +22 -0
  51. tests/models/site/test_defaultMethodClassification.py +6 -0
  52. tests/models/site/test_defaultMethodClassificationDescription.py +6 -0
  53. tests/models/site/test_management.py +4 -4
  54. tests/models/test_cache_sites.py +2 -2
  55. tests/models/utils/test_crop.py +14 -2
  56. tests/orchestrator/strategies/merge/test_merge_list.py +11 -1
  57. {hestia_earth_models-0.65.10.dist-info → hestia_earth_models-0.66.0.dist-info}/LICENSE +0 -0
  58. {hestia_earth_models-0.65.10.dist-info → hestia_earth_models-0.66.0.dist-info}/WHEEL +0 -0
  59. {hestia_earth_models-0.65.10.dist-info → hestia_earth_models-0.66.0.dist-info}/top_level.txt +0 -0
@@ -1768,7 +1768,7 @@
1768
1768
  "@type": "Term",
1769
1769
  "name": "Generic crop, seed",
1770
1770
  "@id": "genericCropSeed",
1771
- "_score": 25.442797
1771
+ "_score": 25.417622
1772
1772
  }
1773
1773
  ]
1774
1774
  },
@@ -2004,157 +2004,157 @@
2004
2004
  "@type": "Term",
2005
2005
  "name": "Glass or high accessible cover",
2006
2006
  "@id": "glassOrHighAccessibleCover",
2007
- "_score": 64.880936
2007
+ "_score": 64.78735
2008
2008
  },
2009
2009
  {
2010
2010
  "@type": "Term",
2011
2011
  "name": "Sea or ocean",
2012
2012
  "@id": "seaOrOcean",
2013
- "_score": 54.485077
2013
+ "_score": 54.49869
2014
2014
  },
2015
2015
  {
2016
2016
  "@type": "Term",
2017
2017
  "name": "River or stream",
2018
2018
  "@id": "riverOrStream",
2019
- "_score": 52.44936
2019
+ "_score": 52.33286
2020
2020
  },
2021
2021
  {
2022
2022
  "@type": "Term",
2023
2023
  "name": "Other natural vegetation",
2024
2024
  "@id": "otherNaturalVegetation",
2025
- "_score": 43.05355
2025
+ "_score": 43.071865
2026
2026
  },
2027
2027
  {
2028
2028
  "@type": "Term",
2029
2029
  "name": "Agri-food processor",
2030
2030
  "@id": "agriFoodProcessor",
2031
- "_score": 42.06176
2031
+ "_score": 41.75925
2032
2032
  },
2033
2033
  {
2034
2034
  "@type": "Term",
2035
2035
  "name": "Food retailer",
2036
2036
  "@id": "foodRetailer",
2037
- "_score": 41.523476
2037
+ "_score": 41.196114
2038
2038
  },
2039
2039
  {
2040
2040
  "@type": "Term",
2041
2041
  "name": "Natural forest",
2042
2042
  "@id": "naturalForest",
2043
- "_score": 32.727547
2043
+ "_score": 32.262592
2044
2044
  },
2045
2045
  {
2046
2046
  "@type": "Term",
2047
2047
  "name": "Permanent pasture",
2048
2048
  "@id": "permanentPasture",
2049
- "_score": 28.69685
2049
+ "_score": 28.761673
2050
2050
  },
2051
2051
  {
2052
2052
  "@type": "Term",
2053
2053
  "name": "Animal housing",
2054
2054
  "@id": "animalHousing",
2055
- "_score": 27.970219
2055
+ "_score": 27.809086
2056
2056
  },
2057
2057
  {
2058
2058
  "@type": "Term",
2059
2059
  "name": "Root or tuber crop plant",
2060
2060
  "@id": "rootOrTuberCropPlant",
2061
- "_score": 27.436356
2061
+ "_score": 27.444334
2062
2062
  },
2063
2063
  {
2064
2064
  "@type": "Term",
2065
2065
  "name": "High intensity grazing pasture",
2066
2066
  "@id": "highIntensityGrazingPasture",
2067
- "_score": 24.657646
2067
+ "_score": 24.399143
2068
2068
  },
2069
2069
  {
2070
2070
  "@type": "Term",
2071
2071
  "name": "Forest",
2072
2072
  "@id": "forest",
2073
- "_score": 20.619314
2073
+ "_score": 20.347794
2074
2074
  },
2075
2075
  {
2076
2076
  "@type": "Term",
2077
2077
  "name": "Permanent cropland",
2078
2078
  "@id": "permanentCropland",
2079
- "_score": 20.092436
2079
+ "_score": 20.196049
2080
2080
  },
2081
2081
  {
2082
2082
  "@type": "Term",
2083
2083
  "name": "Other land",
2084
2084
  "@id": "otherLand",
2085
- "_score": 19.758362
2085
+ "_score": 19.765242
2086
2086
  },
2087
2087
  {
2088
2088
  "@type": "Term",
2089
2089
  "name": "Plantation forest",
2090
2090
  "@id": "plantationForest",
2091
- "_score": 19.308796
2092
- },
2093
- {
2094
- "@type": "Term",
2095
- "name": "Sea kale plant",
2096
- "@id": "seaKalePlant",
2097
- "_score": 18.2738
2091
+ "_score": 19.03003
2098
2092
  },
2099
2093
  {
2100
2094
  "@type": "Term",
2101
2095
  "name": "Lake",
2102
2096
  "@id": "lake",
2103
- "_score": 18.255703
2097
+ "_score": 18.264688
2104
2098
  },
2105
2099
  {
2106
2100
  "@type": "Term",
2107
- "name": "Red sea plume alga",
2108
- "@id": "redSeaPlumeAlga",
2109
- "_score": 18.100435
2101
+ "name": "Sea kale plant",
2102
+ "@id": "seaKalePlant",
2103
+ "_score": 18.135399
2110
2104
  },
2111
2105
  {
2112
2106
  "@type": "Term",
2113
2107
  "name": "Native pasture",
2114
2108
  "@id": "nativePasture",
2115
- "_score": 17.83277
2109
+ "_score": 17.863037
2116
2110
  },
2117
2111
  {
2118
2112
  "@type": "Term",
2119
2113
  "name": "Improved pasture",
2120
2114
  "@id": "improvedPasture",
2121
- "_score": 17.396095
2115
+ "_score": 17.444498
2116
+ },
2117
+ {
2118
+ "@type": "Term",
2119
+ "name": "Red sea plume alga",
2120
+ "@id": "redSeaPlumeAlga",
2121
+ "_score": 17.235117
2122
2122
  },
2123
2123
  {
2124
2124
  "@type": "Term",
2125
2125
  "name": "Nominally managed pasture",
2126
2126
  "@id": "nominallyManagedPasture",
2127
- "_score": 16.762512
2127
+ "_score": 16.79026
2128
2128
  },
2129
2129
  {
2130
2130
  "@type": "Term",
2131
2131
  "name": "Severely degraded pasture",
2132
2132
  "@id": "severelyDegradedPasture",
2133
- "_score": 16.237305
2133
+ "_score": 16.32682
2134
2134
  },
2135
2135
  {
2136
2136
  "@type": "Term",
2137
2137
  "name": "Pond",
2138
2138
  "@id": "pond",
2139
- "_score": 15.637012
2139
+ "_score": 15.64036
2140
2140
  },
2141
2141
  {
2142
2142
  "@type": "Term",
2143
2143
  "name": "River tamarind tree",
2144
2144
  "@id": "riverTamarindTree",
2145
- "_score": 15.435883
2145
+ "_score": 15.445333
2146
2146
  },
2147
2147
  {
2148
2148
  "@type": "Term",
2149
2149
  "name": "Annual cropland",
2150
2150
  "@id": "annualCropland",
2151
- "_score": 9.802788
2151
+ "_score": 9.808704
2152
2152
  },
2153
2153
  {
2154
2154
  "@type": "Term",
2155
2155
  "name": "Cropland",
2156
2156
  "@id": "cropland",
2157
- "_score": 9.768499
2157
+ "_score": 9.772207
2158
2158
  }
2159
2159
  ]
2160
2160
  },
@@ -18,7 +18,6 @@ REQUIREMENTS = {
18
18
  "@type": "Cycle",
19
19
  "products": [{
20
20
  "@type": "Product",
21
- "primary": "True",
22
21
  "value": "> 0",
23
22
  "economicValueShare": "> 0"
24
23
  }],
@@ -2,7 +2,8 @@ from hestia_earth.models.log import logRequirements, logShouldRun
2
2
  from hestia_earth.models.utils.indicator import _new_indicator
3
3
  from hestia_earth.models.utils.impact_assessment import get_product, get_site
4
4
  from hestia_earth.models.utils.cycle import land_occupation_per_kg
5
- from hestia_earth.models.utils.site import get_land_cover_term_id
5
+ from hestia_earth.models.utils.site import get_land_cover_term_id as get_landCover_term_id_from_site_type
6
+ from hestia_earth.models.utils.crop import get_landCover_term_id
6
7
  from . import MODEL
7
8
 
8
9
  REQUIREMENTS = {
@@ -16,7 +17,6 @@ REQUIREMENTS = {
16
17
  "cycleDuration": "",
17
18
  "products": [{
18
19
  "@type": "Product",
19
- "primary": "True",
20
20
  "value": "> 0",
21
21
  "economicValueShare": "> 0"
22
22
  }],
@@ -46,8 +46,8 @@ RETURNS = {
46
46
  TERM_ID = 'landOccupationDuringCycle'
47
47
 
48
48
 
49
- def _indicator(term_id: str, value: float, land_covert_term_id: str):
50
- indicator = _new_indicator(term_id, MODEL, land_covert_term_id)
49
+ def _indicator(term_id: str, value: float, land_cover_term_id: str):
50
+ indicator = _new_indicator(term_id, MODEL, land_cover_term_id)
51
51
  indicator['value'] = value
52
52
  return indicator
53
53
 
@@ -56,18 +56,21 @@ def _should_run(impact_assessment: dict):
56
56
  product = get_product(impact_assessment)
57
57
  cycle = impact_assessment.get('cycle', {})
58
58
  site = get_site(impact_assessment)
59
- land_covert_term_id = get_land_cover_term_id(site.get('siteType'))
59
+ land_cover_from_product = get_landCover_term_id(product.get('term', {}))
60
+ land_cover_from_site = get_landCover_term_id_from_site_type(site.get('siteType'))
61
+ land_cover_term_id = land_cover_from_product or land_cover_from_site
60
62
  land_occupation_m2_kg = land_occupation_per_kg(MODEL, TERM_ID, cycle, site, product)
61
63
 
62
64
  logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
63
65
  land_occupation_kg=land_occupation_m2_kg,
64
- land_covert_term_id=land_covert_term_id)
66
+ land_cover_from_product=land_cover_from_product,
67
+ land_cover_from_site=land_cover_from_site)
65
68
 
66
- should_run = all([land_covert_term_id, land_occupation_m2_kg is not None])
69
+ should_run = all([land_cover_term_id, land_occupation_m2_kg is not None])
67
70
  logShouldRun(impact_assessment, MODEL, TERM_ID, should_run)
68
- return should_run, land_occupation_m2_kg, land_covert_term_id
71
+ return should_run, land_occupation_m2_kg, land_cover_term_id
69
72
 
70
73
 
71
74
  def run(impact_assessment: dict):
72
- should_run, land_occupation_kg, land_covert_term_id = _should_run(impact_assessment)
73
- return [_indicator(TERM_ID, land_occupation_kg, land_covert_term_id)] if should_run else []
75
+ should_run, land_occupation_kg, land_cover_term_id = _should_run(impact_assessment)
76
+ return [_indicator(TERM_ID, land_occupation_kg, land_cover_term_id)] if should_run else []
@@ -5,6 +5,8 @@ When gap-filling `management` node on Site, the
5
5
  `defaultMethodClassification` and `defaultMethodClassificationDescription` fields become required.
6
6
  This model will use the first value in the `management` node.
7
7
  """
8
+ from hestia_earth.schema import SiteDefaultMethodClassification
9
+
8
10
  from hestia_earth.models.log import logRequirements, logShouldRun
9
11
  from . import MODEL
10
12
 
@@ -20,12 +22,17 @@ MODEL_KEY = 'defaultMethodClassification'
20
22
 
21
23
 
22
24
  def _should_run(site: dict):
23
- methodClassification = next((n.get('methodClassification') for n in site.get('management', [])), None)
25
+ has_management = bool(site.get('management', []))
26
+ methodClassification = next(
27
+ (n.get('methodClassification') for n in site.get('management', [])),
28
+ None
29
+ ) or SiteDefaultMethodClassification.MODELLED.value
24
30
 
25
31
  logRequirements(site, model=MODEL, model_key=MODEL_KEY,
32
+ has_management=has_management,
26
33
  methodClassification=methodClassification)
27
34
 
28
- should_run = all([methodClassification])
35
+ should_run = all([has_management, methodClassification])
29
36
  logShouldRun(site, MODEL, None, should_run, model_key=MODEL_KEY)
30
37
  return should_run, methodClassification
31
38
 
@@ -20,16 +20,18 @@ MODEL_KEY = 'defaultMethodClassificationDescription'
20
20
 
21
21
 
22
22
  def _should_run(site: dict):
23
+ has_management = bool(site.get('management', []))
23
24
  methodClassificationDescription = next((
24
25
  n.get('methodClassificationDescription')
25
26
  for n in site.get('management', [])
26
27
  if n.get('methodClassification')
27
- ), None)
28
+ ), None) or 'Data calculated by merging real land use histories and modelled land use histories for each Site.'
28
29
 
29
30
  logRequirements(site, model=MODEL, model_key=MODEL_KEY,
31
+ has_management=has_management,
30
32
  methodClassificationDescription=methodClassificationDescription)
31
33
 
32
- should_run = all([methodClassificationDescription])
34
+ should_run = all([has_management, methodClassificationDescription])
33
35
  logShouldRun(site, MODEL, None, should_run, model_key=MODEL_KEY)
34
36
  return should_run, methodClassificationDescription
35
37
 
@@ -10,7 +10,8 @@ tillage, cropResidueManagement and landUseManagement.
10
10
  All values are copied from the source node, except for crop and forage terms in which case the dates are copied from the
11
11
  cycle.
12
12
 
13
- Where `startDate` is missing from landCover products, gap-filling is attempted using `endDate` - `maximumCycleDuration`.
13
+ Where `startDate` is missing from landCover products, gap-filling is attempted using
14
+ `endDate` - `cycleDuration` (or `maximumCycleDuration` lookup).
14
15
  This is the `endDate` of the `landCover` product.
15
16
  This ensures no overlapping date ranges.
16
17
  If both `endDate` and `startDate` are missing from the product, these will be gap-filled from the `Cycle`.
@@ -18,9 +19,10 @@ If both `endDate` and `startDate` are missing from the product, these will be ga
18
19
  When nodes are chronologically consecutive with "% area" or "boolean" units and the same term and value, they are
19
20
  condensed into a single node to aid readability.
20
21
  """
22
+ from typing import List
21
23
  from datetime import timedelta, datetime
22
24
  from functools import reduce
23
- from hestia_earth.schema import TermTermType, SiteSiteType
25
+ from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType, COMPLETENESS_MAPPING
24
26
  from hestia_earth.utils.lookup import column_name, get_table_value, download_lookup
25
27
  from hestia_earth.utils.model import filter_list_term_type
26
28
  from hestia_earth.utils.tools import safe_parse_float, flatten
@@ -31,18 +33,17 @@ from hestia_earth.models.utils import _include, _omit, group_by
31
33
  from hestia_earth.models.utils.management import _new_management
32
34
  from hestia_earth.models.utils.term import get_lookup_value
33
35
  from hestia_earth.models.utils.blank_node import condense_nodes, DatestrFormat, _gapfill_datestr, DatestrGapfillMode
36
+ from hestia_earth.models.utils.crop import get_landCover_term_id
34
37
  from hestia_earth.models.utils.site import (
35
38
  related_cycles, get_land_cover_term_id as get_landCover_term_id_from_site_type
36
39
  )
37
40
  from . import MODEL
38
- from ..utils.crop import get_landCover_term_id
39
41
 
40
42
  REQUIREMENTS = {
41
43
  "Site": {
42
44
  "related": {
43
45
  "Cycle": [{
44
46
  "@type": "Cycle",
45
- "startDate": "",
46
47
  "endDate": "",
47
48
  "products": [
48
49
  {
@@ -71,7 +72,11 @@ REQUIREMENTS = {
71
72
  "soilAmendment"
72
73
  ]
73
74
  }
74
- ]
75
+ ],
76
+ "optional": {
77
+ "startDate": "",
78
+ "cycleDuration": ""
79
+ }
75
80
  }]
76
81
  }
77
82
  }
@@ -98,6 +103,14 @@ LOOKUPS = {
98
103
  }
99
104
  MODEL_KEY = 'management'
100
105
 
106
+ _PRACTICES_TERM_TYPES = [
107
+ TermTermType.WATERREGIME,
108
+ TermTermType.TILLAGE,
109
+ TermTermType.CROPRESIDUEMANAGEMENT,
110
+ TermTermType.LANDUSEMANAGEMENT,
111
+ TermTermType.SYSTEM
112
+ ]
113
+ _PRACTICES_COMPLETENESS_MAPPING = COMPLETENESS_MAPPING.get(SchemaType.PRACTICE.value)
101
114
  _ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
102
115
  _INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID = "inorganicNitrogenFertiliserUsed"
103
116
  _ORGANIC_FERTILISER_USED_TERM_ID = "organicFertiliserUsed"
@@ -147,11 +160,13 @@ def management(data: dict):
147
160
  return node
148
161
 
149
162
 
150
- def _get_maximum_cycle_duration(land_cover_id: str):
151
- lookup = download_lookup("crop.csv")
152
- return safe_parse_float(
153
- get_table_value(lookup, column_name('landCoverTermId'), land_cover_id, column_name('maximumCycleDuration'))
154
- )
163
+ def _get_cycle_duration(cycle: dict, land_cover_id: str):
164
+ return cycle.get('cycleDuration') or safe_parse_float(get_table_value(
165
+ download_lookup("crop.csv"),
166
+ column_name('landCoverTermId'),
167
+ land_cover_id,
168
+ column_name('maximumCycleDuration')
169
+ ))
155
170
 
156
171
 
157
172
  def _gap_filled_date_only_str(date_str: str, mode: str = DatestrGapfillMode.END) -> str:
@@ -166,16 +181,16 @@ def _gap_filled_date_obj(date_str: str, mode: str = DatestrGapfillMode.END) -> d
166
181
 
167
182
 
168
183
  def _gap_filled_start_date(land_cover_id: str, end_date: str, cycle: dict) -> dict:
169
- """If possible, gap-fill the startDate based on the endDate - maximumCycleDuration"""
170
- maximum_cycle_duration = _get_maximum_cycle_duration(land_cover_id)
184
+ """If possible, gap-fill the startDate based on the endDate - cycleDuration"""
185
+ cycle_duration = _get_cycle_duration(cycle, land_cover_id)
171
186
  return {
172
187
  "startDate": max(
173
- _gap_filled_date_obj(end_date) - timedelta(days=maximum_cycle_duration)
174
- if maximum_cycle_duration else datetime.fromtimestamp(0),
188
+ _gap_filled_date_obj(end_date) - timedelta(days=cycle_duration)
189
+ if cycle_duration else datetime.fromtimestamp(0),
175
190
  _gap_filled_date_obj(cycle.get("startDate"), mode=DatestrGapfillMode.START)
176
191
  if cycle.get("startDate") else datetime.fromtimestamp(0)
177
192
  )
178
- } if any([maximum_cycle_duration, cycle.get("startDate")]) else {}
193
+ } if any([cycle_duration, cycle.get("startDate")]) else {}
179
194
 
180
195
 
181
196
  def _include_with_date_gap_fill(value: dict, keys: list) -> dict:
@@ -217,12 +232,21 @@ def _copy_item_if_exists(source: dict, keys: list[str] = None, dest: dict = None
217
232
  return reduce(lambda p, c: p | ({c: source[c]} if source.get(c) else {}), keys or [], dest or {})
218
233
 
219
234
 
220
- def _get_relevant_items(cycle: dict, item_name: str, relevant_terms: list):
235
+ def _get_relevant_items(cycle: dict, item_name: str, term_types: List[TermTermType], completeness_mapping: dict = {}):
221
236
  """
222
237
  Get items from the list of cycles with any of the relevant terms.
223
238
  Also adds dates from Cycle.
224
239
  """
225
- items = [
240
+ # filter term types that are no complete
241
+ complete_term_types = term_types if not completeness_mapping else [
242
+ term_type for term_type in term_types
243
+ if any([
244
+ not completeness_mapping.get(term_type.value),
245
+ cycle.get('completeness').get(completeness_mapping.get(term_type.value), False)
246
+ ])
247
+ ]
248
+ blank_nodes = filter_list_term_type(cycle.get(item_name, []), complete_term_types)
249
+ return [
226
250
  _include_with_date_gap_fill(cycle, ["startDate", "endDate"]) |
227
251
  _include(
228
252
  _gap_filled_start_date(
@@ -233,20 +257,19 @@ def _get_relevant_items(cycle: dict, item_name: str, relevant_terms: list):
233
257
  "startDate"
234
258
  ) |
235
259
  item
236
- for item in filter_list_term_type(cycle.get(item_name, []), relevant_terms)
260
+ for item in blank_nodes
237
261
  ]
238
- return items
239
262
 
240
263
 
241
264
  def _process_rule(node: dict, term: dict) -> list:
242
- relevant_terms = []
265
+ term_types = []
243
266
  for column, condition, new_term in _INPUT_RULES[term.get('termType')]:
244
267
  lookup_result = get_lookup_value(term, LOOKUPS[column], model=MODEL, term=term.get('@id'), model_key=MODEL_KEY)
245
268
 
246
269
  if condition(lookup_result):
247
- relevant_terms.append(node | {'id': new_term})
270
+ term_types.append(node | {'id': new_term})
248
271
 
249
- return relevant_terms
272
+ return term_types
250
273
 
251
274
 
252
275
  def _run_from_inputs(site: dict, cycle: dict) -> list:
@@ -307,7 +330,7 @@ def _run_from_landCover(cycle: dict, crop_forage_products: list):
307
330
  )) for product in _get_relevant_items(
308
331
  cycle=cycle,
309
332
  item_name="products",
310
- relevant_terms=[TermTermType.LANDCOVER]
333
+ term_types=[TermTermType.LANDCOVER]
311
334
  )
312
335
  ]
313
336
  return land_cover_products + _run_products(
@@ -337,7 +360,7 @@ def _run_from_crop_forage(cycle: dict, site: dict):
337
360
  products = _get_relevant_items(
338
361
  cycle=cycle,
339
362
  item_name="products",
340
- relevant_terms=[TermTermType.CROP, TermTermType.FORAGE]
363
+ term_types=[TermTermType.CROP, TermTermType.FORAGE]
341
364
  ) if site.get("siteType", "") == SiteSiteType.CROPLAND.value else []
342
365
  # only take products with a matching landCover term
343
366
  products = [p for p in products if get_landCover_term_id(p.get('term', {}))]
@@ -372,13 +395,8 @@ def _run_from_practices(cycle: dict):
372
395
  ) for practice in _get_relevant_items(
373
396
  cycle=cycle,
374
397
  item_name="practices",
375
- relevant_terms=[
376
- TermTermType.WATERREGIME,
377
- TermTermType.TILLAGE,
378
- TermTermType.CROPRESIDUEMANAGEMENT,
379
- TermTermType.LANDUSEMANAGEMENT,
380
- TermTermType.SYSTEM
381
- ]
398
+ term_types=_PRACTICES_TERM_TYPES,
399
+ completeness_mapping=_PRACTICES_COMPLETENESS_MAPPING
382
400
  )
383
401
  ]
384
402
  practices = list(map(_map_to_value, filter(_should_run_practice, practices)))
@@ -93,41 +93,46 @@ def _is_type(value: dict, ee_type: str):
93
93
  ]) if isinstance(params, list) else params.get('ee_type') == ee_type
94
94
 
95
95
 
96
- def list_collections(years: list = [], include_region: bool = False, years_only: bool = False):
96
+ def list_rasters(years: list = [], years_only: bool = False):
97
97
  ee_params = list_ee_params()
98
98
  # only cache `raster` results as can be combined in a single query
99
99
  rasters = [value for value in ee_params if _is_type(value, 'raster')]
100
100
  rasters = _extend_collections(rasters, years or [])
101
101
  rasters = [raster for raster in rasters if not years_only or _is_collection_by_year(raster)]
102
102
 
103
+ return rasters
104
+
105
+
106
+ def list_vectors(sites: list):
107
+ ee_params = list_ee_params()
108
+
109
+ vectors = [value for value in ee_params if _is_type(value, 'vector')]
103
110
  vectors = [
104
- value for value in ee_params if _is_type(value, 'vector') and (
105
- include_region or not value.get('params').get('collection', '').startswith('gadm36')
106
- )
111
+ value for value in vectors
112
+ # name of the model is the key in the data. If the key is present in all sites, we don't need to query
113
+ if all([not s.get(value.get('name')) for s in sites])
107
114
  ]
108
115
  # no vectors are running with specific years
109
- vectors = [] if years_only else _extend_collections(vectors)
116
+ vectors = _extend_collections(vectors)
110
117
 
111
- return (rasters, vectors)
118
+ return vectors
112
119
 
113
120
 
114
121
  def _cache_results(site: dict, area_size: float):
115
122
  # to fetch data related to the year
116
123
  years = cached_value(site, key=CACHE_YEARS_KEY, default=[])
117
- include_region = all([has_coordinates(site), not site.get('region')])
118
- rasters, vectors = list_collections(years, include_region=include_region)
124
+ rasters = list_rasters(years)
125
+ vectors = list_vectors([site])
119
126
 
120
127
  raster_results = _run_query({
121
128
  'ee_type': 'raster',
122
- 'collections': rasters,
123
- **geospatial_data(site)
124
- })
129
+ 'collections': rasters
130
+ } | geospatial_data(site)) if rasters else []
125
131
 
126
132
  vector_results = _run_query({
127
133
  'ee_type': 'vector',
128
- 'collections': vectors,
129
- **geospatial_data(site)
130
- })
134
+ 'collections': vectors
135
+ } | geospatial_data(site)) if vectors else []
131
136
 
132
137
  return cache_site_results(raster_results + vector_results, rasters + vectors, area_size)
133
138
 
@@ -73,7 +73,7 @@ def lookups_logs(model: str, blank_nodes: list, lookups_per_termType: dict, **lo
73
73
 
74
74
  def _reduce_lookups_logs(logs: dict, column: str):
75
75
  lookup_value = get_lookup_value(term, column, model=model, **log_args)
76
- return logs | {column: lookup_value}
76
+ return logs | {column: str(lookup_value).replace(':', '-')}
77
77
 
78
78
  return reduce(_reduce_lookups_logs, lookups, {'id': term_id})
79
79
 
@@ -270,7 +270,8 @@ def get_total_value_converted_with_min_ratio(
270
270
  model: str, term: str, node: dict = {},
271
271
  blank_nodes: list = [],
272
272
  prop_id: str = 'energyContentHigherHeatingValue',
273
- min_ratio: float = 0.8
273
+ min_ratio: float = 0.8,
274
+ is_sum: bool = True
274
275
  ):
275
276
  values = [
276
277
  (
@@ -301,9 +302,14 @@ def get_total_value_converted_with_min_ratio(
301
302
  debugValues(node, model=model, term=term,
302
303
  **logs)
303
304
 
304
- return list_sum([
305
+ total_converted_value = list_sum([
305
306
  value * prop_value for term_id, value, prop_value in values if all([value, prop_value])
306
- ]) * total_value / total_value_with_property if total_value_ratio >= min_ratio else None
307
+ ])
308
+
309
+ return (
310
+ total_converted_value * total_value / total_value_with_property if is_sum
311
+ else total_converted_value / total_value_with_property
312
+ ) if total_value_ratio >= min_ratio else None
307
313
 
308
314
 
309
315
  def get_N_total(nodes: list) -> list:
@@ -67,6 +67,6 @@ def get_landCover_term_id(lookup_term: dict, **log_args) -> str:
67
67
  value = get_lookup_value(lookup_term, 'landCoverTermId', **log_args)
68
68
  return (
69
69
  lookup_term.get("@id") if lookup_term.get("termType") == TermTermType.LANDCOVER.value else
70
- value.split(';')[0] if value else
70
+ value.split(';')[0] if isinstance(value, str) else
71
71
  None
72
72
  )
@@ -291,7 +291,7 @@ def _land_occupation_per_kg(model: str, term_id: str, cycle: dict, product: dict
291
291
  return value
292
292
 
293
293
 
294
- def land_occupation_per_kg(model: str, term_id: str, cycle: dict, site: dict, primary_product: dict):
294
+ def land_occupation_per_kg(model: str, term_id: str, cycle: dict, site: dict, product: dict):
295
295
  """
296
296
  Get the land occupation of the cycle per kg in meter square.
297
297
 
@@ -305,7 +305,7 @@ def land_occupation_per_kg(model: str, term_id: str, cycle: dict, site: dict, pr
305
305
  The `Cycle` as defined in the HESTIA Schema.
306
306
  site : dict
307
307
  The `Site` as defined in the HESTIA Schema.
308
- primary_product : dict
308
+ product : dict
309
309
  The primary `Product` of the `Cycle`.
310
310
 
311
311
  Returns
@@ -325,7 +325,7 @@ def land_occupation_per_kg(model: str, term_id: str, cycle: dict, site: dict, pr
325
325
  SiteSiteType.RIVER_OR_STREAM.value,
326
326
  SiteSiteType.SEA_OR_OCEAN.value
327
327
  ] else (
328
- _land_occupation_per_kg(model, term_id, cycle, primary_product, value) if value is not None else None
328
+ _land_occupation_per_kg(model, term_id, cycle, product, value) if value is not None else None
329
329
  )
330
330
 
331
331
 
@@ -53,7 +53,7 @@ def all_factor_value(
53
53
  all_with_factors = all([v.get('coefficient') is not None for v in values if v.get('value') is not None])
54
54
 
55
55
  for missing_value in missing_values:
56
- debugMissingLookup(lookup_name, 'termid', missing_value, lookup_col, None)
56
+ debugMissingLookup(lookup_name, 'termid', missing_value, lookup_col, None, model=model, term=term_id)
57
57
 
58
58
  debugValues(node, model=model, term=term_id,
59
59
  all_with_factors=all_with_factors,
@@ -1 +1 @@
1
- VERSION = '0.65.10'
1
+ VERSION = '0.66.0'