hestia-earth-models 0.65.10__py3-none-any.whl → 0.66.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hestia_earth/models/cache_sites.py +7 -9
- hestia_earth/models/config/Cycle.json +34 -16
- hestia_earth/models/config/ImpactAssessment.json +12 -0
- hestia_earth/models/config/Site.json +4 -1
- hestia_earth/models/cycle/completeness/freshForage.py +10 -2
- hestia_earth/models/cycle/cropResidueManagement.py +3 -1
- hestia_earth/models/ecoinventV3/__init__.py +2 -1
- hestia_earth/models/environmentalFootprintV3/environmentalFootprintSingleOverallScore.py +135 -0
- hestia_earth/models/environmentalFootprintV3/soilQualityIndexLandTransformation.py +17 -6
- hestia_earth/models/geospatialDatabase/{aware.py → awareWaterBasinId.py} +1 -1
- hestia_earth/models/hestia/landCover.py +57 -39
- hestia_earth/models/hestia/residueRemoved.py +80 -0
- hestia_earth/models/hestia/resourceUse_utils.py +64 -38
- hestia_earth/models/hestia/utils.py +1 -2
- hestia_earth/models/ipcc2019/aboveGroundBiomass.py +33 -12
- hestia_earth/models/ipcc2019/animal/pastureGrass.py +1 -1
- hestia_earth/models/ipcc2019/belowGroundBiomass.py +32 -11
- hestia_earth/models/ipcc2019/ch4ToAirEntericFermentation.py +17 -8
- hestia_earth/models/ipcc2019/co2ToAirCarbonStockChange_utils.py +5 -3
- hestia_earth/models/ipcc2019/organicCarbonPerHa_tier_2_utils.py +27 -17
- hestia_earth/models/ipcc2019/pastureGrass.py +1 -1
- hestia_earth/models/ipcc2019/pastureGrass_utils.py +8 -1
- hestia_earth/models/log.py +1 -1
- hestia_earth/models/mocking/search-results.json +34 -34
- hestia_earth/models/pooreNemecek2018/freshwaterWithdrawalsDuringCycle.py +0 -1
- hestia_earth/models/pooreNemecek2018/landOccupationDuringCycle.py +13 -10
- hestia_earth/models/site/defaultMethodClassification.py +9 -2
- hestia_earth/models/site/defaultMethodClassificationDescription.py +4 -2
- hestia_earth/models/site/management.py +49 -31
- hestia_earth/models/site/pre_checks/cache_geospatialDatabase.py +19 -14
- hestia_earth/models/utils/blank_node.py +10 -4
- hestia_earth/models/utils/crop.py +1 -1
- hestia_earth/models/utils/cycle.py +3 -3
- hestia_earth/models/utils/lookup.py +1 -1
- hestia_earth/models/version.py +1 -1
- hestia_earth/orchestrator/strategies/merge/merge_list.py +17 -6
- {hestia_earth_models-0.65.10.dist-info → hestia_earth_models-0.66.0.dist-info}/METADATA +1 -1
- {hestia_earth_models-0.65.10.dist-info → hestia_earth_models-0.66.0.dist-info}/RECORD +59 -54
- tests/models/environmentalFootprintV3/test_environmentalFootprintSingleOverallScore.py +92 -0
- tests/models/environmentalFootprintV3/test_soilQualityIndexLandTransformation.py +4 -19
- tests/models/faostat2018/product/test_price.py +1 -1
- tests/models/geospatialDatabase/{test_aware.py → test_awareWaterBasinId.py} +1 -1
- tests/models/hestia/test_landCover.py +4 -2
- tests/models/hestia/test_landTransformation20YearAverageDuringCycle.py +3 -1
- tests/models/hestia/test_residueRemoved.py +20 -0
- tests/models/ipcc2019/test_aboveGroundBiomass.py +3 -1
- tests/models/ipcc2019/test_belowGroundBiomass.py +4 -2
- tests/models/ipcc2019/test_organicCarbonPerHa.py +94 -1
- tests/models/pooreNemecek2018/test_landOccupationDuringCycle.py +1 -3
- tests/models/site/pre_checks/test_cache_geospatialDatabase.py +22 -0
- tests/models/site/test_defaultMethodClassification.py +6 -0
- tests/models/site/test_defaultMethodClassificationDescription.py +6 -0
- tests/models/site/test_management.py +4 -4
- tests/models/test_cache_sites.py +2 -2
- tests/models/utils/test_crop.py +14 -2
- tests/orchestrator/strategies/merge/test_merge_list.py +11 -1
- {hestia_earth_models-0.65.10.dist-info → hestia_earth_models-0.66.0.dist-info}/LICENSE +0 -0
- {hestia_earth_models-0.65.10.dist-info → hestia_earth_models-0.66.0.dist-info}/WHEEL +0 -0
- {hestia_earth_models-0.65.10.dist-info → hestia_earth_models-0.66.0.dist-info}/top_level.txt +0 -0
@@ -1768,7 +1768,7 @@
|
|
1768
1768
|
"@type": "Term",
|
1769
1769
|
"name": "Generic crop, seed",
|
1770
1770
|
"@id": "genericCropSeed",
|
1771
|
-
"_score": 25.
|
1771
|
+
"_score": 25.417622
|
1772
1772
|
}
|
1773
1773
|
]
|
1774
1774
|
},
|
@@ -2004,157 +2004,157 @@
|
|
2004
2004
|
"@type": "Term",
|
2005
2005
|
"name": "Glass or high accessible cover",
|
2006
2006
|
"@id": "glassOrHighAccessibleCover",
|
2007
|
-
"_score": 64.
|
2007
|
+
"_score": 64.78735
|
2008
2008
|
},
|
2009
2009
|
{
|
2010
2010
|
"@type": "Term",
|
2011
2011
|
"name": "Sea or ocean",
|
2012
2012
|
"@id": "seaOrOcean",
|
2013
|
-
"_score": 54.
|
2013
|
+
"_score": 54.49869
|
2014
2014
|
},
|
2015
2015
|
{
|
2016
2016
|
"@type": "Term",
|
2017
2017
|
"name": "River or stream",
|
2018
2018
|
"@id": "riverOrStream",
|
2019
|
-
"_score": 52.
|
2019
|
+
"_score": 52.33286
|
2020
2020
|
},
|
2021
2021
|
{
|
2022
2022
|
"@type": "Term",
|
2023
2023
|
"name": "Other natural vegetation",
|
2024
2024
|
"@id": "otherNaturalVegetation",
|
2025
|
-
"_score": 43.
|
2025
|
+
"_score": 43.071865
|
2026
2026
|
},
|
2027
2027
|
{
|
2028
2028
|
"@type": "Term",
|
2029
2029
|
"name": "Agri-food processor",
|
2030
2030
|
"@id": "agriFoodProcessor",
|
2031
|
-
"_score":
|
2031
|
+
"_score": 41.75925
|
2032
2032
|
},
|
2033
2033
|
{
|
2034
2034
|
"@type": "Term",
|
2035
2035
|
"name": "Food retailer",
|
2036
2036
|
"@id": "foodRetailer",
|
2037
|
-
"_score": 41.
|
2037
|
+
"_score": 41.196114
|
2038
2038
|
},
|
2039
2039
|
{
|
2040
2040
|
"@type": "Term",
|
2041
2041
|
"name": "Natural forest",
|
2042
2042
|
"@id": "naturalForest",
|
2043
|
-
"_score": 32.
|
2043
|
+
"_score": 32.262592
|
2044
2044
|
},
|
2045
2045
|
{
|
2046
2046
|
"@type": "Term",
|
2047
2047
|
"name": "Permanent pasture",
|
2048
2048
|
"@id": "permanentPasture",
|
2049
|
-
"_score": 28.
|
2049
|
+
"_score": 28.761673
|
2050
2050
|
},
|
2051
2051
|
{
|
2052
2052
|
"@type": "Term",
|
2053
2053
|
"name": "Animal housing",
|
2054
2054
|
"@id": "animalHousing",
|
2055
|
-
"_score": 27.
|
2055
|
+
"_score": 27.809086
|
2056
2056
|
},
|
2057
2057
|
{
|
2058
2058
|
"@type": "Term",
|
2059
2059
|
"name": "Root or tuber crop plant",
|
2060
2060
|
"@id": "rootOrTuberCropPlant",
|
2061
|
-
"_score": 27.
|
2061
|
+
"_score": 27.444334
|
2062
2062
|
},
|
2063
2063
|
{
|
2064
2064
|
"@type": "Term",
|
2065
2065
|
"name": "High intensity grazing pasture",
|
2066
2066
|
"@id": "highIntensityGrazingPasture",
|
2067
|
-
"_score": 24.
|
2067
|
+
"_score": 24.399143
|
2068
2068
|
},
|
2069
2069
|
{
|
2070
2070
|
"@type": "Term",
|
2071
2071
|
"name": "Forest",
|
2072
2072
|
"@id": "forest",
|
2073
|
-
"_score": 20.
|
2073
|
+
"_score": 20.347794
|
2074
2074
|
},
|
2075
2075
|
{
|
2076
2076
|
"@type": "Term",
|
2077
2077
|
"name": "Permanent cropland",
|
2078
2078
|
"@id": "permanentCropland",
|
2079
|
-
"_score": 20.
|
2079
|
+
"_score": 20.196049
|
2080
2080
|
},
|
2081
2081
|
{
|
2082
2082
|
"@type": "Term",
|
2083
2083
|
"name": "Other land",
|
2084
2084
|
"@id": "otherLand",
|
2085
|
-
"_score": 19.
|
2085
|
+
"_score": 19.765242
|
2086
2086
|
},
|
2087
2087
|
{
|
2088
2088
|
"@type": "Term",
|
2089
2089
|
"name": "Plantation forest",
|
2090
2090
|
"@id": "plantationForest",
|
2091
|
-
"_score": 19.
|
2092
|
-
},
|
2093
|
-
{
|
2094
|
-
"@type": "Term",
|
2095
|
-
"name": "Sea kale plant",
|
2096
|
-
"@id": "seaKalePlant",
|
2097
|
-
"_score": 18.2738
|
2091
|
+
"_score": 19.03003
|
2098
2092
|
},
|
2099
2093
|
{
|
2100
2094
|
"@type": "Term",
|
2101
2095
|
"name": "Lake",
|
2102
2096
|
"@id": "lake",
|
2103
|
-
"_score": 18.
|
2097
|
+
"_score": 18.264688
|
2104
2098
|
},
|
2105
2099
|
{
|
2106
2100
|
"@type": "Term",
|
2107
|
-
"name": "
|
2108
|
-
"@id": "
|
2109
|
-
"_score": 18.
|
2101
|
+
"name": "Sea kale plant",
|
2102
|
+
"@id": "seaKalePlant",
|
2103
|
+
"_score": 18.135399
|
2110
2104
|
},
|
2111
2105
|
{
|
2112
2106
|
"@type": "Term",
|
2113
2107
|
"name": "Native pasture",
|
2114
2108
|
"@id": "nativePasture",
|
2115
|
-
"_score": 17.
|
2109
|
+
"_score": 17.863037
|
2116
2110
|
},
|
2117
2111
|
{
|
2118
2112
|
"@type": "Term",
|
2119
2113
|
"name": "Improved pasture",
|
2120
2114
|
"@id": "improvedPasture",
|
2121
|
-
"_score": 17.
|
2115
|
+
"_score": 17.444498
|
2116
|
+
},
|
2117
|
+
{
|
2118
|
+
"@type": "Term",
|
2119
|
+
"name": "Red sea plume alga",
|
2120
|
+
"@id": "redSeaPlumeAlga",
|
2121
|
+
"_score": 17.235117
|
2122
2122
|
},
|
2123
2123
|
{
|
2124
2124
|
"@type": "Term",
|
2125
2125
|
"name": "Nominally managed pasture",
|
2126
2126
|
"@id": "nominallyManagedPasture",
|
2127
|
-
"_score": 16.
|
2127
|
+
"_score": 16.79026
|
2128
2128
|
},
|
2129
2129
|
{
|
2130
2130
|
"@type": "Term",
|
2131
2131
|
"name": "Severely degraded pasture",
|
2132
2132
|
"@id": "severelyDegradedPasture",
|
2133
|
-
"_score": 16.
|
2133
|
+
"_score": 16.32682
|
2134
2134
|
},
|
2135
2135
|
{
|
2136
2136
|
"@type": "Term",
|
2137
2137
|
"name": "Pond",
|
2138
2138
|
"@id": "pond",
|
2139
|
-
"_score": 15.
|
2139
|
+
"_score": 15.64036
|
2140
2140
|
},
|
2141
2141
|
{
|
2142
2142
|
"@type": "Term",
|
2143
2143
|
"name": "River tamarind tree",
|
2144
2144
|
"@id": "riverTamarindTree",
|
2145
|
-
"_score": 15.
|
2145
|
+
"_score": 15.445333
|
2146
2146
|
},
|
2147
2147
|
{
|
2148
2148
|
"@type": "Term",
|
2149
2149
|
"name": "Annual cropland",
|
2150
2150
|
"@id": "annualCropland",
|
2151
|
-
"_score": 9.
|
2151
|
+
"_score": 9.808704
|
2152
2152
|
},
|
2153
2153
|
{
|
2154
2154
|
"@type": "Term",
|
2155
2155
|
"name": "Cropland",
|
2156
2156
|
"@id": "cropland",
|
2157
|
-
"_score": 9.
|
2157
|
+
"_score": 9.772207
|
2158
2158
|
}
|
2159
2159
|
]
|
2160
2160
|
},
|
@@ -2,7 +2,8 @@ from hestia_earth.models.log import logRequirements, logShouldRun
|
|
2
2
|
from hestia_earth.models.utils.indicator import _new_indicator
|
3
3
|
from hestia_earth.models.utils.impact_assessment import get_product, get_site
|
4
4
|
from hestia_earth.models.utils.cycle import land_occupation_per_kg
|
5
|
-
from hestia_earth.models.utils.site import get_land_cover_term_id
|
5
|
+
from hestia_earth.models.utils.site import get_land_cover_term_id as get_landCover_term_id_from_site_type
|
6
|
+
from hestia_earth.models.utils.crop import get_landCover_term_id
|
6
7
|
from . import MODEL
|
7
8
|
|
8
9
|
REQUIREMENTS = {
|
@@ -16,7 +17,6 @@ REQUIREMENTS = {
|
|
16
17
|
"cycleDuration": "",
|
17
18
|
"products": [{
|
18
19
|
"@type": "Product",
|
19
|
-
"primary": "True",
|
20
20
|
"value": "> 0",
|
21
21
|
"economicValueShare": "> 0"
|
22
22
|
}],
|
@@ -46,8 +46,8 @@ RETURNS = {
|
|
46
46
|
TERM_ID = 'landOccupationDuringCycle'
|
47
47
|
|
48
48
|
|
49
|
-
def _indicator(term_id: str, value: float,
|
50
|
-
indicator = _new_indicator(term_id, MODEL,
|
49
|
+
def _indicator(term_id: str, value: float, land_cover_term_id: str):
|
50
|
+
indicator = _new_indicator(term_id, MODEL, land_cover_term_id)
|
51
51
|
indicator['value'] = value
|
52
52
|
return indicator
|
53
53
|
|
@@ -56,18 +56,21 @@ def _should_run(impact_assessment: dict):
|
|
56
56
|
product = get_product(impact_assessment)
|
57
57
|
cycle = impact_assessment.get('cycle', {})
|
58
58
|
site = get_site(impact_assessment)
|
59
|
-
|
59
|
+
land_cover_from_product = get_landCover_term_id(product.get('term', {}))
|
60
|
+
land_cover_from_site = get_landCover_term_id_from_site_type(site.get('siteType'))
|
61
|
+
land_cover_term_id = land_cover_from_product or land_cover_from_site
|
60
62
|
land_occupation_m2_kg = land_occupation_per_kg(MODEL, TERM_ID, cycle, site, product)
|
61
63
|
|
62
64
|
logRequirements(impact_assessment, model=MODEL, term=TERM_ID,
|
63
65
|
land_occupation_kg=land_occupation_m2_kg,
|
64
|
-
|
66
|
+
land_cover_from_product=land_cover_from_product,
|
67
|
+
land_cover_from_site=land_cover_from_site)
|
65
68
|
|
66
|
-
should_run = all([
|
69
|
+
should_run = all([land_cover_term_id, land_occupation_m2_kg is not None])
|
67
70
|
logShouldRun(impact_assessment, MODEL, TERM_ID, should_run)
|
68
|
-
return should_run, land_occupation_m2_kg,
|
71
|
+
return should_run, land_occupation_m2_kg, land_cover_term_id
|
69
72
|
|
70
73
|
|
71
74
|
def run(impact_assessment: dict):
|
72
|
-
should_run, land_occupation_kg,
|
73
|
-
return [_indicator(TERM_ID, land_occupation_kg,
|
75
|
+
should_run, land_occupation_kg, land_cover_term_id = _should_run(impact_assessment)
|
76
|
+
return [_indicator(TERM_ID, land_occupation_kg, land_cover_term_id)] if should_run else []
|
@@ -5,6 +5,8 @@ When gap-filling `management` node on Site, the
|
|
5
5
|
`defaultMethodClassification` and `defaultMethodClassificationDescription` fields become required.
|
6
6
|
This model will use the first value in the `management` node.
|
7
7
|
"""
|
8
|
+
from hestia_earth.schema import SiteDefaultMethodClassification
|
9
|
+
|
8
10
|
from hestia_earth.models.log import logRequirements, logShouldRun
|
9
11
|
from . import MODEL
|
10
12
|
|
@@ -20,12 +22,17 @@ MODEL_KEY = 'defaultMethodClassification'
|
|
20
22
|
|
21
23
|
|
22
24
|
def _should_run(site: dict):
|
23
|
-
|
25
|
+
has_management = bool(site.get('management', []))
|
26
|
+
methodClassification = next(
|
27
|
+
(n.get('methodClassification') for n in site.get('management', [])),
|
28
|
+
None
|
29
|
+
) or SiteDefaultMethodClassification.MODELLED.value
|
24
30
|
|
25
31
|
logRequirements(site, model=MODEL, model_key=MODEL_KEY,
|
32
|
+
has_management=has_management,
|
26
33
|
methodClassification=methodClassification)
|
27
34
|
|
28
|
-
should_run = all([methodClassification])
|
35
|
+
should_run = all([has_management, methodClassification])
|
29
36
|
logShouldRun(site, MODEL, None, should_run, model_key=MODEL_KEY)
|
30
37
|
return should_run, methodClassification
|
31
38
|
|
@@ -20,16 +20,18 @@ MODEL_KEY = 'defaultMethodClassificationDescription'
|
|
20
20
|
|
21
21
|
|
22
22
|
def _should_run(site: dict):
|
23
|
+
has_management = bool(site.get('management', []))
|
23
24
|
methodClassificationDescription = next((
|
24
25
|
n.get('methodClassificationDescription')
|
25
26
|
for n in site.get('management', [])
|
26
27
|
if n.get('methodClassification')
|
27
|
-
), None)
|
28
|
+
), None) or 'Data calculated by merging real land use histories and modelled land use histories for each Site.'
|
28
29
|
|
29
30
|
logRequirements(site, model=MODEL, model_key=MODEL_KEY,
|
31
|
+
has_management=has_management,
|
30
32
|
methodClassificationDescription=methodClassificationDescription)
|
31
33
|
|
32
|
-
should_run = all([methodClassificationDescription])
|
34
|
+
should_run = all([has_management, methodClassificationDescription])
|
33
35
|
logShouldRun(site, MODEL, None, should_run, model_key=MODEL_KEY)
|
34
36
|
return should_run, methodClassificationDescription
|
35
37
|
|
@@ -10,7 +10,8 @@ tillage, cropResidueManagement and landUseManagement.
|
|
10
10
|
All values are copied from the source node, except for crop and forage terms in which case the dates are copied from the
|
11
11
|
cycle.
|
12
12
|
|
13
|
-
Where `startDate` is missing from landCover products, gap-filling is attempted using
|
13
|
+
Where `startDate` is missing from landCover products, gap-filling is attempted using
|
14
|
+
`endDate` - `cycleDuration` (or `maximumCycleDuration` lookup).
|
14
15
|
This is the `endDate` of the `landCover` product.
|
15
16
|
This ensures no overlapping date ranges.
|
16
17
|
If both `endDate` and `startDate` are missing from the product, these will be gap-filled from the `Cycle`.
|
@@ -18,9 +19,10 @@ If both `endDate` and `startDate` are missing from the product, these will be ga
|
|
18
19
|
When nodes are chronologically consecutive with "% area" or "boolean" units and the same term and value, they are
|
19
20
|
condensed into a single node to aid readability.
|
20
21
|
"""
|
22
|
+
from typing import List
|
21
23
|
from datetime import timedelta, datetime
|
22
24
|
from functools import reduce
|
23
|
-
from hestia_earth.schema import TermTermType, SiteSiteType
|
25
|
+
from hestia_earth.schema import SchemaType, TermTermType, SiteSiteType, COMPLETENESS_MAPPING
|
24
26
|
from hestia_earth.utils.lookup import column_name, get_table_value, download_lookup
|
25
27
|
from hestia_earth.utils.model import filter_list_term_type
|
26
28
|
from hestia_earth.utils.tools import safe_parse_float, flatten
|
@@ -31,18 +33,17 @@ from hestia_earth.models.utils import _include, _omit, group_by
|
|
31
33
|
from hestia_earth.models.utils.management import _new_management
|
32
34
|
from hestia_earth.models.utils.term import get_lookup_value
|
33
35
|
from hestia_earth.models.utils.blank_node import condense_nodes, DatestrFormat, _gapfill_datestr, DatestrGapfillMode
|
36
|
+
from hestia_earth.models.utils.crop import get_landCover_term_id
|
34
37
|
from hestia_earth.models.utils.site import (
|
35
38
|
related_cycles, get_land_cover_term_id as get_landCover_term_id_from_site_type
|
36
39
|
)
|
37
40
|
from . import MODEL
|
38
|
-
from ..utils.crop import get_landCover_term_id
|
39
41
|
|
40
42
|
REQUIREMENTS = {
|
41
43
|
"Site": {
|
42
44
|
"related": {
|
43
45
|
"Cycle": [{
|
44
46
|
"@type": "Cycle",
|
45
|
-
"startDate": "",
|
46
47
|
"endDate": "",
|
47
48
|
"products": [
|
48
49
|
{
|
@@ -71,7 +72,11 @@ REQUIREMENTS = {
|
|
71
72
|
"soilAmendment"
|
72
73
|
]
|
73
74
|
}
|
74
|
-
]
|
75
|
+
],
|
76
|
+
"optional": {
|
77
|
+
"startDate": "",
|
78
|
+
"cycleDuration": ""
|
79
|
+
}
|
75
80
|
}]
|
76
81
|
}
|
77
82
|
}
|
@@ -98,6 +103,14 @@ LOOKUPS = {
|
|
98
103
|
}
|
99
104
|
MODEL_KEY = 'management'
|
100
105
|
|
106
|
+
_PRACTICES_TERM_TYPES = [
|
107
|
+
TermTermType.WATERREGIME,
|
108
|
+
TermTermType.TILLAGE,
|
109
|
+
TermTermType.CROPRESIDUEMANAGEMENT,
|
110
|
+
TermTermType.LANDUSEMANAGEMENT,
|
111
|
+
TermTermType.SYSTEM
|
112
|
+
]
|
113
|
+
_PRACTICES_COMPLETENESS_MAPPING = COMPLETENESS_MAPPING.get(SchemaType.PRACTICE.value)
|
101
114
|
_ANIMAL_MANURE_USED_TERM_ID = "animalManureUsed"
|
102
115
|
_INORGANIC_NITROGEN_FERTILISER_USED_TERM_ID = "inorganicNitrogenFertiliserUsed"
|
103
116
|
_ORGANIC_FERTILISER_USED_TERM_ID = "organicFertiliserUsed"
|
@@ -147,11 +160,13 @@ def management(data: dict):
|
|
147
160
|
return node
|
148
161
|
|
149
162
|
|
150
|
-
def
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
163
|
+
def _get_cycle_duration(cycle: dict, land_cover_id: str):
|
164
|
+
return cycle.get('cycleDuration') or safe_parse_float(get_table_value(
|
165
|
+
download_lookup("crop.csv"),
|
166
|
+
column_name('landCoverTermId'),
|
167
|
+
land_cover_id,
|
168
|
+
column_name('maximumCycleDuration')
|
169
|
+
))
|
155
170
|
|
156
171
|
|
157
172
|
def _gap_filled_date_only_str(date_str: str, mode: str = DatestrGapfillMode.END) -> str:
|
@@ -166,16 +181,16 @@ def _gap_filled_date_obj(date_str: str, mode: str = DatestrGapfillMode.END) -> d
|
|
166
181
|
|
167
182
|
|
168
183
|
def _gap_filled_start_date(land_cover_id: str, end_date: str, cycle: dict) -> dict:
|
169
|
-
"""If possible, gap-fill the startDate based on the endDate -
|
170
|
-
|
184
|
+
"""If possible, gap-fill the startDate based on the endDate - cycleDuration"""
|
185
|
+
cycle_duration = _get_cycle_duration(cycle, land_cover_id)
|
171
186
|
return {
|
172
187
|
"startDate": max(
|
173
|
-
_gap_filled_date_obj(end_date) - timedelta(days=
|
174
|
-
if
|
188
|
+
_gap_filled_date_obj(end_date) - timedelta(days=cycle_duration)
|
189
|
+
if cycle_duration else datetime.fromtimestamp(0),
|
175
190
|
_gap_filled_date_obj(cycle.get("startDate"), mode=DatestrGapfillMode.START)
|
176
191
|
if cycle.get("startDate") else datetime.fromtimestamp(0)
|
177
192
|
)
|
178
|
-
} if any([
|
193
|
+
} if any([cycle_duration, cycle.get("startDate")]) else {}
|
179
194
|
|
180
195
|
|
181
196
|
def _include_with_date_gap_fill(value: dict, keys: list) -> dict:
|
@@ -217,12 +232,21 @@ def _copy_item_if_exists(source: dict, keys: list[str] = None, dest: dict = None
|
|
217
232
|
return reduce(lambda p, c: p | ({c: source[c]} if source.get(c) else {}), keys or [], dest or {})
|
218
233
|
|
219
234
|
|
220
|
-
def _get_relevant_items(cycle: dict, item_name: str,
|
235
|
+
def _get_relevant_items(cycle: dict, item_name: str, term_types: List[TermTermType], completeness_mapping: dict = {}):
|
221
236
|
"""
|
222
237
|
Get items from the list of cycles with any of the relevant terms.
|
223
238
|
Also adds dates from Cycle.
|
224
239
|
"""
|
225
|
-
|
240
|
+
# filter term types that are no complete
|
241
|
+
complete_term_types = term_types if not completeness_mapping else [
|
242
|
+
term_type for term_type in term_types
|
243
|
+
if any([
|
244
|
+
not completeness_mapping.get(term_type.value),
|
245
|
+
cycle.get('completeness').get(completeness_mapping.get(term_type.value), False)
|
246
|
+
])
|
247
|
+
]
|
248
|
+
blank_nodes = filter_list_term_type(cycle.get(item_name, []), complete_term_types)
|
249
|
+
return [
|
226
250
|
_include_with_date_gap_fill(cycle, ["startDate", "endDate"]) |
|
227
251
|
_include(
|
228
252
|
_gap_filled_start_date(
|
@@ -233,20 +257,19 @@ def _get_relevant_items(cycle: dict, item_name: str, relevant_terms: list):
|
|
233
257
|
"startDate"
|
234
258
|
) |
|
235
259
|
item
|
236
|
-
for item in
|
260
|
+
for item in blank_nodes
|
237
261
|
]
|
238
|
-
return items
|
239
262
|
|
240
263
|
|
241
264
|
def _process_rule(node: dict, term: dict) -> list:
|
242
|
-
|
265
|
+
term_types = []
|
243
266
|
for column, condition, new_term in _INPUT_RULES[term.get('termType')]:
|
244
267
|
lookup_result = get_lookup_value(term, LOOKUPS[column], model=MODEL, term=term.get('@id'), model_key=MODEL_KEY)
|
245
268
|
|
246
269
|
if condition(lookup_result):
|
247
|
-
|
270
|
+
term_types.append(node | {'id': new_term})
|
248
271
|
|
249
|
-
return
|
272
|
+
return term_types
|
250
273
|
|
251
274
|
|
252
275
|
def _run_from_inputs(site: dict, cycle: dict) -> list:
|
@@ -307,7 +330,7 @@ def _run_from_landCover(cycle: dict, crop_forage_products: list):
|
|
307
330
|
)) for product in _get_relevant_items(
|
308
331
|
cycle=cycle,
|
309
332
|
item_name="products",
|
310
|
-
|
333
|
+
term_types=[TermTermType.LANDCOVER]
|
311
334
|
)
|
312
335
|
]
|
313
336
|
return land_cover_products + _run_products(
|
@@ -337,7 +360,7 @@ def _run_from_crop_forage(cycle: dict, site: dict):
|
|
337
360
|
products = _get_relevant_items(
|
338
361
|
cycle=cycle,
|
339
362
|
item_name="products",
|
340
|
-
|
363
|
+
term_types=[TermTermType.CROP, TermTermType.FORAGE]
|
341
364
|
) if site.get("siteType", "") == SiteSiteType.CROPLAND.value else []
|
342
365
|
# only take products with a matching landCover term
|
343
366
|
products = [p for p in products if get_landCover_term_id(p.get('term', {}))]
|
@@ -372,13 +395,8 @@ def _run_from_practices(cycle: dict):
|
|
372
395
|
) for practice in _get_relevant_items(
|
373
396
|
cycle=cycle,
|
374
397
|
item_name="practices",
|
375
|
-
|
376
|
-
|
377
|
-
TermTermType.TILLAGE,
|
378
|
-
TermTermType.CROPRESIDUEMANAGEMENT,
|
379
|
-
TermTermType.LANDUSEMANAGEMENT,
|
380
|
-
TermTermType.SYSTEM
|
381
|
-
]
|
398
|
+
term_types=_PRACTICES_TERM_TYPES,
|
399
|
+
completeness_mapping=_PRACTICES_COMPLETENESS_MAPPING
|
382
400
|
)
|
383
401
|
]
|
384
402
|
practices = list(map(_map_to_value, filter(_should_run_practice, practices)))
|
@@ -93,41 +93,46 @@ def _is_type(value: dict, ee_type: str):
|
|
93
93
|
]) if isinstance(params, list) else params.get('ee_type') == ee_type
|
94
94
|
|
95
95
|
|
96
|
-
def
|
96
|
+
def list_rasters(years: list = [], years_only: bool = False):
|
97
97
|
ee_params = list_ee_params()
|
98
98
|
# only cache `raster` results as can be combined in a single query
|
99
99
|
rasters = [value for value in ee_params if _is_type(value, 'raster')]
|
100
100
|
rasters = _extend_collections(rasters, years or [])
|
101
101
|
rasters = [raster for raster in rasters if not years_only or _is_collection_by_year(raster)]
|
102
102
|
|
103
|
+
return rasters
|
104
|
+
|
105
|
+
|
106
|
+
def list_vectors(sites: list):
|
107
|
+
ee_params = list_ee_params()
|
108
|
+
|
109
|
+
vectors = [value for value in ee_params if _is_type(value, 'vector')]
|
103
110
|
vectors = [
|
104
|
-
value for value in
|
105
|
-
|
106
|
-
)
|
111
|
+
value for value in vectors
|
112
|
+
# name of the model is the key in the data. If the key is present in all sites, we don't need to query
|
113
|
+
if all([not s.get(value.get('name')) for s in sites])
|
107
114
|
]
|
108
115
|
# no vectors are running with specific years
|
109
|
-
vectors =
|
116
|
+
vectors = _extend_collections(vectors)
|
110
117
|
|
111
|
-
return
|
118
|
+
return vectors
|
112
119
|
|
113
120
|
|
114
121
|
def _cache_results(site: dict, area_size: float):
|
115
122
|
# to fetch data related to the year
|
116
123
|
years = cached_value(site, key=CACHE_YEARS_KEY, default=[])
|
117
|
-
|
118
|
-
|
124
|
+
rasters = list_rasters(years)
|
125
|
+
vectors = list_vectors([site])
|
119
126
|
|
120
127
|
raster_results = _run_query({
|
121
128
|
'ee_type': 'raster',
|
122
|
-
'collections': rasters
|
123
|
-
|
124
|
-
})
|
129
|
+
'collections': rasters
|
130
|
+
} | geospatial_data(site)) if rasters else []
|
125
131
|
|
126
132
|
vector_results = _run_query({
|
127
133
|
'ee_type': 'vector',
|
128
|
-
'collections': vectors
|
129
|
-
|
130
|
-
})
|
134
|
+
'collections': vectors
|
135
|
+
} | geospatial_data(site)) if vectors else []
|
131
136
|
|
132
137
|
return cache_site_results(raster_results + vector_results, rasters + vectors, area_size)
|
133
138
|
|
@@ -73,7 +73,7 @@ def lookups_logs(model: str, blank_nodes: list, lookups_per_termType: dict, **lo
|
|
73
73
|
|
74
74
|
def _reduce_lookups_logs(logs: dict, column: str):
|
75
75
|
lookup_value = get_lookup_value(term, column, model=model, **log_args)
|
76
|
-
return logs | {column: lookup_value}
|
76
|
+
return logs | {column: str(lookup_value).replace(':', '-')}
|
77
77
|
|
78
78
|
return reduce(_reduce_lookups_logs, lookups, {'id': term_id})
|
79
79
|
|
@@ -270,7 +270,8 @@ def get_total_value_converted_with_min_ratio(
|
|
270
270
|
model: str, term: str, node: dict = {},
|
271
271
|
blank_nodes: list = [],
|
272
272
|
prop_id: str = 'energyContentHigherHeatingValue',
|
273
|
-
min_ratio: float = 0.8
|
273
|
+
min_ratio: float = 0.8,
|
274
|
+
is_sum: bool = True
|
274
275
|
):
|
275
276
|
values = [
|
276
277
|
(
|
@@ -301,9 +302,14 @@ def get_total_value_converted_with_min_ratio(
|
|
301
302
|
debugValues(node, model=model, term=term,
|
302
303
|
**logs)
|
303
304
|
|
304
|
-
|
305
|
+
total_converted_value = list_sum([
|
305
306
|
value * prop_value for term_id, value, prop_value in values if all([value, prop_value])
|
306
|
-
])
|
307
|
+
])
|
308
|
+
|
309
|
+
return (
|
310
|
+
total_converted_value * total_value / total_value_with_property if is_sum
|
311
|
+
else total_converted_value / total_value_with_property
|
312
|
+
) if total_value_ratio >= min_ratio else None
|
307
313
|
|
308
314
|
|
309
315
|
def get_N_total(nodes: list) -> list:
|
@@ -67,6 +67,6 @@ def get_landCover_term_id(lookup_term: dict, **log_args) -> str:
|
|
67
67
|
value = get_lookup_value(lookup_term, 'landCoverTermId', **log_args)
|
68
68
|
return (
|
69
69
|
lookup_term.get("@id") if lookup_term.get("termType") == TermTermType.LANDCOVER.value else
|
70
|
-
value.split(';')[0] if value else
|
70
|
+
value.split(';')[0] if isinstance(value, str) else
|
71
71
|
None
|
72
72
|
)
|
@@ -291,7 +291,7 @@ def _land_occupation_per_kg(model: str, term_id: str, cycle: dict, product: dict
|
|
291
291
|
return value
|
292
292
|
|
293
293
|
|
294
|
-
def land_occupation_per_kg(model: str, term_id: str, cycle: dict, site: dict,
|
294
|
+
def land_occupation_per_kg(model: str, term_id: str, cycle: dict, site: dict, product: dict):
|
295
295
|
"""
|
296
296
|
Get the land occupation of the cycle per kg in meter square.
|
297
297
|
|
@@ -305,7 +305,7 @@ def land_occupation_per_kg(model: str, term_id: str, cycle: dict, site: dict, pr
|
|
305
305
|
The `Cycle` as defined in the HESTIA Schema.
|
306
306
|
site : dict
|
307
307
|
The `Site` as defined in the HESTIA Schema.
|
308
|
-
|
308
|
+
product : dict
|
309
309
|
The primary `Product` of the `Cycle`.
|
310
310
|
|
311
311
|
Returns
|
@@ -325,7 +325,7 @@ def land_occupation_per_kg(model: str, term_id: str, cycle: dict, site: dict, pr
|
|
325
325
|
SiteSiteType.RIVER_OR_STREAM.value,
|
326
326
|
SiteSiteType.SEA_OR_OCEAN.value
|
327
327
|
] else (
|
328
|
-
_land_occupation_per_kg(model, term_id, cycle,
|
328
|
+
_land_occupation_per_kg(model, term_id, cycle, product, value) if value is not None else None
|
329
329
|
)
|
330
330
|
|
331
331
|
|
@@ -53,7 +53,7 @@ def all_factor_value(
|
|
53
53
|
all_with_factors = all([v.get('coefficient') is not None for v in values if v.get('value') is not None])
|
54
54
|
|
55
55
|
for missing_value in missing_values:
|
56
|
-
debugMissingLookup(lookup_name, 'termid', missing_value, lookup_col, None)
|
56
|
+
debugMissingLookup(lookup_name, 'termid', missing_value, lookup_col, None, model=model, term=term_id)
|
57
57
|
|
58
58
|
debugValues(node, model=model, term=term_id,
|
59
59
|
all_with_factors=all_with_factors,
|
hestia_earth/models/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
VERSION = '0.
|
1
|
+
VERSION = '0.66.0'
|