hestia-earth-utils 0.16.9__py3-none-any.whl → 0.16.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hestia_earth/utils/api.py +78 -36
- hestia_earth/utils/blank_node.py +101 -60
- hestia_earth/utils/calculation_status.py +45 -35
- hestia_earth/utils/cycle.py +7 -7
- hestia_earth/utils/date.py +7 -2
- hestia_earth/utils/descriptive_stats.py +10 -6
- hestia_earth/utils/emission.py +26 -15
- hestia_earth/utils/lookup.py +62 -28
- hestia_earth/utils/lookup_utils.py +89 -63
- hestia_earth/utils/model.py +45 -40
- hestia_earth/utils/pipeline.py +179 -90
- hestia_earth/utils/pivot/_shared.py +16 -12
- hestia_earth/utils/pivot/pivot_csv.py +35 -18
- hestia_earth/utils/pivot/pivot_json.py +34 -18
- hestia_earth/utils/request.py +17 -6
- hestia_earth/utils/stats.py +89 -68
- hestia_earth/utils/storage/_azure_client.py +17 -6
- hestia_earth/utils/storage/_local_client.py +8 -3
- hestia_earth/utils/storage/_s3_client.py +27 -22
- hestia_earth/utils/storage/_sns_client.py +7 -2
- hestia_earth/utils/term.py +5 -5
- hestia_earth/utils/tools.py +50 -21
- hestia_earth/utils/version.py +1 -1
- {hestia_earth_utils-0.16.9.dist-info → hestia_earth_utils-0.16.10.dist-info}/METADATA +1 -1
- hestia_earth_utils-0.16.10.dist-info/RECORD +33 -0
- hestia_earth_utils-0.16.9.dist-info/RECORD +0 -33
- {hestia_earth_utils-0.16.9.data → hestia_earth_utils-0.16.10.data}/scripts/hestia-format-upload +0 -0
- {hestia_earth_utils-0.16.9.data → hestia_earth_utils-0.16.10.data}/scripts/hestia-pivot-csv +0 -0
- {hestia_earth_utils-0.16.9.dist-info → hestia_earth_utils-0.16.10.dist-info}/WHEEL +0 -0
- {hestia_earth_utils-0.16.9.dist-info → hestia_earth_utils-0.16.10.dist-info}/top_level.txt +0 -0
hestia_earth/utils/emission.py
CHANGED
|
@@ -17,7 +17,9 @@ from .lookup_utils import (
|
|
|
17
17
|
)
|
|
18
18
|
|
|
19
19
|
|
|
20
|
-
def emissions_in_system_boundary(
|
|
20
|
+
def emissions_in_system_boundary(
|
|
21
|
+
termType: TermTermType = TermTermType.EMISSION,
|
|
22
|
+
) -> List[str]:
|
|
21
23
|
"""
|
|
22
24
|
Get all emissions included in HESTIA system boundary.
|
|
23
25
|
|
|
@@ -33,23 +35,30 @@ def emissions_in_system_boundary(termType: TermTermType = TermTermType.EMISSION)
|
|
|
33
35
|
|
|
34
36
|
def cycle_emission_is_in_system_boundary(cycle: dict):
|
|
35
37
|
def filter_term(term_id: str):
|
|
36
|
-
return is_in_system_boundary(term_id) and all(
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
38
|
+
return is_in_system_boundary(term_id) and all(
|
|
39
|
+
map(
|
|
40
|
+
lambda check: check(cycle, term_id),
|
|
41
|
+
[
|
|
42
|
+
is_siteType_allowed,
|
|
43
|
+
is_site_measurement_id_allowed,
|
|
44
|
+
is_product_termType_allowed,
|
|
45
|
+
is_product_id_allowed,
|
|
46
|
+
is_input_termType_allowed,
|
|
47
|
+
is_input_id_allowed,
|
|
48
|
+
is_practice_termType_allowed,
|
|
49
|
+
is_practice_id_allowed,
|
|
50
|
+
is_transformation_termType_allowed,
|
|
51
|
+
is_transformation_id_allowed,
|
|
52
|
+
],
|
|
53
|
+
)
|
|
54
|
+
)
|
|
48
55
|
|
|
49
56
|
return filter_term
|
|
50
57
|
|
|
51
58
|
|
|
52
|
-
def cycle_emissions_in_system_boundary(
|
|
59
|
+
def cycle_emissions_in_system_boundary(
|
|
60
|
+
cycle: dict, termType: TermTermType = TermTermType.EMISSION
|
|
61
|
+
):
|
|
53
62
|
"""
|
|
54
63
|
Get all emissions relevant for the Cycle, included in HESTIA system boundary.
|
|
55
64
|
|
|
@@ -60,4 +69,6 @@ def cycle_emissions_in_system_boundary(cycle: dict, termType: TermTermType = Ter
|
|
|
60
69
|
"""
|
|
61
70
|
lookup = download_lookup(f"{termType.value}.csv")
|
|
62
71
|
# find all emissions in system boundary
|
|
63
|
-
return list(
|
|
72
|
+
return list(
|
|
73
|
+
filter(cycle_emission_is_in_system_boundary(cycle), lookup_term_ids(lookup))
|
|
74
|
+
)
|
hestia_earth/utils/lookup.py
CHANGED
|
@@ -7,7 +7,7 @@ import pandas as pd
|
|
|
7
7
|
from .storage import _load_from_storage
|
|
8
8
|
from .request import request_url, web_url
|
|
9
9
|
|
|
10
|
-
_GLOSSARY_FOLDER =
|
|
10
|
+
_GLOSSARY_FOLDER = "glossary/lookups"
|
|
11
11
|
_memory = {}
|
|
12
12
|
|
|
13
13
|
|
|
@@ -18,7 +18,7 @@ def _memory_wrapper(key: str, func):
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
def _read_csv(value: str) -> pd.DataFrame:
|
|
21
|
-
return pd.read_csv(value, na_values=[
|
|
21
|
+
return pd.read_csv(value, na_values=["-", ""])
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def _read_csv_from_string(data: str) -> pd.DataFrame:
|
|
@@ -26,7 +26,7 @@ def _read_csv_from_string(data: str) -> pd.DataFrame:
|
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
def is_missing_value(value):
|
|
29
|
-
return pd.isna(value) or value is None or value ==
|
|
29
|
+
return pd.isna(value) or value is None or value == "" or value == "-"
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
def load_lookup(filepath: str, keep_in_memory: bool = False):
|
|
@@ -45,7 +45,10 @@ def load_lookup(filepath: str, keep_in_memory: bool = False):
|
|
|
45
45
|
numpy.recarray
|
|
46
46
|
The `numpy.recarray` converted from the csv content.
|
|
47
47
|
"""
|
|
48
|
-
|
|
48
|
+
|
|
49
|
+
def load():
|
|
50
|
+
return _read_csv(filepath)
|
|
51
|
+
|
|
49
52
|
return _memory_wrapper(filepath, load) if keep_in_memory else load()
|
|
50
53
|
|
|
51
54
|
|
|
@@ -54,12 +57,12 @@ def _download_lookup_data(filename: str):
|
|
|
54
57
|
|
|
55
58
|
def fallback():
|
|
56
59
|
url = request_url(f"{web_url()}/{filepath}")
|
|
57
|
-
data = requests.get(url).content.decode(
|
|
58
|
-
return data if data and
|
|
60
|
+
data = requests.get(url).content.decode("utf-8")
|
|
61
|
+
return data if data and "<html" not in data else None
|
|
59
62
|
|
|
60
63
|
try:
|
|
61
64
|
data = _load_from_storage(filepath, glossary=True)
|
|
62
|
-
return data.decode(
|
|
65
|
+
return data.decode("utf-8") if data else None
|
|
63
66
|
except ImportError:
|
|
64
67
|
return fallback()
|
|
65
68
|
|
|
@@ -82,6 +85,7 @@ def download_lookup(filename: str, keep_in_memory: bool = True):
|
|
|
82
85
|
numpy.recarray
|
|
83
86
|
The `numpy.recarray` converted from the csv content.
|
|
84
87
|
"""
|
|
88
|
+
|
|
85
89
|
def load():
|
|
86
90
|
data = _download_lookup_data(filename)
|
|
87
91
|
return _read_csv_from_string(data) if data else None
|
|
@@ -100,12 +104,12 @@ def column_name(key: str):
|
|
|
100
104
|
|
|
101
105
|
|
|
102
106
|
def _parse_value(value: str):
|
|
103
|
-
"""
|
|
107
|
+
"""Automatically converts the value to float or bool if possible"""
|
|
104
108
|
try:
|
|
105
109
|
return (
|
|
106
|
-
True
|
|
107
|
-
|
|
108
|
-
float(value)
|
|
110
|
+
True
|
|
111
|
+
if str(value).lower() == "true"
|
|
112
|
+
else False if str(value).lower() == "false" else float(value)
|
|
109
113
|
)
|
|
110
114
|
except Exception:
|
|
111
115
|
return value
|
|
@@ -116,7 +120,13 @@ def _get_single_table_value(df: pd.DataFrame, col_match: str, col_match_with, co
|
|
|
116
120
|
return None if filtered_df.empty else filtered_df[col_val].iloc[0]
|
|
117
121
|
|
|
118
122
|
|
|
119
|
-
def get_table_value(
|
|
123
|
+
def get_table_value(
|
|
124
|
+
lookup: pd.DataFrame,
|
|
125
|
+
col_match: str,
|
|
126
|
+
col_match_with: str,
|
|
127
|
+
col_val: Any,
|
|
128
|
+
default_value="",
|
|
129
|
+
):
|
|
120
130
|
"""
|
|
121
131
|
Get a value matched by one or more columns from a `numpy.recarray`.
|
|
122
132
|
|
|
@@ -166,7 +176,11 @@ def find_term_ids_by(lookup: pd.DataFrame, col_match: str, col_match_with: str):
|
|
|
166
176
|
The list of `term.id` that matched the expected column value.
|
|
167
177
|
"""
|
|
168
178
|
filtered_df = lookup[lookup[col_match] == col_match_with]
|
|
169
|
-
term_ids =
|
|
179
|
+
term_ids = (
|
|
180
|
+
filtered_df["term.id"].unique().tolist()
|
|
181
|
+
if "term.id" in filtered_df.columns
|
|
182
|
+
else []
|
|
183
|
+
)
|
|
170
184
|
return list(map(str, term_ids))
|
|
171
185
|
|
|
172
186
|
|
|
@@ -190,10 +204,15 @@ def extract_grouped_data(data: str, key: str) -> str:
|
|
|
190
204
|
str
|
|
191
205
|
The value found or `None` if no match.
|
|
192
206
|
"""
|
|
193
|
-
grouped_data =
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
207
|
+
grouped_data = (
|
|
208
|
+
reduce(
|
|
209
|
+
lambda prev, curr: {**prev, **{curr.split(":")[0]: curr.split(":")[1]}},
|
|
210
|
+
data.split(";"),
|
|
211
|
+
{},
|
|
212
|
+
)
|
|
213
|
+
if data is not None and isinstance(data, str) and len(data) > 1
|
|
214
|
+
else {}
|
|
215
|
+
)
|
|
197
216
|
value = grouped_data.get(key)
|
|
198
217
|
return None if is_missing_value(value) else _parse_value(value)
|
|
199
218
|
|
|
@@ -218,17 +237,28 @@ def extract_grouped_data_closest_date(data: str, year: int) -> str:
|
|
|
218
237
|
str
|
|
219
238
|
The closest value found.
|
|
220
239
|
"""
|
|
221
|
-
data_by_date =
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
240
|
+
data_by_date = (
|
|
241
|
+
reduce(
|
|
242
|
+
lambda prev, curr: (
|
|
243
|
+
{**prev, **{curr.split(":")[0]: curr.split(":")[1]}}
|
|
244
|
+
if len(curr) > 0 and not is_missing_value(curr.split(":")[1])
|
|
245
|
+
else prev
|
|
246
|
+
),
|
|
247
|
+
data.split(";"),
|
|
248
|
+
{},
|
|
249
|
+
)
|
|
250
|
+
if data is not None and isinstance(data, str) and len(data) > 1
|
|
251
|
+
else {}
|
|
252
|
+
)
|
|
229
253
|
dist_years = list(data_by_date.keys())
|
|
230
|
-
closest_year =
|
|
231
|
-
|
|
254
|
+
closest_year = (
|
|
255
|
+
min(dist_years, key=lambda x: abs(int(x) - year))
|
|
256
|
+
if len(dist_years) > 0
|
|
257
|
+
else None
|
|
258
|
+
)
|
|
259
|
+
return (
|
|
260
|
+
None if closest_year is None else _parse_value(data_by_date.get(closest_year))
|
|
261
|
+
)
|
|
232
262
|
|
|
233
263
|
|
|
234
264
|
def lookup_term_ids(lookup: pd.DataFrame):
|
|
@@ -245,7 +275,11 @@ def lookup_term_ids(lookup: pd.DataFrame):
|
|
|
245
275
|
list[str]
|
|
246
276
|
The `term.id` values from the lookup.
|
|
247
277
|
"""
|
|
248
|
-
return
|
|
278
|
+
return (
|
|
279
|
+
list(map(str, lookup["term.id"].tolist()))
|
|
280
|
+
if "term.id" in lookup.columns
|
|
281
|
+
else []
|
|
282
|
+
)
|
|
249
283
|
|
|
250
284
|
|
|
251
285
|
def lookup_columns(lookup: pd.DataFrame):
|
|
@@ -6,153 +6,173 @@ from .lookup import _download_lookup_data, download_lookup, get_table_value
|
|
|
6
6
|
from .api import download_hestia
|
|
7
7
|
from .tools import non_empty_list, flatten
|
|
8
8
|
|
|
9
|
-
_ALLOW_ALL =
|
|
9
|
+
_ALLOW_ALL = "all"
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
@lru_cache()
|
|
13
13
|
def _allowed_mapping_data():
|
|
14
|
-
data = _download_lookup_data(
|
|
14
|
+
data = _download_lookup_data("allowed-mapping.json")
|
|
15
15
|
data = json.loads(data) if data else {}
|
|
16
16
|
return data
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
def _is_site(site: dict):
|
|
20
|
-
return site.get(
|
|
20
|
+
return site.get("@type", site.get("type")) == "Site" if site else None
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def _get_sites(node: dict):
|
|
24
|
-
site = node.get(
|
|
25
|
-
other_sites = node.get(
|
|
24
|
+
site = node.get("site", node.get("cycle", {}).get("site"))
|
|
25
|
+
other_sites = node.get("otherSites", node.get("cycle", {}).get("otherSites", []))
|
|
26
26
|
return non_empty_list([site] + other_sites)
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
def _get_site_types(node: dict):
|
|
30
30
|
sites = [node] if _is_site(node) else _get_sites(node)
|
|
31
|
-
return non_empty_list([site.get(
|
|
31
|
+
return non_empty_list([site.get("siteType") for site in sites])
|
|
32
32
|
|
|
33
33
|
|
|
34
34
|
def _get_site_measurements(node: dict):
|
|
35
35
|
sites = [node] if _is_site(node) else _get_sites(node)
|
|
36
|
-
return flatten([non_empty_list(site.get(
|
|
36
|
+
return flatten([non_empty_list(site.get("measurements", [])) for site in sites])
|
|
37
37
|
|
|
38
38
|
|
|
39
|
-
def _blank_node_term_values(blank_nodes: list, key: str =
|
|
40
|
-
return non_empty_list([v.get(
|
|
39
|
+
def _blank_node_term_values(blank_nodes: list, key: str = "@id"):
|
|
40
|
+
return non_empty_list([v.get("term", {}).get(key) for v in blank_nodes])
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
@lru_cache()
|
|
44
44
|
def _allowed_model_mapping(model: str, term_id: str, column: str):
|
|
45
45
|
mapping = _allowed_mapping_data()
|
|
46
|
-
value =
|
|
47
|
-
|
|
48
|
-
|
|
46
|
+
value = (
|
|
47
|
+
mapping.get(term_id, {}).get(model, {}).get(column)
|
|
48
|
+
if mapping
|
|
49
|
+
else get_table_value(
|
|
50
|
+
download_lookup(
|
|
51
|
+
f"{(download_hestia(term_id) or {}).get('termType')}-model-{column}.csv"
|
|
52
|
+
),
|
|
53
|
+
"term.id",
|
|
54
|
+
term_id,
|
|
55
|
+
column,
|
|
56
|
+
)
|
|
49
57
|
)
|
|
50
|
-
return (value or _ALLOW_ALL).split(
|
|
58
|
+
return (value or _ALLOW_ALL).split(";") if isinstance(value, str) else _ALLOW_ALL
|
|
51
59
|
|
|
52
60
|
|
|
53
61
|
def _is_model_value_allowed(model: str, term_id: str, values: list, lookup_column: str):
|
|
54
62
|
allowed_values = _allowed_model_mapping(model, term_id, lookup_column)
|
|
55
|
-
return any([
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
]) or any([value in allowed_values for value in values])
|
|
63
|
+
return any([_ALLOW_ALL in allowed_values, len(values) == 0]) or any(
|
|
64
|
+
[value in allowed_values for value in values]
|
|
65
|
+
)
|
|
59
66
|
|
|
60
67
|
|
|
61
68
|
def is_model_siteType_allowed(model: str, term_id: str, data: dict):
|
|
62
69
|
values = _get_site_types(data)
|
|
63
|
-
return _is_model_value_allowed(model, term_id, values,
|
|
70
|
+
return _is_model_value_allowed(model, term_id, values, "siteTypesAllowed")
|
|
64
71
|
|
|
65
72
|
|
|
66
73
|
def is_model_product_id_allowed(model: str, term_id: str, data: dict):
|
|
67
|
-
values = _blank_node_term_values(data.get(
|
|
68
|
-
return _is_model_value_allowed(model, term_id, values,
|
|
74
|
+
values = _blank_node_term_values(data.get("products", []))
|
|
75
|
+
return _is_model_value_allowed(model, term_id, values, "productTermIdsAllowed")
|
|
69
76
|
|
|
70
77
|
|
|
71
78
|
def is_model_measurement_id_allowed(model: str, term_id: str, data: dict):
|
|
72
|
-
values = _blank_node_term_values(data.get(
|
|
73
|
-
return _is_model_value_allowed(model, term_id, values,
|
|
79
|
+
values = _blank_node_term_values(data.get("measurements", []))
|
|
80
|
+
return _is_model_value_allowed(model, term_id, values, "measurementIdsAllowed")
|
|
74
81
|
|
|
75
82
|
|
|
76
83
|
@lru_cache()
|
|
77
84
|
def _allowed_mapping(term_id: str, column: str):
|
|
78
85
|
mapping = _allowed_mapping_data()
|
|
79
|
-
value =
|
|
80
|
-
|
|
81
|
-
|
|
86
|
+
value = (
|
|
87
|
+
mapping.get(term_id, {}).get(column)
|
|
88
|
+
if mapping
|
|
89
|
+
else get_table_value(
|
|
90
|
+
download_lookup(f"{(download_hestia(term_id) or {}).get('termType')}.csv"),
|
|
91
|
+
"term.id",
|
|
92
|
+
term_id,
|
|
93
|
+
column,
|
|
94
|
+
)
|
|
82
95
|
)
|
|
83
|
-
return (value or _ALLOW_ALL).split(
|
|
96
|
+
return (value or _ALLOW_ALL).split(";") if isinstance(value, str) else _ALLOW_ALL
|
|
84
97
|
|
|
85
98
|
|
|
86
99
|
def _is_term_value_allowed(term_id: str, values: list, lookup_column: str):
|
|
87
100
|
allowed_values = _allowed_mapping(term_id, lookup_column)
|
|
88
|
-
return any([
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
]) or any([value in allowed_values for value in values])
|
|
101
|
+
return any([_ALLOW_ALL in allowed_values, len(values) == 0]) or any(
|
|
102
|
+
[value in allowed_values for value in values]
|
|
103
|
+
)
|
|
92
104
|
|
|
93
105
|
|
|
94
106
|
def is_siteType_allowed(data: dict, term_id: str):
|
|
95
107
|
values = _get_site_types(data)
|
|
96
|
-
return _is_term_value_allowed(term_id, values,
|
|
108
|
+
return _is_term_value_allowed(term_id, values, "siteTypesAllowed")
|
|
97
109
|
|
|
98
110
|
|
|
99
111
|
def is_site_measurement_id_allowed(data: dict, term_id: str):
|
|
100
112
|
measurements = _get_site_measurements(data)
|
|
101
|
-
values = _blank_node_term_values(measurements, key=
|
|
102
|
-
return _is_term_value_allowed(term_id, values,
|
|
113
|
+
values = _blank_node_term_values(measurements, key="@id")
|
|
114
|
+
return _is_term_value_allowed(term_id, values, "siteMeasurementIdsAllowed")
|
|
103
115
|
|
|
104
116
|
|
|
105
117
|
def is_product_termType_allowed(data: dict, term_id: str):
|
|
106
|
-
products = data.get(
|
|
107
|
-
values = _blank_node_term_values(products, key=
|
|
108
|
-
return _is_term_value_allowed(term_id, values,
|
|
118
|
+
products = data.get("products", [])
|
|
119
|
+
values = _blank_node_term_values(products, key="termType")
|
|
120
|
+
return _is_term_value_allowed(term_id, values, "productTermTypesAllowed")
|
|
109
121
|
|
|
110
122
|
|
|
111
123
|
def is_product_id_allowed(data: dict, term_id: str):
|
|
112
|
-
products = data.get(
|
|
113
|
-
values = _blank_node_term_values(products, key=
|
|
114
|
-
return _is_term_value_allowed(term_id, values,
|
|
124
|
+
products = data.get("products", [])
|
|
125
|
+
values = _blank_node_term_values(products, key="@id")
|
|
126
|
+
return _is_term_value_allowed(term_id, values, "productTermIdsAllowed")
|
|
115
127
|
|
|
116
128
|
|
|
117
129
|
def is_input_termType_allowed(data: dict, term_id: str):
|
|
118
|
-
inputs = data.get(
|
|
119
|
-
values = _blank_node_term_values(inputs, key=
|
|
120
|
-
return _is_term_value_allowed(term_id, values,
|
|
130
|
+
inputs = data.get("inputs", [])
|
|
131
|
+
values = _blank_node_term_values(inputs, key="termType")
|
|
132
|
+
return _is_term_value_allowed(term_id, values, "inputTermTypesAllowed")
|
|
121
133
|
|
|
122
134
|
|
|
123
135
|
def is_input_id_allowed(data: dict, term_id: str):
|
|
124
|
-
inputs = data.get(
|
|
125
|
-
values = _blank_node_term_values(inputs, key=
|
|
126
|
-
return _is_term_value_allowed(term_id, values,
|
|
136
|
+
inputs = data.get("inputs", [])
|
|
137
|
+
values = _blank_node_term_values(inputs, key="@id")
|
|
138
|
+
return _is_term_value_allowed(term_id, values, "inputTermIdsAllowed")
|
|
127
139
|
|
|
128
140
|
|
|
129
141
|
def is_practice_termType_allowed(data: dict, term_id: str):
|
|
130
|
-
practices = data.get(
|
|
131
|
-
values = _blank_node_term_values(practices, key=
|
|
132
|
-
return _is_term_value_allowed(term_id, values,
|
|
142
|
+
practices = data.get("practices", [])
|
|
143
|
+
values = _blank_node_term_values(practices, key="termType")
|
|
144
|
+
return _is_term_value_allowed(term_id, values, "practiceTermTypesAllowed")
|
|
133
145
|
|
|
134
146
|
|
|
135
147
|
def is_practice_id_allowed(data: dict, term_id: str):
|
|
136
|
-
practices = data.get(
|
|
137
|
-
values = _blank_node_term_values(practices, key=
|
|
138
|
-
return _is_term_value_allowed(term_id, values,
|
|
148
|
+
practices = data.get("practices", [])
|
|
149
|
+
values = _blank_node_term_values(practices, key="@id")
|
|
150
|
+
return _is_term_value_allowed(term_id, values, "practiceTermIdsAllowed")
|
|
139
151
|
|
|
140
152
|
|
|
141
153
|
def is_transformation_termType_allowed(data: dict, term_id: str):
|
|
142
|
-
is_transformation =
|
|
143
|
-
|
|
144
|
-
|
|
154
|
+
is_transformation = (
|
|
155
|
+
data.get("@type", data.get("type")) == SchemaType.TRANSFORMATION.value
|
|
156
|
+
)
|
|
157
|
+
values = non_empty_list([data.get("term", {}).get("termType")])
|
|
158
|
+
return not is_transformation or _is_term_value_allowed(
|
|
159
|
+
term_id, values, "transformationTermTypesAllowed"
|
|
160
|
+
)
|
|
145
161
|
|
|
146
162
|
|
|
147
163
|
def is_transformation_id_allowed(data: dict, term_id: str):
|
|
148
|
-
is_transformation =
|
|
149
|
-
|
|
150
|
-
|
|
164
|
+
is_transformation = (
|
|
165
|
+
data.get("@type", data.get("type")) == SchemaType.TRANSFORMATION.value
|
|
166
|
+
)
|
|
167
|
+
values = non_empty_list([data.get("term", {}).get("@id")])
|
|
168
|
+
return not is_transformation or _is_term_value_allowed(
|
|
169
|
+
term_id, values, "transformationTermIdsAllowed"
|
|
170
|
+
)
|
|
151
171
|
|
|
152
172
|
|
|
153
173
|
def is_node_type_allowed(data: dict, term_id: str):
|
|
154
|
-
values = non_empty_list([data.get(
|
|
155
|
-
return _is_term_value_allowed(term_id, values,
|
|
174
|
+
values = non_empty_list([data.get("@type", data.get("type"))])
|
|
175
|
+
return _is_term_value_allowed(term_id, values, "typesAllowed")
|
|
156
176
|
|
|
157
177
|
|
|
158
178
|
@lru_cache()
|
|
@@ -171,10 +191,16 @@ def is_in_system_boundary(term_id: str) -> bool:
|
|
|
171
191
|
True if the Term is included in the HESTIA system boundary, False otherwise.
|
|
172
192
|
"""
|
|
173
193
|
mapping = _allowed_mapping_data()
|
|
174
|
-
column =
|
|
175
|
-
value =
|
|
176
|
-
|
|
177
|
-
|
|
194
|
+
column = "inHestiaDefaultSystemBoundary"
|
|
195
|
+
value = (
|
|
196
|
+
mapping.get(term_id, {}).get(column)
|
|
197
|
+
if mapping
|
|
198
|
+
else get_table_value(
|
|
199
|
+
download_lookup(f"{(download_hestia(term_id) or {}).get('termType')}.csv"),
|
|
200
|
+
"term.id",
|
|
201
|
+
term_id,
|
|
202
|
+
column,
|
|
203
|
+
)
|
|
178
204
|
)
|
|
179
205
|
# handle numpy bool from table value
|
|
180
206
|
return not (not value)
|
hestia_earth/utils/model.py
CHANGED
|
@@ -4,8 +4,8 @@ from hestia_earth.schema import TermTermType, NodeType
|
|
|
4
4
|
from .tools import get_dict_key
|
|
5
5
|
|
|
6
6
|
_LINKED_NODE_KEYS = {
|
|
7
|
-
NodeType.SOURCE.value: [
|
|
8
|
-
NodeType.TERM.value: [
|
|
7
|
+
NodeType.SOURCE.value: ["name"],
|
|
8
|
+
NodeType.TERM.value: ["name", "termType", "units"],
|
|
9
9
|
}
|
|
10
10
|
|
|
11
11
|
|
|
@@ -23,12 +23,14 @@ def linked_node(node: dict):
|
|
|
23
23
|
dict
|
|
24
24
|
The node with only: `@type`, `@id`, `name`, `termType` and `units` fields.
|
|
25
25
|
"""
|
|
26
|
-
node_type = node.get(
|
|
27
|
-
keys = [
|
|
26
|
+
node_type = node.get("@type", "")
|
|
27
|
+
keys = ["@type", "@id"] + _LINKED_NODE_KEYS.get(node_type, [])
|
|
28
28
|
return {key: node[key] for key in keys if key in node}
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
def find_term_match(
|
|
31
|
+
def find_term_match(
|
|
32
|
+
values: list, term_id: str, default_val: Any = {}, match_params: dict = {}
|
|
33
|
+
):
|
|
32
34
|
"""
|
|
33
35
|
Return the element in a list which matches the `Term` with the given `@id`.
|
|
34
36
|
|
|
@@ -48,17 +50,19 @@ def find_term_match(values: list, term_id: str, default_val: Any = {}, match_par
|
|
|
48
50
|
dict
|
|
49
51
|
The matching object.
|
|
50
52
|
"""
|
|
53
|
+
|
|
51
54
|
def is_match(value: dict):
|
|
52
|
-
return all(
|
|
53
|
-
value.get(
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
])
|
|
55
|
+
return all(
|
|
56
|
+
[value.get("term", {}).get("@id") == term_id]
|
|
57
|
+
+ [get_dict_key(value, k) == v for k, v in match_params.items()]
|
|
58
|
+
)
|
|
57
59
|
|
|
58
60
|
return next((v for v in values if is_match(v)), default_val)
|
|
59
61
|
|
|
60
62
|
|
|
61
|
-
def filter_list_term_type(
|
|
63
|
+
def filter_list_term_type(
|
|
64
|
+
values: list, term_type: Union[TermTermType, str, List[TermTermType], List[str]]
|
|
65
|
+
) -> list:
|
|
62
66
|
"""
|
|
63
67
|
Filters the values by filtering by the `term.termType` property.
|
|
64
68
|
|
|
@@ -76,7 +80,9 @@ def filter_list_term_type(values: list, term_type: Union[TermTermType, str, List
|
|
|
76
80
|
"""
|
|
77
81
|
term_types = [t for t in term_type] if isinstance(term_type, list) else [term_type]
|
|
78
82
|
term_types = [(t if isinstance(t, str) else t.value) for t in term_types]
|
|
79
|
-
return list(
|
|
83
|
+
return list(
|
|
84
|
+
filter(lambda i: i.get("term", {}).get("termType") in term_types, values)
|
|
85
|
+
)
|
|
80
86
|
|
|
81
87
|
|
|
82
88
|
def find_primary_product(cycle: dict) -> dict:
|
|
@@ -93,53 +99,52 @@ def find_primary_product(cycle: dict) -> dict:
|
|
|
93
99
|
dict
|
|
94
100
|
The primary `Product`.
|
|
95
101
|
"""
|
|
96
|
-
products = cycle.get(
|
|
97
|
-
return
|
|
102
|
+
products = cycle.get("products", [])
|
|
103
|
+
return (
|
|
104
|
+
next((p for p in products if p.get("primary", False)), products[0])
|
|
105
|
+
if len(products) > 0
|
|
106
|
+
else None
|
|
107
|
+
)
|
|
98
108
|
|
|
99
109
|
|
|
100
|
-
def _convert_m3_to_kg(value: float, **kwargs):
|
|
110
|
+
def _convert_m3_to_kg(value: float, **kwargs):
|
|
111
|
+
return value * kwargs.get("density")
|
|
101
112
|
|
|
102
113
|
|
|
103
|
-
def _convert_m3_to_l(value: float, **kwargs):
|
|
114
|
+
def _convert_m3_to_l(value: float, **kwargs):
|
|
115
|
+
return value * 1000
|
|
104
116
|
|
|
105
117
|
|
|
106
|
-
def _convert_kg_to_m3(value: float, **kwargs):
|
|
118
|
+
def _convert_kg_to_m3(value: float, **kwargs):
|
|
119
|
+
return value / kwargs.get("density")
|
|
107
120
|
|
|
108
121
|
|
|
109
|
-
def _convert_kg_to_l(value: float, **kwargs):
|
|
122
|
+
def _convert_kg_to_l(value: float, **kwargs):
|
|
123
|
+
return value / kwargs.get("density") * 1000
|
|
110
124
|
|
|
111
125
|
|
|
112
|
-
def _convert_liter_to_kg(value: float, **kwargs):
|
|
126
|
+
def _convert_liter_to_kg(value: float, **kwargs):
|
|
127
|
+
return value * kwargs.get("density") / 1000
|
|
113
128
|
|
|
114
129
|
|
|
115
|
-
def _convert_liter_to_m3(value: float, **kwargs):
|
|
130
|
+
def _convert_liter_to_m3(value: float, **kwargs):
|
|
131
|
+
return value / 1000
|
|
116
132
|
|
|
117
133
|
|
|
118
|
-
def _convert_mj_to_kwh(value: float, **kwargs):
|
|
134
|
+
def _convert_mj_to_kwh(value: float, **kwargs):
|
|
135
|
+
return value / 3.6
|
|
119
136
|
|
|
120
137
|
|
|
121
|
-
def _convert_kwh_to_mj(value: float, **kwargs):
|
|
138
|
+
def _convert_kwh_to_mj(value: float, **kwargs):
|
|
139
|
+
return value * 3.6
|
|
122
140
|
|
|
123
141
|
|
|
124
142
|
CONVERTERS = {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
},
|
|
129
|
-
|
|
130
|
-
'm3': _convert_kg_to_m3,
|
|
131
|
-
'L': _convert_kg_to_l
|
|
132
|
-
},
|
|
133
|
-
'L': {
|
|
134
|
-
'kg': _convert_liter_to_kg,
|
|
135
|
-
'm3': _convert_liter_to_m3
|
|
136
|
-
},
|
|
137
|
-
'kWh': {
|
|
138
|
-
'MJ': _convert_kwh_to_mj
|
|
139
|
-
},
|
|
140
|
-
'MJ': {
|
|
141
|
-
'kWh': _convert_mj_to_kwh
|
|
142
|
-
}
|
|
143
|
+
"m3": {"kg": _convert_m3_to_kg, "L": _convert_m3_to_l},
|
|
144
|
+
"kg": {"m3": _convert_kg_to_m3, "L": _convert_kg_to_l},
|
|
145
|
+
"L": {"kg": _convert_liter_to_kg, "m3": _convert_liter_to_m3},
|
|
146
|
+
"kWh": {"MJ": _convert_kwh_to_mj},
|
|
147
|
+
"MJ": {"kWh": _convert_mj_to_kwh},
|
|
143
148
|
}
|
|
144
149
|
|
|
145
150
|
|