hestia-earth-utils 0.16.14__tar.gz → 0.16.16__tar.gz

{hestia_earth_utils-0.16.14 → hestia_earth_utils-0.16.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.14
+Version: 0.16.16
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team
@@ -13,7 +13,7 @@ Requires-Dist: hestia-earth-schema>=35.0.1
 Requires-Dist: requests>=2.24.0
 Requires-Dist: urllib3~=1.26.0
 Requires-Dist: python-dateutil>=2.8.1
-Requires-Dist: pandas>=2
+Requires-Dist: pandas<3,>=2
 Requires-Dist: flatten_json
 Dynamic: author
 Dynamic: author-email

{hestia_earth_utils-0.16.14 → hestia_earth_utils-0.16.16}/hestia_earth/utils/blank_node.py

@@ -186,6 +186,38 @@ def get_node_value(
 _BLANK_NODE_GROUPING_KEYS = {TermTermType.EMISSION: ["methodModel"]}
 
 
+def _pluralize_key(key: str):
+    return key + ("" if key.endswith("s") else "s")
+
+
+def _blank_node_ids(values: list):
+    return sorted(list(set(list(map(lambda v: v.get("@id"), values)))))
+
+
+def _blank_node_sub_values(blank_nodes: list, key: str):
+    values = flatten(map(lambda v: v.get(key, []), blank_nodes))
+    return {_pluralize_key(key): _blank_node_ids(values)} if values else {}
+
+
+def _blank_node_data(blank_nodes: list):
+    value = get_node_value(
+        {
+            "term": blank_nodes[0].get("term"),
+            "value": list(map(get_node_value, blank_nodes)),
+        }
+    )
+    sub_values = ["inputs", "operation", "transformation"]
+    has_cycle_value = any(
+        [
+            all([get_node_value(v) is not None, not v.get("transformation")])
+            for v in blank_nodes
+        ]
+    )
+    return {"value": value, "hasCycleValue": has_cycle_value} | reduce(
+        lambda p, c: p | _blank_node_sub_values(blank_nodes, c), sub_values, {}
+    )
+
+
 def get_blank_nodes_calculation_status(
     node: dict, list_key: str, termType: TermTermType
 ):
@@ -214,20 +246,6 @@ def get_blank_nodes_calculation_status(
     blank_nodes_by_term = group_by_keys(blank_nodes, ["term"])
     blank_nodes_grouping_keys = _BLANK_NODE_GROUPING_KEYS.get(termType) or []
 
-    def blank_node_data(blank_nodes: list):
-        value = get_node_value(
-            {
-                "term": blank_nodes[0].get("term"),
-                "value": list(map(get_node_value, blank_nodes)),
-            }
-        )
-        inputs = flatten(map(lambda v: v.get("inputs", []), blank_nodes))
-        return {"value": value} | (
-            {"inputs": sorted(list(map(lambda v: v.get("@id"), inputs)))}
-            if inputs
-            else {}
-        )
-
     def map_blank_node(term_id: str):
         values = blank_nodes_by_term.get(term_id, [])
         grouped_blank_nodes = (
@@ -239,9 +257,9 @@ def get_blank_nodes_calculation_status(
             {}
             if not values
             else (
-                {k: blank_node_data(v) for k, v in grouped_blank_nodes.items()}
+                {k: _blank_node_data(v) for k, v in grouped_blank_nodes.items()}
                 if grouped_blank_nodes
-                else blank_node_data([values[0]])
+                else _blank_node_data([values[0]])
             )
         )
 

{hestia_earth_utils-0.16.14 → hestia_earth_utils-0.16.16}/hestia_earth/utils/cycle.py

@@ -4,10 +4,16 @@ from .tools import flatten
 from .blank_node import get_blank_nodes_calculation_status
 
 
-def _extend_missing_inputs(value: dict, input_ids: set):
-    included_inputs = set(flatten([v.get("inputs", []) for v in value.values()]))
-    missing_inputs = input_ids - included_inputs
-    return {"missingInputs": sorted(list(missing_inputs))} if missing_inputs else {}
+def _extend_missing_values(
+    value: dict, all_values: set, key: str, must_have_key: bool = False
+):
+    included_values = set(flatten([v.get(key, []) for v in value.values()]))
+    missing_values = all_values - included_values
+    return (
+        {"missing" + key.capitalize(): sorted(list(missing_values))}
+        if all([missing_values, not must_have_key or included_values])
+        else {}
+    )
 
 
 def get_cycle_emissions_calculation_status(cycle: dict):
@@ -30,7 +36,20 @@ def get_cycle_emissions_calculation_status(cycle: dict):
         cycle, "emissions", TermTermType.EMISSION
     )
     input_ids = set([v.get("term", {}).get("@id") for v in cycle.get("inputs", [])])
+    transformation_ids = set(
+        [v.get("term", {}).get("@id") for v in cycle.get("transformations", [])]
+    )
     return {
-        k: v | (_extend_missing_inputs(v, input_ids) if "InputsProduction" in k else {})
+        k: v
+        | (
+            _extend_missing_values(v, input_ids, "inputs")
+            if "InputsProduction" in k
+            else {}
+        )
+        | (
+            _extend_missing_values(
+                v, transformation_ids, "transformations", must_have_key=True
+            )
+        )
         for k, v in status.items()
     }

{hestia_earth_utils-0.16.14 → hestia_earth_utils-0.16.16}/hestia_earth/utils/pivot/_shared.py

@@ -6,7 +6,6 @@ from flatten_json import flatten as flatten_json
 
 from ..tools import list_sum
 
-
 EXCLUDE_FIELDS = ["@type", "type", "@context"]
 EXCLUDE_PRIVATE_FIELDS = [
     "added",

hestia_earth_utils-0.16.16/hestia_earth/utils/version.py

@@ -0,0 +1 @@
+VERSION = "0.16.16"

{hestia_earth_utils-0.16.14 → hestia_earth_utils-0.16.16}/hestia_earth_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.14
+Version: 0.16.16
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team
@@ -13,7 +13,7 @@ Requires-Dist: hestia-earth-schema>=35.0.1
 Requires-Dist: requests>=2.24.0
 Requires-Dist: urllib3~=1.26.0
 Requires-Dist: python-dateutil>=2.8.1
-Requires-Dist: pandas>=2
+Requires-Dist: pandas<3,>=2
 Requires-Dist: flatten_json
 Dynamic: author
 Dynamic: author-email

{hestia_earth_utils-0.16.14 → hestia_earth_utils-0.16.16}/hestia_earth_utils.egg-info/requires.txt

@@ -2,5 +2,5 @@ hestia-earth-schema>=35.0.1
 requests>=2.24.0
 urllib3~=1.26.0
 python-dateutil>=2.8.1
-pandas>=2
+pandas<3,>=2
 flatten_json

{hestia_earth_utils-0.16.14 → hestia_earth_utils-0.16.16}/tests/test_api.py

@@ -17,7 +17,6 @@ from hestia_earth.utils.api import (
     find_term_ids_by_names,
 )
 
-
 fake_related_response = {"results": [[{"@id": "related_id"}]]}
 fake_download_response = {"@id": "id", "@type": "type"}
 

{hestia_earth_utils-0.16.14 → hestia_earth_utils-0.16.16}/tests/test_blank_node.py

@@ -1,14 +1,14 @@
 import os
+import json
+from pytest import mark
+from unittest.mock import patch
+from hestia_earth.schema import TermTermType
 
-# import json
-# from pytest import mark
-# from hestia_earth.schema import TermTermType
-
-from tests.utils import fixtures_path
+from tests.utils import EMISSION_IDS, fixtures_path
 from hestia_earth.utils.blank_node import (
     get_node_value,
     ArrayTreatment,
-    # get_blank_nodes_calculation_status,
+    get_blank_nodes_calculation_status,
 )
 
 fixtures_folder = os.path.join(fixtures_path, "blank_node")
@@ -57,18 +57,33 @@ def test_get_node_value():
     assert get_node_value(blank_node, "value", array_treatment=ArrayTreatment.SUM) == 30
 
 
-# @mark.parametrize(
-#     'folder,list_key,termType',
-#     [
-#         ('cycle', 'emissions', TermTermType.EMISSION),
-#     ]
-# )
-# def test_get_blank_nodes_calculation_status(folder: str, list_key: str, termType: TermTermType):
-#     with open(f"{calculation_status_folder}/{folder}/node.jsonld", encoding='utf-8') as f:
-#         node = json.load(f)
-
-#     with open(f"{calculation_status_folder}/{folder}/{list_key}-{termType.value}.json", encoding='utf-8') as f:
-#         expected = json.load(f)
-
-#     result = get_blank_nodes_calculation_status(node, list_key=list_key, termType=termType)
-#     assert result == expected, folder
+@mark.parametrize(
+    "folder,list_key,termType",
+    [
+        ("cycle", "emissions", TermTermType.EMISSION),
+        ("cycle-with-transformations", "emissions", TermTermType.EMISSION),
+    ],
+)
+@patch(
+    "hestia_earth.utils.blank_node.cycle_emissions_in_system_boundary",
+    return_value=EMISSION_IDS,
+)
+def test_get_blank_nodes_calculation_status(
+    mock_, folder: str, list_key: str, termType: TermTermType
+):
+    with open(
+        f"{calculation_status_folder}/{folder}/node.jsonld", encoding="utf-8"
+    ) as f:
+        node = json.load(f)
+
+    with open(
+        f"{calculation_status_folder}/{folder}/{list_key}-{termType.value}.json",
+        encoding="utf-8",
+    ) as f:
+        expected = json.load(f)
+
+    result = get_blank_nodes_calculation_status(
+        node, list_key=list_key, termType=termType
+    )
+    print(json.dumps(result, indent=2))
+    assert result == expected, folder

hestia_earth_utils-0.16.16/tests/test_calculation_status.py

@@ -0,0 +1,30 @@
+import os
+import json
+
+from tests.utils import fixtures_path
+from hestia_earth.utils.calculation_status import _emissions_with_status
+
+fixtures_folder = os.path.join(fixtures_path, "calculation_status")
+
+
+def test_emissions_with_status():
+    with open(os.path.join(fixtures_folder, "nodes.json")) as f:
+        nodes = json.load(f)
+
+    result = _emissions_with_status(nodes[0])
+    assert result == {
+        "emissions-total": 202,
+        "emissions-complete": 55,
+        "emissions-incomplete": 1,
+        "emissions-missing": 146,
+        "emissions": result["emissions"],  # ignore
+    }
+
+    result = _emissions_with_status(nodes[1])
+    assert result == {
+        "emissions-total": 202,
+        "emissions-complete": 0,
+        "emissions-incomplete": 13,
+        "emissions-missing": 189,
+        "emissions": result["emissions"],  # ignore
+    }

hestia_earth_utils-0.16.16/tests/test_cycle.py

@@ -0,0 +1,38 @@
+import os
+import json
+import pytest
+from unittest.mock import patch
+
+from tests.utils import EMISSION_IDS, fixtures_path
+from hestia_earth.utils.cycle import get_cycle_emissions_calculation_status
+
+fixtures_folder = os.path.join(fixtures_path, "blank_node", "calculation_status")
+_folders = [
+    d
+    for d in os.listdir(fixtures_folder)
+    if os.path.isdir(os.path.join(fixtures_folder, d))
+]
+
+
+@pytest.mark.parametrize("folder", _folders)
+@patch(
+    "hestia_earth.utils.blank_node.cycle_emissions_in_system_boundary",
+    return_value=EMISSION_IDS,
+)
+def test_get_cycle_emissions_calculation_status(mock_, folder: str):
+    with open(
+        os.path.join(fixtures_folder, folder, "node.jsonld"), encoding="utf-8"
+    ) as f:
+        cycle = json.load(f)
+
+    with open(
+        os.path.join(
+            fixtures_folder, folder, "emissions-emission-with-missing-inputs.json"
+        ),
+        encoding="utf-8",
+    ) as f:
+        expected = json.load(f)
+
+    result = get_cycle_emissions_calculation_status(cycle)
+    print(json.dumps(result, indent=2))
+    assert result == expected

hestia_earth_utils-0.16.14/hestia_earth/utils/version.py

@@ -1 +0,0 @@
-VERSION = "0.16.14"

hestia_earth_utils-0.16.14/tests/test_calculation_status.py

@@ -1,40 +0,0 @@
-# import os
-# import json
-
-# from tests.utils import fixtures_path
-# from hestia_earth.utils.calculation_status import _emissions_with_status, get_nodes_calculations_status_dataframe
-
-# fixtures_folder = os.path.join(fixtures_path, 'calculation_status')
-
-
-# def test_emissions_with_status():
-#     with open(os.path.join(fixtures_folder, 'nodes.json')) as f:
-#         nodes = json.load(f)
-
-#     result = _emissions_with_status(nodes[0])
-#     assert result == {
-#         'emissions-total': 193,
-#         'emissions-complete': 56,
-#         'emissions-incomplete': 1,
-#         'emissions-missing': 136,
-#         'emissions': result['emissions']  # ignore
-#     }
-
-#     result = _emissions_with_status(nodes[1])
-#     assert result == {
-#         'emissions-total': 193,
-#         'emissions-complete': 0,
-#         'emissions-incomplete': 13,
-#         'emissions-missing': 180,
-#         'emissions': result['emissions']  # ignore
-#     }
-
-
-# def test_get_nodes_calculations_status_dataframe():
-#     with open(os.path.join(fixtures_folder, 'nodes.json')) as f:
-#         nodes = json.load(f)
-
-#     expected = open(os.path.join(fixtures_folder, 'result.csv'), 'r').read()
-
-#     df = get_nodes_calculations_status_dataframe(nodes, file_format='csv')
-#     assert df.to_csv(None, index=None) == expected

hestia_earth_utils-0.16.14/tests/test_cycle.py

@@ -1,18 +0,0 @@
-# import os
-# import json
-
-# from tests.utils import fixtures_path
-# from hestia_earth.utils.cycle import get_cycle_emissions_calculation_status
-
-
-# def test_get_cycle_emissions_calculation_status():
-#     folder = os.path.join(fixtures_path, 'blank_node', 'calculation_status', 'cycle')
-
-#     with open(f"{folder}/node.jsonld", encoding='utf-8') as f:
-#         cycle = json.load(f)
-
-#     with open(f"{folder}/emissions-emission-with-missing-inputs.json", encoding='utf-8') as f:
-#         expected = json.load(f)
-
-#     result = get_cycle_emissions_calculation_status(cycle)
-#     assert result == expected