hestia-earth-utils 0.16.13__tar.gz → 0.16.14__tar.gz

{hestia_earth_utils-0.16.13/hestia_earth_utils.egg-info → hestia_earth_utils-0.16.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.13
+Version: 0.16.14
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

hestia_earth_utils-0.16.14/hestia_earth/utils/pivot/_shared.py

@@ -0,0 +1,110 @@
+import json
+import numpy as np
+import pandas as pd
+from hestia_earth.schema import SCHEMA_TYPES, NODE_TYPES, EmissionMethodTier
+from flatten_json import flatten as flatten_json
+
+from ..tools import list_sum
+
+
+EXCLUDE_FIELDS = ["@type", "type", "@context"]
+EXCLUDE_PRIVATE_FIELDS = [
+    "added",
+    "addedVersion",
+    "updated",
+    "updatedVersion",
+    "aggregatedVersion",
+    "_cache",
+]
+
+
+# assuming column labels always camelCase
+def _get_node_type_label(node_type):
+    return node_type[0].lower() + node_type[1:]
+
+
+def _get_node_type_from_label(node_type):
+    return node_type[0].upper() + node_type[1:]
+
+
+def _is_blank_node(data: dict):
+    node_type = data.get("@type") or data.get("type")
+    return node_type in SCHEMA_TYPES and node_type not in NODE_TYPES
+
+
+def _with_csv_formatting(dct):
+    """
+    Use as object_hook when parsing a JSON node: json.loads(node, object_hook=_with_csv_formatting).
+    Ensures parsed JSON has field values formatted according to hestia csv conventions.
+    """
+    if "boundary" in dct:
+        dct["boundary"] = json.dumps(dct["boundary"])
+    for key, value in dct.items():
+        if _is_scalar_list(value):
+            dct[key] = ";".join([str(el) for el in value])
+    return dct
+
+
+def _is_scalar_list(value):
+    if not isinstance(value, list):
+        return False
+    all_scalar = True
+    for element in value:
+        if not np.isscalar(element):
+            all_scalar = False
+            break
+    return all_scalar
+
+
+def _filter_not_relevant(blank_node: dict):
+    return blank_node.get("methodTier") != EmissionMethodTier.NOT_RELEVANT.value
+
+
+def _filter_emissions_not_relevant(node: dict):
+    """
+    Ignore all emissions where `methodTier=not relevant` to save space.
+    """
+    return node | (
+        {
+            key: list(filter(_filter_not_relevant, node[key]))
+            for key in ["emissions", "emissionsResourceUse"]
+            if key in node
+        }
+    )
+
+
+def _filter_zero_value(blank_node: dict):
+    value = blank_node.get("value")
+    value = (
+        list_sum(blank_node.get("value"), default=-1)
+        if isinstance(value, list)
+        else value
+    )
+    return value != 0
+
+
+def _filter_zero_values(node: dict):
+    """
+    Ignore all blank nodes where `value=0` to save space.
+    """
+    return node | (
+        {
+            key: list(filter(_filter_zero_value, value))
+            for key, value in node.items()
+            if isinstance(value, list)
+            and isinstance(value[0], dict)
+            and _is_blank_node(value[0])
+        }
+    )
+
+
+def nodes_to_df(nodes: list[dict]):
+    nodes_flattened = [
+        flatten_json(
+            dict([(_get_node_type_label(node.get("@type", node.get("type"))), node)]),
+            ".",
+        )
+        for node in nodes
+    ]
+
+    return pd.json_normalize(nodes_flattened)

{hestia_earth_utils-0.16.13 → hestia_earth_utils-0.16.14}/hestia_earth/utils/pivot/pivot_csv.py

@@ -5,7 +5,6 @@ import numpy as np
 import pandas as pd
 from hestia_earth.schema import UNIQUENESS_FIELDS, Term, NODE_TYPES
 from hestia_earth.schema.utils.sort import get_sort_key, SORT_CONFIG
-from flatten_json import flatten as flatten_json
 
 # __package__ = "hestia_earth.utils" # required to run interactively in vscode
 from ..api import find_term_ids_by_names
@@ -14,6 +13,9 @@ from ._shared import (
     EXCLUDE_PRIVATE_FIELDS,
     _with_csv_formatting,
     _filter_emissions_not_relevant,
+    _get_node_type_label,
+    _get_node_type_from_label,
+    nodes_to_df,
 )
 
 
@@ -36,15 +38,6 @@ def _get_blank_node_uniqueness_fields():
 BLANK_NODE_UNIQUENESS_FIELDS = _get_blank_node_uniqueness_fields()
 
 
-# assuming column labels always camelCase
-def _get_node_type_label(node_type):
-    return node_type[0].lower() + node_type[1:]
-
-
-def _get_node_type_from_label(node_type):
-    return node_type[0].upper() + node_type[1:]
-
-
 def _get_names(df):
     names = []
     for node_type, array_fields in BLANK_NODE_UNIQUENESS_FIELDS.items():
@@ -283,18 +276,6 @@ def _format_and_pivot(df_in):
     return df_out
 
 
-def nodes_to_df(nodes: list[dict]):
-    nodes_flattened = [
-        flatten_json(
-            dict([(_get_node_type_label(node.get("@type", node.get("type"))), node)]),
-            ".",
-        )
-        for node in nodes
-    ]
-
-    return pd.json_normalize(nodes_flattened)
-
-
 def pivot_nodes(nodes: list[dict]):
     """
     Pivot array of nodes in dict format (e.g under the 'nodes' key of a .hestia file)

{hestia_earth_utils-0.16.13 → hestia_earth_utils-0.16.14}/hestia_earth/utils/pivot/pivot_json.py

@@ -11,6 +11,7 @@ from ._shared import (
     EXCLUDE_PRIVATE_FIELDS,
     _with_csv_formatting,
     _filter_emissions_not_relevant,
+    _filter_zero_values,
 )
 
 pivot_exclude_fields = Term().fields
@@ -22,6 +23,8 @@ term_exclude_fields = Term().fields
 del term_exclude_fields["name"]
 term_exclude_fields.update({k: "" for k in EXCLUDE_PRIVATE_FIELDS})
 
+include_all_unique_keys = ["emissions", "emissionsResourceUse"]
+
 # Treat properties uniqueness fields as special case for now
 PROPERTIES_VIRTUAL_UNIQUENESS_FIELD = "propertyValues"
 ADAPTED_UNIQUENESS_FIELDS = deepcopy(UNIQUENESS_FIELDS)
@@ -90,6 +93,7 @@ def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa:
 
     # print('\n', level, 'fields_to_pivot', fields_to_pivot)
     for field, uniqueness_fields in fields_to_pivot:
+        include_all_unique_fields = field in include_all_unique_keys
         # print('\nbefore processing node field', level, field, node[field], '\n')
         # Compress lists of 'Node' nodes to dict with single @id key.
         # The compressed field matches uniqueness fields like cycle.emissions.inputs.@id.
@@ -155,14 +159,13 @@ def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa:
             del term_data["combined_fields"][id_key]
             # print('combined_fields', field, term_id, term_data['combined_fields'], '\n')
             fields_to_include = {
-                field: any(
-                    by_idx[idx].get(field) != by_idx[indexes[0]].get(field)
-                    for idx in indexes
+                k: include_all_unique_fields
+                or any(
+                    by_idx[idx].get(k) != by_idx[indexes[0]].get(k) for idx in indexes
                 )
-                for field in term_data["combined_fields"].keys()
-                if field in uniqueness_fields
-                or field != "value"
-                and field.split(".")[-1] not in pivot_exclude_fields
+                for k in term_data["combined_fields"].keys()
+                if k in uniqueness_fields
+                or (k != "value" and k.split(".")[-1] not in pivot_exclude_fields)
             }
             # print('fields_to_include', level, field, term_id, fields_to_include, '\n')
             for idx in indexes:
@@ -179,19 +182,23 @@ def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa:
                     )
                 ]
                 # print('distingishing_field_fields', level, field, term_id, distingishing_field_fields, '\n')
-                unanimous_fields = {
-                    field: term_data["combined_fields"][field]
-                    for field, not_unanimous in fields_to_include.items()
-                    if field not in distingishing_field_fields
-                    and not not_unanimous
-                    and field is not PROPERTIES_VIRTUAL_UNIQUENESS_FIELD
-                }
                 # print('unanimous_fields', level, field, term_id, unanimous_fields, '\n')
+                unanimous_fields = (
+                    {}
+                    if include_all_unique_fields
+                    else {
+                        field: term_data["combined_fields"][field]
+                        for field, not_unanimous in fields_to_include.items()
+                        if field not in distingishing_field_fields
+                        and not not_unanimous
+                        and field is not PROPERTIES_VIRTUAL_UNIQUENESS_FIELD
+                    }
+                )
                 differentiated_fields = {
                     field: term[field]
                     for field, not_unanimous in fields_to_include.items()
                     if field not in distingishing_field_fields
-                    and not_unanimous
+                    and (include_all_unique_fields or not_unanimous)
                     and field in term
                 }
                 # print('differentiated_fields', level, field, term_id, differentiated_fields, '\n')
@@ -223,19 +230,35 @@ def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa:
     return pivoted_node
 
 
-def pivot_node(node: dict):
+def pivot_node(
+    node: dict,
+    include_emissions_not_relevant: bool = False,
+    include_zero_values: bool = True,
+):
     """
     Pivot single node in dict format parsed with object_hook=_with_csv_formatting
     """
-    return _do_pivot(_filter_emissions_not_relevant(node))
+    node = (
+        node if include_emissions_not_relevant else _filter_emissions_not_relevant(node)
+    )
+    node = node if include_zero_values else _filter_zero_values(node)
+    return _do_pivot(node)
 
 
-def pivot_json_node(json_node: str):
+def pivot_json_node(
+    json_node: str,
+    include_emissions_not_relevant: bool = False,
+    include_zero_values: bool = True,
+):
     """
     Pivot single schema-compliant unparsed json string node
     """
     node = json.loads(json_node, object_hook=_with_csv_formatting)
-    return pivot_node(node)
+    return pivot_node(
+        node,
+        include_emissions_not_relevant=include_emissions_not_relevant,
+        include_zero_values=include_zero_values,
+    )
 
 
 def pivot_hestia_file(hestia_file: str):
@@ -246,8 +269,19 @@ def pivot_hestia_file(hestia_file: str):
     return pivot_nodes(parsed.get("nodes", []))
 
 
-def pivot_nodes(nodes: list[dict]):
+def pivot_nodes(
+    nodes: list[dict],
+    include_emissions_not_relevant: bool = False,
+    include_zero_values: bool = True,
+):
     """
     Pivot multiple nodes in dict format parsed with object_hook=_with_csv_formatting
     """
-    return [pivot_node(node) for node in nodes]
+    return [
+        pivot_node(
+            node,
+            include_emissions_not_relevant=include_emissions_not_relevant,
+            include_zero_values=include_zero_values,
+        )
+        for node in nodes
+    ]

hestia_earth_utils-0.16.14/hestia_earth/utils/version.py

@@ -0,0 +1 @@
+VERSION = "0.16.14"

{hestia_earth_utils-0.16.13 → hestia_earth_utils-0.16.14/hestia_earth_utils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.13
+Version: 0.16.14
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

hestia_earth_utils-0.16.13/hestia_earth/utils/pivot/_shared.py

@@ -1,55 +0,0 @@
-import json
-import numpy as np
-from hestia_earth.schema import EmissionMethodTier
-
-
-EXCLUDE_FIELDS = ["@type", "type", "@context"]
-EXCLUDE_PRIVATE_FIELDS = [
-    "added",
-    "addedVersion",
-    "updated",
-    "updatedVersion",
-    "aggregatedVersion",
-    "_cache",
-]
-
-
-def _with_csv_formatting(dct):
-    """
-    Use as object_hook when parsing a JSON node: json.loads(node, object_hook=_with_csv_formatting).
-    Ensures parsed JSON has field values formatted according to hestia csv conventions.
-    """
-    if "boundary" in dct:
-        dct["boundary"] = json.dumps(dct["boundary"])
-    for key, value in dct.items():
-        if _is_scalar_list(value):
-            dct[key] = ";".join([str(el) for el in value])
-    return dct
-
-
-def _is_scalar_list(value):
-    if not isinstance(value, list):
-        return False
-    all_scalar = True
-    for element in value:
-        if not np.isscalar(element):
-            all_scalar = False
-            break
-    return all_scalar
-
-
-def _filter_not_relevant(blank_node: dict):
-    return blank_node.get("methodTier") != EmissionMethodTier.NOT_RELEVANT.value
-
-
-def _filter_emissions_not_relevant(node: dict):
-    """
-    Ignore all emissions where `methodTier=not relevant` so save space.
-    """
-    return node | (
-        {
-            key: list(filter(_filter_not_relevant, node[key]))
-            for key in ["emissions", "emissionsResourceUse"]
-            if key in node
-        }
-    )

hestia_earth_utils-0.16.13/hestia_earth/utils/version.py

@@ -1 +0,0 @@
-VERSION = "0.16.13"