hestia-earth-utils 0.16.12__py3-none-any.whl → 0.16.14__py3-none-any.whl

hestia_earth/utils/date.py

@@ -1,6 +1,15 @@
+from calendar import monthrange
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
+from enum import Enum
+from math import floor
+from typing import Any, Callable, Literal, Optional, Union
+
 from dateutil.parser import parse
 import re
 
+from .tools import is_list_like, safe_parse_date
+
 SECOND = 1
 MINUTE = 60 * SECOND
 HOUR = 60 * MINUTE
@@ -12,6 +21,8 @@ def diff_in_days(from_date: str, to_date: str) -> float:
     """
     Return the difference in days between two dates.
 
+    Deprecated, use `diff_in` function with `unit = TimeUnit.DAY` instead.
+
     Parameters
     ----------
     from_date : str
@@ -32,6 +43,8 @@ def diff_in_years(from_date: str, to_date: str) -> float:
     """
     Return the difference in years between two dates.
 
+    Deprecated, use `diff_in` function with `unit = TimeUnit.YEAR` instead.
+
     Parameters
     ----------
     from_date : str
@@ -51,6 +64,8 @@ def is_in_days(date: str) -> bool:
     """
     Check if the date as a string contains year, month and day.
 
+    Deprecated, use `validate_datestr_format` with `valid_format = DatestrFormat.YEAR_MONTH_DAY` instead.
+
     Parameters
     ----------
     date : str
@@ -71,6 +86,8 @@ def is_in_months(date: str) -> bool:
     """
     Check if the date as a string contains year, month but no day.
 
+    Deprecated, use `validate_datestr_format` with `valid_format = DatestrFormat.YEAR_MONTH` instead.
+
     Parameters
     ----------
     date : str
@@ -84,3 +101,425 @@ def is_in_months(date: str) -> bool:
     return (
         date is not None and re.compile(r"^[\d]{4}\-[\d]{2}$").match(date) is not None
     )
+
+
+OLDEST_DATE = "1800"
+
+
+class TimeUnit(Enum):
+    YEAR = "year"
+    MONTH = "month"
+    DAY = "day"
+    HOUR = "hour"
+    MINUTE = "minute"
+    SECOND = "second"
+
+
+class DatestrFormat(Enum):
+    """
+    Enum representing ISO date formats permitted by HESTIA.
+
+    See: https://en.wikipedia.org/wiki/ISO_8601
+    """
+
+    YEAR = r"%Y"
+    YEAR_MONTH = r"%Y-%m"
+    YEAR_MONTH_DAY = r"%Y-%m-%d"
+    YEAR_MONTH_DAY_HOUR_MINUTE_SECOND = r"%Y-%m-%dT%H:%M:%S"
+    MONTH = r"--%m"
+    MONTH_DAY = r"--%m-%d"
+
+
+DatestrGapfillMode = Literal["start", "middle", "end"]
+
+
+DATESTR_FORMAT_TO_EXPECTED_LENGTH = {
+    DatestrFormat.YEAR: len("2001"),
+    DatestrFormat.YEAR_MONTH: len("2001-01"),
+    DatestrFormat.YEAR_MONTH_DAY: len("2001-01-01"),
+    DatestrFormat.YEAR_MONTH_DAY_HOUR_MINUTE_SECOND: len("2001-01-01T00:00:00"),
+    DatestrFormat.MONTH: len("--01"),
+    DatestrFormat.MONTH_DAY: len("--01-01"),
+}
+
+
+TIME_UNIT_TO_DATESTR_FORMAT = {
+    TimeUnit.YEAR: DatestrFormat.YEAR,
+    TimeUnit.MONTH: DatestrFormat.YEAR_MONTH,
+    TimeUnit.DAY: DatestrFormat.YEAR_MONTH_DAY,
+    TimeUnit.HOUR: DatestrFormat.YEAR_MONTH_DAY_HOUR_MINUTE_SECOND,
+    TimeUnit.MINUTE: DatestrFormat.YEAR_MONTH_DAY_HOUR_MINUTE_SECOND,
+    TimeUnit.SECOND: DatestrFormat.YEAR_MONTH_DAY_HOUR_MINUTE_SECOND,
+}
+"""
+Minimum Datestr format required to express DatetimeUnit.
+"""
+
+
+_SECONDS_IN_MINUTE = 60  # 60 seconds in a minute
+_MINUTES_IN_HOUR = 60  # 60 minutes in an hour
+_HOURS_IN_DAY = 24  # 24 hours in a day
+_MONTHS_IN_YEAR = 12  # 12 months in a year
+
+_DAYS_IN_YEAR = YEAR  # average days in a year (365.2425)
+_DAYS_IN_MONTH = (
+    _DAYS_IN_YEAR / _MONTHS_IN_YEAR
+)  # average days in a month (365.2425/12)
+
+
+DATETIME_UNIT_CONVERSIONS: dict[str, dict[str, float]] = {
+    TimeUnit.YEAR.value: {
+        TimeUnit.MONTH.value: _MONTHS_IN_YEAR,
+        TimeUnit.DAY.value: _DAYS_IN_YEAR,
+        TimeUnit.HOUR.value: _DAYS_IN_YEAR * _HOURS_IN_DAY,
+        TimeUnit.MINUTE.value: _DAYS_IN_YEAR * _HOURS_IN_DAY * _MINUTES_IN_HOUR,
+        TimeUnit.SECOND.value: _DAYS_IN_YEAR
+        * _HOURS_IN_DAY
+        * _MINUTES_IN_HOUR
+        * _SECONDS_IN_MINUTE,
+    },
+    TimeUnit.MONTH.value: {
+        TimeUnit.YEAR.value: 1 / _MONTHS_IN_YEAR,
+        TimeUnit.DAY.value: _DAYS_IN_MONTH,
+        TimeUnit.HOUR.value: _DAYS_IN_MONTH * _HOURS_IN_DAY,
+        TimeUnit.MINUTE.value: _DAYS_IN_MONTH * _HOURS_IN_DAY * _MINUTES_IN_HOUR,
+        TimeUnit.SECOND.value: _DAYS_IN_MONTH
+        * _HOURS_IN_DAY
+        * _MINUTES_IN_HOUR
+        * _SECONDS_IN_MINUTE,
+    },
+    TimeUnit.DAY.value: {
+        TimeUnit.YEAR.value: 1 / _DAYS_IN_YEAR,
+        TimeUnit.MONTH.value: 1 / _DAYS_IN_MONTH,
+        TimeUnit.HOUR.value: _HOURS_IN_DAY,
+        TimeUnit.MINUTE.value: _HOURS_IN_DAY * _MINUTES_IN_HOUR,
+        TimeUnit.SECOND.value: _HOURS_IN_DAY * _MINUTES_IN_HOUR * _SECONDS_IN_MINUTE,
+    },
+    TimeUnit.HOUR.value: {
+        TimeUnit.YEAR.value: 1 / (_HOURS_IN_DAY * _DAYS_IN_YEAR),
+        TimeUnit.MONTH.value: 1 / (_HOURS_IN_DAY * _DAYS_IN_MONTH),
+        TimeUnit.DAY.value: 1 / (_HOURS_IN_DAY),
+        TimeUnit.MINUTE.value: _MINUTES_IN_HOUR,
+        TimeUnit.SECOND.value: _MINUTES_IN_HOUR * _SECONDS_IN_MINUTE,
+    },
+    TimeUnit.MINUTE.value: {
+        TimeUnit.YEAR.value: 1 / (_MINUTES_IN_HOUR * _HOURS_IN_DAY * _DAYS_IN_YEAR),
+        TimeUnit.MONTH.value: 1 / (_MINUTES_IN_HOUR * _HOURS_IN_DAY * _DAYS_IN_MONTH),
+        TimeUnit.DAY.value: 1 / (_MINUTES_IN_HOUR * _HOURS_IN_DAY),
+        TimeUnit.HOUR.value: 1 / _MINUTES_IN_HOUR,
+        TimeUnit.SECOND.value: _SECONDS_IN_MINUTE,
+    },
+    TimeUnit.SECOND.value: {
+        TimeUnit.YEAR.value: 1
+        / (_SECONDS_IN_MINUTE * _MINUTES_IN_HOUR * _HOURS_IN_DAY * _DAYS_IN_YEAR),
+        TimeUnit.MONTH.value: 1
+        / (_SECONDS_IN_MINUTE * _MINUTES_IN_HOUR * _HOURS_IN_DAY * _DAYS_IN_MONTH),
+        TimeUnit.DAY.value: 1 / (_SECONDS_IN_MINUTE * _MINUTES_IN_HOUR * _HOURS_IN_DAY),
+        TimeUnit.HOUR.value: 1 / (_SECONDS_IN_MINUTE * _MINUTES_IN_HOUR),
+        TimeUnit.MINUTE.value: 1 / _SECONDS_IN_MINUTE,
+    },
+}
+"""
+A dict of TimeUnit conversion factors with format:
+```
+{
+    source (str): {
+        dest (str): conversion_factor (float)
+    }
+}
+```
+"""
+
+
+def _get_time_unit_conversion(
+    src_unit: TimeUnit, dest_unit: TimeUnit, default_value: float = 1
+):
+    src_key = src_unit if isinstance(src_unit, str) else src_unit.value
+    dest_key = dest_unit if isinstance(dest_unit, str) else dest_unit.value
+    return DATETIME_UNIT_CONVERSIONS.get(src_key, {}).get(dest_key, default_value)
+
+
+def convert_duration(
+    duration: float,
+    src_unit: TimeUnit,
+    dest_unit: TimeUnit,
+    default_conversion_factor: float = 1,
+):
+    conversion_factor = _get_time_unit_conversion(
+        src_unit, dest_unit, default_conversion_factor
+    )
+    return duration * conversion_factor
+
+
+def _check_datestr_format(datestr: str, format: DatestrFormat) -> bool:
+    """
+    Use `datetime.strptime` to determine if a datestr is in a particular ISO format.
+    """
+    try:
+        expected_length = DATESTR_FORMAT_TO_EXPECTED_LENGTH.get(format, 0)
+        format_str = format.value
+        return len(datestr) == expected_length and bool(
+            datetime.strptime(datestr, format_str)
+        )
+    except ValueError:
+        return False
+
+
+def _get_datestr_format(
+    datestr: str, default: Optional[Any] = None
+) -> Union[DatestrFormat, Any, None]:
+    """
+    Check a datestr against each ISO format permitted by the HESTIA schema and
+    return the matching format.
+    """
+    return next(
+        (
+            date_format
+            for date_format in DatestrFormat
+            if _check_datestr_format(str(datestr), date_format)
+        ),
+        default,
+    )
+
+
+def validate_datestr_format(
+    datestr: str,
+    valid_format: Union[DatestrFormat, list[DatestrFormat]] = [
+        DatestrFormat.YEAR,
+        DatestrFormat.YEAR_MONTH,
+        DatestrFormat.YEAR_MONTH_DAY,
+    ],
+):
+    valid_formats = valid_format if is_list_like(valid_format) else [valid_format]
+    format_ = _get_datestr_format(datestr)
+    return format_ in valid_formats
+
+
+def _gapfill_datestr_start(datestr: str, *_) -> str:
+    """
+    Gapfill an incomplete datestr with the earliest possible date and time.
+
+    Datestr will snap to the start of the year/month/day as appropriate.
+    """
+    return datestr + "YYYY-01-01T00:00:00"[len(datestr) :]
+
+
+def _days_in_month(datestr: str) -> int:
+    """
+    Get the number of days in the datestr's month. If datestr invalid, return minimum value of 28.
+    """
+    datetime = safe_parse_date(datestr)
+    return monthrange(datetime.year, datetime.month)[1] if datetime else 28
+
+
+def _gapfill_datestr_end(datestr: str, format: DatestrFormat) -> str:
+    """
+    Gapfill an incomplete datestr with the latest possible date and time.
+
+    Datestr will snap to the end of the year/month/day as appropriate.
+    """
+    days = _days_in_month(datestr) if format == DatestrFormat.YEAR_MONTH else 31
+    completion_str = f"YYYY-12-{days}T23:59:59"
+    return datestr + completion_str[len(datestr) :]
+
+
+def _gapfill_datestr_middle(datestr: str, format: DatestrFormat) -> str:
+    """
+    Gap-fill an incomplete datestr with the middle value, halfway between the latest and earliest values.
+    """
+    start_date_obj = datetime.strptime(
+        _gapfill_datestr_start(datestr),
+        DatestrFormat.YEAR_MONTH_DAY_HOUR_MINUTE_SECOND.value,
+    )
+    end_date_obj = datetime.strptime(
+        _gapfill_datestr_end(datestr, format=format),
+        DatestrFormat.YEAR_MONTH_DAY_HOUR_MINUTE_SECOND.value,
+    )
+    middle_date = start_date_obj + (end_date_obj - start_date_obj) / 2
+    return datetime.strftime(
+        middle_date, DatestrFormat.YEAR_MONTH_DAY_HOUR_MINUTE_SECOND.value
+    )
+
+
+DATESTR_GAPFILL_MODE_TO_GAPFILL_FUNCTION: dict[DatestrGapfillMode, Callable] = {
+    "start": _gapfill_datestr_start,
+    "middle": _gapfill_datestr_middle,
+    "end": _gapfill_datestr_end,
+}
+
+_VALID_GAPFILL_DATE_FORMATS = {
+    DatestrFormat.YEAR,
+    DatestrFormat.YEAR_MONTH,
+    DatestrFormat.YEAR_MONTH_DAY,
+}
+
+
+def gapfill_datestr(datestr: str, mode: DatestrGapfillMode = "start") -> str:
+    """
+    Gapfill incomplete datestrs and returns them in the format `YYYY-MM-DDTHH:mm:ss`.
+    """
+    datestr_ = str(datestr)
+    format_ = _get_datestr_format(datestr_)
+    should_run = format_ in _VALID_GAPFILL_DATE_FORMATS
+    return (
+        None
+        if datestr is None
+        else (
+            DATESTR_GAPFILL_MODE_TO_GAPFILL_FUNCTION[mode](datestr_, format_)
+            if should_run
+            else datestr_
+        )
+    )
+
+
+def convert_datestr(
+    datestr: str,
+    target_format: DatestrFormat,
+    gapfill_mode: DatestrGapfillMode = "start",
+) -> str:
+    should_run = validate_datestr_format(datestr, _VALID_GAPFILL_DATE_FORMATS)
+    return (
+        datetime.strptime(
+            gapfill_datestr(datestr, gapfill_mode),
+            DatestrFormat.YEAR_MONTH_DAY_HOUR_MINUTE_SECOND.value,
+        ).strftime(target_format.value)
+        if should_run
+        else datestr
+    )
+
+
+def parse_gapfilled_datestr(
+    datestr: str, gapfill_mode: DatestrGapfillMode = "start", default: Any = None
+):
+    return safe_parse_date(gapfill_datestr(datestr, mode=gapfill_mode), default=default)
+
+
+def datestrs_match(
+    datestr_a: str, datestr_b: str, mode: DatestrGapfillMode = "start"
+) -> bool:
+    """
+    Comparison of non-gap-filled string dates.
+    example: For end dates, '2010' would match '2010-12-31', but not '2010-01-01'
+    """
+    return gapfill_datestr(datestr=datestr_a, mode=mode) == gapfill_datestr(
+        datestr=datestr_b, mode=mode
+    )
+
+
+def _diff_in_years_calendar(a: datetime, b: datetime, *, add_second: bool, **_) -> int:
+    reverse = a > b
+    b_ = (
+        b
+        if not add_second
+        else b - relativedelta(seconds=1) if reverse else b + relativedelta(seconds=1)
+    )
+    diff = relativedelta(b_, a)
+    return diff.years
+
+
+def _diff_in_months_calendar(a: datetime, b: datetime, *, add_second: bool, **_) -> int:
+    reverse = a > b
+    b_ = (
+        b
+        if not add_second
+        else b - relativedelta(seconds=1) if reverse else b + relativedelta(seconds=1)
+    )
+    diff = relativedelta(b_, a)
+    return diff.years * 12 + diff.months
+
+
+def _diff(
+    a: datetime, b: datetime, *, unit: TimeUnit, add_second: bool, complete_only: bool
+) -> Union[float, int]:
+    reverse = a > b
+    b_ = (
+        b
+        if not add_second
+        else b - relativedelta(seconds=1) if reverse else b + relativedelta(seconds=1)
+    )
+    diff = convert_duration((b_ - a).total_seconds(), TimeUnit.SECOND, unit)
+    return floor(diff) if complete_only else diff
+
+
+DIFF_FUNCTION = {
+    (TimeUnit.YEAR, True): _diff_in_years_calendar,
+    (TimeUnit.MONTH, True): _diff_in_months_calendar,
+}
+"""
+(unit: TimeUnit, calendar: bool): Callable
+"""
+
+
+def diff_in(
+    a: Union[datetime, str],
+    b: Union[datetime, str],
+    unit: TimeUnit,
+    add_second=False,
+    calendar=False,
+    gapfill_mode: DatestrGapfillMode = "start",
+):
+    """
+    Calculate the difference between two dates.
+
+    This function does NOT return the absolute difference. If `b` is before `a` the function will return a negative
+    value.
+
+    If dates are passed as datestrings, they will be parsed into datetime objects. Caution is advised when using
+    datestrings with formats `--MM` and `--MM-DD` as these might be parsed in unexpected ways.
+
+    Parameters
+    ----------
+    a : datetime | str
+        The first date.
+
+    b: datetime | str
+        The second date.
+
+    unit : TimeUnit
+        The time unit to calculate the diff in.
+
+    add_second : bool, optional, default = `False`
+        A flag to determine whether to add one second to diff results.
+
+        Set to `True` in cases where you are calculating the duration of nodes with incomplete datestrings.
+
+        For example, a node with `"startDate"` = `"2000"` and `"endDate"` = `"2001"` will ordinarily be assumed to take
+        place over the entirety of 2000 and 2001 (i.e., from `"2000-01-01T00-00-00"` to `"2001-12-31T23-59-59"`).
+        However, If `add_second` = `False`, the diff in days will be slightly less than 731 because the final second of
+        2001-12-31 is not accounted for. If `True` the diff will be exactly 731.
+
+    calendar : bool, optional, default = `False`
+        A flag to determine whether to use calendar time units.
+
+        If `True` the diff in years between `"2000"` and `"2001"` will be exactly 1, if `False` the diff will be
+        slightly over 1 because a leap year is longer than the average year.
+
+        If `True` the diff in months between `"2000-02"` and `"2000-03"` will be exactly 1, if `False` the diff will be
+        approximately 0.95 because February is shorter than the average month.
+
+        For all units, if `True`, only complete units will be counted, For example, the diff in days between
+        `"2000-01-01:00:00:00"` and `"2000-01-01:12:00:00"` will be 0. If `False` the diff will be 0.5.
+
+    gapfill_mode : DatestrGapfillMode, optional, default = `"start"`
+        How to gapfill incomplete datestrings (`"start"`, `"middle"` or `"end"`).
+
+    Returns
+    -------
+    diff : float | int
+        The difference between the dates in the selected units.
+    """
+    a_, b_ = (
+        (
+            d
+            if isinstance(d, datetime)
+            else parse_gapfilled_datestr(d, gapfill_mode=gapfill_mode)
+        )
+        for d in (a, b)
+    )
+
+    diff_func = DIFF_FUNCTION.get(
+        (unit, calendar),
+        lambda *_, **kwargs: _diff(a_, b_, **kwargs, complete_only=calendar),
+    )
+
+    return diff_func(a_, b_, unit=unit, add_second=add_second)

hestia_earth/utils/pivot/_shared.py

@@ -1,6 +1,10 @@
 import json
 import numpy as np
-from hestia_earth.schema import EmissionMethodTier
+import pandas as pd
+from hestia_earth.schema import SCHEMA_TYPES, NODE_TYPES, EmissionMethodTier
+from flatten_json import flatten as flatten_json
+
+from ..tools import list_sum
 
 
 EXCLUDE_FIELDS = ["@type", "type", "@context"]
@@ -14,6 +18,20 @@ EXCLUDE_PRIVATE_FIELDS = [
 ]
 
 
+# assuming column labels always camelCase
+def _get_node_type_label(node_type):
+    return node_type[0].lower() + node_type[1:]
+
+
+def _get_node_type_from_label(node_type):
+    return node_type[0].upper() + node_type[1:]
+
+
+def _is_blank_node(data: dict):
+    node_type = data.get("@type") or data.get("type")
+    return node_type in SCHEMA_TYPES and node_type not in NODE_TYPES
+
+
 def _with_csv_formatting(dct):
     """
     Use as object_hook when parsing a JSON node: json.loads(node, object_hook=_with_csv_formatting).
@@ -44,7 +62,7 @@ def _filter_not_relevant(blank_node: dict):
 
 def _filter_emissions_not_relevant(node: dict):
     """
-    Ignore all emissions where `methodTier=not relevant` so save space.
+    Ignore all emissions where `methodTier=not relevant` to save space.
     """
     return node | (
         {
@@ -53,3 +71,40 @@ def _filter_emissions_not_relevant(node: dict):
             if key in node
         }
     )
+
+
+def _filter_zero_value(blank_node: dict):
+    value = blank_node.get("value")
+    value = (
+        list_sum(blank_node.get("value"), default=-1)
+        if isinstance(value, list)
+        else value
+    )
+    return value != 0
+
+
+def _filter_zero_values(node: dict):
+    """
+    Ignore all blank nodes where `value=0` to save space.
+    """
+    return node | (
+        {
+            key: list(filter(_filter_zero_value, value))
+            for key, value in node.items()
+            if isinstance(value, list)
+            and isinstance(value[0], dict)
+            and _is_blank_node(value[0])
+        }
+    )
+
+
+def nodes_to_df(nodes: list[dict]):
+    nodes_flattened = [
+        flatten_json(
+            dict([(_get_node_type_label(node.get("@type", node.get("type"))), node)]),
+            ".",
+        )
+        for node in nodes
+    ]
+
+    return pd.json_normalize(nodes_flattened)

hestia_earth/utils/pivot/pivot_csv.py

@@ -5,7 +5,6 @@ import numpy as np
 import pandas as pd
 from hestia_earth.schema import UNIQUENESS_FIELDS, Term, NODE_TYPES
 from hestia_earth.schema.utils.sort import get_sort_key, SORT_CONFIG
-from flatten_json import flatten as flatten_json
 
 # __package__ = "hestia_earth.utils" # required to run interactively in vscode
 from ..api import find_term_ids_by_names
@@ -14,6 +13,9 @@ from ._shared import (
     EXCLUDE_PRIVATE_FIELDS,
     _with_csv_formatting,
     _filter_emissions_not_relevant,
+    _get_node_type_label,
+    _get_node_type_from_label,
+    nodes_to_df,
 )
 
 
@@ -36,15 +38,6 @@ def _get_blank_node_uniqueness_fields():
 BLANK_NODE_UNIQUENESS_FIELDS = _get_blank_node_uniqueness_fields()
 
 
-# assuming column labels always camelCase
-def _get_node_type_label(node_type):
-    return node_type[0].lower() + node_type[1:]
-
-
-def _get_node_type_from_label(node_type):
-    return node_type[0].upper() + node_type[1:]
-
-
 def _get_names(df):
     names = []
     for node_type, array_fields in BLANK_NODE_UNIQUENESS_FIELDS.items():
@@ -283,18 +276,6 @@ def _format_and_pivot(df_in):
     return df_out
 
 
-def nodes_to_df(nodes: list[dict]):
-    nodes_flattened = [
-        flatten_json(
-            dict([(_get_node_type_label(node.get("@type", node.get("type"))), node)]),
-            ".",
-        )
-        for node in nodes
-    ]
-
-    return pd.json_normalize(nodes_flattened)
-
-
 def pivot_nodes(nodes: list[dict]):
     """
     Pivot array of nodes in dict format (e.g under the 'nodes' key of a .hestia file)

hestia_earth/utils/pivot/pivot_json.py

@@ -11,6 +11,7 @@ from ._shared import (
     EXCLUDE_PRIVATE_FIELDS,
     _with_csv_formatting,
     _filter_emissions_not_relevant,
+    _filter_zero_values,
 )
 
 pivot_exclude_fields = Term().fields
@@ -22,6 +23,8 @@ term_exclude_fields = Term().fields
 del term_exclude_fields["name"]
 term_exclude_fields.update({k: "" for k in EXCLUDE_PRIVATE_FIELDS})
 
+include_all_unique_keys = ["emissions", "emissionsResourceUse"]
+
 # Treat properties uniqueness fields as special case for now
 PROPERTIES_VIRTUAL_UNIQUENESS_FIELD = "propertyValues"
 ADAPTED_UNIQUENESS_FIELDS = deepcopy(UNIQUENESS_FIELDS)
@@ -90,6 +93,7 @@ def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa:
 
     # print('\n', level, 'fields_to_pivot', fields_to_pivot)
     for field, uniqueness_fields in fields_to_pivot:
+        include_all_unique_fields = field in include_all_unique_keys
         # print('\nbefore processing node field', level, field, node[field], '\n')
         # Compress lists of 'Node' nodes to dict with single @id key.
         # The compressed field matches uniqueness fields like cycle.emissions.inputs.@id.
@@ -155,14 +159,13 @@ def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa:
             del term_data["combined_fields"][id_key]
             # print('combined_fields', field, term_id, term_data['combined_fields'], '\n')
             fields_to_include = {
-                field: any(
-                    by_idx[idx].get(field) != by_idx[indexes[0]].get(field)
-                    for idx in indexes
+                k: include_all_unique_fields
+                or any(
+                    by_idx[idx].get(k) != by_idx[indexes[0]].get(k) for idx in indexes
                 )
-                for field in term_data["combined_fields"].keys()
-                if field in uniqueness_fields
-                or field != "value"
-                and field.split(".")[-1] not in pivot_exclude_fields
+                for k in term_data["combined_fields"].keys()
+                if k in uniqueness_fields
+                or (k != "value" and k.split(".")[-1] not in pivot_exclude_fields)
             }
             # print('fields_to_include', level, field, term_id, fields_to_include, '\n')
             for idx in indexes:
@@ -179,19 +182,23 @@ def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa:
                     )
                 ]
                 # print('distingishing_field_fields', level, field, term_id, distingishing_field_fields, '\n')
-                unanimous_fields = {
-                    field: term_data["combined_fields"][field]
-                    for field, not_unanimous in fields_to_include.items()
-                    if field not in distingishing_field_fields
-                    and not not_unanimous
-                    and field is not PROPERTIES_VIRTUAL_UNIQUENESS_FIELD
-                }
                 # print('unanimous_fields', level, field, term_id, unanimous_fields, '\n')
+                unanimous_fields = (
+                    {}
+                    if include_all_unique_fields
+                    else {
+                        field: term_data["combined_fields"][field]
+                        for field, not_unanimous in fields_to_include.items()
+                        if field not in distingishing_field_fields
+                        and not not_unanimous
+                        and field is not PROPERTIES_VIRTUAL_UNIQUENESS_FIELD
+                    }
+                )
                 differentiated_fields = {
                     field: term[field]
                     for field, not_unanimous in fields_to_include.items()
                     if field not in distingishing_field_fields
-                    and not_unanimous
+                    and (include_all_unique_fields or not_unanimous)
                     and field in term
                 }
                 # print('differentiated_fields', level, field, term_id, differentiated_fields, '\n')
@@ -223,19 +230,35 @@ def _do_pivot(node, parent_node_type=None, parent_field=None, level=0):  # noqa:
     return pivoted_node
 
 
-def pivot_node(node: dict):
+def pivot_node(
+    node: dict,
+    include_emissions_not_relevant: bool = False,
+    include_zero_values: bool = True,
+):
     """
     Pivot single node in dict format parsed with object_hook=_with_csv_formatting
     """
-    return _do_pivot(_filter_emissions_not_relevant(node))
+    node = (
+        node if include_emissions_not_relevant else _filter_emissions_not_relevant(node)
+    )
+    node = node if include_zero_values else _filter_zero_values(node)
+    return _do_pivot(node)
 
 
-def pivot_json_node(json_node: str):
+def pivot_json_node(
+    json_node: str,
+    include_emissions_not_relevant: bool = False,
+    include_zero_values: bool = True,
+):
     """
     Pivot single schema-compliant unparsed json string node
     """
     node = json.loads(json_node, object_hook=_with_csv_formatting)
-    return pivot_node(node)
+    return pivot_node(
+        node,
+        include_emissions_not_relevant=include_emissions_not_relevant,
+        include_zero_values=include_zero_values,
+    )
 
 
 def pivot_hestia_file(hestia_file: str):
@@ -246,8 +269,19 @@ def pivot_hestia_file(hestia_file: str):
     return pivot_nodes(parsed.get("nodes", []))
 
 
-def pivot_nodes(nodes: list[dict]):
+def pivot_nodes(
+    nodes: list[dict],
+    include_emissions_not_relevant: bool = False,
+    include_zero_values: bool = True,
+):
     """
     Pivot multiple nodes in dict format parsed with object_hook=_with_csv_formatting
     """
-    return [pivot_node(node) for node in nodes]
+    return [
+        pivot_node(
+            node,
+            include_emissions_not_relevant=include_emissions_not_relevant,
+            include_zero_values=include_zero_values,
+        )
+        for node in nodes
+    ]

hestia_earth/utils/tools.py

@@ -1,9 +1,11 @@
-import time
+from collections.abc import Generator, Iterable
 from dateutil.parser import parse
-from statistics import mean
 from functools import reduce
-from math import log10, floor
 import numpy
+from math import log10, floor
+from statistics import mean
+import time
+from typing import Literal
 from hestia_earth.schema import NodeType
 
 
@@ -268,3 +270,67 @@ def pick(value: dict, keys: list) -> dict:
 
 def unique_values(values: list, key: str = "@id"):
     return list({v[key]: v for v in values}.values())
+
+
+def is_list_like(obj) -> bool:
+    """
+    Return `True` if the input arg is an instance of an `Iterable` (excluding `str` and `bytes`) or a `Generator`, else
+    return `False`.
+    """
+    return isinstance(obj, (Iterable, Generator)) and not isinstance(obj, (str, bytes))
+
+
+TO_LIST_LIKE_CONSTRUCTOR = {"list": list, "set": set, "tuple": tuple}
+
+
+def _as_list_like(obj, to: Literal["list", "set", "tuple"] = "list"):
+    """
+    Convert an object to either a list, set or tuple.
+
+    If the object is list-like, convert it to the target iterable. If the object is not list-like, wrap the
+    object in the iterable.
+
+    `str` and `bytes` objects are not consider list-like and, therefore, will be wrapped.
+    """
+    constructor = TO_LIST_LIKE_CONSTRUCTOR.get(to, list)
+    return (
+        obj
+        if isinstance(obj, constructor)
+        else constructor(obj) if is_list_like(obj) else constructor([obj])
+    )
+
+
+def as_list(obj) -> list:
+    """
+    Convert an object to a list.
+
+    If the object is a list, return it. Else, if the object is list-like, convert it into a list. Else, wrap the object
+    in a list (e.g., `[obj]`).
+
+    `str` and `bytes` objects are not consider list-like and, therefore, will be wrapped (e.g., `"abc"` -> `["abc"]`).
+    """
+    return _as_list_like(obj, "list")
+
+
+def as_set(obj) -> set:
+    """
+    Convert an object to a set.
+
+    If the object is a set, return it. Else, if the object is list-like, convert it into a set. Else, wrap the object
+    in a set (e.g., `{obj}`).
+
+    `str` and `bytes` objects are not consider list-like and, therefore, will be wrapped (e.g., `"abc"` -> `{"abc"}`).
+    """
+    return _as_list_like(obj, "set")
+
+
+def as_tuple(obj) -> tuple:
+    """
+    Convert an object to a tuple.
+
+    If the object is a tuple, return it. Else, if the object is list-like, convert it into a tuple. Else, wrap the
+    object in a tuple (e.g., `(obj, )`)
+
+    `str` and `bytes` objects are not consider list-like and, therefore, will be wrapped (e.g., `"abc"` -> `("abc", )`).
+    """
+    return _as_list_like(obj, "tuple")

hestia_earth/utils/version.py

@@ -1 +1 @@
-VERSION = "0.16.12"
+VERSION = "0.16.14"

{hestia_earth_utils-0.16.12.dist-info → hestia_earth_utils-0.16.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.12
+Version: 0.16.14
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

{hestia_earth_utils-0.16.12.dist-info → hestia_earth_utils-0.16.14.dist-info}/RECORD

@@ -3,7 +3,7 @@ hestia_earth/utils/api.py,sha256=b6g87ylAgdWvwPlDeZDv74UGfXVe1KFXap5-Rv5daSE,992
 hestia_earth/utils/blank_node.py,sha256=K_8wWACiwOqEJR71ClQFInzsIdDH3UUSwrtCZoh9V-o,7854
 hestia_earth/utils/calculation_status.py,sha256=f5b05cEFXMfFI1clirIt7v3Y9H2Nja66GYv1NCyZjf0,2381
 hestia_earth/utils/cycle.py,sha256=oo0CesLMblL8ewI4s7eXdyNjr9R9df4Vyr0iRXCYFu4,1326
-hestia_earth/utils/date.py,sha256=IVqZjIJda8lUbDZkEbaNN5FBQoTHdNudgKXpngm-2a4,1847
+hestia_earth/utils/date.py,sha256=cwcGtYVh2K5rH7k5uO1Olovy-hzIDrvsfaiR94ZlyO0,16165
 hestia_earth/utils/descriptive_stats.py,sha256=YvDI6EWCcZWw8yCxYhqyzMCDqCu2X8DjvygmMK_AVvc,1633
 hestia_earth/utils/emission.py,sha256=rHHf5vwe-RxTOaOJ9N0MuyJOLpDnPjjxv6MHYSpPgcU,2165
 hestia_earth/utils/lookup.py,sha256=SArKqjqs_Yt5cC6TIk12WVQNToun2spZ_iNUm4fX9FA,8274
@@ -14,20 +14,20 @@ hestia_earth/utils/request.py,sha256=EfitmS13abPxfl5gSAOyHRUYhk4R2Rfv94fF9lvOT00
 hestia_earth/utils/stats.py,sha256=vTNyKcMKmX0DoodM9QEG7HF8qm2Wf-4ckMWQFWZ1VgE,34729
 hestia_earth/utils/table.py,sha256=MOJDo5fQPRDogAty_UXbO9-EXFwz97m0f7--mOM17lQ,2363
 hestia_earth/utils/term.py,sha256=aBVYuYv55nPqJPyt5mN4Fz652s_1hwUPckNUZX0pMP8,1064
-hestia_earth/utils/tools.py,sha256=b8_ZJ4lfRVwOFXzl4_Tbah1FmrGnzDx5QVgVLQmQUz4,5372
-hestia_earth/utils/version.py,sha256=z2tz1TqNic_wxJ0cdufZWrE4ryfXE9jZgzJKKWhdm4E,20
+hestia_earth/utils/tools.py,sha256=WMx05cBtBR8mYQnLLBA2cgF1x2tI41514diFDeX4gLQ,7533
+hestia_earth/utils/version.py,sha256=R_I9V6MKTrP_WY9UKq_aQBdaU5xIiWzkcyC-foZlSnM,20
 hestia_earth/utils/pivot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hestia_earth/utils/pivot/_shared.py,sha256=8-AZAAitXU53BYpOFFGnuCy5VBSphyClgIqGU-kg6CM,1436
-hestia_earth/utils/pivot/pivot_csv.py,sha256=TtePpo9L_GKISpQMW9dwjBv2tHHE822rb-j6_bflVOA,12251
-hestia_earth/utils/pivot/pivot_json.py,sha256=CrFwIxRXuUemdr5-k7weloB5rHeTLX5yB9K-I3xDVmk,9977
+hestia_earth/utils/pivot/_shared.py,sha256=ob8T7bziZ6YffpRx_srzzH0h96z2GZJB5uVe9NxALIg,2828
+hestia_earth/utils/pivot/pivot_csv.py,sha256=1zGYuC_xlrLoX5FYSG3mt5HSc03CO-0p8J0oAmyI1VI,11778
+hestia_earth/utils/pivot/pivot_json.py,sha256=yWbsuFeTQJPk4H2b7ibVALGJa3qfKGyNuPeqM8yrcsI,11030
 hestia_earth/utils/storage/__init__.py,sha256=uNX6_EHWWnNUIm4Ng7L43-cQmuc6NGFAxXye85saIXQ,922
 hestia_earth/utils/storage/_azure_client.py,sha256=mseexhzjteRDzzoFe2fEXe9MYLmvjG3sfl9-eha4ZW0,1353
 hestia_earth/utils/storage/_local_client.py,sha256=KbYqTfniIU5R5J1m_unCQip9kOz9EGIGI0OH0QvD8eo,551
 hestia_earth/utils/storage/_s3_client.py,sha256=8TCxiHfxE7G8kdp3CnEFrxgmPwfPyci3-blsowE2T7o,3146
 hestia_earth/utils/storage/_sns_client.py,sha256=pvtXYw-sQ8ns3mlDz7ld9iZp3FYSm0xSXSXMJ5IPnBc,380
-hestia_earth_utils-0.16.12.data/scripts/hestia-format-upload,sha256=IhLAHHPJqRgUcht-M_EUEsRMbRbMfshig07o488zscM,703
-hestia_earth_utils-0.16.12.data/scripts/hestia-pivot-csv,sha256=0YBuGuyPO8rytod6iwWEKiQdSlr9JLuD001k6U5t6no,1163
-hestia_earth_utils-0.16.12.dist-info/METADATA,sha256=CZegpFMoj0MWumacYaibHi1BCg6IDaJDuIDtMlEzT3A,1870
-hestia_earth_utils-0.16.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-hestia_earth_utils-0.16.12.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
-hestia_earth_utils-0.16.12.dist-info/RECORD,,
+hestia_earth_utils-0.16.14.data/scripts/hestia-format-upload,sha256=IhLAHHPJqRgUcht-M_EUEsRMbRbMfshig07o488zscM,703
+hestia_earth_utils-0.16.14.data/scripts/hestia-pivot-csv,sha256=0YBuGuyPO8rytod6iwWEKiQdSlr9JLuD001k6U5t6no,1163
+hestia_earth_utils-0.16.14.dist-info/METADATA,sha256=4i7b1a12W_HBp3NtPrR1Kr6ztiedReV_svrcDcnjKEY,1870
+hestia_earth_utils-0.16.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+hestia_earth_utils-0.16.14.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
+hestia_earth_utils-0.16.14.dist-info/RECORD,,