hestia-earth-utils 0.16.1__py3-none-any.whl → 0.16.3__py3-none-any.whl

hestia_earth/utils/csv_utils.py

@@ -0,0 +1,72 @@
+import io
+import csv
+import re
+import numpy as np
+
+_MISSING_VALUE = '-'
+_MISSING = -99999
+_DELIMITER = ','
+_QUOTE_CHAR = '"'
+ENCODING = 'ISO-8859-1'
+# default: " !#$%&'()*+,-./:;<=>?@[\\]^{|}~"
+_DELETE_CHARS = " !#$%&'()*,./:;<=>?@^{|}~"
+
+
+def is_missing_value(value): return value == _MISSING_VALUE or value == _MISSING or value == str(_MISSING)
+
+
+def _replace_missing_values(value: str): return str(_MISSING) if str(value) == _MISSING_VALUE else value
+
+
+def _replace_chars(value: str): return re.sub(f'[{re.escape(_DELETE_CHARS)}]', '', value.replace(' ', '_'))
+
+
+def _text_to_csv(csv_content: str):
+    return csv.reader(io.StringIO(csv_content.strip()), delimiter=_DELIMITER, quotechar=_QUOTE_CHAR)
+
+
+def _csv_reader_converter(field_str_bytes):
+    field_str = field_str_bytes if isinstance(field_str_bytes, str) else field_str_bytes.decode('utf-8')
+    reader = _text_to_csv(field_str)
+
+    try:
+        return _replace_missing_values(next(reader)[0].strip())
+    except StopIteration:
+        return str(_MISSING)
+
+
+def _get_columns(csv_content: str):
+    try:
+        reader = _text_to_csv(csv_content)
+        names = next(reader)
+        return list(map(_replace_chars, names))
+    except StopIteration:
+        return []
+
+
+def csv_str_to_recarray(csv_content: str) -> np.recarray:
+    names = _get_columns(csv_content)
+    num_cols = len(names)
+
+    converters_dict = {
+        i: _csv_reader_converter
+        for i in range(num_cols)
+    }
+
+    # TODO: find the maximum column size instead of using 1000
+    max_size = 1000
+    return np.loadtxt(
+        io.StringIO(csv_content.strip()),
+        delimiter=_DELIMITER,
+        quotechar=_QUOTE_CHAR,
+        skiprows=1,
+        converters=converters_dict,
+        dtype=[(name, f"U{max_size}") for name in names],
+        encoding=ENCODING
+    ).view(np.recarray)
+
+
+def csv_file_to_recarray(filepath: str):
+    with open(filepath, 'r', encoding=ENCODING) as f:
+        content = f.read()
+    return csv_str_to_recarray(content)

hestia_earth/utils/lookup.py

@@ -1,51 +1,16 @@
 from functools import reduce
-from io import StringIO
 from typing import Union
-import re
 import requests
-import csv
 import numpy
+import traceback
 
 from .storage import _load_from_storage
 from .request import request_url, web_url
+from .csv_utils import csv_str_to_recarray, csv_file_to_recarray, is_missing_value, _replace_chars
 
-DELIMITER = '\t'
-ENCODING = 'ISO-8859-1'
-GLOSSARY_FOLDER = 'glossary/lookups'
+_GLOSSARY_FOLDER = 'glossary/lookups'
 _memory = {}
-MISSING_VALUE = '-'
-MISSING = -99999
 _INDEX_COL = 'termid'
-# default: " !#$%&'()*+,-./:;<=>?@[\\]^{|}~"
-_DELETE_CHARS = " !#$%&'()*,./:;<=>?@^{|}~"
-
-
-def _is_missing_value(value): return value == MISSING_VALUE or value == MISSING or value == str(MISSING)
-
-
-def _replace_missing_values(value: str): return str(MISSING) if str(value) == '-' else value
-
-
-def _rewrite_csv_file_as_tab(filepath: str):
-    with open(filepath, 'r', encoding=ENCODING) as fp:
-        reader = csv.reader(fp)
-        for row in reader:
-            yield DELIMITER.join(list(map(_replace_missing_values, row)))
-
-
-def _rewrite_csv_text_as_tab(text: str):
-    reader = csv.reader(StringIO(text))
-    for row in reader:
-        yield DELIMITER.join(list(map(_replace_missing_values, row)))
-
-
-def _recfromcsv(data): return numpy.recfromcsv(data,
-                                               missing_values=MISSING_VALUE,
-                                               filling_values=MISSING,
-                                               delimiter=DELIMITER,
-                                               encoding=ENCODING,
-                                               case_sensitive=True,
-                                               deletechars=_DELETE_CHARS)
 
 
 def _memory_wrapper(key: str, func):
@@ -70,12 +35,12 @@ def load_lookup(filepath: str, keep_in_memory: bool = False):
     numpy.recarray
         The `numpy.recarray` converted from the csv content.
     """
-    def load(): return _recfromcsv(_rewrite_csv_file_as_tab(filepath))
+    def load(): return csv_file_to_recarray(filepath)
     return _memory_wrapper(filepath, load) if keep_in_memory else load()
 
 
 def _download_lookup_data(filename: str):
-    filepath = f"{GLOSSARY_FOLDER}/{filename}"
+    filepath = f"{_GLOSSARY_FOLDER}/{filename}"
 
     def fallback():
         url = request_url(f"{web_url()}/{filepath}")
@@ -121,12 +86,14 @@ def download_lookup(filename: str, keep_in_memory: bool = True, build_index: boo
     """
     def load():
         data = _download_lookup_data(filename)
-        rec = _recfromcsv(_rewrite_csv_text_as_tab(data)) if data else None
+        rec = csv_str_to_recarray(data) if data else None
         return (_build_index(rec) if build_index else rec) if data else None
 
     try:
         return _memory_wrapper(filename, load) if keep_in_memory else load()
     except Exception:
+        stack = traceback.format_exc()
+        print(stack)
         return None
 
 
@@ -144,7 +111,19 @@ def column_name(key: str):
     str
         The column name that can be used in `get_table_value`.
     """
-    return re.sub('[' + re.escape(_DELETE_CHARS) + ']', '', key.replace(' ', '_')) if key else ''
+    return _replace_chars(key) if key else ''
+
+
+def _parse_value(value: str):
+    """ Automatically converts the value to float or bool if possible """
+    try:
+        return (
+            True if str(value).lower() == 'true' else
+            False if str(value).lower() == 'false' else
+            float(value)
+        )
+    except Exception:
+        return value
 
 
 def _get_single_table_value(data: Union[dict, numpy.recarray], col_match: str, col_match_with, col_val):
@@ -163,7 +142,7 @@ def _get_multiple_table_values(data: Union[dict, numpy.recarray], col_match: str
     return reduce(reducer, enumerate(col_match), data)[col_val][0]
 
 
-def get_table_value(lookup: Union[dict, numpy.recarray], col_match: str, col_match_with, col_val):
+def get_table_value(lookup: Union[dict, numpy.recarray], col_match: str, col_match_with, col_val, default_value=''):
     """
     Get a value matched by one or more columns from a `numpy.recarray`.
 
@@ -179,6 +158,8 @@ def get_table_value(lookup: Union[dict, numpy.recarray], col_match: str, col_mat
         Can be a single `str` or a list of `str`. If a list is used, must be the same length as `col_match`.
     col_val: str
         The column which contains the value to look for.
+    default_value : Any
+        A value to return when none if found in the data.
 
     Returns
     -------
@@ -191,7 +172,7 @@ def get_table_value(lookup: Union[dict, numpy.recarray], col_match: str, col_mat
             _get_single_table_value(lookup, col_match, col_match_with, col_val) if single else
             _get_multiple_table_values(lookup, col_match, col_match_with, col_val)
         )
-        return None if _is_missing_value(value) else value
+        return default_value if is_missing_value(value) else _parse_value(value)
     except Exception:
         return None
 
@@ -251,7 +232,7 @@ def extract_grouped_data(data: str, key: str) -> str:
         **{curr.split(':')[0]: curr.split(':')[1]}
     }, data.split(';'), {}) if data is not None and isinstance(data, str) and len(data) > 1 else {}
     value = grouped_data.get(key)
-    return None if _is_missing_value(value) else value
+    return None if is_missing_value(value) else _parse_value(value)
 
 
 def extract_grouped_data_closest_date(data: str, year: int) -> str:
@@ -278,13 +259,13 @@ def extract_grouped_data_closest_date(data: str, year: int) -> str:
         lambda prev, curr: {
             **prev,
             **{curr.split(':')[0]: curr.split(':')[1]}
-        } if len(curr) > 0 and not _is_missing_value(curr.split(':')[1]) else prev,
+        } if len(curr) > 0 and not is_missing_value(curr.split(':')[1]) else prev,
         data.split(';'),
         {}
     ) if data is not None and isinstance(data, str) and len(data) > 1 else {}
     dist_years = list(data_by_date.keys())
     closest_year = min(dist_years, key=lambda x: abs(int(x) - year)) if len(dist_years) > 0 else None
-    return None if closest_year is None else data_by_date.get(closest_year)
+    return None if closest_year is None else _parse_value(data_by_date.get(closest_year))
 
 
 def lookup_term_ids(lookup: Union[dict, numpy.recarray]):

hestia_earth/utils/pivot/pivot_csv.py

@@ -2,6 +2,7 @@ import copy
 import json
 import re
 import numpy as np
+import pandas as pd
 from hestia_earth.schema import UNIQUENESS_FIELDS, Term, NODE_TYPES
 from hestia_earth.schema.utils.sort import get_sort_key, SORT_CONFIG
 from flatten_json import flatten as flatten_json
@@ -11,17 +12,6 @@ from ..api import find_term_ids_by_names
 from ._shared import EXCLUDE_FIELDS, EXCLUDE_PRIVATE_FIELDS, _with_csv_formatting, _filter_emissions_not_relevant
 
 
-PANDAS_IMPORT_ERROR_MSG = "Run `pip install pandas>=1.2` to use this functionality"
-try:
-    import pandas as pd
-
-    version = [int(x) for x in pd.__version__.split('+')[0].split(".")]
-    if version[0] < 1 or (version[0] == 1 and version[1] < 2):
-        raise ImportError(PANDAS_IMPORT_ERROR_MSG)
-except ImportError:
-    raise ImportError(PANDAS_IMPORT_ERROR_MSG)
-
-
 # We only want to pivot array items containing blank nodes
 # Assume these are all fields with uniqueness fields not of type Node
 def _get_blank_node_uniqueness_fields():

hestia_earth/utils/table.py

@@ -1,22 +1,12 @@
 from functools import reduce
 import numpy as np
+import pandas as pd
 from hestia_earth.schema import NodeType
 
 # __package__ = "hestia_earth.utils" # required to run interactively in vscode
 from .tools import flatten
 
 
-PANDAS_IMPORT_ERROR_MSG = "Run `pip install pandas>=1.2` to use this functionality"
-try:
-    import pandas as pd
-
-    version = [int(x) for x in pd.__version__.split('+')[0].split(".")]
-    if version[0] < 1 or (version[0] == 1 and version[1] < 2):
-        raise ImportError(PANDAS_IMPORT_ERROR_MSG)
-except ImportError:
-    raise ImportError(PANDAS_IMPORT_ERROR_MSG)
-
-
 def _replace_ids(df):
     # in columns, first letter is always lower case
     node_types = [e.value[0].lower() + e.value[1:] for e in NodeType]
@@ -74,11 +64,6 @@ def format_for_upload(filepath: str):
     pandas.DataFrame
         Formatted pandas dataframe
     """
-    try:
-        import pandas as pd
-    except ImportError:
-        raise ImportError("Run `pip install pandas~=1.2.0` to use this functionality")
-
     df = pd.read_csv(filepath, index_col=None, na_values="")
 
     # replace @id with id for top-level Node

hestia_earth/utils/version.py

@@ -1 +1 @@
-VERSION = '0.16.1'
+VERSION = '0.16.3'

{hestia_earth_utils-0.16.1.dist-info → hestia_earth_utils-0.16.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.1
+Version: 0.16.3
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team
@@ -13,8 +13,12 @@ Requires-Dist: hestia-earth-schema>=35.0.1
 Requires-Dist: requests>=2.24.0
 Requires-Dist: urllib3~=1.26.0
 Requires-Dist: python-dateutil>=2.8.1
-Requires-Dist: numpy<2,>=1.25.0
+Requires-Dist: numpy>=2
 Requires-Dist: flatten_json
+Provides-Extra: pivot-csv
+Requires-Dist: pandas>=2; extra == "pivot-csv"
+Provides-Extra: table
+Requires-Dist: pandas>=2; extra == "table"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier
@@ -22,6 +26,7 @@ Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: license
+Dynamic: provides-extra
 Dynamic: requires-dist
 Dynamic: requires-python
 Dynamic: summary

{hestia_earth_utils-0.16.1.dist-info → hestia_earth_utils-0.16.3.dist-info}/RECORD

@@ -2,32 +2,33 @@ hestia_earth/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 hestia_earth/utils/api.py,sha256=y0gw5pCCHNnFIhM62Hok_5eDtH3QDAZdkye_1mANMNs,9654
 hestia_earth/utils/blank_node.py,sha256=1wc9zUkOvFhJS-YmuKexfIdYxfsp5KyJczLmHlW559Q,7375
 hestia_earth/utils/calculation_status.py,sha256=X7lbgVMD9luH1gj9lEcxd3_P2-u7e8ZPGCvX1czPZUo,2238
+hestia_earth/utils/csv_utils.py,sha256=nb_ihJaTj3K5hO7cxXO1xjTLVGVX1P13m9SgquO5-XY,1990
 hestia_earth/utils/cycle.py,sha256=rFLRL9X4KQ1UrE6fEPA_gV8KmwzrZpR3Ce56zg41lRk,1326
 hestia_earth/utils/date.py,sha256=SPQ69uxHiv1o3BqIkBKkM5XX_CmS20CB7g6u2rhsdh8,1807
 hestia_earth/utils/descriptive_stats.py,sha256=EMVwFvg2OnZgKRAfireAoWY2EbrSvqR0V0bK9B53p28,1583
 hestia_earth/utils/emission.py,sha256=BhBitooLTxZSh82S982v2QfPxxTF1kmGClG_uHyWdz4,1981
-hestia_earth/utils/lookup.py,sha256=0RLqy3HPzkbhkRaO7fYoHU0jKhAYzI6QHMptMEbqTlg,10344
+hestia_earth/utils/lookup.py,sha256=iZnNQn13_WWFWc3LyL4qPTiL2IA48Db3y4eat5kmcuw,9599
 hestia_earth/utils/lookup_utils.py,sha256=_k3RZ1pK-gw7jq8wn9HrPWfDl4FlEWRb8bXmgaARu0w,6716
 hestia_earth/utils/model.py,sha256=uUcrF07XmBzqLni8VSaP0HoebJnQ57kk0EOmhwYMbfI,4637
 hestia_earth/utils/pipeline.py,sha256=O-6DPtK0U1lJ51LFGa1gM6pjkBJUfxOjNjY8LxQPXV0,9588
 hestia_earth/utils/request.py,sha256=bu7hkWKmFdXl2_Feawiam_x32whlclA9oP0asJyC69k,626
 hestia_earth/utils/stats.py,sha256=4t3op10xDJbGxWJEY1Jtyl302PYWyMFwLpsSkMlzQn8,34667
-hestia_earth/utils/table.py,sha256=RrTt-KF_QzjKiCpaAueoG6La1FG-Iusxw5NMDpoRBpQ,2861
+hestia_earth/utils/table.py,sha256=MOJDo5fQPRDogAty_UXbO9-EXFwz97m0f7--mOM17lQ,2363
 hestia_earth/utils/term.py,sha256=6LiUSc6KX3IOkfWF6fYkQ2tENCO8ENljcdDypxU6WtA,1060
 hestia_earth/utils/tools.py,sha256=9GaUJwxL-CTzEOGnRFkUQDVFelPevQSxXrf25vssCVo,4990
-hestia_earth/utils/version.py,sha256=xSEZ3N7McvEb7jAV8u8vNc-uFNjN6zu3mnEj2i-XYyk,19
+hestia_earth/utils/version.py,sha256=2f5woLdAmKvk1TRWtU3KMO3z6ac2wJDnGXIOq8M0TMk,19
 hestia_earth/utils/pivot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hestia_earth/utils/pivot/_shared.py,sha256=JnyIOzpans45DE2hSa9-4yvNhq8t08lx1IAWGJi6WPQ,1397
-hestia_earth/utils/pivot/pivot_csv.py,sha256=zaiDcig4I5lVSHPZ-2bXKKBcIRrayA0GUaw0c8H3D-w,12371
+hestia_earth/utils/pivot/pivot_csv.py,sha256=7f6kMqeb1b3RKANLGeDgVu8G5WC-vXIijHnsJhO-CjI,12022
 hestia_earth/utils/pivot/pivot_json.py,sha256=GBu5CFgCNdFjAuKGNsk2Phgds-xp4iREa5YIrplpFwA,9801
 hestia_earth/utils/storage/__init__.py,sha256=uNX6_EHWWnNUIm4Ng7L43-cQmuc6NGFAxXye85saIXQ,922
 hestia_earth/utils/storage/_azure_client.py,sha256=sevCZni04eknMql2DgUsWG23f7u0KvsXP7me1ZUBy00,1274
 hestia_earth/utils/storage/_local_client.py,sha256=IbzziUKY0QS3ybHFfgEpELqvafa7hQnZ-DdGdjQuypE,515
 hestia_earth/utils/storage/_s3_client.py,sha256=B2yTsf-VfHcRLCKTMes4S_nCXxrZad9umyZx3b5Pu_c,3181
 hestia_earth/utils/storage/_sns_client.py,sha256=LowUatj78Egu6_Id6Rr7hZjfZx1WguS3lozB3yAwSps,347
-hestia_earth_utils-0.16.1.data/scripts/hestia-format-upload,sha256=IhLAHHPJqRgUcht-M_EUEsRMbRbMfshig07o488zscM,703
-hestia_earth_utils-0.16.1.data/scripts/hestia-pivot-csv,sha256=0YBuGuyPO8rytod6iwWEKiQdSlr9JLuD001k6U5t6no,1163
-hestia_earth_utils-0.16.1.dist-info/METADATA,sha256=ofiEuFahZ5b4fyOxvnJ0N40gck62LEBKWsZiSWaSzFU,1876
-hestia_earth_utils-0.16.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-hestia_earth_utils-0.16.1.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
-hestia_earth_utils-0.16.1.dist-info/RECORD,,
+hestia_earth_utils-0.16.3.data/scripts/hestia-format-upload,sha256=IhLAHHPJqRgUcht-M_EUEsRMbRbMfshig07o488zscM,703
+hestia_earth_utils-0.16.3.data/scripts/hestia-pivot-csv,sha256=0YBuGuyPO8rytod6iwWEKiQdSlr9JLuD001k6U5t6no,1163
+hestia_earth_utils-0.16.3.dist-info/METADATA,sha256=mP1MsmL8u1Ybatz47kHMAEzVi3hdSXMsrJXfCPfVglg,2030
+hestia_earth_utils-0.16.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+hestia_earth_utils-0.16.3.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
+hestia_earth_utils-0.16.3.dist-info/RECORD,,