hestia-earth-utils 0.16.3__py3-none-any.whl → 0.16.5__py3-none-any.whl

hestia_earth/utils/csv_utils.py

@@ -25,16 +25,6 @@ def _text_to_csv(csv_content: str):
     return csv.reader(io.StringIO(csv_content.strip()), delimiter=_DELIMITER, quotechar=_QUOTE_CHAR)
 
 
-def _csv_reader_converter(field_str_bytes):
-    field_str = field_str_bytes if isinstance(field_str_bytes, str) else field_str_bytes.decode('utf-8')
-    reader = _text_to_csv(field_str)
-
-    try:
-        return _replace_missing_values(next(reader)[0].strip())
-    except StopIteration:
-        return str(_MISSING)
-
-
 def _get_columns(csv_content: str):
     try:
         reader = _text_to_csv(csv_content)
@@ -44,26 +34,48 @@ def _get_columns(csv_content: str):
         return []
 
 
-def csv_str_to_recarray(csv_content: str) -> np.recarray:
+def _get_rows(csv_content: str):
+    string_io = io.StringIO(csv_content.strip())
+    try:
+        next(string_io)
+    except StopIteration:
+        return
+
+    return csv.reader(string_io, delimiter=_DELIMITER, quotechar=_QUOTE_CHAR)
+
+
+def _csv_str_to_recarray_chunks_numpy(csv_content: str, chunk_size: int = 5):
     names = _get_columns(csv_content)
     num_cols = len(names)
 
-    converters_dict = {
-        i: _csv_reader_converter
-        for i in range(num_cols)
-    }
-
-    # TODO: find the maximum column size instead of using 1000
     max_size = 1000
-    return np.loadtxt(
-        io.StringIO(csv_content.strip()),
-        delimiter=_DELIMITER,
-        quotechar=_QUOTE_CHAR,
-        skiprows=1,
-        converters=converters_dict,
-        dtype=[(name, f"U{max_size}") for name in names],
-        encoding=ENCODING
-    ).view(np.recarray)
+    dtype = [(name, f"U{max_size}") for name in names]
+
+    reader = _get_rows(csv_content)
+
+    # 4. Process the file in batches
+    chunk_rows = []
+    for row in reader:
+        if not row:
+            continue
+        if len(row) != num_cols:
+            continue
+
+        # replace missing values
+        processed_row = tuple(_replace_missing_values(field) for field in row)
+        chunk_rows.append(processed_row)
+
+        if len(chunk_rows) >= chunk_size:
+            yield np.array(chunk_rows, dtype=dtype).view(np.recarray)
+            chunk_rows = []
+
+    if chunk_rows:
+        yield np.array(chunk_rows, dtype=dtype).view(np.recarray)
+
+
+def csv_str_to_recarray(csv_content: str) -> np.recarray:
+    array_rows = list(_csv_str_to_recarray_chunks_numpy(csv_content))
+    return np.hstack(array_rows).view(np.recarray)
 
 
 def csv_file_to_recarray(filepath: str):

hestia_earth/utils/lookup.py

@@ -2,7 +2,6 @@ from functools import reduce
 from typing import Union
 import requests
 import numpy
-import traceback
 
 from .storage import _load_from_storage
 from .request import request_url, web_url
@@ -92,8 +91,6 @@ def download_lookup(filename: str, keep_in_memory: bool = True, build_index: boo
     try:
         return _memory_wrapper(filename, load) if keep_in_memory else load()
     except Exception:
-        stack = traceback.format_exc()
-        print(stack)
         return None
 
 

hestia_earth/utils/version.py

@@ -1 +1 @@
-VERSION = '0.16.3'
+VERSION = '0.16.5'

{hestia_earth_utils-0.16.3.dist-info → hestia_earth_utils-0.16.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.3
+Version: 0.16.5
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

{hestia_earth_utils-0.16.3.dist-info → hestia_earth_utils-0.16.5.dist-info}/RECORD

@@ -2,12 +2,12 @@ hestia_earth/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 hestia_earth/utils/api.py,sha256=y0gw5pCCHNnFIhM62Hok_5eDtH3QDAZdkye_1mANMNs,9654
 hestia_earth/utils/blank_node.py,sha256=1wc9zUkOvFhJS-YmuKexfIdYxfsp5KyJczLmHlW559Q,7375
 hestia_earth/utils/calculation_status.py,sha256=X7lbgVMD9luH1gj9lEcxd3_P2-u7e8ZPGCvX1czPZUo,2238
-hestia_earth/utils/csv_utils.py,sha256=nb_ihJaTj3K5hO7cxXO1xjTLVGVX1P13m9SgquO5-XY,1990
+hestia_earth/utils/csv_utils.py,sha256=BK-tci1sALmsxamSR1Y7f9O6ajTTdhggLC2pBEWhYME,2310
 hestia_earth/utils/cycle.py,sha256=rFLRL9X4KQ1UrE6fEPA_gV8KmwzrZpR3Ce56zg41lRk,1326
 hestia_earth/utils/date.py,sha256=SPQ69uxHiv1o3BqIkBKkM5XX_CmS20CB7g6u2rhsdh8,1807
 hestia_earth/utils/descriptive_stats.py,sha256=EMVwFvg2OnZgKRAfireAoWY2EbrSvqR0V0bK9B53p28,1583
 hestia_earth/utils/emission.py,sha256=BhBitooLTxZSh82S982v2QfPxxTF1kmGClG_uHyWdz4,1981
-hestia_earth/utils/lookup.py,sha256=iZnNQn13_WWFWc3LyL4qPTiL2IA48Db3y4eat5kmcuw,9599
+hestia_earth/utils/lookup.py,sha256=XKmxFEH9o1Rhi4oTLteQnMAwNXiObjSX7pMfrUw8q1I,9522
 hestia_earth/utils/lookup_utils.py,sha256=_k3RZ1pK-gw7jq8wn9HrPWfDl4FlEWRb8bXmgaARu0w,6716
 hestia_earth/utils/model.py,sha256=uUcrF07XmBzqLni8VSaP0HoebJnQ57kk0EOmhwYMbfI,4637
 hestia_earth/utils/pipeline.py,sha256=O-6DPtK0U1lJ51LFGa1gM6pjkBJUfxOjNjY8LxQPXV0,9588
@@ -16,7 +16,7 @@ hestia_earth/utils/stats.py,sha256=4t3op10xDJbGxWJEY1Jtyl302PYWyMFwLpsSkMlzQn8,3
 hestia_earth/utils/table.py,sha256=MOJDo5fQPRDogAty_UXbO9-EXFwz97m0f7--mOM17lQ,2363
 hestia_earth/utils/term.py,sha256=6LiUSc6KX3IOkfWF6fYkQ2tENCO8ENljcdDypxU6WtA,1060
 hestia_earth/utils/tools.py,sha256=9GaUJwxL-CTzEOGnRFkUQDVFelPevQSxXrf25vssCVo,4990
-hestia_earth/utils/version.py,sha256=2f5woLdAmKvk1TRWtU3KMO3z6ac2wJDnGXIOq8M0TMk,19
+hestia_earth/utils/version.py,sha256=RKa3Cna34LUgL3Ye-ubIdZ9B2mS9iFURrl2snrPa3uY,19
 hestia_earth/utils/pivot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 hestia_earth/utils/pivot/_shared.py,sha256=JnyIOzpans45DE2hSa9-4yvNhq8t08lx1IAWGJi6WPQ,1397
 hestia_earth/utils/pivot/pivot_csv.py,sha256=7f6kMqeb1b3RKANLGeDgVu8G5WC-vXIijHnsJhO-CjI,12022
@@ -26,9 +26,9 @@ hestia_earth/utils/storage/_azure_client.py,sha256=sevCZni04eknMql2DgUsWG23f7u0K
 hestia_earth/utils/storage/_local_client.py,sha256=IbzziUKY0QS3ybHFfgEpELqvafa7hQnZ-DdGdjQuypE,515
 hestia_earth/utils/storage/_s3_client.py,sha256=B2yTsf-VfHcRLCKTMes4S_nCXxrZad9umyZx3b5Pu_c,3181
 hestia_earth/utils/storage/_sns_client.py,sha256=LowUatj78Egu6_Id6Rr7hZjfZx1WguS3lozB3yAwSps,347
-hestia_earth_utils-0.16.3.data/scripts/hestia-format-upload,sha256=IhLAHHPJqRgUcht-M_EUEsRMbRbMfshig07o488zscM,703
-hestia_earth_utils-0.16.3.data/scripts/hestia-pivot-csv,sha256=0YBuGuyPO8rytod6iwWEKiQdSlr9JLuD001k6U5t6no,1163
-hestia_earth_utils-0.16.3.dist-info/METADATA,sha256=mP1MsmL8u1Ybatz47kHMAEzVi3hdSXMsrJXfCPfVglg,2030
-hestia_earth_utils-0.16.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-hestia_earth_utils-0.16.3.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
-hestia_earth_utils-0.16.3.dist-info/RECORD,,
+hestia_earth_utils-0.16.5.data/scripts/hestia-format-upload,sha256=IhLAHHPJqRgUcht-M_EUEsRMbRbMfshig07o488zscM,703
+hestia_earth_utils-0.16.5.data/scripts/hestia-pivot-csv,sha256=0YBuGuyPO8rytod6iwWEKiQdSlr9JLuD001k6U5t6no,1163
+hestia_earth_utils-0.16.5.dist-info/METADATA,sha256=PAHew6LMon2UUdlJlGUSE7plh0VzfcBXXsV3SPzLL5A,2030
+hestia_earth_utils-0.16.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+hestia_earth_utils-0.16.5.dist-info/top_level.txt,sha256=q0QxKEYx9uLpAD5ZtC7Ypq29smEPfOzEAn7Xv8XHGOQ,13
+hestia_earth_utils-0.16.5.dist-info/RECORD,,