hestia-earth-utils 0.16.3__tar.gz → 0.16.5__tar.gz

{hestia_earth_utils-0.16.3/hestia_earth_utils.egg-info → hestia_earth_utils-0.16.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.3
+Version: 0.16.5
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

{hestia_earth_utils-0.16.3 → hestia_earth_utils-0.16.5}/hestia_earth/utils/csv_utils.py

@@ -25,16 +25,6 @@ def _text_to_csv(csv_content: str):
     return csv.reader(io.StringIO(csv_content.strip()), delimiter=_DELIMITER, quotechar=_QUOTE_CHAR)
 
 
-def _csv_reader_converter(field_str_bytes):
-    field_str = field_str_bytes if isinstance(field_str_bytes, str) else field_str_bytes.decode('utf-8')
-    reader = _text_to_csv(field_str)
-
-    try:
-        return _replace_missing_values(next(reader)[0].strip())
-    except StopIteration:
-        return str(_MISSING)
-
-
 def _get_columns(csv_content: str):
     try:
         reader = _text_to_csv(csv_content)
@@ -44,26 +34,48 @@ def _get_columns(csv_content: str):
         return []
 
 
-def csv_str_to_recarray(csv_content: str) -> np.recarray:
+def _get_rows(csv_content: str):
+    string_io = io.StringIO(csv_content.strip())
+    try:
+        next(string_io)
+    except StopIteration:
+        return
+
+    return csv.reader(string_io, delimiter=_DELIMITER, quotechar=_QUOTE_CHAR)
+
+
+def _csv_str_to_recarray_chunks_numpy(csv_content: str, chunk_size: int = 5):
     names = _get_columns(csv_content)
     num_cols = len(names)
 
-    converters_dict = {
-        i: _csv_reader_converter
-        for i in range(num_cols)
-    }
-
-    # TODO: find the maximum column size instead of using 1000
     max_size = 1000
-    return np.loadtxt(
-        io.StringIO(csv_content.strip()),
-        delimiter=_DELIMITER,
-        quotechar=_QUOTE_CHAR,
-        skiprows=1,
-        converters=converters_dict,
-        dtype=[(name, f"U{max_size}") for name in names],
-        encoding=ENCODING
-    ).view(np.recarray)
+    dtype = [(name, f"U{max_size}") for name in names]
+
+    reader = _get_rows(csv_content)
+
+    # 4. Process the file in batches
+    chunk_rows = []
+    for row in reader:
+        if not row:
+            continue
+        if len(row) != num_cols:
+            continue
+
+        # replace missing values
+        processed_row = tuple(_replace_missing_values(field) for field in row)
+        chunk_rows.append(processed_row)
+
+        if len(chunk_rows) >= chunk_size:
+            yield np.array(chunk_rows, dtype=dtype).view(np.recarray)
+            chunk_rows = []
+
+    if chunk_rows:
+        yield np.array(chunk_rows, dtype=dtype).view(np.recarray)
+
+
+def csv_str_to_recarray(csv_content: str) -> np.recarray:
+    array_rows = list(_csv_str_to_recarray_chunks_numpy(csv_content))
+    return np.hstack(array_rows).view(np.recarray)
 
 
 def csv_file_to_recarray(filepath: str):

{hestia_earth_utils-0.16.3 → hestia_earth_utils-0.16.5}/hestia_earth/utils/lookup.py

@@ -2,7 +2,6 @@ from functools import reduce
 from typing import Union
 import requests
 import numpy
-import traceback
 
 from .storage import _load_from_storage
 from .request import request_url, web_url
@@ -92,8 +91,6 @@ def download_lookup(filename: str, keep_in_memory: bool = True, build_index: boo
     try:
         return _memory_wrapper(filename, load) if keep_in_memory else load()
     except Exception:
-        stack = traceback.format_exc()
-        print(stack)
         return None
 
 

hestia_earth_utils-0.16.5/hestia_earth/utils/version.py

@@ -0,0 +1 @@
+VERSION = '0.16.5'

{hestia_earth_utils-0.16.3 → hestia_earth_utils-0.16.5/hestia_earth_utils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.3
+Version: 0.16.5
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

{hestia_earth_utils-0.16.3 → hestia_earth_utils-0.16.5}/tests/test_lookup.py

@@ -154,3 +154,10 @@ def test_get_data_advanced():
     lookup = download_lookup('liveAnimal.csv')
     value = get_table_value(lookup, 'termid', 'sheepRam', column_name('ratioCPregnancyNetEnergyPregnancyIpcc2019'))
     assert value == ''
+
+
+def test_grouping_with_comma():
+    lookup = download_lookup('animalProduct.csv')
+    term_id = 'meatChickenReadyToCookWeight'
+    value = get_table_value(lookup, 'termid', term_id, column_name('animalProductGroupingFAO'))
+    assert value == 'Meat of chickens, fresh or chilled'

hestia_earth_utils-0.16.3/hestia_earth/utils/version.py

@@ -1 +0,0 @@
-VERSION = '0.16.3'