hestia-earth-utils 0.16.4__tar.gz → 0.16.6__tar.gz

{hestia_earth_utils-0.16.4/hestia_earth_utils.egg-info → hestia_earth_utils-0.16.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.4
+Version: 0.16.6
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

{hestia_earth_utils-0.16.4 → hestia_earth_utils-0.16.6}/hestia_earth/utils/csv_utils.py

@@ -25,14 +25,6 @@ def _text_to_csv(csv_content: str):
     return csv.reader(io.StringIO(csv_content.strip()), delimiter=_DELIMITER, quotechar=_QUOTE_CHAR)
 
 
-def _csv_reader_converter(field_str_bytes):
-    try:
-        field_str = field_str_bytes if isinstance(field_str_bytes, str) else field_str_bytes.decode('utf-8')
-        return _replace_missing_values(field_str)
-    except Exception:
-        return str(_MISSING)
-
-
 def _get_columns(csv_content: str):
     try:
         reader = _text_to_csv(csv_content)
@@ -42,26 +34,48 @@ def _get_columns(csv_content: str):
         return []
 
 
-def csv_str_to_recarray(csv_content: str) -> np.recarray:
+def _get_rows(csv_content: str):
+    string_io = io.StringIO(csv_content.strip())
+    try:
+        next(string_io)
+    except StopIteration:
+        return
+
+    return csv.reader(string_io, delimiter=_DELIMITER, quotechar=_QUOTE_CHAR)
+
+
+def _csv_str_to_recarray_chunks_numpy(csv_content: str, chunk_size: int = 5):
     names = _get_columns(csv_content)
     num_cols = len(names)
 
-    converters_dict = {
-        i: _csv_reader_converter
-        for i in range(num_cols)
-    }
-
-    # TODO: find the maximum column size instead of using 1000
     max_size = 1000
-    return np.loadtxt(
-        io.StringIO(csv_content.strip()),
-        delimiter=_DELIMITER,
-        quotechar=_QUOTE_CHAR,
-        skiprows=1,
-        converters=converters_dict,
-        dtype=[(name, f"U{max_size}") for name in names],
-        encoding=ENCODING
-    ).view(np.recarray)
+    dtype = [(name, f"U{max_size}") for name in names]
+
+    reader = _get_rows(csv_content)
+
+    # 4. Process the file in batches
+    chunk_rows = []
+    for row in reader:
+        if not row:
+            continue
+        if len(row) != num_cols:
+            continue
+
+        # replace missing values
+        processed_row = tuple(_replace_missing_values(field) for field in row)
+        chunk_rows.append(processed_row)
+
+        if len(chunk_rows) >= chunk_size:
+            yield np.array(chunk_rows, dtype=dtype).view(np.recarray)
+            chunk_rows = []
+
+    if chunk_rows:
+        yield np.array(chunk_rows, dtype=dtype).view(np.recarray)
+
+
+def csv_str_to_recarray(csv_content: str) -> np.recarray:
+    array_rows = list(_csv_str_to_recarray_chunks_numpy(csv_content))
+    return np.hstack(array_rows).view(np.recarray)
 
 
 def csv_file_to_recarray(filepath: str):

{hestia_earth_utils-0.16.4 → hestia_earth_utils-0.16.6}/hestia_earth/utils/lookup.py

@@ -43,7 +43,8 @@ def _download_lookup_data(filename: str):
 
     def fallback():
         url = request_url(f"{web_url()}/{filepath}")
-        return requests.get(url).content.decode('utf-8')
+        data = requests.get(url).content.decode('utf-8')
+        return data if data and '<html' not in data else None
 
     try:
         data = _load_from_storage(filepath, glossary=True)

hestia_earth_utils-0.16.6/hestia_earth/utils/version.py

@@ -0,0 +1 @@
+VERSION = '0.16.6'

{hestia_earth_utils-0.16.4 → hestia_earth_utils-0.16.6/hestia_earth_utils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hestia_earth_utils
-Version: 0.16.4
+Version: 0.16.6
 Summary: HESTIA's utils library
 Home-page: https://gitlab.com/hestia-earth/hestia-utils
 Author: HESTIA Team

hestia_earth_utils-0.16.4/hestia_earth/utils/version.py

@@ -1 +0,0 @@
-VERSION = '0.16.4'