iker-python-common 1.0.15__tar.gz → 1.0.16__tar.gz

{iker_python_common-1.0.15 → iker_python_common-1.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: iker-python-common
-Version: 1.0.15
+Version: 1.0.16
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12

iker_python_common-1.0.16/resources/unittest/csv/data.csv

@@ -0,0 +1,7 @@
+dummy_str,dummy_bool,dummy_int,dummy_float,dummy_datetime,dummy_params
+foo,True,1,1.0,2020-01-01T00:00:00,
+bar,False,-1,-1.0,2020-01-01T00:00:00,key_1=value_1;key_2;!key_3
+baz,\N,100,inf,2020-01-01T00:00:00,\N
+,\N,-100,-inf,2020-01-01T00:00:00,
+\N,\N,0,nan,2020-01-01T00:00:00,\N
+\N,\N,\N,\N,\N,\N

iker_python_common-1.0.16/resources/unittest/csv/data.tsv

@@ -0,0 +1,7 @@
+dummy_str	dummy_bool	dummy_int	dummy_float	dummy_datetime	dummy_params
+foo	True	1	1.0	2020-01-01T00:00:00	
+bar	False	-1	-1.0	2020-01-01T00:00:00	key_1:value_1#key_2#-key_3
+baz	<null>	100	inf	2020-01-01T00:00:00	<null>
+	<null>	-100	-inf	2020-01-01T00:00:00	
+<null>	<null>	0	nan	2020-01-01T00:00:00	<null>
+<null>	<null>	<null>	<null>	<null>	<null>

iker_python_common-1.0.16/src/iker/common/utils/csv.py

@@ -0,0 +1,99 @@
+import os
+from collections.abc import Callable, Generator, Iterable, Mapping, Sequence
+from typing import Any
+
+__all__ = [
+    "CSVColumn",
+    "CSVView",
+    "column",
+    "view",
+]
+
+
+class CSVColumn(object):
+    def __init__(
+        self,
+        name: str,
+        loader: Callable[[str], Any] = str,
+        dumper: Callable[[Any], str] = str,
+        null_str: str = "",
+    ):
+        self.name = name
+        self.loader = loader
+        self.dumper = dumper
+        self.null_str = null_str
+
+
+class CSVView(object):
+    def __init__(self, schema: Sequence[CSVColumn], *, row_delim: str = "\n", col_delim: str = ","):
+        self.schema = schema
+        self.row_delim = row_delim
+        self.col_delim = col_delim
+
+    def load_lines(
+        self,
+        lines: Iterable[str],
+        has_header: bool = True,
+        ret_dict: bool = False,
+    ) -> Generator[list[Any] | dict[str, Any], None, None]:
+        rows_iter = iter(lines)
+        if has_header:
+            header_row = next(rows_iter)
+            header_cols = header_row.split(self.col_delim)
+            if len(self.schema) != len(header_cols):
+                raise ValueError("size of the schema is not identical to size of the columns")
+            for c, header_col in zip(self.schema, header_cols):
+                if c.name != header_col:
+                    raise ValueError("name of the schema is not equal to the name of the columns")
+        for row in rows_iter:
+            cols = row.split(self.col_delim)
+            if len(self.schema) != len(cols):
+                continue
+            if ret_dict:
+                yield {c.name: None if col == c.null_str else c.loader(col) for c, col in zip(self.schema, cols)}
+            else:
+                yield [None if col == c.null_str else c.loader(col) for c, col in zip(self.schema, cols)]
+
+    def dump_lines(
+        self,
+        data: Iterable[Sequence[Any] | Mapping[str, Any]],
+        has_header: bool = True,
+    ) -> Generator[str, None, None]:
+        if has_header:
+            yield self.col_delim.join(c.name for c in self.schema)
+        for cols in data:
+            if isinstance(cols, Sequence):
+                if len(self.schema) != len(cols):
+                    raise ValueError("size of the schema is not identical to size of the columns")
+                yield self.col_delim.join(c.null_str if col is None else c.dumper(col)
+                                          for c, col in zip(self.schema, cols))
+            if isinstance(cols, Mapping):
+                yield self.col_delim.join(c.null_str if cols.get(c.name) is None else c.dumper(cols.get(c.name))
+                                          for c in self.schema)
+
+    def load_file(
+        self,
+        file_path: os.PathLike | str,
+        has_header: bool = True,
+        ret_dict: bool = False,
+        **kwargs,
+    ) -> Generator[list[Any] | dict[str, Any], None, None]:
+        with open(file_path, mode="r", **kwargs) as fh:
+            lines = fh.read().split(self.row_delim)
+            yield from self.load_lines(lines, has_header, ret_dict)
+
+    def dump_file(
+        self,
+        data: Iterable[Sequence[Any] | Mapping[str, Any]],
+        file_path: os.PathLike | str,
+        has_header: bool = True,
+        **kwargs,
+    ) -> None:
+        with open(file_path, mode="w", **kwargs) as fh:
+            for line in self.dump_lines(data, has_header):
+                fh.write(line)
+                fh.write(self.row_delim)
+
+
+column = CSVColumn
+view = CSVView

{iker_python_common-1.0.15 → iker_python_common-1.0.16}/src/iker_python_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: iker-python-common
-Version: 1.0.15
+Version: 1.0.16
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12

{iker_python_common-1.0.15 → iker_python_common-1.0.16}/src/iker_python_common.egg-info/SOURCES.txt

@@ -7,6 +7,8 @@ pyproject.toml
 setup.py
 .github/workflows/pr.yml
 .github/workflows/push.yml
+resources/unittest/csv/data.csv
+resources/unittest/csv/data.tsv
 resources/unittest/shutils/dir.baz/file.bar.baz
 resources/unittest/shutils/dir.baz/file.foo.bar
 resources/unittest/shutils/dir.baz/file.foo.baz
@@ -21,6 +23,7 @@ src/iker/common/__init__.py
 src/iker/common/utils/__init__.py
 src/iker/common/utils/argutils.py
 src/iker/common/utils/config.py
+src/iker/common/utils/csv.py
 src/iker/common/utils/dbutils.py
 src/iker/common/utils/dockerutils.py
 src/iker/common/utils/dtutils.py
@@ -47,6 +50,7 @@ test/iker_tests/docker_fixtures.py
 test/iker_tests/iker_test.py
 test/iker_tests/common/utils/argutils_test.py
 test/iker_tests/common/utils/config_test.py
+test/iker_tests/common/utils/csv_test.py
 test/iker_tests/common/utils/dbutils_test.py
 test/iker_tests/common/utils/dockerutils_test.py
 test/iker_tests/common/utils/dtutils_test.py

iker_python_common-1.0.16/test/iker_tests/common/utils/csv_test.py

@@ -0,0 +1,241 @@
+import datetime
+import itertools
+import math
+import os.path
+import unittest
+
+import ddt
+
+from iker.common.utils import csv
+from iker.common.utils.dtutils import dt_format_iso, dt_parse_iso
+from iker.common.utils.jsonutils import json_compare
+from iker.common.utils.strutils import make_params_string, parse_params_string
+from iker.common.utils.strutils import parse_bool
+
+from iker_tests import resources_directory
+
+
+@ddt.ddt
+class CSVTest(unittest.TestCase):
+
+    def test_lines(self):
+        view = csv.view(
+            [
+                csv.column("dummy_str", loader=str, dumper=str, null_str=r"\N"),
+                csv.column("dummy_bool", loader=parse_bool, dumper=str, null_str=r"\N"),
+                csv.column("dummy_int", loader=int, dumper=str, null_str=r"\N"),
+                csv.column("dummy_float", loader=float, dumper=str, null_str=r"\N"),
+                csv.column("dummy_datetime",
+                           loader=lambda x: dt_parse_iso(x).timestamp(),
+                           dumper=lambda x: dt_format_iso(datetime.datetime.fromtimestamp(x, tz=datetime.timezone.utc)),
+                           null_str=r"\N"),
+                csv.column("dummy_params",
+                           loader=lambda x: parse_params_string(x, delim=";", kv_delim="=", neg_prefix="!"),
+                           dumper=lambda x: make_params_string(x, delim=";", kv_delim="=", neg_prefix="!"),
+                           null_str=r"\N"),
+            ],
+        )
+
+        lines = [
+            r"dummy_str,dummy_bool,dummy_int,dummy_float,dummy_datetime,dummy_params",
+            r"foo,True,1,1.0,2020-01-01T00:00:00,",
+            r"bar,False,-1,-1.0,2020-01-01T00:00:00,key_1=value_1;key_2;!key_3",
+            r"baz,\N,100,inf,2020-01-01T00:00:00,\N",
+            r",\N,-100,-inf,2020-01-01T00:00:00,",
+            r"\N,\N,0,nan,2020-01-01T00:00:00,\N",
+            r"\N,\N,\N,\N,\N,\N",
+        ]
+
+        lines_no_header = [
+            r"foo,True,1,1.0,2020-01-01T00:00:00,",
+            r"bar,False,-1,-1.0,2020-01-01T00:00:00,key_1=value_1;key_2;!key_3",
+            r"baz,\N,100,inf,2020-01-01T00:00:00,\N",
+            r",\N,-100,-inf,2020-01-01T00:00:00,",
+            r"\N,\N,0,nan,2020-01-01T00:00:00,\N",
+            r"\N,\N,\N,\N,\N,\N",
+        ]
+
+        time = dt_parse_iso("2020-01-01T00:00:00").timestamp()
+
+        list_data = [
+            ["foo", True, 1, 1.0, time, {}],
+            ["bar", False, -1, -1.0, time, {"key_1": "value_1", "key_2": True, "key_3": False}],
+            ["baz", None, 100, math.inf, time, None],
+            ["", None, -100, -math.inf, time, {}],
+            [None, None, 0, math.nan, time, None],
+            [None, None, None, None, None, None],
+        ]
+
+        dict_data = [
+            {
+                "dummy_str": "foo",
+                "dummy_bool": True,
+                "dummy_int": 1,
+                "dummy_float": 1.0,
+                "dummy_datetime": time,
+                "dummy_params": {},
+            },
+            {
+                "dummy_str": "bar",
+                "dummy_bool": False,
+                "dummy_int": -1,
+                "dummy_float": -1.0,
+                "dummy_datetime": time,
+                "dummy_params": {"key_1": "value_1", "key_2": True, "key_3": False},
+            },
+            {
+                "dummy_str": "baz",
+                "dummy_bool": None,
+                "dummy_int": 100,
+                "dummy_float": math.inf,
+                "dummy_datetime": time,
+                "dummy_params": None,
+            },
+            {
+                "dummy_str": "",
+                "dummy_bool": None,
+                "dummy_int": -100,
+                "dummy_float": -math.inf,
+                "dummy_datetime": time,
+                "dummy_params": {},
+            },
+            {
+                "dummy_str": None,
+                "dummy_bool": None,
+                "dummy_int": 0,
+                "dummy_float": math.nan,
+                "dummy_datetime": time,
+                "dummy_params": None,
+            },
+            {
+                "dummy_str": None,
+                "dummy_bool": None,
+                "dummy_int": None,
+                "dummy_float": None,
+                "dummy_datetime": None,
+                "dummy_params": None,
+            },
+        ]
+
+        for a, e in zip(view.load_lines(lines, has_header=True), list_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.load_lines(lines, has_header=True, ret_dict=True), dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(list_data, has_header=True), lines):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(dict_data, has_header=True), lines):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.load_lines(lines_no_header, has_header=False), list_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.load_lines(lines_no_header, has_header=False, ret_dict=True), dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(list_data, has_header=False), lines_no_header):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(dict_data, has_header=False), lines_no_header):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(view.load_lines(lines, has_header=True, ret_dict=True), has_header=True),
+                        lines):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.load_lines(view.dump_lines(list_data, has_header=True), has_header=True, ret_dict=True),
+                        dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(view.load_lines(lines_no_header, has_header=False, ret_dict=True),
+                                        has_header=False),
+                        lines_no_header):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.load_lines(view.dump_lines(list_data, has_header=False), has_header=False, ret_dict=True),
+                        dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(itertools.chain(view.load_lines(lines, has_header=True),
+                                        view.load_lines(lines, has_header=True, ret_dict=True)),
+                        list_data + dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(itertools.chain(view.dump_lines(list_data, has_header=True),
+                                        view.dump_lines(dict_data, has_header=True)),
+                        lines + lines):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(itertools.chain(view.load_lines(lines, has_header=True),
+                                        view.load_lines(lines, has_header=True, ret_dict=True),
+                                        view.load_lines(lines_no_header, has_header=False),
+                                        view.load_lines(lines_no_header, has_header=False, ret_dict=True)),
+                        list_data + dict_data + list_data + dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(itertools.chain(view.dump_lines(list_data, has_header=True),
+                                        view.dump_lines(dict_data, has_header=True),
+                                        view.dump_lines(list_data, has_header=False),
+                                        view.dump_lines(dict_data, has_header=False)),
+                        lines + lines + lines_no_header + lines_no_header):
+            self.assertTrue(json_compare(a, e))
+
+    def test_file(self):
+        view_csv = csv.view(
+            [
+                csv.column("dummy_str", loader=str, dumper=str, null_str=r"\N"),
+                csv.column("dummy_bool", loader=parse_bool, dumper=str, null_str=r"\N"),
+                csv.column("dummy_int", loader=int, dumper=str, null_str=r"\N"),
+                csv.column("dummy_float", loader=float, dumper=str, null_str=r"\N"),
+                csv.column("dummy_datetime",
+                           loader=lambda x: dt_parse_iso(x).timestamp(),
+                           dumper=lambda x: dt_format_iso(datetime.datetime.fromtimestamp(x, tz=datetime.timezone.utc)),
+                           null_str=r"\N"),
+                csv.column("dummy_params",
+                           loader=lambda x: parse_params_string(x, delim=";", kv_delim="=", neg_prefix="!"),
+                           dumper=lambda x: make_params_string(x, delim=";", kv_delim="=", neg_prefix="!"),
+                           null_str=r"\N"),
+            ],
+        )
+
+        view_tsv = csv.view(
+            [
+                csv.column("dummy_str", loader=str, dumper=str, null_str=r"<null>"),
+                csv.column("dummy_bool", loader=parse_bool, dumper=str, null_str=r"<null>"),
+                csv.column("dummy_int", loader=int, dumper=str, null_str=r"<null>"),
+                csv.column("dummy_float", loader=float, dumper=str, null_str=r"<null>"),
+                csv.column("dummy_datetime",
+                           loader=lambda x: dt_parse_iso(x).timestamp(),
+                           dumper=lambda x: dt_format_iso(datetime.datetime.fromtimestamp(x, tz=datetime.timezone.utc)),
+                           null_str=r"<null>"),
+                csv.column("dummy_params",
+                           loader=lambda x: parse_params_string(x, delim="#", kv_delim=":", neg_prefix="-"),
+                           dumper=lambda x: make_params_string(x, delim="#", kv_delim=":", neg_prefix="-"),
+                           null_str=r"<null>"),
+            ],
+            col_delim="\t",
+        )
+
+        time = dt_parse_iso("2020-01-01T00:00:00").timestamp()
+
+        data = [
+            ["foo", True, 1, 1.0, time, {}],
+            ["bar", False, -1, -1.0, time, {"key_1": "value_1", "key_2": True, "key_3": False}],
+            ["baz", None, 100, math.inf, time, None],
+            ["", None, -100, -math.inf, time, {}],
+            [None, None, 0, math.nan, time, None],
+            [None, None, None, None, None, None],
+        ]
+
+        for c, t, d in zip(view_csv.load_file(os.path.join(resources_directory, "unittest/csv/data.csv"),
+                                              has_header=True,
+                                              encoding="utf-8"),
+                           view_tsv.load_file(os.path.join(resources_directory, "unittest/csv/data.tsv"),
+                                              has_header=True,
+                                              encoding="utf-8"),
+                           data):
+            self.assertTrue(json_compare(c, d))
+            self.assertTrue(json_compare(t, d))
+            self.assertTrue(json_compare(c, t))