iker-python-common 1.0.15__tar.gz → 1.0.17__tar.gz

{iker_python_common-1.0.15 → iker_python_common-1.0.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: iker-python-common
-Version: 1.0.15
+Version: 1.0.17
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12

iker_python_common-1.0.17/resources/unittest/csv/data.csv

@@ -0,0 +1,7 @@
+dummy_str,dummy_bool,dummy_int,dummy_float,dummy_datetime,dummy_params
+foo,True,1,1.0,2020-01-01T00:00:00,
+bar,False,-1,-1.0,2020-01-01T00:00:00,key_1=value_1;key_2;!key_3
+baz,\N,100,inf,2020-01-01T00:00:00,\N
+,\N,-100,-inf,2020-01-01T00:00:00,
+\N,\N,0,nan,2020-01-01T00:00:00,\N
+\N,\N,\N,\N,\N,\N

iker_python_common-1.0.17/resources/unittest/csv/data.tsv

@@ -0,0 +1,7 @@
+dummy_str	dummy_bool	dummy_int	dummy_float	dummy_datetime	dummy_params
+foo	True	1	1.0	2020-01-01T00:00:00	
+bar	False	-1	-1.0	2020-01-01T00:00:00	key_1:value_1#key_2#-key_3
+baz	<null>	100	inf	2020-01-01T00:00:00	<null>
+	<null>	-100	-inf	2020-01-01T00:00:00	
+<null>	<null>	0	nan	2020-01-01T00:00:00	<null>
+<null>	<null>	<null>	<null>	<null>	<null>

iker_python_common-1.0.17/src/iker/common/utils/csv.py

@@ -0,0 +1,99 @@
+import os
+from collections.abc import Callable, Generator, Iterable, Mapping, Sequence
+from typing import Any
+
+__all__ = [
+    "CSVColumn",
+    "CSVView",
+    "column",
+    "view",
+]
+
+
+class CSVColumn(object):
+    def __init__(
+        self,
+        name: str,
+        loader: Callable[[str], Any] = str,
+        dumper: Callable[[Any], str] = str,
+        null_str: str = "",
+    ):
+        self.name = name
+        self.loader = loader
+        self.dumper = dumper
+        self.null_str = null_str
+
+
+class CSVView(object):
+    def __init__(self, schema: Sequence[CSVColumn], *, row_delim: str = "\n", col_delim: str = ","):
+        self.schema = schema
+        self.row_delim = row_delim
+        self.col_delim = col_delim
+
+    def load_lines(
+        self,
+        lines: Iterable[str],
+        has_header: bool = True,
+        ret_dict: bool = False,
+    ) -> Generator[list[Any] | dict[str, Any], None, None]:
+        rows_iter = iter(lines)
+        if has_header:
+            header_row = next(rows_iter)
+            header_cols = header_row.split(self.col_delim)
+            if len(self.schema) != len(header_cols):
+                raise ValueError("size of the schema is not identical to size of the columns")
+            for c, header_col in zip(self.schema, header_cols):
+                if c.name != header_col:
+                    raise ValueError("name of the schema is not equal to the name of the columns")
+        for row in rows_iter:
+            cols = row.split(self.col_delim)
+            if len(self.schema) != len(cols):
+                continue
+            if ret_dict:
+                yield {c.name: None if col == c.null_str else c.loader(col) for c, col in zip(self.schema, cols)}
+            else:
+                yield [None if col == c.null_str else c.loader(col) for c, col in zip(self.schema, cols)]
+
+    def dump_lines(
+        self,
+        data: Iterable[Sequence[Any] | Mapping[str, Any]],
+        has_header: bool = True,
+    ) -> Generator[str, None, None]:
+        if has_header:
+            yield self.col_delim.join(c.name for c in self.schema)
+        for cols in data:
+            if isinstance(cols, Sequence):
+                if len(self.schema) != len(cols):
+                    raise ValueError("size of the schema is not identical to size of the columns")
+                yield self.col_delim.join(c.null_str if col is None else c.dumper(col)
+                                          for c, col in zip(self.schema, cols))
+            if isinstance(cols, Mapping):
+                yield self.col_delim.join(c.null_str if cols.get(c.name) is None else c.dumper(cols.get(c.name))
+                                          for c in self.schema)
+
+    def load_file(
+        self,
+        file_path: os.PathLike | str,
+        has_header: bool = True,
+        ret_dict: bool = False,
+        **kwargs,
+    ) -> Generator[list[Any] | dict[str, Any], None, None]:
+        with open(file_path, mode="r", **kwargs) as fh:
+            lines = fh.read().split(self.row_delim)
+            yield from self.load_lines(lines, has_header, ret_dict)
+
+    def dump_file(
+        self,
+        data: Iterable[Sequence[Any] | Mapping[str, Any]],
+        file_path: os.PathLike | str,
+        has_header: bool = True,
+        **kwargs,
+    ) -> None:
+        with open(file_path, mode="w", **kwargs) as fh:
+            for line in self.dump_lines(data, has_header):
+                fh.write(line)
+                fh.write(self.row_delim)
+
+
+column = CSVColumn
+view = CSVView

{iker_python_common-1.0.15 → iker_python_common-1.0.17}/src/iker/common/utils/randutils.py

@@ -3,9 +3,12 @@ import math
 import random
 import string
 import sys
+from collections.abc import Callable, Sequence
+from typing import TypeVar
 
 from iker.common.utils.dtutils import dt_utc_max, dt_utc_min
 from iker.common.utils.funcutils import memorized, singleton
+from iker.common.utils.sequtils import head_or_none
 
 __all__ = [
     "max_int",
@@ -14,6 +17,8 @@ __all__ = [
     "randomizer",
 ]
 
+T = TypeVar("T")
+
 
 @singleton
 def max_int() -> int:
@@ -142,6 +147,21 @@ class Randomizer(object):
 
         return generate_json_object(0)
 
+    def sample(self, population: Sequence[T], count_func: Callable[[T], int] = None, k: int = None) -> list[T] | T:
+        counts = list(map(count_func, population)) if callable(count_func) else None
+        result = self.random.sample(population, counts=counts, k=k or 1)
+        return result if k is not None else head_or_none(result)
+
+    def choose(self, population: Sequence[T], weight_func: Callable[[T], float] = None, k: int = None) -> list[T] | T:
+        weights = list(map(weight_func, population)) if callable(weight_func) else None
+        result = self.random.choices(population, weights=weights, k=k or 1)
+        return result if k is not None else head_or_none(result)
+
+    def shuffle(self, data: Sequence[T]) -> list[T]:
+        clone = list(item for item in data)
+        self.random.shuffle(clone)
+        return clone
+
 
 @memorized
 def randomizer(seed: int = 0) -> Randomizer:

{iker_python_common-1.0.15 → iker_python_common-1.0.17}/src/iker_python_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: iker-python-common
-Version: 1.0.15
+Version: 1.0.17
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12

{iker_python_common-1.0.15 → iker_python_common-1.0.17}/src/iker_python_common.egg-info/SOURCES.txt

@@ -7,6 +7,8 @@ pyproject.toml
 setup.py
 .github/workflows/pr.yml
 .github/workflows/push.yml
+resources/unittest/csv/data.csv
+resources/unittest/csv/data.tsv
 resources/unittest/shutils/dir.baz/file.bar.baz
 resources/unittest/shutils/dir.baz/file.foo.bar
 resources/unittest/shutils/dir.baz/file.foo.baz
@@ -21,6 +23,7 @@ src/iker/common/__init__.py
 src/iker/common/utils/__init__.py
 src/iker/common/utils/argutils.py
 src/iker/common/utils/config.py
+src/iker/common/utils/csv.py
 src/iker/common/utils/dbutils.py
 src/iker/common/utils/dockerutils.py
 src/iker/common/utils/dtutils.py
@@ -47,6 +50,7 @@ test/iker_tests/docker_fixtures.py
 test/iker_tests/iker_test.py
 test/iker_tests/common/utils/argutils_test.py
 test/iker_tests/common/utils/config_test.py
+test/iker_tests/common/utils/csv_test.py
 test/iker_tests/common/utils/dbutils_test.py
 test/iker_tests/common/utils/dockerutils_test.py
 test/iker_tests/common/utils/dtutils_test.py

iker_python_common-1.0.17/test/iker_tests/common/utils/csv_test.py

@@ -0,0 +1,241 @@
+import datetime
+import itertools
+import math
+import os.path
+import unittest
+
+import ddt
+
+from iker.common.utils import csv
+from iker.common.utils.dtutils import dt_format_iso, dt_parse_iso
+from iker.common.utils.jsonutils import json_compare
+from iker.common.utils.strutils import make_params_string, parse_params_string
+from iker.common.utils.strutils import parse_bool
+
+from iker_tests import resources_directory
+
+
+@ddt.ddt
+class CSVTest(unittest.TestCase):
+
+    def test_lines(self):
+        view = csv.view(
+            [
+                csv.column("dummy_str", loader=str, dumper=str, null_str=r"\N"),
+                csv.column("dummy_bool", loader=parse_bool, dumper=str, null_str=r"\N"),
+                csv.column("dummy_int", loader=int, dumper=str, null_str=r"\N"),
+                csv.column("dummy_float", loader=float, dumper=str, null_str=r"\N"),
+                csv.column("dummy_datetime",
+                           loader=lambda x: dt_parse_iso(x).timestamp(),
+                           dumper=lambda x: dt_format_iso(datetime.datetime.fromtimestamp(x, tz=datetime.timezone.utc)),
+                           null_str=r"\N"),
+                csv.column("dummy_params",
+                           loader=lambda x: parse_params_string(x, delim=";", kv_delim="=", neg_prefix="!"),
+                           dumper=lambda x: make_params_string(x, delim=";", kv_delim="=", neg_prefix="!"),
+                           null_str=r"\N"),
+            ],
+        )
+
+        lines = [
+            r"dummy_str,dummy_bool,dummy_int,dummy_float,dummy_datetime,dummy_params",
+            r"foo,True,1,1.0,2020-01-01T00:00:00,",
+            r"bar,False,-1,-1.0,2020-01-01T00:00:00,key_1=value_1;key_2;!key_3",
+            r"baz,\N,100,inf,2020-01-01T00:00:00,\N",
+            r",\N,-100,-inf,2020-01-01T00:00:00,",
+            r"\N,\N,0,nan,2020-01-01T00:00:00,\N",
+            r"\N,\N,\N,\N,\N,\N",
+        ]
+
+        lines_no_header = [
+            r"foo,True,1,1.0,2020-01-01T00:00:00,",
+            r"bar,False,-1,-1.0,2020-01-01T00:00:00,key_1=value_1;key_2;!key_3",
+            r"baz,\N,100,inf,2020-01-01T00:00:00,\N",
+            r",\N,-100,-inf,2020-01-01T00:00:00,",
+            r"\N,\N,0,nan,2020-01-01T00:00:00,\N",
+            r"\N,\N,\N,\N,\N,\N",
+        ]
+
+        time = dt_parse_iso("2020-01-01T00:00:00").timestamp()
+
+        list_data = [
+            ["foo", True, 1, 1.0, time, {}],
+            ["bar", False, -1, -1.0, time, {"key_1": "value_1", "key_2": True, "key_3": False}],
+            ["baz", None, 100, math.inf, time, None],
+            ["", None, -100, -math.inf, time, {}],
+            [None, None, 0, math.nan, time, None],
+            [None, None, None, None, None, None],
+        ]
+
+        dict_data = [
+            {
+                "dummy_str": "foo",
+                "dummy_bool": True,
+                "dummy_int": 1,
+                "dummy_float": 1.0,
+                "dummy_datetime": time,
+                "dummy_params": {},
+            },
+            {
+                "dummy_str": "bar",
+                "dummy_bool": False,
+                "dummy_int": -1,
+                "dummy_float": -1.0,
+                "dummy_datetime": time,
+                "dummy_params": {"key_1": "value_1", "key_2": True, "key_3": False},
+            },
+            {
+                "dummy_str": "baz",
+                "dummy_bool": None,
+                "dummy_int": 100,
+                "dummy_float": math.inf,
+                "dummy_datetime": time,
+                "dummy_params": None,
+            },
+            {
+                "dummy_str": "",
+                "dummy_bool": None,
+                "dummy_int": -100,
+                "dummy_float": -math.inf,
+                "dummy_datetime": time,
+                "dummy_params": {},
+            },
+            {
+                "dummy_str": None,
+                "dummy_bool": None,
+                "dummy_int": 0,
+                "dummy_float": math.nan,
+                "dummy_datetime": time,
+                "dummy_params": None,
+            },
+            {
+                "dummy_str": None,
+                "dummy_bool": None,
+                "dummy_int": None,
+                "dummy_float": None,
+                "dummy_datetime": None,
+                "dummy_params": None,
+            },
+        ]
+
+        for a, e in zip(view.load_lines(lines, has_header=True), list_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.load_lines(lines, has_header=True, ret_dict=True), dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(list_data, has_header=True), lines):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(dict_data, has_header=True), lines):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.load_lines(lines_no_header, has_header=False), list_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.load_lines(lines_no_header, has_header=False, ret_dict=True), dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(list_data, has_header=False), lines_no_header):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(dict_data, has_header=False), lines_no_header):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(view.load_lines(lines, has_header=True, ret_dict=True), has_header=True),
+                        lines):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.load_lines(view.dump_lines(list_data, has_header=True), has_header=True, ret_dict=True),
+                        dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.dump_lines(view.load_lines(lines_no_header, has_header=False, ret_dict=True),
+                                        has_header=False),
+                        lines_no_header):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(view.load_lines(view.dump_lines(list_data, has_header=False), has_header=False, ret_dict=True),
+                        dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(itertools.chain(view.load_lines(lines, has_header=True),
+                                        view.load_lines(lines, has_header=True, ret_dict=True)),
+                        list_data + dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(itertools.chain(view.dump_lines(list_data, has_header=True),
+                                        view.dump_lines(dict_data, has_header=True)),
+                        lines + lines):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(itertools.chain(view.load_lines(lines, has_header=True),
+                                        view.load_lines(lines, has_header=True, ret_dict=True),
+                                        view.load_lines(lines_no_header, has_header=False),
+                                        view.load_lines(lines_no_header, has_header=False, ret_dict=True)),
+                        list_data + dict_data + list_data + dict_data):
+            self.assertTrue(json_compare(a, e))
+
+        for a, e in zip(itertools.chain(view.dump_lines(list_data, has_header=True),
+                                        view.dump_lines(dict_data, has_header=True),
+                                        view.dump_lines(list_data, has_header=False),
+                                        view.dump_lines(dict_data, has_header=False)),
+                        lines + lines + lines_no_header + lines_no_header):
+            self.assertTrue(json_compare(a, e))
+
+    def test_file(self):
+        view_csv = csv.view(
+            [
+                csv.column("dummy_str", loader=str, dumper=str, null_str=r"\N"),
+                csv.column("dummy_bool", loader=parse_bool, dumper=str, null_str=r"\N"),
+                csv.column("dummy_int", loader=int, dumper=str, null_str=r"\N"),
+                csv.column("dummy_float", loader=float, dumper=str, null_str=r"\N"),
+                csv.column("dummy_datetime",
+                           loader=lambda x: dt_parse_iso(x).timestamp(),
+                           dumper=lambda x: dt_format_iso(datetime.datetime.fromtimestamp(x, tz=datetime.timezone.utc)),
+                           null_str=r"\N"),
+                csv.column("dummy_params",
+                           loader=lambda x: parse_params_string(x, delim=";", kv_delim="=", neg_prefix="!"),
+                           dumper=lambda x: make_params_string(x, delim=";", kv_delim="=", neg_prefix="!"),
+                           null_str=r"\N"),
+            ],
+        )
+
+        view_tsv = csv.view(
+            [
+                csv.column("dummy_str", loader=str, dumper=str, null_str=r"<null>"),
+                csv.column("dummy_bool", loader=parse_bool, dumper=str, null_str=r"<null>"),
+                csv.column("dummy_int", loader=int, dumper=str, null_str=r"<null>"),
+                csv.column("dummy_float", loader=float, dumper=str, null_str=r"<null>"),
+                csv.column("dummy_datetime",
+                           loader=lambda x: dt_parse_iso(x).timestamp(),
+                           dumper=lambda x: dt_format_iso(datetime.datetime.fromtimestamp(x, tz=datetime.timezone.utc)),
+                           null_str=r"<null>"),
+                csv.column("dummy_params",
+                           loader=lambda x: parse_params_string(x, delim="#", kv_delim=":", neg_prefix="-"),
+                           dumper=lambda x: make_params_string(x, delim="#", kv_delim=":", neg_prefix="-"),
+                           null_str=r"<null>"),
+            ],
+            col_delim="\t",
+        )
+
+        time = dt_parse_iso("2020-01-01T00:00:00").timestamp()
+
+        data = [
+            ["foo", True, 1, 1.0, time, {}],
+            ["bar", False, -1, -1.0, time, {"key_1": "value_1", "key_2": True, "key_3": False}],
+            ["baz", None, 100, math.inf, time, None],
+            ["", None, -100, -math.inf, time, {}],
+            [None, None, 0, math.nan, time, None],
+            [None, None, None, None, None, None],
+        ]
+
+        for c, t, d in zip(view_csv.load_file(os.path.join(resources_directory, "unittest/csv/data.csv"),
+                                              has_header=True,
+                                              encoding="utf-8"),
+                           view_tsv.load_file(os.path.join(resources_directory, "unittest/csv/data.tsv"),
+                                              has_header=True,
+                                              encoding="utf-8"),
+                           data):
+            self.assertTrue(json_compare(c, d))
+            self.assertTrue(json_compare(t, d))
+            self.assertTrue(json_compare(c, t))

{iker_python_common-1.0.15 → iker_python_common-1.0.17}/test/iker_tests/common/utils/randutils_test.py

@@ -1,3 +1,4 @@
+import itertools
 import unittest
 
 import ddt
@@ -130,3 +131,39 @@ class RandUtilsTest(unittest.TestCase):
             result = randomizer().random_time(begin, end)
 
             self.assertTrue(dt_utc_min().timetz() <= begin <= result <= end <= dt_utc_max().timetz())
+
+    def test_sample(self):
+        for _ in range(1000):
+            population = list(itertools.accumulate(randomizer().next_int(1, 10)
+                                                   for _ in range(randomizer().next_int(100, 200))))
+            count_func = lambda x: 1 + x % 3
+            k = randomizer().next_int(1, len(population))
+
+            result = randomizer().sample(population, count_func, k)
+
+            self.assertTrue(len(result), k)
+            self.assertTrue(all(elem in population for elem in result))
+            self.assertTrue(all(result.count(elem) <= count_func(elem) for elem in result))
+
+    def test_choose(self):
+        for _ in range(1000):
+            population = list(itertools.accumulate(randomizer().next_int(1, 10)
+                                                   for _ in range(randomizer().next_int(100, 200))))
+            weight_func = lambda x: float(1 + x % 3)
+            k = randomizer().next_int(1, len(population))
+
+            result = randomizer().choose(population, weight_func, k)
+
+            self.assertTrue(len(result), k)
+            self.assertTrue(all(elem in population for elem in result))
+
+    def test_shuffle(self):
+        for _ in range(1000):
+            data = list(itertools.accumulate(randomizer().next_int(1, 10)
+                                             for _ in range(randomizer().next_int(100, 200))))
+
+            result = randomizer().shuffle(data)
+
+            self.assertIsNot(result, data)
+            self.assertTrue(len(result), len(data))
+            self.assertTrue(all(elem in data for elem in result))