iker-python-common 1.0.35__tar.gz → 1.0.57__tar.gz

iker_python_common-1.0.57/.github/workflows/pr.yml

@@ -0,0 +1,32 @@
+name: Pull Request
+
+on:
+  pull_request:
+    branches: [ "master" ]
+
+env:
+  BUILD_NUMBER: ${{ github.run_number }}
+
+jobs:
+  build-python:
+    runs-on: ubuntu-latest
+    container:
+      image: ruyangshou/iker-basedev-dev:latest
+      options: --user github
+    strategy:
+      matrix:
+        python-version: [ "3.11", "3.12", "3.13", "3.14" ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "${{ matrix.python-version }}"
+
+      - name: Build and Test
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[test,all]
+          python -m pytest . --cov --cov-report xml --cov-config pyproject.toml

{iker_python_common-1.0.35 → iker_python_common-1.0.57}/.github/workflows/push.yml

@@ -10,21 +10,22 @@ env:
 jobs:
   push:
     runs-on: ubuntu-latest
+    container:
+      image: ruyangshou/iker-basedev-dev:latest
+      options: --user github
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.13"
-
-      - name: Checkout
-        uses: actions/checkout@v4
+          python-version: "3.14"
 
       - name: Build and Upload
         run: |
-          sudo apt-get update
-          sudo apt-get install libxml2-dev libxslt1-dev llvm-14-dev
           python -m pip install --upgrade pip build twine
-          python -m pip install .[test]
+          python -m pip install .[test,all]
           python -m pytest . --cov --cov-report xml --cov-config pyproject.toml
           python -m build -sw .
           python -m twine upload --username __token__ --password ${{ secrets.PYPI_TOKEN }} dist/*

{iker_python_common-1.0.35 → iker_python_common-1.0.57}/PKG-INFO

@@ -1,17 +1,21 @@
 Metadata-Version: 2.4
 Name: iker-python-common
-Version: 1.0.35
+Version: 1.0.57
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Python: <3.14,>=3.11
+Classifier: Programming Language :: Python :: 3.14
+Requires-Python: <3.15,>=3.11
 Requires-Dist: boto3>=1.35
 Requires-Dist: docker>=7.1
 Requires-Dist: numpy>=1.26
 Requires-Dist: psycopg>=3.2
 Requires-Dist: pymysql>=1.1
 Requires-Dist: sqlalchemy>=1.4
+Requires-Dist: typing-extensions; python_version < "3.12"
+Provides-Extra: all
+Requires-Dist: iker-python-common; extra == "all"
 Provides-Extra: test
 Requires-Dist: ddt>=1.7; extra == "test"
 Requires-Dist: moto[all,ec2,s3]>=5.0; extra == "test"

{iker_python_common-1.0.35 → iker_python_common-1.0.57}/pyproject.toml

@@ -1,20 +1,42 @@
 [build-system]
 requires = [
-    "setuptools>=75.0",
-    "setuptools-scm>=8.0",
+    "setuptools>=80.0",
+    "setuptools-scm>=9.0",
     "iker-python-setup>=1.0",
 ]
 build-backend = "setuptools.build_meta"
 
+[dependency-groups]
+dev = [
+    "boto3>=1.35",
+    "docker>=7.1",
+    "numpy>=1.26",
+    "psycopg>=3.2",
+    "pymysql>=1.1",
+    "sqlalchemy>=1.4",
+    "typing-extensions; python_version<'3.12'"
+]
+test = [
+    "ddt>=1.7",
+    "moto[ec2,s3,all]>=5.0",
+    "pytest-cov>=5.0",
+    "pytest-mysql>=3.0",
+    "pytest-order>=1.3",
+    "pytest-postgresql>=6.1",
+    "pytest>=8.3",
+    "sqlalchemy>=2.0",
+]
+
 [project]
 name = "iker-python-common"
 dynamic = ["version"]
-requires-python = ">=3.11,<3.14"
+requires-python = ">=3.11,<3.15"
 classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
 ]
 dependencies = [
     "boto3>=1.35",
@@ -23,9 +45,13 @@ dependencies = [
     "psycopg>=3.2",
     "pymysql>=1.1",
     "sqlalchemy>=1.4",
+    "typing-extensions; python_version<'3.12'"
 ]
 
 [project.optional-dependencies]
+all = [
+    "iker-python-common",
+]
 test = [
     "ddt>=1.7",
     "moto[ec2,s3,all]>=5.0",

iker_python_common-1.0.57/src/iker/common/utils/argutils.py

@@ -0,0 +1,233 @@
+import argparse
+import dataclasses
+import inspect
+import typing
+from collections.abc import Sequence
+from typing import Any
+
+from iker.common.utils.typeutils import is_identical_type
+
+__all__ = [
+    "ParserTreeNode",
+    "ParserTree",
+    "ArgParseSpec",
+    "argparse_spec",
+    "make_argparse"
+]
+
+import sys
+
+
+class ParserTreeNode(object):
+    """
+    Represents a node in the parser tree, holding a command, its parser, and any child nodes. Each node may have
+    subparsers and a list of child nodes representing subcommands.
+
+    :param command: The command string for this node.
+    :param parser: The ``ArgumentParser`` associated with this node.
+    """
+
+    def __init__(self, command: str, parser: argparse.ArgumentParser):
+        self.command = command
+        self.parser = parser
+        self.subparsers = None
+        self.child_nodes: list[ParserTreeNode] = []
+
+
+def construct_parser_tree(
+    root_node: ParserTreeNode,
+    command_chain: list[str],
+    command_key_prefix: str,
+    **kwargs,
+) -> list[ParserTreeNode]:
+    """
+    Constructs a parser tree by traversing or creating nodes for each command in the command chain. Returns the path
+    from the ``root_node`` to the last node in the chain.
+
+    :param root_node: The root node of the parser tree.
+    :param command_chain: A list of command strings representing the path.
+    :param command_key_prefix: Prefix for command keys in the parser.
+    :param kwargs: Additional keyword arguments for parser creation.
+    :return: A list of ``ParserTreeNode`` objects representing the path from root to the last command.
+    """
+    node_path = [root_node]
+    if len(command_chain) == 0:
+        return node_path
+
+    node = root_node
+    for depth, command in enumerate(command_chain):
+        if node.subparsers is None:
+            node.subparsers = node.parser.add_subparsers(dest=f"{command_key_prefix}:{depth}")
+        for child_node in node.child_nodes:
+            if child_node.command == command:
+                node = child_node
+                break
+        else:
+            if depth == len(command_chain) - 1:
+                child_parser = node.subparsers.add_parser(command, **kwargs)
+            else:
+                child_parser = node.subparsers.add_parser(command)
+            child_node = ParserTreeNode(command, child_parser)
+            node.child_nodes.append(child_node)
+            node = child_node
+        node_path.append(node)
+
+    return node_path
+
+
+class ParserTree(object):
+    """
+    Represents a tree structure for managing ``argparse`` parsers and subcommands. Provides methods to add subcommand
+    parsers and parse arguments, returning the command chain and parsed namespace.
+
+    :param root_parser: The root ``ArgumentParser``.
+    :param command_key_prefix: Prefix for command keys in the parser tree.
+    """
+
+    def __init__(self, root_parser: argparse.ArgumentParser, command_key_prefix: str = "command"):
+        self.root_node = ParserTreeNode("", root_parser)
+        self.command_key_prefix = command_key_prefix
+
+    def add_subcommand_parser(self, command_chain: list[str], **kwargs) -> argparse.ArgumentParser:
+        """
+        Adds a subcommand parser for the specified command chain, creating intermediate nodes as needed.
+
+        :param command_chain: A list of command strings representing the subcommand path.
+        :param kwargs: Additional keyword arguments for parser creation.
+        :return: The ``ArgumentParser`` for the last command in the chain.
+        """
+        *_, last_node = construct_parser_tree(self.root_node, command_chain, self.command_key_prefix, **kwargs)
+        return last_node.parser
+
+    def parse_args(self, args: list[str] | None = None) -> tuple[list[str], argparse.Namespace]:
+        """
+        Parses the provided argument list, returning the command chain and the parsed namespace.
+
+        :param args: The list of arguments to parse. If ``None``, parses ``sys.argv``.
+        :return: A tuple containing the list of command strings and the parsed ``Namespace``.
+        """
+        # Before Python 3.12 the ``exit_on_error`` attribute does not take effect properly
+        # if unknown arguments encountered. We have to employ this workaround
+        if sys.version_info < (3, 12):
+            if self.root_node.parser.exit_on_error:
+                known_args_namespace = self.root_node.parser.parse_args(args)
+            else:
+                known_args_namespace, unknown_args = self.root_node.parser.parse_known_args(args)
+                if len(unknown_args or []) > 0:
+                    raise argparse.ArgumentError(None, f"unrecognized arguments '{unknown_args}'")
+        else:
+            known_args_namespace = self.root_node.parser.parse_args(args)
+
+        command_pairs = []
+        namespace = argparse.Namespace()
+        for key, value in dict(vars(known_args_namespace)).items():
+            if key.startswith(self.command_key_prefix) and value is not None:
+                command_pairs.append((key, value))
+            else:
+                setattr(namespace, key, value)
+
+        return list(command for _, command in sorted(command_pairs)), namespace
+
+
+@dataclasses.dataclass(frozen=True)
+class ArgParseSpec(object):
+    """
+    Specification for an argument to be added to an ``ArgumentParser``. Allows detailed configuration of argument
+    properties such as ``flag``, ``name``, ``type``, ``action``, ``default``, ``choices``, and help text.
+
+    :param flag: The optional flag for the argument (e.g., '-f').
+    :param name: The name of the argument (e.g., '--file').
+    :param action: The action to be taken by the argument parser.
+    :param default: The default value for the argument.
+    :param type: The type of the argument value.
+    :param choices: A list of valid choices for the argument.
+    :param required: Whether the argument is required.
+    :param help: The help text for the argument.
+    """
+    flag: str | None = None
+    name: str | None = None
+    action: str | None = None
+    default: Any = None
+    type: typing.Type | None = None
+    choices: list[Any] | None = None
+    required: bool | None = None
+    help: str | None = None
+
+    def make_kwargs(self) -> dict[str, Any]:
+        """
+        Constructs a dictionary of keyword arguments for ``ArgumentParser.add_argument``, omitting any that are
+        ``None``.
+
+        :return: A dictionary of argument properties suitable for ``ArgumentParser.add_argument``.
+        """
+        kwargs = dict(
+            action=self.action,
+            default=self.default,
+            type=self.type,
+            choices=self.choices,
+            required=self.required,
+            help=self.help,
+        )
+
+        return {key: value for key, value in kwargs.items() if value is not None}
+
+
+argparse_spec = ArgParseSpec
+
+
+def make_argparse(func, parser: argparse.ArgumentParser = None) -> argparse.ArgumentParser:
+    """
+    Automatically generates an ``ArgumentParser`` for the given function by inspecting its signature and parameter
+    annotations. Supports ``ArgParseSpec`` for detailed argument configuration.
+
+    :param func: The function whose parameters will be used to generate arguments.
+    :param parser: An optional ``ArgumentParser`` to add arguments to. If ``None``, a new parser is created.
+    :return: The ``ArgumentParser`` with arguments added based on the function signature.
+    """
+    if parser is None:
+        parser = argparse.ArgumentParser()
+
+    def is_type_of(a: Any, *bs) -> bool:
+        return any(is_identical_type(a, b, strict_optional=False, covariant=True) for b in bs)
+
+    sig = inspect.signature(func)
+    for name, param in sig.parameters.items():
+
+        arg_name = f"--{name.replace('_', '-')}"
+
+        if param.annotation is None:
+            arg_type = str
+        elif is_type_of(param.annotation, str, Sequence[str]):
+            arg_type = str
+        elif is_type_of(param.annotation, int, Sequence[int]):
+            arg_type = int
+        elif is_type_of(param.annotation, float, Sequence[float]):
+            arg_type = float
+        elif is_type_of(param.annotation, bool, Sequence[bool]):
+            arg_type = bool
+        else:
+            arg_type = str
+
+        arg_action = "append" if typing.get_origin(param.annotation) in {list, Sequence} else None
+        arg_default = None if param.default is inspect.Parameter.empty else param.default
+
+        if isinstance(arg_default, ArgParseSpec):
+            spec = arg_default
+            spec = dataclasses.replace(spec,
+                                       name=spec.name or arg_name,
+                                       type=spec.type if spec.type is not None else arg_type,
+                                       action=spec.action if spec.action is not None else arg_action)
+
+            if spec.flag is None:
+                parser.add_argument(spec.name, **spec.make_kwargs())
+            else:
+                parser.add_argument(spec.flag, spec.name, **spec.make_kwargs())
+
+        else:
+            parser.add_argument(arg_name,
+                                type=arg_type,
+                                action=arg_action,
+                                required=arg_default is None,
+                                default=arg_default)
+
+    return parser

iker_python_common-1.0.57/src/iker/common/utils/csv.py

@@ -0,0 +1,224 @@
+import os
+from collections.abc import Callable, Generator, Iterable, Mapping, Sequence
+from typing import Any
+from typing import overload
+
+__all__ = [
+    "CSVColumn",
+    "CSVView",
+    "column",
+    "view",
+]
+
+
+class CSVColumn(object):
+    """
+    Represents a column in a CSV schema, including its name, loader function, dumper function, and the string used for
+    null values.
+
+    :param name: The name of the column.
+    :param loader: A function to convert a string to the column's value type.
+    :param dumper: A function to convert the column's value to a string.
+    :param null_str: The string representation of a null value in this column.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        loader: Callable[[str], Any] = str,
+        dumper: Callable[[Any], str] = str,
+        null_str: str = "",
+    ):
+        self.name = name
+        self.loader = loader
+        self.dumper = dumper
+        self.null_str = null_str
+
+
+class CSVView(object):
+    """
+    Represents a view for reading and writing CSV data according to a schema. Supports loading and dumping lines or
+    files, with options for headers and dictionary or list output.
+
+    :param schema: The sequence of ``CSVColumn`` objects defining the schema.
+    :param row_delim: The delimiter for rows (default is ``'\n'``).
+    :param col_delim: The delimiter for columns (default is ``','``).
+    """
+
+    def __init__(self, schema: Sequence[CSVColumn], *, row_delim: str = "\n", col_delim: str = ","):
+        self.schema = schema
+        self.row_delim = row_delim
+        self.col_delim = col_delim
+
+    @overload
+    def load_lines(
+        self,
+        lines: Iterable[str],
+        has_header: bool,
+        ret_dict: False = False,
+    ) -> Generator[list[Any], None, None]:
+        ...
+
+    @overload
+    def load_lines(
+        self,
+        lines: Iterable[str],
+        ret_dict: False = False,
+    ) -> Generator[list[Any], None, None]:
+        ...
+
+    @overload
+    def load_lines(
+        self,
+        lines: Iterable[str],
+        has_header: bool,
+        ret_dict: True = True,
+    ) -> Generator[dict[str, Any], None, None]:
+        ...
+
+    @overload
+    def load_lines(
+        self,
+        lines: Iterable[str],
+        ret_dict: True = True,
+    ) -> Generator[dict[str, Any], None, None]:
+        ...
+
+    def load_lines(
+        self,
+        lines: Iterable[str],
+        has_header: bool = True,
+        ret_dict: bool = False,
+    ) -> Generator[list[Any] | dict[str, Any], None, None]:
+        """
+        Loads CSV data from an iterable of lines, optionally using the first line as a header. Returns each row as a
+        list or dictionary, depending on ``ret_dict``.
+
+        :param lines: An iterable of CSV lines.
+        :param has_header: Whether the first line is a header.
+        :param ret_dict: Whether to return rows as dictionaries (``True``) or lists (``False``).
+        :return: A generator yielding each row as a list or dictionary.
+        """
+        rows_iter = iter(lines)
+        if has_header:
+            header_row = next(rows_iter)
+            header_cols = header_row.split(self.col_delim)
+            if len(self.schema) != len(header_cols):
+                raise ValueError("size of the schema is not identical to size of the columns")
+            for c, header_col in zip(self.schema, header_cols):
+                if c.name != header_col:
+                    raise ValueError("name of the schema is not equal to the name of the columns")
+        for row in rows_iter:
+            cols = row.split(self.col_delim)
+            if len(self.schema) != len(cols):
+                continue
+            if ret_dict:
+                yield {c.name: None if col == c.null_str else c.loader(col) for c, col in zip(self.schema, cols)}
+            else:
+                yield [None if col == c.null_str else c.loader(col) for c, col in zip(self.schema, cols)]
+
+    def dump_lines(
+        self,
+        data: Iterable[Sequence[Any] | Mapping[str, Any]],
+        has_header: bool = True,
+    ) -> Generator[str, None, None]:
+        """
+        Dumps data to CSV lines according to the schema, optionally including a header row.
+
+        :param data: An iterable of rows, each as a sequence or mapping.
+        :param has_header: Whether to include a header row.
+        :return: A generator yielding CSV lines as strings.
+        """
+        if has_header:
+            yield self.col_delim.join(c.name for c in self.schema)
+        for cols in data:
+            if isinstance(cols, Sequence):
+                if len(self.schema) != len(cols):
+                    raise ValueError("size of the schema is not identical to size of the columns")
+                yield self.col_delim.join(c.null_str if col is None else c.dumper(col)
+                                          for c, col in zip(self.schema, cols))
+            if isinstance(cols, Mapping):
+                yield self.col_delim.join(c.null_str if cols.get(c.name) is None else c.dumper(cols.get(c.name))
+                                          for c in self.schema)
+
+    @overload
+    def load_file(
+        self,
+        file_path: os.PathLike | str,
+        has_header: bool,
+        ret_dict: False = False,
+        **kwargs,
+    ) -> Generator[list[Any], None, None]:
+        ...
+
+    @overload
+    def load_file(
+        self,
+        file_path: os.PathLike | str,
+        ret_dict: False = False,
+        **kwargs,
+    ) -> Generator[list[Any], None, None]:
+        ...
+
+    @overload
+    def load_file(
+        self,
+        file_path: os.PathLike | str,
+        has_header: bool,
+        ret_dict: True = True,
+        **kwargs,
+    ) -> Generator[dict[str, Any], None, None]:
+        ...
+
+    @overload
+    def load_file(
+        self,
+        file_path: os.PathLike | str,
+        ret_dict: True = True,
+        **kwargs,
+    ) -> Generator[dict[str, Any], None, None]:
+        ...
+
+    def load_file(
+        self,
+        file_path: os.PathLike | str,
+        has_header: bool = True,
+        ret_dict: bool = False,
+        **kwargs,
+    ) -> Generator[list[Any] | dict[str, Any], None, None]:
+        """
+        Loads CSV data from a file, splitting by row delimiter and using the ``schema`` for parsing.
+
+        :param file_path: The path to the CSV file.
+        :param has_header: Whether the first line is a header.
+        :param ret_dict: Whether to return rows as dictionaries (``True``) or lists (``False``).
+        :param kwargs: Additional keyword arguments for file opening.
+        :return: A generator yielding each row as a list or dictionary.
+        """
+        with open(file_path, mode="r", **kwargs) as fh:
+            lines = fh.read().split(self.row_delim)
+            yield from self.load_lines(lines, has_header, ret_dict)
+
+    def dump_file(
+        self,
+        data: Iterable[Sequence[Any] | Mapping[str, Any]],
+        file_path: os.PathLike | str,
+        has_header: bool = True,
+        **kwargs,
+    ) -> None:
+        """
+        Dumps data to a CSV file according to the ``schema``, optionally including a header row.
+
+        :param data: An iterable of rows, each as a sequence or mapping.
+        :param file_path: The path to the output CSV file.
+        :param has_header: Whether to include a header row.
+        :param kwargs: Additional keyword arguments for file opening.
+        """
+        with open(file_path, mode="w", **kwargs) as fh:
+            for line in self.dump_lines(data, has_header):
+                fh.write(line)
+                fh.write(self.row_delim)
+
+
+column = CSVColumn
+view = CSVView