iker-python-common 1.0.1__py3-none-any.whl

iker/common/__init__.py

@@ -0,0 +1,6 @@
+import importlib.metadata
+
+try:
+    __version__ = importlib.metadata.version("iker-python-common")
+except importlib.metadata.PackageNotFoundError:
+    __version__ = "unknown"

iker/common/core/exceptions.py

@@ -0,0 +1,64 @@
+import traceback
+
+
+class BaseTraceableException(Exception):
+    """
+    Base class of traceable exceptions and errors
+    """
+
+    def __init__(self, message: str, *args, **kwargs):
+        super(BaseTraceableException, self).__init__()
+        self.message = message
+        self.args = args
+        self.cause = kwargs.get("cause")
+        self.traceback = "" if kwargs.get("cause") is None else traceback.format_exc()
+
+    def __str__(self) -> str:
+        return self.message % self.args
+
+    def __traceback(self):
+        if self.cause is None or not isinstance(self.cause, BaseTraceableException):
+            return self.traceback
+        return self.traceback + self.cause.__traceback()
+
+    @staticmethod
+    def format_traceback(error):
+        if isinstance(error, BaseTraceableException):
+            return traceback.format_exc() + error.__traceback()
+        return traceback.format_exc()
+
+
+class RuntimeException(BaseTraceableException):
+    """
+    Represents runtime exception
+    """
+
+    def __init__(self, message: str, *args, **kwargs):
+        super(RuntimeException, self).__init__(message, *args, **kwargs)
+
+
+class InvalidTypeException(BaseTraceableException):
+    """
+    Represents invalid type exception
+    """
+
+    def __init__(self, message: str, *args, **kwargs):
+        super(InvalidTypeException, self).__init__(message, *args, **kwargs)
+
+
+class InvalidKeyException(BaseTraceableException):
+    """
+    Represents invalid key exception
+    """
+
+    def __init__(self, message: str, *args, **kwargs):
+        super(InvalidKeyException, self).__init__(message, *args, **kwargs)
+
+
+class InvalidValueException(BaseTraceableException):
+    """
+    Represents invalid value exception
+    """
+
+    def __init__(self, message: str, *args, **kwargs):
+        super(InvalidValueException, self).__init__(message, *args, **kwargs)

iker/common/utils/config.py

@@ -0,0 +1,117 @@
+import configparser
+from typing import Self
+
+from iker.common.utils import logger
+from iker.common.utils.strutils import is_blank, trim_to_empty
+
+
+class Config(object):
+    def __init__(self, config_path: str = None):
+        self.config_path = trim_to_empty(config_path)
+        self.config_parser: configparser.RawConfigParser = configparser.RawConfigParser(strict=False)
+
+    def __len__(self):
+        return sum(len(self.config_parser.options(section)) for section in self.config_parser.sections())
+
+    def update(self, tuples: list[tuple[str, str, str]], *, overwrite: bool = False):
+        for section, option, value in tuples:
+            if not self.config_parser.has_section(section):
+                self.config_parser.add_section(section)
+            if overwrite or not self.config_parser.has_option(section, option):
+                self.config_parser.set(section, option, value)
+
+    def restore(self) -> bool:
+        self.config_parser = configparser.RawConfigParser(strict=False)
+        if is_blank(self.config_path):
+            return False
+        try:
+            self.config_parser.read(self.config_path)
+            return True
+        except IOError as e:
+            logger.exception("Failed to restore config from file <%s>", self.config_path)
+        return False
+
+    def persist(self) -> bool:
+        if is_blank(self.config_path):
+            return False
+        try:
+            with open(self.config_path, "w") as fh:
+                self.config_parser.write(fh)
+            return True
+        except IOError as e:
+            logger.exception("Failed to persist config to file <%s>", self.config_path)
+        return False
+
+    def has_section(self, section: str) -> bool:
+        return self.config_parser.has_section(section)
+
+    def has(self, section: str, option: str) -> bool:
+        return self.config_parser.has_option(section, option)
+
+    def get(self, section: str, option: str, default_value: str = None) -> str:
+        if self.config_parser.has_option(section, option):
+            return self.config_parser.get(section, option)
+        return default_value
+
+    def getint(self, section: str, option: str, default_value: int = None) -> int:
+        if self.config_parser.has_option(section, option):
+            return self.config_parser.getint(section, option)
+        return default_value
+
+    def getfloat(self, section: str, option: str, default_value: float = None) -> float:
+        if self.config_parser.has_option(section, option):
+            return self.config_parser.getfloat(section, option)
+        return default_value
+
+    def getboolean(self, section: str, option: str, default_value: bool = None) -> bool:
+        if self.config_parser.has_option(section, option):
+            return self.config_parser.getboolean(section, option)
+        return default_value
+
+    def set(self, section: str, option: str, value: str):
+        if not self.config_parser.has_section(section):
+            self.config_parser.add_section(section)
+        self.config_parser.set(section, option, value)
+
+    def sections(self) -> list[str]:
+        return self.config_parser.sections()
+
+    def options(self, section: str) -> list[str]:
+        if not self.config_parser.has_section(section):
+            return []
+        return self.config_parser.options(section)
+
+    def tuples(self) -> list[tuple[str, str, str]]:
+        result = []
+        for section in self.config_parser.sections():
+            for option in self.config_parser.options(section):
+                value = self.config_parser.get(section, option)
+                result.append((section, option, value))
+        return result
+
+
+class ConfigVisitor(object):
+    def __init__(self, config: Config, section: str, prefix: str = "", separator: str = "."):
+        self.config = config
+        self.section = section
+        self.prefix = prefix
+        self.separator = separator
+
+    def __str__(self):
+        return self.config.get(self.section, self.prefix)
+
+    def __int__(self):
+        return self.config.getint(self.section, self.prefix)
+
+    def __float__(self):
+        return self.config.getfloat(self.section, self.prefix)
+
+    def __bool__(self):
+        return self.config.getboolean(self.section, self.prefix)
+
+    def __getattr__(self, suffix: str) -> Self:
+        return self[suffix]
+
+    def __getitem__(self, suffix: str) -> Self:
+        new_prefix = suffix if is_blank(self.prefix) else self.separator.join([self.prefix, suffix])
+        return ConfigVisitor(self.config, self.section, new_prefix, self.separator)

iker/common/utils/dbutils.py

@@ -0,0 +1,203 @@
+import contextlib
+import dataclasses
+import datetime
+import urllib.parse
+from typing import Any, ContextManager, Type
+
+import psycopg2
+import psycopg2.extensions
+import sqlalchemy
+import sqlalchemy.ext.compiler
+import sqlalchemy.orm
+
+from iker.common.utils.funcutils import singleton
+from iker.common.utils.sequtils import head_or_none
+from iker.common.utils.strutils import is_blank
+
+__all__ = [
+    "DBAdapter",
+    "orm_to_dict",
+    "orm_clone",
+    "to_pg_date",
+    "to_pg_time",
+    "to_pg_ts",
+    "to_pg_ts",
+    "pg_date_max",
+    "pg_ts_max",
+    "mysql_insert_ignore",
+    "postgresql_insert_on_conflict_do_nothing",
+]
+
+
+class DBAdapter(object):
+    """
+    Database adapter
+    """
+
+    class Drivers:
+        Mysql = "mysql+mysqldb"
+        Postgresql = "postgresql+psycopg2"
+
+    def __init__(
+        self,
+        driver: str,
+        host: str,
+        port: int,
+        user: str,
+        password: str,
+        database: str,
+        engine_opts: dict[str, Any] | None = None,
+        session_opts: dict[str, Any] | None = None,
+    ):
+        self.driver = driver
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+        self.database = database
+        self.engine_opts = engine_opts or {}
+        self.session_opts = session_opts or {}
+
+    @property
+    def connection_string(self) -> str:
+        port_part = "" if self.port is None else (":%d" % self.port)
+        user_part = urllib.parse.quote(self.user, safe="")
+        password_part = "" if is_blank(self.password) else (":%s" % urllib.parse.quote(self.password, safe=""))
+        database_part = urllib.parse.quote(self.database, safe="")
+
+        return f"{self.driver}://{user_part}{password_part}@{self.host}{port_part}/{database_part}"
+
+    @property
+    def engine(self) -> sqlalchemy.Engine:
+        return sqlalchemy.create_engine(self.connection_string, **self.engine_opts)
+
+    def make_connection(self):
+        return self.engine.connect()
+
+    def make_session(self, **kwargs) -> ContextManager[sqlalchemy.orm.Session]:
+        return contextlib.closing(sqlalchemy.orm.sessionmaker(self.engine, **{**self.session_opts, **kwargs})())
+
+    def create_model(self, orm_base: Type[sqlalchemy.orm.DeclarativeBase]):
+        orm_base.metadata.create_all(self.engine)
+
+    def drop_model(self, orm_base: Type[sqlalchemy.orm.DeclarativeBase]):
+        orm_base.metadata.drop_all(self.engine)
+
+    def execute(self, sql: str, **params):
+        """
+        Executes the given SQL statement with the specific parameters
+
+        :param sql: SQL statement to execute
+        :param params: parameters dict
+        """
+        with self.make_connection() as connection:
+            connection.execute(sqlalchemy.text(sql), params)
+            connection.commit()
+
+    def query_all(self, sql: str, **params) -> list[tuple]:
+        """
+        Executes the given SQL query with the specific parameters and returns all the results
+
+        :param sql: SQL query to execute
+        :param params: parameters dict
+        :return: result tuples list
+        """
+        with self.make_connection() as connection:
+            with contextlib.closing(connection.execute(sqlalchemy.text(sql), params)) as proxy:
+                return [item for item in proxy.fetchall()]
+
+    def query_first(self, sql: str, **params) -> tuple | None:
+        """
+        Executes the given SQL query with the specific parameters and returns the first result tuple
+
+        :param sql: SQL query to execute
+        :param params: parameters dict
+        :return: the first result tuple
+        """
+        return head_or_none(self.query_all(sql, **params))
+
+
+def orm_to_dict(orm: sqlalchemy.orm.DeclarativeBase, exclude: set[str] = None) -> dict[str, Any]:
+    if not isinstance(orm, sqlalchemy.orm.DeclarativeBase):
+        raise TypeError('not a SQLAlchemy ORM declarative base')
+
+    mapper = sqlalchemy.inspect(type(orm))
+    return dict((c.key, getattr(orm, c.key)) for c in mapper.columns if c.key not in (exclude or set()))
+
+
+def orm_clone(orm: sqlalchemy.orm.DeclarativeBase, exclude: set[str] = None, no_autoincrement: bool = False):
+    if not isinstance(orm, sqlalchemy.orm.DeclarativeBase):
+        raise TypeError('not a SQLAlchemy ORM declarative base')
+
+    mapper = sqlalchemy.inspect(type(orm))
+    exclude = exclude or (set(c.key for c in mapper.columns if c.autoincrement is True) if no_autoincrement else set())
+    fields = orm_to_dict(orm, exclude)
+
+    if not dataclasses.is_dataclass(orm):
+        return type(orm)(**fields)
+
+    init_fields = dict((field.name, fields.get(field.name)) for field in dataclasses.fields(orm) if field.init)
+
+    new_orm = type(orm)(**init_fields)
+    for name, value in fields.items():
+        if name not in init_fields:
+            setattr(new_orm, name, value)
+    return new_orm
+
+
+def to_pg_date(dt: datetime.datetime | datetime.date | int | float):
+    if isinstance(dt, (datetime.datetime, datetime.date)):
+        return psycopg2.extensions.DateFromPy(dt)
+    elif isinstance(dt, (int, float)):
+        return psycopg2.DateFromTicks(dt)
+    raise TypeError("should be one of 'datetime.datetime', 'datetime.date', 'int', 'float'")
+
+
+def to_pg_time(dt: datetime.time | int | float):
+    if isinstance(dt, datetime.time):
+        return psycopg2.extensions.TimeFromPy(dt)
+    elif isinstance(dt, (int, float)):
+        return psycopg2.TimeFromTicks(dt)
+    raise TypeError("should be one of 'datetime.time', 'int', 'float'")
+
+
+def to_pg_ts(dt: datetime.datetime | datetime.date | int | float):
+    if isinstance(dt, (datetime.datetime, datetime.date)):
+        return psycopg2.extensions.TimestampFromPy(dt)
+    elif isinstance(dt, (int, float)):
+        return psycopg2.TimestampFromTicks(dt)
+    raise TypeError("should be one of 'datetime.datetime', 'datetime.date', 'int', 'float'")
+
+
+@singleton
+def pg_date_max():
+    return psycopg2.Date(9999, 12, 31)
+
+
+@singleton
+def pg_ts_max():
+    return psycopg2.Timestamp(9999, 12, 31, 23, 59, 59.999, tzinfo=datetime.timezone.utc)
+
+
+def mysql_insert_ignore(enabled: bool = True):
+    @sqlalchemy.ext.compiler.compiles(sqlalchemy.sql.Insert, "mysql")
+    def dispatch(insert: sqlalchemy.sql.Insert, compiler: sqlalchemy.sql.compiler.SQLCompiler, **kwargs) -> str:
+        if not enabled:
+            return compiler.visit_insert(insert, **kwargs)
+
+        return compiler.visit_insert(insert.prefix_with("IGNORE"), **kwargs)
+
+
+def postgresql_insert_on_conflict_do_nothing(enabled: bool = True):
+    @sqlalchemy.ext.compiler.compiles(sqlalchemy.sql.Insert, "postgresql")
+    def dispatch(insert: sqlalchemy.sql.Insert, compiler: sqlalchemy.sql.compiler.SQLCompiler, **kwargs) -> str:
+        if not enabled:
+            return compiler.visit_insert(insert, **kwargs)
+
+        statement = compiler.visit_insert(insert, **kwargs)
+        # If we have a "RETURNING" clause, we must insert before it
+        returning_position = statement.find("RETURNING")
+        if returning_position >= 0:
+            return statement[:returning_position] + " ON CONFLICT DO NOTHING " + statement[returning_position:]
+        else:
+            return statement + " ON CONFLICT DO NOTHING"

iker/common/utils/dockerutils.py

@@ -0,0 +1,223 @@
+import contextlib
+import re
+from dataclasses import dataclass
+from typing import Any, ContextManager, Iterable
+
+import docker
+import docker.errors
+import docker.models.containers
+import docker.models.images
+import requests.exceptions
+
+from iker.common.utils import logger
+from iker.common.utils.strutils import parse_int_or, trim_to_empty
+
+__all__ = [
+    "ImageName",
+    "docker_create_client",
+    "docker_build_image",
+    "docker_get_image",
+    "docker_pull_image",
+    "docker_fetch_image",
+    "docker_run_detached",
+]
+
+
+@dataclass
+class ImageName(object):
+    registry_host: str | None
+    registry_port: int | None
+    components: list[str]
+    tag: str | None
+
+    @property
+    def registry(self) -> str:
+        if self.registry_host is None and self.registry_port is None:
+            return ""
+        if self.registry_port is None:
+            return self.registry_host
+        return f"{self.registry_host}:{self.registry_port}"
+
+    @property
+    def repository(self) -> str:
+        return "/".join(self.components)
+
+    @staticmethod
+    def parse(s: str):
+
+        # Registry absent version
+        matcher = re.compile(r"^(?P<components>[\w-]+(/[\w-]+)*)(:(?P<tag>[\w._-]+))?$")
+        match = matcher.match(s)
+        if match:
+            return ImageName(registry_host=None,
+                             registry_port=None,
+                             components=trim_to_empty(match.group("components")).split("/"),
+                             tag=match.group("tag"))
+
+        # Registry present version
+        matcher = re.compile(
+            r"^(?:(?P<host>[\w.-]+)(?::(?P<port>\d+))?/)?(?P<components>[\w-]+(/[\w-]+)*)(:(?P<tag>[\w._-]+))?$")
+        match = matcher.match(s)
+        if match:
+            return ImageName(registry_host=match.group("host"),
+                             registry_port=parse_int_or(match.group("port"), None),
+                             components=trim_to_empty(match.group("components")).split("/"),
+                             tag=match.group("tag"))
+
+        return None
+
+
+def docker_create_client(registry: str, username: str, password: str) -> ContextManager[docker.DockerClient]:
+    try:
+        client = docker.DockerClient()
+        client.login(registry=registry, username=username, password=password, reauth=True)
+        return contextlib.closing(client)
+    except docker.errors.APIError:
+        logger.exception("Failed to login Docker server <%s>", registry)
+        raise
+
+
+def docker_build_image(
+    client: docker.DockerClient,
+    tag: str,
+    path: str,
+    dockerfile: str,
+    build_args: dict[str, str],
+) -> tuple[docker.models.images.Image, Iterable[Any]]:
+    try:
+        return client.images.build(tag=tag,
+                                   path=path,
+                                   dockerfile=dockerfile,
+                                   buildargs=build_args,
+                                   rm=True,
+                                   forcerm=True,
+                                   nocache=True)
+
+    except docker.errors.BuildError:
+        logger.exception("Failed to build image <%s>", tag)
+        raise
+    except docker.errors.APIError:
+        logger.exception("Docker server returns an error while building image <%s>", tag)
+        raise
+    except Exception:
+        logger.exception("Unexpected error occurred while building image <%s>", tag)
+        raise
+
+
+def docker_get_image(
+    client: docker.DockerClient,
+    image: str,
+) -> docker.models.images.Image:
+    try:
+        return client.images.get(image)
+    except docker.errors.ImageNotFound:
+        logger.exception("Image <%s> is not found from local repository", image)
+        raise
+    except docker.errors.APIError:
+        logger.exception("Docker server returns an error while getting image <%s>", image)
+        raise
+    except Exception:
+        logger.exception("Unexpected error occurred while getting image <%s>", image)
+        raise
+
+
+def docker_pull_image(
+    client: docker.DockerClient,
+    image: str,
+    fallback_local: bool = False,
+) -> docker.models.images.Image:
+    try:
+        return client.images.pull(image)
+    except docker.errors.ImageNotFound:
+        if not fallback_local:
+            logger.exception("Image <%s> is not found from remote repository", image)
+            raise
+        logger.warning("Image <%s> is not found from remote repository, try local repository instead", image)
+    except docker.errors.APIError:
+        if not fallback_local:
+            logger.exception("Docker server returns an error while pulling image <%s>", image)
+            raise
+        logger.warning("Docker server returns an error while pulling image <%s>, try local repository instead", image)
+    except Exception:
+        logger.exception("Unexpected error occurred while pulling image <%s>", image)
+        raise
+
+    return docker_get_image(client, image)
+
+
+def docker_fetch_image(
+    client: docker.DockerClient,
+    image: str,
+    force_pull: bool = False,
+) -> docker.models.images.Image:
+    if force_pull:
+        return docker_pull_image(client, image, fallback_local=True)
+    else:
+        try:
+            return docker_get_image(client, image)
+        except Exception:
+            return docker_pull_image(client, image, fallback_local=False)
+
+
+def docker_run_detached(
+    client: docker.DockerClient,
+    image: str,
+    name: str,
+    command: str | list[str],
+    volumes: dict[str, dict[str, str]],
+    environment: dict[str, str],
+    extra_hosts: dict[str, str],
+    timeout: int,
+    **kwargs,
+) -> tuple[dict[str, Any], Any]:
+    @contextlib.contextmanager
+    def managed_docker_run(client: docker.DockerClient, **kwargs) -> docker.models.containers.Container:
+        container_model = client.containers.run(**kwargs)
+        try:
+            yield container_model
+        finally:
+            if container_model.status != "exited":
+                try:
+                    container_model.stop()
+                except docker.errors.DockerException:
+                    pass
+                try:
+                    container_model.wait()
+                except docker.errors.DockerException:
+                    pass
+            try:
+                container_model.remove()
+            except docker.errors.DockerException:
+                pass
+
+    try:
+        run_args = kwargs.copy()
+        run_args.update(dict(image=image,
+                             name=name,
+                             command=command,
+                             volumes=volumes,
+                             environment=environment,
+                             extra_hosts=extra_hosts,
+                             detach=True))
+
+        with managed_docker_run(client, **run_args) as container_model:
+            result = container_model.wait(timeout=timeout)
+            logs = container_model.logs()
+
+            return result, logs
+
+    except requests.exceptions.ReadTimeout:
+        logger.exception("Running container <%s> of image <%s> exceed the timeout", name, image)
+        raise
+    except docker.errors.ImageNotFound:
+        logger.exception("Image <%s> is not found", image)
+        raise
+    except docker.errors.ContainerError:
+        logger.exception("Failed to run container <%s> of image <%s>", name, image)
+        raise
+    except docker.errors.APIError:
+        logger.exception("Docker server returns an error while running container <%s> of image <%s>", name, image)
+        raise
+    except Exception:
+        logger.exception("Unexpected error occurred while running container <%s> of image <%s>", image)
+        raise