holado 0.5.2__tar.gz → 0.6.0__tar.gz

{holado-0.5.2 → holado-0.6.0}/.gitlab-ci.yml

@@ -1,15 +1,16 @@
 stages:
-  - dockerhub-build-publish
-  - gitlab-docker-build-publish
-#  - pypi-build-publish
+  - publish-holado-dockerhub
+  - publish-docker-controller-dockerhub
+  - publish-holado-gitlab
+#  - publish-holado-pypi
 
 variables:
   DOCKER_IMAGE_TAG: "26.1"
   PYTHON_IMAGE_TAG: "3.12"
 
 
-dockerhub-build-publish-latest:
-  stage: dockerhub-build-publish
+publish-holado-dockerhub-latest:
+  stage: publish-holado-dockerhub
 
   image:
     name: docker:${DOCKER_IMAGE_TAG}
@@ -27,9 +28,8 @@ dockerhub-build-publish-latest:
       when: never
     - if: $CI_COMMIT_BRANCH == "main"
 
-
-dockerhub-build-publish-tag:
-  stage: dockerhub-build-publish
+publish-holado-dockerhub-tag:
+  stage: publish-holado-dockerhub
 
   image:
     name: docker:${DOCKER_IMAGE_TAG}
@@ -43,11 +43,48 @@ dockerhub-build-publish-tag:
     - docker push ${IMAGE_TAG}
   
   rules:
-    - if: $CI_COMMIT_BRANCH == "main" && $CI_COMMIT_TAG
+    - if: $CI_COMMIT_TAG
+
+
+publish-docker-controller-dockerhub-latest:
+  stage: publish-docker-controller-dockerhub
+
+  image:
+    name: docker:${DOCKER_IMAGE_TAG}
+    
+  variables:
+    IMAGE_TAG: index.docker.io/holado/docker_controller:latest
+
+  script:
+    - docker build ./src/holado_docker/tools/docker_controller/server/ -t ${IMAGE_TAG}
+    - echo "$DOCKERHUB_TOKEN" | docker login docker.io -u holado --password-stdin
+    - docker push ${IMAGE_TAG}
+  
+  rules:
+    - if: $CI_COMMIT_TAG
+      when: never
+    - if: $CI_COMMIT_BRANCH == "main"
+
+publish-docker-controller-dockerhub-tag:
+  stage: publish-docker-controller-dockerhub
+
+  image:
+    name: docker:${DOCKER_IMAGE_TAG}
+    
+  variables:
+    IMAGE_TAG: index.docker.io/holado/docker_controller:$CI_COMMIT_TAG
+
+  script:
+    - docker build ./src/holado_docker/tools/docker_controller/server/ -t ${IMAGE_TAG}
+    - echo "$DOCKERHUB_TOKEN" | docker login docker.io -u holado --password-stdin
+    - docker push ${IMAGE_TAG}
+  
+  rules:
+    - if: $CI_COMMIT_TAG
 
 
-gitlab-docker-build-publish:
-  stage: gitlab-docker-build-publish
+publish-holado-gitlab:
+  stage: publish-holado-gitlab
 
   image:
     name: docker:${DOCKER_IMAGE_TAG}
@@ -60,8 +97,8 @@ gitlab-docker-build-publish:
     - docker push ${IMAGE_TAG}
     
 
-# pypi-build-publish:
-#   stage: pypi-build-publish
+# publish-holado-pypi:
+#   stage: publish-holado-pypi
 # 
 #   image:
 #     name: python:${PYTHON_IMAGE_TAG}

{holado-0.5.2 → holado-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: holado
-Version: 0.5.2
+Version: 0.6.0
 Summary: HolAdo framework
 Project-URL: Homepage, https://gitlab.com/holado_framework/python
 Project-URL: Issues, https://gitlab.com/holado_framework/python/-/issues
@@ -20,6 +20,8 @@ Requires-Dist: psutil
 Requires-Dist: python-dateutil
 Provides-Extra: ais
 Requires-Dist: pyais; extra == 'ais'
+Provides-Extra: api-connexion
+Requires-Dist: connexion[flask,swagger-ui,uvicorn]; extra == 'api-connexion'
 Provides-Extra: db
 Requires-Dist: pypika; extra == 'db'
 Requires-Dist: sql-metadata; extra == 'db'
@@ -58,8 +60,8 @@ Requires-Dist: pysftp; extra == 'sftp'
 Requires-Dist: sftpserver; extra == 'sftp'
 Provides-Extra: ssl
 Requires-Dist: certifi; extra == 'ssl'
-Requires-Dist: pip-system-certs; extra == 'ssl'
 Requires-Dist: pyopenssl; extra == 'ssl'
+Requires-Dist: truststore; extra == 'ssl'
 Provides-Extra: ws-suds
 Requires-Dist: suds; extra == 'ws-suds'
 Provides-Extra: ws-zeep

{holado-0.5.2 → holado-0.6.0}/README.rst

@@ -11,8 +11,8 @@ Howto run tests from docker image got from registry.gitlab.com
 ==============
 
 - docker login -u XXX registry.gitlab.com
-- docker pull registry.gitlab.com/testing-solution/python:main
-- docker run --rm -it registry.gitlab.com/testing-solution/python:main /bin/sh -c "cd /code/testing-solution/python; ./run_test.sh"
+- docker pull registry.gitlab.com/holado_framework/python:main
+- docker run --rm -it registry.gitlab.com/holado_framework/python:main /bin/sh -c "cd /code/holado/python; ./run_test.sh"
 
 
 Howto create a gitlab runner

holado-0.6.0/build/Dockerfile_python3.13_alpine3.20

@@ -0,0 +1,75 @@
+# Note: HolAdo tests are currently failing with Python 3.13.0
+
+### Build
+
+#FROM python:3.10.15-alpine3.20 AS build
+#FROM python:3.11.10-alpine3.20 AS build
+#FROM python:3.12.7-alpine3.20 AS build
+FROM python:3.13-alpine3.20 AS build
+
+# Install to build python requirements
+RUN apk update \
+    && apk --no-cache --update add build-base \
+    && apk --update add alpine-sdk \
+    && apk add libffi-dev \
+    && apk add postgresql-dev gcc python3-dev musl-dev \
+    && apk add protobuf-dev
+
+# Add tools
+RUN apk --no-cache add bash \
+    && apk --no-cache add nano \
+    && apk --no-cache add curl
+
+# Create user
+RUN addgroup appuser \
+    && adduser -G appuser -s /bin/bash -D appuser
+
+# Create /code folder
+RUN mkdir /code \
+    && chown -R appuser:appuser /code
+
+# Switch to user appuser
+USER appuser
+
+# Copy HolAdo sources
+COPY --chown=appuser . /code/holado/python
+WORKDIR /code/holado/python
+
+# Create python venv and install requirements
+RUN python -m venv /code/env \
+    && source /code/env/bin/activate \
+    && pip install -r requirements_docker.txt
+
+
+
+### Runtime
+
+#FROM python:3.10.15-alpine3.20 AS runtime
+#FROM python:3.11.10-alpine3.20 AS runtime
+#FROM python:3.12.7-alpine3.20 AS runtime
+FROM python:3.13-alpine3.20 AS runtime
+
+# Install for python requirements
+RUN apk update \
+    && apk --no-cache add libstdc++ \
+    && apk --no-cache add postgresql-dev gcc python3-dev musl-dev
+
+# Add tools
+RUN apk --no-cache add bash \
+    && apk --no-cache add nano \
+    && apk --no-cache add curl
+
+# Create and switch to user appuser as in build
+RUN addgroup appuser \
+    && adduser -G appuser -s /bin/bash -D appuser
+USER appuser
+
+# Copy /code from build
+COPY --chown=appuser --from=build /code /code
+
+# Set working directory in HolAdo
+WORKDIR /code/holado/python
+
+# Activate permanently python venv
+ENV PATH=/code/env/bin:$PATH
+

holado-0.6.0/build/Dockerfile_with_docker

@@ -0,0 +1,69 @@
+### Build
+
+FROM python:3.12.9-alpine3.21 AS build
+
+# Install to build python requirements
+RUN apk update \
+    && apk --no-cache --update add build-base \
+    && apk --update add alpine-sdk \
+    && apk add libffi-dev \
+    && apk add postgresql-dev gcc python3-dev musl-dev \
+    && apk add protobuf-dev
+
+# Add tools
+RUN apk --no-cache add bash \
+    && apk --no-cache add nano \
+    && apk --no-cache add curl
+
+# Create user
+RUN addgroup appuser \
+    && adduser -G appuser -s /bin/bash -D appuser
+
+# Create /code folder
+RUN mkdir /code \
+    && chown -R appuser:appuser /code
+
+# Switch to user appuser
+USER appuser
+
+# Copy HolAdo sources
+COPY --chown=appuser . /code/holado/python
+WORKDIR /code/holado/python
+
+# Create python venv and install requirements
+RUN python -m venv /code/env \
+    && source /code/env/bin/activate \
+    && pip install -r requirements.txt
+
+
+
+### Runtime
+
+FROM python:3.12.9-alpine3.21 AS runtime
+
+# Install for python requirements
+RUN apk update \
+    && apk --no-cache add libstdc++ \
+    && apk --no-cache add postgresql-dev gcc python3-dev musl-dev
+
+# Add tools
+RUN apk --no-cache add bash \
+    && apk --no-cache add nano \
+    && apk --no-cache add curl
+
+# Create and switch to user appuser as in build
+RUN addgroup appuser \
+    && adduser -G appuser -s /bin/bash -D appuser
+USER appuser
+
+# Copy /code from build
+COPY --chown=appuser --from=build /code /code
+
+# Set working directory in HolAdo
+WORKDIR /code/holado/python
+
+# Set environement variable HOLADO_PATH, so that installed holado sources are used instead of holado package
+ENV HOLADO_PATH=/code/holado/python
+
+# Activate permanently python venv
+ENV PATH=/code/env/bin:$PATH

holado-0.6.0/packaging/VERSION

@@ -0,0 +1 @@
+0.6.0

{holado-0.5.2 → holado-0.6.0}/pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
 
 [project.optional-dependencies]
 ais = ["pyais"]
+api-connexion = ["connexion[flask,swagger-ui,uvicorn]"]
 db = ["pypika", "sql_metadata"]
 db-postgresql = ["psycopg", "psycopg_binary"]
 docker = ["docker"]
@@ -44,7 +45,7 @@ rabbitmq = ["pika"]
 redis = ["redis", "redis-om"]
 s3 = ["minio", "boto3", "moto", "flask", "flask_cors"]
 sftp = ["pysftp", "sftpserver"]
-ssl = ["pip-system-certs", "certifi", "pyOpenSSL"]
+ssl = ["truststore", "certifi", "pyOpenSSL"]
 ws-suds = ["suds"]
 ws-zeep = ["zeep"]
 yaml = ["pyyaml"]

{holado-0.5.2 → holado-0.6.0}/requirements_docker.txt

@@ -74,6 +74,9 @@ truststore
 certifi
 pyOpenSSL
 
+# For API module
+connexion[flask,swagger-ui,uvicorn]
+
 
 ### DB modules
 

holado-0.6.0/src/holado_api/README.txt

@@ -0,0 +1,6 @@
+Tools:
+	https://openapi.tools/
+
+Generate code from OpenAPI spec:
+	https://openapi-generator.tech/ -> https://github.com/OpenAPITools/openapi-generator
+

holado-0.6.0/src/holado_api/client/README.txt

@@ -0,0 +1,2 @@
+Generate client from OpenAPI:
+	https://github.com/openapi-generators/openapi-python-client

holado-0.6.0/src/holado_api/server/README.txt

@@ -0,0 +1,7 @@
+Article comparing 20 frameworks:
+	https://zuplo.com/blog/2024/11/04/top-20-python-api-frameworks-with-openapi
+
+For API-First developement with OpenAPI:
+	Connexion
+	With FastAPI: https://medium.com/@georgedimitropulos/generate-python-fastapi-server-from-openapi-file-099bfa944d3b
+	PyAPI: https://pyapi-server.readthedocs.io/en/latest/

holado-0.6.0/src/holado_api/tests/behave/steps/server/connexion_server_steps.py

@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+# from holado.common.context.session_context import SessionContext
+# from holado_test.behave.behave import *  # @UnusedWildImport
+# import logging
+# from holado_test.scenario.step_tools import StepTools
+# from holado_test.behave.scenario.behave_step_tools import BehaveStepTools
+# from holado_value.common.tables.converters.value_table_converter import ValueTableConverter
+# from holado_core.common.exceptions.technical_exception import TechnicalException
+# from connexion.apps.flask import FlaskApp
+#
+# logger = logging.getLogger(__name__)
+#
+#
+# def __get_session_context():
+#     return SessionContext.instance()
+#
+# def __get_scenario_context():
+#     return __get_session_context().get_scenario_context()
+#
+# def __get_text_interpreter():
+#     return __get_scenario_context().get_text_interpreter()
+#
+# def __get_variable_manager():
+#     return __get_scenario_context().get_variable_manager()
+#
+#
+#
+# @Given(r"(?P<var_name>{Variable}) = new Connexion API server")
+# def step_impl(context, var_name):
+#     var_name = StepTools.evaluate_variable_name(var_name)
+#     table = BehaveStepTools.get_step_table(context)
+#
+#     kwargs = ValueTableConverter.convert_name_value_table_2_json_object(table)
+#     if 'project_path' not in kwargs:
+#         raise TechnicalException("Parameter 'project_path' is required")
+#     if 'openapi_yaml_path' not in kwargs:
+#         raise TechnicalException("Parameter 'openapi_yaml_path' is required")
+#     if 'name' not in kwargs:
+#         kwargs['name'] = f"Connexion API server ({kwargs['project_path']})"
+#
+#     server = FlaskApp(**kwargs)
+#
+#     __get_variable_manager().register_variable(var_name, server)
+#
+#
+# @Step(r"start \(Connexion server: (?P<var_server>{Variable})\)")
+# def step_impl(context, var_server):
+#     server = StepTools.evaluate_variable_value(var_server)
+#
+#     #TODO: leverage automatic reloading of your application, you need to provide the application as an import string. In most cases, this can be achieved as follows:
+#     #     from pathlib import Path
+#     #     app.run(f"{Path(__file__).stem}:app")
+#     server.run()
+#
+# # @Step(r"stop \(Django server: (?P<var_server>{Variable})\)")
+# # def step_impl(context, var_server):
+# #     server = StepTools.evaluate_variable_value(var_server)
+# #     server.stop()
+#
+#
+

{holado-0.5.2 → holado-0.6.0}/src/holado_db/tools/db/clients/base/db_client.py

@@ -22,8 +22,6 @@ from holado_core.common.tools.tools import Tools
 from holado_core.common.tables.table import Table
 from holado_db.tools.db.query.base.query_builder import QueryBuilder
 from holado_data.data.generator.base import BaseGenerator
-from sql_metadata.keywords_lists import QueryType
-import sql_metadata
 from holado_python.standard_library.typing import Typing
 import pypika.queries
 
@@ -223,6 +221,9 @@ class DBClient(object):
         return res
     
     def is_query_type(self, sql_or_query, query_type):
+        from sql_metadata.keywords_lists import QueryType
+        import sql_metadata
+        
         if isinstance(sql_or_query, pypika.queries.QueryBuilder):
             sql = self.query_builder.to_sql(sql_or_query)
         else:

{holado-0.5.2 → holado-0.6.0}/src/holado_docker/sdk/docker/docker_client.py

@@ -43,39 +43,61 @@ class DockerClient(object):
     def client(self):
         return self.__client
     
-    def has_container(self, name, in_list = True):
+    def has_container(self, name, in_list=True, reset_if_removed=True):
         # Note: Even if name exists in __containers, it is possible that the container has been removed
         if in_list:
             c = self.__get_container_from_list(name)
             res = c is not None
+            
+            if reset_if_removed and not res and name in self.__containers:
+                del self.__containers[name]
         else:
             res = name in self.__containers
+            
+            if reset_if_removed and res:
+                if self.__containers[name].status == "removed":
+                    del self.__containers[name]
+                res = False
         return res
     
-    def get_container(self, name):
+    def get_container(self, name, reset_if_removed=True):
+        # Reset container if removed
+        if reset_if_removed and name in self.__containers:
+            if self.__containers[name].status == "removed":
+                del self.__containers[name]
+        
+        # Get container from list if needed
         if name not in self.__containers:
             c = self.__get_container_from_list(name)
             if c:
                 self.__containers[name] = DockerContainer(self, c)
+        
         return self.__containers.get(name)
     
     def get_all_container_names(self):
-        for _ in range(3):
+        res = []
+        for c in self.__client.containers.list(all=True):
             try:
-                return [c.name for c in self.__client.containers.list(all=True)]
+                c_name = c.name
             except docker.errors.NotFound:
-                pass
+                # Container 'c' doesn't exist anymore
+                continue
+            
+            res.append(c_name)
+        return res
     
     def __get_container_from_list(self, name):
         res = None
-        for _ in range(3):
+        for c in self.__client.containers.list(all=True):
             try:
-                for c in self.__client.containers.list(all=True):
-                    if c.name == name:
-                        res = c
-                        break
+                c_name = c.name
             except docker.errors.NotFound:
-                pass
+                # Container 'c' doesn't exist anymore
+                continue
+            
+            if c_name == name:
+                res = c
+                break
         return res
     
     def has_volume(self, name, in_list = False):
@@ -133,17 +155,35 @@ class DockerClient(object):
                 time.sleep(1)
                 if container.status == "running":
                     break
-        if container.status != "running":
-            raise TechnicalException("Failed to run container of name '{}' (status: {})".format(name, container.status))
-        else:
-            if Tools.do_log(logger, logging.DEBUG):
-                logger.debug(f"Run docker container '{name}' with image '{image}' and arguments {kwargs}{', and wait running status' if wait_running else ''}")
+            if container.status != "running":
+                raise TechnicalException("Failed to run container of name '{}' (status: {})".format(name, container.status))
+        if Tools.do_log(logger, logging.DEBUG):
+            logger.debug(f"Run docker container '{name}' with image '{image}' and arguments {kwargs}{', and wait running status' if wait_running else ''}")
             
         # Set properties
         container.auto_stop = auto_stop
         
         return container
     
+    def restart_container(self, name, wait_running=True, **kwargs):
+        if Tools.do_log(logger, logging.DEBUG):
+            logger.debug(f"Restarting docker container '{name}' with arguments {kwargs}{', and waiting running status' if wait_running else ''}")
+        container = self.get_container(name)
+        if not container:
+            raise FunctionalException("Container of name '{}' doesn't exist")
+        
+        container.restart(**kwargs)
+        
+        if wait_running:
+            for _ in range(120):
+                time.sleep(1)
+                if container.status == "running":
+                    break
+            if container.status != "running":
+                raise TechnicalException("Failed to restart container of name '{}' (status: {})".format(name, container.status))
+        if Tools.do_log(logger, logging.DEBUG):
+            logger.debug(f"Restarted docker container '{name}' with arguments {kwargs}{', and waited running status' if wait_running else ''}")
+    
     def start_container(self, name, wait_running=True, **kwargs):
         if Tools.do_log(logger, logging.DEBUG):
             logger.debug(f"Starting docker container '{name}' with arguments {kwargs}{', and waiting running status' if wait_running else ''}")
@@ -158,11 +198,10 @@ class DockerClient(object):
                 time.sleep(1)
                 if container.status == "running":
                     break
-        if container.status != "running":
-            raise TechnicalException("Failed to run container of name '{}' (status: {})".format(name, container.status))
-        else:
-            if Tools.do_log(logger, logging.DEBUG):
-                logger.debug(f"Started docker container '{name}' with arguments {kwargs}{', and waited running status' if wait_running else ''}")
+            if container.status != "running":
+                raise TechnicalException("Failed to start container of name '{}' (status: {})".format(name, container.status))
+        if Tools.do_log(logger, logging.DEBUG):
+            logger.debug(f"Started docker container '{name}' with arguments {kwargs}{', and waited running status' if wait_running else ''}")
         
     def stop_container(self, name):
         if Tools.do_log(logger, logging.DEBUG):
@@ -215,7 +254,7 @@ class DockerContainer(DeleteableObject):
         
     @property
     def container(self):
-        self.__container
+        return self.__container
         
     @property
     def status(self):
@@ -233,6 +272,9 @@ class DockerContainer(DeleteableObject):
     def auto_stop(self, auto_stop):
         self.__auto_stop = auto_stop
     
+    def restart(self, **kwargs):
+        return self.__container.restart(**kwargs)
+    
     def start(self, **kwargs):
         return self.__container.start(**kwargs)
     

holado-0.6.0/src/holado_docker/tests/behave/steps/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+
+from .tools.docker_controller.client_steps import *