holado 0.2.7__py3-none-any.whl → 0.3.0__py3-none-any.whl

holado_core/tests/behave/steps/common/tables_steps.py

@@ -37,6 +37,8 @@ from holado_core.common.tools.converters.converter import Converter
 from holado_test.behave.scenario.behave_step_tools import BehaveStepTools
 from holado_value.common.tables.converters.value_table_converter import ValueTableConverter
 from holado_python.standard_library.typing import Typing
+from holado_value.common.tables.value_table_manager import ValueTableManager
+from holado_core.common.tables.converters.table_converter import TableConverter
 
 logger = logging.getLogger(__name__)
 
@@ -110,6 +112,21 @@ def step_impl(context, var_name, obj_str):
     
     __get_variable_manager().register_variable(var_name, table)
 
+@Step(r"(?P<var_name>{Variable}) = convert table with header (?P<table>{Variable}) to list of dictionary(?: \((?P<as_generator_str>as generator)\))?")
+def step_impl(context, var_name, table, as_generator_str):
+    var_name = StepTools.evaluate_variable_name(var_name)
+    table = StepTools.evaluate_scenario_parameter(table)
+    as_generator = as_generator_str is not None
+    if not TableManager.is_table_with_header(table):
+        raise TechnicalException(f"Table must be a table with header (obtained type: {Typing.get_object_class_fullname(table)})")
+    
+    if ValueTableManager.is_value_table(table):
+        res = ValueTableConverter.convert_table_with_header_to_dict_list(table, as_generator=as_generator)
+    else:
+        res = TableConverter.convert_table_with_header_to_dict_list(table, as_generator=as_generator)
+    
+    __get_variable_manager().register_variable(var_name, res)
+
 @Step(r'(?P<var_name>{Variable}) = convert object (?P<obj_str>{Variable}) to name/value table')
 def step_impl(context, var_name, obj_str):
     var_name = StepTools.evaluate_variable_name(var_name)
@@ -256,7 +273,7 @@ def step_impl(context, var_name, table_varname):
     table_new_columns = BehaveStepTools.convert_step_table_2_value_table_with_header(context.table, do_eval_once=False)
 
     res_table = copy.copy(table)
-    is_res_value_table = isinstance(res_table, ValueTable) or isinstance(res_table, ValueTableWithHeader)
+    is_res_value_table = ValueTableManager.is_value_table(res_table)
     
     res_col_indexes = res_table.get_column_indexes_by_string_content()
     
@@ -292,7 +309,7 @@ def step_impl(context, var_name, table_varname):
     table_new_rows = BehaveStepTools.convert_step_table_2_value_table_with_header(context.table)
 
     res_table = copy.copy(table)
-    is_res_value_table = isinstance(res_table, ValueTable) or isinstance(res_table, ValueTableWithHeader)
+    is_res_value_table = ValueTableManager.is_value_table(res_table)
     
     # Verify tables headers
     header_comp = StringTableRowComparator()
@@ -317,6 +334,7 @@ def step_impl(context, var_name, table_varname, col_name):
     col_name = StepTools.evaluate_scenario_parameter(col_name)
 
     res_table = copy.copy(table)
+    is_value_table = ValueTableManager.is_value_table(res_table)
     col = res_table.get_column(name=col_name)
     
     tr_indexes = table_replace.get_column_indexes_by_string_content()
@@ -328,8 +346,10 @@ def step_impl(context, var_name, table_varname, col_name):
             cell_rep_value = row_replace[tr_indexes["Replace Value"]] if "Replace Value" in tr_indexes else None
             
             if cell_cond_value is not None:
+                # If condition value is 'DEFAULT' or cell value is equal to condition value
                 cond = (cell_cond_value.content_type == ValueTypes.Symbol and cell_cond_value.content == 'DEFAULT'
-                        or cell_cond_value.value_type != ValueTypes.NotApplicable and cell.content == cell_cond_value.value)
+                        or cell_cond_value.value_type != ValueTypes.NotApplicable 
+                            and (is_value_table and cell.value == cell_cond_value.value or not is_value_table and cell.content == cell_cond_value.value) )
             elif cell_cond_expr is not None:
                 cond = tcell_comparator.equals(cell, cell_cond_expr, raise_exception = False)
             else:
@@ -339,7 +359,10 @@ def step_impl(context, var_name, table_varname, col_name):
                 if cell_rep_value.value_type != ValueTypes.NotApplicable:
                     # Note: As 'col' is created with a copy of 'res_table' cells, 'cell' is also a 'res_table' cell.
                     #       Thus, modifying 'cell' is modifying the cell in 'res_table'
-                    cell.content = cell_rep_value.value
+                    if is_value_table:
+                        cell.value = cell_rep_value.value
+                    else:
+                        cell.content = cell_rep_value.value
                     break
                 else:
                     continue

holado_data/data/generator/generator_manager.py

@@ -0,0 +1,39 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import logging
+from holado_core.common.tools.converters.converter import Converter
+from holado_data.data.generator.base import BaseGenerator
+
+logger = logging.getLogger(__name__)
+
+
+class GeneratorManager(object):
+    """
+    Manage generators.
+    """
+    
+    @classmethod
+    def convert_to_list(cls, generator_or_list):
+        if generator_or_list is None:
+            return None
+        elif Converter.is_list(generator_or_list):
+            return generator_or_list
+        elif isinstance(generator_or_list, Generator):
+            return Converter.to_list(generator_or_list)
+        else:
+            raise TechnicalException(f"Unexpected generator (or list) type '{Typing.get_object_class_fullname(generator_or_list)}'")
+
+
+
+

holado_data/tests/behave/steps/data/generator_steps.py

@@ -15,7 +15,7 @@
 
 from holado_test.scenario.step_tools import StepTools
 from holado.common.context.session_context import SessionContext
-from holado_test.behave.behave import *
+from holado_test.behave.behave import *  # @UnusedWildImport
 from holado_scripting.text.interpreter.text_interpreter import TextInterpreter
 from holado_scripting.common.tools.variable_manager import VariableManager
 from holado_test.common.context.scenario_context import ScenarioContext

holado_data/tests/behave/steps/tools/utils_steps.py

@@ -15,12 +15,11 @@
 
 from holado_test.scenario.step_tools import StepTools
 from holado.common.context.session_context import SessionContext
-from holado_test.behave.behave import *
+from holado_test.behave.behave import *  # @UnusedWildImport
 from holado_scripting.text.interpreter.text_interpreter import TextInterpreter
 from holado_scripting.common.tools.variable_manager import VariableManager
 from holado_test.common.context.scenario_context import ScenarioContext
 import logging
-from holado_data.data.generator.python_generator import PythonGenerator
 import math
 from holado_core.common.exceptions.technical_exception import TechnicalException
 

holado_db/tests/behave/steps/tools/db/db_client_steps.py

@@ -15,7 +15,7 @@
 
 from holado_test.scenario.step_tools import StepTools
 from holado.common.context.session_context import SessionContext
-from holado_test.behave.behave import *
+from holado_test.behave.behave import *  # @UnusedWildImport
 from holado_core.common.exceptions.technical_exception import TechnicalException
 from holado_scripting.text.interpreter.text_interpreter import TextInterpreter
 from holado_scripting.common.tools.variable_manager import VariableManager

holado_db/tests/behave/steps/tools/db/postgresql_client_steps.py

@@ -14,7 +14,7 @@
 
 
 from holado.common.context.session_context import SessionContext
-from holado_test.behave.behave import *
+from holado_test.behave.behave import *  # @UnusedWildImport
 from holado_scripting.text.interpreter.text_interpreter import TextInterpreter
 from holado_scripting.common.tools.variable_manager import VariableManager
 from holado_test.common.context.scenario_context import ScenarioContext

holado_db/tests/behave/steps/tools/db/sqlite_client_steps.py

@@ -14,7 +14,7 @@
 
 
 from holado.common.context.session_context import SessionContext
-from holado_test.behave.behave import *
+from holado_test.behave.behave import *  # @UnusedWildImport
 from holado_scripting.text.interpreter.text_interpreter import TextInterpreter
 from holado_scripting.common.tools.variable_manager import VariableManager
 from holado_test.common.context.scenario_context import ScenarioContext

holado_db/tools/db/clients/base/db_client.py

@@ -21,6 +21,11 @@ import abc
 from holado_core.common.tools.tools import Tools
 from holado_core.common.tables.table import Table
 from holado_db.tools.db.query.base.query_builder import QueryBuilder
+from holado_data.data.generator.base import BaseGenerator
+from sql_metadata.keywords_lists import QueryType
+import sql_metadata
+from holado_python.standard_library.typing import Typing
+import pypika.queries
 
 logger = logging.getLogger(__name__)
 
@@ -77,21 +82,49 @@ class DBClient(object):
     def _verify_is_connected(self):
         if not self.is_connected:
             raise FunctionalException(f"DB Client '{self.name}' is not connected")
+    
+    def is_query_type(self, sql_or_query, query_type):
+        if isinstance(sql_or_query, pypika.queries.QueryBuilder):
+            sql = self.query_builder.to_sql(sql_or_query)
+        else:
+            sql = sql_or_query
+        if not isinstance(sql, str):
+            raise TechnicalException(f"Unexpected type '{Typing.get_object_class_fullname(sql)}' for parameter sql_or_query")
+        
+        if isinstance(query_type, str):
+            query_type = QueryType[query_type.upper()]
+        if not isinstance(query_type, QueryType):
+            raise TechnicalException(f"Unmanage query_type of type '{Typing.get_object_class_fullname(query_type)}'")
         
+        p = sql_metadata.Parser(sql)
+        return p.query_type == query_type
+    
     def execute_query(self, query, *args, **kwargs):
         sql = self.query_builder.to_sql(query)
-        return self.execute(sql, *args, **kwargs)
+        
+        # Force do_commit to False for select queries
+        do_commit = kwargs.pop('do_commit', None)
+        
+        return self.execute(sql, *args, do_commit=do_commit, **kwargs)
         
     def execute(self, sql, *args, **kwargs):
-        # Manage commit & auto commit
-        do_commit = None
-        if 'do_commit' in kwargs:
-            do_commit = kwargs.pop('do_commit')
-        if do_commit is None:
-            do_commit = self.__auto_commit
+        # Manage specific parameters
+        if self.is_query_type(sql, 'select'):
+            do_commit = False
+        else:
+            do_commit = kwargs.pop('do_commit', None)
+            if do_commit is None:
+                do_commit = self.__auto_commit
+        result_as_dict_list = kwargs.pop('result_as_dict_list', False)
+        as_generator = kwargs.pop('as_generator', False)
+        if as_generator and do_commit:
+            raise TechnicalException(f"'do_commit=True' and 'as_generator=True' are incompatible")
+        if as_generator and not result_as_dict_list:
+            raise TechnicalException(f"'as_generator=True' is possible only with 'result_as_dict_list=True'")
             
         self._verify_is_connected()
         
+        # Execute
         if Tools.do_log(logger, logging.DEBUG):
             logger.debug(f"[{self.name}] Executing SQL [{sql}] with parameters [{args if args else ''}{kwargs if kwargs else ''}]...")
         try:
@@ -105,24 +138,41 @@ class DBClient(object):
             self.rollback()
             raise TechnicalException(f"[{self.name}] Error while executing SQL [{sql}] (args: {args} ; kwargs: {kwargs})") from exc
         
+        # Get result
         if self.cursor.description:
             field_names = [field[0] for field in self.cursor.description]
             
-            res = TableWithHeader()
-            res.header = TableRow(cells_content=field_names)
-            
-            row_values = self.cursor.fetchone()
-            while row_values:
-                res.add_row(cells_content=row_values)
+            if result_as_dict_list:
+                class Cursor2DictGenerator(BaseGenerator):
+                    def __init__(self, field_names, cursor):
+                        super().__init__(name="DB cursor to dict generator")
+                        self.__field_names = field_names
+                        self.__cursor = cursor
+                        self.__cursor_iter = iter(self.__cursor)
+                    
+                    def __next__(self):
+                        row_values = next(self.__cursor_iter)
+                        return dict(zip(self.__field_names, row_values))
+                
+                gen = Cursor2DictGenerator(field_names, self.cursor)
+                if as_generator:
+                    res = gen
+                else:
+                    res = [e for e in gen]
+            else:
+                res = TableWithHeader()
+                res.header = TableRow(cells_content=field_names)
+                
                 row_values = self.cursor.fetchone()
+                while row_values:
+                    res.add_row(cells_content=row_values)
+                    row_values = self.cursor.fetchone()
         elif self.cursor.rowcount > 0:
             res = self.cursor.rowcount
         else:
             res = None
         
-        if Tools.do_log(logger, logging.DEBUG):
-            logger.debug(f"[{self.name}] Executed SQL [{sql}] with parameters [{args if args else ''}{kwargs if kwargs else ''}] => [{res}]")
-        self.__log_sql_result(res)
+        self.__log_sql_result(res, message=f"Executed SQL [{sql}] with parameters [{args if args else ''}{kwargs if kwargs else ''}]")
         
         # Manage commit
         if do_commit:
@@ -137,17 +187,19 @@ class DBClient(object):
     def _get_base_exception_type(self):
         raise NotImplementedError()
     
-    def __log_sql_result(self, sql_result, limit_rows=10):
+    def __log_sql_result(self, sql_result, message="SQL result", limit_rows=10):
         if Tools.do_log(logger, logging.DEBUG):
             res_str = self.__represent_sql_result(sql_result, limit_rows=limit_rows)
             if '\n' in res_str:
-                logger.debug(f"[{self.name}] SQL result:\n{Tools.indent_string(4, res_str)}")
+                logger.debug(f"[{self.name}] {message}:\n{Tools.indent_string(4, res_str)}")
             else:
-                logger.debug(f"[{self.name}] SQL result: {res_str}")
+                logger.debug(f"[{self.name}] {message} => {res_str}")
     
     def __represent_sql_result(self, sql_result, limit_rows = 10):
         if isinstance(sql_result, Table):
             return sql_result.represent(limit_rows=limit_rows)
+        elif isinstance(sql_result, list) and limit_rows > 0 and len(sql_result) > limit_rows:
+            return str(sql_result[:limit_rows])[:-1] + f", ...({len(sql_result)-limit_rows})]"
         else:
             return str(sql_result)
     
@@ -168,31 +220,32 @@ class DBClient(object):
         query, values = self.query_builder.insert(table_name, data)
         return self.execute_query(query, *values, do_commit=do_commit)
     
-    def update(self, table_name, data: dict, where_data: dict, do_commit=None):
+    def update(self, table_name, data: dict, where_data: dict=None, where_compare_data: list=None, do_commit=None):
         """
         Update given data.
         Parameters 'data' and 'where_data' have to be dictionaries with keys equal to table column names.
         """
-        query, values = self.query_builder.update(table_name, data, where_data)
+        query, values = self.query_builder.update(table_name, data, where_data=where_data, where_compare_data=where_compare_data)
         return self.execute_query(query, *values, do_commit=do_commit)
     
-    def select(self, table_name, where_data: dict=None, sql_return="*"):
+    def select(self, table_name, where_data: dict=None, where_compare_data: list=None, sql_return="*", **kwargs):
         """
         Select by filtering on given where data.
-        Parameter 'where_data' has to be a dictionary with keys equal to table column names.
+        @param where_data: dictionary of (field_name, value) for simple where clauses.
+        @param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
         """
-        query, values = self.query_builder.select(table_name, where_data, sql_return)
-        return self.execute_query(query, *values, do_commit=False)
+        query, values = self.query_builder.select(table_name, where_data=where_data, where_compare_data=where_compare_data, sql_return=sql_return)
+        return self.execute_query(query, *values, do_commit=False, **kwargs)
     
-    def delete(self, table_name, where_data: dict=None, do_commit=None):
+    def delete(self, table_name, where_data: dict=None, where_compare_data: list=None, do_commit=None):
         """
         Delete by filtering on given where data.
         Parameter 'where_data' has to be a dictionary with keys equal to table column names.
         """
-        query, values = self.query_builder.delete(table_name, where_data)
+        query, values = self.query_builder.delete(table_name, where_data=where_data, where_compare_data=where_compare_data)
         return self.execute_query(query, *values, do_commit=do_commit)
     
-    def set_or_update_json_key_value(self, table_name, field_name, json_key, json_value, where_data):
+    def set_or_update_json_key_value(self, table_name, field_name, json_key, json_value, where_data: dict=None, where_compare_data: list=None):
         """
         Set or update a json field with key=value.
         """

holado_db/tools/db/clients/postgresql/postgresql_client.py

@@ -15,6 +15,8 @@ import logging
 from holado_db.tools.db.clients.base.db_client import DBClient
 from holado_core.common.tables.table_with_header import TableWithHeader
 from holado_core.common.tools.tools import Tools
+import json
+from pypika.terms import Function, LiteralValue
 
 logger = logging.getLogger(__name__)
 
@@ -27,6 +29,12 @@ except Exception as exc:
     with_psycopg = False
 
 
+# from pypika import Function
+class JsonbSet(Function):
+    def __init__(self, name, *args, **kwargs):
+        super().__init__('jsonb_set', LiteralValue(name), *args, **kwargs)
+
+
 class PostgreSQLClient(DBClient):
     @classmethod
     def is_available(cls):
@@ -48,18 +56,20 @@ class PostgreSQLClient(DBClient):
         result = self.execute(f"SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename = '{table_name}'")
         return result is not None
         
-    def set_or_update_json_key_value(self, table_name, field_name, json_key, json_value, where_data: dict=None):
+    def set_or_update_json_key_value(self, table_name, field_name, json_key, json_value, where_data: dict=None, where_compare_data: list=None):
         # Help on JSON column: https://www.databasestar.com/postgresql-json/#How_to_Update_JSON_Data_in_PostgreSQL
-        result = self.select(table_name, where_data, sql_return=field_name)
+        result = self.select(table_name, where_data=where_data, where_compare_data=where_compare_data, sql_return=field_name)
         is_set = isinstance(result, TableWithHeader) and result[0][0].content is not None
         if not is_set:
-            self.update(table_name, {field_name: f'{{"{json_key}":"{json_value}"}}'}, where_data, do_commit=True)
+            self.update(table_name, {field_name: f'{{"{json_key}":"{json_value}"}}'}, where_data=where_data, where_compare_data=where_compare_data, do_commit=True)
         else:
-            result = self.select(table_name, where_data, sql_return=f"{field_name} ->> '{json_key}'")
-            is_key_set = isinstance(result, TableWithHeader) and result[0][0].content is not None
+            result = self.select(table_name, where_data=where_data, where_compare_data=where_compare_data, sql_return=f"{field_name}")
+            is_key_set = False
+            if isinstance(result, TableWithHeader) and result[0][0].content is not None:
+                is_key_set = json_key in result[0][0].content
             if is_key_set:
-                self.update(table_name, {field_name: f"JSONB_SET({field_name}, '{json_key}', '\"{json_value}\"')"}, where_data, do_commit=True)
+                self.update(table_name, {field_name: JsonbSet(f'{field_name}', f'{{"{json_key}"}}', f'"{json_value}"')}, where_data=where_data, where_compare_data=where_compare_data, do_commit=True)
             else:
-                self.update(table_name, {field_name: f'{field_name} || {{"{json_key}":"{json_value}"}}'}, where_data, do_commit=True)
+                self.update(table_name, {field_name: f'{field_name} || {{"{json_key}":"{json_value}"}}'}, where_data=where_data, where_compare_data=where_compare_data, do_commit=True)
     
     

holado_db/tools/db/query/base/query_builder.py

@@ -13,11 +13,33 @@
 
 import logging
 import abc
+from enum import Enum
 
 
 logger = logging.getLogger(__name__)
 
 
+class DBCompareOperator(str, Enum):
+    Different = ("!=", "Equal")
+    Equal = ("==", "Different")
+    Inferior = ("<", "SuperiorOrEqual")
+    InferiorOrEqual = ("<=", "Superior")
+    Superior = (">", "InferiorOrEqual")
+    SuperiorOrEqual = (">=", "Inferior")
+    In = ("in", "NotIn")
+    NotIn = ("not in", "In")
+    
+    def __new__(cls, value, not_name):
+        obj = str.__new__(cls, [value])
+        obj._value_ = value
+        obj.__not_name = not_name
+        return obj
+    
+    @property
+    def not_(self):
+        return DBCompareOperator[self.__not_name]
+    
+
 
 class QueryBuilder():
     """
@@ -41,31 +63,35 @@ class QueryBuilder():
     def db_client(self, client):
         self.__db_client = client
     
-    def select(self, table_name, where_data: dict=None, sql_return="*"):
+    def select(self, table_name, where_data: dict=None, where_compare_data: list=None, sql_return="*"):
         """
         Simple query & values builder of a select by filtering on given where data.
-        Parameter 'where_data' has to be a dictionary with keys equal to table column names.
+        @param where_data: dictionary of (field_name, value) for simple where clauses.
+        @param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
         """
         raise NotImplementedError
 
     def insert(self, table_name, data: dict):
         """
         Simple query & values builder of an insert of given data.
-        Parameter 'data' has to be a dictionary with keys equal to table column names.
+        @param data: data to insert as dictionary of (field_name, value).
         """
         raise NotImplementedError
     
-    def update(self, table_name, data: dict, where_data: dict):
+    def update(self, table_name, data: dict, where_data: dict=None, where_compare_data: list=None):
         """
         Simple query & values builder of an update of given data.
-        Parameters 'data' and 'where_data' have to be dictionaries with keys equal to table column names.
+        @param data: data to update as dictionary of (field_name, value).
+        @param where_data: dictionary of (field_name, value) for simple where clauses.
+        @param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
         """
         raise NotImplementedError
     
-    def delete(self, table_name, where_data: dict=None):
+    def delete(self, table_name, where_data: dict=None, where_compare_data: list=None):
         """
         Simple query & values builder of a delete by filtering on given where data.
-        Parameter 'where_data' has to be a dictionary with keys equal to table column names.
+        @param where_data: dictionary of (field_name, value) for simple where clauses.
+        @param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
         """
         raise NotImplementedError
     
@@ -75,12 +101,37 @@ class QueryBuilder():
         """
         raise NotImplementedError
     
+    def where_compare(self, query, values, *, field_name=None, operator:DBCompareOperator=None, value=None, list_of_field_operator_value=None):
+        """
+        Add where clause with field value comparison to current couple (query, values), and return a new couple (query, values).
+        To add multiple clauses in a single call, define list_of_field_operator_value as a list of tuples (field_name, operator, value).
+        """
+        if list_of_field_operator_value is not None:
+            res_query, res_values = query, values
+            for f_name, op, val in list_of_field_operator_value:
+                res_query, res_values = self.where_compare(res_query, res_values, field_name=f_name, operator=op, value=val)
+            return res_query, res_values
+        else:
+            raise NotImplementedError
+        
     def where_in(self, query, values, field_name, field_values, not_in=False):
         """
         Add where in clause to current couple (query, values), and return a new couple (query, values).
         """
         raise NotImplementedError
         
+    def where_is_null(self, query, values, field_name, is_not_null=False):
+        """
+        Add where is null clause to current couple (query, values), and return a new couple (query, values).
+        """
+        raise NotImplementedError
+    
+    def where_json_value(self, query, values, field_name, key, value, as_text_value=False):
+        """
+        Add where clause on a json field key value to current couple (query, values), and return a new couple (query, values).
+        """
+        raise NotImplementedError
+        
     
     def to_sql(self, query):
         raise NotImplementedError