holado 0.13.22__py3-none-any.whl

holado_db/tools/db/clients/postgresql/postgresql_client.py

@@ -0,0 +1,84 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2023 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import logging
+from holado_db.tools.db.clients.base.db_client import DBClient
+from holado_core.common.tables.table_with_header import TableWithHeader
+from holado_core.common.tools.tools import Tools
+from holado_db.tools.db.clients.postgresql.postgresql_audit import PostgreSQLTriggerAuditManager
+from holado_db.tools.db.query.pypika.pypika_query_builder import PypikaQueryBuilder
+from holado_core.common.exceptions.technical_exception import TechnicalException
+
+logger = logging.getLogger(__name__)
+
+try:
+    import psycopg
+    with_psycopg = True
+except Exception as exc:
+    if Tools.do_log(logger, logging.DEBUG):
+        logger.debug(f"PostgreSQLClient is not available. Initialization failed on error: {exc}")
+    with_psycopg = False
+
+if PypikaQueryBuilder.is_available():
+    from pypika.terms import Function, LiteralValue
+    
+    # from pypika import Function
+    class JsonbSet(Function):
+        def __init__(self, name, *args, **kwargs):
+            super().__init__('jsonb_set', LiteralValue(name), *args, **kwargs)
+
+
+class PostgreSQLClient(DBClient):
+    @classmethod
+    def is_available(cls):
+        return with_psycopg
+
+    def __init__(self, name, connect_kwargs):
+        super().__init__(name if name else 'PostgreSQL', connect_kwargs)
+        
+    def _new_audit_manager(self):
+        return PostgreSQLTriggerAuditManager("Audit with triggers", self)
+        
+    def _get_base_exception_type(self):
+        return psycopg.Error
+    
+    def _connect(self, **kwargs):
+        return psycopg.connect(**kwargs)
+        
+    def _get_sql_placeholder(self):
+        return "%s"
+        
+    def exist_table(self, table_name):
+        result = self.execute(f"SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename = '{table_name}'")
+        return result is not None
+        
+    def set_or_update_json_key_value(self, table_name, field_name, json_key, json_value, where_data: dict=None, where_compare_data: list=None):
+        # Help on JSON column: https://www.databasestar.com/postgresql-json/#How_to_Update_JSON_Data_in_PostgreSQL
+        result = self.select(table_name, where_data=where_data, where_compare_data=where_compare_data, sql_return=field_name)
+        is_set = isinstance(result, TableWithHeader) and result[0][0].content is not None
+        if not is_set:
+            self.update(table_name, {field_name: f'{{"{json_key}":"{json_value}"}}'}, where_data=where_data, where_compare_data=where_compare_data, do_commit=True)
+        else:
+            result = self.select(table_name, where_data=where_data, where_compare_data=where_compare_data, sql_return=f"{field_name}")
+            is_key_set = False
+            if isinstance(result, TableWithHeader) and result[0][0].content is not None:
+                is_key_set = json_key in result[0][0].content
+            if is_key_set:
+                if not PypikaQueryBuilder.is_available():
+                    # Note: This case cannot appear currently (04/08/2025), but can appear if another QueryBuilder is managed that is not using pypika
+                    raise TechnicalException(f"Missing dependence: pypika")
+                self.update(table_name, {field_name: JsonbSet(f'{field_name}', f'{{"{json_key}"}}', f'"{json_value}"')}, where_data=where_data, where_compare_data=where_compare_data, do_commit=True)
+            else:
+                self.update(table_name, {field_name: f'{field_name} || {{"{json_key}":"{json_value}"}}'}, where_data=where_data, where_compare_data=where_compare_data, do_commit=True)
+    
+    

holado_db/tools/db/clients/sqlite/sqlite_audit.py

@@ -0,0 +1,70 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2023 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import logging
+from holado.common.handlers.undefined import default_value
+from holado_db.tools.db.clients.base.db_audit import TriggerAuditManager
+
+logger = logging.getLogger(__name__)
+
+
+class SQLite3TriggerAuditManager(TriggerAuditManager):
+    def __init__(self, name, db_client):
+        super().__init__(name, db_client)
+    
+    def _get_audit_table_sql_create(self, audit_table_name=default_value):
+        audit_table_name = self._get_audit_table_name(audit_table_name)
+        return f"""CREATE TABLE {audit_table_name} (
+                id INTEGER PRIMARY KEY,
+                table_name TEXT,
+                record_id INTEGER,
+                operation_type TEXT,
+                changed_at TEXT,
+                previous_values TEXT,
+                new_values TEXT
+            )"""
+    
+    def _get_drop_trigger_sql_of_audit_table_operation(self, table_name, operation_type, audit_table_name):
+        trigger_name = self.get_trigger_name(table_name, operation_type)
+        return f"DROP TRIGGER IF EXISTS {trigger_name}"
+    
+    def _get_create_trigger_sql_to_audit_table_operation(self, table_name, operation_type, audit_table_name):
+        audit_table_name = self._get_audit_table_name(audit_table_name)
+        trigger_name = self.get_trigger_name(table_name, operation_type)
+        trigger_insert_sql = self.__get_trigger_insert_sql(table_name, operation_type, audit_table_name)
+        return f"""CREATE TRIGGER {trigger_name} AFTER {operation_type.upper()} ON {table_name}
+                BEGIN
+                    {trigger_insert_sql}
+                END;
+                """
+    
+    def __get_trigger_insert_sql(self, table_name, operation_type, audit_table_name):
+        values = [f"'{table_name}'",
+                  "NEW.rowid" if operation_type in ['insert', 'update'] else "OLD.rowid",
+                  f"'{operation_type[0]}'",
+                  "DATETIME('now')",
+                  self.__get_row_version_json_object_sql(table_name, 'OLD') if operation_type in ['update', 'delete'] else "NULL",
+                  self.__get_row_version_json_object_sql(table_name, 'NEW') if operation_type in ['insert', 'update'] else "NULL",
+                 ]
+        
+        return f"""INSERT INTO {audit_table_name} (table_name, record_id, operation_type, changed_at, previous_values, new_values) 
+                   VALUES ({','.join(values)});"""
+    
+    def __get_row_version_json_object_sql(self, table_name, row_version):
+        col_names = self._db_client.get_table_column_names(table_name)
+        json_object_args = []
+        for cn in col_names:
+            json_object_args.append(f"'{cn}'")
+            json_object_args.append(f"{row_version}.{cn}")
+        return f"json_object({','.join(json_object_args)})"
+

holado_db/tools/db/clients/sqlite/sqlite_client.py

@@ -0,0 +1,60 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2023 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import logging
+from holado_db.tools.db.clients.base.db_client import DBClient
+from holado_core.common.exceptions.functional_exception import FunctionalException
+from holado_core.common.tools.tools import Tools
+from holado_db.tools.db.clients.sqlite.sqlite_audit import SQLite3TriggerAuditManager
+
+logger = logging.getLogger(__name__)
+
+try:
+    import sqlite3
+    with_sqlite3 = True
+except Exception as exc:
+    if Tools.do_log(logger, logging.DEBUG):
+        logger.debug(f"SQLite3Client is not available. Initialization failed on error: {exc}")
+    with_sqlite3 = False
+
+
+class SQLite3Client(DBClient):
+    @classmethod
+    def is_available(cls):
+        return with_sqlite3
+
+    def __init__(self, name, connect_kwargs):
+        super().__init__(name if name else 'SQLite3', connect_kwargs)
+        
+    def _new_audit_manager(self):
+        return SQLite3TriggerAuditManager("Audit with triggers", self)
+    
+    def _get_base_exception_type(self):
+        return sqlite3.Error
+        
+    def _connect(self, **kwargs):
+        try:
+            connect_kwargs = dict(kwargs)
+            database = connect_kwargs.pop("database")
+            return sqlite3.connect(database, **connect_kwargs)
+        except Exception as exc:
+            raise FunctionalException(f"Failed to connect with kwargs={kwargs}.\nUsage:\n{Tools.indent_string(4, sqlite3.connect.__doc__)}") from exc  # @UndefinedVariable
+        
+    def _get_sql_placeholder(self):
+        return "?"
+        
+    def exist_table(self, table_name):
+        result = self.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'")
+        return result is not None and result.nb_rows > 0
+        
+        

holado_db/tools/db/db_manager.py

@@ -0,0 +1,109 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import logging
+from holado_db.tools.db.clients.sqlite.sqlite_client import SQLite3Client
+from holado_db.tools.db.clients.postgresql.postgresql_client import PostgreSQLClient
+from holado_core.common.exceptions.technical_exception import TechnicalException
+from holado_core.common.handlers.abstracts.get_or_create import GetOrCreateVariableObject
+from holado_core.common.tools.tools import Tools
+from holado.common.context.session_context import SessionContext
+
+
+logger = logging.getLogger(__name__)
+
+
+
+class DBManager(GetOrCreateVariableObject):
+    """
+    Manage DB clients, agnostic to managed DB.
+    
+    It manages an instance by thread for DB types (ex: sqlite3) which clients don't support multithreading.
+    Note: Even if sqlite3 can support multithreading (with 'connect' parameter 'check_same_thread=False'),
+          current implementation of DBManager keeps sqlite3 default behavior.
+    """
+    
+    def __init__(self, name):
+        super().__init__(name if name else "DBManager")
+        
+        self.__func_query_manager = None
+        self.__db_info_by_db_type = {}
+    
+    def initialize(self, func_variable_manager, func_query_manager):
+        super().initialize(func_variable_manager)
+        self.__func_query_manager = func_query_manager
+        
+        if SQLite3Client.is_available():
+            self.register_db_client("sqlite3", SQLite3Client, do_support_multithreading=False)
+        else:
+            if Tools.do_log(logger, logging.DEBUG):
+                logger.debug("SQLite3 package is not installed, this type of DB is not available")
+        if PostgreSQLClient.is_available():
+            self.register_db_client("postgresql", PostgreSQLClient)
+        else:
+            if Tools.do_log(logger, logging.DEBUG):
+                logger.debug("PostgreSQL package is not installed, this type of DB is not available")
+    
+    @property
+    def __query_manager(self):
+        return self.__func_query_manager()
+    
+    def register_db_client(self, db_type, new_client_func, do_support_multithreading=True):
+        self.__db_info_by_db_type[db_type] = (new_client_func, do_support_multithreading)
+        if Tools.do_log(logger, logging.DEBUG):
+            logger.debug(f"Registered DB type '{db_type}'")
+    
+    def get_or_create(self, name, *args, **kwargs):
+        # Manager DB types that need a client by thread
+        db_type = self.__find_db_type(args, kwargs)
+        if db_type is not None and not self.__db_info_by_db_type[db_type][1]:
+            thread_db_manager = SessionContext.instance().multitask_manager.get_thread_context().db_manager
+            if thread_db_manager is not self:
+                return thread_db_manager.get_or_create(name, *args, **kwargs)
+            else:
+                # Prefix name by thread UID, since same variable manager is shared by all thread context DBManagers
+                thread_uid = SessionContext.instance().multitask_manager.get_thread_uid()
+                real_name = f"{thread_uid}-{name}"
+        else:
+            real_name = name
+        
+        return super().get_or_create(real_name, *args, **kwargs)
+    
+    def __find_db_type(self, args, kwargs):
+        if kwargs and 'db_type' in kwargs:
+            return kwargs['db_type']
+        elif args and len(args) > 0 and args[0] in self.__db_info_by_db_type:
+            return args[0]
+        else:
+            return None
+    
+    def _goc_new_object(self, name, db_type, connect_kwargs):
+        res = self.new_client(name, db_type, connect_kwargs)
+        res.connect()
+        return res
+    
+    def new_client(self, name, db_type, connect_kwargs):
+        if db_type in self.__db_info_by_db_type:
+            try:
+                res = self.__db_info_by_db_type[db_type][0](name, connect_kwargs)
+            except Exception as exc:
+                raise TechnicalException(f"Failed to create client for DB type '{db_type}' with connect parameters: {connect_kwargs}") from exc
+        else:
+            raise TechnicalException(f"Unmanaged DB type '{db_type}'")
+        
+        res.query_builder = self.__query_manager.new_default_query_builder(db_type, res)
+        return res
+        
+        
+        
+        

holado_db/tools/db/query/base/query_builder.py

@@ -0,0 +1,138 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import logging
+import abc
+from enum import Enum
+
+
+logger = logging.getLogger(__name__)
+
+
+class DBCompareOperator(str, Enum):
+    Different = ("!=", "Equal")
+    Equal = ("==", "Different")
+    Inferior = ("<", "SuperiorOrEqual")
+    InferiorOrEqual = ("<=", "Superior")
+    Superior = (">", "InferiorOrEqual")
+    SuperiorOrEqual = (">=", "Inferior")
+    In = ("in", "NotIn")
+    NotIn = ("not in", "In")
+    
+    def __new__(cls, value, not_name):
+        obj = str.__new__(cls, [value])
+        obj._value_ = value
+        obj.__not_name = not_name
+        return obj
+    
+    @property
+    def not_(self):
+        return DBCompareOperator[self.__not_name]
+    
+
+
+class QueryBuilder():
+    """
+    Generic Query builder.
+    """
+    __metaclass__ = abc.ABCMeta
+    
+    def __init__(self, name, db_client=None):
+        self.__name = name
+        self.__db_client = db_client
+    
+    @property
+    def name(self):
+        return self.__name
+    
+    @property
+    def db_client(self):
+        return self.__db_client
+    
+    @db_client.setter
+    def db_client(self, client):
+        self.__db_client = client
+    
+    def select(self, table_name, where_data: dict=None, where_compare_data: list=None, sql_return="*"):
+        """
+        Simple query & values builder of a select by filtering on given where data.
+        @param where_data: dictionary of (field_name, value) for simple where clauses.
+        @param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
+        """
+        raise NotImplementedError
+
+    def insert(self, table_name, data: dict):
+        """
+        Simple query & values builder of an insert of given data.
+        @param data: data to insert as dictionary of (field_name, value).
+        """
+        raise NotImplementedError
+    
+    def update(self, table_name, data: dict, where_data: dict=None, where_compare_data: list=None):
+        """
+        Simple query & values builder of an update of given data.
+        @param data: data to update as dictionary of (field_name, value).
+        @param where_data: dictionary of (field_name, value) for simple where clauses.
+        @param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
+        """
+        raise NotImplementedError
+    
+    def delete(self, table_name, where_data: dict=None, where_compare_data: list=None):
+        """
+        Simple query & values builder of a delete by filtering on given where data.
+        @param where_data: dictionary of (field_name, value) for simple where clauses.
+        @param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
+        """
+        raise NotImplementedError
+    
+    def where(self, query, values, where_data: dict):
+        """
+        Add where clauses to current couple (query, values), and return a new couple (query, values).
+        """
+        raise NotImplementedError
+    
+    def where_compare(self, query, values, *, field_name=None, operator:DBCompareOperator=None, value=None, list_of_field_operator_value=None):
+        """
+        Add where clause with field value comparison to current couple (query, values), and return a new couple (query, values).
+        To add multiple clauses in a single call, define list_of_field_operator_value as a list of tuples (field_name, operator, value).
+        """
+        if list_of_field_operator_value is not None:
+            res_query, res_values = query, values
+            for f_name, op, val in list_of_field_operator_value:
+                res_query, res_values = self.where_compare(res_query, res_values, field_name=f_name, operator=op, value=val)
+            return res_query, res_values
+        else:
+            raise NotImplementedError
+        
+    def where_in(self, query, values, field_name, field_values, not_in=False):
+        """
+        Add where in clause to current couple (query, values), and return a new couple (query, values).
+        """
+        raise NotImplementedError
+        
+    def where_is_null(self, query, values, field_name, is_not_null=False):
+        """
+        Add where is null clause to current couple (query, values), and return a new couple (query, values).
+        """
+        raise NotImplementedError
+    
+    def where_json_value(self, query, values, field_name, key, value, as_text_value=False):
+        """
+        Add where clause on a json field key value to current couple (query, values), and return a new couple (query, values).
+        """
+        raise NotImplementedError
+        
+    
+    def to_sql(self, query):
+        raise NotImplementedError
+