holado 0.2.8__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of holado might be problematic. Click here for more details.
- holado/common/handlers/undefined.py +7 -1
- {holado-0.2.8.dist-info → holado-0.3.0.dist-info}/METADATA +4 -1
- {holado-0.2.8.dist-info → holado-0.3.0.dist-info}/RECORD +106 -99
- holado_ais/ais/ais_messages.py +97 -6
- holado_ais/tests/behave/steps/ais/ais_manager_steps.py +1 -1
- holado_ais/tests/behave/steps/ais/ais_messages_steps.py +45 -6
- holado_binary/ipc/bit_series.py +3 -3
- holado_binary/tests/behave/steps/ipc/binary_steps.py +1 -1
- holado_binary/tests/behave/steps/ipc/bit_series_steps.py +4 -3
- holado_context/tests/behave/steps/private/common/context_steps.py +1 -1
- holado_core/common/resource/persisted_data_manager.py +13 -16
- holado_core/common/resource/resource_manager.py +10 -10
- holado_core/common/tables/converters/table_converter.py +47 -9
- holado_core/common/tables/table_manager.py +6 -7
- holado_core/common/tables/table_with_header.py +6 -0
- holado_core/tests/behave/steps/common/common_steps.py +2 -1
- holado_core/tests/behave/steps/common/config_steps.py +1 -1
- holado_core/tests/behave/steps/common/resource_steps.py +1 -1
- holado_core/tests/behave/steps/common/tables_steps.py +18 -2
- holado_data/data/generator/generator_manager.py +39 -0
- holado_data/tests/behave/steps/data/generator_steps.py +1 -1
- holado_data/tests/behave/steps/tools/utils_steps.py +1 -2
- holado_db/tests/behave/steps/tools/db/db_client_steps.py +1 -1
- holado_db/tests/behave/steps/tools/db/postgresql_client_steps.py +1 -1
- holado_db/tests/behave/steps/tools/db/sqlite_client_steps.py +1 -1
- holado_db/tools/db/clients/base/db_client.py +81 -28
- holado_db/tools/db/clients/postgresql/postgresql_client.py +17 -7
- holado_db/tools/db/query/base/query_builder.py +58 -7
- holado_db/tools/db/query/pypika/pypika_query_builder.py +73 -21
- holado_docker/tests/behave/steps/tools/docker_steps.py +1 -1
- holado_grpc/tests/behave/steps/api/grpc_client_steps.py +1 -1
- holado_grpc/tests/behave/steps/private/api/grpc_steps.py +1 -1
- holado_json/tests/behave/steps/ipc/json_steps.py +1 -1
- holado_keycloak/tests/behave/steps/tools/keycloak_client_steps.py +1 -1
- holado_multitask/tests/behave/steps/multiprocessing_steps.py +1 -1
- holado_multitask/tests/behave/steps/multithreading_steps.py +1 -1
- holado_protobuf/ipc/protobuf/types/google/protobuf.py +1 -1
- holado_protobuf/tests/behave/steps/ipc/protobuf_steps.py +1 -1
- holado_python/common/tools/datetime.py +31 -12
- holado_python/standard_library/socket/blocking_socket.py +37 -24
- holado_python/standard_library/socket/message_socket.py +11 -3
- holado_python/standard_library/socket/non_blocking_socket.py +24 -24
- holado_python/standard_library/socket/socket.py +132 -19
- holado_python/standard_library/ssl/resources/certificates/NOTES.txt +1 -1
- holado_python/standard_library/ssl/resources/certificates/rootCACert.pem +24 -0
- holado_python/standard_library/ssl/resources/certificates/tcpbin.crt +21 -0
- holado_python/standard_library/ssl/resources/certificates/tcpbin.key +28 -0
- holado_python/standard_library/ssl/ssl.py +138 -21
- holado_python/tests/behave/steps/convert_steps.py +1 -1
- holado_python/tests/behave/steps/iterable_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/csv_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/datetime_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/hashlib_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/multiprocessing_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/queue_steps.py +1 -1
- holado_python/tests/behave/steps/standard_library/socket_steps.py +132 -18
- holado_python/tests/behave/steps/standard_library/ssl_steps.py +87 -16
- holado_rabbitmq/tests/behave/steps/tools/rabbitmq_client_steps.py +48 -20
- holado_rabbitmq/tests/behave/steps/tools/rabbitmq_server_steps.py +1 -1
- holado_rabbitmq/tools/rabbitmq/rabbitmq_client.py +19 -13
- holado_rabbitmq/tools/rabbitmq/rabbitmq_manager.py +2 -29
- holado_redis/tests/behave/steps/tools/redis_client_steps.py +1 -1
- holado_rest/tests/behave/steps/api/rest_client_steps.py +1 -1
- holado_rest/tests/behave/steps/private/api/rest_steps.py +1 -1
- holado_s3/tests/behave/steps/private/tools/s3_steps.py +1 -1
- holado_s3/tests/behave/steps/tools/s3_client_steps.py +1 -1
- holado_s3/tests/behave/steps/tools/s3_server_steps.py +1 -1
- holado_scripting/tests/behave/steps/common/tools/variable_convert_steps.py +3 -2
- holado_scripting/tests/behave/steps/common/tools/variable_new_steps.py +1 -1
- holado_scripting/tests/behave/steps/common/tools/variable_steps.py +1 -1
- holado_scripting/tests/behave/steps/common/tools/variable_verify_steps.py +1 -1
- holado_scripting/tests/behave/steps/scenario/function_steps.py +1 -1
- holado_scripting/tests/behave/steps/scenario/if_steps.py +1 -1
- holado_scripting/tests/behave/steps/scenario/loop_steps.py +1 -1
- holado_sftp/tests/behave/steps/private/tools/sftp_steps.py +1 -1
- holado_sftp/tests/behave/steps/tools/sftp_client_steps.py +1 -1
- holado_sftp/tests/behave/steps/tools/sftp_server_steps.py +1 -1
- holado_swagger/tests/behave/steps/swagger_hub/mockserver_steps.py +1 -1
- holado_system/system/command/command.py +14 -9
- holado_system/tests/behave/steps/system/commands_steps.py +1 -1
- holado_system/tests/behave/steps/system/file_steps.py +1 -1
- holado_system/tests/behave/steps/system/system_steps.py +1 -1
- holado_test/scenario/step_tools.py +1 -1
- holado_test/scenario/tester_tools.py +6 -3
- holado_test/tests/behave/steps/scenario/exception_steps.py +1 -1
- holado_test/tests/behave/steps/scenario/scenario_steps.py +1 -1
- holado_test/tests/behave/steps/scenario/tester_steps.py +4 -4
- holado_value/common/tables/converters/value_table_converter.py +52 -8
- holado_value/common/tables/value_table_manager.py +0 -10
- holado_ws/tests/behave/steps/api/web_service_steps.py +1 -1
- holado_yaml/tests/behave/steps/yaml_steps.py +1 -1
- holado_yaml/yaml/yaml_manager.py +2 -2
- test_holado/features/NonReg/common/tables/table.feature +30 -24
- test_holado/features/NonReg/holado_ais/ais_message-bitarray_to_nmea.feature +1 -1
- test_holado/features/NonReg/holado_python/standard_library/socket/local_echo_server/socket_reset.feature +191 -0
- test_holado/features/NonReg/holado_python/standard_library/{socket_with_ssl.feature → socket/local_echo_server/socket_with_tls_and_verify.feature} +53 -30
- test_holado/features/NonReg/holado_python/standard_library/socket/local_echo_server/socket_with_tls_without_verify.feature +299 -0
- test_holado/features/NonReg/holado_python/standard_library/{socket.feature → socket/local_echo_server/socket_without_tls.feature} +2 -2
- test_holado/features/NonReg/holado_python/standard_library/socket/tcpbin.com/socket_with_mtls.feature +214 -0
- test_holado/features/NonReg/holado_python/standard_library/socket/tcpbin.com/socket_with_tls.feature +184 -0
- test_holado/features/NonReg/holado_python/standard_library/socket/tcpbin.com/socket_without_tls.feature +169 -0
- test_holado/features/NonReg/tools/RabbitMQ.feature +9 -9
- test_holado/features/NonReg/tools/RabbitMQ_steps.feature +8 -8
- test_holado/logging.conf +5 -3
- holado_core/common/transport/__init__.py +0 -0
- holado_core/common/transport/crc.py +0 -40
- {holado-0.2.8.dist-info → holado-0.3.0.dist-info}/WHEEL +0 -0
- {holado-0.2.8.dist-info → holado-0.3.0.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
|
|
2
|
+
#################################################
|
|
3
|
+
# HolAdo (Holistic Automation do)
|
|
4
|
+
#
|
|
5
|
+
# (C) Copyright 2021-2025 by Eric Klumpp
|
|
6
|
+
#
|
|
7
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
8
|
+
#
|
|
9
|
+
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
10
|
+
|
|
11
|
+
# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
|
|
12
|
+
#################################################
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
from holado_core.common.tools.converters.converter import Converter
|
|
16
|
+
from holado_data.data.generator.base import BaseGenerator
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class GeneratorManager(object):
|
|
22
|
+
"""
|
|
23
|
+
Manage generators.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def convert_to_list(cls, generator_or_list):
|
|
28
|
+
if generator_or_list is None:
|
|
29
|
+
return None
|
|
30
|
+
elif Converter.is_list(generator_or_list):
|
|
31
|
+
return generator_or_list
|
|
32
|
+
elif isinstance(generator_or_list, Generator):
|
|
33
|
+
return Converter.to_list(generator_or_list)
|
|
34
|
+
else:
|
|
35
|
+
raise TechnicalException(f"Unexpected generator (or list) type '{Typing.get_object_class_fullname(generator_or_list)}'")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
from holado_test.scenario.step_tools import StepTools
|
|
17
17
|
from holado.common.context.session_context import SessionContext
|
|
18
|
-
from holado_test.behave.behave import *
|
|
18
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
19
19
|
from holado_scripting.text.interpreter.text_interpreter import TextInterpreter
|
|
20
20
|
from holado_scripting.common.tools.variable_manager import VariableManager
|
|
21
21
|
from holado_test.common.context.scenario_context import ScenarioContext
|
|
@@ -15,12 +15,11 @@
|
|
|
15
15
|
|
|
16
16
|
from holado_test.scenario.step_tools import StepTools
|
|
17
17
|
from holado.common.context.session_context import SessionContext
|
|
18
|
-
from holado_test.behave.behave import *
|
|
18
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
19
19
|
from holado_scripting.text.interpreter.text_interpreter import TextInterpreter
|
|
20
20
|
from holado_scripting.common.tools.variable_manager import VariableManager
|
|
21
21
|
from holado_test.common.context.scenario_context import ScenarioContext
|
|
22
22
|
import logging
|
|
23
|
-
from holado_data.data.generator.python_generator import PythonGenerator
|
|
24
23
|
import math
|
|
25
24
|
from holado_core.common.exceptions.technical_exception import TechnicalException
|
|
26
25
|
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
from holado_test.scenario.step_tools import StepTools
|
|
17
17
|
from holado.common.context.session_context import SessionContext
|
|
18
|
-
from holado_test.behave.behave import *
|
|
18
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
19
19
|
from holado_core.common.exceptions.technical_exception import TechnicalException
|
|
20
20
|
from holado_scripting.text.interpreter.text_interpreter import TextInterpreter
|
|
21
21
|
from holado_scripting.common.tools.variable_manager import VariableManager
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
from holado.common.context.session_context import SessionContext
|
|
17
|
-
from holado_test.behave.behave import *
|
|
17
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
18
18
|
from holado_scripting.text.interpreter.text_interpreter import TextInterpreter
|
|
19
19
|
from holado_scripting.common.tools.variable_manager import VariableManager
|
|
20
20
|
from holado_test.common.context.scenario_context import ScenarioContext
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
from holado.common.context.session_context import SessionContext
|
|
17
|
-
from holado_test.behave.behave import *
|
|
17
|
+
from holado_test.behave.behave import * # @UnusedWildImport
|
|
18
18
|
from holado_scripting.text.interpreter.text_interpreter import TextInterpreter
|
|
19
19
|
from holado_scripting.common.tools.variable_manager import VariableManager
|
|
20
20
|
from holado_test.common.context.scenario_context import ScenarioContext
|
|
@@ -21,6 +21,11 @@ import abc
|
|
|
21
21
|
from holado_core.common.tools.tools import Tools
|
|
22
22
|
from holado_core.common.tables.table import Table
|
|
23
23
|
from holado_db.tools.db.query.base.query_builder import QueryBuilder
|
|
24
|
+
from holado_data.data.generator.base import BaseGenerator
|
|
25
|
+
from sql_metadata.keywords_lists import QueryType
|
|
26
|
+
import sql_metadata
|
|
27
|
+
from holado_python.standard_library.typing import Typing
|
|
28
|
+
import pypika.queries
|
|
24
29
|
|
|
25
30
|
logger = logging.getLogger(__name__)
|
|
26
31
|
|
|
@@ -77,21 +82,49 @@ class DBClient(object):
|
|
|
77
82
|
def _verify_is_connected(self):
|
|
78
83
|
if not self.is_connected:
|
|
79
84
|
raise FunctionalException(f"DB Client '{self.name}' is not connected")
|
|
85
|
+
|
|
86
|
+
def is_query_type(self, sql_or_query, query_type):
|
|
87
|
+
if isinstance(sql_or_query, pypika.queries.QueryBuilder):
|
|
88
|
+
sql = self.query_builder.to_sql(sql_or_query)
|
|
89
|
+
else:
|
|
90
|
+
sql = sql_or_query
|
|
91
|
+
if not isinstance(sql, str):
|
|
92
|
+
raise TechnicalException(f"Unexpected type '{Typing.get_object_class_fullname(sql)}' for parameter sql_or_query")
|
|
93
|
+
|
|
94
|
+
if isinstance(query_type, str):
|
|
95
|
+
query_type = QueryType[query_type.upper()]
|
|
96
|
+
if not isinstance(query_type, QueryType):
|
|
97
|
+
raise TechnicalException(f"Unmanage query_type of type '{Typing.get_object_class_fullname(query_type)}'")
|
|
80
98
|
|
|
99
|
+
p = sql_metadata.Parser(sql)
|
|
100
|
+
return p.query_type == query_type
|
|
101
|
+
|
|
81
102
|
def execute_query(self, query, *args, **kwargs):
|
|
82
103
|
sql = self.query_builder.to_sql(query)
|
|
83
|
-
|
|
104
|
+
|
|
105
|
+
# Force do_commit to False for select queries
|
|
106
|
+
do_commit = kwargs.pop('do_commit', None)
|
|
107
|
+
|
|
108
|
+
return self.execute(sql, *args, do_commit=do_commit, **kwargs)
|
|
84
109
|
|
|
85
110
|
def execute(self, sql, *args, **kwargs):
|
|
86
|
-
# Manage
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
do_commit
|
|
111
|
+
# Manage specific parameters
|
|
112
|
+
if self.is_query_type(sql, 'select'):
|
|
113
|
+
do_commit = False
|
|
114
|
+
else:
|
|
115
|
+
do_commit = kwargs.pop('do_commit', None)
|
|
116
|
+
if do_commit is None:
|
|
117
|
+
do_commit = self.__auto_commit
|
|
118
|
+
result_as_dict_list = kwargs.pop('result_as_dict_list', False)
|
|
119
|
+
as_generator = kwargs.pop('as_generator', False)
|
|
120
|
+
if as_generator and do_commit:
|
|
121
|
+
raise TechnicalException(f"'do_commit=True' and 'as_generator=True' are incompatible")
|
|
122
|
+
if as_generator and not result_as_dict_list:
|
|
123
|
+
raise TechnicalException(f"'as_generator=True' is possible only with 'result_as_dict_list=True'")
|
|
92
124
|
|
|
93
125
|
self._verify_is_connected()
|
|
94
126
|
|
|
127
|
+
# Execute
|
|
95
128
|
if Tools.do_log(logger, logging.DEBUG):
|
|
96
129
|
logger.debug(f"[{self.name}] Executing SQL [{sql}] with parameters [{args if args else ''}{kwargs if kwargs else ''}]...")
|
|
97
130
|
try:
|
|
@@ -105,24 +138,41 @@ class DBClient(object):
|
|
|
105
138
|
self.rollback()
|
|
106
139
|
raise TechnicalException(f"[{self.name}] Error while executing SQL [{sql}] (args: {args} ; kwargs: {kwargs})") from exc
|
|
107
140
|
|
|
141
|
+
# Get result
|
|
108
142
|
if self.cursor.description:
|
|
109
143
|
field_names = [field[0] for field in self.cursor.description]
|
|
110
144
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
145
|
+
if result_as_dict_list:
|
|
146
|
+
class Cursor2DictGenerator(BaseGenerator):
|
|
147
|
+
def __init__(self, field_names, cursor):
|
|
148
|
+
super().__init__(name="DB cursor to dict generator")
|
|
149
|
+
self.__field_names = field_names
|
|
150
|
+
self.__cursor = cursor
|
|
151
|
+
self.__cursor_iter = iter(self.__cursor)
|
|
152
|
+
|
|
153
|
+
def __next__(self):
|
|
154
|
+
row_values = next(self.__cursor_iter)
|
|
155
|
+
return dict(zip(self.__field_names, row_values))
|
|
156
|
+
|
|
157
|
+
gen = Cursor2DictGenerator(field_names, self.cursor)
|
|
158
|
+
if as_generator:
|
|
159
|
+
res = gen
|
|
160
|
+
else:
|
|
161
|
+
res = [e for e in gen]
|
|
162
|
+
else:
|
|
163
|
+
res = TableWithHeader()
|
|
164
|
+
res.header = TableRow(cells_content=field_names)
|
|
165
|
+
|
|
117
166
|
row_values = self.cursor.fetchone()
|
|
167
|
+
while row_values:
|
|
168
|
+
res.add_row(cells_content=row_values)
|
|
169
|
+
row_values = self.cursor.fetchone()
|
|
118
170
|
elif self.cursor.rowcount > 0:
|
|
119
171
|
res = self.cursor.rowcount
|
|
120
172
|
else:
|
|
121
173
|
res = None
|
|
122
174
|
|
|
123
|
-
|
|
124
|
-
logger.debug(f"[{self.name}] Executed SQL [{sql}] with parameters [{args if args else ''}{kwargs if kwargs else ''}] => [{res}]")
|
|
125
|
-
self.__log_sql_result(res)
|
|
175
|
+
self.__log_sql_result(res, message=f"Executed SQL [{sql}] with parameters [{args if args else ''}{kwargs if kwargs else ''}]")
|
|
126
176
|
|
|
127
177
|
# Manage commit
|
|
128
178
|
if do_commit:
|
|
@@ -137,17 +187,19 @@ class DBClient(object):
|
|
|
137
187
|
def _get_base_exception_type(self):
|
|
138
188
|
raise NotImplementedError()
|
|
139
189
|
|
|
140
|
-
def __log_sql_result(self, sql_result, limit_rows=10):
|
|
190
|
+
def __log_sql_result(self, sql_result, message="SQL result", limit_rows=10):
|
|
141
191
|
if Tools.do_log(logger, logging.DEBUG):
|
|
142
192
|
res_str = self.__represent_sql_result(sql_result, limit_rows=limit_rows)
|
|
143
193
|
if '\n' in res_str:
|
|
144
|
-
logger.debug(f"[{self.name}]
|
|
194
|
+
logger.debug(f"[{self.name}] {message}:\n{Tools.indent_string(4, res_str)}")
|
|
145
195
|
else:
|
|
146
|
-
logger.debug(f"[{self.name}]
|
|
196
|
+
logger.debug(f"[{self.name}] {message} => {res_str}")
|
|
147
197
|
|
|
148
198
|
def __represent_sql_result(self, sql_result, limit_rows = 10):
|
|
149
199
|
if isinstance(sql_result, Table):
|
|
150
200
|
return sql_result.represent(limit_rows=limit_rows)
|
|
201
|
+
elif isinstance(sql_result, list) and limit_rows > 0 and len(sql_result) > limit_rows:
|
|
202
|
+
return str(sql_result[:limit_rows])[:-1] + f", ...({len(sql_result)-limit_rows})]"
|
|
151
203
|
else:
|
|
152
204
|
return str(sql_result)
|
|
153
205
|
|
|
@@ -168,31 +220,32 @@ class DBClient(object):
|
|
|
168
220
|
query, values = self.query_builder.insert(table_name, data)
|
|
169
221
|
return self.execute_query(query, *values, do_commit=do_commit)
|
|
170
222
|
|
|
171
|
-
def update(self, table_name, data: dict, where_data: dict, do_commit=None):
|
|
223
|
+
def update(self, table_name, data: dict, where_data: dict=None, where_compare_data: list=None, do_commit=None):
|
|
172
224
|
"""
|
|
173
225
|
Update given data.
|
|
174
226
|
Parameters 'data' and 'where_data' have to be dictionaries with keys equal to table column names.
|
|
175
227
|
"""
|
|
176
|
-
query, values = self.query_builder.update(table_name, data, where_data)
|
|
228
|
+
query, values = self.query_builder.update(table_name, data, where_data=where_data, where_compare_data=where_compare_data)
|
|
177
229
|
return self.execute_query(query, *values, do_commit=do_commit)
|
|
178
230
|
|
|
179
|
-
def select(self, table_name, where_data: dict=None, sql_return="*"):
|
|
231
|
+
def select(self, table_name, where_data: dict=None, where_compare_data: list=None, sql_return="*", **kwargs):
|
|
180
232
|
"""
|
|
181
233
|
Select by filtering on given where data.
|
|
182
|
-
|
|
234
|
+
@param where_data: dictionary of (field_name, value) for simple where clauses.
|
|
235
|
+
@param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
|
|
183
236
|
"""
|
|
184
|
-
query, values = self.query_builder.select(table_name, where_data, sql_return)
|
|
185
|
-
return self.execute_query(query, *values, do_commit=False)
|
|
237
|
+
query, values = self.query_builder.select(table_name, where_data=where_data, where_compare_data=where_compare_data, sql_return=sql_return)
|
|
238
|
+
return self.execute_query(query, *values, do_commit=False, **kwargs)
|
|
186
239
|
|
|
187
|
-
def delete(self, table_name, where_data: dict=None, do_commit=None):
|
|
240
|
+
def delete(self, table_name, where_data: dict=None, where_compare_data: list=None, do_commit=None):
|
|
188
241
|
"""
|
|
189
242
|
Delete by filtering on given where data.
|
|
190
243
|
Parameter 'where_data' has to be a dictionary with keys equal to table column names.
|
|
191
244
|
"""
|
|
192
|
-
query, values = self.query_builder.delete(table_name, where_data)
|
|
245
|
+
query, values = self.query_builder.delete(table_name, where_data=where_data, where_compare_data=where_compare_data)
|
|
193
246
|
return self.execute_query(query, *values, do_commit=do_commit)
|
|
194
247
|
|
|
195
|
-
def set_or_update_json_key_value(self, table_name, field_name, json_key, json_value, where_data):
|
|
248
|
+
def set_or_update_json_key_value(self, table_name, field_name, json_key, json_value, where_data: dict=None, where_compare_data: list=None):
|
|
196
249
|
"""
|
|
197
250
|
Set or update a json field with key=value.
|
|
198
251
|
"""
|
|
@@ -15,6 +15,8 @@ import logging
|
|
|
15
15
|
from holado_db.tools.db.clients.base.db_client import DBClient
|
|
16
16
|
from holado_core.common.tables.table_with_header import TableWithHeader
|
|
17
17
|
from holado_core.common.tools.tools import Tools
|
|
18
|
+
import json
|
|
19
|
+
from pypika.terms import Function, LiteralValue
|
|
18
20
|
|
|
19
21
|
logger = logging.getLogger(__name__)
|
|
20
22
|
|
|
@@ -27,6 +29,12 @@ except Exception as exc:
|
|
|
27
29
|
with_psycopg = False
|
|
28
30
|
|
|
29
31
|
|
|
32
|
+
# from pypika import Function
|
|
33
|
+
class JsonbSet(Function):
|
|
34
|
+
def __init__(self, name, *args, **kwargs):
|
|
35
|
+
super().__init__('jsonb_set', LiteralValue(name), *args, **kwargs)
|
|
36
|
+
|
|
37
|
+
|
|
30
38
|
class PostgreSQLClient(DBClient):
|
|
31
39
|
@classmethod
|
|
32
40
|
def is_available(cls):
|
|
@@ -48,18 +56,20 @@ class PostgreSQLClient(DBClient):
|
|
|
48
56
|
result = self.execute(f"SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename = '{table_name}'")
|
|
49
57
|
return result is not None
|
|
50
58
|
|
|
51
|
-
def set_or_update_json_key_value(self, table_name, field_name, json_key, json_value, where_data: dict=None):
|
|
59
|
+
def set_or_update_json_key_value(self, table_name, field_name, json_key, json_value, where_data: dict=None, where_compare_data: list=None):
|
|
52
60
|
# Help on JSON column: https://www.databasestar.com/postgresql-json/#How_to_Update_JSON_Data_in_PostgreSQL
|
|
53
|
-
result = self.select(table_name, where_data, sql_return=field_name)
|
|
61
|
+
result = self.select(table_name, where_data=where_data, where_compare_data=where_compare_data, sql_return=field_name)
|
|
54
62
|
is_set = isinstance(result, TableWithHeader) and result[0][0].content is not None
|
|
55
63
|
if not is_set:
|
|
56
|
-
self.update(table_name, {field_name: f'{{"{json_key}":"{json_value}"}}'}, where_data, do_commit=True)
|
|
64
|
+
self.update(table_name, {field_name: f'{{"{json_key}":"{json_value}"}}'}, where_data=where_data, where_compare_data=where_compare_data, do_commit=True)
|
|
57
65
|
else:
|
|
58
|
-
result = self.select(table_name, where_data, sql_return=f"{field_name}
|
|
59
|
-
is_key_set =
|
|
66
|
+
result = self.select(table_name, where_data=where_data, where_compare_data=where_compare_data, sql_return=f"{field_name}")
|
|
67
|
+
is_key_set = False
|
|
68
|
+
if isinstance(result, TableWithHeader) and result[0][0].content is not None:
|
|
69
|
+
is_key_set = json_key in result[0][0].content
|
|
60
70
|
if is_key_set:
|
|
61
|
-
self.update(table_name, {field_name: f
|
|
71
|
+
self.update(table_name, {field_name: JsonbSet(f'{field_name}', f'{{"{json_key}"}}', f'"{json_value}"')}, where_data=where_data, where_compare_data=where_compare_data, do_commit=True)
|
|
62
72
|
else:
|
|
63
|
-
self.update(table_name, {field_name: f'{field_name} || {{"{json_key}":"{json_value}"}}'}, where_data, do_commit=True)
|
|
73
|
+
self.update(table_name, {field_name: f'{field_name} || {{"{json_key}":"{json_value}"}}'}, where_data=where_data, where_compare_data=where_compare_data, do_commit=True)
|
|
64
74
|
|
|
65
75
|
|
|
@@ -13,11 +13,33 @@
|
|
|
13
13
|
|
|
14
14
|
import logging
|
|
15
15
|
import abc
|
|
16
|
+
from enum import Enum
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
logger = logging.getLogger(__name__)
|
|
19
20
|
|
|
20
21
|
|
|
22
|
+
class DBCompareOperator(str, Enum):
|
|
23
|
+
Different = ("!=", "Equal")
|
|
24
|
+
Equal = ("==", "Different")
|
|
25
|
+
Inferior = ("<", "SuperiorOrEqual")
|
|
26
|
+
InferiorOrEqual = ("<=", "Superior")
|
|
27
|
+
Superior = (">", "InferiorOrEqual")
|
|
28
|
+
SuperiorOrEqual = (">=", "Inferior")
|
|
29
|
+
In = ("in", "NotIn")
|
|
30
|
+
NotIn = ("not in", "In")
|
|
31
|
+
|
|
32
|
+
def __new__(cls, value, not_name):
|
|
33
|
+
obj = str.__new__(cls, [value])
|
|
34
|
+
obj._value_ = value
|
|
35
|
+
obj.__not_name = not_name
|
|
36
|
+
return obj
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def not_(self):
|
|
40
|
+
return DBCompareOperator[self.__not_name]
|
|
41
|
+
|
|
42
|
+
|
|
21
43
|
|
|
22
44
|
class QueryBuilder():
|
|
23
45
|
"""
|
|
@@ -41,31 +63,35 @@ class QueryBuilder():
|
|
|
41
63
|
def db_client(self, client):
|
|
42
64
|
self.__db_client = client
|
|
43
65
|
|
|
44
|
-
def select(self, table_name, where_data: dict=None, sql_return="*"):
|
|
66
|
+
def select(self, table_name, where_data: dict=None, where_compare_data: list=None, sql_return="*"):
|
|
45
67
|
"""
|
|
46
68
|
Simple query & values builder of a select by filtering on given where data.
|
|
47
|
-
|
|
69
|
+
@param where_data: dictionary of (field_name, value) for simple where clauses.
|
|
70
|
+
@param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
|
|
48
71
|
"""
|
|
49
72
|
raise NotImplementedError
|
|
50
73
|
|
|
51
74
|
def insert(self, table_name, data: dict):
|
|
52
75
|
"""
|
|
53
76
|
Simple query & values builder of an insert of given data.
|
|
54
|
-
|
|
77
|
+
@param data: data to insert as dictionary of (field_name, value).
|
|
55
78
|
"""
|
|
56
79
|
raise NotImplementedError
|
|
57
80
|
|
|
58
|
-
def update(self, table_name, data: dict, where_data: dict):
|
|
81
|
+
def update(self, table_name, data: dict, where_data: dict=None, where_compare_data: list=None):
|
|
59
82
|
"""
|
|
60
83
|
Simple query & values builder of an update of given data.
|
|
61
|
-
|
|
84
|
+
@param data: data to update as dictionary of (field_name, value).
|
|
85
|
+
@param where_data: dictionary of (field_name, value) for simple where clauses.
|
|
86
|
+
@param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
|
|
62
87
|
"""
|
|
63
88
|
raise NotImplementedError
|
|
64
89
|
|
|
65
|
-
def delete(self, table_name, where_data: dict=None):
|
|
90
|
+
def delete(self, table_name, where_data: dict=None, where_compare_data: list=None):
|
|
66
91
|
"""
|
|
67
92
|
Simple query & values builder of a delete by filtering on given where data.
|
|
68
|
-
|
|
93
|
+
@param where_data: dictionary of (field_name, value) for simple where clauses.
|
|
94
|
+
@param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
|
|
69
95
|
"""
|
|
70
96
|
raise NotImplementedError
|
|
71
97
|
|
|
@@ -75,12 +101,37 @@ class QueryBuilder():
|
|
|
75
101
|
"""
|
|
76
102
|
raise NotImplementedError
|
|
77
103
|
|
|
104
|
+
def where_compare(self, query, values, *, field_name=None, operator:DBCompareOperator=None, value=None, list_of_field_operator_value=None):
|
|
105
|
+
"""
|
|
106
|
+
Add where clause with field value comparison to current couple (query, values), and return a new couple (query, values).
|
|
107
|
+
To add multiple clauses in a single call, define list_of_field_operator_value as a list of tuples (field_name, operator, value).
|
|
108
|
+
"""
|
|
109
|
+
if list_of_field_operator_value is not None:
|
|
110
|
+
res_query, res_values = query, values
|
|
111
|
+
for f_name, op, val in list_of_field_operator_value:
|
|
112
|
+
res_query, res_values = self.where_compare(res_query, res_values, field_name=f_name, operator=op, value=val)
|
|
113
|
+
return res_query, res_values
|
|
114
|
+
else:
|
|
115
|
+
raise NotImplementedError
|
|
116
|
+
|
|
78
117
|
def where_in(self, query, values, field_name, field_values, not_in=False):
|
|
79
118
|
"""
|
|
80
119
|
Add where in clause to current couple (query, values), and return a new couple (query, values).
|
|
81
120
|
"""
|
|
82
121
|
raise NotImplementedError
|
|
83
122
|
|
|
123
|
+
def where_is_null(self, query, values, field_name, is_not_null=False):
|
|
124
|
+
"""
|
|
125
|
+
Add where is null clause to current couple (query, values), and return a new couple (query, values).
|
|
126
|
+
"""
|
|
127
|
+
raise NotImplementedError
|
|
128
|
+
|
|
129
|
+
def where_json_value(self, query, values, field_name, key, value, as_text_value=False):
|
|
130
|
+
"""
|
|
131
|
+
Add where clause on a json field key value to current couple (query, values), and return a new couple (query, values).
|
|
132
|
+
"""
|
|
133
|
+
raise NotImplementedError
|
|
134
|
+
|
|
84
135
|
|
|
85
136
|
def to_sql(self, query):
|
|
86
137
|
raise NotImplementedError
|
|
@@ -13,10 +13,10 @@
|
|
|
13
13
|
|
|
14
14
|
import logging
|
|
15
15
|
from holado_core.common.exceptions.technical_exception import TechnicalException
|
|
16
|
-
from holado_db.tools.db.query.base.query_builder import QueryBuilder
|
|
16
|
+
from holado_db.tools.db.query.base.query_builder import QueryBuilder, DBCompareOperator
|
|
17
17
|
from holado_core.common.tools.tools import Tools
|
|
18
|
-
from holado_core.common.tools.comparators.comparator import CompareOperator
|
|
19
18
|
from holado_python.standard_library.typing import Typing
|
|
19
|
+
from pypika.terms import Function
|
|
20
20
|
|
|
21
21
|
|
|
22
22
|
logger = logging.getLogger(__name__)
|
|
@@ -44,10 +44,11 @@ class PypikaQueryBuilder(QueryBuilder):
|
|
|
44
44
|
def __init__(self, name, db_client=None):
|
|
45
45
|
super().__init__(name, db_client)
|
|
46
46
|
|
|
47
|
-
def select(self, table_name, where_data: dict=None, sql_return="*"):
|
|
47
|
+
def select(self, table_name, where_data: dict=None, where_compare_data: list=None, sql_return="*"):
|
|
48
48
|
"""
|
|
49
49
|
Simple query & values builder of a select by filtering on given where data.
|
|
50
|
-
|
|
50
|
+
@param where_data: dictionary of (field_name, value) for simple where clauses.
|
|
51
|
+
@param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
|
|
51
52
|
"""
|
|
52
53
|
table = Table(table_name)
|
|
53
54
|
res = Query.from_(table)
|
|
@@ -62,13 +63,15 @@ class PypikaQueryBuilder(QueryBuilder):
|
|
|
62
63
|
|
|
63
64
|
if where_data:
|
|
64
65
|
res, values = self.where(res, values, where_data)
|
|
65
|
-
|
|
66
|
+
if where_compare_data:
|
|
67
|
+
res, values = self.where_compare(res, values, list_of_field_operator_value=where_compare_data)
|
|
68
|
+
|
|
66
69
|
return res, values
|
|
67
70
|
|
|
68
71
|
def insert(self, table_name, data: dict):
|
|
69
72
|
"""
|
|
70
73
|
Simple query & values builder of an insert of given data.
|
|
71
|
-
|
|
74
|
+
@param data: data to insert as dictionary of (field_name, value).
|
|
72
75
|
"""
|
|
73
76
|
col_names = tuple(sorted(data.keys()))
|
|
74
77
|
values = tuple((data[c] for c in col_names))
|
|
@@ -79,29 +82,39 @@ class PypikaQueryBuilder(QueryBuilder):
|
|
|
79
82
|
|
|
80
83
|
return res, values
|
|
81
84
|
|
|
82
|
-
def update(self, table_name, data: dict, where_data: dict):
|
|
85
|
+
def update(self, table_name, data: dict, where_data: dict=None, where_compare_data: list=None):
|
|
83
86
|
"""
|
|
84
87
|
Simple query & values builder of an update of given data.
|
|
85
|
-
|
|
88
|
+
@param data: data to update as dictionary of (field_name, value).
|
|
89
|
+
@param where_data: dictionary of (field_name, value) for simple where clauses.
|
|
90
|
+
@param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
|
|
86
91
|
"""
|
|
87
92
|
col_names = tuple(sorted(data.keys()))
|
|
88
|
-
values = tuple((data[c] for c in col_names))
|
|
89
93
|
sql_placeholder = self.db_client._get_sql_placeholder()
|
|
90
94
|
|
|
91
95
|
table = Table(table_name)
|
|
92
96
|
res = Query.update(table)
|
|
97
|
+
values = []
|
|
93
98
|
for c in col_names:
|
|
94
|
-
|
|
99
|
+
val = data[c]
|
|
100
|
+
if isinstance(val, Function):
|
|
101
|
+
res = res.set(c, val)
|
|
102
|
+
else:
|
|
103
|
+
res = res.set(c, Parameter(sql_placeholder))
|
|
104
|
+
values.append(val)
|
|
95
105
|
|
|
96
106
|
if where_data:
|
|
97
107
|
res, values = self.where(res, values, where_data)
|
|
108
|
+
if where_compare_data:
|
|
109
|
+
res, values = self.where_compare(res, values, list_of_field_operator_value=where_compare_data)
|
|
98
110
|
|
|
99
111
|
return res, values
|
|
100
112
|
|
|
101
|
-
def delete(self, table_name, where_data: dict=None):
|
|
113
|
+
def delete(self, table_name, where_data: dict=None, where_compare_data: list=None):
|
|
102
114
|
"""
|
|
103
115
|
Simple query & values builder of a delete by filtering on given where data.
|
|
104
|
-
|
|
116
|
+
@param where_data: dictionary of (field_name, value) for simple where clauses.
|
|
117
|
+
@param where_compare_data: list of tuples (field_name, operator, value) for where clauses comparing fields with values.
|
|
105
118
|
"""
|
|
106
119
|
table = Table(table_name)
|
|
107
120
|
res = Query.from_(table).delete()
|
|
@@ -109,18 +122,25 @@ class PypikaQueryBuilder(QueryBuilder):
|
|
|
109
122
|
|
|
110
123
|
if where_data:
|
|
111
124
|
res, values = self.where(res, values, where_data)
|
|
125
|
+
if where_compare_data:
|
|
126
|
+
res, values = self.where_compare(res, values, list_of_field_operator_value=where_compare_data)
|
|
112
127
|
|
|
113
128
|
return res, values
|
|
114
129
|
|
|
115
130
|
def where(self, query, values, where_data: dict):
|
|
116
131
|
col_names = tuple(sorted(where_data.keys()))
|
|
117
|
-
where_values = tuple((where_data[c] for c in col_names))
|
|
118
132
|
sql_placeholder = self.db_client._get_sql_placeholder()
|
|
119
133
|
|
|
120
134
|
res = query
|
|
121
135
|
table = self.__get_table(query)
|
|
136
|
+
where_values = []
|
|
122
137
|
for c in col_names:
|
|
123
|
-
|
|
138
|
+
val = where_data[c]
|
|
139
|
+
if val is None:
|
|
140
|
+
res, where_values = self.where_is_null(res, where_values, c)
|
|
141
|
+
else:
|
|
142
|
+
res = res.where(getattr(table, c) == Parameter(sql_placeholder))
|
|
143
|
+
where_values.append(where_data[c])
|
|
124
144
|
|
|
125
145
|
if values is not None:
|
|
126
146
|
values = (*values, *where_values)
|
|
@@ -128,21 +148,31 @@ class PypikaQueryBuilder(QueryBuilder):
|
|
|
128
148
|
values = where_values
|
|
129
149
|
return res, values
|
|
130
150
|
|
|
131
|
-
def where_compare(self, query, values, field_name, operator:
|
|
151
|
+
def where_compare(self, query, values, *, field_name=None, operator:DBCompareOperator=None, value=None, list_of_field_operator_value=None):
|
|
152
|
+
if list_of_field_operator_value is not None:
|
|
153
|
+
return super().where_compare(query, values, list_of_field_operator_value=list_of_field_operator_value)
|
|
154
|
+
|
|
155
|
+
# Manage operators calling other methods
|
|
156
|
+
if operator == DBCompareOperator.In:
|
|
157
|
+
return self.where_in(query, values, field_name, value, not_in=False)
|
|
158
|
+
elif operator == DBCompareOperator.NotIn:
|
|
159
|
+
return self.where_in(query, values, field_name, value, not_in=True)
|
|
160
|
+
|
|
161
|
+
# Manage other operators
|
|
132
162
|
table = self.__get_table(query)
|
|
133
163
|
sql_placeholder = self.db_client._get_sql_placeholder()
|
|
134
164
|
|
|
135
|
-
if operator ==
|
|
165
|
+
if operator == DBCompareOperator.Different:
|
|
136
166
|
res = query.where(getattr(table, field_name) != Parameter(sql_placeholder))
|
|
137
|
-
elif operator ==
|
|
167
|
+
elif operator == DBCompareOperator.Equal:
|
|
138
168
|
res = query.where(getattr(table, field_name) == Parameter(sql_placeholder))
|
|
139
|
-
elif operator ==
|
|
169
|
+
elif operator == DBCompareOperator.Inferior:
|
|
140
170
|
res = query.where(getattr(table, field_name) < Parameter(sql_placeholder))
|
|
141
|
-
elif operator ==
|
|
171
|
+
elif operator == DBCompareOperator.InferiorOrEqual:
|
|
142
172
|
res = query.where(getattr(table, field_name) <= Parameter(sql_placeholder))
|
|
143
|
-
elif operator ==
|
|
173
|
+
elif operator == DBCompareOperator.Superior:
|
|
144
174
|
res = query.where(getattr(table, field_name) > Parameter(sql_placeholder))
|
|
145
|
-
elif operator ==
|
|
175
|
+
elif operator == DBCompareOperator.SuperiorOrEqual:
|
|
146
176
|
res = query.where(getattr(table, field_name) >= Parameter(sql_placeholder))
|
|
147
177
|
else:
|
|
148
178
|
raise TechnicalException(f"Unmanaged compare operator {operator}")
|
|
@@ -161,6 +191,14 @@ class PypikaQueryBuilder(QueryBuilder):
|
|
|
161
191
|
res = query.where(getattr(table, field_name).isin(field_values))
|
|
162
192
|
return res, values
|
|
163
193
|
|
|
194
|
+
def where_is_null(self, query, values, field_name, is_not_null=False):
|
|
195
|
+
table = self.__get_table(query)
|
|
196
|
+
if is_not_null:
|
|
197
|
+
res = query.where(getattr(table, field_name).notnull())
|
|
198
|
+
else:
|
|
199
|
+
res = query.where(getattr(table, field_name).isnull())
|
|
200
|
+
return res, values
|
|
201
|
+
|
|
164
202
|
def where_json_value(self, query, values, field_name, key, value, as_text_value=False):
|
|
165
203
|
table = self.__get_table(query)
|
|
166
204
|
sql_placeholder = self.db_client._get_sql_placeholder()
|
|
@@ -181,6 +219,20 @@ class PypikaQueryBuilder(QueryBuilder):
|
|
|
181
219
|
return query.get_sql()
|
|
182
220
|
else:
|
|
183
221
|
raise TechnicalException(f"Unmanaged query of type {Typing.get_object_class_fullname(query)}")
|
|
222
|
+
|
|
223
|
+
# def is_query_type(self, query, query_type):
|
|
224
|
+
# query_type = query_type.lower()
|
|
225
|
+
#
|
|
226
|
+
# if query_type == 'insert':
|
|
227
|
+
# return query._insert_table is not None
|
|
228
|
+
# elif query_type == 'select':
|
|
229
|
+
# return len(query._selects) > 0
|
|
230
|
+
# elif query_type == 'update':
|
|
231
|
+
# return query._update_table is not None
|
|
232
|
+
# elif query_type == 'delete':
|
|
233
|
+
# return query._delete_from
|
|
234
|
+
# else:
|
|
235
|
+
# raise TechnicalException(f"Unmanaged query type '{query_type}' (managed query types: 'insert', 'select', 'update', 'delete')")
|
|
184
236
|
|
|
185
237
|
def __get_table(self, query):
|
|
186
238
|
if query._from is not None and len(query._from) > 0:
|