iker-python-common 1.0.49__py3-none-any.whl → 1.0.51__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iker/common/utils/dbutils.py +127 -100
- iker/common/utils/dtutils.py +22 -0
- iker/common/utils/testutils.py +1 -1
- {iker_python_common-1.0.49.dist-info → iker_python_common-1.0.51.dist-info}/METADATA +1 -1
- {iker_python_common-1.0.49.dist-info → iker_python_common-1.0.51.dist-info}/RECORD +7 -7
- {iker_python_common-1.0.49.dist-info → iker_python_common-1.0.51.dist-info}/WHEEL +0 -0
- {iker_python_common-1.0.49.dist-info → iker_python_common-1.0.51.dist-info}/top_level.txt +0 -0
iker/common/utils/dbutils.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import contextlib
|
|
2
2
|
import dataclasses
|
|
3
|
-
import
|
|
4
|
-
from typing import Any
|
|
3
|
+
from typing import Any, Sequence
|
|
5
4
|
|
|
6
5
|
import packaging.version
|
|
7
6
|
import psycopg
|
|
@@ -10,12 +9,13 @@ import sqlalchemy
|
|
|
10
9
|
import sqlalchemy.ext.compiler
|
|
11
10
|
import sqlalchemy.orm
|
|
12
11
|
|
|
13
|
-
from iker.common.utils.
|
|
14
|
-
from iker.common.utils.strutils import is_blank
|
|
12
|
+
from iker.common.utils.jsonutils import JsonType
|
|
15
13
|
|
|
16
14
|
__all__ = [
|
|
15
|
+
"Dialects",
|
|
16
|
+
"Drivers",
|
|
17
|
+
"make_scheme",
|
|
17
18
|
"ConnectionMaker",
|
|
18
|
-
"DBAdapter",
|
|
19
19
|
"orm_to_dict",
|
|
20
20
|
"orm_clone",
|
|
21
21
|
"mysql_insert_ignore",
|
|
@@ -23,87 +23,106 @@ __all__ = [
|
|
|
23
23
|
]
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
class Dialects:
|
|
27
|
+
mysql = "mysql"
|
|
28
|
+
postgresql = "postgresql"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Drivers:
|
|
32
|
+
pymysql = pymysql.__name__
|
|
33
|
+
psycopg = psycopg.__name__
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def make_scheme(dialect: str, driver: str | None = None) -> str:
|
|
37
|
+
"""
|
|
38
|
+
Constructs a SQLAlchemy scheme string based on the provided dialect and driver.
|
|
39
|
+
|
|
40
|
+
:param dialect: The database dialect (e.g., 'mysql', 'postgresql').
|
|
41
|
+
:param driver: Optional database driver (e.g., 'pymysql', 'psycopg').
|
|
42
|
+
:return: A SQLAlchemy scheme string.
|
|
43
|
+
"""
|
|
44
|
+
return dialect if driver is None else f"{dialect}+{driver}"
|
|
45
|
+
|
|
46
|
+
|
|
26
47
|
class ConnectionMaker(object):
|
|
27
48
|
"""
|
|
28
49
|
Provides utilities to simplify establishing database connections and sessions, including connection string
|
|
29
50
|
construction, engine and session creation, and model management.
|
|
30
51
|
|
|
31
|
-
:param
|
|
32
|
-
:param host: The database host.
|
|
33
|
-
:param port: The database port.
|
|
34
|
-
:param user: The database user.
|
|
35
|
-
:param password: The database password.
|
|
36
|
-
:param database: The database name.
|
|
52
|
+
:param url: A SQLAlchemy URL string or ``URL`` object representing the database connection.
|
|
37
53
|
:param engine_opts: Optional dictionary of SQLAlchemy engine options.
|
|
38
54
|
:param session_opts: Optional dictionary of SQLAlchemy session options.
|
|
39
55
|
"""
|
|
40
56
|
|
|
41
|
-
class Drivers:
|
|
42
|
-
Mysql = f"mysql+{pymysql.__name__}"
|
|
43
|
-
Postgresql = f"postgresql+{psycopg.__name__}"
|
|
44
|
-
|
|
45
57
|
def __init__(
|
|
46
58
|
self,
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
password: str,
|
|
52
|
-
database: str,
|
|
53
|
-
engine_opts: dict[str, Any] | None = None,
|
|
54
|
-
session_opts: dict[str, Any] | None = None,
|
|
59
|
+
url: sqlalchemy.URL,
|
|
60
|
+
*,
|
|
61
|
+
engine_opts: dict[str, JsonType] | None = None,
|
|
62
|
+
session_opts: dict[str, JsonType] | None = None,
|
|
55
63
|
):
|
|
56
|
-
self.
|
|
57
|
-
self.host = host
|
|
58
|
-
self.port = port
|
|
59
|
-
self.user = user
|
|
60
|
-
self.password = password
|
|
61
|
-
self.database = database
|
|
64
|
+
self.url = url
|
|
62
65
|
self.engine_opts = engine_opts or {}
|
|
63
66
|
self.session_opts = session_opts or {}
|
|
64
67
|
|
|
68
|
+
@staticmethod
|
|
69
|
+
def create(
|
|
70
|
+
driver: str | None = None,
|
|
71
|
+
host: str | None = None,
|
|
72
|
+
port: int | None = None,
|
|
73
|
+
username: str | None = None,
|
|
74
|
+
password: str | None = None,
|
|
75
|
+
database: str | None = None,
|
|
76
|
+
*,
|
|
77
|
+
engine_opts: dict[str, JsonType] | None = None,
|
|
78
|
+
session_opts: dict[str, JsonType] | None = None,
|
|
79
|
+
):
|
|
80
|
+
"""
|
|
81
|
+
Creates a new instance of ``ConnectionMaker`` using the provided parameters to construct a SQLAlchemy URL.
|
|
82
|
+
|
|
83
|
+
:param driver: Optional database driver.
|
|
84
|
+
:param host: The database host (e.g., 'localhost').
|
|
85
|
+
:param port: The database port.
|
|
86
|
+
:param username: The database username.
|
|
87
|
+
:param password: The database password.
|
|
88
|
+
:param database: The name of the database to connect to.
|
|
89
|
+
:param engine_opts: Optional dictionary of SQLAlchemy engine options.
|
|
90
|
+
:param session_opts: Optional dictionary of SQLAlchemy session options.
|
|
91
|
+
"""
|
|
92
|
+
return ConnectionMaker(sqlalchemy.URL.create(drivername=driver,
|
|
93
|
+
host=host,
|
|
94
|
+
port=port,
|
|
95
|
+
username=username,
|
|
96
|
+
password=password,
|
|
97
|
+
database=database),
|
|
98
|
+
engine_opts=engine_opts,
|
|
99
|
+
session_opts=session_opts)
|
|
100
|
+
|
|
65
101
|
@staticmethod
|
|
66
102
|
def from_url(
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
engine_opts: dict[str,
|
|
70
|
-
session_opts: dict[str,
|
|
103
|
+
url: str | sqlalchemy.URL,
|
|
104
|
+
*,
|
|
105
|
+
engine_opts: dict[str, JsonType] | None = None,
|
|
106
|
+
session_opts: dict[str, JsonType] | None = None,
|
|
71
107
|
) -> "ConnectionMaker":
|
|
72
108
|
"""
|
|
73
|
-
Creates a ``ConnectionMaker``
|
|
109
|
+
Creates a new instance of ``ConnectionMaker`` from a SQLAlchemy URL string or object.
|
|
74
110
|
|
|
75
|
-
:param
|
|
76
|
-
:param url: The database URL as a string or ``ParseResult``.
|
|
111
|
+
:param url: A SQLAlchemy URL string or ``URL`` object representing the database connection.
|
|
77
112
|
:param engine_opts: Optional dictionary of SQLAlchemy engine options.
|
|
78
113
|
:param session_opts: Optional dictionary of SQLAlchemy session options.
|
|
79
|
-
:return: A ``ConnectionMaker``
|
|
114
|
+
:return: A new instance of ``ConnectionMaker`` configured with the provided URL and options.
|
|
80
115
|
"""
|
|
81
|
-
|
|
82
|
-
return ConnectionMaker.from_url(driver, urllib.parse.urlparse(url), engine_opts, session_opts)
|
|
83
|
-
if isinstance(url, urllib.parse.ParseResult):
|
|
84
|
-
return ConnectionMaker(driver,
|
|
85
|
-
url.hostname,
|
|
86
|
-
url.port,
|
|
87
|
-
url.username,
|
|
88
|
-
url.password,
|
|
89
|
-
url.path.strip("/"),
|
|
90
|
-
engine_opts,
|
|
91
|
-
session_opts)
|
|
92
|
-
raise ValueError("malformed parameter 'url'")
|
|
116
|
+
return ConnectionMaker(sqlalchemy.make_url(url), engine_opts=engine_opts, session_opts=session_opts)
|
|
93
117
|
|
|
94
118
|
@property
|
|
95
119
|
def connection_string(self) -> str:
|
|
96
120
|
"""
|
|
97
|
-
Constructs
|
|
121
|
+
Constructs a SQLAlchemy connection string for the database using the provided parameters.
|
|
98
122
|
|
|
99
|
-
:return:
|
|
123
|
+
:return: A string representing the database connection.
|
|
100
124
|
"""
|
|
101
|
-
|
|
102
|
-
user_part = urllib.parse.quote(self.user, safe="")
|
|
103
|
-
password_part = "" if is_blank(self.password) else (":%s" % urllib.parse.quote(self.password, safe=""))
|
|
104
|
-
database_part = urllib.parse.quote(self.database, safe="")
|
|
105
|
-
|
|
106
|
-
return f"{self.driver}://{user_part}{password_part}@{self.host}{port_part}/{database_part}"
|
|
125
|
+
return self.url.render_as_string(hide_password=False)
|
|
107
126
|
|
|
108
127
|
@property
|
|
109
128
|
def engine(self) -> sqlalchemy.Engine:
|
|
@@ -114,7 +133,7 @@ class ConnectionMaker(object):
|
|
|
114
133
|
"""
|
|
115
134
|
return sqlalchemy.create_engine(self.connection_string, **self.engine_opts)
|
|
116
135
|
|
|
117
|
-
def make_connection(self):
|
|
136
|
+
def make_connection(self) -> sqlalchemy.Connection:
|
|
118
137
|
"""
|
|
119
138
|
Establishes and returns a new database connection using the SQLAlchemy engine.
|
|
120
139
|
|
|
@@ -165,11 +184,11 @@ class ConnectionMaker(object):
|
|
|
165
184
|
:param params: The parameters dictionary for the SQL statement.
|
|
166
185
|
:return: None.
|
|
167
186
|
"""
|
|
168
|
-
with self.
|
|
169
|
-
|
|
170
|
-
|
|
187
|
+
with self.make_session() as session:
|
|
188
|
+
session.execute(sqlalchemy.text(sql), params)
|
|
189
|
+
session.commit()
|
|
171
190
|
|
|
172
|
-
def query_all(self, sql: str, **params) ->
|
|
191
|
+
def query_all(self, sql: str, **params) -> Sequence[sqlalchemy.Row[tuple[Any, ...]]]:
|
|
173
192
|
"""
|
|
174
193
|
Executes the given SQL query with the specified parameters and returns all result tuples.
|
|
175
194
|
|
|
@@ -177,23 +196,22 @@ class ConnectionMaker(object):
|
|
|
177
196
|
:param params: The parameters dictionary for the SQL query.
|
|
178
197
|
:return: A list of result tuples.
|
|
179
198
|
"""
|
|
180
|
-
with self.
|
|
181
|
-
|
|
182
|
-
|
|
199
|
+
with self.make_session() as session:
|
|
200
|
+
result = session.execute(sqlalchemy.text(sql), params)
|
|
201
|
+
return result.all()
|
|
183
202
|
|
|
184
|
-
def query_first(self, sql: str, **params) -> tuple | None:
|
|
203
|
+
def query_first(self, sql: str, **params) -> sqlalchemy.Row[tuple[Any, ...]] | None:
|
|
185
204
|
"""
|
|
186
|
-
Executes the given SQL query with the specified parameters and returns the first result tuple, or ``None``
|
|
187
|
-
results are found.
|
|
205
|
+
Executes the given SQL query with the specified parameters and returns the first result tuple, or ``None``
|
|
206
|
+
if no results are found.
|
|
188
207
|
|
|
189
208
|
:param sql: The SQL query to execute.
|
|
190
209
|
:param params: The parameters dictionary for the SQL query.
|
|
191
210
|
:return: The first result tuple, or ``None`` if no results are found.
|
|
192
211
|
"""
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
DBAdapter = ConnectionMaker
|
|
212
|
+
with self.make_session() as session:
|
|
213
|
+
result = session.execute(sqlalchemy.text(sql), params)
|
|
214
|
+
return result.first()
|
|
197
215
|
|
|
198
216
|
|
|
199
217
|
def orm_to_dict(orm, exclude: set[str] = None) -> dict[str, Any]:
|
|
@@ -241,40 +259,49 @@ def orm_clone(orm, exclude: set[str] = None, no_autoincrement: bool = False):
|
|
|
241
259
|
return new_orm
|
|
242
260
|
|
|
243
261
|
|
|
244
|
-
|
|
262
|
+
@contextlib.contextmanager
|
|
263
|
+
def mysql_insert_ignore():
|
|
245
264
|
"""
|
|
246
|
-
Registers a SQLAlchemy compiler extension to add ``IGNORE`` to MySQL ``INSERT`` statements
|
|
247
|
-
|
|
248
|
-
:param enabled: Whether to enable the ``IGNORE`` prefix for MySQL ``INSERT`` statements.
|
|
249
|
-
:return: None.
|
|
265
|
+
Registers a SQLAlchemy compiler extension to add ``IGNORE`` to MySQL ``INSERT`` statements.
|
|
250
266
|
"""
|
|
251
267
|
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
268
|
+
def context(enabled: bool):
|
|
269
|
+
@sqlalchemy.ext.compiler.compiles(sqlalchemy.sql.Insert, Dialects.mysql)
|
|
270
|
+
def dispatch(insert: sqlalchemy.sql.Insert, compiler: sqlalchemy.sql.compiler.SQLCompiler, **kwargs) -> str:
|
|
271
|
+
if not enabled:
|
|
272
|
+
return compiler.visit_insert(insert, **kwargs)
|
|
256
273
|
|
|
257
|
-
|
|
274
|
+
return compiler.visit_insert(insert.prefix_with("IGNORE"), **kwargs)
|
|
258
275
|
|
|
276
|
+
context(True)
|
|
277
|
+
try:
|
|
278
|
+
yield
|
|
279
|
+
finally:
|
|
280
|
+
context(False)
|
|
259
281
|
|
|
260
|
-
def postgresql_insert_on_conflict_do_nothing(enabled: bool = True):
|
|
261
|
-
"""
|
|
262
|
-
Registers a SQLAlchemy compiler extension to add ``ON CONFLICT DO NOTHING`` to PostgreSQL ``INSERT`` statements if
|
|
263
|
-
``enabled``.
|
|
264
282
|
|
|
265
|
-
|
|
266
|
-
|
|
283
|
+
@contextlib.contextmanager
|
|
284
|
+
def postgresql_insert_on_conflict_do_nothing():
|
|
285
|
+
"""
|
|
286
|
+
Registers a SQLAlchemy compiler extension to add ``ON CONFLICT DO NOTHING`` to Postgresql ``INSERT`` statements.
|
|
267
287
|
"""
|
|
268
288
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
289
|
+
def context(enabled: bool):
|
|
290
|
+
@sqlalchemy.ext.compiler.compiles(sqlalchemy.sql.Insert, Dialects.postgresql)
|
|
291
|
+
def dispatch(insert: sqlalchemy.sql.Insert, compiler: sqlalchemy.sql.compiler.SQLCompiler, **kwargs) -> str:
|
|
292
|
+
if not enabled:
|
|
293
|
+
return compiler.visit_insert(insert, **kwargs)
|
|
294
|
+
|
|
295
|
+
statement = compiler.visit_insert(insert, **kwargs)
|
|
296
|
+
# If we have a ``RETURNING`` clause, we must insert before it
|
|
297
|
+
returning_position = statement.find("RETURNING")
|
|
298
|
+
if returning_position >= 0:
|
|
299
|
+
return statement[:returning_position] + " ON CONFLICT DO NOTHING " + statement[returning_position:]
|
|
300
|
+
else:
|
|
301
|
+
return statement + " ON CONFLICT DO NOTHING"
|
|
302
|
+
|
|
303
|
+
context(True)
|
|
304
|
+
try:
|
|
305
|
+
yield
|
|
306
|
+
finally:
|
|
307
|
+
context(False)
|
iker/common/utils/dtutils.py
CHANGED
|
@@ -24,6 +24,8 @@ __all__ = [
|
|
|
24
24
|
"dt_utc",
|
|
25
25
|
"td_to_us",
|
|
26
26
|
"td_from_us",
|
|
27
|
+
"td_to_time",
|
|
28
|
+
"td_from_time",
|
|
27
29
|
"dt_to_ts",
|
|
28
30
|
"dt_to_ts_us",
|
|
29
31
|
"dt_from_ts",
|
|
@@ -229,6 +231,26 @@ def td_from_us(us: int) -> datetime.timedelta:
|
|
|
229
231
|
return datetime.timedelta(microseconds=us)
|
|
230
232
|
|
|
231
233
|
|
|
234
|
+
def td_to_time(td: datetime.timedelta) -> datetime.time:
|
|
235
|
+
"""
|
|
236
|
+
Returns a ``time`` object representing the given ``timedelta``.
|
|
237
|
+
|
|
238
|
+
:param td: The ``timedelta`` to convert.
|
|
239
|
+
:return: The corresponding ``time`` object.
|
|
240
|
+
"""
|
|
241
|
+
return (dt_utc_min() + td).timetz()
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def td_from_time(t: datetime.time) -> datetime.timedelta:
|
|
245
|
+
"""
|
|
246
|
+
Returns a ``timedelta`` representing the given ``time``.
|
|
247
|
+
|
|
248
|
+
:param t: The ``time`` to convert.
|
|
249
|
+
:return: The corresponding ``timedelta``.
|
|
250
|
+
"""
|
|
251
|
+
return datetime.timedelta(hours=t.hour, minutes=t.minute, seconds=t.second, microseconds=t.microsecond)
|
|
252
|
+
|
|
253
|
+
|
|
232
254
|
def dt_to_td(dt: datetime.datetime) -> datetime.timedelta:
|
|
233
255
|
"""
|
|
234
256
|
Returns the ``timedelta`` between the given ``datetime`` and the POSIX epoch.
|
iker/common/utils/testutils.py
CHANGED
|
@@ -69,7 +69,7 @@ class ApproxNestedMapping(ApproxNestedMixin, ApproxMapping):
|
|
|
69
69
|
yield from mapping._yield_comparisons(actual[k])
|
|
70
70
|
|
|
71
71
|
|
|
72
|
-
def nested_approx(expected, rel: float = None, abs: float = None, nan_ok: bool = False):
|
|
72
|
+
def nested_approx(expected, rel: float = None, abs: float = None, nan_ok: bool = False) -> ApproxBase:
|
|
73
73
|
if isinstance(expected, dict):
|
|
74
74
|
return ApproxNestedMapping(expected, rel, abs, nan_ok)
|
|
75
75
|
if isinstance(expected, (tuple, list)):
|
|
@@ -3,9 +3,9 @@ iker/common/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
|
|
|
3
3
|
iker/common/utils/argutils.py,sha256=hMLNqdZs_Kjc2hw4Npm6N47RivP6JRNzCKIbJr1jYy8,9274
|
|
4
4
|
iker/common/utils/config.py,sha256=z8rLqli961A-qAV9EaELp-pKuhNUNaq1Btdv-uwG7_I,4690
|
|
5
5
|
iker/common/utils/csv.py,sha256=_V9OUrKcojec2L-hWagEIVnL2uvGjyJAFTrD7tHNr48,7573
|
|
6
|
-
iker/common/utils/dbutils.py,sha256=
|
|
6
|
+
iker/common/utils/dbutils.py,sha256=zXZVJCz7HZPityFRF7sHRRMpMraegV_hyYnzApUUPhY,11852
|
|
7
7
|
iker/common/utils/dockerutils.py,sha256=n2WuzXaZB6_WocSljvPOnfExSIjIHRUbuWp2oBbaPKQ,8004
|
|
8
|
-
iker/common/utils/dtutils.py,sha256=
|
|
8
|
+
iker/common/utils/dtutils.py,sha256=86vbaa4pgcBWERZvTfJ92PKB3IimxP6tf0O11ho2Ffk,12554
|
|
9
9
|
iker/common/utils/funcutils.py,sha256=A08f5wjoLgLQKyRJcYeWJnqVm2QcerIx0l-Se2600bc,5869
|
|
10
10
|
iker/common/utils/jsonutils.py,sha256=xYKimWtsqQKiQDQr3EMIhpGrmWSNPzgZR1Sdz60CxSo,16536
|
|
11
11
|
iker/common/utils/logger.py,sha256=FJaai6Sbchy4wKHcUMUCrrkBcXvIxq4qByERZ_TJBps,3881
|
|
@@ -17,9 +17,9 @@ iker/common/utils/sequtils.py,sha256=MfYL82TygBRlGK4sw0RJcoB5P3hO6l7saVH5oZ3YDP8
|
|
|
17
17
|
iker/common/utils/shutils.py,sha256=44_Qkzkhrs9LsfDflsaY_4Va0IpVLU3o8K_NvqCB04w,7859
|
|
18
18
|
iker/common/utils/span.py,sha256=yiXqk86cLKxkMdkO3pAHEfU5bUvHsGo3p--pAWo_yfM,5999
|
|
19
19
|
iker/common/utils/strutils.py,sha256=Tu_qFeH3K-SfwvMxdrZAc9iLPV8ZmtX4ntyyFGNslf8,5094
|
|
20
|
-
iker/common/utils/testutils.py,sha256=
|
|
20
|
+
iker/common/utils/testutils.py,sha256=2VieV5yeCDntSKQSpIeyqRT8BZmZYE_ArMeQz3g7fXY,5568
|
|
21
21
|
iker/common/utils/typeutils.py,sha256=RVkYkFRgDrx77OHFH7PavMV0AIB0S8ly40rs4g7JWE4,8220
|
|
22
|
-
iker_python_common-1.0.
|
|
23
|
-
iker_python_common-1.0.
|
|
24
|
-
iker_python_common-1.0.
|
|
25
|
-
iker_python_common-1.0.
|
|
22
|
+
iker_python_common-1.0.51.dist-info/METADATA,sha256=Yb0B55UC6sBod8XCPOu5SZFk_JpS5nyQ9uaWz9LtclE,1001
|
|
23
|
+
iker_python_common-1.0.51.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
24
|
+
iker_python_common-1.0.51.dist-info/top_level.txt,sha256=4_B8Prfc_lxFafFYTQThIU1ZqOYQ4pHHHnJ_fQ_oHs8,5
|
|
25
|
+
iker_python_common-1.0.51.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|