velocity-python 0.0.153__py3-none-any.whl → 0.0.158__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of velocity-python might be problematic. Click here for more details.
- velocity/__init__.py +1 -1
- velocity/db/core/decorators.py +27 -8
- velocity/db/core/engine.py +3 -2
- velocity/db/core/table.py +103 -49
- velocity/db/core/transaction.py +9 -4
- velocity/db/servers/mysql/sql.py +32 -10
- velocity/db/servers/postgres/sql.py +165 -91
- velocity/db/servers/sqlite/sql.py +34 -10
- velocity/db/servers/sqlserver/sql.py +31 -12
- velocity/db/tests/postgres/test_table_comprehensive.py +3 -1
- velocity/db/tests/test_db_utils.py +49 -0
- velocity/db/utils.py +67 -4
- {velocity_python-0.0.153.dist-info → velocity_python-0.0.158.dist-info}/METADATA +1 -1
- {velocity_python-0.0.153.dist-info → velocity_python-0.0.158.dist-info}/RECORD +17 -17
- {velocity_python-0.0.153.dist-info → velocity_python-0.0.158.dist-info}/WHEEL +0 -0
- {velocity_python-0.0.153.dist-info → velocity_python-0.0.158.dist-info}/licenses/LICENSE +0 -0
- {velocity_python-0.0.153.dist-info → velocity_python-0.0.158.dist-info}/top_level.txt +0 -0
velocity/__init__.py
CHANGED
velocity/db/core/decorators.py
CHANGED
|
@@ -4,10 +4,27 @@ from functools import wraps
|
|
|
4
4
|
from velocity.db import exceptions
|
|
5
5
|
|
|
6
6
|
|
|
7
|
+
_PRIMARY_KEY_PATTERNS = (
|
|
8
|
+
"primary key",
|
|
9
|
+
"key 'primary'",
|
|
10
|
+
'key "primary"',
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _is_primary_key_duplicate(error):
|
|
15
|
+
"""Return True when the duplicate-key error is targeting the primary key."""
|
|
16
|
+
|
|
17
|
+
message = str(error or "")
|
|
18
|
+
lowered = message.lower()
|
|
19
|
+
|
|
20
|
+
if "sys_id" in lowered:
|
|
21
|
+
return True
|
|
22
|
+
|
|
23
|
+
return any(pattern in lowered for pattern in _PRIMARY_KEY_PATTERNS)
|
|
24
|
+
|
|
25
|
+
|
|
7
26
|
def retry_on_dup_key(func):
|
|
8
|
-
"""
|
|
9
|
-
Retries a function call if it raises DbDuplicateKeyError, up to max_retries.
|
|
10
|
-
"""
|
|
27
|
+
"""Retry when insert/update fails because the primary key already exists."""
|
|
11
28
|
|
|
12
29
|
@wraps(func)
|
|
13
30
|
def retry_decorator(self, *args, **kwds):
|
|
@@ -19,10 +36,12 @@ def retry_on_dup_key(func):
|
|
|
19
36
|
result = func(self, *args, **kwds)
|
|
20
37
|
self.tx.release_savepoint(sp, cursor=self.cursor())
|
|
21
38
|
return result
|
|
22
|
-
except exceptions.DbDuplicateKeyError:
|
|
39
|
+
except exceptions.DbDuplicateKeyError as error:
|
|
23
40
|
self.tx.rollback_savepoint(sp, cursor=self.cursor())
|
|
24
41
|
if "sys_id" in kwds.get("data", {}):
|
|
25
42
|
raise
|
|
43
|
+
if not _is_primary_key_duplicate(error):
|
|
44
|
+
raise
|
|
26
45
|
retries += 1
|
|
27
46
|
if retries >= max_retries:
|
|
28
47
|
raise
|
|
@@ -34,9 +53,7 @@ def retry_on_dup_key(func):
|
|
|
34
53
|
|
|
35
54
|
|
|
36
55
|
def reset_id_on_dup_key(func):
|
|
37
|
-
"""
|
|
38
|
-
Wraps an INSERT/UPSERT to reset the sys_id sequence on duplicate key collisions.
|
|
39
|
-
"""
|
|
56
|
+
"""Retry sys_id sequence bump only when the primary key collides."""
|
|
40
57
|
|
|
41
58
|
@wraps(func)
|
|
42
59
|
def reset_decorator(self, *args, retries=0, **kwds):
|
|
@@ -45,10 +62,12 @@ def reset_id_on_dup_key(func):
|
|
|
45
62
|
result = func(self, *args, **kwds)
|
|
46
63
|
self.tx.release_savepoint(sp, cursor=self.cursor())
|
|
47
64
|
return result
|
|
48
|
-
except exceptions.DbDuplicateKeyError:
|
|
65
|
+
except exceptions.DbDuplicateKeyError as error:
|
|
49
66
|
self.tx.rollback_savepoint(sp, cursor=self.cursor())
|
|
50
67
|
if "sys_id" in kwds.get("data", {}):
|
|
51
68
|
raise
|
|
69
|
+
if not _is_primary_key_duplicate(error):
|
|
70
|
+
raise
|
|
52
71
|
if retries < 3:
|
|
53
72
|
backoff_time = (2**retries) * 0.01 + random.uniform(0, 0.02)
|
|
54
73
|
time.sleep(backoff_time)
|
velocity/db/core/engine.py
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
import re
|
|
3
|
-
import os
|
|
4
3
|
from contextlib import contextmanager
|
|
5
4
|
from functools import wraps
|
|
6
5
|
from velocity.db import exceptions
|
|
7
6
|
from velocity.db.core.transaction import Transaction
|
|
7
|
+
from velocity.db.utils import mask_config_for_display
|
|
8
8
|
|
|
9
9
|
import logging
|
|
10
10
|
|
|
@@ -27,7 +27,8 @@ class Engine:
|
|
|
27
27
|
self.__schema_locked = schema_locked
|
|
28
28
|
|
|
29
29
|
def __str__(self):
|
|
30
|
-
|
|
30
|
+
safe_config = mask_config_for_display(self.config)
|
|
31
|
+
return f"[{self.sql.server}] engine({safe_config})"
|
|
31
32
|
|
|
32
33
|
def connect(self):
|
|
33
34
|
"""
|
velocity/db/core/table.py
CHANGED
|
@@ -24,7 +24,24 @@ class Query:
|
|
|
24
24
|
return self.sql
|
|
25
25
|
|
|
26
26
|
|
|
27
|
+
SYSTEM_COLUMN_NAMES = (
|
|
28
|
+
"sys_id",
|
|
29
|
+
"sys_created",
|
|
30
|
+
"sys_modified",
|
|
31
|
+
"sys_modified_by",
|
|
32
|
+
"sys_modified_row",
|
|
33
|
+
"sys_modified_count",
|
|
34
|
+
"sys_dirty",
|
|
35
|
+
"sys_table",
|
|
36
|
+
"description",
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
_SYSTEM_COLUMN_SET = {name.lower() for name in SYSTEM_COLUMN_NAMES}
|
|
40
|
+
|
|
41
|
+
|
|
27
42
|
class Table:
|
|
43
|
+
SYSTEM_COLUMNS = SYSTEM_COLUMN_NAMES
|
|
44
|
+
|
|
28
45
|
"""
|
|
29
46
|
Provides an interface for performing CRUD and metadata operations on a DB table.
|
|
30
47
|
"""
|
|
@@ -94,9 +111,15 @@ class Table:
|
|
|
94
111
|
|
|
95
112
|
def columns(self):
|
|
96
113
|
"""
|
|
97
|
-
Returns column names
|
|
114
|
+
Returns non-system column names.
|
|
98
115
|
"""
|
|
99
|
-
return [col for col in self.sys_columns() if not
|
|
116
|
+
return [col for col in self.sys_columns() if not self.is_system_column(col)]
|
|
117
|
+
|
|
118
|
+
@staticmethod
|
|
119
|
+
def is_system_column(column_name):
|
|
120
|
+
if not column_name:
|
|
121
|
+
return False
|
|
122
|
+
return column_name.lower() in _SYSTEM_COLUMN_SET or column_name.lower().startswith("sys_")
|
|
100
123
|
|
|
101
124
|
@return_default(None, (exceptions.DbObjectExistsError,))
|
|
102
125
|
def create_index(
|
|
@@ -217,12 +240,8 @@ class Table:
|
|
|
217
240
|
return self.name in [f"{x[0]}.{x[1]}" for x in result.as_tuple()]
|
|
218
241
|
return self.name in [x[1] for x in result.as_tuple()]
|
|
219
242
|
|
|
220
|
-
def
|
|
221
|
-
"""
|
|
222
|
-
Ensure the sys_modified_count column and trigger exist for this table.
|
|
223
|
-
|
|
224
|
-
Returns early when the column is already present unless `force=True` is provided.
|
|
225
|
-
"""
|
|
243
|
+
def ensure_system_columns(self, **kwds):
|
|
244
|
+
"""Ensure Velocity system columns and triggers exist for this table."""
|
|
226
245
|
force = kwds.get("force", False)
|
|
227
246
|
|
|
228
247
|
try:
|
|
@@ -230,15 +249,21 @@ class Table:
|
|
|
230
249
|
except Exception:
|
|
231
250
|
columns = []
|
|
232
251
|
|
|
233
|
-
|
|
234
|
-
has_row_column = "sys_modified_row" in columns
|
|
252
|
+
sql_method = getattr(self.sql, "ensure_system_columns", None)
|
|
235
253
|
|
|
236
|
-
if
|
|
237
|
-
|
|
254
|
+
if sql_method is None:
|
|
255
|
+
raise AttributeError(
|
|
256
|
+
f"{self.sql.__class__.__name__} does not implement ensure_system_columns"
|
|
257
|
+
)
|
|
238
258
|
|
|
239
|
-
|
|
240
|
-
self.name,
|
|
259
|
+
result = sql_method(
|
|
260
|
+
self.name, existing_columns=columns, force=force
|
|
241
261
|
)
|
|
262
|
+
|
|
263
|
+
if not result:
|
|
264
|
+
return
|
|
265
|
+
|
|
266
|
+
sql, vals = result
|
|
242
267
|
if kwds.get("sql_only", False):
|
|
243
268
|
return sql, vals
|
|
244
269
|
self.tx.execute(sql, vals, cursor=self.cursor())
|
|
@@ -321,29 +346,65 @@ class Table:
|
|
|
321
346
|
sys_id = self.tx.execute(sql, vals).scalar()
|
|
322
347
|
return self.row(sys_id, lock=lock)
|
|
323
348
|
|
|
349
|
+
def _normalize_lookup_where(self, where):
|
|
350
|
+
if where is None:
|
|
351
|
+
raise Exception("None is not allowed as a primary key.")
|
|
352
|
+
if isinstance(where, Row):
|
|
353
|
+
return dict(where.pk)
|
|
354
|
+
if isinstance(where, int):
|
|
355
|
+
return {"sys_id": where}
|
|
356
|
+
if not isinstance(where, Mapping):
|
|
357
|
+
raise TypeError(
|
|
358
|
+
"Lookup criteria must be an int, Row, or mapping of column -> value."
|
|
359
|
+
)
|
|
360
|
+
return dict(where)
|
|
361
|
+
|
|
362
|
+
def _select_sys_ids(
|
|
363
|
+
self,
|
|
364
|
+
where,
|
|
365
|
+
*,
|
|
366
|
+
lock=None,
|
|
367
|
+
orderby=None,
|
|
368
|
+
skip_locked=None,
|
|
369
|
+
limit=2,
|
|
370
|
+
):
|
|
371
|
+
select_kwargs = {
|
|
372
|
+
"where": where,
|
|
373
|
+
"lock": lock,
|
|
374
|
+
"orderby": orderby,
|
|
375
|
+
"skip_locked": skip_locked,
|
|
376
|
+
}
|
|
377
|
+
if limit is not None:
|
|
378
|
+
select_kwargs["qty"] = limit
|
|
379
|
+
return self.select("sys_id", **select_kwargs).all()
|
|
380
|
+
|
|
381
|
+
def _clean_where_for_insert(self, where):
|
|
382
|
+
clean = {}
|
|
383
|
+
for key, val in where.items():
|
|
384
|
+
if not isinstance(key, str):
|
|
385
|
+
continue
|
|
386
|
+
if set("<>!=%").intersection(key):
|
|
387
|
+
continue
|
|
388
|
+
clean.setdefault(key, val)
|
|
389
|
+
return clean
|
|
390
|
+
|
|
324
391
|
def get(self, where, lock=None, use_where=False):
|
|
325
392
|
"""
|
|
326
393
|
Gets or creates a row matching `where`. If multiple rows match, raises DuplicateRowsFoundError.
|
|
327
394
|
If none match, a new row is created with the non-operator aspects of `where`.
|
|
328
395
|
"""
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
if isinstance(where, int):
|
|
332
|
-
where = {"sys_id": where}
|
|
333
|
-
result = self.select("sys_id", where=where, lock=lock).all()
|
|
396
|
+
lookup = self._normalize_lookup_where(where)
|
|
397
|
+
result = self._select_sys_ids(lookup, lock=lock, limit=2)
|
|
334
398
|
if len(result) > 1:
|
|
335
|
-
sql = self.select("sys_id", sql_only=True, where=
|
|
399
|
+
sql = self.select("sys_id", sql_only=True, where=lookup, lock=lock)
|
|
336
400
|
raise exceptions.DuplicateRowsFoundError(
|
|
337
401
|
f"More than one entry found. {sql}"
|
|
338
402
|
)
|
|
339
403
|
if not result:
|
|
340
|
-
new_data =
|
|
341
|
-
for k in list(new_data.keys()):
|
|
342
|
-
if set("<>!=%").intersection(k):
|
|
343
|
-
new_data.pop(k)
|
|
404
|
+
new_data = self._clean_where_for_insert(lookup)
|
|
344
405
|
return self.new(new_data, lock=lock)
|
|
345
406
|
if use_where:
|
|
346
|
-
return Row(self,
|
|
407
|
+
return Row(self, lookup, lock=lock)
|
|
347
408
|
return Row(self, result[0]["sys_id"], lock=lock)
|
|
348
409
|
|
|
349
410
|
@return_default(None)
|
|
@@ -352,24 +413,21 @@ class Table:
|
|
|
352
413
|
Finds a single row matching `where`, or returns None if none found unless
|
|
353
414
|
``raise_if_missing`` is True. Raises DuplicateRowsFoundError if multiple rows match.
|
|
354
415
|
"""
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
if isinstance(where, int):
|
|
358
|
-
where = {"sys_id": where}
|
|
359
|
-
result = self.select("sys_id", where=where, lock=lock).all()
|
|
416
|
+
lookup = self._normalize_lookup_where(where)
|
|
417
|
+
result = self._select_sys_ids(lookup, lock=lock, limit=2)
|
|
360
418
|
if not result:
|
|
361
419
|
if raise_if_missing:
|
|
362
420
|
raise LookupError(
|
|
363
|
-
f"No rows found in `{self.name}` for criteria: {
|
|
421
|
+
f"No rows found in `{self.name}` for criteria: {lookup!r}"
|
|
364
422
|
)
|
|
365
423
|
return None
|
|
366
424
|
if len(result) > 1:
|
|
367
|
-
sql = self.select("sys_id", sql_only=True, where=
|
|
425
|
+
sql = self.select("sys_id", sql_only=True, where=lookup, lock=lock)
|
|
368
426
|
raise exceptions.DuplicateRowsFoundError(
|
|
369
427
|
f"More than one entry found. {sql}"
|
|
370
428
|
)
|
|
371
429
|
if use_where:
|
|
372
|
-
return Row(self,
|
|
430
|
+
return Row(self, lookup, lock=lock)
|
|
373
431
|
return Row(self, result[0]["sys_id"], lock=lock)
|
|
374
432
|
|
|
375
433
|
one = find
|
|
@@ -387,23 +445,21 @@ class Table:
|
|
|
387
445
|
"""
|
|
388
446
|
Finds the first matching row (by `orderby`) or creates one if `create_new=True` and none found.
|
|
389
447
|
"""
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
448
|
+
lookup = self._normalize_lookup_where(where)
|
|
449
|
+
results = self._select_sys_ids(
|
|
450
|
+
lookup,
|
|
451
|
+
lock=lock,
|
|
452
|
+
orderby=orderby,
|
|
453
|
+
skip_locked=skip_locked,
|
|
454
|
+
limit=1,
|
|
455
|
+
)
|
|
397
456
|
if not results:
|
|
398
457
|
if create_new:
|
|
399
|
-
new_data =
|
|
400
|
-
for k in list(new_data.keys()):
|
|
401
|
-
if set("<>!=%").intersection(k):
|
|
402
|
-
new_data.pop(k)
|
|
458
|
+
new_data = self._clean_where_for_insert(lookup)
|
|
403
459
|
return self.new(new_data, lock=lock)
|
|
404
460
|
return None
|
|
405
461
|
if use_where:
|
|
406
|
-
return Row(self,
|
|
462
|
+
return Row(self, lookup, lock=lock)
|
|
407
463
|
return Row(self, results[0]["sys_id"], lock=lock)
|
|
408
464
|
|
|
409
465
|
def primary_keys(self):
|
|
@@ -484,10 +540,8 @@ class Table:
|
|
|
484
540
|
if not isinstance(columns, dict):
|
|
485
541
|
raise Exception("Columns must be a dict.")
|
|
486
542
|
columns = self.lower_keys(columns)
|
|
487
|
-
|
|
488
|
-
for
|
|
489
|
-
if k not in self.sys_columns():
|
|
490
|
-
diff.append(k)
|
|
543
|
+
existing_columns = {col.lower() for col in self.sys_columns()}
|
|
544
|
+
diff = [col for col in columns.keys() if col not in existing_columns]
|
|
491
545
|
if diff:
|
|
492
546
|
newcols = {key: columns[key] for key in diff}
|
|
493
547
|
sql, vals = self.sql.alter_add(self.name, newcols)
|
velocity/db/core/transaction.py
CHANGED
|
@@ -6,6 +6,7 @@ from velocity.db.core.result import Result
|
|
|
6
6
|
from velocity.db.core.column import Column
|
|
7
7
|
from velocity.db.core.database import Database
|
|
8
8
|
from velocity.db.core.sequence import Sequence
|
|
9
|
+
from velocity.db.utils import mask_config_for_display
|
|
9
10
|
from velocity.misc.db import randomword
|
|
10
11
|
|
|
11
12
|
debug = False
|
|
@@ -22,10 +23,14 @@ class Transaction:
|
|
|
22
23
|
self.__pg_types = {}
|
|
23
24
|
|
|
24
25
|
def __str__(self):
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
config = mask_config_for_display(self.engine.config)
|
|
27
|
+
|
|
28
|
+
if isinstance(config, dict):
|
|
29
|
+
server = config.get("host", config.get("server"))
|
|
30
|
+
database = config.get("database", config.get("dbname"))
|
|
31
|
+
return f"{self.engine.sql.server}.transaction({server}:{database})"
|
|
32
|
+
|
|
33
|
+
return f"{self.engine.sql.server}.transaction({config})"
|
|
29
34
|
|
|
30
35
|
def __enter__(self):
|
|
31
36
|
return self
|
velocity/db/servers/mysql/sql.py
CHANGED
|
@@ -450,19 +450,40 @@ END;
|
|
|
450
450
|
return "\n".join(statements), tuple()
|
|
451
451
|
|
|
452
452
|
@classmethod
|
|
453
|
-
def
|
|
454
|
-
"""Ensure
|
|
453
|
+
def ensure_system_columns(cls, name, existing_columns=None, force=False):
|
|
454
|
+
"""Ensure MySQL tables maintain the Velocity system metadata."""
|
|
455
|
+
existing_columns = {col.lower() for col in existing_columns or []}
|
|
456
|
+
|
|
455
457
|
table_identifier = quote(name)
|
|
456
458
|
base_name = name.split(".")[-1].replace("`", "")
|
|
457
459
|
base_name_sql = base_name.replace("'", "''")
|
|
458
460
|
trigger_prefix = re.sub(r"[^0-9A-Za-z_]+", "_", f"cc_sysmod_{base_name}")
|
|
459
461
|
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
462
|
+
has_count = "sys_modified_count" in existing_columns
|
|
463
|
+
|
|
464
|
+
add_column = not has_count
|
|
465
|
+
recreate_triggers = force or add_column
|
|
466
|
+
|
|
467
|
+
if not recreate_triggers and not force:
|
|
468
|
+
return None
|
|
469
|
+
|
|
470
|
+
statements = []
|
|
471
|
+
|
|
472
|
+
if add_column:
|
|
473
|
+
statements.append(
|
|
474
|
+
f"ALTER TABLE {table_identifier} ADD COLUMN IF NOT EXISTS `sys_modified_count` INT NOT NULL DEFAULT 0;"
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
statements.append(
|
|
478
|
+
f"UPDATE {table_identifier} SET `sys_modified_count` = 0 WHERE `sys_modified_count` IS NULL;"
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
statements.append(f"DROP TRIGGER IF EXISTS {trigger_prefix}_bi;")
|
|
482
|
+
statements.append(f"DROP TRIGGER IF EXISTS {trigger_prefix}_bu;")
|
|
483
|
+
|
|
484
|
+
statements.extend(
|
|
485
|
+
[
|
|
486
|
+
f"""
|
|
466
487
|
CREATE TRIGGER {trigger_prefix}_bi
|
|
467
488
|
BEFORE INSERT ON {table_identifier}
|
|
468
489
|
FOR EACH ROW
|
|
@@ -474,7 +495,7 @@ BEGIN
|
|
|
474
495
|
SET NEW.sys_table = '{base_name_sql}';
|
|
475
496
|
END;
|
|
476
497
|
""".strip(),
|
|
477
|
-
|
|
498
|
+
f"""
|
|
478
499
|
CREATE TRIGGER {trigger_prefix}_bu
|
|
479
500
|
BEFORE UPDATE ON {table_identifier}
|
|
480
501
|
FOR EACH ROW
|
|
@@ -491,7 +512,8 @@ BEGIN
|
|
|
491
512
|
SET NEW.sys_table = '{base_name_sql}';
|
|
492
513
|
END;
|
|
493
514
|
""".strip(),
|
|
494
|
-
|
|
515
|
+
]
|
|
516
|
+
)
|
|
495
517
|
|
|
496
518
|
return "\n".join(statements), tuple()
|
|
497
519
|
|
|
@@ -23,6 +23,8 @@ system_fields = [
|
|
|
23
23
|
"sys_created",
|
|
24
24
|
"sys_modified",
|
|
25
25
|
"sys_modified_by",
|
|
26
|
+
"sys_modified_row",
|
|
27
|
+
"sys_modified_count",
|
|
26
28
|
"sys_dirty",
|
|
27
29
|
"sys_table",
|
|
28
30
|
"description",
|
|
@@ -799,37 +801,13 @@ class SQL(BaseSQLDialect):
|
|
|
799
801
|
def drop_database(cls, name):
|
|
800
802
|
return f"drop database if exists {name}", tuple()
|
|
801
803
|
|
|
802
|
-
@
|
|
803
|
-
def
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
else:
|
|
807
|
-
fqtn = f"public.{TableHelper.quote(name)}"
|
|
808
|
-
schema, table = fqtn.split(".")
|
|
809
|
-
name = fqtn.replace(".", "_")
|
|
810
|
-
sql = []
|
|
811
|
-
if drop:
|
|
812
|
-
sql.append(cls.drop_table(fqtn)[0])
|
|
813
|
-
sql.append(
|
|
814
|
-
f"""
|
|
815
|
-
CREATE TABLE {fqtn} (
|
|
816
|
-
sys_id BIGSERIAL PRIMARY KEY,
|
|
817
|
-
sys_created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
818
|
-
sys_modified TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
819
|
-
sys_modified_by TEXT NOT NULL DEFAULT 'SYSTEM',
|
|
820
|
-
sys_modified_row TIMESTAMP NOT NULL DEFAULT CLOCK_TIMESTAMP(),
|
|
821
|
-
sys_modified_count INTEGER NOT NULL DEFAULT 0,
|
|
822
|
-
sys_dirty BOOLEAN NOT NULL DEFAULT FALSE,
|
|
823
|
-
sys_table TEXT NOT NULL,
|
|
824
|
-
description TEXT
|
|
825
|
-
);
|
|
826
|
-
|
|
827
|
-
SELECT SETVAL(PG_GET_SERIAL_SEQUENCE('{fqtn}', 'sys_id'),1000,TRUE);
|
|
828
|
-
|
|
829
|
-
CREATE OR REPLACE FUNCTION {schema}.on_sys_modified()
|
|
804
|
+
@staticmethod
|
|
805
|
+
def _sys_modified_function_sql(schema_identifier):
|
|
806
|
+
return f"""
|
|
807
|
+
CREATE OR REPLACE FUNCTION {schema_identifier}.on_sys_modified()
|
|
830
808
|
RETURNS TRIGGER AS
|
|
831
809
|
$BODY$
|
|
832
|
-
|
|
810
|
+
BEGIN
|
|
833
811
|
IF (TG_OP = 'INSERT') THEN
|
|
834
812
|
NEW.sys_table := TG_TABLE_NAME;
|
|
835
813
|
NEW.sys_created := transaction_timestamp();
|
|
@@ -847,19 +825,58 @@ class SQL(BaseSQLDialect):
|
|
|
847
825
|
NEW.sys_dirty := TRUE;
|
|
848
826
|
END IF;
|
|
849
827
|
NEW.sys_modified := transaction_timestamp();
|
|
828
|
+
NEW.sys_modified_row := clock_timestamp();
|
|
850
829
|
NEW.sys_modified_count := COALESCE(OLD.sys_modified_count, 0) + 1;
|
|
851
830
|
END IF;
|
|
852
831
|
END IF;
|
|
853
|
-
|
|
854
832
|
RETURN NEW;
|
|
855
|
-
|
|
833
|
+
END;
|
|
856
834
|
$BODY$
|
|
857
835
|
LANGUAGE plpgsql VOLATILE
|
|
858
836
|
COST 100;
|
|
837
|
+
"""
|
|
838
|
+
|
|
839
|
+
@classmethod
|
|
840
|
+
def create_table(cls, name, columns={}, drop=False):
|
|
841
|
+
if "." in name:
|
|
842
|
+
fqtn = TableHelper.quote(name)
|
|
843
|
+
else:
|
|
844
|
+
fqtn = f"public.{TableHelper.quote(name)}"
|
|
859
845
|
|
|
860
|
-
|
|
846
|
+
schema, table = fqtn.split(".")
|
|
847
|
+
schema_unquoted = schema.replace('"', "")
|
|
848
|
+
table_unquoted = table.replace('"', "")
|
|
849
|
+
trigger_name = (
|
|
850
|
+
f"on_update_row_{schema_unquoted}_{table_unquoted}".replace(".", "_")
|
|
851
|
+
)
|
|
852
|
+
trigger_identifier = TableHelper.quote(trigger_name)
|
|
853
|
+
schema_identifier = TableHelper.quote(schema_unquoted)
|
|
854
|
+
sql = []
|
|
855
|
+
if drop:
|
|
856
|
+
sql.append(cls.drop_table(fqtn)[0])
|
|
857
|
+
sql.append(
|
|
858
|
+
f"""
|
|
859
|
+
CREATE TABLE {fqtn} (
|
|
860
|
+
sys_id BIGSERIAL PRIMARY KEY,
|
|
861
|
+
sys_created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
862
|
+
sys_modified TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
863
|
+
sys_modified_by TEXT NOT NULL DEFAULT 'SYSTEM',
|
|
864
|
+
sys_modified_row TIMESTAMP NOT NULL DEFAULT CLOCK_TIMESTAMP(),
|
|
865
|
+
sys_modified_count INTEGER NOT NULL DEFAULT 0,
|
|
866
|
+
sys_dirty BOOLEAN NOT NULL DEFAULT FALSE,
|
|
867
|
+
sys_table TEXT NOT NULL,
|
|
868
|
+
description TEXT
|
|
869
|
+
);
|
|
870
|
+
|
|
871
|
+
SELECT SETVAL(PG_GET_SERIAL_SEQUENCE('{fqtn}', 'sys_id'),1000,TRUE);
|
|
872
|
+
|
|
873
|
+
{cls._sys_modified_function_sql(schema_identifier)}
|
|
874
|
+
|
|
875
|
+
DROP TRIGGER IF EXISTS {trigger_identifier} ON {fqtn};
|
|
876
|
+
|
|
877
|
+
CREATE TRIGGER {trigger_identifier}
|
|
861
878
|
BEFORE INSERT OR UPDATE ON {fqtn}
|
|
862
|
-
FOR EACH ROW EXECUTE PROCEDURE {
|
|
879
|
+
FOR EACH ROW EXECUTE PROCEDURE {schema_identifier}.on_sys_modified();
|
|
863
880
|
|
|
864
881
|
"""
|
|
865
882
|
)
|
|
@@ -876,81 +893,138 @@ class SQL(BaseSQLDialect):
|
|
|
876
893
|
return sql, tuple()
|
|
877
894
|
|
|
878
895
|
@classmethod
|
|
879
|
-
def
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
896
|
+
def ensure_system_columns(cls, name, existing_columns=None, force=False):
|
|
897
|
+
"""Ensure all Velocity system columns and triggers exist for the table."""
|
|
898
|
+
existing_columns = {
|
|
899
|
+
col.lower() for col in (existing_columns or [])
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
required_columns = [
|
|
903
|
+
"sys_id",
|
|
904
|
+
"sys_created",
|
|
905
|
+
"sys_modified",
|
|
906
|
+
"sys_modified_by",
|
|
907
|
+
"sys_modified_row",
|
|
908
|
+
"sys_modified_count",
|
|
909
|
+
"sys_dirty",
|
|
910
|
+
"sys_table",
|
|
911
|
+
"description",
|
|
912
|
+
]
|
|
913
|
+
|
|
914
|
+
missing_columns = [
|
|
915
|
+
column for column in required_columns if column not in existing_columns
|
|
916
|
+
]
|
|
917
|
+
|
|
918
|
+
if not missing_columns and not force:
|
|
919
|
+
return None
|
|
920
|
+
|
|
883
921
|
if "." in name:
|
|
884
922
|
schema_name, table_name = name.split(".", 1)
|
|
885
923
|
else:
|
|
886
924
|
schema_name = cls.default_schema
|
|
887
925
|
table_name = name
|
|
888
926
|
|
|
889
|
-
|
|
890
|
-
|
|
927
|
+
schema_unquoted = schema_name.replace('"', "")
|
|
928
|
+
table_unquoted = table_name.replace('"', "")
|
|
929
|
+
|
|
930
|
+
schema_identifier = TableHelper.quote(schema_unquoted)
|
|
931
|
+
table_identifier = TableHelper.quote(table_unquoted)
|
|
891
932
|
fqtn = f"{schema_identifier}.{table_identifier}"
|
|
892
933
|
|
|
893
934
|
trigger_name = (
|
|
894
|
-
f"on_update_row_{
|
|
895
|
-
.replace(".", "_")
|
|
896
|
-
.replace('"', "")
|
|
935
|
+
f"on_update_row_{schema_unquoted}_{table_unquoted}".replace(".", "_")
|
|
897
936
|
)
|
|
898
937
|
trigger_identifier = TableHelper.quote(trigger_name)
|
|
899
|
-
|
|
900
|
-
row_column_name = TableHelper.quote("sys_modified_row")
|
|
901
|
-
|
|
938
|
+
|
|
902
939
|
statements = []
|
|
903
|
-
|
|
940
|
+
added_columns = set()
|
|
941
|
+
columns_after = set(existing_columns)
|
|
942
|
+
|
|
943
|
+
if "sys_id" in missing_columns:
|
|
904
944
|
statements.append(
|
|
905
|
-
f"ALTER TABLE {fqtn} ADD COLUMN {
|
|
945
|
+
f"ALTER TABLE {fqtn} ADD COLUMN {TableHelper.quote('sys_id')} BIGSERIAL PRIMARY KEY;"
|
|
906
946
|
)
|
|
907
|
-
|
|
947
|
+
added_columns.add("sys_id")
|
|
948
|
+
columns_after.add("sys_id")
|
|
949
|
+
|
|
950
|
+
column_definitions = {
|
|
951
|
+
"sys_created": "TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP",
|
|
952
|
+
"sys_modified": "TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP",
|
|
953
|
+
"sys_modified_by": "TEXT NOT NULL DEFAULT 'SYSTEM'",
|
|
954
|
+
"sys_modified_row": "TIMESTAMP NOT NULL DEFAULT CLOCK_TIMESTAMP()",
|
|
955
|
+
"sys_modified_count": "INTEGER NOT NULL DEFAULT 0",
|
|
956
|
+
"sys_dirty": "BOOLEAN NOT NULL DEFAULT FALSE",
|
|
957
|
+
"sys_table": "TEXT",
|
|
958
|
+
"description": "TEXT",
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
for column, definition in column_definitions.items():
|
|
962
|
+
if column in missing_columns:
|
|
963
|
+
statements.append(
|
|
964
|
+
f"ALTER TABLE {fqtn} ADD COLUMN {TableHelper.quote(column)} {definition};"
|
|
965
|
+
)
|
|
966
|
+
added_columns.add(column)
|
|
967
|
+
columns_after.add(column)
|
|
968
|
+
|
|
969
|
+
default_map = {
|
|
970
|
+
"sys_created": "CURRENT_TIMESTAMP",
|
|
971
|
+
"sys_modified": "CURRENT_TIMESTAMP",
|
|
972
|
+
"sys_modified_by": "'SYSTEM'",
|
|
973
|
+
"sys_modified_row": "CLOCK_TIMESTAMP()",
|
|
974
|
+
"sys_modified_count": "0",
|
|
975
|
+
"sys_dirty": "FALSE",
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
for column, default_sql in default_map.items():
|
|
979
|
+
if column in columns_after and (force or column in added_columns):
|
|
980
|
+
quoted_column = TableHelper.quote(column)
|
|
981
|
+
statements.append(
|
|
982
|
+
f"UPDATE {fqtn} SET {quoted_column} = {default_sql} WHERE {quoted_column} IS NULL;"
|
|
983
|
+
)
|
|
984
|
+
statements.append(
|
|
985
|
+
f"ALTER TABLE {fqtn} ALTER COLUMN {quoted_column} SET DEFAULT {default_sql};"
|
|
986
|
+
)
|
|
987
|
+
|
|
988
|
+
if "sys_table" in columns_after and (force or "sys_table" in added_columns):
|
|
989
|
+
quoted_column = TableHelper.quote("sys_table")
|
|
990
|
+
table_literal = table_unquoted.replace("'", "''")
|
|
908
991
|
statements.append(
|
|
909
|
-
f"
|
|
992
|
+
f"UPDATE {fqtn} SET {quoted_column} = COALESCE({quoted_column}, '{table_literal}') WHERE {quoted_column} IS NULL;"
|
|
910
993
|
)
|
|
911
994
|
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
COST 100;
|
|
946
|
-
""",
|
|
947
|
-
f"DROP TRIGGER IF EXISTS {trigger_identifier} ON {fqtn};",
|
|
948
|
-
f"""
|
|
949
|
-
CREATE TRIGGER {trigger_identifier}
|
|
950
|
-
BEFORE INSERT OR UPDATE ON {fqtn}
|
|
951
|
-
FOR EACH ROW EXECUTE PROCEDURE {schema_identifier}.on_sys_modified();
|
|
952
|
-
""",
|
|
953
|
-
])
|
|
995
|
+
not_null_columns = {
|
|
996
|
+
"sys_created",
|
|
997
|
+
"sys_modified",
|
|
998
|
+
"sys_modified_by",
|
|
999
|
+
"sys_modified_row",
|
|
1000
|
+
"sys_modified_count",
|
|
1001
|
+
"sys_dirty",
|
|
1002
|
+
"sys_table",
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
for column in not_null_columns:
|
|
1006
|
+
if column in columns_after and (force or column in added_columns):
|
|
1007
|
+
statements.append(
|
|
1008
|
+
f"ALTER TABLE {fqtn} ALTER COLUMN {TableHelper.quote(column)} SET NOT NULL;"
|
|
1009
|
+
)
|
|
1010
|
+
|
|
1011
|
+
reset_trigger = force or bool(added_columns)
|
|
1012
|
+
|
|
1013
|
+
if reset_trigger:
|
|
1014
|
+
statements.append(
|
|
1015
|
+
f"DROP TRIGGER IF EXISTS {trigger_identifier} ON {fqtn};"
|
|
1016
|
+
)
|
|
1017
|
+
statements.append(cls._sys_modified_function_sql(schema_identifier))
|
|
1018
|
+
statements.append(
|
|
1019
|
+
f"""
|
|
1020
|
+
CREATE TRIGGER {trigger_identifier}
|
|
1021
|
+
BEFORE INSERT OR UPDATE ON {fqtn}
|
|
1022
|
+
FOR EACH ROW EXECUTE PROCEDURE {schema_identifier}.on_sys_modified();
|
|
1023
|
+
"""
|
|
1024
|
+
)
|
|
1025
|
+
|
|
1026
|
+
if not statements:
|
|
1027
|
+
return None
|
|
954
1028
|
|
|
955
1029
|
sql = sqlparse.format(
|
|
956
1030
|
" ".join(statements), reindent=True, keyword_case="upper"
|
|
@@ -431,18 +431,40 @@ END;
|
|
|
431
431
|
return "\n".join(statements), tuple()
|
|
432
432
|
|
|
433
433
|
@classmethod
|
|
434
|
-
def
|
|
435
|
-
"""Ensure
|
|
434
|
+
def ensure_system_columns(cls, name, existing_columns=None, force=False):
|
|
435
|
+
"""Ensure SQLite tables maintain the Velocity system triggers/columns."""
|
|
436
|
+
existing_columns = {col.lower() for col in existing_columns or []}
|
|
437
|
+
|
|
436
438
|
table_identifier = quote(name)
|
|
437
439
|
base_name = name.split(".")[-1].replace('"', "")
|
|
438
440
|
base_name_sql = base_name.replace("'", "''")
|
|
439
441
|
trigger_prefix = re.sub(r"[^0-9A-Za-z_]+", "_", f"cc_sysmod_{base_name}")
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
442
|
+
|
|
443
|
+
has_count = "sys_modified_count" in existing_columns
|
|
444
|
+
|
|
445
|
+
add_column = not has_count
|
|
446
|
+
recreate_triggers = force or add_column
|
|
447
|
+
|
|
448
|
+
if not recreate_triggers and not force:
|
|
449
|
+
return None
|
|
450
|
+
|
|
451
|
+
statements = []
|
|
452
|
+
|
|
453
|
+
if add_column:
|
|
454
|
+
statements.append(
|
|
455
|
+
f"ALTER TABLE {table_identifier} ADD COLUMN sys_modified_count INTEGER NOT NULL DEFAULT 0;"
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
statements.append(
|
|
459
|
+
f"UPDATE {table_identifier} SET sys_modified_count = 0 WHERE sys_modified_count IS NULL;"
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
statements.append(f"DROP TRIGGER IF EXISTS {trigger_prefix}_ai;")
|
|
463
|
+
statements.append(f"DROP TRIGGER IF EXISTS {trigger_prefix}_au;")
|
|
464
|
+
|
|
465
|
+
statements.extend(
|
|
466
|
+
[
|
|
467
|
+
f"""
|
|
446
468
|
CREATE TRIGGER {trigger_prefix}_ai
|
|
447
469
|
AFTER INSERT ON {table_identifier}
|
|
448
470
|
FOR EACH ROW
|
|
@@ -456,7 +478,7 @@ BEGIN
|
|
|
456
478
|
WHERE rowid = NEW.rowid;
|
|
457
479
|
END;
|
|
458
480
|
""".strip(),
|
|
459
|
-
|
|
481
|
+
f"""
|
|
460
482
|
CREATE TRIGGER {trigger_prefix}_au
|
|
461
483
|
AFTER UPDATE ON {table_identifier}
|
|
462
484
|
FOR EACH ROW
|
|
@@ -470,7 +492,9 @@ BEGIN
|
|
|
470
492
|
WHERE rowid = NEW.rowid;
|
|
471
493
|
END;
|
|
472
494
|
""".strip(),
|
|
473
|
-
|
|
495
|
+
]
|
|
496
|
+
)
|
|
497
|
+
|
|
474
498
|
return "\n".join(statements), tuple()
|
|
475
499
|
|
|
476
500
|
@classmethod
|
|
@@ -485,10 +485,10 @@ END;
|
|
|
485
485
|
return "\n".join(statements), tuple()
|
|
486
486
|
|
|
487
487
|
@classmethod
|
|
488
|
-
def
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
488
|
+
def ensure_system_columns(cls, name, existing_columns=None, force=False):
|
|
489
|
+
"""Ensure SQL Server tables maintain Velocity system metadata."""
|
|
490
|
+
existing_columns = {col.lower() for col in existing_columns or []}
|
|
491
|
+
|
|
492
492
|
if "." in name:
|
|
493
493
|
schema, table_name = name.split(".", 1)
|
|
494
494
|
else:
|
|
@@ -501,17 +501,35 @@ END;
|
|
|
501
501
|
table_name_sql = table_name.replace("'", "''")
|
|
502
502
|
trigger_prefix = re.sub(r"[^0-9A-Za-z_]+", "_", f"CC_SYS_MOD_{table_name}")
|
|
503
503
|
|
|
504
|
+
has_count = "sys_modified_count" in existing_columns
|
|
505
|
+
|
|
506
|
+
add_column = not has_count
|
|
507
|
+
recreate_triggers = force or add_column
|
|
508
|
+
|
|
509
|
+
if not recreate_triggers and not force:
|
|
510
|
+
return None
|
|
511
|
+
|
|
504
512
|
statements = []
|
|
505
|
-
|
|
513
|
+
|
|
514
|
+
if add_column:
|
|
506
515
|
statements.append(
|
|
507
516
|
f"IF COL_LENGTH(N'{object_name}', 'sys_modified_count') IS NULL BEGIN ALTER TABLE {table_identifier} ADD sys_modified_count INT NOT NULL CONSTRAINT DF_{trigger_prefix}_COUNT DEFAULT (0); END;"
|
|
508
517
|
)
|
|
509
518
|
|
|
510
|
-
statements.
|
|
511
|
-
f"UPDATE {table_identifier} SET sys_modified_count = 0 WHERE sys_modified_count IS NULL;"
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
519
|
+
statements.append(
|
|
520
|
+
f"UPDATE {table_identifier} SET sys_modified_count = 0 WHERE sys_modified_count IS NULL;"
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
statements.append(
|
|
524
|
+
f"IF OBJECT_ID(N'{schema_identifier}.{trigger_prefix}_insert', N'TR') IS NOT NULL DROP TRIGGER {schema_identifier}.{trigger_prefix}_insert;"
|
|
525
|
+
)
|
|
526
|
+
statements.append(
|
|
527
|
+
f"IF OBJECT_ID(N'{schema_identifier}.{trigger_prefix}_update', N'TR') IS NOT NULL DROP TRIGGER {schema_identifier}.{trigger_prefix}_update;"
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
statements.extend(
|
|
531
|
+
[
|
|
532
|
+
f"""
|
|
515
533
|
CREATE TRIGGER {schema_identifier}.{trigger_prefix}_insert
|
|
516
534
|
ON {table_identifier}
|
|
517
535
|
AFTER INSERT
|
|
@@ -528,7 +546,7 @@ BEGIN
|
|
|
528
546
|
INNER JOIN inserted AS i ON t.sys_id = i.sys_id;
|
|
529
547
|
END;
|
|
530
548
|
""".strip(),
|
|
531
|
-
|
|
549
|
+
f"""
|
|
532
550
|
CREATE TRIGGER {schema_identifier}.{trigger_prefix}_update
|
|
533
551
|
ON {table_identifier}
|
|
534
552
|
AFTER UPDATE
|
|
@@ -546,7 +564,8 @@ BEGIN
|
|
|
546
564
|
INNER JOIN deleted AS d ON d.sys_id = i.sys_id;
|
|
547
565
|
END;
|
|
548
566
|
""".strip(),
|
|
549
|
-
|
|
567
|
+
]
|
|
568
|
+
)
|
|
550
569
|
|
|
551
570
|
return "\n".join(statements), tuple()
|
|
552
571
|
|
|
@@ -199,7 +199,9 @@ class TestTableComprehensive(CommonPostgresTest):
|
|
|
199
199
|
|
|
200
200
|
# Test column filtering (non-sys columns)
|
|
201
201
|
filtered_columns = table.columns()
|
|
202
|
-
sys_column_count = len(
|
|
202
|
+
sys_column_count = len(
|
|
203
|
+
[col for col in filtered_columns if table.is_system_column(col)]
|
|
204
|
+
)
|
|
203
205
|
self.assertEqual(sys_column_count, 0)
|
|
204
206
|
|
|
205
207
|
def test_table_row_count(self, tx):
|
|
@@ -11,6 +11,8 @@ from src.velocity.db.utils import (
|
|
|
11
11
|
safe_sort_rows,
|
|
12
12
|
group_by_fields,
|
|
13
13
|
safe_sort_grouped_rows,
|
|
14
|
+
mask_config_for_display,
|
|
15
|
+
mask_sensitive_in_string,
|
|
14
16
|
)
|
|
15
17
|
|
|
16
18
|
|
|
@@ -216,6 +218,53 @@ class TestDatabaseUtils(unittest.TestCase):
|
|
|
216
218
|
expected = ["2024-06", "2024-12", None]
|
|
217
219
|
self.assertEqual(exp_dates, expected)
|
|
218
220
|
|
|
221
|
+
def test_mask_config_for_display_redacts_direct_passwords(self):
|
|
222
|
+
"""Ensure direct password/token fields are masked."""
|
|
223
|
+
config = {
|
|
224
|
+
"host": "db.local",
|
|
225
|
+
"password": "supersecret",
|
|
226
|
+
"token": "abc123",
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
masked = mask_config_for_display(config)
|
|
230
|
+
|
|
231
|
+
self.assertEqual(masked["host"], "db.local")
|
|
232
|
+
self.assertEqual(masked["password"], "*****")
|
|
233
|
+
self.assertEqual(masked["token"], "*****")
|
|
234
|
+
|
|
235
|
+
def test_mask_config_for_display_handles_nested_structures(self):
|
|
236
|
+
"""Verify masking applies to nested dicts, lists, tuples, and DSN strings."""
|
|
237
|
+
config = {
|
|
238
|
+
"options": {
|
|
239
|
+
"passwd": "innersecret",
|
|
240
|
+
"hosts": [
|
|
241
|
+
{"url": "postgresql://user:pwd@localhost/db"},
|
|
242
|
+
("token=xyz",),
|
|
243
|
+
],
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
masked = mask_config_for_display(config)
|
|
248
|
+
|
|
249
|
+
self.assertEqual(masked["options"]["passwd"], "*****")
|
|
250
|
+
self.assertEqual(
|
|
251
|
+
masked["options"]["hosts"][0]["url"],
|
|
252
|
+
"postgresql://user:*****@localhost/db",
|
|
253
|
+
)
|
|
254
|
+
self.assertEqual(masked["options"]["hosts"][1][0], "token=*****")
|
|
255
|
+
|
|
256
|
+
def test_mask_sensitive_in_string_redacts_key_value_pairs(self):
|
|
257
|
+
"""Key/value DSN parameters should be redacted."""
|
|
258
|
+
dsn = "host=db password=abc123;user=test"
|
|
259
|
+
masked = mask_sensitive_in_string(dsn)
|
|
260
|
+
self.assertEqual(masked, "host=db password=*****;user=test")
|
|
261
|
+
|
|
262
|
+
def test_mask_sensitive_in_string_redacts_url_credentials(self):
|
|
263
|
+
"""URL style credentials should hide the password portion."""
|
|
264
|
+
url = "postgresql://user:secret@host/db"
|
|
265
|
+
masked = mask_sensitive_in_string(url)
|
|
266
|
+
self.assertEqual(masked, "postgresql://user:*****@host/db")
|
|
267
|
+
|
|
219
268
|
|
|
220
269
|
if __name__ == "__main__":
|
|
221
270
|
unittest.main()
|
velocity/db/utils.py
CHANGED
|
@@ -1,12 +1,75 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Database utility functions for common operations.
|
|
1
|
+
"""Utility helpers for velocity.db modules.
|
|
3
2
|
|
|
4
|
-
This module provides
|
|
5
|
-
|
|
3
|
+
This module provides helpers for redacting sensitive configuration values along
|
|
4
|
+
with common collection utilities used across the velocity database codebase.
|
|
6
5
|
"""
|
|
7
6
|
|
|
7
|
+
import re
|
|
8
8
|
from typing import Any, Callable, List
|
|
9
9
|
|
|
10
|
+
_SENSITIVE_KEYWORDS = {
|
|
11
|
+
"password",
|
|
12
|
+
"passwd",
|
|
13
|
+
"pwd",
|
|
14
|
+
"secret",
|
|
15
|
+
"token",
|
|
16
|
+
"apikey",
|
|
17
|
+
"api_key",
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
_SENSITIVE_PATTERNS = [
|
|
21
|
+
re.compile(r"(password\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
22
|
+
re.compile(r"(passwd\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
23
|
+
re.compile(r"(pwd\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
24
|
+
re.compile(r"(secret\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
25
|
+
re.compile(r"(token\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
26
|
+
re.compile(r"(api[_-]?key\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
_URL_CREDENTIAL_PATTERN = re.compile(r"(://[^:\s]+:)([^@/\s]+)")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def mask_sensitive_in_string(value: str) -> str:
|
|
33
|
+
"""Return ``value`` with credential-like substrings redacted."""
|
|
34
|
+
|
|
35
|
+
if not value:
|
|
36
|
+
return value
|
|
37
|
+
|
|
38
|
+
masked = value
|
|
39
|
+
for pattern in _SENSITIVE_PATTERNS:
|
|
40
|
+
masked = pattern.sub(lambda match: match.group(1) + "*****", masked)
|
|
41
|
+
|
|
42
|
+
return _URL_CREDENTIAL_PATTERN.sub(r"\1*****", masked)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def mask_config_for_display(config: Any) -> Any:
|
|
46
|
+
"""Return ``config`` with common secret fields masked for logging/str()."""
|
|
47
|
+
|
|
48
|
+
if isinstance(config, dict):
|
|
49
|
+
masked = {}
|
|
50
|
+
for key, value in config.items():
|
|
51
|
+
if isinstance(key, str) and _contains_sensitive_keyword(key):
|
|
52
|
+
masked[key] = "*****"
|
|
53
|
+
else:
|
|
54
|
+
masked[key] = mask_config_for_display(value)
|
|
55
|
+
return masked
|
|
56
|
+
|
|
57
|
+
if isinstance(config, tuple):
|
|
58
|
+
return tuple(mask_config_for_display(item) for item in config)
|
|
59
|
+
|
|
60
|
+
if isinstance(config, list):
|
|
61
|
+
return [mask_config_for_display(item) for item in config]
|
|
62
|
+
|
|
63
|
+
if isinstance(config, str):
|
|
64
|
+
return mask_sensitive_in_string(config)
|
|
65
|
+
|
|
66
|
+
return config
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _contains_sensitive_keyword(key: str) -> bool:
|
|
70
|
+
lowered = key.lower()
|
|
71
|
+
return any(token in lowered for token in _SENSITIVE_KEYWORDS)
|
|
72
|
+
|
|
10
73
|
|
|
11
74
|
def safe_sort_key_none_last(field_name: str) -> Callable[[dict], tuple]:
|
|
12
75
|
"""
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
velocity/__init__.py,sha256=
|
|
1
|
+
velocity/__init__.py,sha256=BblI9g448OMQsducxdjwOZ1xg9Ii6BTbRNFIckwZyAY,147
|
|
2
2
|
velocity/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
3
|
velocity/app/invoices.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
4
|
velocity/app/orders.py,sha256=C7ewngMpO8nD3ul_82o4FhZBdRkWvJtnuEbEJUKDCno,6151
|
|
@@ -28,17 +28,17 @@ velocity/aws/tests/test_lambda_handler_json_serialization.py,sha256=VuikDD_tbRbA
|
|
|
28
28
|
velocity/aws/tests/test_response.py,sha256=sCg6qOSDWNncnmkuf4N3gO5fxQQ3PyFNMRsYy195nKo,6546
|
|
29
29
|
velocity/db/__init__.py,sha256=7XRUHY2af0HL1jvL0SAMpxSe5a2Phbkm-YLJCvC1C_0,739
|
|
30
30
|
velocity/db/exceptions.py,sha256=LbgYJfY2nCBu7tBC_U1BwdczypJ24S6CUyREG_N5gO0,2345
|
|
31
|
-
velocity/db/utils.py,sha256=
|
|
31
|
+
velocity/db/utils.py,sha256=qfnxN67B91pIXqMEp8mP5RuzFlBI20zwMMWBoHHO29g,8864
|
|
32
32
|
velocity/db/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
33
|
velocity/db/core/column.py,sha256=tAr8tL3a2nyaYpNHhGl508FrY_pGZTzyYgjAV5CEBv4,4092
|
|
34
34
|
velocity/db/core/database.py,sha256=3zNGItklu9tZCKsbx2T2vCcU1so8AL9PPL0DLjvaz6s,3554
|
|
35
|
-
velocity/db/core/decorators.py,sha256=
|
|
36
|
-
velocity/db/core/engine.py,sha256=
|
|
35
|
+
velocity/db/core/decorators.py,sha256=GpMw_2BkWvEKgEfsPnCnlN7uyUvIXOJX_TeYCp0O_2I,5804
|
|
36
|
+
velocity/db/core/engine.py,sha256=84jz1taX-t4nqXN99kz9WyczpgsRLGFNnc_hpaCi3lE,20769
|
|
37
37
|
velocity/db/core/result.py,sha256=b0ie3yZAOj9S57x32uFFGKZ95zhImmZ0iXl0X1qYszc,12813
|
|
38
38
|
velocity/db/core/row.py,sha256=GOWm-HEBPCBwdqMHMBRc41m0Hoht4vRVQLkvdogX1fU,7729
|
|
39
39
|
velocity/db/core/sequence.py,sha256=VMBc0ZjGnOaWTwKW6xMNTdP8rZ2umQ8ml4fHTTwuGq4,3904
|
|
40
|
-
velocity/db/core/table.py,sha256=
|
|
41
|
-
velocity/db/core/transaction.py,sha256=
|
|
40
|
+
velocity/db/core/table.py,sha256=BN_tY2bVSLCGmr0el0wAcm6binOxAc1OBvco3DI7_zY,43379
|
|
41
|
+
velocity/db/core/transaction.py,sha256=LDwwAXk666sBdT4J01bvu3rAYfWwxvxVVQUBbQlX-hY,7155
|
|
42
42
|
velocity/db/servers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
43
43
|
velocity/db/servers/tablehelper.py,sha256=Q48ObN5KD_U2sBP0GUcjaQjKeE4Hr351sPQirwQ0_1s,22163
|
|
44
44
|
velocity/db/servers/base/__init__.py,sha256=5--XJUeEAm7O6Ns2C_ODCr5TjFhdAge-zApZCT0LGTQ,285
|
|
@@ -49,27 +49,27 @@ velocity/db/servers/base/types.py,sha256=3LBxFCD35eeIsIqftpAJh0JjUVonDYemz2n6AMt
|
|
|
49
49
|
velocity/db/servers/mysql/__init__.py,sha256=mASO5JB0xkzYngwx2X79yyKifYRqxIdfKFWutIHuw7k,2661
|
|
50
50
|
velocity/db/servers/mysql/operators.py,sha256=wHmVSPxlPGbOdvQEmsfKhD25H8djovSbNcmacLHDVkI,1273
|
|
51
51
|
velocity/db/servers/mysql/reserved.py,sha256=s-aFMwYJpZ_1FBcCMU8fOdhml2ET58-59ZnUm7iw5OU,3312
|
|
52
|
-
velocity/db/servers/mysql/sql.py,sha256=
|
|
52
|
+
velocity/db/servers/mysql/sql.py,sha256=pVUuxQnwxbCDHBltX7vjoVSyGodHm_eYpU0XfF0QwL4,22681
|
|
53
53
|
velocity/db/servers/mysql/types.py,sha256=BMQf4TpsRo1JN-yOl1nSItTO-Juu2piSTNy5o_djBeM,3486
|
|
54
54
|
velocity/db/servers/postgres/__init__.py,sha256=6YcTLXposmsrEaJgdUAM_QgD1TZDSILQrGcwWZ-dibk,2457
|
|
55
55
|
velocity/db/servers/postgres/operators.py,sha256=y9k6enReeR5hJxU_lYYR2epoaw4qCxEqmYJJ5jjaVWA,1166
|
|
56
56
|
velocity/db/servers/postgres/reserved.py,sha256=5tKLaqFV-HrWRj-nsrxl5KGbmeM3ukn_bPZK36XEu8M,3648
|
|
57
|
-
velocity/db/servers/postgres/sql.py,sha256=
|
|
57
|
+
velocity/db/servers/postgres/sql.py,sha256=zmn8cd1kxo7xAhR6BbP4XYx4m8Y7cTI-Ypw7qGGf22I,56789
|
|
58
58
|
velocity/db/servers/postgres/types.py,sha256=W71x8iRx-IIJkQSjb29k-KGkqp-QS6SxB0BHYXd4k8w,6955
|
|
59
59
|
velocity/db/servers/sqlite/__init__.py,sha256=EIx09YN1-Vm-4CXVcEf9DBgvd8FhIN9rEqIaSRrEcIk,2293
|
|
60
60
|
velocity/db/servers/sqlite/operators.py,sha256=VzZgph8RrnHkIVqqWGqnJwcafgBzc_8ZQp-M8tMl-mw,1221
|
|
61
61
|
velocity/db/servers/sqlite/reserved.py,sha256=4vOI06bjt8wg9KxdzDTF-iOd-ewY23NvSzthpdty2fA,1298
|
|
62
|
-
velocity/db/servers/sqlite/sql.py,sha256=
|
|
62
|
+
velocity/db/servers/sqlite/sql.py,sha256=FAWhzfM-D25Qaug1omMlcnu2NEHvnMfZIf7yR7HA-TQ,21456
|
|
63
63
|
velocity/db/servers/sqlite/types.py,sha256=jpCJeV25x4Iytf6D6GXgK3hVYFAAFV4WKJC-d-m4kdU,3102
|
|
64
64
|
velocity/db/servers/sqlserver/__init__.py,sha256=LN8OycN7W8da_ZPRYnPQ-O3Bv_xjret9qV1ZCitZlOU,2708
|
|
65
65
|
velocity/db/servers/sqlserver/operators.py,sha256=xK8_doDLssS38SRs1NoAI7XTO0-gNGMDS76nTVru4kE,1104
|
|
66
66
|
velocity/db/servers/sqlserver/reserved.py,sha256=Gn5n9DjxcjM-7PsIZPYigD6XLvMAYGnz1IrPuN7Dp2Y,2120
|
|
67
|
-
velocity/db/servers/sqlserver/sql.py,sha256=
|
|
67
|
+
velocity/db/servers/sqlserver/sql.py,sha256=Uh_XbBKJw5NZUoDxMYQjS26pXSA5OJj3XDsGPZ1S9Io,26754
|
|
68
68
|
velocity/db/servers/sqlserver/types.py,sha256=FAODYEO137m-WugpM89f9bQN9q6S2cjjUaz0a9gfE6M,3745
|
|
69
69
|
velocity/db/tests/__init__.py,sha256=7-hilWb43cKnSnCeXcjFG-6LpziN5k443IpsIvuevP0,24
|
|
70
70
|
velocity/db/tests/common_db_test.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
71
71
|
velocity/db/tests/test_cursor_rowcount_fix.py,sha256=mZRL1SBb9Knh67CSFyvfwj_LAarE_ilfVwpQHW18Yy8,5507
|
|
72
|
-
velocity/db/tests/test_db_utils.py,sha256=
|
|
72
|
+
velocity/db/tests/test_db_utils.py,sha256=UmBh2Bmz95GI6uYwEnilcz3gmuxPgLVpsj_6ekSnkWY,10297
|
|
73
73
|
velocity/db/tests/test_postgres.py,sha256=NoBydNkGmXn8olXwva4C4sYV3cKzERd6Df0wHixxoyE,15554
|
|
74
74
|
velocity/db/tests/test_postgres_unchanged.py,sha256=rNcy7S_HXazi_MjU8QjRZO4q8dULMeG4tg6eN-rPPz8,2998
|
|
75
75
|
velocity/db/tests/test_process_error_robustness.py,sha256=CZr_co_o6PK7dejOr_gwdn0iKTzjWPTY5k-PwJ6oh9s,11361
|
|
@@ -96,7 +96,7 @@ velocity/db/tests/postgres/test_schema_locking_unit.py,sha256=5Ifa7BEiR1zJwttuuV
|
|
|
96
96
|
velocity/db/tests/postgres/test_sequence.py,sha256=UoI4x2z8RvucuvZk4Tf1Ue_obtRHt0QCX0ae87iQ7mY,672
|
|
97
97
|
velocity/db/tests/postgres/test_sql_comprehensive.py,sha256=6eSnAMvnC257OD1EnUmiuXwzhuZlKrdK_W30OEu1yAY,18474
|
|
98
98
|
velocity/db/tests/postgres/test_table.py,sha256=D55TpJl0fh8b9Q-ijS3Cj6BeXrS_XQs8qfJFu3G2WL8,2306
|
|
99
|
-
velocity/db/tests/postgres/test_table_comprehensive.py,sha256=
|
|
99
|
+
velocity/db/tests/postgres/test_table_comprehensive.py,sha256=NxpOFTjgjqMG5WMwxzRdsphSCq6VQDqItV95i5yplPk,24038
|
|
100
100
|
velocity/db/tests/postgres/test_transaction.py,sha256=Hek8rXmz7Cuz1-Fmmgq7eyhMG9GYKkCKpUUMx5prnjc,2406
|
|
101
101
|
velocity/db/tests/sql/__init__.py,sha256=evafiIjAB0jyhqZ8HfiPgRujXJkRpQ7a34Bjac4qyv8,12
|
|
102
102
|
velocity/db/tests/sql/common.py,sha256=bXRob_RcZoonjCcwY712muraqGiW6HRMSpz5OOtixUM,5811
|
|
@@ -122,8 +122,8 @@ velocity/misc/tests/test_merge.py,sha256=Vm5_jY5cVczw0hZF-3TYzmxFw81heJOJB-dvhCg
|
|
|
122
122
|
velocity/misc/tests/test_oconv.py,sha256=fy4DwWGn_v486r2d_3ACpuBD-K1oOngNq1HJCGH7X-M,4694
|
|
123
123
|
velocity/misc/tests/test_original_error.py,sha256=iWSd18tckOA54LoPQOGV5j9LAz2W-3_ZOwmyZ8-4YQc,1742
|
|
124
124
|
velocity/misc/tests/test_timer.py,sha256=l9nrF84kHaFofvQYKInJmfoqC01wBhsUB18lVBgXCoo,2758
|
|
125
|
-
velocity_python-0.0.
|
|
126
|
-
velocity_python-0.0.
|
|
127
|
-
velocity_python-0.0.
|
|
128
|
-
velocity_python-0.0.
|
|
129
|
-
velocity_python-0.0.
|
|
125
|
+
velocity_python-0.0.158.dist-info/licenses/LICENSE,sha256=aoN245GG8s9oRUU89KNiGTU4_4OtnNmVi4hQeChg6rM,1076
|
|
126
|
+
velocity_python-0.0.158.dist-info/METADATA,sha256=LI9OtDa0mQD0qua0MhVSVa-EtZYMY3D5dksal4bSQT8,34266
|
|
127
|
+
velocity_python-0.0.158.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
128
|
+
velocity_python-0.0.158.dist-info/top_level.txt,sha256=JW2vJPmodgdgSz7H6yoZvnxF8S3fTMIv-YJWCT1sNW0,9
|
|
129
|
+
velocity_python-0.0.158.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|