velocity-python 0.0.155__py3-none-any.whl → 0.0.161__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- velocity/__init__.py +1 -1
- velocity/aws/handlers/context.py +87 -3
- velocity/db/core/decorators.py +28 -9
- velocity/db/core/engine.py +3 -2
- velocity/db/core/result.py +3 -1
- velocity/db/core/table.py +215 -11
- velocity/db/core/transaction.py +9 -4
- velocity/db/tests/test_db_utils.py +49 -0
- velocity/db/utils.py +67 -4
- {velocity_python-0.0.155.dist-info → velocity_python-0.0.161.dist-info}/METADATA +1 -1
- {velocity_python-0.0.155.dist-info → velocity_python-0.0.161.dist-info}/RECORD +14 -14
- {velocity_python-0.0.155.dist-info → velocity_python-0.0.161.dist-info}/WHEEL +0 -0
- {velocity_python-0.0.155.dist-info → velocity_python-0.0.161.dist-info}/licenses/LICENSE +0 -0
- {velocity_python-0.0.155.dist-info → velocity_python-0.0.161.dist-info}/top_level.txt +0 -0
velocity/__init__.py
CHANGED
velocity/aws/handlers/context.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import os
|
|
2
3
|
import boto3
|
|
3
4
|
import uuid
|
|
@@ -24,6 +25,8 @@ class Context:
|
|
|
24
25
|
self.__aws_event = aws_event
|
|
25
26
|
self.__aws_context = aws_context
|
|
26
27
|
self.__log = log
|
|
28
|
+
self._job_record_cache = {}
|
|
29
|
+
self._job_cancelled_flag = False
|
|
27
30
|
|
|
28
31
|
def postdata(self, keys=-1, default=None):
|
|
29
32
|
if keys == -1:
|
|
@@ -87,9 +90,9 @@ class Context:
|
|
|
87
90
|
if self.postdata("job_id"):
|
|
88
91
|
# Sanitize data before storing in database
|
|
89
92
|
sanitized_data = self._sanitize_job_data(data)
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
)
|
|
93
|
+
job_id = self.postdata("job_id")
|
|
94
|
+
tx.table("aws_job_activity").update(sanitized_data, {"job_id": job_id})
|
|
95
|
+
self._job_record_cache.pop(job_id, None)
|
|
93
96
|
tx.commit()
|
|
94
97
|
|
|
95
98
|
def create_job(self, tx, job_data=None):
|
|
@@ -98,6 +101,9 @@ class Context:
|
|
|
98
101
|
return
|
|
99
102
|
sanitized_data = self._sanitize_job_data(job_data)
|
|
100
103
|
tx.table("aws_job_activity").insert(sanitized_data)
|
|
104
|
+
job_id = sanitized_data.get("job_id")
|
|
105
|
+
if job_id:
|
|
106
|
+
self._job_record_cache.pop(job_id, None)
|
|
101
107
|
tx.commit()
|
|
102
108
|
|
|
103
109
|
def _sanitize_job_data(self, data):
|
|
@@ -183,6 +189,84 @@ class Context:
|
|
|
183
189
|
|
|
184
190
|
return sanitized
|
|
185
191
|
|
|
192
|
+
def _get_job_record(self, tx, job_id=None, refresh=False):
|
|
193
|
+
job_id = job_id or self.postdata("job_id")
|
|
194
|
+
if not job_id:
|
|
195
|
+
return None
|
|
196
|
+
|
|
197
|
+
if refresh or job_id not in self._job_record_cache:
|
|
198
|
+
record = tx.table("aws_job_activity").find({"job_id": job_id})
|
|
199
|
+
if record is not None:
|
|
200
|
+
self._job_record_cache[job_id] = record
|
|
201
|
+
elif job_id in self._job_record_cache:
|
|
202
|
+
del self._job_record_cache[job_id]
|
|
203
|
+
|
|
204
|
+
return self._job_record_cache.get(job_id)
|
|
205
|
+
|
|
206
|
+
def is_job_cancel_requested(self, tx, force_refresh=False):
|
|
207
|
+
job = self._get_job_record(tx, refresh=force_refresh)
|
|
208
|
+
if not job:
|
|
209
|
+
return False
|
|
210
|
+
|
|
211
|
+
status = (job.get("status") or "").lower()
|
|
212
|
+
if status in {"cancelrequested", "cancelled"}:
|
|
213
|
+
return True
|
|
214
|
+
|
|
215
|
+
message_raw = job.get("message")
|
|
216
|
+
if not message_raw:
|
|
217
|
+
return False
|
|
218
|
+
|
|
219
|
+
if isinstance(message_raw, dict):
|
|
220
|
+
message = message_raw
|
|
221
|
+
else:
|
|
222
|
+
try:
|
|
223
|
+
message = json.loads(message_raw)
|
|
224
|
+
except (TypeError, ValueError, json.JSONDecodeError):
|
|
225
|
+
return False
|
|
226
|
+
|
|
227
|
+
return bool(message.get("cancel_requested") or message.get("cancelled"))
|
|
228
|
+
|
|
229
|
+
def mark_job_cancelled(self, tx, detail=None, requested_by=None):
|
|
230
|
+
job_id = self.postdata("job_id")
|
|
231
|
+
if not job_id:
|
|
232
|
+
return
|
|
233
|
+
|
|
234
|
+
job = self._get_job_record(tx, refresh=True) or {}
|
|
235
|
+
message_raw = job.get("message")
|
|
236
|
+
if isinstance(message_raw, dict):
|
|
237
|
+
message = dict(message_raw)
|
|
238
|
+
else:
|
|
239
|
+
try:
|
|
240
|
+
message = json.loads(message_raw) if message_raw else {}
|
|
241
|
+
except (TypeError, ValueError, json.JSONDecodeError):
|
|
242
|
+
message = {}
|
|
243
|
+
|
|
244
|
+
message.update(
|
|
245
|
+
{
|
|
246
|
+
"detail": detail or "Job cancelled",
|
|
247
|
+
"cancelled": True,
|
|
248
|
+
}
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
tx.table("aws_job_activity").update(
|
|
252
|
+
{
|
|
253
|
+
"status": "Cancelled",
|
|
254
|
+
"message": to_json(message),
|
|
255
|
+
"handler_complete_timestamp": datetime.now(),
|
|
256
|
+
"sys_modified": datetime.now(),
|
|
257
|
+
"sys_modified_by": requested_by
|
|
258
|
+
or self.session().get("email_address")
|
|
259
|
+
or "system",
|
|
260
|
+
},
|
|
261
|
+
{"job_id": job_id},
|
|
262
|
+
)
|
|
263
|
+
tx.commit()
|
|
264
|
+
self._job_record_cache.pop(job_id, None)
|
|
265
|
+
self._job_cancelled_flag = True
|
|
266
|
+
|
|
267
|
+
def was_job_cancelled(self):
|
|
268
|
+
return self._job_cancelled_flag
|
|
269
|
+
|
|
186
270
|
def enqueue(self, action, payload={}, user=None, suppress_job_activity=False):
|
|
187
271
|
"""
|
|
188
272
|
Enqueue jobs to SQS with independent job activity tracking.
|
velocity/db/core/decorators.py
CHANGED
|
@@ -4,10 +4,27 @@ from functools import wraps
|
|
|
4
4
|
from velocity.db import exceptions
|
|
5
5
|
|
|
6
6
|
|
|
7
|
+
_PRIMARY_KEY_PATTERNS = (
|
|
8
|
+
"primary key",
|
|
9
|
+
"key 'primary'",
|
|
10
|
+
'key "primary"',
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _is_primary_key_duplicate(error):
|
|
15
|
+
"""Return True when the duplicate-key error is targeting the primary key."""
|
|
16
|
+
|
|
17
|
+
message = str(error or "")
|
|
18
|
+
lowered = message.lower()
|
|
19
|
+
|
|
20
|
+
if "sys_id" in lowered:
|
|
21
|
+
return True
|
|
22
|
+
|
|
23
|
+
return any(pattern in lowered for pattern in _PRIMARY_KEY_PATTERNS)
|
|
24
|
+
|
|
25
|
+
|
|
7
26
|
def retry_on_dup_key(func):
|
|
8
|
-
"""
|
|
9
|
-
Retries a function call if it raises DbDuplicateKeyError, up to max_retries.
|
|
10
|
-
"""
|
|
27
|
+
"""Retry when insert/update fails because the primary key already exists."""
|
|
11
28
|
|
|
12
29
|
@wraps(func)
|
|
13
30
|
def retry_decorator(self, *args, **kwds):
|
|
@@ -19,10 +36,12 @@ def retry_on_dup_key(func):
|
|
|
19
36
|
result = func(self, *args, **kwds)
|
|
20
37
|
self.tx.release_savepoint(sp, cursor=self.cursor())
|
|
21
38
|
return result
|
|
22
|
-
except exceptions.DbDuplicateKeyError:
|
|
39
|
+
except exceptions.DbDuplicateKeyError as error:
|
|
23
40
|
self.tx.rollback_savepoint(sp, cursor=self.cursor())
|
|
24
41
|
if "sys_id" in kwds.get("data", {}):
|
|
25
42
|
raise
|
|
43
|
+
if not _is_primary_key_duplicate(error):
|
|
44
|
+
raise
|
|
26
45
|
retries += 1
|
|
27
46
|
if retries >= max_retries:
|
|
28
47
|
raise
|
|
@@ -34,9 +53,7 @@ def retry_on_dup_key(func):
|
|
|
34
53
|
|
|
35
54
|
|
|
36
55
|
def reset_id_on_dup_key(func):
|
|
37
|
-
"""
|
|
38
|
-
Wraps an INSERT/UPSERT to reset the sys_id sequence on duplicate key collisions.
|
|
39
|
-
"""
|
|
56
|
+
"""Retry sys_id sequence bump only when the primary key collides."""
|
|
40
57
|
|
|
41
58
|
@wraps(func)
|
|
42
59
|
def reset_decorator(self, *args, retries=0, **kwds):
|
|
@@ -45,10 +62,12 @@ def reset_id_on_dup_key(func):
|
|
|
45
62
|
result = func(self, *args, **kwds)
|
|
46
63
|
self.tx.release_savepoint(sp, cursor=self.cursor())
|
|
47
64
|
return result
|
|
48
|
-
except exceptions.DbDuplicateKeyError:
|
|
65
|
+
except exceptions.DbDuplicateKeyError as error:
|
|
49
66
|
self.tx.rollback_savepoint(sp, cursor=self.cursor())
|
|
50
67
|
if "sys_id" in kwds.get("data", {}):
|
|
51
68
|
raise
|
|
69
|
+
if not _is_primary_key_duplicate(error):
|
|
70
|
+
raise
|
|
52
71
|
if retries < 3:
|
|
53
72
|
backoff_time = (2**retries) * 0.01 + random.uniform(0, 0.02)
|
|
54
73
|
time.sleep(backoff_time)
|
|
@@ -128,7 +147,7 @@ def create_missing(func):
|
|
|
128
147
|
for i, arg in enumerate(args):
|
|
129
148
|
if isinstance(arg, dict):
|
|
130
149
|
data.update(arg)
|
|
131
|
-
self.alter(data)
|
|
150
|
+
self.alter(data, mode="add")
|
|
132
151
|
return func(self, *args, **kwds)
|
|
133
152
|
except exceptions.DbTableMissingError as e:
|
|
134
153
|
self.tx.rollback_savepoint(sp, cursor=self.cursor())
|
velocity/db/core/engine.py
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import inspect
|
|
2
2
|
import re
|
|
3
|
-
import os
|
|
4
3
|
from contextlib import contextmanager
|
|
5
4
|
from functools import wraps
|
|
6
5
|
from velocity.db import exceptions
|
|
7
6
|
from velocity.db.core.transaction import Transaction
|
|
7
|
+
from velocity.db.utils import mask_config_for_display
|
|
8
8
|
|
|
9
9
|
import logging
|
|
10
10
|
|
|
@@ -27,7 +27,8 @@ class Engine:
|
|
|
27
27
|
self.__schema_locked = schema_locked
|
|
28
28
|
|
|
29
29
|
def __str__(self):
|
|
30
|
-
|
|
30
|
+
safe_config = mask_config_for_display(self.config)
|
|
31
|
+
return f"[{self.sql.server}] engine({safe_config})"
|
|
31
32
|
|
|
32
33
|
def connect(self):
|
|
33
34
|
"""
|
velocity/db/core/result.py
CHANGED
|
@@ -349,7 +349,9 @@ class Result:
|
|
|
349
349
|
return row
|
|
350
350
|
except StopIteration:
|
|
351
351
|
return default
|
|
352
|
-
|
|
352
|
+
|
|
353
|
+
one_or_none = one
|
|
354
|
+
|
|
353
355
|
def get_table_data(self, headers=True):
|
|
354
356
|
"""
|
|
355
357
|
Builds a two-dimensional list: first row is column headers, subsequent rows are data.
|
velocity/db/core/table.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import re
|
|
1
2
|
import sqlparse
|
|
2
3
|
from collections.abc import Iterable, Mapping
|
|
3
4
|
from velocity.db import exceptions
|
|
@@ -38,6 +39,103 @@ SYSTEM_COLUMN_NAMES = (
|
|
|
38
39
|
|
|
39
40
|
_SYSTEM_COLUMN_SET = {name.lower() for name in SYSTEM_COLUMN_NAMES}
|
|
40
41
|
|
|
42
|
+
_NULLABLE_TRUE = {"YES", "TRUE", "T", "1", "Y"}
|
|
43
|
+
_NULLABLE_FALSE = {"NO", "FALSE", "F", "0", "N"}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _normalize_sql_type(value):
|
|
47
|
+
"""Return a simplified SQL type identifier for comparison purposes."""
|
|
48
|
+
|
|
49
|
+
if value is None:
|
|
50
|
+
return None
|
|
51
|
+
|
|
52
|
+
normalized = re.sub(r"\s+", " ", str(value).strip()).upper()
|
|
53
|
+
|
|
54
|
+
if not normalized:
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
if normalized.startswith("CHARACTER VARYING") or normalized.startswith("VARCHAR"):
|
|
58
|
+
return "TEXT"
|
|
59
|
+
if normalized.startswith("CHAR(") or normalized == "CHARACTER" or normalized == "BPCHAR":
|
|
60
|
+
return "TEXT"
|
|
61
|
+
if normalized.startswith("NUMERIC(") or normalized.startswith("DECIMAL("):
|
|
62
|
+
return "NUMERIC"
|
|
63
|
+
if normalized == "TIMESTAMP":
|
|
64
|
+
return "TIMESTAMP WITHOUT TIME ZONE"
|
|
65
|
+
if normalized.startswith("TIMESTAMP WITHOUT TIME ZONE"):
|
|
66
|
+
return "TIMESTAMP WITHOUT TIME ZONE"
|
|
67
|
+
if normalized in {"TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE"}:
|
|
68
|
+
return "TIMESTAMP WITH TIME ZONE"
|
|
69
|
+
if normalized.startswith("TIME WITHOUT TIME ZONE"):
|
|
70
|
+
return "TIME WITHOUT TIME ZONE"
|
|
71
|
+
if normalized in {"TIME WITH TIME ZONE", "TIMETZ"}:
|
|
72
|
+
return "TIME WITH TIME ZONE"
|
|
73
|
+
if normalized == "BOOL":
|
|
74
|
+
return "BOOLEAN"
|
|
75
|
+
return normalized
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _types_equivalent(current, expected):
|
|
79
|
+
return _normalize_sql_type(current) == _normalize_sql_type(expected)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _is_nullable_flag(value):
|
|
83
|
+
if isinstance(value, bool):
|
|
84
|
+
return value
|
|
85
|
+
if value is None:
|
|
86
|
+
return None
|
|
87
|
+
text = str(value).strip().upper()
|
|
88
|
+
if text in _NULLABLE_TRUE:
|
|
89
|
+
return True
|
|
90
|
+
if text in _NULLABLE_FALSE:
|
|
91
|
+
return False
|
|
92
|
+
return None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _parse_column_spec(spec, default_nullable):
|
|
96
|
+
"""Normalize user-provided column specification into a common structure."""
|
|
97
|
+
|
|
98
|
+
nullable = default_nullable
|
|
99
|
+
nullable_specified = False
|
|
100
|
+
alter_sql = None
|
|
101
|
+
using_expression = None
|
|
102
|
+
|
|
103
|
+
base = spec
|
|
104
|
+
override_options = {}
|
|
105
|
+
|
|
106
|
+
if isinstance(spec, tuple) and len(spec) == 2 and isinstance(spec[1], Mapping):
|
|
107
|
+
base, override_options = spec
|
|
108
|
+
|
|
109
|
+
options = {}
|
|
110
|
+
if isinstance(base, Mapping):
|
|
111
|
+
options.update(base)
|
|
112
|
+
base = options.get("type", options.get("value"))
|
|
113
|
+
|
|
114
|
+
if override_options:
|
|
115
|
+
options.update(override_options)
|
|
116
|
+
|
|
117
|
+
type_hint = options.get("type", base)
|
|
118
|
+
if type_hint is None and "value" in options:
|
|
119
|
+
type_hint = options["value"]
|
|
120
|
+
|
|
121
|
+
add_value = options.get("add_value", options.get("value", type_hint))
|
|
122
|
+
|
|
123
|
+
if "nullable" in options:
|
|
124
|
+
nullable = bool(options["nullable"])
|
|
125
|
+
nullable_specified = True
|
|
126
|
+
|
|
127
|
+
alter_sql = options.get("sql")
|
|
128
|
+
using_expression = options.get("using")
|
|
129
|
+
|
|
130
|
+
return {
|
|
131
|
+
"type_hint": type_hint,
|
|
132
|
+
"add_value": add_value,
|
|
133
|
+
"nullable": nullable,
|
|
134
|
+
"nullable_specified": nullable_specified,
|
|
135
|
+
"alter_sql": alter_sql,
|
|
136
|
+
"using": using_expression,
|
|
137
|
+
}
|
|
138
|
+
|
|
41
139
|
|
|
42
140
|
class Table:
|
|
43
141
|
SYSTEM_COLUMNS = SYSTEM_COLUMN_NAMES
|
|
@@ -430,7 +528,7 @@ class Table:
|
|
|
430
528
|
return Row(self, lookup, lock=lock)
|
|
431
529
|
return Row(self, result[0]["sys_id"], lock=lock)
|
|
432
530
|
|
|
433
|
-
one = find
|
|
531
|
+
one = one_or_none = find
|
|
434
532
|
|
|
435
533
|
@return_default(None)
|
|
436
534
|
def first(
|
|
@@ -535,22 +633,128 @@ class Table:
|
|
|
535
633
|
@create_missing
|
|
536
634
|
def alter(self, columns, **kwds):
|
|
537
635
|
"""
|
|
538
|
-
|
|
636
|
+
Create or update columns so they match the supplied specification.
|
|
539
637
|
"""
|
|
638
|
+
|
|
540
639
|
if not isinstance(columns, dict):
|
|
541
640
|
raise Exception("Columns must be a dict.")
|
|
641
|
+
|
|
642
|
+
mode = kwds.pop("mode", "smart")
|
|
643
|
+
mode = (mode or "smart").lower()
|
|
644
|
+
if mode not in {"smart", "add"}:
|
|
645
|
+
raise ValueError(f"Unsupported alter mode: {mode}")
|
|
646
|
+
|
|
647
|
+
sql_only = kwds.get("sql_only", False)
|
|
648
|
+
default_nullable = kwds.get("null_allowed", True)
|
|
649
|
+
|
|
542
650
|
columns = self.lower_keys(columns)
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
651
|
+
existing_columns = {col.lower() for col in self.sys_columns()}
|
|
652
|
+
|
|
653
|
+
to_add = {}
|
|
654
|
+
add_specs = {}
|
|
655
|
+
statements = []
|
|
656
|
+
null_statements = []
|
|
657
|
+
type_statements = []
|
|
658
|
+
custom_statements = []
|
|
659
|
+
|
|
660
|
+
for column_name, raw_spec in columns.items():
|
|
661
|
+
spec = _parse_column_spec(raw_spec, default_nullable)
|
|
662
|
+
|
|
663
|
+
if column_name not in existing_columns:
|
|
664
|
+
to_add[column_name] = spec["add_value"]
|
|
665
|
+
add_specs[column_name] = spec
|
|
666
|
+
continue
|
|
667
|
+
|
|
668
|
+
if mode == "add":
|
|
669
|
+
continue
|
|
670
|
+
|
|
671
|
+
column_obj = self.column(column_name)
|
|
672
|
+
current_type = column_obj.sql_type
|
|
673
|
+
type_hint = spec["type_hint"]
|
|
674
|
+
|
|
675
|
+
if not (isinstance(type_hint, str) and type_hint.startswith("@@")) and type_hint is not None:
|
|
676
|
+
expected_type = self.sql.types.get_type(type_hint)
|
|
677
|
+
if expected_type and not _types_equivalent(current_type, expected_type):
|
|
678
|
+
if spec["using"]:
|
|
679
|
+
clause = f"TYPE {expected_type} USING {spec['using']}"
|
|
680
|
+
sql, vals = self.sql.alter_column_by_sql(self.name, column_name, clause)
|
|
681
|
+
else:
|
|
682
|
+
nullable_flag = _is_nullable_flag(column_obj.is_nullable)
|
|
683
|
+
if nullable_flag is None:
|
|
684
|
+
nullable_flag = True
|
|
685
|
+
server_name = getattr(self.sql, "server", "").lower()
|
|
686
|
+
if server_name.startswith("postgre"):
|
|
687
|
+
type_argument = type_hint
|
|
688
|
+
else:
|
|
689
|
+
type_argument = expected_type or type_hint
|
|
690
|
+
sql, vals = self.sql.alter_column_by_type(
|
|
691
|
+
self.name,
|
|
692
|
+
column_name,
|
|
693
|
+
type_argument,
|
|
694
|
+
nullable_flag,
|
|
695
|
+
)
|
|
696
|
+
type_statements.append((sql, vals))
|
|
697
|
+
|
|
698
|
+
if spec["alter_sql"]:
|
|
699
|
+
sql, vals = self.sql.alter_column_by_sql(self.name, column_name, spec["alter_sql"])
|
|
700
|
+
custom_statements.append((sql, vals))
|
|
701
|
+
|
|
702
|
+
if spec["nullable_specified"]:
|
|
703
|
+
desired_nullable = spec["nullable"]
|
|
704
|
+
current_nullable = _is_nullable_flag(column_obj.is_nullable)
|
|
705
|
+
if desired_nullable and current_nullable is False:
|
|
706
|
+
sql, vals = self.sql.alter_column_by_sql(
|
|
707
|
+
self.name, column_name, "DROP NOT NULL"
|
|
708
|
+
)
|
|
709
|
+
null_statements.append((sql, vals))
|
|
710
|
+
elif not desired_nullable and current_nullable is True:
|
|
711
|
+
sql, vals = self.sql.alter_column_by_sql(
|
|
712
|
+
self.name, column_name, "SET NOT NULL"
|
|
713
|
+
)
|
|
714
|
+
null_statements.append((sql, vals))
|
|
715
|
+
|
|
716
|
+
if to_add:
|
|
717
|
+
sql, vals = self.sql.alter_add(self.name, to_add, default_nullable)
|
|
718
|
+
statements.append((sql, vals))
|
|
719
|
+
|
|
720
|
+
for column_name, spec in add_specs.items():
|
|
721
|
+
if spec["nullable_specified"] and not spec["nullable"] and default_nullable:
|
|
722
|
+
sql, vals = self.sql.alter_column_by_sql(
|
|
723
|
+
self.name, column_name, "SET NOT NULL"
|
|
724
|
+
)
|
|
725
|
+
null_statements.append((sql, vals))
|
|
726
|
+
elif spec["nullable_specified"] and spec["nullable"] and not default_nullable:
|
|
727
|
+
sql, vals = self.sql.alter_column_by_sql(
|
|
728
|
+
self.name, column_name, "DROP NOT NULL"
|
|
729
|
+
)
|
|
730
|
+
null_statements.append((sql, vals))
|
|
731
|
+
|
|
732
|
+
statements.extend(type_statements)
|
|
733
|
+
statements.extend(custom_statements)
|
|
734
|
+
statements.extend(null_statements)
|
|
735
|
+
|
|
736
|
+
if not statements:
|
|
737
|
+
return None
|
|
738
|
+
|
|
739
|
+
if sql_only:
|
|
740
|
+
if len(statements) == 1:
|
|
741
|
+
return statements[0]
|
|
742
|
+
return statements
|
|
743
|
+
|
|
744
|
+
for sql, vals in statements:
|
|
745
|
+
if not sql:
|
|
746
|
+
continue
|
|
552
747
|
self.tx.execute(sql, vals, cursor=self.cursor())
|
|
553
748
|
|
|
749
|
+
def alter_add(self, columns, **kwds):
|
|
750
|
+
"""
|
|
751
|
+
Add missing columns without modifying existing column definitions.
|
|
752
|
+
"""
|
|
753
|
+
|
|
754
|
+
kwds = dict(kwds)
|
|
755
|
+
kwds["mode"] = "add"
|
|
756
|
+
return self.alter(columns, **kwds)
|
|
757
|
+
|
|
554
758
|
@create_missing
|
|
555
759
|
def alter_type(self, column, type_or_value, nullable=True, **kwds):
|
|
556
760
|
"""
|
velocity/db/core/transaction.py
CHANGED
|
@@ -6,6 +6,7 @@ from velocity.db.core.result import Result
|
|
|
6
6
|
from velocity.db.core.column import Column
|
|
7
7
|
from velocity.db.core.database import Database
|
|
8
8
|
from velocity.db.core.sequence import Sequence
|
|
9
|
+
from velocity.db.utils import mask_config_for_display
|
|
9
10
|
from velocity.misc.db import randomword
|
|
10
11
|
|
|
11
12
|
debug = False
|
|
@@ -22,10 +23,14 @@ class Transaction:
|
|
|
22
23
|
self.__pg_types = {}
|
|
23
24
|
|
|
24
25
|
def __str__(self):
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
config = mask_config_for_display(self.engine.config)
|
|
27
|
+
|
|
28
|
+
if isinstance(config, dict):
|
|
29
|
+
server = config.get("host", config.get("server"))
|
|
30
|
+
database = config.get("database", config.get("dbname"))
|
|
31
|
+
return f"{self.engine.sql.server}.transaction({server}:{database})"
|
|
32
|
+
|
|
33
|
+
return f"{self.engine.sql.server}.transaction({config})"
|
|
29
34
|
|
|
30
35
|
def __enter__(self):
|
|
31
36
|
return self
|
|
@@ -11,6 +11,8 @@ from src.velocity.db.utils import (
|
|
|
11
11
|
safe_sort_rows,
|
|
12
12
|
group_by_fields,
|
|
13
13
|
safe_sort_grouped_rows,
|
|
14
|
+
mask_config_for_display,
|
|
15
|
+
mask_sensitive_in_string,
|
|
14
16
|
)
|
|
15
17
|
|
|
16
18
|
|
|
@@ -216,6 +218,53 @@ class TestDatabaseUtils(unittest.TestCase):
|
|
|
216
218
|
expected = ["2024-06", "2024-12", None]
|
|
217
219
|
self.assertEqual(exp_dates, expected)
|
|
218
220
|
|
|
221
|
+
def test_mask_config_for_display_redacts_direct_passwords(self):
|
|
222
|
+
"""Ensure direct password/token fields are masked."""
|
|
223
|
+
config = {
|
|
224
|
+
"host": "db.local",
|
|
225
|
+
"password": "supersecret",
|
|
226
|
+
"token": "abc123",
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
masked = mask_config_for_display(config)
|
|
230
|
+
|
|
231
|
+
self.assertEqual(masked["host"], "db.local")
|
|
232
|
+
self.assertEqual(masked["password"], "*****")
|
|
233
|
+
self.assertEqual(masked["token"], "*****")
|
|
234
|
+
|
|
235
|
+
def test_mask_config_for_display_handles_nested_structures(self):
|
|
236
|
+
"""Verify masking applies to nested dicts, lists, tuples, and DSN strings."""
|
|
237
|
+
config = {
|
|
238
|
+
"options": {
|
|
239
|
+
"passwd": "innersecret",
|
|
240
|
+
"hosts": [
|
|
241
|
+
{"url": "postgresql://user:pwd@localhost/db"},
|
|
242
|
+
("token=xyz",),
|
|
243
|
+
],
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
masked = mask_config_for_display(config)
|
|
248
|
+
|
|
249
|
+
self.assertEqual(masked["options"]["passwd"], "*****")
|
|
250
|
+
self.assertEqual(
|
|
251
|
+
masked["options"]["hosts"][0]["url"],
|
|
252
|
+
"postgresql://user:*****@localhost/db",
|
|
253
|
+
)
|
|
254
|
+
self.assertEqual(masked["options"]["hosts"][1][0], "token=*****")
|
|
255
|
+
|
|
256
|
+
def test_mask_sensitive_in_string_redacts_key_value_pairs(self):
|
|
257
|
+
"""Key/value DSN parameters should be redacted."""
|
|
258
|
+
dsn = "host=db password=abc123;user=test"
|
|
259
|
+
masked = mask_sensitive_in_string(dsn)
|
|
260
|
+
self.assertEqual(masked, "host=db password=*****;user=test")
|
|
261
|
+
|
|
262
|
+
def test_mask_sensitive_in_string_redacts_url_credentials(self):
|
|
263
|
+
"""URL style credentials should hide the password portion."""
|
|
264
|
+
url = "postgresql://user:secret@host/db"
|
|
265
|
+
masked = mask_sensitive_in_string(url)
|
|
266
|
+
self.assertEqual(masked, "postgresql://user:*****@host/db")
|
|
267
|
+
|
|
219
268
|
|
|
220
269
|
if __name__ == "__main__":
|
|
221
270
|
unittest.main()
|
velocity/db/utils.py
CHANGED
|
@@ -1,12 +1,75 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Database utility functions for common operations.
|
|
1
|
+
"""Utility helpers for velocity.db modules.
|
|
3
2
|
|
|
4
|
-
This module provides
|
|
5
|
-
|
|
3
|
+
This module provides helpers for redacting sensitive configuration values along
|
|
4
|
+
with common collection utilities used across the velocity database codebase.
|
|
6
5
|
"""
|
|
7
6
|
|
|
7
|
+
import re
|
|
8
8
|
from typing import Any, Callable, List
|
|
9
9
|
|
|
10
|
+
_SENSITIVE_KEYWORDS = {
|
|
11
|
+
"password",
|
|
12
|
+
"passwd",
|
|
13
|
+
"pwd",
|
|
14
|
+
"secret",
|
|
15
|
+
"token",
|
|
16
|
+
"apikey",
|
|
17
|
+
"api_key",
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
_SENSITIVE_PATTERNS = [
|
|
21
|
+
re.compile(r"(password\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
22
|
+
re.compile(r"(passwd\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
23
|
+
re.compile(r"(pwd\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
24
|
+
re.compile(r"(secret\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
25
|
+
re.compile(r"(token\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
26
|
+
re.compile(r"(api[_-]?key\s*=\s*)([^\s;]+)", re.IGNORECASE),
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
_URL_CREDENTIAL_PATTERN = re.compile(r"(://[^:\s]+:)([^@/\s]+)")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def mask_sensitive_in_string(value: str) -> str:
|
|
33
|
+
"""Return ``value`` with credential-like substrings redacted."""
|
|
34
|
+
|
|
35
|
+
if not value:
|
|
36
|
+
return value
|
|
37
|
+
|
|
38
|
+
masked = value
|
|
39
|
+
for pattern in _SENSITIVE_PATTERNS:
|
|
40
|
+
masked = pattern.sub(lambda match: match.group(1) + "*****", masked)
|
|
41
|
+
|
|
42
|
+
return _URL_CREDENTIAL_PATTERN.sub(r"\1*****", masked)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def mask_config_for_display(config: Any) -> Any:
|
|
46
|
+
"""Return ``config`` with common secret fields masked for logging/str()."""
|
|
47
|
+
|
|
48
|
+
if isinstance(config, dict):
|
|
49
|
+
masked = {}
|
|
50
|
+
for key, value in config.items():
|
|
51
|
+
if isinstance(key, str) and _contains_sensitive_keyword(key):
|
|
52
|
+
masked[key] = "*****"
|
|
53
|
+
else:
|
|
54
|
+
masked[key] = mask_config_for_display(value)
|
|
55
|
+
return masked
|
|
56
|
+
|
|
57
|
+
if isinstance(config, tuple):
|
|
58
|
+
return tuple(mask_config_for_display(item) for item in config)
|
|
59
|
+
|
|
60
|
+
if isinstance(config, list):
|
|
61
|
+
return [mask_config_for_display(item) for item in config]
|
|
62
|
+
|
|
63
|
+
if isinstance(config, str):
|
|
64
|
+
return mask_sensitive_in_string(config)
|
|
65
|
+
|
|
66
|
+
return config
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _contains_sensitive_keyword(key: str) -> bool:
|
|
70
|
+
lowered = key.lower()
|
|
71
|
+
return any(token in lowered for token in _SENSITIVE_KEYWORDS)
|
|
72
|
+
|
|
10
73
|
|
|
11
74
|
def safe_sort_key_none_last(field_name: str) -> Callable[[dict], tuple]:
|
|
12
75
|
"""
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
velocity/__init__.py,sha256=
|
|
1
|
+
velocity/__init__.py,sha256=yQuGM3AN4czqhK3Leb3ndVePPePCWYzTSOPANqmAQf0,147
|
|
2
2
|
velocity/app/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
3
|
velocity/app/invoices.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
4
|
velocity/app/orders.py,sha256=C7ewngMpO8nD3ul_82o4FhZBdRkWvJtnuEbEJUKDCno,6151
|
|
@@ -12,7 +12,7 @@ velocity/aws/__init__.py,sha256=0nEsX68Q4I7Z7ybECJnNWsC8QhWOijn4NpaFgyzyfXA,685
|
|
|
12
12
|
velocity/aws/amplify.py,sha256=VgSgon0XxboIozz0tKyUohgLIigelhe4W7EH8kwbnLg,15330
|
|
13
13
|
velocity/aws/handlers/__init__.py,sha256=4-NKj8dBzjYEdIlNdfm_Ip5mI0oOGcpjjBcMwU42yhQ,227
|
|
14
14
|
velocity/aws/handlers/base_handler.py,sha256=bapdzWss5lXesoLPsVwJo9hQMZLdz7XOubo3sK70xC8,7960
|
|
15
|
-
velocity/aws/handlers/context.py,sha256=
|
|
15
|
+
velocity/aws/handlers/context.py,sha256=yL1oA8y6q0b1Vg3KUB0XWJ0hqnToScZuBXEHpWif8V0,11259
|
|
16
16
|
velocity/aws/handlers/exceptions.py,sha256=i4wcB8ZSWUHglX2xnesDlWLsU9AMYU72cHCWRBDmjQ8,361
|
|
17
17
|
velocity/aws/handlers/lambda_handler.py,sha256=0wa_CHyJOaI5RsHqB0Ae83-B-SwlKR0qkGUlc7jitQI,4427
|
|
18
18
|
velocity/aws/handlers/response.py,sha256=s2Kw7yv5zAir1mEmfv6yBVIvRcRQ__xyryf1SrvtiRc,9317
|
|
@@ -28,17 +28,17 @@ velocity/aws/tests/test_lambda_handler_json_serialization.py,sha256=VuikDD_tbRbA
|
|
|
28
28
|
velocity/aws/tests/test_response.py,sha256=sCg6qOSDWNncnmkuf4N3gO5fxQQ3PyFNMRsYy195nKo,6546
|
|
29
29
|
velocity/db/__init__.py,sha256=7XRUHY2af0HL1jvL0SAMpxSe5a2Phbkm-YLJCvC1C_0,739
|
|
30
30
|
velocity/db/exceptions.py,sha256=LbgYJfY2nCBu7tBC_U1BwdczypJ24S6CUyREG_N5gO0,2345
|
|
31
|
-
velocity/db/utils.py,sha256=
|
|
31
|
+
velocity/db/utils.py,sha256=qfnxN67B91pIXqMEp8mP5RuzFlBI20zwMMWBoHHO29g,8864
|
|
32
32
|
velocity/db/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
33
|
velocity/db/core/column.py,sha256=tAr8tL3a2nyaYpNHhGl508FrY_pGZTzyYgjAV5CEBv4,4092
|
|
34
34
|
velocity/db/core/database.py,sha256=3zNGItklu9tZCKsbx2T2vCcU1so8AL9PPL0DLjvaz6s,3554
|
|
35
|
-
velocity/db/core/decorators.py,sha256=
|
|
36
|
-
velocity/db/core/engine.py,sha256=
|
|
37
|
-
velocity/db/core/result.py,sha256=
|
|
35
|
+
velocity/db/core/decorators.py,sha256=j2IM063-ejDrIpNwwajxPawTs0A5-DU0-prbPYa4KTM,5816
|
|
36
|
+
velocity/db/core/engine.py,sha256=84jz1taX-t4nqXN99kz9WyczpgsRLGFNnc_hpaCi3lE,20769
|
|
37
|
+
velocity/db/core/result.py,sha256=mW4wQLSMkUF59VXgiAZz0XOGdHwPkd6r8ncSK2iATFk,12848
|
|
38
38
|
velocity/db/core/row.py,sha256=GOWm-HEBPCBwdqMHMBRc41m0Hoht4vRVQLkvdogX1fU,7729
|
|
39
39
|
velocity/db/core/sequence.py,sha256=VMBc0ZjGnOaWTwKW6xMNTdP8rZ2umQ8ml4fHTTwuGq4,3904
|
|
40
|
-
velocity/db/core/table.py,sha256=
|
|
41
|
-
velocity/db/core/transaction.py,sha256=
|
|
40
|
+
velocity/db/core/table.py,sha256=x7D--wyiZjr_xlE5Ug_yp0fkzvkYqYrZMzca78bSrh8,50583
|
|
41
|
+
velocity/db/core/transaction.py,sha256=LDwwAXk666sBdT4J01bvu3rAYfWwxvxVVQUBbQlX-hY,7155
|
|
42
42
|
velocity/db/servers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
43
43
|
velocity/db/servers/tablehelper.py,sha256=Q48ObN5KD_U2sBP0GUcjaQjKeE4Hr351sPQirwQ0_1s,22163
|
|
44
44
|
velocity/db/servers/base/__init__.py,sha256=5--XJUeEAm7O6Ns2C_ODCr5TjFhdAge-zApZCT0LGTQ,285
|
|
@@ -69,7 +69,7 @@ velocity/db/servers/sqlserver/types.py,sha256=FAODYEO137m-WugpM89f9bQN9q6S2cjjUa
|
|
|
69
69
|
velocity/db/tests/__init__.py,sha256=7-hilWb43cKnSnCeXcjFG-6LpziN5k443IpsIvuevP0,24
|
|
70
70
|
velocity/db/tests/common_db_test.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
71
71
|
velocity/db/tests/test_cursor_rowcount_fix.py,sha256=mZRL1SBb9Knh67CSFyvfwj_LAarE_ilfVwpQHW18Yy8,5507
|
|
72
|
-
velocity/db/tests/test_db_utils.py,sha256=
|
|
72
|
+
velocity/db/tests/test_db_utils.py,sha256=UmBh2Bmz95GI6uYwEnilcz3gmuxPgLVpsj_6ekSnkWY,10297
|
|
73
73
|
velocity/db/tests/test_postgres.py,sha256=NoBydNkGmXn8olXwva4C4sYV3cKzERd6Df0wHixxoyE,15554
|
|
74
74
|
velocity/db/tests/test_postgres_unchanged.py,sha256=rNcy7S_HXazi_MjU8QjRZO4q8dULMeG4tg6eN-rPPz8,2998
|
|
75
75
|
velocity/db/tests/test_process_error_robustness.py,sha256=CZr_co_o6PK7dejOr_gwdn0iKTzjWPTY5k-PwJ6oh9s,11361
|
|
@@ -122,8 +122,8 @@ velocity/misc/tests/test_merge.py,sha256=Vm5_jY5cVczw0hZF-3TYzmxFw81heJOJB-dvhCg
|
|
|
122
122
|
velocity/misc/tests/test_oconv.py,sha256=fy4DwWGn_v486r2d_3ACpuBD-K1oOngNq1HJCGH7X-M,4694
|
|
123
123
|
velocity/misc/tests/test_original_error.py,sha256=iWSd18tckOA54LoPQOGV5j9LAz2W-3_ZOwmyZ8-4YQc,1742
|
|
124
124
|
velocity/misc/tests/test_timer.py,sha256=l9nrF84kHaFofvQYKInJmfoqC01wBhsUB18lVBgXCoo,2758
|
|
125
|
-
velocity_python-0.0.
|
|
126
|
-
velocity_python-0.0.
|
|
127
|
-
velocity_python-0.0.
|
|
128
|
-
velocity_python-0.0.
|
|
129
|
-
velocity_python-0.0.
|
|
125
|
+
velocity_python-0.0.161.dist-info/licenses/LICENSE,sha256=aoN245GG8s9oRUU89KNiGTU4_4OtnNmVi4hQeChg6rM,1076
|
|
126
|
+
velocity_python-0.0.161.dist-info/METADATA,sha256=g74KjjNchM3sz3UzhciMNOxZYIOSQhzXujp0KLKVczU,34266
|
|
127
|
+
velocity_python-0.0.161.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
128
|
+
velocity_python-0.0.161.dist-info/top_level.txt,sha256=JW2vJPmodgdgSz7H6yoZvnxF8S3fTMIv-YJWCT1sNW0,9
|
|
129
|
+
velocity_python-0.0.161.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|