limitsdb 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- limitsdb/__init__.py +0 -0
- limitsdb/cli/__init__.py +0 -0
- limitsdb/cli/ldb_crypt.py +44 -0
- limitsdb/cli/ldb_impl.py +207 -0
- limitsdb/cli/ldb_init.py +60 -0
- limitsdb/cli/ldb_run.py +53 -0
- limitsdb/core/__init__.py +0 -0
- limitsdb/core/ldb_config_loader.py +198 -0
- limitsdb/core/ldb_crypto.py +91 -0
- limitsdb/core/ldb_errors.py +30 -0
- limitsdb/core/ldb_ilm_config.py +228 -0
- limitsdb/core/ldb_logger.py +57 -0
- limitsdb/core/ldb_meta.py +31 -0
- limitsdb/core/ldb_params_config.py +459 -0
- limitsdb/core/ldb_runner.py +810 -0
- limitsdb/core/ldb_status.py +8 -0
- limitsdb/core/ldb_utils.py +332 -0
- limitsdb/db/__init__.py +0 -0
- limitsdb/db/ldb_engine_loader.py +16 -0
- limitsdb/db/ldb_engines.py +347 -0
- limitsdb/db/oracle/__init__.py +0 -0
- limitsdb/db/oracle/ldb_conf.sql +151 -0
- limitsdb/db/oracle/ldb_engine_impl.py +1505 -0
- limitsdb/resources/__init__.py +0 -0
- limitsdb/resources/ilm.example.yml +71 -0
- limitsdb-0.5.0.dist-info/METADATA +586 -0
- limitsdb-0.5.0.dist-info/RECORD +32 -0
- limitsdb-0.5.0.dist-info/WHEEL +4 -0
- limitsdb-0.5.0.dist-info/entry_points.txt +6 -0
- limitsdb-0.5.0.dist-info/licenses/LICENSE +118 -0
- limitsdb-0.5.0.dist-info/licenses/LICENSE-DUAL.md +59 -0
- limitsdb-0.5.0.dist-info/licenses/NOTICE +12 -0
|
@@ -0,0 +1,1505 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import Any, ClassVar, Literal, cast
|
|
5
|
+
|
|
6
|
+
import oracledb
|
|
7
|
+
|
|
8
|
+
from limitsdb.core.ldb_errors import DatabaseConnectionError, ExecutionError
|
|
9
|
+
from limitsdb.core.ldb_logger import get_logger
|
|
10
|
+
from limitsdb.core.ldb_params_config import Config
|
|
11
|
+
from limitsdb.core.ldb_status import Status
|
|
12
|
+
from limitsdb.core.ldb_utils import get_effective_credentials, indent_lines, join_wrapped, nvl
|
|
13
|
+
from limitsdb.db.ldb_engines import (
|
|
14
|
+
ColumnDefinition,
|
|
15
|
+
ColumnMetadata,
|
|
16
|
+
DatabaseEngine,
|
|
17
|
+
DatabaseLinkDefinition,
|
|
18
|
+
IndexMap,
|
|
19
|
+
RoleDefinition,
|
|
20
|
+
SequenceDefinition,
|
|
21
|
+
TableDefinition,
|
|
22
|
+
UserDefinition,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
SUPPORTING_OBJECT_LIST = [
|
|
26
|
+
"TYPE t_referencing_tables",
|
|
27
|
+
"PROCEDURE check_save_status",
|
|
28
|
+
"PROCEDURE check_referencing_tables",
|
|
29
|
+
]
|
|
30
|
+
CHECK_SAVE_STATUS_PROC = """
|
|
31
|
+
CREATE OR REPLACE PROCEDURE check_save_status(
|
|
32
|
+
p_owner IN VARCHAR2,
|
|
33
|
+
p_table_name IN VARCHAR2,
|
|
34
|
+
p_process_date IN DATE,
|
|
35
|
+
p_action IN VARCHAR2,
|
|
36
|
+
p_status IN VARCHAR2,
|
|
37
|
+
p_process_start IN DATE,
|
|
38
|
+
p_chunk_start IN DATE,
|
|
39
|
+
p_process_end IN DATE,
|
|
40
|
+
p_message IN VARCHAR2,
|
|
41
|
+
p_rows_processed IN VARCHAR2,
|
|
42
|
+
p_plsql IN VARCHAR2,
|
|
43
|
+
p_sqlcode IN OUT NUMBER,
|
|
44
|
+
p_out_message OUT VARCHAR2
|
|
45
|
+
)
|
|
46
|
+
IS
|
|
47
|
+
l_status VARCHAR2(10);
|
|
48
|
+
l_process_start DATE;
|
|
49
|
+
l_process_date DATE;
|
|
50
|
+
l_rowid ROWID;
|
|
51
|
+
in_use EXCEPTION;
|
|
52
|
+
PRAGMA EXCEPTION_INIT(in_use, -54);
|
|
53
|
+
BEGIN
|
|
54
|
+
IF NVL(p_sqlcode, 0) NOT IN (-20001, -20002, -20003) THEN
|
|
55
|
+
BEGIN
|
|
56
|
+
SELECT ctl_status, ctl_process_date, ctl_process_start, ROWID
|
|
57
|
+
INTO l_status, l_process_date, l_process_start, l_rowid
|
|
58
|
+
FROM ldb_ctl
|
|
59
|
+
WHERE ctl_owner = p_owner
|
|
60
|
+
AND ctl_table_name = p_table_name
|
|
61
|
+
FOR UPDATE NOWAIT;
|
|
62
|
+
|
|
63
|
+
IF p_status = 'TSTART'
|
|
64
|
+
AND l_status != 'TEND'
|
|
65
|
+
AND (SYSDATE - l_process_start) * 3600 * 24 < 5 THEN
|
|
66
|
+
p_sqlcode := -20002;
|
|
67
|
+
p_out_message := 'Table ' || p_table_name || ' is currently in process without lock';
|
|
68
|
+
RETURN;
|
|
69
|
+
ELSIF p_status = 'TSTART'
|
|
70
|
+
AND l_status = 'TEND'
|
|
71
|
+
AND l_process_date = p_process_date THEN
|
|
72
|
+
ROLLBACK;
|
|
73
|
+
p_sqlcode := -20003;
|
|
74
|
+
p_out_message := 'Table ' || p_table_name ||
|
|
75
|
+
' is already processed for date ' ||
|
|
76
|
+
TO_CHAR(p_process_date, 'YYYYMMDD');
|
|
77
|
+
RETURN;
|
|
78
|
+
END IF;
|
|
79
|
+
|
|
80
|
+
UPDATE ldb_ctl
|
|
81
|
+
SET ctl_process_date = p_process_date,
|
|
82
|
+
ctl_action = p_action,
|
|
83
|
+
ctl_status = p_status,
|
|
84
|
+
ctl_process_start = NVL(p_process_start, ctl_process_start),
|
|
85
|
+
ctl_process_end = p_process_end,
|
|
86
|
+
ctl_rows_processed = DECODE(p_status, 'TSTART', 0, ctl_rows_processed) +
|
|
87
|
+
p_rows_processed,
|
|
88
|
+
ctl_plsql = NVL(p_plsql, ctl_plsql)
|
|
89
|
+
WHERE ROWID = l_rowid;
|
|
90
|
+
EXCEPTION
|
|
91
|
+
WHEN in_use THEN
|
|
92
|
+
p_sqlcode := -20001;
|
|
93
|
+
p_out_message := 'Table ' || p_table_name || ' is currently in process';
|
|
94
|
+
RETURN;
|
|
95
|
+
WHEN NO_DATA_FOUND THEN
|
|
96
|
+
INSERT INTO ldb_ctl (
|
|
97
|
+
ctl_owner,
|
|
98
|
+
ctl_table_name,
|
|
99
|
+
ctl_process_date,
|
|
100
|
+
ctl_action,
|
|
101
|
+
ctl_status,
|
|
102
|
+
ctl_process_start,
|
|
103
|
+
ctl_process_end,
|
|
104
|
+
ctl_rows_processed,
|
|
105
|
+
ctl_plsql
|
|
106
|
+
)
|
|
107
|
+
VALUES (
|
|
108
|
+
p_owner,
|
|
109
|
+
p_table_name,
|
|
110
|
+
p_process_date,
|
|
111
|
+
p_action,
|
|
112
|
+
p_status,
|
|
113
|
+
l_process_start,
|
|
114
|
+
NULL,
|
|
115
|
+
0,
|
|
116
|
+
p_plsql
|
|
117
|
+
);
|
|
118
|
+
END;
|
|
119
|
+
END IF;
|
|
120
|
+
|
|
121
|
+
INSERT INTO ldb_log (
|
|
122
|
+
log_id,
|
|
123
|
+
log_owner,
|
|
124
|
+
log_table_name,
|
|
125
|
+
log_process_date,
|
|
126
|
+
log_action,
|
|
127
|
+
log_status,
|
|
128
|
+
log_process_start,
|
|
129
|
+
log_process_end,
|
|
130
|
+
log_message,
|
|
131
|
+
log_rows_processed,
|
|
132
|
+
log_plsql
|
|
133
|
+
)
|
|
134
|
+
VALUES (
|
|
135
|
+
ldb_log_id.NEXTVAL,
|
|
136
|
+
p_owner,
|
|
137
|
+
p_table_name,
|
|
138
|
+
p_process_date,
|
|
139
|
+
p_action,
|
|
140
|
+
p_status,
|
|
141
|
+
NVL(p_chunk_start, p_process_start),
|
|
142
|
+
p_process_end,
|
|
143
|
+
p_message,
|
|
144
|
+
p_rows_processed,
|
|
145
|
+
p_plsql
|
|
146
|
+
);
|
|
147
|
+
END check_save_status;
|
|
148
|
+
"""
|
|
149
|
+
T_REFERENCING_TABLES_TYPE = """
|
|
150
|
+
CREATE OR REPLACE TYPE t_referencing_tables AS TABLE OF VARCHAR2(100);
|
|
151
|
+
"""
|
|
152
|
+
CHECK_REFERENCING_TABLES_PROC = """
|
|
153
|
+
CREATE OR REPLACE PROCEDURE check_referencing_tables(
|
|
154
|
+
p_referencing_tables IN t_referencing_tables,
|
|
155
|
+
p_process_date IN DATE
|
|
156
|
+
)
|
|
157
|
+
IS
|
|
158
|
+
l_ref_table VARCHAR2(100);
|
|
159
|
+
l_ref_owner VARCHAR2(100);
|
|
160
|
+
l_ref_table_name VARCHAR2(100);
|
|
161
|
+
l_dummy NUMBER;
|
|
162
|
+
BEGIN
|
|
163
|
+
FOR i IN 1 .. p_referencing_tables.COUNT LOOP
|
|
164
|
+
l_ref_table := p_referencing_tables(i);
|
|
165
|
+
l_ref_owner := SUBSTR(l_ref_table, 1, INSTR(l_ref_table, '.') - 1);
|
|
166
|
+
l_ref_table_name := SUBSTR(l_ref_table, INSTR(l_ref_table, '.') + 1);
|
|
167
|
+
BEGIN
|
|
168
|
+
SELECT 1
|
|
169
|
+
INTO l_dummy
|
|
170
|
+
FROM ldb_ctl
|
|
171
|
+
WHERE ctl_owner = l_ref_owner
|
|
172
|
+
AND ctl_table_name = l_ref_table_name
|
|
173
|
+
AND ctl_process_date = p_process_date
|
|
174
|
+
AND ctl_status = 'TEND';
|
|
175
|
+
EXCEPTION
|
|
176
|
+
WHEN NO_DATA_FOUND THEN
|
|
177
|
+
RAISE_APPLICATION_ERROR(
|
|
178
|
+
-20004,
|
|
179
|
+
'Referencing table ' || l_ref_table ||
|
|
180
|
+
' was not fully processed for date ' ||
|
|
181
|
+
TO_CHAR(p_process_date, 'YYYYMMDD')
|
|
182
|
+
);
|
|
183
|
+
END;
|
|
184
|
+
END LOOP;
|
|
185
|
+
END check_referencing_tables;
|
|
186
|
+
"""
|
|
187
|
+
logger = get_logger("oracle.engine")
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
class OracleEngine(DatabaseEngine):
|
|
191
|
+
"""Oracle DB engine with methods for connection, configuration loading, and PL/SQL generation."""
|
|
192
|
+
|
|
193
|
+
# states and class constants
|
|
194
|
+
_connection_envs: ClassVar[dict[int, str]] = {}
|
|
195
|
+
REQUIRED_SYSTEM_PRIVILEGES: tuple[str, ...] = (
|
|
196
|
+
"CREATE SESSION",
|
|
197
|
+
"ALTER SESSION",
|
|
198
|
+
"CREATE TABLE",
|
|
199
|
+
"CREATE PROCEDURE",
|
|
200
|
+
"CREATE TYPE",
|
|
201
|
+
"CREATE DATABASE LINK",
|
|
202
|
+
"CREATE SEQUENCE",
|
|
203
|
+
"RESUMABLE",
|
|
204
|
+
"ALTER USER",
|
|
205
|
+
"CREATE SYNONYM",
|
|
206
|
+
"CREATE VIEW",
|
|
207
|
+
"CREATE ROLE",
|
|
208
|
+
"CREATE TRIGGER",
|
|
209
|
+
"CREATE MATERIALIZED VIEW",
|
|
210
|
+
"QUERY REWRITE",
|
|
211
|
+
)
|
|
212
|
+
_REGULAR_IDENTIFIER = re.compile(r"[A-Z_#$][A-Z0-9_#$]*") # only DD uppercase identifiers are unquoted
|
|
213
|
+
_PREFIXED_IDENTIFIER = re.compile(r'@("(?:""|[^"])*"|[A-Za-z_#$][A-Za-z0-9_#$]*)')
|
|
214
|
+
_PRIVATE_DDL_MARK = re.compile(r"##(.*?)##")
|
|
215
|
+
|
|
216
|
+
# public API
|
|
217
|
+
@staticmethod
|
|
218
|
+
def get_connection(
|
|
219
|
+
config: Config, *, admin: bool = False, env: Literal["SOURCE", "HISTORY"] | None = None
|
|
220
|
+
) -> oracledb.Connection:
|
|
221
|
+
"""Returns an Oracle connection using provided config.
|
|
222
|
+
Args:
|
|
223
|
+
config: Database config object.
|
|
224
|
+
admin: Whether to use admin credentials.
|
|
225
|
+
Returns:
|
|
226
|
+
An active oracledb.Connection."""
|
|
227
|
+
user, password, dsn = get_effective_credentials(config, admin=admin, env=env)
|
|
228
|
+
try:
|
|
229
|
+
connection = oracledb.connect(user=user, password=password, dsn=dsn)
|
|
230
|
+
OracleEngine._register_connection_env(connection, env)
|
|
231
|
+
return cast(oracledb.Connection, connection)
|
|
232
|
+
except Exception as exc:
|
|
233
|
+
logger.critical("Failed to connect to Oracle DB.", exc_info=True)
|
|
234
|
+
raise DatabaseConnectionError("Failed to connect to Oracle database") from exc
|
|
235
|
+
|
|
236
|
+
@staticmethod
|
|
237
|
+
def close_connection(conn: oracledb.Connection) -> None:
|
|
238
|
+
"""Closes the Oracle connection.
|
|
239
|
+
Args:
|
|
240
|
+
conn: Active Oracle connection."""
|
|
241
|
+
try:
|
|
242
|
+
if conn:
|
|
243
|
+
OracleEngine._register_connection_env(conn, None)
|
|
244
|
+
conn.close()
|
|
245
|
+
except Exception:
|
|
246
|
+
logger.critical("Failed to close Oracle DB connection.", exc_info=True)
|
|
247
|
+
|
|
248
|
+
@staticmethod
|
|
249
|
+
def get_table_columns(
|
|
250
|
+
conn: oracledb.Connection, owner: str, table_name: str
|
|
251
|
+
) -> tuple[list[str], dict[str, ColumnDefinition]]:
|
|
252
|
+
"""Retrieves column names and metadata for a given table.
|
|
253
|
+
Args:
|
|
254
|
+
conn: Active Oracle connection.
|
|
255
|
+
owner: Schema owner of the table.
|
|
256
|
+
table_name: Table name.
|
|
257
|
+
Returns:
|
|
258
|
+
Tuple containing a list of column names and a dict of column metadata."""
|
|
259
|
+
cursor: oracledb.Cursor | None = None
|
|
260
|
+
columns: list[str] = []
|
|
261
|
+
metadata: dict[str, ColumnDefinition] = {}
|
|
262
|
+
try:
|
|
263
|
+
cursor = conn.cursor()
|
|
264
|
+
cursor.execute(
|
|
265
|
+
"""select column_id, column_name, data_type, data_length, data_precision, data_scale, nullable, data_default, char_length, char_used
|
|
266
|
+
from all_tab_columns
|
|
267
|
+
where owner = :1 and table_name = :2 and column_id is not null
|
|
268
|
+
order by column_id""",
|
|
269
|
+
[owner, table_name],
|
|
270
|
+
)
|
|
271
|
+
for row in cast(
|
|
272
|
+
Iterable[tuple[int, str, str, int | None, int | None, int | None, str, str | None, str | None, str]],
|
|
273
|
+
cursor,
|
|
274
|
+
):
|
|
275
|
+
(
|
|
276
|
+
column_id,
|
|
277
|
+
column_name,
|
|
278
|
+
data_type,
|
|
279
|
+
data_length,
|
|
280
|
+
data_precision,
|
|
281
|
+
data_scale,
|
|
282
|
+
nullable,
|
|
283
|
+
data_default,
|
|
284
|
+
char_length,
|
|
285
|
+
char_used,
|
|
286
|
+
) = row
|
|
287
|
+
dtype = data_type.lower()
|
|
288
|
+
length: int | None = None
|
|
289
|
+
if dtype in ("varchar2", "varchar", "char"):
|
|
290
|
+
if char_used == "C" and char_length is not None:
|
|
291
|
+
length = int(char_length)
|
|
292
|
+
elif data_length is not None:
|
|
293
|
+
length = int(data_length)
|
|
294
|
+
precision = int(data_precision) if data_precision is not None else None
|
|
295
|
+
scale = int(data_scale) if data_scale is not None else None
|
|
296
|
+
metadata[column_name] = ColumnDefinition(
|
|
297
|
+
name=column_name,
|
|
298
|
+
data_type=dtype,
|
|
299
|
+
id=column_id,
|
|
300
|
+
length=length,
|
|
301
|
+
precision=precision,
|
|
302
|
+
scale=scale,
|
|
303
|
+
nullable=(nullable == "Y"),
|
|
304
|
+
default=data_default.strip() if isinstance(data_default, str) else None,
|
|
305
|
+
)
|
|
306
|
+
columns.append(column_name)
|
|
307
|
+
finally:
|
|
308
|
+
if cursor:
|
|
309
|
+
cursor.close()
|
|
310
|
+
return columns, metadata
|
|
311
|
+
|
|
312
|
+
@staticmethod
|
|
313
|
+
def get_date_condition(date_expr: str, months_keep_src: int) -> str:
|
|
314
|
+
"""Returns a date condition for the given date expression and months to keep.
|
|
315
|
+
Args:
|
|
316
|
+
date_expr: Date expression to evaluate.
|
|
317
|
+
months_keep_src: Months to keep.
|
|
318
|
+
Returns:
|
|
319
|
+
Date condition as string."""
|
|
320
|
+
return f"({date_expr} < add_months(l_process_date,-{months_keep_src}))"
|
|
321
|
+
|
|
322
|
+
@staticmethod
|
|
323
|
+
def get_fallback_expression(expression: str, fallback: str) -> str:
|
|
324
|
+
"""Return Oracle's null-safe fallback expression."""
|
|
325
|
+
return f"coalesce({expression}, {fallback})"
|
|
326
|
+
|
|
327
|
+
@staticmethod
|
|
328
|
+
def get_identifiers_from_expression(expression: str) -> set[str]:
|
|
329
|
+
"""Returns a set of @prefixxed identifiers found in the given expression.
|
|
330
|
+
Args:
|
|
331
|
+
expression: The expression string to process.
|
|
332
|
+
Returns:
|
|
333
|
+
A set of identifier strings found in the expression."""
|
|
334
|
+
identifiers: set[str] = set()
|
|
335
|
+
for match in OracleEngine._PREFIXED_IDENTIFIER.finditer(expression):
|
|
336
|
+
ident = match.group(1)
|
|
337
|
+
if ident.startswith('"') and ident.endswith('"'):
|
|
338
|
+
ident = ident[1:-1].replace('""', '"')
|
|
339
|
+
else:
|
|
340
|
+
ident = ident.upper()
|
|
341
|
+
identifiers.add(ident)
|
|
342
|
+
return identifiers
|
|
343
|
+
|
|
344
|
+
@staticmethod
|
|
345
|
+
def rewrite_expression_identifiers(expression: str, replacements: Mapping[str, str], *, qualifier: str) -> str:
|
|
346
|
+
"""Rewrite ``@`` identifiers to qualified persisted Oracle columns."""
|
|
347
|
+
|
|
348
|
+
def replace_identifier(match: re.Match[str]) -> str:
|
|
349
|
+
token = match.group(1)
|
|
350
|
+
if token.startswith('"') and token.endswith('"'):
|
|
351
|
+
identifier = token[1:-1].replace('""', '"')
|
|
352
|
+
else:
|
|
353
|
+
identifier = token.upper()
|
|
354
|
+
replacement = replacements.get(identifier)
|
|
355
|
+
if replacement is None:
|
|
356
|
+
return match.group(0)
|
|
357
|
+
return f"{qualifier}.{OracleEngine._format_identifier(replacement)}"
|
|
358
|
+
|
|
359
|
+
return OracleEngine._PREFIXED_IDENTIFIER.sub(replace_identifier, expression)
|
|
360
|
+
|
|
361
|
+
@staticmethod
|
|
362
|
+
def get_column_type(column: ColumnDefinition) -> str:
|
|
363
|
+
dtype = column.data_type.lower()
|
|
364
|
+
if dtype in ("string", "varchar", "varchar2"):
|
|
365
|
+
length = column.length or 255
|
|
366
|
+
return f"VARCHAR2({length})"
|
|
367
|
+
if dtype == "char":
|
|
368
|
+
length = column.length or 1
|
|
369
|
+
return f"CHAR({length})"
|
|
370
|
+
if dtype in ("number", "numeric", "decimal"):
|
|
371
|
+
if column.precision is not None and column.scale is not None:
|
|
372
|
+
return f"NUMBER({column.precision},{column.scale})"
|
|
373
|
+
if column.precision is not None:
|
|
374
|
+
return f"NUMBER({column.precision})"
|
|
375
|
+
return "NUMBER"
|
|
376
|
+
if dtype in ("integer", "int"):
|
|
377
|
+
return "NUMBER(10)"
|
|
378
|
+
if dtype == "date":
|
|
379
|
+
return "DATE"
|
|
380
|
+
if dtype == "clob":
|
|
381
|
+
return "CLOB"
|
|
382
|
+
raise ValueError(f"Unsupported column data type: {column.data_type}")
|
|
383
|
+
|
|
384
|
+
@staticmethod
|
|
385
|
+
def generate_sql_block(config: Config, table_cnf: Mapping[str, Any], process_date: str) -> str:
|
|
386
|
+
"""Generates a PL/SQL block for table processing with optional chunking and LOB handling.
|
|
387
|
+
Args:
|
|
388
|
+
config: Configuration object.
|
|
389
|
+
table_cnf: Processed table configuration object.
|
|
390
|
+
process_date: Process date in 'YYYYMMDD' format.
|
|
391
|
+
Returns:
|
|
392
|
+
PL/SQL block as string."""
|
|
393
|
+
execution = config.execution
|
|
394
|
+
administration = config.administration
|
|
395
|
+
cnd0 = table_cnf["conds"][0]
|
|
396
|
+
source_owner, history_owner, table_name = cnd0["source_owner"], cnd0["history_owner"], cnd0["table_name"]
|
|
397
|
+
history_hint_expr = cnd0.get("history_hint_expr")
|
|
398
|
+
hint_expr = cnd0["hint_expr"]
|
|
399
|
+
if execution.action == "HISTORY_ILM":
|
|
400
|
+
hint_expr = nvl(history_hint_expr, hint_expr)
|
|
401
|
+
has_lob_columns = cnd0["has_lob_columns"] == "Y"
|
|
402
|
+
other_columns, referencing_tables = table_cnf["other_columns"], table_cnf["referencing_tables"]
|
|
403
|
+
other_cols_alias = [col["name"] for col in other_columns]
|
|
404
|
+
other_cols_exprs = [col["expr"] for col in other_columns]
|
|
405
|
+
query_expr, table_columns, months_keep_history_max = (
|
|
406
|
+
table_cnf["query_expr"],
|
|
407
|
+
table_cnf["table_columns"],
|
|
408
|
+
table_cnf["months_keep_history_max"],
|
|
409
|
+
)
|
|
410
|
+
referencing_tables = ", ".join([f"'{rt[0]}.{rt[1]}'" for rt in referencing_tables])
|
|
411
|
+
source_ilm = execution.action == "SOURCE_ILM"
|
|
412
|
+
runtime_owner = source_owner if source_ilm else history_owner
|
|
413
|
+
ins_cols = join_wrapped(", ", table_columns + other_cols_alias, 200)
|
|
414
|
+
ins_vals = join_wrapped(", ", [f"r_rec(i).{col}" for col in table_columns + other_cols_alias], 200)
|
|
415
|
+
if execution.generate_script:
|
|
416
|
+
process_date = "&process_date"
|
|
417
|
+
plsql = f"""declare
|
|
418
|
+
l_source_owner varchar2(50) := '{source_owner}';
|
|
419
|
+
l_history_owner varchar2(50) := '{history_owner}';
|
|
420
|
+
l_action varchar2(11) := '{execution.action}';
|
|
421
|
+
l_mode varchar2(10) := '{execution.mode}';
|
|
422
|
+
l_message varchar2(200) := case
|
|
423
|
+
when l_mode = 'EXECUTE' then null
|
|
424
|
+
when l_mode = 'SCRIPT' then 'SCRIPT'
|
|
425
|
+
when l_mode = 'PREVIEW' then 'PREVIEW'
|
|
426
|
+
else l_mode
|
|
427
|
+
end;
|
|
428
|
+
l_table_name varchar2(50) := '{table_name}';
|
|
429
|
+
l_process_date date := to_date('{process_date}', 'YYYYMMDD');
|
|
430
|
+
l_referencing_tables t_referencing_tables := t_referencing_tables({referencing_tables});
|
|
431
|
+
l_process_start date;
|
|
432
|
+
l_record_count pls_integer := 0;
|
|
433
|
+
l_plsql clob := {"null" if execution.generate_script else ":plsql_code"};
|
|
434
|
+
l_sqlcode number := null;
|
|
435
|
+
l_out_message varchar2(200) := null;"""
|
|
436
|
+
if not has_lob_columns:
|
|
437
|
+
other_cols_select = [
|
|
438
|
+
f"{expression} AS {OracleEngine._format_identifier(alias)}"
|
|
439
|
+
for expression, alias in zip(other_cols_exprs, other_cols_alias, strict=True)
|
|
440
|
+
]
|
|
441
|
+
cols_expr = ", ".join(["A.*", *other_cols_select])
|
|
442
|
+
plsql += f"""
|
|
443
|
+
l_chunk_size pls_integer := {execution.chunk_size}; l_chunk_start date;
|
|
444
|
+
cursor c_records is
|
|
445
|
+
select /*+ {hint_expr} */ A.rowid{", " + cols_expr if source_ilm else ""}
|
|
446
|
+
{indent_lines(query_expr, 8)};
|
|
447
|
+
type t_records is table of c_records%rowtype index by pls_integer; r_rec t_records;"""
|
|
448
|
+
plsql += f"""
|
|
449
|
+
begin
|
|
450
|
+
l_process_start := sysdate;
|
|
451
|
+
check_save_status(l_source_owner, l_table_name, l_process_date, l_action, '{Status.TABLE_START}', l_process_start, null, null, l_message, 0, l_plsql, l_sqlcode, l_out_message);
|
|
452
|
+
if l_sqlcode is not null then raise_application_error(l_sqlcode, l_out_message); end if;"""
|
|
453
|
+
if source_ilm or not execution.use_added_columns:
|
|
454
|
+
plsql += """
|
|
455
|
+
check_referencing_tables(l_referencing_tables, l_process_date); commit;"""
|
|
456
|
+
if not has_lob_columns or not source_ilm:
|
|
457
|
+
plsql += f"""
|
|
458
|
+
open c_records;
|
|
459
|
+
loop
|
|
460
|
+
l_chunk_start := sysdate;
|
|
461
|
+
check_save_status(l_source_owner, l_table_name, l_process_date, l_action, '{Status.CHUNK_START}', null, l_chunk_start, null, l_message, 0, null, l_sqlcode, l_out_message);
|
|
462
|
+
fetch c_records bulk collect into r_rec limit l_chunk_size;
|
|
463
|
+
if r_rec.count <= 0 then
|
|
464
|
+
exit;
|
|
465
|
+
end if;"""
|
|
466
|
+
if execution.mode in ("EXECUTE", "SCRIPT"):
|
|
467
|
+
if source_ilm and nvl(months_keep_history_max, 1) > 0:
|
|
468
|
+
plsql += f"""
|
|
469
|
+
for i in 1 .. r_rec.count loop
|
|
470
|
+
insert into {OracleEngine._format_identifier(history_owner)}.{OracleEngine._format_identifier(table_name)}@{OracleEngine._format_identifier(administration.source_to_history_dblink_name)}
|
|
471
|
+
({indent_lines(ins_cols, 12)})
|
|
472
|
+
values ({indent_lines(ins_vals, 12)});
|
|
473
|
+
end loop;"""
|
|
474
|
+
plsql += f"""
|
|
475
|
+
forall i in 1 .. r_rec.count
|
|
476
|
+
delete from {OracleEngine._format_identifier(runtime_owner)}.{OracleEngine._format_identifier(table_name)} where rowid = r_rec(i).rowid;
|
|
477
|
+
l_record_count := l_record_count + r_rec.count;"""
|
|
478
|
+
plsql += f"""
|
|
479
|
+
check_save_status(l_source_owner, l_table_name, l_process_date, l_action, '{Status.CHUNK_END}', null, l_chunk_start, sysdate, l_message, r_rec.count, null, l_sqlcode, l_out_message);
|
|
480
|
+
commit;
|
|
481
|
+
end loop;
|
|
482
|
+
close c_records;"""
|
|
483
|
+
else:
|
|
484
|
+
cols_select = ", ".join([f"a.{col}" for col in table_columns] + other_cols_exprs)
|
|
485
|
+
if execution.mode in ("EXECUTE", "SCRIPT"):
|
|
486
|
+
if source_ilm and nvl(months_keep_history_max, 0) > 0:
|
|
487
|
+
plsql += f"""
|
|
488
|
+
insert into {OracleEngine._format_identifier(history_owner)}.{OracleEngine._format_identifier(table_name)}@{OracleEngine._format_identifier(administration.source_to_history_dblink_name)}({indent_lines(ins_cols, 4)})
|
|
489
|
+
select /*+ {hint_expr} */ {indent_lines(cols_select, 4)}
|
|
490
|
+
{indent_lines(query_expr, 4)};"""
|
|
491
|
+
plsql += f"""
|
|
492
|
+
delete from {OracleEngine._format_identifier(runtime_owner)}.{OracleEngine._format_identifier(table_name)} where rowid in
|
|
493
|
+
(select /*+ {hint_expr} */ a.rowid
|
|
494
|
+
{indent_lines(query_expr, 4)});
|
|
495
|
+
l_record_count := sql%rowcount;"""
|
|
496
|
+
else:
|
|
497
|
+
plsql += f"""
|
|
498
|
+
select /*+ {hint_expr} */ count(*) into l_record_count
|
|
499
|
+
{indent_lines(query_expr, 4)};"""
|
|
500
|
+
table_end_rows = "l_record_count" if has_lob_columns else "0"
|
|
501
|
+
plsql += f"""
|
|
502
|
+
check_save_status(l_source_owner, l_table_name, l_process_date, l_action, '{Status.TABLE_END}', l_process_start, null, sysdate, l_message, {table_end_rows}, null, l_sqlcode, l_out_message);
|
|
503
|
+
commit;
|
|
504
|
+
exception
|
|
505
|
+
when others then
|
|
506
|
+
rollback;
|
|
507
|
+
check_save_status(l_source_owner, l_table_name, l_process_date, l_action, '{Status.ERROR}', l_process_start, null, sysdate, sqlerrm, l_record_count, null, l_sqlcode, l_out_message);
|
|
508
|
+
commit;
|
|
509
|
+
if l_sqlcode not in (-20003) then
|
|
510
|
+
raise;
|
|
511
|
+
end if;
|
|
512
|
+
end;"""
|
|
513
|
+
return plsql
|
|
514
|
+
|
|
515
|
+
@staticmethod
|
|
516
|
+
def get_system_date(conn: oracledb.Connection) -> datetime:
|
|
517
|
+
"""Returns the current system date from Oracle.
|
|
518
|
+
Args:
|
|
519
|
+
conn: Active Oracle connection.
|
|
520
|
+
Returns:
|
|
521
|
+
Current system date as Python date."""
|
|
522
|
+
cursor: oracledb.Cursor | None = None
|
|
523
|
+
try:
|
|
524
|
+
cursor = conn.cursor()
|
|
525
|
+
cursor.execute("SELECT SYSDATE FROM dual")
|
|
526
|
+
sysdate: datetime = cursor.fetchone()[0]
|
|
527
|
+
return sysdate
|
|
528
|
+
except Exception as exc:
|
|
529
|
+
logger.critical("Failed to retrieve system date from Oracle.", exc_info=True)
|
|
530
|
+
raise ExecutionError("Failed to retrieve system date from Oracle") from exc
|
|
531
|
+
finally:
|
|
532
|
+
if cursor:
|
|
533
|
+
cursor.close()
|
|
534
|
+
|
|
535
|
+
@staticmethod
|
|
536
|
+
def sql_block_run(conn: oracledb.Connection, plsql_code: str) -> None:
|
|
537
|
+
"""Executes a PL/SQL block on the database.
|
|
538
|
+
Args:
|
|
539
|
+
conn: Active Oracle connection.
|
|
540
|
+
plsql_code: The PL/SQL block to execute."""
|
|
541
|
+
cursor: oracledb.Cursor | None = None
|
|
542
|
+
try:
|
|
543
|
+
cursor = conn.cursor()
|
|
544
|
+
cursor.setinputsizes(plsql_code=oracledb.CLOB)
|
|
545
|
+
cursor.execute(plsql_code, {"plsql_code": plsql_code})
|
|
546
|
+
finally:
|
|
547
|
+
if cursor:
|
|
548
|
+
cursor.close()
|
|
549
|
+
|
|
550
|
+
@staticmethod
|
|
551
|
+
def get_status(conn: oracledb.Connection, process_date: str) -> list[dict[str, Any]]:
|
|
552
|
+
"""Loads status from ldb_ctl for the given process date.
|
|
553
|
+
Args:
|
|
554
|
+
conn: Active Oracle connection.
|
|
555
|
+
process_date: Date in 'YYYYMMDD' format.
|
|
556
|
+
Returns:
|
|
557
|
+
List of status rows."""
|
|
558
|
+
cursor: oracledb.Cursor | None = None
|
|
559
|
+
try:
|
|
560
|
+
cursor = conn.cursor()
|
|
561
|
+
cursor.execute(
|
|
562
|
+
"""SELECT ctl_owner, ctl_table_name, ctl_status
|
|
563
|
+
FROM ldb_ctl
|
|
564
|
+
WHERE ctl_process_date = TO_DATE(:1, 'YYYYMMDD')""",
|
|
565
|
+
[process_date],
|
|
566
|
+
)
|
|
567
|
+
cols = [col[0].lower() for col in cursor.description]
|
|
568
|
+
return [dict(zip(cols, row, strict=True)) for row in cursor.fetchall()]
|
|
569
|
+
finally:
|
|
570
|
+
if cursor:
|
|
571
|
+
cursor.close()
|
|
572
|
+
|
|
573
|
+
@staticmethod
|
|
574
|
+
def get_rows_processed(conn: oracledb.Connection, owner: str, table_name: str, process_date: str) -> int:
|
|
575
|
+
"""Returns rows processed for a given table and process date from ldb_ctl.
|
|
576
|
+
Args:
|
|
577
|
+
conn: Active Oracle connection.
|
|
578
|
+
owner: Schema owner of the table.
|
|
579
|
+
table_name: Table name.
|
|
580
|
+
process_date: Target process date in 'YYYYMMDD'.
|
|
581
|
+
Returns:
|
|
582
|
+
Number of rows processed or 0 if none found or mismatched date."""
|
|
583
|
+
cursor: oracledb.Cursor | None = None
|
|
584
|
+
try:
|
|
585
|
+
cursor = conn.cursor()
|
|
586
|
+
cursor.execute(
|
|
587
|
+
"""SELECT ctl_rows_processed
|
|
588
|
+
FROM ldb_ctl WHERE ctl_owner = :1 AND ctl_table_name = :2 AND ctl_process_date = TO_DATE(:3, 'YYYYMMDD')""",
|
|
589
|
+
[owner, table_name, process_date],
|
|
590
|
+
)
|
|
591
|
+
result = cursor.fetchone()
|
|
592
|
+
return result[0] if result else 0
|
|
593
|
+
except Exception as exc:
|
|
594
|
+
logger.critical(f"Unexpected error in get_rows_processed for {owner}.{table_name}:", exc_info=True)
|
|
595
|
+
raise ExecutionError(f"Failed to read rows processed for {owner}.{table_name}") from exc
|
|
596
|
+
finally:
|
|
597
|
+
if cursor:
|
|
598
|
+
cursor.close()
|
|
599
|
+
|
|
600
|
+
@staticmethod
|
|
601
|
+
def save_error_status(
|
|
602
|
+
conn: oracledb.Connection,
|
|
603
|
+
config: Config,
|
|
604
|
+
owner: str,
|
|
605
|
+
table_name: str,
|
|
606
|
+
process_date: str,
|
|
607
|
+
process_start: datetime,
|
|
608
|
+
message: str,
|
|
609
|
+
plsql_code: str,
|
|
610
|
+
) -> None:
|
|
611
|
+
cursor: oracledb.Cursor | None = None
|
|
612
|
+
try:
|
|
613
|
+
cursor = conn.cursor()
|
|
614
|
+
process_end = OracleEngine.get_system_date(conn)
|
|
615
|
+
sqlcode = cursor.var(oracledb.NUMBER)
|
|
616
|
+
out_message = cursor.var(oracledb.STRING)
|
|
617
|
+
cursor.callproc(
|
|
618
|
+
"check_save_status",
|
|
619
|
+
[
|
|
620
|
+
owner,
|
|
621
|
+
table_name,
|
|
622
|
+
datetime.strptime(process_date, "%Y%m%d").date(),
|
|
623
|
+
config.execution.action,
|
|
624
|
+
Status.ERROR,
|
|
625
|
+
process_start,
|
|
626
|
+
None,
|
|
627
|
+
process_end,
|
|
628
|
+
message,
|
|
629
|
+
0,
|
|
630
|
+
plsql_code,
|
|
631
|
+
sqlcode,
|
|
632
|
+
out_message,
|
|
633
|
+
],
|
|
634
|
+
)
|
|
635
|
+
conn.commit()
|
|
636
|
+
except Exception as exc:
|
|
637
|
+
logger.critical(f"Unexpected error in process_table for {owner}.{table_name}:", exc_info=True)
|
|
638
|
+
raise ExecutionError(f"Failed to save error status for {owner}.{table_name}") from exc
|
|
639
|
+
finally:
|
|
640
|
+
if cursor:
|
|
641
|
+
cursor.close()
|
|
642
|
+
|
|
643
|
+
@staticmethod
|
|
644
|
+
def all_status_tend(
|
|
645
|
+
conn: oracledb.Connection, tables_config: dict[tuple[str, str], Any], process_date: str
|
|
646
|
+
) -> bool:
|
|
647
|
+
"""Checks if all referenced tables have status "Status.TABLE_END" in ldb_ctl.
|
|
648
|
+
Args:
|
|
649
|
+
conn: Active Oracle connection.
|
|
650
|
+
tables_config: Dict of (owner, table_name) keys representing configured tables.
|
|
651
|
+
process_date: Processing date in 'YYYYMMDD' format.
|
|
652
|
+
Returns:
|
|
653
|
+
True if all tables have status "Status.TABLE_END" for the given process date, False otherwise."""
|
|
654
|
+
cursor: oracledb.Cursor | None = None
|
|
655
|
+
try:
|
|
656
|
+
cursor = conn.cursor()
|
|
657
|
+
for (owner, table_name), table_cnf in tables_config.items():
|
|
658
|
+
if table_cnf["skip"]:
|
|
659
|
+
continue
|
|
660
|
+
cursor.execute(
|
|
661
|
+
f"""SELECT ctl_status FROM ldb_ctl
|
|
662
|
+
WHERE ctl_owner = :1 AND ctl_table_name = :2
|
|
663
|
+
AND ctl_process_date = TO_DATE(:3, 'YYYYMMDD') AND ctl_status = '{Status.TABLE_END}'""",
|
|
664
|
+
[owner, table_name, process_date],
|
|
665
|
+
)
|
|
666
|
+
if not cursor.fetchone():
|
|
667
|
+
return False
|
|
668
|
+
return True
|
|
669
|
+
finally:
|
|
670
|
+
if cursor:
|
|
671
|
+
cursor.close()
|
|
672
|
+
|
|
673
|
+
@staticmethod
|
|
674
|
+
def ensure_users(conn: oracledb.Connection, users: Sequence[UserDefinition]) -> list[str]:
|
|
675
|
+
created: list[str] = []
|
|
676
|
+
if not users:
|
|
677
|
+
return created
|
|
678
|
+
cursor: oracledb.Cursor | None = None
|
|
679
|
+
changed = False
|
|
680
|
+
try:
|
|
681
|
+
cursor = conn.cursor()
|
|
682
|
+
database_default_tablespace = OracleEngine._get_database_default_tablespace(cursor)
|
|
683
|
+
for user in users:
|
|
684
|
+
fmttd_username = OracleEngine._format_identifier(user.name)
|
|
685
|
+
fmttd_default_tablespace = OracleEngine._format_identifier(user.default_tablespace)
|
|
686
|
+
tablespace_for_quota = user.default_tablespace
|
|
687
|
+
if not OracleEngine._user_exists(cursor, user.name):
|
|
688
|
+
sql = f"CREATE USER {fmttd_username} IDENTIFIED BY ##{OracleEngine._quote(user.password)}##"
|
|
689
|
+
if user.default_tablespace:
|
|
690
|
+
sql += f" DEFAULT TABLESPACE {fmttd_default_tablespace}"
|
|
691
|
+
if user.temporary_tablespace:
|
|
692
|
+
sql += f" TEMPORARY TABLESPACE {OracleEngine._format_identifier(user.temporary_tablespace)}"
|
|
693
|
+
logger.debug(f"Creating user with SQL: {sql}")
|
|
694
|
+
OracleEngine._execute_ddl(conn, cursor, sql)
|
|
695
|
+
created.append(fmttd_username)
|
|
696
|
+
changed = True
|
|
697
|
+
if not user.default_tablespace:
|
|
698
|
+
tablespace_for_quota = (
|
|
699
|
+
OracleEngine._get_user_default_tablespace(cursor, user.name) or database_default_tablespace
|
|
700
|
+
)
|
|
701
|
+
else:
|
|
702
|
+
tablespace_for_quota = (
|
|
703
|
+
OracleEngine._get_user_default_tablespace(cursor, user.name) or database_default_tablespace
|
|
704
|
+
)
|
|
705
|
+
admin_option_roles = set(OracleEngine._format_identifier(r) for r in user.roles_with_admin_option)
|
|
706
|
+
for role in user.roles:
|
|
707
|
+
fmttd_role_name = OracleEngine._format_identifier(role)
|
|
708
|
+
requires_admin_option = role in admin_option_roles
|
|
709
|
+
has_role = OracleEngine._user_has_role(cursor, user.name, role)
|
|
710
|
+
if requires_admin_option:
|
|
711
|
+
if not has_role or not OracleEngine._user_has_role_with_admin_option(cursor, user.name, role):
|
|
712
|
+
grant_sql = f"GRANT {fmttd_role_name} TO {fmttd_username} WITH ADMIN OPTION"
|
|
713
|
+
OracleEngine._execute_ddl(conn, cursor, grant_sql)
|
|
714
|
+
changed = True
|
|
715
|
+
elif not has_role:
|
|
716
|
+
grant_sql = f"GRANT {fmttd_role_name} TO {fmttd_username}"
|
|
717
|
+
OracleEngine._execute_ddl(conn, cursor, grant_sql)
|
|
718
|
+
changed = True
|
|
719
|
+
for privilege in user.system_privileges:
|
|
720
|
+
if not OracleEngine._user_has_sys_priv(cursor, user.name, privilege):
|
|
721
|
+
grant_priv_sql = f"GRANT {privilege} TO {fmttd_username}"
|
|
722
|
+
OracleEngine._execute_ddl(conn, cursor, grant_priv_sql)
|
|
723
|
+
changed = True
|
|
724
|
+
if OracleEngine._ensure_unlimited_quota(cursor, user.name, tablespace_for_quota):
|
|
725
|
+
changed = True
|
|
726
|
+
if changed:
|
|
727
|
+
conn.commit()
|
|
728
|
+
except Exception as exc:
|
|
729
|
+
conn.rollback()
|
|
730
|
+
logger.critical("Failed to ensure users.", exc_info=True)
|
|
731
|
+
raise ExecutionError("Failed to ensure Oracle users") from exc
|
|
732
|
+
finally:
|
|
733
|
+
if cursor:
|
|
734
|
+
cursor.close()
|
|
735
|
+
return created
|
|
736
|
+
|
|
737
|
+
@staticmethod
|
|
738
|
+
def ensure_tables(conn: oracledb.Connection, tables: Sequence[TableDefinition]) -> list[str]:
|
|
739
|
+
created: list[str] = []
|
|
740
|
+
if not tables:
|
|
741
|
+
return created
|
|
742
|
+
cursor: oracledb.Cursor | None = None
|
|
743
|
+
changed = False
|
|
744
|
+
try:
|
|
745
|
+
cursor = conn.cursor()
|
|
746
|
+
for table in tables:
|
|
747
|
+
fmttd_owner = OracleEngine._format_identifier(table.owner)
|
|
748
|
+
fmttd_table_name = OracleEngine._format_identifier(table.name)
|
|
749
|
+
if not OracleEngine._table_exists(cursor, table.owner, table.name):
|
|
750
|
+
columns_sql = ",\n ".join(OracleEngine._column_sql(col) for col in table.columns)
|
|
751
|
+
sql = f"CREATE TABLE {fmttd_owner}.{fmttd_table_name} (\n {columns_sql}\n )"
|
|
752
|
+
OracleEngine._execute_ddl(conn, cursor, sql)
|
|
753
|
+
created.append(f"{fmttd_owner}.{fmttd_table_name}")
|
|
754
|
+
changed = True
|
|
755
|
+
if table.primary_key:
|
|
756
|
+
pk_name = f"{table.name}_PK"
|
|
757
|
+
fmttd_pk_name = OracleEngine._format_identifier(pk_name)
|
|
758
|
+
if not OracleEngine._constraint_exists(cursor, table.owner, pk_name):
|
|
759
|
+
cols = ", ".join(OracleEngine._format_identifier(col) for col in table.primary_key)
|
|
760
|
+
sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} ADD CONSTRAINT {fmttd_pk_name} PRIMARY KEY ({cols})"
|
|
761
|
+
OracleEngine._execute_ddl(conn, cursor, sql)
|
|
762
|
+
changed = True
|
|
763
|
+
for index in table.indexes:
|
|
764
|
+
fmttd_idx_name = OracleEngine._format_identifier(index.name)
|
|
765
|
+
if OracleEngine._index_exists(cursor, table.owner, index.name):
|
|
766
|
+
continue
|
|
767
|
+
cols = ", ".join(OracleEngine._format_identifier(col) for col in index.columns)
|
|
768
|
+
unique_kw = "UNIQUE " if index.unique else ""
|
|
769
|
+
sql = f"CREATE {unique_kw}INDEX {fmttd_owner}.{fmttd_idx_name} ON {fmttd_owner}.{fmttd_table_name} ({cols})"
|
|
770
|
+
OracleEngine._execute_ddl(conn, cursor, sql)
|
|
771
|
+
changed = True
|
|
772
|
+
if changed:
|
|
773
|
+
conn.commit()
|
|
774
|
+
except Exception as exc:
|
|
775
|
+
conn.rollback()
|
|
776
|
+
logger.critical("Failed to ensure tables.", exc_info=True)
|
|
777
|
+
raise ExecutionError("Failed to ensure Oracle tables") from exc
|
|
778
|
+
finally:
|
|
779
|
+
if cursor:
|
|
780
|
+
cursor.close()
|
|
781
|
+
return created
|
|
782
|
+
|
|
783
|
+
@staticmethod
|
|
784
|
+
def ensure_table_structure(conn: oracledb.Connection, table: TableDefinition, table_cnf: Mapping[str, Any]) -> None:
|
|
785
|
+
cursor: oracledb.Cursor | None = None
|
|
786
|
+
fmttd_owner = OracleEngine._format_identifier(table.owner)
|
|
787
|
+
fmttd_table_name = OracleEngine._format_identifier(table.name)
|
|
788
|
+
logger.debug(f"Ensuring structure for table {table.owner}.{table.name} conn: {conn}")
|
|
789
|
+
try:
|
|
790
|
+
cursor = conn.cursor()
|
|
791
|
+
if not OracleEngine._table_exists(cursor, table.owner, table.name):
|
|
792
|
+
fmttd_columns_sql = ",\n ".join(OracleEngine._column_sql(col) for col in table.columns)
|
|
793
|
+
sql = f"CREATE TABLE {fmttd_owner}.{fmttd_table_name} (\n {fmttd_columns_sql}\n )"
|
|
794
|
+
OracleEngine._execute_ddl(conn, cursor, sql)
|
|
795
|
+
else:
|
|
796
|
+
_, existing_columns = OracleEngine.get_table_columns(conn, table.owner, table.name)
|
|
797
|
+
for column in table.columns:
|
|
798
|
+
column_definition = OracleEngine._column_sql(column)
|
|
799
|
+
if column.name not in existing_columns:
|
|
800
|
+
sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} ADD ({column_definition})"
|
|
801
|
+
OracleEngine._execute_ddl(conn, cursor, sql)
|
|
802
|
+
else:
|
|
803
|
+
existing = existing_columns[column.name]
|
|
804
|
+
if OracleEngine._column_needs_update(existing, column):
|
|
805
|
+
sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} MODIFY ({column_definition})"
|
|
806
|
+
OracleEngine._execute_ddl(conn, cursor, sql)
|
|
807
|
+
existing_indexes = OracleEngine._get_table_indexes(cursor, fmttd_owner, fmttd_table_name)
|
|
808
|
+
desired_pk = tuple(table.primary_key or ())
|
|
809
|
+
fmttd_desired_pk = tuple(OracleEngine._format_identifier(col) for col in desired_pk)
|
|
810
|
+
existing_pk_name, existing_pk_cols, existing_pk_index = OracleEngine._get_primary_key_info(
|
|
811
|
+
cursor, table.owner, table.name
|
|
812
|
+
)
|
|
813
|
+
fmttd_existing_pk_cols = tuple(OracleEngine._format_identifier(col) for col in existing_pk_cols)
|
|
814
|
+
fmttd_existing_pk_index = OracleEngine._format_identifier(existing_pk_index) if existing_pk_index else None
|
|
815
|
+
desired_constraint_name = f"{table.name}_PK"
|
|
816
|
+
fmttd_desired_constraint_name = OracleEngine._format_identifier(desired_constraint_name)
|
|
817
|
+
if desired_pk:
|
|
818
|
+
if not existing_pk_cols:
|
|
819
|
+
existing_index_info = existing_indexes.get(desired_constraint_name)
|
|
820
|
+
if existing_index_info and (existing_index_info[0] != desired_pk or not existing_index_info[1]):
|
|
821
|
+
drop_sql = f"DROP INDEX {fmttd_owner}.{fmttd_desired_constraint_name}"
|
|
822
|
+
OracleEngine._execute_ddl(conn, cursor, drop_sql)
|
|
823
|
+
existing_indexes.pop(desired_constraint_name, None)
|
|
824
|
+
add_pk_sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} ADD CONSTRAINT {fmttd_desired_constraint_name} PRIMARY KEY ({', '.join(fmttd_desired_pk)})"
|
|
825
|
+
OracleEngine._execute_ddl(conn, cursor, add_pk_sql)
|
|
826
|
+
elif existing_pk_cols != desired_pk:
|
|
827
|
+
if existing_pk_name:
|
|
828
|
+
drop_pk_sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} DROP CONSTRAINT {OracleEngine._format_identifier(existing_pk_name)}"
|
|
829
|
+
OracleEngine._execute_ddl(conn, cursor, drop_pk_sql)
|
|
830
|
+
index_to_drop = fmttd_existing_pk_index or existing_pk_name
|
|
831
|
+
if index_to_drop:
|
|
832
|
+
existing_indexes = OracleEngine._get_table_indexes(cursor, table.owner, table.name)
|
|
833
|
+
if index_to_drop and index_to_drop in existing_indexes:
|
|
834
|
+
drop_index_sql = f"DROP INDEX {fmttd_owner}.{index_to_drop}"
|
|
835
|
+
OracleEngine._execute_ddl(conn, cursor, drop_index_sql)
|
|
836
|
+
existing_indexes.pop(index_to_drop, None)
|
|
837
|
+
existing_index_info = existing_indexes.get(desired_constraint_name)
|
|
838
|
+
if existing_index_info and (existing_index_info[0] != desired_pk or not existing_index_info[1]):
|
|
839
|
+
drop_conflict_sql = f"DROP INDEX {fmttd_owner}.{fmttd_desired_constraint_name}"
|
|
840
|
+
OracleEngine._execute_ddl(conn, cursor, drop_conflict_sql)
|
|
841
|
+
existing_indexes.pop(desired_constraint_name, None)
|
|
842
|
+
recreate_pk_sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} ADD CONSTRAINT {fmttd_desired_constraint_name} PRIMARY KEY ({', '.join(fmttd_desired_pk)})"
|
|
843
|
+
OracleEngine._execute_ddl(conn, cursor, recreate_pk_sql)
|
|
844
|
+
elif fmttd_existing_pk_cols and existing_pk_name:
|
|
845
|
+
drop_unexpected_pk_sql = (
|
|
846
|
+
f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} DROP CONSTRAINT {existing_pk_name}"
|
|
847
|
+
)
|
|
848
|
+
OracleEngine._execute_ddl(conn, cursor, drop_unexpected_pk_sql)
|
|
849
|
+
index_to_drop = fmttd_existing_pk_index or existing_pk_name
|
|
850
|
+
if index_to_drop:
|
|
851
|
+
existing_indexes = OracleEngine._get_table_indexes(cursor, table.owner, table.name)
|
|
852
|
+
if index_to_drop and index_to_drop in existing_indexes:
|
|
853
|
+
drop_unexpected_index_sql = f"DROP INDEX {fmttd_owner}.{index_to_drop}"
|
|
854
|
+
OracleEngine._execute_ddl(conn, cursor, drop_unexpected_index_sql)
|
|
855
|
+
existing_indexes.pop(index_to_drop, None)
|
|
856
|
+
existing_indexes = OracleEngine._get_table_indexes(cursor, table.owner, table.name)
|
|
857
|
+
desired_indexes = OracleEngine._prepare_desired_indexes(table)
|
|
858
|
+
for index_name, (columns, is_unique) in desired_indexes.items():
|
|
859
|
+
fmttd_index_name = OracleEngine._format_identifier(index_name)
|
|
860
|
+
fmttd_columns = tuple(OracleEngine._format_identifier(col) for col in columns)
|
|
861
|
+
existing_index = existing_indexes.get(index_name)
|
|
862
|
+
if existing_index == (columns, is_unique):
|
|
863
|
+
continue
|
|
864
|
+
if existing_index is not None:
|
|
865
|
+
drop_mismatch_index_sql = f"DROP INDEX {fmttd_owner}.{fmttd_index_name}"
|
|
866
|
+
OracleEngine._execute_ddl(conn, cursor, drop_mismatch_index_sql)
|
|
867
|
+
fmttd_columns_sql = ", ".join(fmttd_columns)
|
|
868
|
+
unique_clause = "UNIQUE " if is_unique else ""
|
|
869
|
+
create_index_sql = f"CREATE {unique_clause}INDEX {fmttd_owner}.{index_name} ON {fmttd_owner}.{fmttd_table_name} ({fmttd_columns_sql})"
|
|
870
|
+
OracleEngine._execute_ddl(conn, cursor, create_index_sql)
|
|
871
|
+
final_pk_name, _, final_pk_index = OracleEngine._get_primary_key_info(cursor, table.owner, table.name)
|
|
872
|
+
expected_indexes = set(desired_indexes.keys())
|
|
873
|
+
if final_pk_name:
|
|
874
|
+
expected_indexes.add(final_pk_name)
|
|
875
|
+
if final_pk_index:
|
|
876
|
+
expected_indexes.add(final_pk_index)
|
|
877
|
+
existing_indexes = OracleEngine._get_table_indexes(cursor, table.owner, table.name)
|
|
878
|
+
for index_name in list(existing_indexes.keys()):
|
|
879
|
+
if index_name not in expected_indexes:
|
|
880
|
+
fmttd_index_name = OracleEngine._format_identifier(index_name)
|
|
881
|
+
logger.info("Dropping unmanaged index %s on %s.%s", fmttd_index_name, fmttd_owner, fmttd_table_name)
|
|
882
|
+
drop_unmanaged_index_sql = f"DROP INDEX {fmttd_owner}.{fmttd_index_name}"
|
|
883
|
+
OracleEngine._execute_ddl(conn, cursor, drop_unmanaged_index_sql)
|
|
884
|
+
except Exception as exc:
|
|
885
|
+
conn.rollback()
|
|
886
|
+
logger.critical("Failed to ensure table structure.", exc_info=True)
|
|
887
|
+
raise ExecutionError("Failed to ensure Oracle table structure") from exc
|
|
888
|
+
finally:
|
|
889
|
+
if cursor:
|
|
890
|
+
cursor.close()
|
|
891
|
+
|
|
892
|
+
@staticmethod
|
|
893
|
+
def get_primary_key_columns(
|
|
894
|
+
conn: oracledb.Connection, owner: str, table_name: str, table_cnf: Mapping[str, Any]
|
|
895
|
+
) -> tuple[str, ...]:
|
|
896
|
+
cursor: oracledb.Cursor | None = None
|
|
897
|
+
try:
|
|
898
|
+
cursor = conn.cursor()
|
|
899
|
+
_, columns, _ = OracleEngine._get_primary_key_info(cursor, owner, table_name)
|
|
900
|
+
if columns:
|
|
901
|
+
return tuple(col for col in columns)
|
|
902
|
+
cursor.execute(
|
|
903
|
+
"""
|
|
904
|
+
SELECT i.index_name, i.uniqueness, c.column_name, s.distinct_keys
|
|
905
|
+
FROM dba_indexes i JOIN dba_ind_columns c ON i.owner = c.index_owner AND i.index_name = c.index_name
|
|
906
|
+
LEFT JOIN dba_ind_statistics s ON s.owner = i.owner AND s.index_name = i.index_name AND s.partition_name IS NULL
|
|
907
|
+
WHERE i.owner = :owner
|
|
908
|
+
AND i.table_name = :table_name
|
|
909
|
+
ORDER BY i.index_name, c.column_position
|
|
910
|
+
""",
|
|
911
|
+
owner=owner,
|
|
912
|
+
table_name=table_name,
|
|
913
|
+
)
|
|
914
|
+
indexes: dict[str, dict[str, Any]] = {}
|
|
915
|
+
for index_name, uniqueness, column_name, distinct_keys in cast(
|
|
916
|
+
Iterable[tuple[str, str, str, int | None]], cursor
|
|
917
|
+
):
|
|
918
|
+
index_info = indexes.setdefault(
|
|
919
|
+
index_name,
|
|
920
|
+
{
|
|
921
|
+
"columns": [],
|
|
922
|
+
"unique": uniqueness == "UNIQUE",
|
|
923
|
+
"distinct_keys": None,
|
|
924
|
+
},
|
|
925
|
+
)
|
|
926
|
+
index_info["columns"].append(column_name)
|
|
927
|
+
if distinct_keys is not None:
|
|
928
|
+
try:
|
|
929
|
+
index_info["distinct_keys"] = int(distinct_keys)
|
|
930
|
+
except (TypeError, ValueError):
|
|
931
|
+
try:
|
|
932
|
+
index_info["distinct_keys"] = int(float(distinct_keys))
|
|
933
|
+
except (TypeError, ValueError):
|
|
934
|
+
index_info["distinct_keys"] = index_info.get("distinct_keys")
|
|
935
|
+
if not indexes:
|
|
936
|
+
return ()
|
|
937
|
+
all_columns = {col for info in indexes.values() for col in info.get("columns", [])}
|
|
938
|
+
columns_metadata = table_cnf.get("columns_metadata", {})
|
|
939
|
+
for col in all_columns:
|
|
940
|
+
if hasattr(table_cnf["metadata"], col):
|
|
941
|
+
logger.warning(f"Index column '{col}' not found in table definition for {owner}.{table_name}.")
|
|
942
|
+
chosen = OracleEngine._choose_best_index_for_primary_key(indexes, columns_metadata)
|
|
943
|
+
if not chosen:
|
|
944
|
+
return ()
|
|
945
|
+
return tuple(chosen)
|
|
946
|
+
finally:
|
|
947
|
+
if cursor:
|
|
948
|
+
cursor.close()
|
|
949
|
+
|
|
950
|
+
@staticmethod
|
|
951
|
+
def ensure_roles(conn: oracledb.Connection, roles: Sequence[RoleDefinition]) -> list[str]:
|
|
952
|
+
created: list[str] = []
|
|
953
|
+
if not roles:
|
|
954
|
+
return created
|
|
955
|
+
cursor: oracledb.Cursor | None = None
|
|
956
|
+
changed = False
|
|
957
|
+
try:
|
|
958
|
+
cursor = conn.cursor()
|
|
959
|
+
for role in roles:
|
|
960
|
+
fmttd_role_name = OracleEngine._format_identifier(role.name)
|
|
961
|
+
if OracleEngine._role_exists(cursor, role.name):
|
|
962
|
+
continue
|
|
963
|
+
sql = f"CREATE ROLE {fmttd_role_name}"
|
|
964
|
+
OracleEngine._execute_ddl(conn, cursor, sql)
|
|
965
|
+
created.append(fmttd_role_name)
|
|
966
|
+
changed = True
|
|
967
|
+
if changed:
|
|
968
|
+
conn.commit()
|
|
969
|
+
except Exception as exc:
|
|
970
|
+
conn.rollback()
|
|
971
|
+
logger.critical("Failed to ensure roles.", exc_info=True)
|
|
972
|
+
raise ExecutionError("Failed to ensure Oracle roles") from exc
|
|
973
|
+
finally:
|
|
974
|
+
if cursor:
|
|
975
|
+
cursor.close()
|
|
976
|
+
return created
|
|
977
|
+
|
|
978
|
+
@staticmethod
|
|
979
|
+
def ensure_sequences(conn: oracledb.Connection, sequences: Sequence[SequenceDefinition]) -> list[str]:
|
|
980
|
+
created: list[str] = []
|
|
981
|
+
if not sequences:
|
|
982
|
+
return created
|
|
983
|
+
cursor: oracledb.Cursor | None = None
|
|
984
|
+
changed = False
|
|
985
|
+
try:
|
|
986
|
+
cursor = conn.cursor()
|
|
987
|
+
for sequence in sequences:
|
|
988
|
+
fmttd_owner = OracleEngine._format_identifier(sequence.owner)
|
|
989
|
+
fmttd_sequence_name = OracleEngine._format_identifier(sequence.name)
|
|
990
|
+
if OracleEngine._sequence_exists(cursor, sequence.owner, sequence.name):
|
|
991
|
+
continue
|
|
992
|
+
sql = (
|
|
993
|
+
f"CREATE SEQUENCE {fmttd_owner}.{fmttd_sequence_name} "
|
|
994
|
+
f"START WITH {sequence.start_with} INCREMENT BY {sequence.increment_by}"
|
|
995
|
+
)
|
|
996
|
+
if sequence.minvalue is not None:
|
|
997
|
+
sql += f" MINVALUE {sequence.minvalue}"
|
|
998
|
+
else:
|
|
999
|
+
sql += " NOMINVALUE"
|
|
1000
|
+
if sequence.maxvalue is not None:
|
|
1001
|
+
sql += f" MAXVALUE {sequence.maxvalue}"
|
|
1002
|
+
else:
|
|
1003
|
+
sql += " NOMAXVALUE"
|
|
1004
|
+
sql += " CYCLE" if sequence.cycle else " NOCYCLE"
|
|
1005
|
+
if sequence.cache is not None:
|
|
1006
|
+
sql += f" CACHE {sequence.cache}"
|
|
1007
|
+
else:
|
|
1008
|
+
sql += " NOCACHE"
|
|
1009
|
+
OracleEngine._execute_ddl(conn, cursor, sql)
|
|
1010
|
+
created.append(f"{fmttd_owner}.{fmttd_sequence_name}")
|
|
1011
|
+
changed = True
|
|
1012
|
+
if changed:
|
|
1013
|
+
conn.commit()
|
|
1014
|
+
except Exception as exc:
|
|
1015
|
+
conn.rollback()
|
|
1016
|
+
logger.critical("Failed to ensure sequences.", exc_info=True)
|
|
1017
|
+
raise ExecutionError("Failed to ensure Oracle sequences") from exc
|
|
1018
|
+
finally:
|
|
1019
|
+
if cursor:
|
|
1020
|
+
cursor.close()
|
|
1021
|
+
return created
|
|
1022
|
+
|
|
1023
|
+
@staticmethod
|
|
1024
|
+
def ensure_database_links(conn: oracledb.Connection, links: Sequence[DatabaseLinkDefinition]) -> list[str]:
|
|
1025
|
+
created: list[str] = []
|
|
1026
|
+
if not links:
|
|
1027
|
+
return created
|
|
1028
|
+
cursor: oracledb.Cursor | None = None
|
|
1029
|
+
changed = False
|
|
1030
|
+
try:
|
|
1031
|
+
cursor = conn.cursor()
|
|
1032
|
+
for link in links:
|
|
1033
|
+
fmttd_link_name = OracleEngine._format_identifier(link.name)
|
|
1034
|
+
if OracleEngine._db_link_exists(cursor, link.name):
|
|
1035
|
+
continue
|
|
1036
|
+
fmttd_username = OracleEngine._format_identifier(link.username)
|
|
1037
|
+
fmttd_password = OracleEngine._quote(link.password)
|
|
1038
|
+
literal_dsn_literal = OracleEngine._quote_literal(link.dsn)
|
|
1039
|
+
sql = f"CREATE DATABASE LINK {fmttd_link_name} CONNECT TO {fmttd_username} IDENTIFIED BY ##{fmttd_password}## USING {literal_dsn_literal}"
|
|
1040
|
+
OracleEngine._execute_ddl(conn, cursor, sql)
|
|
1041
|
+
created.append(fmttd_link_name)
|
|
1042
|
+
changed = True
|
|
1043
|
+
if changed:
|
|
1044
|
+
conn.commit()
|
|
1045
|
+
except Exception as exc:
|
|
1046
|
+
conn.rollback()
|
|
1047
|
+
logger.critical("Failed to ensure database links.", exc_info=True)
|
|
1048
|
+
raise ExecutionError("Failed to ensure Oracle database links") from exc
|
|
1049
|
+
finally:
|
|
1050
|
+
if cursor:
|
|
1051
|
+
cursor.close()
|
|
1052
|
+
return created
|
|
1053
|
+
|
|
1054
|
+
@staticmethod
|
|
1055
|
+
def ensure_table_privileges(
|
|
1056
|
+
conn: oracledb.Connection, role: str, tables: Sequence[tuple[str, str]], privileges: Sequence[str]
|
|
1057
|
+
) -> list[str]:
|
|
1058
|
+
cursor: oracledb.Cursor | None = None
|
|
1059
|
+
granted: list[str] = []
|
|
1060
|
+
changed = False
|
|
1061
|
+
if not tables:
|
|
1062
|
+
return granted
|
|
1063
|
+
fmt_role = OracleEngine._format_identifier(role)
|
|
1064
|
+
try:
|
|
1065
|
+
cursor = conn.cursor()
|
|
1066
|
+
for owner, table_name in tables:
|
|
1067
|
+
fmttd_owner = OracleEngine._format_identifier(owner)
|
|
1068
|
+
fmttd_table_name = OracleEngine._format_identifier(table_name)
|
|
1069
|
+
for privilege in privileges:
|
|
1070
|
+
privilege_upper = privilege.upper()
|
|
1071
|
+
if OracleEngine._role_has_table_priv(
|
|
1072
|
+
cursor, role.upper(), owner.upper(), table_name.upper(), privilege_upper
|
|
1073
|
+
):
|
|
1074
|
+
continue
|
|
1075
|
+
sql = f"GRANT {privilege_upper} ON {fmttd_owner}.{fmttd_table_name} TO {fmt_role}"
|
|
1076
|
+
OracleEngine._execute_ddl(conn, cursor, sql)
|
|
1077
|
+
granted.append(f"{fmttd_owner}.{fmttd_table_name}:{privilege_upper}")
|
|
1078
|
+
changed = True
|
|
1079
|
+
if changed:
|
|
1080
|
+
conn.commit()
|
|
1081
|
+
except Exception as exc:
|
|
1082
|
+
conn.rollback()
|
|
1083
|
+
logger.critical("Failed to ensure table privileges.", exc_info=True)
|
|
1084
|
+
raise ExecutionError("Failed to ensure Oracle table privileges") from exc
|
|
1085
|
+
finally:
|
|
1086
|
+
if cursor:
|
|
1087
|
+
cursor.close()
|
|
1088
|
+
return granted
|
|
1089
|
+
|
|
1090
|
+
@staticmethod
|
|
1091
|
+
def ensure_supporting_objects(conn: oracledb.Connection, owner: str) -> list[str]:
|
|
1092
|
+
cursor: oracledb.Cursor | None = None
|
|
1093
|
+
try:
|
|
1094
|
+
cursor = conn.cursor()
|
|
1095
|
+
cursor.execute(f"ALTER SESSION SET CURRENT_SCHEMA = {OracleEngine._format_identifier(owner)}")
|
|
1096
|
+
for statement in (T_REFERENCING_TABLES_TYPE, CHECK_SAVE_STATUS_PROC, CHECK_REFERENCING_TABLES_PROC):
|
|
1097
|
+
OracleEngine._execute_ddl(conn, cursor, statement)
|
|
1098
|
+
return SUPPORTING_OBJECT_LIST
|
|
1099
|
+
except Exception as exc:
|
|
1100
|
+
logger.critical("Failed to ensure supporting PL/SQL objects.", exc_info=True)
|
|
1101
|
+
raise ExecutionError("Failed to ensure Oracle supporting PL/SQL objects") from exc
|
|
1102
|
+
finally:
|
|
1103
|
+
if cursor:
|
|
1104
|
+
cursor.close()
|
|
1105
|
+
|
|
1106
|
+
@staticmethod
|
|
1107
|
+
def get_identifier_str(identifier: str) -> str:
|
|
1108
|
+
"""Returns the identifier string required to query dictionary views.
|
|
1109
|
+
Args:
|
|
1110
|
+
indentifier: The identifier string to process.
|
|
1111
|
+
Returns:
|
|
1112
|
+
The identifier string without angle brackets.
|
|
1113
|
+
"""
|
|
1114
|
+
identifier = identifier.strip()
|
|
1115
|
+
if identifier.startswith('"') and identifier.endswith('"'):
|
|
1116
|
+
return identifier[1:-1]
|
|
1117
|
+
return identifier.upper()
|
|
1118
|
+
|
|
1119
|
+
@staticmethod
|
|
1120
|
+
def get_ldb_columns_expressions() -> tuple[str, str]:
|
|
1121
|
+
"""Returns the expressions for the LDB process date and insert date columns.
|
|
1122
|
+
Returns:
|
|
1123
|
+
A tuple containing the process date expression and insert date expression.
|
|
1124
|
+
"""
|
|
1125
|
+
return "l_process_date", "sysdate"
|
|
1126
|
+
|
|
1127
|
+
@staticmethod
|
|
1128
|
+
def _column_sql(column: ColumnDefinition) -> str:
|
|
1129
|
+
col_name = OracleEngine._format_identifier(column.name)
|
|
1130
|
+
col_type = OracleEngine.get_column_type(column)
|
|
1131
|
+
default_clause = f" DEFAULT {column.default}" if column.default is not None else ""
|
|
1132
|
+
nullable_clause = "" if column.nullable else " NOT NULL"
|
|
1133
|
+
return f"{col_name} {col_type}{default_clause}{nullable_clause}"
|
|
1134
|
+
|
|
1135
|
+
# format / expressions helpers
|
|
1136
|
+
@staticmethod
|
|
1137
|
+
def _format_identifier(name: str | None) -> str:
|
|
1138
|
+
"""Formats an Oracle identifier, quoting if necessary.
|
|
1139
|
+
Args:
|
|
1140
|
+
name: The identifier name as appears data dictionary.
|
|
1141
|
+
Returns:
|
|
1142
|
+
Formatted identifier, quoted if necessary."""
|
|
1143
|
+
if not name:
|
|
1144
|
+
return '""'
|
|
1145
|
+
if OracleEngine._REGULAR_IDENTIFIER.fullmatch(name):
|
|
1146
|
+
return name.lower()
|
|
1147
|
+
return OracleEngine._quote(name)
|
|
1148
|
+
|
|
1149
|
+
@staticmethod
|
|
1150
|
+
def _quote(identifier: str) -> str:
|
|
1151
|
+
escaped = identifier.replace('"', '""')
|
|
1152
|
+
return f'"{escaped}"'
|
|
1153
|
+
|
|
1154
|
+
@staticmethod
|
|
1155
|
+
def _quote_literal(value: str) -> str:
|
|
1156
|
+
escaped = value.replace("'", "''")
|
|
1157
|
+
return f"'{escaped}'"
|
|
1158
|
+
|
|
1159
|
+
# metadata and privilege helpers
|
|
1160
|
+
@staticmethod
|
|
1161
|
+
def _object_exists(cursor: oracledb.Cursor, query: str, params: Sequence[Any]) -> bool:
|
|
1162
|
+
cursor.execute(query, tuple(params))
|
|
1163
|
+
return cursor.fetchone() is not None
|
|
1164
|
+
|
|
1165
|
+
@staticmethod
|
|
1166
|
+
def _user_has_role(cursor: oracledb.Cursor, username: str, role: str) -> bool:
|
|
1167
|
+
return OracleEngine._object_exists(
|
|
1168
|
+
cursor, "SELECT 1 FROM dba_role_privs WHERE grantee = :1 AND granted_role = :2", [username, role]
|
|
1169
|
+
)
|
|
1170
|
+
|
|
1171
|
+
@staticmethod
|
|
1172
|
+
def _user_has_role_with_admin_option(cursor: oracledb.Cursor, username: str, role: str) -> bool:
|
|
1173
|
+
return OracleEngine._object_exists(
|
|
1174
|
+
cursor,
|
|
1175
|
+
"SELECT 1 FROM dba_role_privs WHERE grantee = :1 AND granted_role = :2 AND admin_option = 'YES'",
|
|
1176
|
+
[username, role],
|
|
1177
|
+
)
|
|
1178
|
+
|
|
1179
|
+
@staticmethod
|
|
1180
|
+
def _user_has_sys_priv(cursor: oracledb.Cursor, username: str, privilege: str) -> bool:
|
|
1181
|
+
return OracleEngine._object_exists(
|
|
1182
|
+
cursor, "SELECT 1 FROM dba_sys_privs WHERE grantee = :1 AND privilege = :2", [username, privilege]
|
|
1183
|
+
)
|
|
1184
|
+
|
|
1185
|
+
@staticmethod
|
|
1186
|
+
def _role_has_table_priv(cursor: oracledb.Cursor, role: str, owner: str, table_name: str, privilege: str) -> bool:
|
|
1187
|
+
return OracleEngine._object_exists(
|
|
1188
|
+
cursor,
|
|
1189
|
+
"""
|
|
1190
|
+
SELECT 1
|
|
1191
|
+
FROM dba_tab_privs
|
|
1192
|
+
WHERE grantee = :grantee
|
|
1193
|
+
AND owner = :owner
|
|
1194
|
+
AND table_name = :table_name
|
|
1195
|
+
AND privilege = :privilege
|
|
1196
|
+
""",
|
|
1197
|
+
[role, owner, table_name, privilege],
|
|
1198
|
+
)
|
|
1199
|
+
|
|
1200
|
+
@staticmethod
|
|
1201
|
+
def _role_exists(cursor: oracledb.Cursor, role: str) -> bool:
|
|
1202
|
+
return OracleEngine._object_exists(cursor, "SELECT 1 FROM dba_roles WHERE role = :1", [role])
|
|
1203
|
+
|
|
1204
|
+
@staticmethod
|
|
1205
|
+
def _user_exists(cursor: oracledb.Cursor, username: str) -> bool:
|
|
1206
|
+
return OracleEngine._object_exists(cursor, "SELECT 1 FROM dba_users WHERE username = :1", [username])
|
|
1207
|
+
|
|
1208
|
+
@staticmethod
|
|
1209
|
+
def _get_database_default_tablespace(cursor: oracledb.Cursor) -> str | None:
|
|
1210
|
+
cursor.execute(
|
|
1211
|
+
"SELECT property_value FROM database_properties WHERE property_name = 'DEFAULT_PERMANENT_TABLESPACE'"
|
|
1212
|
+
)
|
|
1213
|
+
row = cursor.fetchone()
|
|
1214
|
+
if row and row[0]:
|
|
1215
|
+
return str(row[0])
|
|
1216
|
+
return None
|
|
1217
|
+
|
|
1218
|
+
@staticmethod
|
|
1219
|
+
def _get_user_default_tablespace(cursor: oracledb.Cursor, username: str) -> str | None:
|
|
1220
|
+
cursor.execute("SELECT default_tablespace FROM dba_users WHERE username = :username", username=username)
|
|
1221
|
+
row = cursor.fetchone()
|
|
1222
|
+
if row and row[0]:
|
|
1223
|
+
return str(row[0])
|
|
1224
|
+
return None
|
|
1225
|
+
|
|
1226
|
+
@staticmethod
|
|
1227
|
+
def _ensure_unlimited_quota(cursor: oracledb.Cursor, username: str, tablespace: str | None) -> bool:
|
|
1228
|
+
if not tablespace:
|
|
1229
|
+
return False
|
|
1230
|
+
tablespace_name = OracleEngine._format_identifier(tablespace)
|
|
1231
|
+
cursor.execute(
|
|
1232
|
+
"""
|
|
1233
|
+
SELECT max_bytes
|
|
1234
|
+
FROM dba_ts_quotas
|
|
1235
|
+
WHERE username = :username
|
|
1236
|
+
AND tablespace_name = :tablespace
|
|
1237
|
+
""",
|
|
1238
|
+
username=username,
|
|
1239
|
+
tablespace=tablespace_name,
|
|
1240
|
+
)
|
|
1241
|
+
row = cursor.fetchone()
|
|
1242
|
+
if row and row[0] == -1:
|
|
1243
|
+
return False
|
|
1244
|
+
sql = f"ALTER USER {username} QUOTA UNLIMITED ON {tablespace_name}"
|
|
1245
|
+
OracleEngine._execute_ddl(cursor.connection, cursor, sql)
|
|
1246
|
+
return True
|
|
1247
|
+
|
|
1248
|
+
@staticmethod
|
|
1249
|
+
def _table_exists(cursor: oracledb.Cursor, owner: str, table_name: str) -> bool:
|
|
1250
|
+
return OracleEngine._object_exists(
|
|
1251
|
+
cursor,
|
|
1252
|
+
"SELECT 1 FROM dba_tables WHERE owner = :1 AND table_name = :2",
|
|
1253
|
+
[owner, table_name],
|
|
1254
|
+
)
|
|
1255
|
+
|
|
1256
|
+
@staticmethod
|
|
1257
|
+
def _index_exists(cursor: oracledb.Cursor, owner: str, index_name: str) -> bool:
|
|
1258
|
+
return OracleEngine._object_exists(
|
|
1259
|
+
cursor,
|
|
1260
|
+
"SELECT 1 FROM dba_indexes WHERE owner = :1 AND index_name = :2",
|
|
1261
|
+
[owner, index_name],
|
|
1262
|
+
)
|
|
1263
|
+
|
|
1264
|
+
@staticmethod
|
|
1265
|
+
def _constraint_exists(cursor: oracledb.Cursor, owner: str, constraint_name: str) -> bool:
|
|
1266
|
+
return OracleEngine._object_exists(
|
|
1267
|
+
cursor,
|
|
1268
|
+
"SELECT 1 FROM dba_constraints WHERE owner = :1 AND constraint_name = :2",
|
|
1269
|
+
[owner, constraint_name],
|
|
1270
|
+
)
|
|
1271
|
+
|
|
1272
|
+
@staticmethod
|
|
1273
|
+
def _sequence_exists(cursor: oracledb.Cursor, owner: str, sequence_name: str) -> bool:
|
|
1274
|
+
return OracleEngine._object_exists(
|
|
1275
|
+
cursor,
|
|
1276
|
+
"SELECT 1 FROM dba_sequences WHERE sequence_owner = :1 AND sequence_name = :2",
|
|
1277
|
+
[owner, sequence_name],
|
|
1278
|
+
)
|
|
1279
|
+
|
|
1280
|
+
@staticmethod
|
|
1281
|
+
def _register_connection_env(conn: oracledb.Connection, env: str | None) -> None:
|
|
1282
|
+
if env:
|
|
1283
|
+
OracleEngine._connection_envs[id(conn)] = env
|
|
1284
|
+
else:
|
|
1285
|
+
OracleEngine._connection_envs.pop(id(conn), None)
|
|
1286
|
+
|
|
1287
|
+
@staticmethod
|
|
1288
|
+
def _get_connection_env(conn: oracledb.Connection) -> str | None:
|
|
1289
|
+
return OracleEngine._connection_envs.get(id(conn))
|
|
1290
|
+
|
|
1291
|
+
@staticmethod
|
|
1292
|
+
def _execute_ddl(conn: oracledb.Connection, cursor: oracledb.Cursor, statement: str) -> None:
|
|
1293
|
+
log_statement = OracleEngine._PRIVATE_DDL_MARK.sub("****", statement)
|
|
1294
|
+
if len(log_statement) > 100:
|
|
1295
|
+
log_statement = log_statement[:100] + " ... [truncated]"
|
|
1296
|
+
statement = OracleEngine._PRIVATE_DDL_MARK.sub(lambda m: m.group(1), statement)
|
|
1297
|
+
logger.info("Executing %s DDL: %s", OracleEngine._get_connection_env(conn), " ".join(log_statement.split()))
|
|
1298
|
+
cursor.execute(statement)
|
|
1299
|
+
|
|
1300
|
+
@staticmethod
|
|
1301
|
+
def _db_link_exists(cursor: oracledb.Cursor, name: str) -> bool:
|
|
1302
|
+
return OracleEngine._object_exists(cursor, "SELECT 1 FROM user_db_links WHERE db_link = :1", [name])
|
|
1303
|
+
|
|
1304
|
+
@staticmethod
|
|
1305
|
+
def _get_primary_key_info(
|
|
1306
|
+
cursor: oracledb.Cursor, owner: str, table_name: str
|
|
1307
|
+
) -> tuple[str | None, tuple[str, ...], str | None]:
|
|
1308
|
+
cursor.execute(
|
|
1309
|
+
"""
|
|
1310
|
+
SELECT constraint_name, index_name
|
|
1311
|
+
FROM dba_constraints
|
|
1312
|
+
WHERE owner = :1
|
|
1313
|
+
AND table_name = :2
|
|
1314
|
+
AND constraint_type = 'P'
|
|
1315
|
+
""",
|
|
1316
|
+
[owner, table_name],
|
|
1317
|
+
)
|
|
1318
|
+
cursor.execute(
|
|
1319
|
+
"""
|
|
1320
|
+
SELECT constraint_name, index_name
|
|
1321
|
+
FROM dba_constraints
|
|
1322
|
+
WHERE owner = :1
|
|
1323
|
+
AND table_name = :2
|
|
1324
|
+
AND constraint_type = 'P'
|
|
1325
|
+
""",
|
|
1326
|
+
[owner, table_name],
|
|
1327
|
+
)
|
|
1328
|
+
row = cursor.fetchone()
|
|
1329
|
+
if not row:
|
|
1330
|
+
return None, (), None
|
|
1331
|
+
constraint_name, index_name = row
|
|
1332
|
+
cursor.execute(
|
|
1333
|
+
"SELECT column_name FROM dba_cons_columns WHERE owner = :1 AND constraint_name = :2 ORDER BY position",
|
|
1334
|
+
[owner, constraint_name],
|
|
1335
|
+
)
|
|
1336
|
+
columns = tuple(r[0] for r in cast(Iterable[tuple[str]], cursor))
|
|
1337
|
+
return constraint_name, columns, index_name
|
|
1338
|
+
|
|
1339
|
+
@staticmethod
|
|
1340
|
+
def _column_needs_update(existing: ColumnDefinition, desired: ColumnDefinition) -> bool:
|
|
1341
|
+
desired_type = desired.data_type.lower()
|
|
1342
|
+
existing_type = existing.data_type.lower()
|
|
1343
|
+
if existing_type != desired_type:
|
|
1344
|
+
return True
|
|
1345
|
+
if desired_type in ("varchar2", "varchar", "char"):
|
|
1346
|
+
desired_length = desired.length or 0
|
|
1347
|
+
existing_length = existing.length or 0
|
|
1348
|
+
if desired_length and desired_length > existing_length:
|
|
1349
|
+
return True
|
|
1350
|
+
if desired_type in ("number", "numeric", "decimal"):
|
|
1351
|
+
desired_precision = desired.precision
|
|
1352
|
+
desired_scale = desired.scale or 0
|
|
1353
|
+
existing_precision = existing.precision
|
|
1354
|
+
existing_scale = existing.scale or 0
|
|
1355
|
+
if desired_precision is not None and existing_precision is not None:
|
|
1356
|
+
if existing_precision < desired_precision:
|
|
1357
|
+
return True
|
|
1358
|
+
if desired_precision is not None and existing_precision is None:
|
|
1359
|
+
# Existing column allows maximum precision; no change required.
|
|
1360
|
+
pass
|
|
1361
|
+
if desired_scale > existing_scale:
|
|
1362
|
+
return True
|
|
1363
|
+
if desired_type in ("integer", "int"):
|
|
1364
|
+
desired_precision = desired.precision or 10
|
|
1365
|
+
existing_precision = existing.precision or 0
|
|
1366
|
+
if desired_precision > existing_precision:
|
|
1367
|
+
return True
|
|
1368
|
+
return False
|
|
1369
|
+
|
|
1370
|
+
@staticmethod
|
|
1371
|
+
def _get_table_indexes(
|
|
1372
|
+
cursor: oracledb.Cursor, owner: str, table_name: str
|
|
1373
|
+
) -> dict[str, tuple[tuple[str, ...], bool]]:
|
|
1374
|
+
cursor.execute(
|
|
1375
|
+
"""
|
|
1376
|
+
SELECT i.index_name, i.uniqueness, c.column_name
|
|
1377
|
+
FROM dba_indexes i JOIN dba_ind_columns c ON i.owner = c.index_owner AND i.index_name = c.index_name WHERE i.owner = :owner
|
|
1378
|
+
AND i.table_name = :table_name
|
|
1379
|
+
ORDER BY i.index_name, c.column_position
|
|
1380
|
+
""",
|
|
1381
|
+
owner=owner,
|
|
1382
|
+
table_name=table_name,
|
|
1383
|
+
)
|
|
1384
|
+
indexes: dict[str, dict[str, Any]] = {}
|
|
1385
|
+
for index_name, uniqueness, column_name in cast(Iterable[tuple[str, str, str]], cursor):
|
|
1386
|
+
index_info = indexes.setdefault(index_name, {"columns": [], "unique": uniqueness == "UNIQUE"})
|
|
1387
|
+
index_info["columns"].append(column_name)
|
|
1388
|
+
return {name: (tuple(info["columns"]), bool(info["unique"])) for name, info in indexes.items()}
|
|
1389
|
+
|
|
1390
|
+
@staticmethod
|
|
1391
|
+
def _determine_process_date_column(table: TableDefinition) -> str | None:
|
|
1392
|
+
available_columns = {column.name for column in table.columns}
|
|
1393
|
+
for candidate in ("LDB_PROCESS_DATE", "LDB_PROCESS_DATE"):
|
|
1394
|
+
if candidate in available_columns:
|
|
1395
|
+
return candidate
|
|
1396
|
+
return None
|
|
1397
|
+
|
|
1398
|
+
@staticmethod
|
|
1399
|
+
def _prepare_desired_indexes(
|
|
1400
|
+
table: TableDefinition,
|
|
1401
|
+
) -> dict[str, tuple[tuple[str, ...], bool]]:
|
|
1402
|
+
desired_pk = tuple(table.primary_key or ())
|
|
1403
|
+
process_date_column = OracleEngine._determine_process_date_column(table)
|
|
1404
|
+
desired_indexes: dict[str, tuple[tuple[str, ...], bool]] = {}
|
|
1405
|
+
for index in table.indexes:
|
|
1406
|
+
columns = tuple(index.columns)
|
|
1407
|
+
if process_date_column and process_date_column not in columns and (not desired_pk or columns != desired_pk):
|
|
1408
|
+
columns = (*columns, process_date_column)
|
|
1409
|
+
desired_indexes[index.name] = (columns, index.unique)
|
|
1410
|
+
return desired_indexes
|
|
1411
|
+
|
|
1412
|
+
@staticmethod
|
|
1413
|
+
def _choose_best_index_for_primary_key(
|
|
1414
|
+
indexes: IndexMap, column_metadata: ColumnMetadata
|
|
1415
|
+
) -> tuple[str, ...] | None:
|
|
1416
|
+
candidates: list[dict[str, Any]] = []
|
|
1417
|
+
for index_name, info in indexes.items():
|
|
1418
|
+
columns: Sequence[str] = info.get("columns", [])
|
|
1419
|
+
if not columns:
|
|
1420
|
+
continue
|
|
1421
|
+
column_defs: list[ColumnDefinition] = []
|
|
1422
|
+
for col in columns:
|
|
1423
|
+
column_def = column_metadata.get(col)
|
|
1424
|
+
if column_def is None:
|
|
1425
|
+
column_defs = []
|
|
1426
|
+
break
|
|
1427
|
+
column_defs.append(column_def)
|
|
1428
|
+
if not column_defs:
|
|
1429
|
+
continue
|
|
1430
|
+
candidates.append(
|
|
1431
|
+
{
|
|
1432
|
+
"name": index_name,
|
|
1433
|
+
"columns": tuple(col.name for col in column_defs),
|
|
1434
|
+
"unique": bool(info.get("unique")),
|
|
1435
|
+
"distinct_keys": info.get("distinct_keys"),
|
|
1436
|
+
"all_not_null": all(not col.nullable for col in column_defs),
|
|
1437
|
+
}
|
|
1438
|
+
)
|
|
1439
|
+
if not candidates:
|
|
1440
|
+
return None
|
|
1441
|
+
|
|
1442
|
+
def score(value: Any) -> int:
|
|
1443
|
+
if value is None:
|
|
1444
|
+
return -1
|
|
1445
|
+
if isinstance(value, int):
|
|
1446
|
+
return value
|
|
1447
|
+
if isinstance(value, float):
|
|
1448
|
+
return int(value)
|
|
1449
|
+
try:
|
|
1450
|
+
return int(value)
|
|
1451
|
+
except (TypeError, ValueError):
|
|
1452
|
+
return -1
|
|
1453
|
+
|
|
1454
|
+
def choose(candidate_list: list[dict[str, Any]]) -> tuple[str, ...] | None:
|
|
1455
|
+
if not candidate_list:
|
|
1456
|
+
return None
|
|
1457
|
+
ordered = sorted(
|
|
1458
|
+
candidate_list,
|
|
1459
|
+
key=lambda item: (
|
|
1460
|
+
-score(item.get("distinct_keys")),
|
|
1461
|
+
len(item["columns"]),
|
|
1462
|
+
item["name"],
|
|
1463
|
+
),
|
|
1464
|
+
)
|
|
1465
|
+
best = ordered[0]
|
|
1466
|
+
return tuple(best["columns"])
|
|
1467
|
+
|
|
1468
|
+
not_null_unique = [c for c in candidates if c["unique"] and c["all_not_null"]]
|
|
1469
|
+
choice = choose(not_null_unique)
|
|
1470
|
+
if choice:
|
|
1471
|
+
return choice
|
|
1472
|
+
unique_candidates = [c for c in candidates if c["unique"]]
|
|
1473
|
+
choice = choose(unique_candidates)
|
|
1474
|
+
if choice:
|
|
1475
|
+
return choice
|
|
1476
|
+
non_unique = [c for c in candidates if not c["unique"]]
|
|
1477
|
+
return choose(non_unique)
|
|
1478
|
+
|
|
1479
|
+
@staticmethod
|
|
1480
|
+
def _drop_constraints_by_type(
|
|
1481
|
+
cursor: oracledb.Cursor, owner: str, table_name: str, constraint_types: Sequence[str]
|
|
1482
|
+
) -> list[str]:
|
|
1483
|
+
dropped: list[str] = []
|
|
1484
|
+
if not constraint_types:
|
|
1485
|
+
return dropped
|
|
1486
|
+
type_list = ", ".join(f"'{constraint_type}'" for constraint_type in constraint_types)
|
|
1487
|
+
cursor.execute(
|
|
1488
|
+
f"""
|
|
1489
|
+
SELECT constraint_name
|
|
1490
|
+
FROM dba_constraints
|
|
1491
|
+
WHERE owner = :owner
|
|
1492
|
+
AND table_name = :table_name
|
|
1493
|
+
AND constraint_type IN ({type_list})
|
|
1494
|
+
""",
|
|
1495
|
+
owner=owner,
|
|
1496
|
+
table_name=table_name,
|
|
1497
|
+
)
|
|
1498
|
+
fmttd_owner = OracleEngine._format_identifier(owner)
|
|
1499
|
+
fmttd_table_name = OracleEngine._format_identifier(table_name)
|
|
1500
|
+
for (constraint_name,) in cast(Iterable[tuple[str]], cursor):
|
|
1501
|
+
fmttd_constraint_name = OracleEngine._format_identifier(constraint_name)
|
|
1502
|
+
sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} DROP CONSTRAINT {fmttd_constraint_name}"
|
|
1503
|
+
OracleEngine._execute_ddl(cursor.connection, cursor, sql)
|
|
1504
|
+
dropped.append(fmttd_constraint_name)
|
|
1505
|
+
return dropped
|