limitsdb 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1505 @@
1
+ import re
2
+ from collections.abc import Iterable, Mapping, Sequence
3
+ from datetime import datetime
4
+ from typing import Any, ClassVar, Literal, cast
5
+
6
+ import oracledb
7
+
8
+ from limitsdb.core.ldb_errors import DatabaseConnectionError, ExecutionError
9
+ from limitsdb.core.ldb_logger import get_logger
10
+ from limitsdb.core.ldb_params_config import Config
11
+ from limitsdb.core.ldb_status import Status
12
+ from limitsdb.core.ldb_utils import get_effective_credentials, indent_lines, join_wrapped, nvl
13
+ from limitsdb.db.ldb_engines import (
14
+ ColumnDefinition,
15
+ ColumnMetadata,
16
+ DatabaseEngine,
17
+ DatabaseLinkDefinition,
18
+ IndexMap,
19
+ RoleDefinition,
20
+ SequenceDefinition,
21
+ TableDefinition,
22
+ UserDefinition,
23
+ )
24
+
25
+ SUPPORTING_OBJECT_LIST = [
26
+ "TYPE t_referencing_tables",
27
+ "PROCEDURE check_save_status",
28
+ "PROCEDURE check_referencing_tables",
29
+ ]
30
+ CHECK_SAVE_STATUS_PROC = """
31
+ CREATE OR REPLACE PROCEDURE check_save_status(
32
+ p_owner IN VARCHAR2,
33
+ p_table_name IN VARCHAR2,
34
+ p_process_date IN DATE,
35
+ p_action IN VARCHAR2,
36
+ p_status IN VARCHAR2,
37
+ p_process_start IN DATE,
38
+ p_chunk_start IN DATE,
39
+ p_process_end IN DATE,
40
+ p_message IN VARCHAR2,
41
+ p_rows_processed IN VARCHAR2,
42
+ p_plsql IN VARCHAR2,
43
+ p_sqlcode IN OUT NUMBER,
44
+ p_out_message OUT VARCHAR2
45
+ )
46
+ IS
47
+ l_status VARCHAR2(10);
48
+ l_process_start DATE;
49
+ l_process_date DATE;
50
+ l_rowid ROWID;
51
+ in_use EXCEPTION;
52
+ PRAGMA EXCEPTION_INIT(in_use, -54);
53
+ BEGIN
54
+ IF NVL(p_sqlcode, 0) NOT IN (-20001, -20002, -20003) THEN
55
+ BEGIN
56
+ SELECT ctl_status, ctl_process_date, ctl_process_start, ROWID
57
+ INTO l_status, l_process_date, l_process_start, l_rowid
58
+ FROM ldb_ctl
59
+ WHERE ctl_owner = p_owner
60
+ AND ctl_table_name = p_table_name
61
+ FOR UPDATE NOWAIT;
62
+
63
+ IF p_status = 'TSTART'
64
+ AND l_status != 'TEND'
65
+ AND (SYSDATE - l_process_start) * 3600 * 24 < 5 THEN
66
+ p_sqlcode := -20002;
67
+ p_out_message := 'Table ' || p_table_name || ' is currently in process without lock';
68
+ RETURN;
69
+ ELSIF p_status = 'TSTART'
70
+ AND l_status = 'TEND'
71
+ AND l_process_date = p_process_date THEN
72
+ ROLLBACK;
73
+ p_sqlcode := -20003;
74
+ p_out_message := 'Table ' || p_table_name ||
75
+ ' is already processed for date ' ||
76
+ TO_CHAR(p_process_date, 'YYYYMMDD');
77
+ RETURN;
78
+ END IF;
79
+
80
+ UPDATE ldb_ctl
81
+ SET ctl_process_date = p_process_date,
82
+ ctl_action = p_action,
83
+ ctl_status = p_status,
84
+ ctl_process_start = NVL(p_process_start, ctl_process_start),
85
+ ctl_process_end = p_process_end,
86
+ ctl_rows_processed = DECODE(p_status, 'TSTART', 0, ctl_rows_processed) +
87
+ p_rows_processed,
88
+ ctl_plsql = NVL(p_plsql, ctl_plsql)
89
+ WHERE ROWID = l_rowid;
90
+ EXCEPTION
91
+ WHEN in_use THEN
92
+ p_sqlcode := -20001;
93
+ p_out_message := 'Table ' || p_table_name || ' is currently in process';
94
+ RETURN;
95
+ WHEN NO_DATA_FOUND THEN
96
+ INSERT INTO ldb_ctl (
97
+ ctl_owner,
98
+ ctl_table_name,
99
+ ctl_process_date,
100
+ ctl_action,
101
+ ctl_status,
102
+ ctl_process_start,
103
+ ctl_process_end,
104
+ ctl_rows_processed,
105
+ ctl_plsql
106
+ )
107
+ VALUES (
108
+ p_owner,
109
+ p_table_name,
110
+ p_process_date,
111
+ p_action,
112
+ p_status,
113
+ l_process_start,
114
+ NULL,
115
+ 0,
116
+ p_plsql
117
+ );
118
+ END;
119
+ END IF;
120
+
121
+ INSERT INTO ldb_log (
122
+ log_id,
123
+ log_owner,
124
+ log_table_name,
125
+ log_process_date,
126
+ log_action,
127
+ log_status,
128
+ log_process_start,
129
+ log_process_end,
130
+ log_message,
131
+ log_rows_processed,
132
+ log_plsql
133
+ )
134
+ VALUES (
135
+ ldb_log_id.NEXTVAL,
136
+ p_owner,
137
+ p_table_name,
138
+ p_process_date,
139
+ p_action,
140
+ p_status,
141
+ NVL(p_chunk_start, p_process_start),
142
+ p_process_end,
143
+ p_message,
144
+ p_rows_processed,
145
+ p_plsql
146
+ );
147
+ END check_save_status;
148
+ """
149
+ T_REFERENCING_TABLES_TYPE = """
150
+ CREATE OR REPLACE TYPE t_referencing_tables AS TABLE OF VARCHAR2(100);
151
+ """
152
+ CHECK_REFERENCING_TABLES_PROC = """
153
+ CREATE OR REPLACE PROCEDURE check_referencing_tables(
154
+ p_referencing_tables IN t_referencing_tables,
155
+ p_process_date IN DATE
156
+ )
157
+ IS
158
+ l_ref_table VARCHAR2(100);
159
+ l_ref_owner VARCHAR2(100);
160
+ l_ref_table_name VARCHAR2(100);
161
+ l_dummy NUMBER;
162
+ BEGIN
163
+ FOR i IN 1 .. p_referencing_tables.COUNT LOOP
164
+ l_ref_table := p_referencing_tables(i);
165
+ l_ref_owner := SUBSTR(l_ref_table, 1, INSTR(l_ref_table, '.') - 1);
166
+ l_ref_table_name := SUBSTR(l_ref_table, INSTR(l_ref_table, '.') + 1);
167
+ BEGIN
168
+ SELECT 1
169
+ INTO l_dummy
170
+ FROM ldb_ctl
171
+ WHERE ctl_owner = l_ref_owner
172
+ AND ctl_table_name = l_ref_table_name
173
+ AND ctl_process_date = p_process_date
174
+ AND ctl_status = 'TEND';
175
+ EXCEPTION
176
+ WHEN NO_DATA_FOUND THEN
177
+ RAISE_APPLICATION_ERROR(
178
+ -20004,
179
+ 'Referencing table ' || l_ref_table ||
180
+ ' was not fully processed for date ' ||
181
+ TO_CHAR(p_process_date, 'YYYYMMDD')
182
+ );
183
+ END;
184
+ END LOOP;
185
+ END check_referencing_tables;
186
+ """
187
+ logger = get_logger("oracle.engine")
188
+
189
+
190
+ class OracleEngine(DatabaseEngine):
191
+ """Oracle DB engine with methods for connection, configuration loading, and PL/SQL generation."""
192
+
193
+ # states and class constants
194
+ _connection_envs: ClassVar[dict[int, str]] = {}
195
+ REQUIRED_SYSTEM_PRIVILEGES: tuple[str, ...] = (
196
+ "CREATE SESSION",
197
+ "ALTER SESSION",
198
+ "CREATE TABLE",
199
+ "CREATE PROCEDURE",
200
+ "CREATE TYPE",
201
+ "CREATE DATABASE LINK",
202
+ "CREATE SEQUENCE",
203
+ "RESUMABLE",
204
+ "ALTER USER",
205
+ "CREATE SYNONYM",
206
+ "CREATE VIEW",
207
+ "CREATE ROLE",
208
+ "CREATE TRIGGER",
209
+ "CREATE MATERIALIZED VIEW",
210
+ "QUERY REWRITE",
211
+ )
212
+ _REGULAR_IDENTIFIER = re.compile(r"[A-Z_#$][A-Z0-9_#$]*") # only DD uppercase identifiers are unquoted
213
+ _PREFIXED_IDENTIFIER = re.compile(r'@("(?:""|[^"])*"|[A-Za-z_#$][A-Za-z0-9_#$]*)')
214
+ _PRIVATE_DDL_MARK = re.compile(r"##(.*?)##")
215
+
216
+ # public API
217
+ @staticmethod
218
+ def get_connection(
219
+ config: Config, *, admin: bool = False, env: Literal["SOURCE", "HISTORY"] | None = None
220
+ ) -> oracledb.Connection:
221
+ """Returns an Oracle connection using provided config.
222
+ Args:
223
+ config: Database config object.
224
+ admin: Whether to use admin credentials.
225
+ Returns:
226
+ An active oracledb.Connection."""
227
+ user, password, dsn = get_effective_credentials(config, admin=admin, env=env)
228
+ try:
229
+ connection = oracledb.connect(user=user, password=password, dsn=dsn)
230
+ OracleEngine._register_connection_env(connection, env)
231
+ return cast(oracledb.Connection, connection)
232
+ except Exception as exc:
233
+ logger.critical("Failed to connect to Oracle DB.", exc_info=True)
234
+ raise DatabaseConnectionError("Failed to connect to Oracle database") from exc
235
+
236
+ @staticmethod
237
+ def close_connection(conn: oracledb.Connection) -> None:
238
+ """Closes the Oracle connection.
239
+ Args:
240
+ conn: Active Oracle connection."""
241
+ try:
242
+ if conn:
243
+ OracleEngine._register_connection_env(conn, None)
244
+ conn.close()
245
+ except Exception:
246
+ logger.critical("Failed to close Oracle DB connection.", exc_info=True)
247
+
248
+ @staticmethod
249
+ def get_table_columns(
250
+ conn: oracledb.Connection, owner: str, table_name: str
251
+ ) -> tuple[list[str], dict[str, ColumnDefinition]]:
252
+ """Retrieves column names and metadata for a given table.
253
+ Args:
254
+ conn: Active Oracle connection.
255
+ owner: Schema owner of the table.
256
+ table_name: Table name.
257
+ Returns:
258
+ Tuple containing a list of column names and a dict of column metadata."""
259
+ cursor: oracledb.Cursor | None = None
260
+ columns: list[str] = []
261
+ metadata: dict[str, ColumnDefinition] = {}
262
+ try:
263
+ cursor = conn.cursor()
264
+ cursor.execute(
265
+ """select column_id, column_name, data_type, data_length, data_precision, data_scale, nullable, data_default, char_length, char_used
266
+ from all_tab_columns
267
+ where owner = :1 and table_name = :2 and column_id is not null
268
+ order by column_id""",
269
+ [owner, table_name],
270
+ )
271
+ for row in cast(
272
+ Iterable[tuple[int, str, str, int | None, int | None, int | None, str, str | None, str | None, str]],
273
+ cursor,
274
+ ):
275
+ (
276
+ column_id,
277
+ column_name,
278
+ data_type,
279
+ data_length,
280
+ data_precision,
281
+ data_scale,
282
+ nullable,
283
+ data_default,
284
+ char_length,
285
+ char_used,
286
+ ) = row
287
+ dtype = data_type.lower()
288
+ length: int | None = None
289
+ if dtype in ("varchar2", "varchar", "char"):
290
+ if char_used == "C" and char_length is not None:
291
+ length = int(char_length)
292
+ elif data_length is not None:
293
+ length = int(data_length)
294
+ precision = int(data_precision) if data_precision is not None else None
295
+ scale = int(data_scale) if data_scale is not None else None
296
+ metadata[column_name] = ColumnDefinition(
297
+ name=column_name,
298
+ data_type=dtype,
299
+ id=column_id,
300
+ length=length,
301
+ precision=precision,
302
+ scale=scale,
303
+ nullable=(nullable == "Y"),
304
+ default=data_default.strip() if isinstance(data_default, str) else None,
305
+ )
306
+ columns.append(column_name)
307
+ finally:
308
+ if cursor:
309
+ cursor.close()
310
+ return columns, metadata
311
+
312
+ @staticmethod
313
+ def get_date_condition(date_expr: str, months_keep_src: int) -> str:
314
+ """Returns a date condition for the given date expression and months to keep.
315
+ Args:
316
+ date_expr: Date expression to evaluate.
317
+ months_keep_src: Months to keep.
318
+ Returns:
319
+ Date condition as string."""
320
+ return f"({date_expr} < add_months(l_process_date,-{months_keep_src}))"
321
+
322
+ @staticmethod
323
+ def get_fallback_expression(expression: str, fallback: str) -> str:
324
+ """Return Oracle's null-safe fallback expression."""
325
+ return f"coalesce({expression}, {fallback})"
326
+
327
+ @staticmethod
328
+ def get_identifiers_from_expression(expression: str) -> set[str]:
329
+ """Returns a set of @prefixxed identifiers found in the given expression.
330
+ Args:
331
+ expression: The expression string to process.
332
+ Returns:
333
+ A set of identifier strings found in the expression."""
334
+ identifiers: set[str] = set()
335
+ for match in OracleEngine._PREFIXED_IDENTIFIER.finditer(expression):
336
+ ident = match.group(1)
337
+ if ident.startswith('"') and ident.endswith('"'):
338
+ ident = ident[1:-1].replace('""', '"')
339
+ else:
340
+ ident = ident.upper()
341
+ identifiers.add(ident)
342
+ return identifiers
343
+
344
+ @staticmethod
345
+ def rewrite_expression_identifiers(expression: str, replacements: Mapping[str, str], *, qualifier: str) -> str:
346
+ """Rewrite ``@`` identifiers to qualified persisted Oracle columns."""
347
+
348
+ def replace_identifier(match: re.Match[str]) -> str:
349
+ token = match.group(1)
350
+ if token.startswith('"') and token.endswith('"'):
351
+ identifier = token[1:-1].replace('""', '"')
352
+ else:
353
+ identifier = token.upper()
354
+ replacement = replacements.get(identifier)
355
+ if replacement is None:
356
+ return match.group(0)
357
+ return f"{qualifier}.{OracleEngine._format_identifier(replacement)}"
358
+
359
+ return OracleEngine._PREFIXED_IDENTIFIER.sub(replace_identifier, expression)
360
+
361
+ @staticmethod
362
+ def get_column_type(column: ColumnDefinition) -> str:
363
+ dtype = column.data_type.lower()
364
+ if dtype in ("string", "varchar", "varchar2"):
365
+ length = column.length or 255
366
+ return f"VARCHAR2({length})"
367
+ if dtype == "char":
368
+ length = column.length or 1
369
+ return f"CHAR({length})"
370
+ if dtype in ("number", "numeric", "decimal"):
371
+ if column.precision is not None and column.scale is not None:
372
+ return f"NUMBER({column.precision},{column.scale})"
373
+ if column.precision is not None:
374
+ return f"NUMBER({column.precision})"
375
+ return "NUMBER"
376
+ if dtype in ("integer", "int"):
377
+ return "NUMBER(10)"
378
+ if dtype == "date":
379
+ return "DATE"
380
+ if dtype == "clob":
381
+ return "CLOB"
382
+ raise ValueError(f"Unsupported column data type: {column.data_type}")
383
+
384
+ @staticmethod
385
+ def generate_sql_block(config: Config, table_cnf: Mapping[str, Any], process_date: str) -> str:
386
+ """Generates a PL/SQL block for table processing with optional chunking and LOB handling.
387
+ Args:
388
+ config: Configuration object.
389
+ table_cnf: Processed table configuration object.
390
+ process_date: Process date in 'YYYYMMDD' format.
391
+ Returns:
392
+ PL/SQL block as string."""
393
+ execution = config.execution
394
+ administration = config.administration
395
+ cnd0 = table_cnf["conds"][0]
396
+ source_owner, history_owner, table_name = cnd0["source_owner"], cnd0["history_owner"], cnd0["table_name"]
397
+ history_hint_expr = cnd0.get("history_hint_expr")
398
+ hint_expr = cnd0["hint_expr"]
399
+ if execution.action == "HISTORY_ILM":
400
+ hint_expr = nvl(history_hint_expr, hint_expr)
401
+ has_lob_columns = cnd0["has_lob_columns"] == "Y"
402
+ other_columns, referencing_tables = table_cnf["other_columns"], table_cnf["referencing_tables"]
403
+ other_cols_alias = [col["name"] for col in other_columns]
404
+ other_cols_exprs = [col["expr"] for col in other_columns]
405
+ query_expr, table_columns, months_keep_history_max = (
406
+ table_cnf["query_expr"],
407
+ table_cnf["table_columns"],
408
+ table_cnf["months_keep_history_max"],
409
+ )
410
+ referencing_tables = ", ".join([f"'{rt[0]}.{rt[1]}'" for rt in referencing_tables])
411
+ source_ilm = execution.action == "SOURCE_ILM"
412
+ runtime_owner = source_owner if source_ilm else history_owner
413
+ ins_cols = join_wrapped(", ", table_columns + other_cols_alias, 200)
414
+ ins_vals = join_wrapped(", ", [f"r_rec(i).{col}" for col in table_columns + other_cols_alias], 200)
415
+ if execution.generate_script:
416
+ process_date = "&process_date"
417
+ plsql = f"""declare
418
+ l_source_owner varchar2(50) := '{source_owner}';
419
+ l_history_owner varchar2(50) := '{history_owner}';
420
+ l_action varchar2(11) := '{execution.action}';
421
+ l_mode varchar2(10) := '{execution.mode}';
422
+ l_message varchar2(200) := case
423
+ when l_mode = 'EXECUTE' then null
424
+ when l_mode = 'SCRIPT' then 'SCRIPT'
425
+ when l_mode = 'PREVIEW' then 'PREVIEW'
426
+ else l_mode
427
+ end;
428
+ l_table_name varchar2(50) := '{table_name}';
429
+ l_process_date date := to_date('{process_date}', 'YYYYMMDD');
430
+ l_referencing_tables t_referencing_tables := t_referencing_tables({referencing_tables});
431
+ l_process_start date;
432
+ l_record_count pls_integer := 0;
433
+ l_plsql clob := {"null" if execution.generate_script else ":plsql_code"};
434
+ l_sqlcode number := null;
435
+ l_out_message varchar2(200) := null;"""
436
+ if not has_lob_columns:
437
+ other_cols_select = [
438
+ f"{expression} AS {OracleEngine._format_identifier(alias)}"
439
+ for expression, alias in zip(other_cols_exprs, other_cols_alias, strict=True)
440
+ ]
441
+ cols_expr = ", ".join(["A.*", *other_cols_select])
442
+ plsql += f"""
443
+ l_chunk_size pls_integer := {execution.chunk_size}; l_chunk_start date;
444
+ cursor c_records is
445
+ select /*+ {hint_expr} */ A.rowid{", " + cols_expr if source_ilm else ""}
446
+ {indent_lines(query_expr, 8)};
447
+ type t_records is table of c_records%rowtype index by pls_integer; r_rec t_records;"""
448
+ plsql += f"""
449
+ begin
450
+ l_process_start := sysdate;
451
+ check_save_status(l_source_owner, l_table_name, l_process_date, l_action, '{Status.TABLE_START}', l_process_start, null, null, l_message, 0, l_plsql, l_sqlcode, l_out_message);
452
+ if l_sqlcode is not null then raise_application_error(l_sqlcode, l_out_message); end if;"""
453
+ if source_ilm or not execution.use_added_columns:
454
+ plsql += """
455
+ check_referencing_tables(l_referencing_tables, l_process_date); commit;"""
456
+ if not has_lob_columns or not source_ilm:
457
+ plsql += f"""
458
+ open c_records;
459
+ loop
460
+ l_chunk_start := sysdate;
461
+ check_save_status(l_source_owner, l_table_name, l_process_date, l_action, '{Status.CHUNK_START}', null, l_chunk_start, null, l_message, 0, null, l_sqlcode, l_out_message);
462
+ fetch c_records bulk collect into r_rec limit l_chunk_size;
463
+ if r_rec.count <= 0 then
464
+ exit;
465
+ end if;"""
466
+ if execution.mode in ("EXECUTE", "SCRIPT"):
467
+ if source_ilm and nvl(months_keep_history_max, 1) > 0:
468
+ plsql += f"""
469
+ for i in 1 .. r_rec.count loop
470
+ insert into {OracleEngine._format_identifier(history_owner)}.{OracleEngine._format_identifier(table_name)}@{OracleEngine._format_identifier(administration.source_to_history_dblink_name)}
471
+ ({indent_lines(ins_cols, 12)})
472
+ values ({indent_lines(ins_vals, 12)});
473
+ end loop;"""
474
+ plsql += f"""
475
+ forall i in 1 .. r_rec.count
476
+ delete from {OracleEngine._format_identifier(runtime_owner)}.{OracleEngine._format_identifier(table_name)} where rowid = r_rec(i).rowid;
477
+ l_record_count := l_record_count + r_rec.count;"""
478
+ plsql += f"""
479
+ check_save_status(l_source_owner, l_table_name, l_process_date, l_action, '{Status.CHUNK_END}', null, l_chunk_start, sysdate, l_message, r_rec.count, null, l_sqlcode, l_out_message);
480
+ commit;
481
+ end loop;
482
+ close c_records;"""
483
+ else:
484
+ cols_select = ", ".join([f"a.{col}" for col in table_columns] + other_cols_exprs)
485
+ if execution.mode in ("EXECUTE", "SCRIPT"):
486
+ if source_ilm and nvl(months_keep_history_max, 0) > 0:
487
+ plsql += f"""
488
+ insert into {OracleEngine._format_identifier(history_owner)}.{OracleEngine._format_identifier(table_name)}@{OracleEngine._format_identifier(administration.source_to_history_dblink_name)}({indent_lines(ins_cols, 4)})
489
+ select /*+ {hint_expr} */ {indent_lines(cols_select, 4)}
490
+ {indent_lines(query_expr, 4)};"""
491
+ plsql += f"""
492
+ delete from {OracleEngine._format_identifier(runtime_owner)}.{OracleEngine._format_identifier(table_name)} where rowid in
493
+ (select /*+ {hint_expr} */ a.rowid
494
+ {indent_lines(query_expr, 4)});
495
+ l_record_count := sql%rowcount;"""
496
+ else:
497
+ plsql += f"""
498
+ select /*+ {hint_expr} */ count(*) into l_record_count
499
+ {indent_lines(query_expr, 4)};"""
500
+ table_end_rows = "l_record_count" if has_lob_columns else "0"
501
+ plsql += f"""
502
+ check_save_status(l_source_owner, l_table_name, l_process_date, l_action, '{Status.TABLE_END}', l_process_start, null, sysdate, l_message, {table_end_rows}, null, l_sqlcode, l_out_message);
503
+ commit;
504
+ exception
505
+ when others then
506
+ rollback;
507
+ check_save_status(l_source_owner, l_table_name, l_process_date, l_action, '{Status.ERROR}', l_process_start, null, sysdate, sqlerrm, l_record_count, null, l_sqlcode, l_out_message);
508
+ commit;
509
+ if l_sqlcode not in (-20003) then
510
+ raise;
511
+ end if;
512
+ end;"""
513
+ return plsql
514
+
515
+ @staticmethod
516
+ def get_system_date(conn: oracledb.Connection) -> datetime:
517
+ """Returns the current system date from Oracle.
518
+ Args:
519
+ conn: Active Oracle connection.
520
+ Returns:
521
+ Current system date as Python date."""
522
+ cursor: oracledb.Cursor | None = None
523
+ try:
524
+ cursor = conn.cursor()
525
+ cursor.execute("SELECT SYSDATE FROM dual")
526
+ sysdate: datetime = cursor.fetchone()[0]
527
+ return sysdate
528
+ except Exception as exc:
529
+ logger.critical("Failed to retrieve system date from Oracle.", exc_info=True)
530
+ raise ExecutionError("Failed to retrieve system date from Oracle") from exc
531
+ finally:
532
+ if cursor:
533
+ cursor.close()
534
+
535
+ @staticmethod
536
+ def sql_block_run(conn: oracledb.Connection, plsql_code: str) -> None:
537
+ """Executes a PL/SQL block on the database.
538
+ Args:
539
+ conn: Active Oracle connection.
540
+ plsql_code: The PL/SQL block to execute."""
541
+ cursor: oracledb.Cursor | None = None
542
+ try:
543
+ cursor = conn.cursor()
544
+ cursor.setinputsizes(plsql_code=oracledb.CLOB)
545
+ cursor.execute(plsql_code, {"plsql_code": plsql_code})
546
+ finally:
547
+ if cursor:
548
+ cursor.close()
549
+
550
+ @staticmethod
551
+ def get_status(conn: oracledb.Connection, process_date: str) -> list[dict[str, Any]]:
552
+ """Loads status from ldb_ctl for the given process date.
553
+ Args:
554
+ conn: Active Oracle connection.
555
+ process_date: Date in 'YYYYMMDD' format.
556
+ Returns:
557
+ List of status rows."""
558
+ cursor: oracledb.Cursor | None = None
559
+ try:
560
+ cursor = conn.cursor()
561
+ cursor.execute(
562
+ """SELECT ctl_owner, ctl_table_name, ctl_status
563
+ FROM ldb_ctl
564
+ WHERE ctl_process_date = TO_DATE(:1, 'YYYYMMDD')""",
565
+ [process_date],
566
+ )
567
+ cols = [col[0].lower() for col in cursor.description]
568
+ return [dict(zip(cols, row, strict=True)) for row in cursor.fetchall()]
569
+ finally:
570
+ if cursor:
571
+ cursor.close()
572
+
573
+ @staticmethod
574
+ def get_rows_processed(conn: oracledb.Connection, owner: str, table_name: str, process_date: str) -> int:
575
+ """Returns rows processed for a given table and process date from ldb_ctl.
576
+ Args:
577
+ conn: Active Oracle connection.
578
+ owner: Schema owner of the table.
579
+ table_name: Table name.
580
+ process_date: Target process date in 'YYYYMMDD'.
581
+ Returns:
582
+ Number of rows processed or 0 if none found or mismatched date."""
583
+ cursor: oracledb.Cursor | None = None
584
+ try:
585
+ cursor = conn.cursor()
586
+ cursor.execute(
587
+ """SELECT ctl_rows_processed
588
+ FROM ldb_ctl WHERE ctl_owner = :1 AND ctl_table_name = :2 AND ctl_process_date = TO_DATE(:3, 'YYYYMMDD')""",
589
+ [owner, table_name, process_date],
590
+ )
591
+ result = cursor.fetchone()
592
+ return result[0] if result else 0
593
+ except Exception as exc:
594
+ logger.critical(f"Unexpected error in get_rows_processed for {owner}.{table_name}:", exc_info=True)
595
+ raise ExecutionError(f"Failed to read rows processed for {owner}.{table_name}") from exc
596
+ finally:
597
+ if cursor:
598
+ cursor.close()
599
+
600
+ @staticmethod
601
+ def save_error_status(
602
+ conn: oracledb.Connection,
603
+ config: Config,
604
+ owner: str,
605
+ table_name: str,
606
+ process_date: str,
607
+ process_start: datetime,
608
+ message: str,
609
+ plsql_code: str,
610
+ ) -> None:
611
+ cursor: oracledb.Cursor | None = None
612
+ try:
613
+ cursor = conn.cursor()
614
+ process_end = OracleEngine.get_system_date(conn)
615
+ sqlcode = cursor.var(oracledb.NUMBER)
616
+ out_message = cursor.var(oracledb.STRING)
617
+ cursor.callproc(
618
+ "check_save_status",
619
+ [
620
+ owner,
621
+ table_name,
622
+ datetime.strptime(process_date, "%Y%m%d").date(),
623
+ config.execution.action,
624
+ Status.ERROR,
625
+ process_start,
626
+ None,
627
+ process_end,
628
+ message,
629
+ 0,
630
+ plsql_code,
631
+ sqlcode,
632
+ out_message,
633
+ ],
634
+ )
635
+ conn.commit()
636
+ except Exception as exc:
637
+ logger.critical(f"Unexpected error in process_table for {owner}.{table_name}:", exc_info=True)
638
+ raise ExecutionError(f"Failed to save error status for {owner}.{table_name}") from exc
639
+ finally:
640
+ if cursor:
641
+ cursor.close()
642
+
643
+ @staticmethod
644
+ def all_status_tend(
645
+ conn: oracledb.Connection, tables_config: dict[tuple[str, str], Any], process_date: str
646
+ ) -> bool:
647
+ """Checks if all referenced tables have status "Status.TABLE_END" in ldb_ctl.
648
+ Args:
649
+ conn: Active Oracle connection.
650
+ tables_config: Dict of (owner, table_name) keys representing configured tables.
651
+ process_date: Processing date in 'YYYYMMDD' format.
652
+ Returns:
653
+ True if all tables have status "Status.TABLE_END" for the given process date, False otherwise."""
654
+ cursor: oracledb.Cursor | None = None
655
+ try:
656
+ cursor = conn.cursor()
657
+ for (owner, table_name), table_cnf in tables_config.items():
658
+ if table_cnf["skip"]:
659
+ continue
660
+ cursor.execute(
661
+ f"""SELECT ctl_status FROM ldb_ctl
662
+ WHERE ctl_owner = :1 AND ctl_table_name = :2
663
+ AND ctl_process_date = TO_DATE(:3, 'YYYYMMDD') AND ctl_status = '{Status.TABLE_END}'""",
664
+ [owner, table_name, process_date],
665
+ )
666
+ if not cursor.fetchone():
667
+ return False
668
+ return True
669
+ finally:
670
+ if cursor:
671
+ cursor.close()
672
+
673
+ @staticmethod
674
+ def ensure_users(conn: oracledb.Connection, users: Sequence[UserDefinition]) -> list[str]:
675
+ created: list[str] = []
676
+ if not users:
677
+ return created
678
+ cursor: oracledb.Cursor | None = None
679
+ changed = False
680
+ try:
681
+ cursor = conn.cursor()
682
+ database_default_tablespace = OracleEngine._get_database_default_tablespace(cursor)
683
+ for user in users:
684
+ fmttd_username = OracleEngine._format_identifier(user.name)
685
+ fmttd_default_tablespace = OracleEngine._format_identifier(user.default_tablespace)
686
+ tablespace_for_quota = user.default_tablespace
687
+ if not OracleEngine._user_exists(cursor, user.name):
688
+ sql = f"CREATE USER {fmttd_username} IDENTIFIED BY ##{OracleEngine._quote(user.password)}##"
689
+ if user.default_tablespace:
690
+ sql += f" DEFAULT TABLESPACE {fmttd_default_tablespace}"
691
+ if user.temporary_tablespace:
692
+ sql += f" TEMPORARY TABLESPACE {OracleEngine._format_identifier(user.temporary_tablespace)}"
693
+ logger.debug(f"Creating user with SQL: {sql}")
694
+ OracleEngine._execute_ddl(conn, cursor, sql)
695
+ created.append(fmttd_username)
696
+ changed = True
697
+ if not user.default_tablespace:
698
+ tablespace_for_quota = (
699
+ OracleEngine._get_user_default_tablespace(cursor, user.name) or database_default_tablespace
700
+ )
701
+ else:
702
+ tablespace_for_quota = (
703
+ OracleEngine._get_user_default_tablespace(cursor, user.name) or database_default_tablespace
704
+ )
705
+ admin_option_roles = set(OracleEngine._format_identifier(r) for r in user.roles_with_admin_option)
706
+ for role in user.roles:
707
+ fmttd_role_name = OracleEngine._format_identifier(role)
708
+ requires_admin_option = role in admin_option_roles
709
+ has_role = OracleEngine._user_has_role(cursor, user.name, role)
710
+ if requires_admin_option:
711
+ if not has_role or not OracleEngine._user_has_role_with_admin_option(cursor, user.name, role):
712
+ grant_sql = f"GRANT {fmttd_role_name} TO {fmttd_username} WITH ADMIN OPTION"
713
+ OracleEngine._execute_ddl(conn, cursor, grant_sql)
714
+ changed = True
715
+ elif not has_role:
716
+ grant_sql = f"GRANT {fmttd_role_name} TO {fmttd_username}"
717
+ OracleEngine._execute_ddl(conn, cursor, grant_sql)
718
+ changed = True
719
+ for privilege in user.system_privileges:
720
+ if not OracleEngine._user_has_sys_priv(cursor, user.name, privilege):
721
+ grant_priv_sql = f"GRANT {privilege} TO {fmttd_username}"
722
+ OracleEngine._execute_ddl(conn, cursor, grant_priv_sql)
723
+ changed = True
724
+ if OracleEngine._ensure_unlimited_quota(cursor, user.name, tablespace_for_quota):
725
+ changed = True
726
+ if changed:
727
+ conn.commit()
728
+ except Exception as exc:
729
+ conn.rollback()
730
+ logger.critical("Failed to ensure users.", exc_info=True)
731
+ raise ExecutionError("Failed to ensure Oracle users") from exc
732
+ finally:
733
+ if cursor:
734
+ cursor.close()
735
+ return created
736
+
737
+ @staticmethod
738
+ def ensure_tables(conn: oracledb.Connection, tables: Sequence[TableDefinition]) -> list[str]:
739
+ created: list[str] = []
740
+ if not tables:
741
+ return created
742
+ cursor: oracledb.Cursor | None = None
743
+ changed = False
744
+ try:
745
+ cursor = conn.cursor()
746
+ for table in tables:
747
+ fmttd_owner = OracleEngine._format_identifier(table.owner)
748
+ fmttd_table_name = OracleEngine._format_identifier(table.name)
749
+ if not OracleEngine._table_exists(cursor, table.owner, table.name):
750
+ columns_sql = ",\n ".join(OracleEngine._column_sql(col) for col in table.columns)
751
+ sql = f"CREATE TABLE {fmttd_owner}.{fmttd_table_name} (\n {columns_sql}\n )"
752
+ OracleEngine._execute_ddl(conn, cursor, sql)
753
+ created.append(f"{fmttd_owner}.{fmttd_table_name}")
754
+ changed = True
755
+ if table.primary_key:
756
+ pk_name = f"{table.name}_PK"
757
+ fmttd_pk_name = OracleEngine._format_identifier(pk_name)
758
+ if not OracleEngine._constraint_exists(cursor, table.owner, pk_name):
759
+ cols = ", ".join(OracleEngine._format_identifier(col) for col in table.primary_key)
760
+ sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} ADD CONSTRAINT {fmttd_pk_name} PRIMARY KEY ({cols})"
761
+ OracleEngine._execute_ddl(conn, cursor, sql)
762
+ changed = True
763
+ for index in table.indexes:
764
+ fmttd_idx_name = OracleEngine._format_identifier(index.name)
765
+ if OracleEngine._index_exists(cursor, table.owner, index.name):
766
+ continue
767
+ cols = ", ".join(OracleEngine._format_identifier(col) for col in index.columns)
768
+ unique_kw = "UNIQUE " if index.unique else ""
769
+ sql = f"CREATE {unique_kw}INDEX {fmttd_owner}.{fmttd_idx_name} ON {fmttd_owner}.{fmttd_table_name} ({cols})"
770
+ OracleEngine._execute_ddl(conn, cursor, sql)
771
+ changed = True
772
+ if changed:
773
+ conn.commit()
774
+ except Exception as exc:
775
+ conn.rollback()
776
+ logger.critical("Failed to ensure tables.", exc_info=True)
777
+ raise ExecutionError("Failed to ensure Oracle tables") from exc
778
+ finally:
779
+ if cursor:
780
+ cursor.close()
781
+ return created
782
+
783
+ @staticmethod
784
+ def ensure_table_structure(conn: oracledb.Connection, table: TableDefinition, table_cnf: Mapping[str, Any]) -> None:
785
+ cursor: oracledb.Cursor | None = None
786
+ fmttd_owner = OracleEngine._format_identifier(table.owner)
787
+ fmttd_table_name = OracleEngine._format_identifier(table.name)
788
+ logger.debug(f"Ensuring structure for table {table.owner}.{table.name} conn: {conn}")
789
+ try:
790
+ cursor = conn.cursor()
791
+ if not OracleEngine._table_exists(cursor, table.owner, table.name):
792
+ fmttd_columns_sql = ",\n ".join(OracleEngine._column_sql(col) for col in table.columns)
793
+ sql = f"CREATE TABLE {fmttd_owner}.{fmttd_table_name} (\n {fmttd_columns_sql}\n )"
794
+ OracleEngine._execute_ddl(conn, cursor, sql)
795
+ else:
796
+ _, existing_columns = OracleEngine.get_table_columns(conn, table.owner, table.name)
797
+ for column in table.columns:
798
+ column_definition = OracleEngine._column_sql(column)
799
+ if column.name not in existing_columns:
800
+ sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} ADD ({column_definition})"
801
+ OracleEngine._execute_ddl(conn, cursor, sql)
802
+ else:
803
+ existing = existing_columns[column.name]
804
+ if OracleEngine._column_needs_update(existing, column):
805
+ sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} MODIFY ({column_definition})"
806
+ OracleEngine._execute_ddl(conn, cursor, sql)
807
+ existing_indexes = OracleEngine._get_table_indexes(cursor, fmttd_owner, fmttd_table_name)
808
+ desired_pk = tuple(table.primary_key or ())
809
+ fmttd_desired_pk = tuple(OracleEngine._format_identifier(col) for col in desired_pk)
810
+ existing_pk_name, existing_pk_cols, existing_pk_index = OracleEngine._get_primary_key_info(
811
+ cursor, table.owner, table.name
812
+ )
813
+ fmttd_existing_pk_cols = tuple(OracleEngine._format_identifier(col) for col in existing_pk_cols)
814
+ fmttd_existing_pk_index = OracleEngine._format_identifier(existing_pk_index) if existing_pk_index else None
815
+ desired_constraint_name = f"{table.name}_PK"
816
+ fmttd_desired_constraint_name = OracleEngine._format_identifier(desired_constraint_name)
817
+ if desired_pk:
818
+ if not existing_pk_cols:
819
+ existing_index_info = existing_indexes.get(desired_constraint_name)
820
+ if existing_index_info and (existing_index_info[0] != desired_pk or not existing_index_info[1]):
821
+ drop_sql = f"DROP INDEX {fmttd_owner}.{fmttd_desired_constraint_name}"
822
+ OracleEngine._execute_ddl(conn, cursor, drop_sql)
823
+ existing_indexes.pop(desired_constraint_name, None)
824
+ add_pk_sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} ADD CONSTRAINT {fmttd_desired_constraint_name} PRIMARY KEY ({', '.join(fmttd_desired_pk)})"
825
+ OracleEngine._execute_ddl(conn, cursor, add_pk_sql)
826
+ elif existing_pk_cols != desired_pk:
827
+ if existing_pk_name:
828
+ drop_pk_sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} DROP CONSTRAINT {OracleEngine._format_identifier(existing_pk_name)}"
829
+ OracleEngine._execute_ddl(conn, cursor, drop_pk_sql)
830
+ index_to_drop = fmttd_existing_pk_index or existing_pk_name
831
+ if index_to_drop:
832
+ existing_indexes = OracleEngine._get_table_indexes(cursor, table.owner, table.name)
833
+ if index_to_drop and index_to_drop in existing_indexes:
834
+ drop_index_sql = f"DROP INDEX {fmttd_owner}.{index_to_drop}"
835
+ OracleEngine._execute_ddl(conn, cursor, drop_index_sql)
836
+ existing_indexes.pop(index_to_drop, None)
837
+ existing_index_info = existing_indexes.get(desired_constraint_name)
838
+ if existing_index_info and (existing_index_info[0] != desired_pk or not existing_index_info[1]):
839
+ drop_conflict_sql = f"DROP INDEX {fmttd_owner}.{fmttd_desired_constraint_name}"
840
+ OracleEngine._execute_ddl(conn, cursor, drop_conflict_sql)
841
+ existing_indexes.pop(desired_constraint_name, None)
842
+ recreate_pk_sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} ADD CONSTRAINT {fmttd_desired_constraint_name} PRIMARY KEY ({', '.join(fmttd_desired_pk)})"
843
+ OracleEngine._execute_ddl(conn, cursor, recreate_pk_sql)
844
+ elif fmttd_existing_pk_cols and existing_pk_name:
845
+ drop_unexpected_pk_sql = (
846
+ f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} DROP CONSTRAINT {existing_pk_name}"
847
+ )
848
+ OracleEngine._execute_ddl(conn, cursor, drop_unexpected_pk_sql)
849
+ index_to_drop = fmttd_existing_pk_index or existing_pk_name
850
+ if index_to_drop:
851
+ existing_indexes = OracleEngine._get_table_indexes(cursor, table.owner, table.name)
852
+ if index_to_drop and index_to_drop in existing_indexes:
853
+ drop_unexpected_index_sql = f"DROP INDEX {fmttd_owner}.{index_to_drop}"
854
+ OracleEngine._execute_ddl(conn, cursor, drop_unexpected_index_sql)
855
+ existing_indexes.pop(index_to_drop, None)
856
+ existing_indexes = OracleEngine._get_table_indexes(cursor, table.owner, table.name)
857
+ desired_indexes = OracleEngine._prepare_desired_indexes(table)
858
+ for index_name, (columns, is_unique) in desired_indexes.items():
859
+ fmttd_index_name = OracleEngine._format_identifier(index_name)
860
+ fmttd_columns = tuple(OracleEngine._format_identifier(col) for col in columns)
861
+ existing_index = existing_indexes.get(index_name)
862
+ if existing_index == (columns, is_unique):
863
+ continue
864
+ if existing_index is not None:
865
+ drop_mismatch_index_sql = f"DROP INDEX {fmttd_owner}.{fmttd_index_name}"
866
+ OracleEngine._execute_ddl(conn, cursor, drop_mismatch_index_sql)
867
+ fmttd_columns_sql = ", ".join(fmttd_columns)
868
+ unique_clause = "UNIQUE " if is_unique else ""
869
+ create_index_sql = f"CREATE {unique_clause}INDEX {fmttd_owner}.{index_name} ON {fmttd_owner}.{fmttd_table_name} ({fmttd_columns_sql})"
870
+ OracleEngine._execute_ddl(conn, cursor, create_index_sql)
871
+ final_pk_name, _, final_pk_index = OracleEngine._get_primary_key_info(cursor, table.owner, table.name)
872
+ expected_indexes = set(desired_indexes.keys())
873
+ if final_pk_name:
874
+ expected_indexes.add(final_pk_name)
875
+ if final_pk_index:
876
+ expected_indexes.add(final_pk_index)
877
+ existing_indexes = OracleEngine._get_table_indexes(cursor, table.owner, table.name)
878
+ for index_name in list(existing_indexes.keys()):
879
+ if index_name not in expected_indexes:
880
+ fmttd_index_name = OracleEngine._format_identifier(index_name)
881
+ logger.info("Dropping unmanaged index %s on %s.%s", fmttd_index_name, fmttd_owner, fmttd_table_name)
882
+ drop_unmanaged_index_sql = f"DROP INDEX {fmttd_owner}.{fmttd_index_name}"
883
+ OracleEngine._execute_ddl(conn, cursor, drop_unmanaged_index_sql)
884
+ except Exception as exc:
885
+ conn.rollback()
886
+ logger.critical("Failed to ensure table structure.", exc_info=True)
887
+ raise ExecutionError("Failed to ensure Oracle table structure") from exc
888
+ finally:
889
+ if cursor:
890
+ cursor.close()
891
+
892
+ @staticmethod
893
+ def get_primary_key_columns(
894
+ conn: oracledb.Connection, owner: str, table_name: str, table_cnf: Mapping[str, Any]
895
+ ) -> tuple[str, ...]:
896
+ cursor: oracledb.Cursor | None = None
897
+ try:
898
+ cursor = conn.cursor()
899
+ _, columns, _ = OracleEngine._get_primary_key_info(cursor, owner, table_name)
900
+ if columns:
901
+ return tuple(col for col in columns)
902
+ cursor.execute(
903
+ """
904
+ SELECT i.index_name, i.uniqueness, c.column_name, s.distinct_keys
905
+ FROM dba_indexes i JOIN dba_ind_columns c ON i.owner = c.index_owner AND i.index_name = c.index_name
906
+ LEFT JOIN dba_ind_statistics s ON s.owner = i.owner AND s.index_name = i.index_name AND s.partition_name IS NULL
907
+ WHERE i.owner = :owner
908
+ AND i.table_name = :table_name
909
+ ORDER BY i.index_name, c.column_position
910
+ """,
911
+ owner=owner,
912
+ table_name=table_name,
913
+ )
914
+ indexes: dict[str, dict[str, Any]] = {}
915
+ for index_name, uniqueness, column_name, distinct_keys in cast(
916
+ Iterable[tuple[str, str, str, int | None]], cursor
917
+ ):
918
+ index_info = indexes.setdefault(
919
+ index_name,
920
+ {
921
+ "columns": [],
922
+ "unique": uniqueness == "UNIQUE",
923
+ "distinct_keys": None,
924
+ },
925
+ )
926
+ index_info["columns"].append(column_name)
927
+ if distinct_keys is not None:
928
+ try:
929
+ index_info["distinct_keys"] = int(distinct_keys)
930
+ except (TypeError, ValueError):
931
+ try:
932
+ index_info["distinct_keys"] = int(float(distinct_keys))
933
+ except (TypeError, ValueError):
934
+ index_info["distinct_keys"] = index_info.get("distinct_keys")
935
+ if not indexes:
936
+ return ()
937
+ all_columns = {col for info in indexes.values() for col in info.get("columns", [])}
938
+ columns_metadata = table_cnf.get("columns_metadata", {})
939
+ for col in all_columns:
940
+ if hasattr(table_cnf["metadata"], col):
941
+ logger.warning(f"Index column '{col}' not found in table definition for {owner}.{table_name}.")
942
+ chosen = OracleEngine._choose_best_index_for_primary_key(indexes, columns_metadata)
943
+ if not chosen:
944
+ return ()
945
+ return tuple(chosen)
946
+ finally:
947
+ if cursor:
948
+ cursor.close()
949
+
950
+ @staticmethod
951
+ def ensure_roles(conn: oracledb.Connection, roles: Sequence[RoleDefinition]) -> list[str]:
952
+ created: list[str] = []
953
+ if not roles:
954
+ return created
955
+ cursor: oracledb.Cursor | None = None
956
+ changed = False
957
+ try:
958
+ cursor = conn.cursor()
959
+ for role in roles:
960
+ fmttd_role_name = OracleEngine._format_identifier(role.name)
961
+ if OracleEngine._role_exists(cursor, role.name):
962
+ continue
963
+ sql = f"CREATE ROLE {fmttd_role_name}"
964
+ OracleEngine._execute_ddl(conn, cursor, sql)
965
+ created.append(fmttd_role_name)
966
+ changed = True
967
+ if changed:
968
+ conn.commit()
969
+ except Exception as exc:
970
+ conn.rollback()
971
+ logger.critical("Failed to ensure roles.", exc_info=True)
972
+ raise ExecutionError("Failed to ensure Oracle roles") from exc
973
+ finally:
974
+ if cursor:
975
+ cursor.close()
976
+ return created
977
+
978
+ @staticmethod
979
+ def ensure_sequences(conn: oracledb.Connection, sequences: Sequence[SequenceDefinition]) -> list[str]:
980
+ created: list[str] = []
981
+ if not sequences:
982
+ return created
983
+ cursor: oracledb.Cursor | None = None
984
+ changed = False
985
+ try:
986
+ cursor = conn.cursor()
987
+ for sequence in sequences:
988
+ fmttd_owner = OracleEngine._format_identifier(sequence.owner)
989
+ fmttd_sequence_name = OracleEngine._format_identifier(sequence.name)
990
+ if OracleEngine._sequence_exists(cursor, sequence.owner, sequence.name):
991
+ continue
992
+ sql = (
993
+ f"CREATE SEQUENCE {fmttd_owner}.{fmttd_sequence_name} "
994
+ f"START WITH {sequence.start_with} INCREMENT BY {sequence.increment_by}"
995
+ )
996
+ if sequence.minvalue is not None:
997
+ sql += f" MINVALUE {sequence.minvalue}"
998
+ else:
999
+ sql += " NOMINVALUE"
1000
+ if sequence.maxvalue is not None:
1001
+ sql += f" MAXVALUE {sequence.maxvalue}"
1002
+ else:
1003
+ sql += " NOMAXVALUE"
1004
+ sql += " CYCLE" if sequence.cycle else " NOCYCLE"
1005
+ if sequence.cache is not None:
1006
+ sql += f" CACHE {sequence.cache}"
1007
+ else:
1008
+ sql += " NOCACHE"
1009
+ OracleEngine._execute_ddl(conn, cursor, sql)
1010
+ created.append(f"{fmttd_owner}.{fmttd_sequence_name}")
1011
+ changed = True
1012
+ if changed:
1013
+ conn.commit()
1014
+ except Exception as exc:
1015
+ conn.rollback()
1016
+ logger.critical("Failed to ensure sequences.", exc_info=True)
1017
+ raise ExecutionError("Failed to ensure Oracle sequences") from exc
1018
+ finally:
1019
+ if cursor:
1020
+ cursor.close()
1021
+ return created
1022
+
1023
+ @staticmethod
1024
+ def ensure_database_links(conn: oracledb.Connection, links: Sequence[DatabaseLinkDefinition]) -> list[str]:
1025
+ created: list[str] = []
1026
+ if not links:
1027
+ return created
1028
+ cursor: oracledb.Cursor | None = None
1029
+ changed = False
1030
+ try:
1031
+ cursor = conn.cursor()
1032
+ for link in links:
1033
+ fmttd_link_name = OracleEngine._format_identifier(link.name)
1034
+ if OracleEngine._db_link_exists(cursor, link.name):
1035
+ continue
1036
+ fmttd_username = OracleEngine._format_identifier(link.username)
1037
+ fmttd_password = OracleEngine._quote(link.password)
1038
+ literal_dsn_literal = OracleEngine._quote_literal(link.dsn)
1039
+ sql = f"CREATE DATABASE LINK {fmttd_link_name} CONNECT TO {fmttd_username} IDENTIFIED BY ##{fmttd_password}## USING {literal_dsn_literal}"
1040
+ OracleEngine._execute_ddl(conn, cursor, sql)
1041
+ created.append(fmttd_link_name)
1042
+ changed = True
1043
+ if changed:
1044
+ conn.commit()
1045
+ except Exception as exc:
1046
+ conn.rollback()
1047
+ logger.critical("Failed to ensure database links.", exc_info=True)
1048
+ raise ExecutionError("Failed to ensure Oracle database links") from exc
1049
+ finally:
1050
+ if cursor:
1051
+ cursor.close()
1052
+ return created
1053
+
1054
+ @staticmethod
1055
+ def ensure_table_privileges(
1056
+ conn: oracledb.Connection, role: str, tables: Sequence[tuple[str, str]], privileges: Sequence[str]
1057
+ ) -> list[str]:
1058
+ cursor: oracledb.Cursor | None = None
1059
+ granted: list[str] = []
1060
+ changed = False
1061
+ if not tables:
1062
+ return granted
1063
+ fmt_role = OracleEngine._format_identifier(role)
1064
+ try:
1065
+ cursor = conn.cursor()
1066
+ for owner, table_name in tables:
1067
+ fmttd_owner = OracleEngine._format_identifier(owner)
1068
+ fmttd_table_name = OracleEngine._format_identifier(table_name)
1069
+ for privilege in privileges:
1070
+ privilege_upper = privilege.upper()
1071
+ if OracleEngine._role_has_table_priv(
1072
+ cursor, role.upper(), owner.upper(), table_name.upper(), privilege_upper
1073
+ ):
1074
+ continue
1075
+ sql = f"GRANT {privilege_upper} ON {fmttd_owner}.{fmttd_table_name} TO {fmt_role}"
1076
+ OracleEngine._execute_ddl(conn, cursor, sql)
1077
+ granted.append(f"{fmttd_owner}.{fmttd_table_name}:{privilege_upper}")
1078
+ changed = True
1079
+ if changed:
1080
+ conn.commit()
1081
+ except Exception as exc:
1082
+ conn.rollback()
1083
+ logger.critical("Failed to ensure table privileges.", exc_info=True)
1084
+ raise ExecutionError("Failed to ensure Oracle table privileges") from exc
1085
+ finally:
1086
+ if cursor:
1087
+ cursor.close()
1088
+ return granted
1089
+
1090
+ @staticmethod
1091
+ def ensure_supporting_objects(conn: oracledb.Connection, owner: str) -> list[str]:
1092
+ cursor: oracledb.Cursor | None = None
1093
+ try:
1094
+ cursor = conn.cursor()
1095
+ cursor.execute(f"ALTER SESSION SET CURRENT_SCHEMA = {OracleEngine._format_identifier(owner)}")
1096
+ for statement in (T_REFERENCING_TABLES_TYPE, CHECK_SAVE_STATUS_PROC, CHECK_REFERENCING_TABLES_PROC):
1097
+ OracleEngine._execute_ddl(conn, cursor, statement)
1098
+ return SUPPORTING_OBJECT_LIST
1099
+ except Exception as exc:
1100
+ logger.critical("Failed to ensure supporting PL/SQL objects.", exc_info=True)
1101
+ raise ExecutionError("Failed to ensure Oracle supporting PL/SQL objects") from exc
1102
+ finally:
1103
+ if cursor:
1104
+ cursor.close()
1105
+
1106
+ @staticmethod
1107
+ def get_identifier_str(identifier: str) -> str:
1108
+ """Returns the identifier string required to query dictionary views.
1109
+ Args:
1110
+ indentifier: The identifier string to process.
1111
+ Returns:
1112
+ The identifier string without angle brackets.
1113
+ """
1114
+ identifier = identifier.strip()
1115
+ if identifier.startswith('"') and identifier.endswith('"'):
1116
+ return identifier[1:-1]
1117
+ return identifier.upper()
1118
+
1119
+ @staticmethod
1120
+ def get_ldb_columns_expressions() -> tuple[str, str]:
1121
+ """Returns the expressions for the LDB process date and insert date columns.
1122
+ Returns:
1123
+ A tuple containing the process date expression and insert date expression.
1124
+ """
1125
+ return "l_process_date", "sysdate"
1126
+
1127
+ @staticmethod
1128
+ def _column_sql(column: ColumnDefinition) -> str:
1129
+ col_name = OracleEngine._format_identifier(column.name)
1130
+ col_type = OracleEngine.get_column_type(column)
1131
+ default_clause = f" DEFAULT {column.default}" if column.default is not None else ""
1132
+ nullable_clause = "" if column.nullable else " NOT NULL"
1133
+ return f"{col_name} {col_type}{default_clause}{nullable_clause}"
1134
+
1135
+ # format / expressions helpers
1136
+ @staticmethod
1137
+ def _format_identifier(name: str | None) -> str:
1138
+ """Formats an Oracle identifier, quoting if necessary.
1139
+ Args:
1140
+ name: The identifier name as appears data dictionary.
1141
+ Returns:
1142
+ Formatted identifier, quoted if necessary."""
1143
+ if not name:
1144
+ return '""'
1145
+ if OracleEngine._REGULAR_IDENTIFIER.fullmatch(name):
1146
+ return name.lower()
1147
+ return OracleEngine._quote(name)
1148
+
1149
+ @staticmethod
1150
+ def _quote(identifier: str) -> str:
1151
+ escaped = identifier.replace('"', '""')
1152
+ return f'"{escaped}"'
1153
+
1154
+ @staticmethod
1155
+ def _quote_literal(value: str) -> str:
1156
+ escaped = value.replace("'", "''")
1157
+ return f"'{escaped}'"
1158
+
1159
+ # metadata and privilege helpers
1160
+ @staticmethod
1161
+ def _object_exists(cursor: oracledb.Cursor, query: str, params: Sequence[Any]) -> bool:
1162
+ cursor.execute(query, tuple(params))
1163
+ return cursor.fetchone() is not None
1164
+
1165
+ @staticmethod
1166
+ def _user_has_role(cursor: oracledb.Cursor, username: str, role: str) -> bool:
1167
+ return OracleEngine._object_exists(
1168
+ cursor, "SELECT 1 FROM dba_role_privs WHERE grantee = :1 AND granted_role = :2", [username, role]
1169
+ )
1170
+
1171
+ @staticmethod
1172
+ def _user_has_role_with_admin_option(cursor: oracledb.Cursor, username: str, role: str) -> bool:
1173
+ return OracleEngine._object_exists(
1174
+ cursor,
1175
+ "SELECT 1 FROM dba_role_privs WHERE grantee = :1 AND granted_role = :2 AND admin_option = 'YES'",
1176
+ [username, role],
1177
+ )
1178
+
1179
+ @staticmethod
1180
+ def _user_has_sys_priv(cursor: oracledb.Cursor, username: str, privilege: str) -> bool:
1181
+ return OracleEngine._object_exists(
1182
+ cursor, "SELECT 1 FROM dba_sys_privs WHERE grantee = :1 AND privilege = :2", [username, privilege]
1183
+ )
1184
+
1185
+ @staticmethod
1186
+ def _role_has_table_priv(cursor: oracledb.Cursor, role: str, owner: str, table_name: str, privilege: str) -> bool:
1187
+ return OracleEngine._object_exists(
1188
+ cursor,
1189
+ """
1190
+ SELECT 1
1191
+ FROM dba_tab_privs
1192
+ WHERE grantee = :grantee
1193
+ AND owner = :owner
1194
+ AND table_name = :table_name
1195
+ AND privilege = :privilege
1196
+ """,
1197
+ [role, owner, table_name, privilege],
1198
+ )
1199
+
1200
+ @staticmethod
1201
+ def _role_exists(cursor: oracledb.Cursor, role: str) -> bool:
1202
+ return OracleEngine._object_exists(cursor, "SELECT 1 FROM dba_roles WHERE role = :1", [role])
1203
+
1204
+ @staticmethod
1205
+ def _user_exists(cursor: oracledb.Cursor, username: str) -> bool:
1206
+ return OracleEngine._object_exists(cursor, "SELECT 1 FROM dba_users WHERE username = :1", [username])
1207
+
1208
+ @staticmethod
1209
+ def _get_database_default_tablespace(cursor: oracledb.Cursor) -> str | None:
1210
+ cursor.execute(
1211
+ "SELECT property_value FROM database_properties WHERE property_name = 'DEFAULT_PERMANENT_TABLESPACE'"
1212
+ )
1213
+ row = cursor.fetchone()
1214
+ if row and row[0]:
1215
+ return str(row[0])
1216
+ return None
1217
+
1218
+ @staticmethod
1219
+ def _get_user_default_tablespace(cursor: oracledb.Cursor, username: str) -> str | None:
1220
+ cursor.execute("SELECT default_tablespace FROM dba_users WHERE username = :username", username=username)
1221
+ row = cursor.fetchone()
1222
+ if row and row[0]:
1223
+ return str(row[0])
1224
+ return None
1225
+
1226
+ @staticmethod
1227
+ def _ensure_unlimited_quota(cursor: oracledb.Cursor, username: str, tablespace: str | None) -> bool:
1228
+ if not tablespace:
1229
+ return False
1230
+ tablespace_name = OracleEngine._format_identifier(tablespace)
1231
+ cursor.execute(
1232
+ """
1233
+ SELECT max_bytes
1234
+ FROM dba_ts_quotas
1235
+ WHERE username = :username
1236
+ AND tablespace_name = :tablespace
1237
+ """,
1238
+ username=username,
1239
+ tablespace=tablespace_name,
1240
+ )
1241
+ row = cursor.fetchone()
1242
+ if row and row[0] == -1:
1243
+ return False
1244
+ sql = f"ALTER USER {username} QUOTA UNLIMITED ON {tablespace_name}"
1245
+ OracleEngine._execute_ddl(cursor.connection, cursor, sql)
1246
+ return True
1247
+
1248
+ @staticmethod
1249
+ def _table_exists(cursor: oracledb.Cursor, owner: str, table_name: str) -> bool:
1250
+ return OracleEngine._object_exists(
1251
+ cursor,
1252
+ "SELECT 1 FROM dba_tables WHERE owner = :1 AND table_name = :2",
1253
+ [owner, table_name],
1254
+ )
1255
+
1256
+ @staticmethod
1257
+ def _index_exists(cursor: oracledb.Cursor, owner: str, index_name: str) -> bool:
1258
+ return OracleEngine._object_exists(
1259
+ cursor,
1260
+ "SELECT 1 FROM dba_indexes WHERE owner = :1 AND index_name = :2",
1261
+ [owner, index_name],
1262
+ )
1263
+
1264
+ @staticmethod
1265
+ def _constraint_exists(cursor: oracledb.Cursor, owner: str, constraint_name: str) -> bool:
1266
+ return OracleEngine._object_exists(
1267
+ cursor,
1268
+ "SELECT 1 FROM dba_constraints WHERE owner = :1 AND constraint_name = :2",
1269
+ [owner, constraint_name],
1270
+ )
1271
+
1272
+ @staticmethod
1273
+ def _sequence_exists(cursor: oracledb.Cursor, owner: str, sequence_name: str) -> bool:
1274
+ return OracleEngine._object_exists(
1275
+ cursor,
1276
+ "SELECT 1 FROM dba_sequences WHERE sequence_owner = :1 AND sequence_name = :2",
1277
+ [owner, sequence_name],
1278
+ )
1279
+
1280
+ @staticmethod
1281
+ def _register_connection_env(conn: oracledb.Connection, env: str | None) -> None:
1282
+ if env:
1283
+ OracleEngine._connection_envs[id(conn)] = env
1284
+ else:
1285
+ OracleEngine._connection_envs.pop(id(conn), None)
1286
+
1287
+ @staticmethod
1288
+ def _get_connection_env(conn: oracledb.Connection) -> str | None:
1289
+ return OracleEngine._connection_envs.get(id(conn))
1290
+
1291
+ @staticmethod
1292
+ def _execute_ddl(conn: oracledb.Connection, cursor: oracledb.Cursor, statement: str) -> None:
1293
+ log_statement = OracleEngine._PRIVATE_DDL_MARK.sub("****", statement)
1294
+ if len(log_statement) > 100:
1295
+ log_statement = log_statement[:100] + " ... [truncated]"
1296
+ statement = OracleEngine._PRIVATE_DDL_MARK.sub(lambda m: m.group(1), statement)
1297
+ logger.info("Executing %s DDL: %s", OracleEngine._get_connection_env(conn), " ".join(log_statement.split()))
1298
+ cursor.execute(statement)
1299
+
1300
+ @staticmethod
1301
+ def _db_link_exists(cursor: oracledb.Cursor, name: str) -> bool:
1302
+ return OracleEngine._object_exists(cursor, "SELECT 1 FROM user_db_links WHERE db_link = :1", [name])
1303
+
1304
+ @staticmethod
1305
+ def _get_primary_key_info(
1306
+ cursor: oracledb.Cursor, owner: str, table_name: str
1307
+ ) -> tuple[str | None, tuple[str, ...], str | None]:
1308
+ cursor.execute(
1309
+ """
1310
+ SELECT constraint_name, index_name
1311
+ FROM dba_constraints
1312
+ WHERE owner = :1
1313
+ AND table_name = :2
1314
+ AND constraint_type = 'P'
1315
+ """,
1316
+ [owner, table_name],
1317
+ )
1318
+ cursor.execute(
1319
+ """
1320
+ SELECT constraint_name, index_name
1321
+ FROM dba_constraints
1322
+ WHERE owner = :1
1323
+ AND table_name = :2
1324
+ AND constraint_type = 'P'
1325
+ """,
1326
+ [owner, table_name],
1327
+ )
1328
+ row = cursor.fetchone()
1329
+ if not row:
1330
+ return None, (), None
1331
+ constraint_name, index_name = row
1332
+ cursor.execute(
1333
+ "SELECT column_name FROM dba_cons_columns WHERE owner = :1 AND constraint_name = :2 ORDER BY position",
1334
+ [owner, constraint_name],
1335
+ )
1336
+ columns = tuple(r[0] for r in cast(Iterable[tuple[str]], cursor))
1337
+ return constraint_name, columns, index_name
1338
+
1339
+ @staticmethod
1340
+ def _column_needs_update(existing: ColumnDefinition, desired: ColumnDefinition) -> bool:
1341
+ desired_type = desired.data_type.lower()
1342
+ existing_type = existing.data_type.lower()
1343
+ if existing_type != desired_type:
1344
+ return True
1345
+ if desired_type in ("varchar2", "varchar", "char"):
1346
+ desired_length = desired.length or 0
1347
+ existing_length = existing.length or 0
1348
+ if desired_length and desired_length > existing_length:
1349
+ return True
1350
+ if desired_type in ("number", "numeric", "decimal"):
1351
+ desired_precision = desired.precision
1352
+ desired_scale = desired.scale or 0
1353
+ existing_precision = existing.precision
1354
+ existing_scale = existing.scale or 0
1355
+ if desired_precision is not None and existing_precision is not None:
1356
+ if existing_precision < desired_precision:
1357
+ return True
1358
+ if desired_precision is not None and existing_precision is None:
1359
+ # Existing column allows maximum precision; no change required.
1360
+ pass
1361
+ if desired_scale > existing_scale:
1362
+ return True
1363
+ if desired_type in ("integer", "int"):
1364
+ desired_precision = desired.precision or 10
1365
+ existing_precision = existing.precision or 0
1366
+ if desired_precision > existing_precision:
1367
+ return True
1368
+ return False
1369
+
1370
+ @staticmethod
1371
+ def _get_table_indexes(
1372
+ cursor: oracledb.Cursor, owner: str, table_name: str
1373
+ ) -> dict[str, tuple[tuple[str, ...], bool]]:
1374
+ cursor.execute(
1375
+ """
1376
+ SELECT i.index_name, i.uniqueness, c.column_name
1377
+ FROM dba_indexes i JOIN dba_ind_columns c ON i.owner = c.index_owner AND i.index_name = c.index_name WHERE i.owner = :owner
1378
+ AND i.table_name = :table_name
1379
+ ORDER BY i.index_name, c.column_position
1380
+ """,
1381
+ owner=owner,
1382
+ table_name=table_name,
1383
+ )
1384
+ indexes: dict[str, dict[str, Any]] = {}
1385
+ for index_name, uniqueness, column_name in cast(Iterable[tuple[str, str, str]], cursor):
1386
+ index_info = indexes.setdefault(index_name, {"columns": [], "unique": uniqueness == "UNIQUE"})
1387
+ index_info["columns"].append(column_name)
1388
+ return {name: (tuple(info["columns"]), bool(info["unique"])) for name, info in indexes.items()}
1389
+
1390
+ @staticmethod
1391
+ def _determine_process_date_column(table: TableDefinition) -> str | None:
1392
+ available_columns = {column.name for column in table.columns}
1393
+ for candidate in ("LDB_PROCESS_DATE", "LDB_PROCESS_DATE"):
1394
+ if candidate in available_columns:
1395
+ return candidate
1396
+ return None
1397
+
1398
+ @staticmethod
1399
+ def _prepare_desired_indexes(
1400
+ table: TableDefinition,
1401
+ ) -> dict[str, tuple[tuple[str, ...], bool]]:
1402
+ desired_pk = tuple(table.primary_key or ())
1403
+ process_date_column = OracleEngine._determine_process_date_column(table)
1404
+ desired_indexes: dict[str, tuple[tuple[str, ...], bool]] = {}
1405
+ for index in table.indexes:
1406
+ columns = tuple(index.columns)
1407
+ if process_date_column and process_date_column not in columns and (not desired_pk or columns != desired_pk):
1408
+ columns = (*columns, process_date_column)
1409
+ desired_indexes[index.name] = (columns, index.unique)
1410
+ return desired_indexes
1411
+
1412
+ @staticmethod
1413
+ def _choose_best_index_for_primary_key(
1414
+ indexes: IndexMap, column_metadata: ColumnMetadata
1415
+ ) -> tuple[str, ...] | None:
1416
+ candidates: list[dict[str, Any]] = []
1417
+ for index_name, info in indexes.items():
1418
+ columns: Sequence[str] = info.get("columns", [])
1419
+ if not columns:
1420
+ continue
1421
+ column_defs: list[ColumnDefinition] = []
1422
+ for col in columns:
1423
+ column_def = column_metadata.get(col)
1424
+ if column_def is None:
1425
+ column_defs = []
1426
+ break
1427
+ column_defs.append(column_def)
1428
+ if not column_defs:
1429
+ continue
1430
+ candidates.append(
1431
+ {
1432
+ "name": index_name,
1433
+ "columns": tuple(col.name for col in column_defs),
1434
+ "unique": bool(info.get("unique")),
1435
+ "distinct_keys": info.get("distinct_keys"),
1436
+ "all_not_null": all(not col.nullable for col in column_defs),
1437
+ }
1438
+ )
1439
+ if not candidates:
1440
+ return None
1441
+
1442
+ def score(value: Any) -> int:
1443
+ if value is None:
1444
+ return -1
1445
+ if isinstance(value, int):
1446
+ return value
1447
+ if isinstance(value, float):
1448
+ return int(value)
1449
+ try:
1450
+ return int(value)
1451
+ except (TypeError, ValueError):
1452
+ return -1
1453
+
1454
+ def choose(candidate_list: list[dict[str, Any]]) -> tuple[str, ...] | None:
1455
+ if not candidate_list:
1456
+ return None
1457
+ ordered = sorted(
1458
+ candidate_list,
1459
+ key=lambda item: (
1460
+ -score(item.get("distinct_keys")),
1461
+ len(item["columns"]),
1462
+ item["name"],
1463
+ ),
1464
+ )
1465
+ best = ordered[0]
1466
+ return tuple(best["columns"])
1467
+
1468
+ not_null_unique = [c for c in candidates if c["unique"] and c["all_not_null"]]
1469
+ choice = choose(not_null_unique)
1470
+ if choice:
1471
+ return choice
1472
+ unique_candidates = [c for c in candidates if c["unique"]]
1473
+ choice = choose(unique_candidates)
1474
+ if choice:
1475
+ return choice
1476
+ non_unique = [c for c in candidates if not c["unique"]]
1477
+ return choose(non_unique)
1478
+
1479
+ @staticmethod
1480
+ def _drop_constraints_by_type(
1481
+ cursor: oracledb.Cursor, owner: str, table_name: str, constraint_types: Sequence[str]
1482
+ ) -> list[str]:
1483
+ dropped: list[str] = []
1484
+ if not constraint_types:
1485
+ return dropped
1486
+ type_list = ", ".join(f"'{constraint_type}'" for constraint_type in constraint_types)
1487
+ cursor.execute(
1488
+ f"""
1489
+ SELECT constraint_name
1490
+ FROM dba_constraints
1491
+ WHERE owner = :owner
1492
+ AND table_name = :table_name
1493
+ AND constraint_type IN ({type_list})
1494
+ """,
1495
+ owner=owner,
1496
+ table_name=table_name,
1497
+ )
1498
+ fmttd_owner = OracleEngine._format_identifier(owner)
1499
+ fmttd_table_name = OracleEngine._format_identifier(table_name)
1500
+ for (constraint_name,) in cast(Iterable[tuple[str]], cursor):
1501
+ fmttd_constraint_name = OracleEngine._format_identifier(constraint_name)
1502
+ sql = f"ALTER TABLE {fmttd_owner}.{fmttd_table_name} DROP CONSTRAINT {fmttd_constraint_name}"
1503
+ OracleEngine._execute_ddl(cursor.connection, cursor, sql)
1504
+ dropped.append(fmttd_constraint_name)
1505
+ return dropped