al-data-utils 0.3.1__py3-none-any.whl

al_data_utils/backup_utils/mssql/schema_snapshot/introspect/functions.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+from al_data_utils.sql_utils.sql_utils import UnifiedSQLClient
+from ..models import SqlObjectDef
+
+
+_SQL_FUNCTIONS = r"""
+select
+    s.name as schema_name,
+    o.name as object_name,
+    convert(nvarchar(max), m.definition) as definition
+from sys.objects o
+inner join sys.schemas s
+  on s.schema_id = o.schema_id
+inner join sys.sql_modules m
+  on m.object_id = o.object_id
+where o.is_ms_shipped = 0
+  and o.type in ('FN', 'IF', 'TF', 'FS', 'FT')
+order by s.name, o.name;
+"""
+
+
+def _normalized_set(values: set[str] | None) -> set[str]:
+    if not values:
+        return set()
+    return {str(v).strip().lower() for v in values if str(v).strip()}
+
+
+def _split_qualified(values: set[str]) -> list[tuple[str, str]]:
+    pairs: list[tuple[str, str]] = []
+    for v in sorted(values):
+        if '.' not in v:
+            continue
+        schema, name = v.split('.', 1)
+        schema = schema.strip().lower()
+        name = name.strip().lower()
+        if schema and name:
+            pairs.append((schema, name))
+    return pairs
+
+
+def _build_filter_sql_and_params(
+    *,
+    include_schemas: set[str] | None = None,
+    exclude_schemas: set[str] | None = None,
+    include_objects: set[str] | None = None,
+    exclude_objects: set[str] | None = None,
+) -> tuple[str, dict[str, str]]:
+    inc_schemas = sorted(_normalized_set(include_schemas))
+    exc_schemas = sorted(_normalized_set(exclude_schemas))
+    inc_objects = _split_qualified(_normalized_set(include_objects))
+    exc_objects = _split_qualified(_normalized_set(exclude_objects))
+
+    params: dict[str, str] = {}
+    clauses: list[str] = []
+
+    if inc_schemas:
+        names: list[str] = []
+        for i, schema in enumerate(inc_schemas):
+            key = f'inc_schema_{i}'
+            params[key] = schema
+            names.append(f':{key}')
+        clauses.append(f'lower(s.name) in ({", ".join(names)})')
+
+    if exc_schemas:
+        names = []
+        for i, schema in enumerate(exc_schemas):
+            key = f'exc_schema_{i}'
+            params[key] = schema
+            names.append(f':{key}')
+        clauses.append(f'lower(s.name) not in ({", ".join(names)})')
+
+    if inc_objects:
+        parts: list[str] = []
+        for i, (schema, name) in enumerate(inc_objects):
+            sk = f'inc_obj_schema_{i}'
+            nk = f'inc_obj_name_{i}'
+            params[sk] = schema
+            params[nk] = name
+            parts.append(f'(lower(s.name) = :{sk} and lower(o.name) = :{nk})')
+        clauses.append(f'({" or ".join(parts)})')
+
+    if exc_objects:
+        parts = []
+        for i, (schema, name) in enumerate(exc_objects):
+            sk = f'exc_obj_schema_{i}'
+            nk = f'exc_obj_name_{i}'
+            params[sk] = schema
+            params[nk] = name
+            parts.append(f'(lower(s.name) = :{sk} and lower(o.name) = :{nk})')
+        clauses.append(f'not ({" or ".join(parts)})')
+
+    if not clauses:
+        return '', {}
+
+    return '\n  and ' + '\n  and '.join(clauses), params
+
+
+def _inject_filters(sql: str, filters_sql: str) -> str:
+    if not filters_sql:
+        return sql
+    return sql.replace(
+        'where o.is_ms_shipped = 0', f'where o.is_ms_shipped = 0{filters_sql}'
+    )
+
+
+def get_functions(
+    sql_client: UnifiedSQLClient,
+    *,
+    include_schemas: set[str] | None = None,
+    exclude_schemas: set[str] | None = None,
+    include_objects: set[str] | None = None,
+    exclude_objects: set[str] | None = None,
+) -> list[SqlObjectDef]:
+    filters_sql, params = _build_filter_sql_and_params(
+        include_schemas=include_schemas,
+        exclude_schemas=exclude_schemas,
+        include_objects=include_objects,
+        exclude_objects=exclude_objects,
+    )
+
+    rows = sql_client.execute_command_with_result_to_dicts(
+        _inject_filters(_SQL_FUNCTIONS, filters_sql), params
+    )
+
+    return [
+        SqlObjectDef(
+            schema_name=str(r['schema_name']),
+            object_name=str(r['object_name']),
+            definition=str(r['definition'] or ''),
+        )
+        for r in rows
+    ]

al_data_utils/backup_utils/mssql/schema_snapshot/introspect/indexes.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+from al_data_utils.sql_utils.sql_utils import UnifiedSQLClient
+from ..models import IndexDef
+
+
+_SQL_INDEXES = r"""
+with idx as (
+    select
+        s.name  as schema_name,
+        t.name  as table_name,
+        i.name  as index_name,
+        i.index_id,
+        i.is_unique,
+        i.type_desc,
+        i.has_filter,
+        i.filter_definition,
+        i.is_disabled,
+        i.is_hypothetical
+    from sys.indexes i
+    inner join sys.tables t on t.object_id = i.object_id
+    inner join sys.schemas s on s.schema_id = t.schema_id
+    where i.type_desc <> 'heap'
+      and i.name is not null
+            and i.is_primary_key = 0
+            and i.is_unique_constraint = 0
+      and t.is_ms_shipped = 0
+)
+select
+    idx.schema_name,
+    idx.table_name,
+    idx.index_name,
+    cast(idx.is_unique as bit) as is_unique,
+    idx.type_desc,
+    cast(idx.has_filter as bit) as has_filter,
+    idx.filter_definition,
+    cast(idx.is_disabled as bit) as is_disabled,
+    cast(idx.is_hypothetical as bit) as is_hypothetical,
+
+    stuff((
+        select
+            ', ' + quotename(c.name) + case when ic.is_descending_key = 1 then ' desc' else ' asc' end
+        from sys.index_columns ic
+        inner join sys.columns c
+          on c.object_id = ic.object_id and c.column_id = ic.column_id
+        where ic.object_id = object_id(quotename(idx.schema_name) + '.' + quotename(idx.table_name))
+          and ic.index_id  = idx.index_id
+          and ic.is_included_column = 0
+        order by ic.key_ordinal
+        for xml path(''), type
+    ).value('.', 'nvarchar(max)'), 1, 2, '') as key_columns,
+
+    stuff((
+        select
+            ', ' + quotename(c.name)
+        from sys.index_columns ic
+        inner join sys.columns c
+          on c.object_id = ic.object_id and c.column_id = ic.column_id
+        where ic.object_id = object_id(quotename(idx.schema_name) + '.' + quotename(idx.table_name))
+          and ic.index_id  = idx.index_id
+          and ic.is_included_column = 1
+        order by ic.index_column_id
+        for xml path(''), type
+    ).value('.', 'nvarchar(max)'), 1, 2, '') as include_columns
+from idx
+order by idx.schema_name, idx.table_name, idx.index_name;
+"""
+
+
+def _normalized_set(values: set[str] | None) -> set[str]:
+    if not values:
+        return set()
+    return {str(v).strip().lower() for v in values if str(v).strip()}
+
+
+def _split_qualified(values: set[str]) -> list[tuple[str, str]]:
+    pairs: list[tuple[str, str]] = []
+    for v in sorted(values):
+        if '.' not in v:
+            continue
+        schema, table = v.split('.', 1)
+        schema = schema.strip().lower()
+        table = table.strip().lower()
+        if schema and table:
+            pairs.append((schema, table))
+    return pairs
+
+
+def _build_filter_sql_and_params(
+    *,
+    include_schemas: set[str] | None = None,
+    exclude_schemas: set[str] | None = None,
+    include_tables: set[str] | None = None,
+    exclude_tables: set[str] | None = None,
+) -> tuple[str, dict[str, str]]:
+    inc_schemas = sorted(_normalized_set(include_schemas))
+    exc_schemas = sorted(_normalized_set(exclude_schemas))
+    inc_tables = _split_qualified(_normalized_set(include_tables))
+    exc_tables = _split_qualified(_normalized_set(exclude_tables))
+
+    params: dict[str, str] = {}
+    clauses: list[str] = []
+
+    if inc_schemas:
+        names: list[str] = []
+        for i, schema in enumerate(inc_schemas):
+            key = f'inc_schema_{i}'
+            params[key] = schema
+            names.append(f':{key}')
+        clauses.append(f'lower(s.name) in ({", ".join(names)})')
+
+    if exc_schemas:
+        names = []
+        for i, schema in enumerate(exc_schemas):
+            key = f'exc_schema_{i}'
+            params[key] = schema
+            names.append(f':{key}')
+        clauses.append(f'lower(s.name) not in ({", ".join(names)})')
+
+    if inc_tables:
+        parts: list[str] = []
+        for i, (schema, table) in enumerate(inc_tables):
+            sk = f'inc_tbl_schema_{i}'
+            tk = f'inc_tbl_name_{i}'
+            params[sk] = schema
+            params[tk] = table
+            parts.append(f'(lower(s.name) = :{sk} and lower(t.name) = :{tk})')
+        clauses.append(f'({" or ".join(parts)})')
+
+    if exc_tables:
+        parts = []
+        for i, (schema, table) in enumerate(exc_tables):
+            sk = f'exc_tbl_schema_{i}'
+            tk = f'exc_tbl_name_{i}'
+            params[sk] = schema
+            params[tk] = table
+            parts.append(f'(lower(s.name) = :{sk} and lower(t.name) = :{tk})')
+        clauses.append(f'not ({" or ".join(parts)})')
+
+    if not clauses:
+        return '', {}
+
+    return '\n      and ' + '\n      and '.join(clauses), params
+
+
+def _inject_filters(sql: str, filters_sql: str) -> str:
+    if not filters_sql:
+        return sql
+    return sql.replace(
+        'and t.is_ms_shipped = 0', f'and t.is_ms_shipped = 0{filters_sql}'
+    )
+
+
+def get_indexes(
+    sql_client: UnifiedSQLClient,
+    *,
+    include_schemas: set[str] | None = None,
+    exclude_schemas: set[str] | None = None,
+    include_tables: set[str] | None = None,
+    exclude_tables: set[str] | None = None,
+) -> list[IndexDef]:
+    filters_sql, params = _build_filter_sql_and_params(
+        include_schemas=include_schemas,
+        exclude_schemas=exclude_schemas,
+        include_tables=include_tables,
+        exclude_tables=exclude_tables,
+    )
+    rows = sql_client.execute_command_with_result_to_dicts(
+        _inject_filters(_SQL_INDEXES, filters_sql), params
+    )
+    result: list[IndexDef] = []
+    for r in rows:
+        result.append(
+            IndexDef(
+                schema_name=str(r['schema_name']),
+                table_name=str(r['table_name']),
+                index_name=str(r['index_name']),
+                is_unique=bool(r['is_unique']),
+                type_desc=str(r['type_desc']),
+                has_filter=bool(r['has_filter']),
+                filter_definition=None
+                if r['filter_definition'] is None
+                else str(r['filter_definition']),
+                is_disabled=bool(r['is_disabled']),
+                is_hypothetical=bool(r['is_hypothetical']),
+                key_columns=str(r['key_columns'] or ''),
+                include_columns=str(r['include_columns'] or ''),
+            )
+        )
+    return result

al_data_utils/backup_utils/mssql/schema_snapshot/introspect/sequences.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from al_data_utils.sql_utils.sql_utils import UnifiedSQLClient
+from ..models import SqlObjectDef
+
+
+_SQL_SEQUENCES = r"""
+select
+    s.name as schema_name,
+    seq.name as object_name,
+    ty.name as data_type,
+    convert(nvarchar(100), seq.start_value) as start_value,
+    convert(nvarchar(100), seq.increment) as increment_value,
+    convert(nvarchar(100), seq.minimum_value) as minimum_value,
+    convert(nvarchar(100), seq.maximum_value) as maximum_value,
+    cast(seq.is_cycling as bit) as is_cycling,
+    cast(seq.is_cached as bit) as is_cached,
+    cast(seq.cache_size as bigint) as cache_size
+from sys.sequences seq
+inner join sys.schemas s
+  on s.schema_id = seq.schema_id
+inner join sys.types ty
+  on ty.user_type_id = seq.user_type_id
+order by s.name, seq.name;
+"""
+
+
+def _normalized_set(values: set[str] | None) -> set[str]:
+    if not values:
+        return set()
+    return {str(v).strip().lower() for v in values if str(v).strip()}
+
+
+def _split_qualified(values: set[str]) -> list[tuple[str, str]]:
+    pairs: list[tuple[str, str]] = []
+    for v in sorted(values):
+        if '.' not in v:
+            continue
+        schema, name = v.split('.', 1)
+        schema = schema.strip().lower()
+        name = name.strip().lower()
+        if schema and name:
+            pairs.append((schema, name))
+    return pairs
+
+
+def _build_filter_sql_and_params(
+    *,
+    include_schemas: set[str] | None = None,
+    exclude_schemas: set[str] | None = None,
+    include_objects: set[str] | None = None,
+    exclude_objects: set[str] | None = None,
+) -> tuple[str, dict[str, str]]:
+    inc_schemas = sorted(_normalized_set(include_schemas))
+    exc_schemas = sorted(_normalized_set(exclude_schemas))
+    inc_objects = _split_qualified(_normalized_set(include_objects))
+    exc_objects = _split_qualified(_normalized_set(exclude_objects))
+
+    params: dict[str, str] = {}
+    clauses: list[str] = []
+
+    if inc_schemas:
+        names: list[str] = []
+        for i, schema in enumerate(inc_schemas):
+            key = f'inc_schema_{i}'
+            params[key] = schema
+            names.append(f':{key}')
+        clauses.append(f'lower(s.name) in ({", ".join(names)})')
+
+    if exc_schemas:
+        names = []
+        for i, schema in enumerate(exc_schemas):
+            key = f'exc_schema_{i}'
+            params[key] = schema
+            names.append(f':{key}')
+        clauses.append(f'lower(s.name) not in ({", ".join(names)})')
+
+    if inc_objects:
+        parts: list[str] = []
+        for i, (schema, name) in enumerate(inc_objects):
+            sk = f'inc_obj_schema_{i}'
+            nk = f'inc_obj_name_{i}'
+            params[sk] = schema
+            params[nk] = name
+            parts.append(f'(lower(s.name) = :{sk} and lower(seq.name) = :{nk})')
+        clauses.append(f'({" or ".join(parts)})')
+
+    if exc_objects:
+        parts = []
+        for i, (schema, name) in enumerate(exc_objects):
+            sk = f'exc_obj_schema_{i}'
+            nk = f'exc_obj_name_{i}'
+            params[sk] = schema
+            params[nk] = name
+            parts.append(f'(lower(s.name) = :{sk} and lower(seq.name) = :{nk})')
+        clauses.append(f'not ({" or ".join(parts)})')
+
+    if not clauses:
+        return '', {}
+
+    return '\nwhere ' + '\n  and '.join(clauses), params
+
+
+def _inject_filters(sql: str, filters_sql: str) -> str:
+    if not filters_sql:
+        return sql
+    return sql.replace(
+        'order by s.name, seq.name;', f'{filters_sql}\norder by s.name, seq.name;'
+    )
+
+
+def _emit_sequence_ddl(r: Mapping[str, object]) -> str:
+    schema_name = str(r['schema_name'])
+    object_name = str(r['object_name'])
+    data_type = str(r['data_type'])
+    start_value = str(r.get('start_value') or '1')
+    increment_value = str(r.get('increment_value') or '1')
+    minimum_value = str(r.get('minimum_value') or '')
+    maximum_value = str(r.get('maximum_value') or '')
+    is_cycling = bool(r.get('is_cycling'))
+    is_cached = bool(r.get('is_cached'))
+    cache_size_raw = r.get('cache_size')
+
+    parts: list[str] = [
+        f'create sequence {schema_name}.{object_name}',
+        f'    as {data_type}',
+        f'    start with {start_value}',
+        f'    increment by {increment_value}',
+    ]
+
+    if minimum_value:
+        parts.append(f'    minvalue {minimum_value}')
+    else:
+        parts.append('    no minvalue')
+
+    if maximum_value:
+        parts.append(f'    maxvalue {maximum_value}')
+    else:
+        parts.append('    no maxvalue')
+
+    parts.append('    cycle' if is_cycling else '    no cycle')
+
+    if is_cached and cache_size_raw is not None:
+        cache_size = int(str(cache_size_raw))
+        parts.append(f'    cache {cache_size}')
+    else:
+        parts.append('    no cache')
+
+    return '\n'.join(parts)
+
+
+def get_sequences(
+    sql_client: UnifiedSQLClient,
+    *,
+    include_schemas: set[str] | None = None,
+    exclude_schemas: set[str] | None = None,
+    include_objects: set[str] | None = None,
+    exclude_objects: set[str] | None = None,
+) -> list[SqlObjectDef]:
+    filters_sql, params = _build_filter_sql_and_params(
+        include_schemas=include_schemas,
+        exclude_schemas=exclude_schemas,
+        include_objects=include_objects,
+        exclude_objects=exclude_objects,
+    )
+
+    rows = sql_client.execute_command_with_result_to_dicts(
+        _inject_filters(_SQL_SEQUENCES, filters_sql), params
+    )
+
+    return [
+        SqlObjectDef(
+            schema_name=str(r['schema_name']),
+            object_name=str(r['object_name']),
+            definition=_emit_sequence_ddl(r),
+        )
+        for r in rows
+    ]