al-data-utils 0.3.1__py3-none-any.whl

al_data_utils/backup_utils/mssql/schema_snapshot/introspect/views.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+from collections import defaultdict
+import re
+
+from al_data_utils.sql_utils.sql_utils import UnifiedSQLClient
+from ..models import SqlObjectDef
+
+
+_SQL_VIEWS = r"""
+select
+    s.name as schema_name,
+    v.name as object_name,
+    m.definition as definition
+from sys.views v
+inner join sys.schemas s
+  on s.schema_id = v.schema_id
+inner join sys.sql_modules m
+  on m.object_id = v.object_id
+where v.is_ms_shipped = 0
+order by s.name, v.name;
+"""
+
+
+_SQL_VIEW_PERMISSIONS = r"""
+select
+    s.name as schema_name,
+    v.name as object_name,
+    dp.permission_name,
+    dp.state,
+    grantee.name as grantee_name,
+    c.name as column_name,
+    cast(case when dp.minor_id > 0 then 1 else 0 end as bit) as is_column_level
+from sys.database_permissions dp
+inner join sys.views v
+  on v.object_id = dp.major_id
+inner join sys.schemas s
+  on s.schema_id = v.schema_id
+inner join sys.database_principals grantee
+  on grantee.principal_id = dp.grantee_principal_id
+left outer join sys.columns c
+  on c.object_id = dp.major_id
+ and c.column_id = dp.minor_id
+where dp.class = 1
+  and v.is_ms_shipped = 0
+  and dp.state in ('G', 'W', 'D')
+order by s.name, v.name, dp.permission_name, dp.state, grantee.name, c.column_id;
+"""
+
+
+def _normalized_set(values: set[str] | None) -> set[str]:
+    if not values:
+        return set()
+    return {str(v).strip().lower() for v in values if str(v).strip()}
+
+
+def _split_qualified(values: set[str]) -> list[tuple[str, str]]:
+    pairs: list[tuple[str, str]] = []
+    for v in sorted(values):
+        if '.' not in v:
+            continue
+        schema, name = v.split('.', 1)
+        schema = schema.strip().lower()
+        name = name.strip().lower()
+        if schema and name:
+            pairs.append((schema, name))
+    return pairs
+
+
+def _build_view_filter_sql_and_params(
+    *,
+    include_schemas: set[str] | None = None,
+    exclude_schemas: set[str] | None = None,
+    include_views: set[str] | None = None,
+    exclude_views: set[str] | None = None,
+) -> tuple[str, dict[str, str]]:
+    inc_schemas = sorted(_normalized_set(include_schemas))
+    exc_schemas = sorted(_normalized_set(exclude_schemas))
+    inc_views = _split_qualified(_normalized_set(include_views))
+    exc_views = _split_qualified(_normalized_set(exclude_views))
+
+    params: dict[str, str] = {}
+    clauses: list[str] = []
+
+    if inc_schemas:
+        names: list[str] = []
+        for i, schema in enumerate(inc_schemas):
+            key = f'inc_schema_{i}'
+            params[key] = schema
+            names.append(f':{key}')
+        clauses.append(f'lower(s.name) in ({", ".join(names)})')
+
+    if exc_schemas:
+        names = []
+        for i, schema in enumerate(exc_schemas):
+            key = f'exc_schema_{i}'
+            params[key] = schema
+            names.append(f':{key}')
+        clauses.append(f'lower(s.name) not in ({", ".join(names)})')
+
+    if inc_views:
+        parts: list[str] = []
+        for i, (schema, name) in enumerate(inc_views):
+            sk = f'inc_view_schema_{i}'
+            nk = f'inc_view_name_{i}'
+            params[sk] = schema
+            params[nk] = name
+            parts.append(f'(lower(s.name) = :{sk} and lower(v.name) = :{nk})')
+        clauses.append(f'({" or ".join(parts)})')
+
+    if exc_views:
+        parts = []
+        for i, (schema, name) in enumerate(exc_views):
+            sk = f'exc_view_schema_{i}'
+            nk = f'exc_view_name_{i}'
+            params[sk] = schema
+            params[nk] = name
+            parts.append(f'(lower(s.name) = :{sk} and lower(v.name) = :{nk})')
+        clauses.append(f'not ({" or ".join(parts)})')
+
+    if not clauses:
+        return '', {}
+
+    return '\n  and ' + '\n  and '.join(clauses), params
+
+
+def _inject_view_filters(sql: str, filters_sql: str) -> str:
+    if not filters_sql:
+        return sql
+    return sql.replace(
+        'where v.is_ms_shipped = 0', f'where v.is_ms_shipped = 0{filters_sql}'
+    )
+
+
+def _q(name: str) -> str:
+    n = name.strip()
+    if re.match(r'^[A-Za-z_][A-Za-z0-9_]*$', n):
+        return n
+    return '[' + n.replace(']', ']]') + ']'
+
+
+def _build_permission_statements(
+    sql_client: UnifiedSQLClient,
+    *,
+    filters_sql: str = '',
+    params: dict[str, str] | None = None,
+) -> dict[tuple[str, str], list[str]]:
+    rows = sql_client.execute_command_with_result_to_dicts(
+        _inject_view_filters(_SQL_VIEW_PERMISSIONS, filters_sql), params or {}
+    )
+
+    object_perms: dict[tuple[str, str], list[str]] = defaultdict(list)
+    column_perms: dict[tuple[str, str], list[str]] = defaultdict(list)
+
+    # key: (schema, object, state, perm, grantee)
+    col_group: dict[tuple[str, str, str, str, str], list[str]] = defaultdict(list)
+
+    for r in rows:
+        schema_name = str(r['schema_name'])
+        object_name = str(r['object_name'])
+        permission_name = str(r['permission_name']).lower()
+        state = str(r['state']).upper()
+        grantee = str(r['grantee_name'])
+        is_column_level = bool(r['is_column_level'])
+        column_name = (
+            None if r.get('column_name') is None else str(r.get('column_name'))
+        )
+
+        target = f'{_q(schema_name)}.{_q(object_name)}'
+        grantee_sql = _q(grantee)
+
+        verb = 'grant' if state in {'G', 'W'} else 'deny'
+        with_grant = ' with grant option' if state == 'W' else ''
+
+        if is_column_level and column_name:
+            group_key = (schema_name, object_name, state, permission_name, grantee)
+            col_group[group_key].append(column_name)
+        else:
+            object_perms[(schema_name, object_name)].append(
+                f'{verb} {permission_name} on {target} to {grantee_sql}{with_grant}'
+            )
+
+    for (schema_name, object_name, state, permission_name, grantee), cols in sorted(
+        col_group.items()
+    ):
+        target = f'{_q(schema_name)}.{_q(object_name)}'
+        grantee_sql = _q(grantee)
+        verb = 'grant' if state in {'G', 'W'} else 'deny'
+        with_grant = ' with grant option' if state == 'W' else ''
+        seen_cols: set[str] = set()
+        ordered_cols: list[str] = []
+        for c in cols:
+            lc = c.lower()
+            if lc in seen_cols:
+                continue
+            seen_cols.add(lc)
+            ordered_cols.append(c)
+
+        col_list = ', '.join(_q(c) for c in ordered_cols)
+        column_perms[(schema_name, object_name)].append(
+            f'{verb} {permission_name} ({col_list}) on {target} to {grantee_sql}{with_grant}'
+        )
+
+    merged: dict[tuple[str, str], list[str]] = defaultdict(list)
+    keys = set(object_perms.keys()) | set(column_perms.keys())
+    for k in keys:
+        merged[k].extend(object_perms.get(k, []))
+        merged[k].extend(column_perms.get(k, []))
+
+    return merged
+
+
+def get_views(
+    sql_client: UnifiedSQLClient,
+    *,
+    include_schemas: set[str] | None = None,
+    exclude_schemas: set[str] | None = None,
+    include_views: set[str] | None = None,
+    exclude_views: set[str] | None = None,
+) -> list[SqlObjectDef]:
+    filters_sql, params = _build_view_filter_sql_and_params(
+        include_schemas=include_schemas,
+        exclude_schemas=exclude_schemas,
+        include_views=include_views,
+        exclude_views=exclude_views,
+    )
+
+    rows = sql_client.execute_command_with_result_to_dicts(
+        _inject_view_filters(_SQL_VIEWS, filters_sql), params
+    )
+    perms_by_view = _build_permission_statements(
+        sql_client, filters_sql=filters_sql, params=params
+    )
+    return [
+        SqlObjectDef(
+            schema_name=str(r['schema_name']),
+            object_name=str(r['object_name']),
+            definition=str(r['definition'] or ''),
+            permissions_sql=perms_by_view.get(
+                (str(r['schema_name']), str(r['object_name'])), []
+            ),
+        )
+        for r in rows
+    ]

al_data_utils/backup_utils/mssql/schema_snapshot/io/__init__.py

@@ -0,0 +1,10 @@
+from .layout import table_file_path, object_file_path
+from .writer import write_text_atomic, set_mtime_like, read_text_if_exists
+
+__all__ = [
+    'table_file_path',
+    'object_file_path',
+    'write_text_atomic',
+    'set_mtime_like',
+    'read_text_if_exists',
+]

al_data_utils/backup_utils/mssql/schema_snapshot/io/layout.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def object_file_path(
+    *,
+    root: Path,
+    root_folder_name: str,
+    schema: str,
+    object_folder: str,
+    object_name: str,
+) -> Path:
+    return (
+        root / root_folder_name / schema / object_folder / f'{schema}.{object_name}.sql'
+    )
+
+
+def table_file_path(
+    *, root: Path, root_folder_name: str, schema: str, table: str
+) -> Path:
+    return object_file_path(
+        root=root,
+        root_folder_name=root_folder_name,
+        schema=schema,
+        object_folder='tables',
+        object_name=table,
+    )

al_data_utils/backup_utils/mssql/schema_snapshot/io/writer.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def _normalize_text(text: str, newline: str) -> str:
+    t = text.replace("\r\n", "\n").replace("\r", "\n")
+    if newline != "\n":
+        t = t.replace("\n", newline)
+    if not t.endswith(newline):
+        t += newline
+    return t
+
+
+def write_text_atomic(path: Path, text: str, *, encoding: str, newline: str) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    data = _normalize_text(text, newline)
+
+    with open(tmp, "w", encoding=encoding, newline="") as f:
+        f.write(data)
+
+    os.replace(tmp, path)
+
+
+def set_mtime_like(path: Path, like_path: Path) -> None:
+    try:
+        st = like_path.stat()
+        m = int(st.st_mtime)
+        os.utime(path, (m, m))
+    except FileNotFoundError:
+        return
+
+
+def read_text_if_exists(path: Path, *, encoding: str) -> str | None:
+    try:
+        return path.read_text(encoding=encoding)
+    except FileNotFoundError:
+        return None

al_data_utils/backup_utils/mssql/schema_snapshot/models/__init__.py

@@ -0,0 +1,14 @@
+from .index_def import IndexDef
+from .foreign_key_def import ForeignKeyDef
+from .table_column_def import TableColumnDef
+from .table_def import TableDef, PrimaryKeyDef
+from .sql_object_def import SqlObjectDef
+
+__all__ = [
+    'IndexDef',
+    'ForeignKeyDef',
+    'TableColumnDef',
+    'TableDef',
+    'PrimaryKeyDef',
+    'SqlObjectDef',
+]

al_data_utils/backup_utils/mssql/schema_snapshot/models/foreign_key_def.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ForeignKeyDef:
+    parent_schema: str
+    parent_table: str
+    fk_name: str
+    referenced_schema: str
+    referenced_table: str
+    on_delete: str
+    on_update: str
+    is_not_for_replication: bool
+    is_disabled: bool
+    is_not_trusted: bool
+    parent_columns: str        # formatted: [col1], [col2]
+    referenced_columns: str    # formatted: [col1], [col2]

al_data_utils/backup_utils/mssql/schema_snapshot/models/index_def.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class IndexDef:
+    schema_name: str
+    table_name: str
+    index_name: str
+    is_unique: bool
+    type_desc: str  # CLUSTERED / NONCLUSTERED
+    has_filter: bool
+    filter_definition: Optional[str]
+    is_disabled: bool
+    is_hypothetical: bool
+    key_columns: str           # formatted: [col] asc, [col2] desc
+    include_columns: str       # formatted: [col], [col2]

al_data_utils/backup_utils/mssql/schema_snapshot/models/sql_object_def.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True)
+class SqlObjectDef:
+    schema_name: str
+    object_name: str
+    definition: str
+    permissions_sql: list[str] = field(default_factory=list)

al_data_utils/backup_utils/mssql/schema_snapshot/models/table_column_def.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class TableColumnDef:
+    schema_name: str
+    table_name: str
+    column_name: str
+    ordinal_position: int
+
+    data_type: str
+    max_length: int | None
+    precision: int | None
+    scale: int | None
+
+    is_nullable: bool
+    is_identity: bool
+    identity_seed: int | None
+    identity_increment: int | None
+
+    is_computed: bool
+    computed_definition: str | None
+    collation_name: str | None
+    default_constraint_name: str | None
+    default_definition: str | None

al_data_utils/backup_utils/mssql/schema_snapshot/models/table_def.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .table_column_def import TableColumnDef
+
+
+@dataclass(frozen=True)
+class PrimaryKeyDef:
+    schema_name: str
+    table_name: str
+    constraint_name: str
+    key_columns: str
+    type_desc: str
+
+
+@dataclass(frozen=True)
+class TableDef:
+    schema_name: str
+    table_name: str
+    columns: list[TableColumnDef]
+    primary_key: PrimaryKeyDef | None = None