al-data-utils 0.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. al_data_utils/__init__.py +10 -0
  2. al_data_utils/airflow_utils/__init__.py +13 -0
  3. al_data_utils/airflow_utils/airflow_utils.py +83 -0
  4. al_data_utils/azure_as_utils/__init__.py +31 -0
  5. al_data_utils/azure_as_utils/auth.py +82 -0
  6. al_data_utils/azure_as_utils/azure_as_utils.py +223 -0
  7. al_data_utils/azure_as_utils/client.py +158 -0
  8. al_data_utils/azure_as_utils/errors.py +78 -0
  9. al_data_utils/azure_as_utils/models.py +265 -0
  10. al_data_utils/azure_as_utils/poller.py +121 -0
  11. al_data_utils/backup_utils/__init__.py +9 -0
  12. al_data_utils/backup_utils/mssql/dacpac/__init__.py +1 -0
  13. al_data_utils/backup_utils/mssql/dacpac/sqlpackage.py +207 -0
  14. al_data_utils/backup_utils/mssql/schema_snapshot/__init__.py +8 -0
  15. al_data_utils/backup_utils/mssql/schema_snapshot/config.py +150 -0
  16. al_data_utils/backup_utils/mssql/schema_snapshot/emit/__init__.py +11 -0
  17. al_data_utils/backup_utils/mssql/schema_snapshot/emit/foreign_keys.py +38 -0
  18. al_data_utils/backup_utils/mssql/schema_snapshot/emit/indexes.py +37 -0
  19. al_data_utils/backup_utils/mssql/schema_snapshot/emit/sql_objects.py +17 -0
  20. al_data_utils/backup_utils/mssql/schema_snapshot/emit/tables.py +139 -0
  21. al_data_utils/backup_utils/mssql/schema_snapshot/introspect/__init__.py +19 -0
  22. al_data_utils/backup_utils/mssql/schema_snapshot/introspect/foreign_keys.py +173 -0
  23. al_data_utils/backup_utils/mssql/schema_snapshot/introspect/functions.py +133 -0
  24. al_data_utils/backup_utils/mssql/schema_snapshot/introspect/indexes.py +190 -0
  25. al_data_utils/backup_utils/mssql/schema_snapshot/introspect/sequences.py +180 -0
  26. al_data_utils/backup_utils/mssql/schema_snapshot/introspect/stored_procedures.py +313 -0
  27. al_data_utils/backup_utils/mssql/schema_snapshot/introspect/synonyms.py +135 -0
  28. al_data_utils/backup_utils/mssql/schema_snapshot/introspect/tables.py +251 -0
  29. al_data_utils/backup_utils/mssql/schema_snapshot/introspect/views.py +244 -0
  30. al_data_utils/backup_utils/mssql/schema_snapshot/io/__init__.py +10 -0
  31. al_data_utils/backup_utils/mssql/schema_snapshot/io/layout.py +28 -0
  32. al_data_utils/backup_utils/mssql/schema_snapshot/io/writer.py +40 -0
  33. al_data_utils/backup_utils/mssql/schema_snapshot/models/__init__.py +14 -0
  34. al_data_utils/backup_utils/mssql/schema_snapshot/models/foreign_key_def.py +19 -0
  35. al_data_utils/backup_utils/mssql/schema_snapshot/models/index_def.py +19 -0
  36. al_data_utils/backup_utils/mssql/schema_snapshot/models/sql_object_def.py +11 -0
  37. al_data_utils/backup_utils/mssql/schema_snapshot/models/table_column_def.py +27 -0
  38. al_data_utils/backup_utils/mssql/schema_snapshot/models/table_def.py +22 -0
  39. al_data_utils/backup_utils/mssql/schema_snapshot/snapshot_processor.py +754 -0
  40. al_data_utils/backup_utils/mssql/scripter/__init__.py +1 -0
  41. al_data_utils/config_utils/__init__.py +6 -0
  42. al_data_utils/config_utils/config_utils.py +31 -0
  43. al_data_utils/files_sync_utils/__init__.py +15 -0
  44. al_data_utils/files_sync_utils/files_sync_utils.py +567 -0
  45. al_data_utils/geo_utils/__init__.py +0 -0
  46. al_data_utils/git_utils/__init__.py +27 -0
  47. al_data_utils/git_utils/git_utils.py +142 -0
  48. al_data_utils/logging_utils/__init__.py +3 -0
  49. al_data_utils/logging_utils/logging_utils.py +39 -0
  50. al_data_utils/ones_api_utils/__init__.py +3 -0
  51. al_data_utils/ones_api_utils/ones_api_utils.py +61 -0
  52. al_data_utils/sql_utils/__init__.py +3 -0
  53. al_data_utils/sql_utils/sql_utils.py +158 -0
  54. al_data_utils/tg_utils/__init__.py +3 -0
  55. al_data_utils/tg_utils/tg_utils.py +45 -0
  56. al_data_utils-0.3.1.dist-info/METADATA +157 -0
  57. al_data_utils-0.3.1.dist-info/RECORD +60 -0
  58. al_data_utils-0.3.1.dist-info/WHEEL +5 -0
  59. al_data_utils-0.3.1.dist-info/licenses/LICENSE +21 -0
  60. al_data_utils-0.3.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,142 @@
1
+ from __future__ import annotations
2
+ import os
3
+ import subprocess
4
+ from pathlib import Path
5
+ from typing import Iterable, Optional
6
+
7
+ from al_data_utils import logging_utils
8
+
9
+
10
+ class GitError(RuntimeError):
11
+ pass
12
+
13
+
14
+ def _run_git(args: Iterable[str], cwd: Optional[os.PathLike[str] | str] = None) -> str:
15
+ cmd = ['git', *args]
16
+
17
+ # nice readable command for logs
18
+ cmd_str = ' '.join(cmd)
19
+ cwd_str = str(cwd) if cwd else os.getcwd()
20
+
21
+ logging_utils.info(f'[git] run: {cmd_str} (cwd={cwd_str})')
22
+
23
+ try:
24
+ res = subprocess.run(
25
+ cmd,
26
+ cwd=cwd,
27
+ check=True,
28
+ stdout=subprocess.PIPE,
29
+ stderr=subprocess.PIPE,
30
+ text=True,
31
+ )
32
+ out = (res.stdout or '').strip()
33
+ err = (res.stderr or '').strip()
34
+
35
+ # log only if anything interesting
36
+ if out:
37
+ logging_utils.info(f'[git] ok: {cmd_str} | stdout: {out}')
38
+ elif err:
39
+ # some git commands write non-error info to stderr
40
+ logging_utils.info(f'[git] ok: {cmd_str} | stderr: {err}')
41
+ else:
42
+ logging_utils.info(f'[git] ok: {cmd_str}')
43
+
44
+ return out
45
+
46
+ except subprocess.CalledProcessError as exc:
47
+ out = (exc.stdout or '').strip()
48
+ err = (exc.stderr or '').strip()
49
+ msg = err or out or 'git command failed'
50
+ logging_utils.error(f'[git] fail: {cmd_str} | {msg}')
51
+ raise GitError(msg) from exc
52
+
53
+
54
+ def _is_nothing_to_commit(msg: str) -> bool:
55
+ m = (msg or '').lower()
56
+ return (
57
+ 'nothing to commit' in m
58
+ or 'working tree clean' in m
59
+ or 'no changes added to commit' in m
60
+ )
61
+
62
+
63
+ def commit_all_changes(path: os.PathLike[str] | str, message: str) -> bool:
64
+ """
65
+ Stage all changes and commit.
66
+ Returns True if a commit was created, False if there was nothing to commit.
67
+ """
68
+ _run_git(['add', '--all'], cwd=path)
69
+
70
+ # Fast check: if clean after add => nothing to commit
71
+ if is_clean(path):
72
+ logging_utils.info('[git] nothing to commit (repo is clean)')
73
+ return False
74
+
75
+ try:
76
+ _run_git(['commit', '-m', message], cwd=path)
77
+ return True
78
+ except GitError as e:
79
+ # Extra safety: treat "nothing to commit" as non-fatal
80
+ if _is_nothing_to_commit(str(e)):
81
+ logging_utils.info('[git] nothing to commit (commit skipped)')
82
+ return False
83
+ raise
84
+
85
+
86
+ def push_changes(
87
+ path: os.PathLike[str] | str, remote: str = 'origin', branch: Optional[str] = None
88
+ ) -> None:
89
+ args = ['push', remote]
90
+ if branch:
91
+ args.append(branch)
92
+ _run_git(args, cwd=path)
93
+
94
+
95
+ def is_git_repo(path: os.PathLike[str] | str) -> bool:
96
+ try:
97
+ _run_git(['rev-parse', '--is-inside-work-tree'], cwd=path)
98
+ return True
99
+ except GitError:
100
+ return False
101
+
102
+
103
+ def repo_root(path: os.PathLike[str] | str) -> Path:
104
+ out = _run_git(['rev-parse', '--show-toplevel'], cwd=path)
105
+ return Path(out)
106
+
107
+
108
+ def current_branch(path: os.PathLike[str] | str) -> str:
109
+ return _run_git(['rev-parse', '--abbrev-ref', 'HEAD'], cwd=path)
110
+
111
+
112
+ def current_commit(path: os.PathLike[str] | str, short: bool = True) -> str:
113
+ args = ['rev-parse']
114
+ if short:
115
+ args.append('--short')
116
+ args.append('HEAD')
117
+ return _run_git(args, cwd=path)
118
+
119
+
120
+ def is_clean(path: os.PathLike[str] | str) -> bool:
121
+ return _run_git(['status', '--porcelain'], cwd=path) == ''
122
+
123
+
124
+ def list_tags(path: os.PathLike[str] | str) -> list[str]:
125
+ out = _run_git(['tag', '--list'], cwd=path)
126
+ return [t for t in out.splitlines() if t]
127
+
128
+
129
+ def list_branches(path: os.PathLike[str] | str, remote: bool = False) -> list[str]:
130
+ args = ['branch', '--list']
131
+ if remote:
132
+ args = ['branch', '--remote', '--list']
133
+ out = _run_git(args, cwd=path)
134
+ return [b.strip().lstrip('* ').strip() for b in out.splitlines() if b.strip()]
135
+
136
+
137
+ def file_last_commit(
138
+ path: os.PathLike[str] | str, file_path: os.PathLike[str] | str
139
+ ) -> str:
140
+ return _run_git(
141
+ ['log', '-n', '1', '--pretty=format:%H', '--', str(file_path)], cwd=path
142
+ )
@@ -0,0 +1,3 @@
1
+ from .logging_utils import configure_logging, error, info, warning
2
+
3
+ __all__ = ["configure_logging", "info", "warning", "error"]
@@ -0,0 +1,39 @@
1
+ """
2
+ module encapsulating logging config
3
+
4
+ stand-alone module makes sure the config is called first
5
+ and shared across modules
6
+
7
+ expected usage:
8
+
9
+ from al_data_utils.logging_utils import info, warning, error
10
+
11
+ info("info message")
12
+ warning("warning message")
13
+ error("error message")
14
+ """
15
+
16
+ import logging
17
+ from logging import error, info, warning # re-exported
18
+
19
+
20
+ def configure_logging(level: int = logging.INFO) -> None:
21
+ """
22
+ Configure root logging once.
23
+ In environments like Airflow, their logging may already be configured.
24
+ """
25
+ root = logging.getLogger()
26
+ if root.handlers:
27
+ # already configured by something else (or by earlier import)
28
+ return
29
+
30
+ logging.basicConfig(
31
+ level=level,
32
+ format="[{asctime}] {{{name}}} {levelname} - {message}",
33
+ style="{",
34
+ datefmt="%Y-%m-%d %H:%M:%S %z",
35
+ )
36
+
37
+
38
+ # Configure on import (keeps your "stand-alone module" behavior)
39
+ configure_logging()
@@ -0,0 +1,3 @@
1
+ from .ones_api_utils import OneSAPIClient
2
+
3
+ __all__ = ['OneSAPIClient']
@@ -0,0 +1,61 @@
1
+ import requests
2
+ from requests.auth import HTTPBasicAuth
3
+ import traceback
4
+
5
+ from al_data_utils import logging_utils
6
+
7
+
8
+ class OneSAPIClient:
9
+ def __init__(self, login, password, base_url):
10
+ self.base_url = base_url
11
+
12
+ self._login = login
13
+ self._password = password
14
+
15
+ def get_data_from_1S_api(self, payload, path):
16
+ """Retrieve all objects from the API."""
17
+
18
+ url = f'{self.base_url}{path}'
19
+ headers = self._browser_headers()
20
+
21
+ try:
22
+ response = requests.post(
23
+ url,
24
+ auth=HTTPBasicAuth(self._login, self._password),
25
+ headers=headers,
26
+ json=payload,
27
+ )
28
+
29
+ if response.status_code == 200:
30
+ logging_utils.info(':) Successfully retrieved data.')
31
+ data = response.json()
32
+ response = {
33
+ 'run_result': 200,
34
+ 'run_response': {'success': True, 'errors': []},
35
+ 'data': data,
36
+ }
37
+
38
+ else:
39
+ logging_utils.error(
40
+ f'Failed to retrieve data. Status code: {response.status_code}'
41
+ )
42
+ content = response.content.decode('utf-8')
43
+ logging_utils.error(content)
44
+ response = {
45
+ 'run_result': 400,
46
+ 'run_response': {'success': False, 'errors': content},
47
+ 'rows_loaded': 0,
48
+ }
49
+ except Exception as e:
50
+ response = {
51
+ 'run_result': 400,
52
+ 'run_response': {'success': False, 'errors': str(e)},
53
+ 'rows_loaded': 0,
54
+ }
55
+ traceback.print_exc()
56
+
57
+ return response
58
+
59
+ def _browser_headers(self):
60
+ """Return headers mimicking a browser request."""
61
+ return {'Authorization': 'Basic', 'Content-Type': 'application/json'}
@@ -0,0 +1,3 @@
1
+ from .sql_utils import (SQLDialect, UnifiedSQLClient, tsql_nstr)
2
+
3
+ __all__ = ['SQLDialect', 'UnifiedSQLClient', 'tsql_nstr']
@@ -0,0 +1,158 @@
1
+ import pandas as pd
2
+ from sqlalchemy import create_engine, event, text
3
+ from urllib.parse import quote_plus
4
+ from enum import Flag, auto
5
+
6
+
7
+ class SQLDialect(Flag):
8
+ POSTGRESQL = auto()
9
+ MYSQL = auto()
10
+ MSSQL = auto()
11
+ SNOWFLAKE = auto()
12
+
13
+
14
+ class UnifiedSQLClient:
15
+ def __init__(self, dialect: SQLDialect | str, params: dict) -> None:
16
+ """
17
+ Initialize the Unified SQL client with support for PostgreSQL, MySQL, MSSQL, and Snowflake.
18
+
19
+ Args:
20
+ dialect (SQLDialect): Enum value indicating which SQL dialect to use ('postgresql', 'mysql', 'mssql', 'snowflake').
21
+
22
+ params (dict): Dictionary of connection parameters. Required keys vary by dialect:
23
+
24
+ Common:
25
+ - host (str): Database host (not required for Snowflake).
26
+ - port (int or str): Database port.
27
+ - database (str): Database name.
28
+ - username (str): Database username.
29
+ - password (str): Database password or key passphrase (if using Snowflake private key).
30
+
31
+ MSSQL-specific:
32
+ - driver (str): ODBC driver name (e.g., 'ODBC Driver 17 for SQL Server').
33
+
34
+ Snowflake-specific:
35
+ - account (str): Snowflake account identifier (e.g., 'abc-xy12345').
36
+ - warehouse (str): Snowflake warehouse name.
37
+ - private_key (str, optional): PEM-encoded private key string for key-pair authentication.
38
+ - passphrase (str, optional): Passphrase to decrypt the private key.
39
+ """
40
+
41
+ self.dialect = self.parse_dialect(dialect)
42
+
43
+ host = quote_plus(params.get('host', params.get('server', '')))
44
+ database = quote_plus(params.get('database', ''))
45
+ username = quote_plus(params.get('username', params.get('user', '')))
46
+ password = quote_plus(params.get('password', ''))
47
+ driver = quote_plus(params.get('driver', ''))
48
+ if 'driver' not in params:
49
+ from al_data_utils import airflow_utils
50
+
51
+ driver = airflow_utils.get_variable_value('core_mssql_odbc_driver')
52
+
53
+ connect_args = {}
54
+
55
+ if self.dialect == SQLDialect.POSTGRESQL:
56
+ port = params.get('port', 5432)
57
+ self.connection_string = (
58
+ f'postgresql+psycopg2://{username}:{password}@{host}:{port}/{database}'
59
+ )
60
+
61
+ elif self.dialect == SQLDialect.MYSQL:
62
+ port = params.get('port', 3306)
63
+ charset = params.get('charset', 'utf8mb4')
64
+ self.connection_string = f'mysql+pymysql://{username}:{password}@{host}:{port}/{database}?charset={charset}'
65
+
66
+ elif self.dialect == SQLDialect.MSSQL:
67
+ port = params.get('port', 1433)
68
+ encrypt = params.get('encrypt', 'yes')
69
+ self.connection_string = f'mssql+pyodbc://{username}:{password}@{host}:{port}/{database}?driver={driver.replace(" ", "+")}&TrustServerCertificate=yes&Encrypt={encrypt}'
70
+ elif self.dialect == SQLDialect.SNOWFLAKE:
71
+ warehouse = params['warehouse']
72
+ account = params['account']
73
+ if 'private_key' in params:
74
+ from modules import key_auth_utils
75
+
76
+ private_key_bytes = key_auth_utils.get_private_key(
77
+ params['private_key'], params['key_password'], result_format='DER'
78
+ )
79
+
80
+ self.connection_string = (
81
+ f'snowflake://{username}@{account}?warehouse={warehouse}/{database}'
82
+ )
83
+
84
+ connect_args = {'private_key': private_key_bytes}
85
+
86
+ else:
87
+ self.connection_string = f'snowflake://{username}:{password}@{account}?warehouse={warehouse}/{database}'
88
+
89
+ else:
90
+ raise ValueError(f'Unsupported dialect: {dialect}')
91
+
92
+ self.engine = create_engine(self.connection_string, connect_args=connect_args)
93
+
94
+ @staticmethod
95
+ def parse_dialect(value: str | SQLDialect) -> SQLDialect:
96
+ if isinstance(value, SQLDialect):
97
+ return value
98
+ try:
99
+ return SQLDialect[value.upper()]
100
+ except KeyError as e:
101
+ raise ValueError(f'Unknown dialect: {value}') from e
102
+
103
+ def read_sql_to_df(self, query: str, params: dict | None = None) -> pd.DataFrame:
104
+ """Run a query and return a pandas DataFrame."""
105
+ with self.engine.connect() as conn:
106
+ df = pd.read_sql(query, conn, params=params)
107
+ return df
108
+
109
+ def write_df_to_sql(
110
+ self,
111
+ df: pd.DataFrame,
112
+ table_name: str,
113
+ schema_name: str = 'dbo',
114
+ if_exists: str = 'append',
115
+ index: bool = False,
116
+ ):
117
+ """Write a pandas DataFrame to a SQL table."""
118
+ with self.engine.begin() as conn:
119
+ df.to_sql(
120
+ table_name,
121
+ schema=schema_name,
122
+ con=conn,
123
+ if_exists=if_exists,
124
+ index=index,
125
+ )
126
+
127
+ def execute_command(self, command: str, params: dict | None = None) -> None:
128
+ """Execute a raw SQL command (e.g., INSERT, UPDATE, or stored procedure)."""
129
+ with self.engine.begin() as conn:
130
+ conn.execute(text(command), params)
131
+
132
+ def execute_command_with_result_to_df(
133
+ self, query: str, params: dict | None = None
134
+ ) -> pd.DataFrame:
135
+ """Run a query and return a pandas DataFrame."""
136
+ with self.engine.begin() as conn:
137
+ df = pd.read_sql(query, conn, params=params)
138
+ return df
139
+
140
+ def execute_command_with_result_to_dicts(
141
+ self, query: str, params: dict[str, str] | None = None
142
+ ) -> list[dict[str, str]]:
143
+ """Run a query and return a list of dicts (one per row)."""
144
+ with self.engine.begin() as conn:
145
+ result = conn.execute(text(query), params or {})
146
+ if result.returns_rows:
147
+ dc = [dict(row) for row in result.mappings()]
148
+ else:
149
+ dc = []
150
+ return dc
151
+
152
+
153
+ def tsql_nstr(s) -> str:
154
+ """Make a safe NVARCHAR literal (N'...') or NULL."""
155
+ if s is None:
156
+ return 'NULL'
157
+ # double single quotes inside the literal
158
+ return "N'{}'".format(str(s).replace("'", "''"))
@@ -0,0 +1,3 @@
1
+ from .tg_utils import (TelegramBot, TelegramBotException)
2
+
3
+ __all__ = ['TelegramBot', 'TelegramBotException']
@@ -0,0 +1,45 @@
1
+ from typing import Union
2
+ import requests
3
+
4
+
5
+ class TelegramBot:
6
+ """Telegram Bot API wrapper
7
+
8
+ Examples:
9
+ >>> TelegramBot('YOuRAwEsomeBOtToKen', '@myprettydebugchat').send_message('Hi, darling')
10
+ >>> TelegramBot('YOuRAwEsomeBOtToKen').send_message('Hi, darling', chat_id=-1762374628374)
11
+ """
12
+
13
+ API_ENDPOINT = 'https://api.telegram.org/bot{}/'
14
+
15
+ def __init__(self, token: str, default_chat_id: Union[int, str] = None):
16
+ self.token = token
17
+ self._base_url = TelegramBot.API_ENDPOINT.format(token)
18
+ self.default_chat_id = default_chat_id
19
+
20
+ def send_message(self, message: str, chat_id: Union[int, str] = None):
21
+ method = 'sendMessage'
22
+
23
+ response = requests.post(
24
+ url='https://api.telegram.org/bot{0}/{1}'.format(self.token, method),
25
+ data={'chat_id': chat_id or self.default_chat_id, 'text': message},
26
+ ).json()
27
+
28
+ if not response.get('ok'):
29
+ raise TelegramBotException(response)
30
+
31
+ def on_fail_callback(self, context, chat_id: Union[int, str] = None):
32
+ message = f"""
33
+ 😡 Task has failed.
34
+ Task: {context.get('task_instance').task_id}
35
+ DAG: {context.get('task_instance').dag_id}
36
+ Execution Time: {context.get('execution_date')}
37
+ Exception: {str(context.get('exception'))[:255]}
38
+ """
39
+
40
+ self.send_message(message=message, chat_id=chat_id or self.default_chat_id)
41
+
42
+
43
+ class TelegramBotException(Exception):
44
+ def __init__(self, *args, **kwargs):
45
+ super().__init__((args, kwargs))
@@ -0,0 +1,157 @@
1
+ Metadata-Version: 2.4
2
+ Name: al-data-utils
3
+ Version: 0.3.1
4
+ Summary: Reusable utilities for data engineering (Airflow, Azure, Snowflake, SQL, config).
5
+ Author-email: AL Data Labs <pypi@aldatalabs.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://aldatalabs.com
8
+ Keywords: airflow,data-engineering,azure,snowflake,sql
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Topic :: Software Development :: Libraries
12
+ Requires-Python: >=3.10
13
+ Description-Content-Type: text/markdown
14
+ License-File: LICENSE
15
+ Dynamic: license-file
16
+
17
+ # al-data-utils
18
+
19
+ Reusable Python utilities for data and automation workflows.
20
+
21
+ This repository contains independent utility modules for sync operations, git automation, SQL work, logging helpers, and MSSQL schema snapshot generation.
22
+
23
+ ## Python and Setup
24
+
25
+ - Python: 3.10+
26
+ - Packaging: setuptools (src layout)
27
+
28
+ Install locally in editable mode:
29
+
30
+ ```bash
31
+ pip install -e .
32
+ ```
33
+
34
+ ## Project Structure
35
+
36
+ - src/al_data_utils
37
+ - examples
38
+ - development_guides
39
+ - tools
40
+
41
+ Main utility groups under src/al_data_utils:
42
+
43
+ - airflow_utils
44
+ - backup_utils
45
+ - config_utils
46
+ - files_sync_utils
47
+ - geo_utils
48
+ - git_utils
49
+ - logging_utils
50
+ - ones_api_utils
51
+ - sql_utils
52
+ - tg_utils
53
+
54
+ ## Examples
55
+
56
+ Runnable examples are available in the `examples` folder for the main utility groups.
57
+ Use them as reference implementations for local scripts and automation jobs.
58
+
59
+ ## MSSQL Schema Snapshot
60
+
61
+ The snapshot tool exports SQL objects from MSSQL into a filesystem layout and syncs staging output to destination folders.
62
+
63
+ Current behavior highlights:
64
+
65
+ - Supports single DB and multi-DB runs in one execution.
66
+ - Always writes outputs into per-DB folders.
67
+ - Uses staging first, then syncs to final output.
68
+ - Supports cleanup of staging folders.
69
+ - Supports sync log retention cleanup.
70
+ - Supports include and exclude filters per object type.
71
+ - Supports DataGrip template override files.
72
+
73
+ Output shape:
74
+
75
+ ```text
76
+ <MSSQL_OUT_DIR>/<DatabaseName>/schemas/<schema>/
77
+ tables/
78
+ views/
79
+ routines/
80
+ sequences/
81
+ synonyms/
82
+ ```
83
+
84
+ ### Snapshot Environment Variables
85
+
86
+ Core connection and output:
87
+
88
+ - MSSQL_OUT_DIR
89
+ - MSSQL_SERVER
90
+ - MSSQL_DATABASE
91
+ - MSSQL_DATABASES
92
+ - MSSQL_AUTH_MODE
93
+ - MSSQL_USER
94
+ - MSSQL_PASSWORD
95
+ - MSSQL_DRIVER
96
+
97
+ Snapshot behavior:
98
+
99
+ - MSSQL_ENABLE_TABLE_DEFINITIONS
100
+ - MSSQL_ENABLE_INDEXES
101
+ - MSSQL_ENABLE_FOREIGN_KEYS
102
+ - MSSQL_ENABLE_VIEWS
103
+ - MSSQL_ENABLE_STORED_PROCEDURES
104
+ - MSSQL_ENABLE_FUNCTIONS
105
+ - MSSQL_ENABLE_SEQUENCES
106
+ - MSSQL_ENABLE_SYNONYMS
107
+ - MSSQL_CLEANUP_STAGING
108
+ - MSSQL_SYNC_LOG_RETENTION_DAYS
109
+
110
+ Filters:
111
+
112
+ - MSSQL_INCLUDE_SCHEMAS
113
+ - MSSQL_EXCLUDE_SCHEMAS
114
+ - MSSQL_INCLUDE_TABLES
115
+ - MSSQL_EXCLUDE_TABLES
116
+ - MSSQL_INCLUDE_VIEWS
117
+ - MSSQL_EXCLUDE_VIEWS
118
+ - MSSQL_INCLUDE_STORED_PROCEDURES
119
+ - MSSQL_EXCLUDE_STORED_PROCEDURES
120
+ - MSSQL_INCLUDE_FUNCTIONS
121
+ - MSSQL_EXCLUDE_FUNCTIONS
122
+ - MSSQL_INCLUDE_SEQUENCES
123
+ - MSSQL_EXCLUDE_SEQUENCES
124
+ - MSSQL_INCLUDE_SYNONYMS
125
+ - MSSQL_EXCLUDE_SYNONYMS
126
+
127
+ Notes:
128
+
129
+ - If MSSQL_DATABASES is set, it is treated as a comma-separated list.
130
+ - If MSSQL_DATABASES is empty, MSSQL_DATABASE is used.
131
+ - If MSSQL_DATABASE contains commas, it is also treated as a list for backward compatibility.
132
+
133
+ ## File Sync Utilities
134
+
135
+ `files_sync_utils` provides:
136
+
137
+ - Inventory building
138
+ - Plan generation
139
+ - Copy and update execution
140
+ - Extra destination cleanup rules
141
+ - Activity CSV logging
142
+ - Log retention cleanup
143
+
144
+ ## Git Utilities
145
+
146
+ `git_utils` wraps common git automation operations with consistent behavior and error handling.
147
+
148
+ ## Development Guides
149
+
150
+ - development_guides/dev_guide.txt
151
+ - development_guides/auth_key_guide.txt
152
+
153
+ These guides contain release flow notes and auth-related instructions.
154
+
155
+ ## Changelog
156
+
157
+ See CHANGELOG.md for release notes.
@@ -0,0 +1,60 @@
1
+ al_data_utils/__init__.py,sha256=l5Mdphl5Gh_pIo8qYMTyUxTQAdkvuiGkCQwXPfkea9o,180
2
+ al_data_utils/airflow_utils/__init__.py,sha256=Yq-Yy9rNmm4NTC2Qo0XLT-j9-b4h4pKrkoXY6l1uOcI,283
3
+ al_data_utils/airflow_utils/airflow_utils.py,sha256=fcCRzVogkTDBnnRovX8_3Qv4pnkPnoBRiOVNq-B-Dac,2470
4
+ al_data_utils/azure_as_utils/__init__.py,sha256=AYWuh37QwiPdStM7uHvW2NFzjR1cIBxScSOAh8F9WpM,789
5
+ al_data_utils/azure_as_utils/auth.py,sha256=2NBJN6TOV5jUpahcj5uqw3LEQHA3iWyiwQyeDKBsvI8,2825
6
+ al_data_utils/azure_as_utils/azure_as_utils.py,sha256=mtkw47RDHCQ9ppFafdhn-p9rqL8HvHAvkCcCQGML-8U,8875
7
+ al_data_utils/azure_as_utils/client.py,sha256=GJGAb6lj5AuyKRAyVIAZ4kdAlfI5A7VKZea0MqorB6A,6162
8
+ al_data_utils/azure_as_utils/errors.py,sha256=IYU20hNhGtvoM_6LB9bYl1LOsh6Jn9TB_3hJP10TgYg,2513
9
+ al_data_utils/azure_as_utils/models.py,sha256=5Q9IvWFh5hgAbhs2INg9dI7NFdOccbh0AT_btJedIw0,10299
10
+ al_data_utils/azure_as_utils/poller.py,sha256=-JQSb5PEgk-AsbQTGX0zpU8iRUo6Wdpfcri9XnSSBdA,4626
11
+ al_data_utils/backup_utils/__init__.py,sha256=ivVRrdutosWUQ6az7qYaPKsOy1UNS9TLE9ixbaV4PjI,381
12
+ al_data_utils/backup_utils/mssql/dacpac/__init__.py,sha256=_tuqsMspDnkc8vM2BLIjyBe0dOwVonQcyYSlwupLihY,77
13
+ al_data_utils/backup_utils/mssql/dacpac/sqlpackage.py,sha256=NTTZjRBn_oL2HStOeGexTzx0ianDYvAAC4Ch6FEwWjg,6227
14
+ al_data_utils/backup_utils/mssql/schema_snapshot/__init__.py,sha256=ek95VFS_psM3kpiC3g3oIJTBBebXSk4uPpeABN-Vgfg,215
15
+ al_data_utils/backup_utils/mssql/schema_snapshot/config.py,sha256=HUGKDiz690qR1IRywmwR3S6zU1XBJYOL7ZqRShghU6M,5451
16
+ al_data_utils/backup_utils/mssql/schema_snapshot/snapshot_processor.py,sha256=9lE7Naq2hl6PA0HUftlljx86VxKbZ7ZoW1Ql_2E68aE,26635
17
+ al_data_utils/backup_utils/mssql/schema_snapshot/emit/__init__.py,sha256=qlHd-Kndr0IDI8iJNBBp_lshNkeGdVGDx3pBPdXUbKo,288
18
+ al_data_utils/backup_utils/mssql/schema_snapshot/emit/foreign_keys.py,sha256=k2IQp4ByXavacvOvYXGc2ZXG64TROuHSUV_qVVv93to,1056
19
+ al_data_utils/backup_utils/mssql/schema_snapshot/emit/indexes.py,sha256=gF8gEY09jsaVlpEt2ro6qikBaxLbqicHlQC_tVGf3Qw,1069
20
+ al_data_utils/backup_utils/mssql/schema_snapshot/emit/sql_objects.py,sha256=47QVpNg2epyanB3EXT_BiAl_5Phy9pi2N0LRtbXrw_o,434
21
+ al_data_utils/backup_utils/mssql/schema_snapshot/emit/tables.py,sha256=0S6V_1SvSKcA4nOr09do_6te1_hIGhTSvjDnSpcbDhA,4168
22
+ al_data_utils/backup_utils/mssql/schema_snapshot/introspect/__init__.py,sha256=d6Yvh-6JO9QrxAI8N_WFLRd-a0XSmILnVNUNEGsVMBU,501
23
+ al_data_utils/backup_utils/mssql/schema_snapshot/introspect/foreign_keys.py,sha256=-EyeL5YRPDuuWoG6caZfGS7IF3j0QhTQ84rv4NTgu2g,6149
24
+ al_data_utils/backup_utils/mssql/schema_snapshot/introspect/functions.py,sha256=nZK9bqBwjM4uBSaKt_Gs8daj-NOmGLRRDUwSH-ygFrw,4194
25
+ al_data_utils/backup_utils/mssql/schema_snapshot/introspect/indexes.py,sha256=MUs728kqTaGCb4l7fIl0lSGNNwpPa0Bbgjt0k-dqMR0,6534
26
+ al_data_utils/backup_utils/mssql/schema_snapshot/introspect/sequences.py,sha256=Ps22XFiMUts2Ye7yV1vxzy6TxbGRUUnM2MPdS83mQpI,5847
27
+ al_data_utils/backup_utils/mssql/schema_snapshot/introspect/stored_procedures.py,sha256=shdi_OZeOMXtUwwGso1msqMGGoI8gXjalw8ZK570PDU,10667
28
+ al_data_utils/backup_utils/mssql/schema_snapshot/introspect/synonyms.py,sha256=rcyT82ddwvQd7RaFVZbnhkeaQPScRDwRLTbmAkV7fgY,4320
29
+ al_data_utils/backup_utils/mssql/schema_snapshot/introspect/tables.py,sha256=5H7QgAyiUWTdP2DVhCtzLYETR_V6ReFitSdmkJxEfzM,8926
30
+ al_data_utils/backup_utils/mssql/schema_snapshot/introspect/views.py,sha256=O3B6FQWmRr1BuYrNq7bE1PK69FjUlPbLqbLtH5ECi3g,8087
31
+ al_data_utils/backup_utils/mssql/schema_snapshot/io/__init__.py,sha256=om0ki99fKptoalzw-It_bHUqGjAaJRe9jJL44io5qPU,275
32
+ al_data_utils/backup_utils/mssql/schema_snapshot/io/layout.py,sha256=52u6V-vwg-PDBQxiLyTDb7G2dtQqE1bfsM38gvmDId8,624
33
+ al_data_utils/backup_utils/mssql/schema_snapshot/io/writer.py,sha256=BiESwNLW_yZ9V6TceUcV0DOwV7ShesctfL3f77K190E,1071
34
+ al_data_utils/backup_utils/mssql/schema_snapshot/models/__init__.py,sha256=Y6aIhnMlhjpuAQPFDe4buJxyfSBWRZbMCNvwHSIOvN0,353
35
+ al_data_utils/backup_utils/mssql/schema_snapshot/models/foreign_key_def.py,sha256=tFB5_66UQg0EI6xqSaCNw90aNoDlhDS7DUAJD2ga5Q4,487
36
+ al_data_utils/backup_utils/mssql/schema_snapshot/models/index_def.py,sha256=hjnKyacS3XCynXXh3lLhWv0aG6aPQNzF0UK2MIhIto4,518
37
+ al_data_utils/backup_utils/mssql/schema_snapshot/models/sql_object_def.py,sha256=ehsn5Y5ZBK6i1FfiwXdU_HtMGiVUOYEiFOU1FnNkuds,257
38
+ al_data_utils/backup_utils/mssql/schema_snapshot/models/table_column_def.py,sha256=WH9nipBAXwjWz3yjbz12Hh0dn0xzckrEmHuRkuPyRrU,603
39
+ al_data_utils/backup_utils/mssql/schema_snapshot/models/table_def.py,sha256=OPZ5ngP7XRAituFNQa0jaW6jVz_wtUziPfiZ9zJrl2k,453
40
+ al_data_utils/backup_utils/mssql/scripter/__init__.py,sha256=RkwDreYTHuYloJr2wpjPY_Ggp2LmhLrkanUtCKFwyv8,52
41
+ al_data_utils/config_utils/__init__.py,sha256=HrzEu11bJhikSvP6seI3dR1ZJnm4bqtNg8G930MLWLo,109
42
+ al_data_utils/config_utils/config_utils.py,sha256=DMiIK7HMdLUmLAUrrd36iYks0WUHXZsB7idxjyFKLhA,1005
43
+ al_data_utils/files_sync_utils/__init__.py,sha256=WzoysPTOXc6s8VZzDRnr95XZ5hj8vPULQsLr08momiU,268
44
+ al_data_utils/files_sync_utils/files_sync_utils.py,sha256=ehor1N634BqPLTZKxCPeM9a_tJNpjHicHq9q4n_dGTQ,19724
45
+ al_data_utils/geo_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
46
+ al_data_utils/git_utils/__init__.py,sha256=T26XLhi4BTpFyFpg0NNF4H7sbE5P6MGJ647mpWJROxs,487
47
+ al_data_utils/git_utils/git_utils.py,sha256=ovfCy9dxvA1pMe5e4sVNzjK8U0j8UHvNKgoeqnI1hF0,4213
48
+ al_data_utils/logging_utils/__init__.py,sha256=HsgS_gpsmsF4QOVacINzIsVHxVuAGpVWyXGI6gn5st0,129
49
+ al_data_utils/logging_utils/logging_utils.py,sha256=d-HB6fRDYqs3r7faFoJ9qt4yO_xEbS2H9ulE6K4JcwQ,992
50
+ al_data_utils/ones_api_utils/__init__.py,sha256=qSm5LIycuuyVs2q8AMM82dJbksLhLnIlfZYsYZPxYVI,72
51
+ al_data_utils/ones_api_utils/ones_api_utils.py,sha256=20CW5YdrWzObL8oFcdkvMnDQ8CN0gOcCAoeKd56zLwk,1984
52
+ al_data_utils/sql_utils/__init__.py,sha256=UF73sf9UD7hU5DIGEuY7x2GhqDYVIo9mpyb45vVUJqY,125
53
+ al_data_utils/sql_utils/sql_utils.py,sha256=wAH-BbFLb6x8uL5B6xE3xzM2giUt6qHcv1buV0tlbcQ,6375
54
+ al_data_utils/tg_utils/__init__.py,sha256=GqloghgtS6OgL6IRnqXTvYFs7FgxnL6vMcSMGcmyEtw,110
55
+ al_data_utils/tg_utils/tg_utils.py,sha256=eyk9v__Mh5pSQJp_9GHfmOQw1i9C7mdKYISI6-7I0TM,1551
56
+ al_data_utils-0.3.1.dist-info/licenses/LICENSE,sha256=FSoDw09XQtrZEbEgid_0dBku6vuuPj3r7XPxcTklOfo,1090
57
+ al_data_utils-0.3.1.dist-info/METADATA,sha256=k8E6oLadDmhS7EQ140u7cxsAZqn-xN5W7rbvK2GktXA,3827
58
+ al_data_utils-0.3.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
59
+ al_data_utils-0.3.1.dist-info/top_level.txt,sha256=Ade88LWXWws-LCIeYmUHHRRN6Eq8MTM0Di5BXxA6ir0,14
60
+ al_data_utils-0.3.1.dist-info/RECORD,,