al-data-utils 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- al_data_utils/__init__.py +10 -0
- al_data_utils/airflow_utils/__init__.py +13 -0
- al_data_utils/airflow_utils/airflow_utils.py +83 -0
- al_data_utils/azure_as_utils/__init__.py +31 -0
- al_data_utils/azure_as_utils/auth.py +82 -0
- al_data_utils/azure_as_utils/azure_as_utils.py +223 -0
- al_data_utils/azure_as_utils/client.py +158 -0
- al_data_utils/azure_as_utils/errors.py +78 -0
- al_data_utils/azure_as_utils/models.py +265 -0
- al_data_utils/azure_as_utils/poller.py +121 -0
- al_data_utils/backup_utils/__init__.py +9 -0
- al_data_utils/backup_utils/mssql/dacpac/__init__.py +1 -0
- al_data_utils/backup_utils/mssql/dacpac/sqlpackage.py +207 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/__init__.py +8 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/config.py +150 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/emit/__init__.py +11 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/emit/foreign_keys.py +38 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/emit/indexes.py +37 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/emit/sql_objects.py +17 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/emit/tables.py +139 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/introspect/__init__.py +19 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/introspect/foreign_keys.py +173 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/introspect/functions.py +133 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/introspect/indexes.py +190 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/introspect/sequences.py +180 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/introspect/stored_procedures.py +313 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/introspect/synonyms.py +135 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/introspect/tables.py +251 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/introspect/views.py +244 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/io/__init__.py +10 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/io/layout.py +28 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/io/writer.py +40 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/models/__init__.py +14 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/models/foreign_key_def.py +19 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/models/index_def.py +19 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/models/sql_object_def.py +11 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/models/table_column_def.py +27 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/models/table_def.py +22 -0
- al_data_utils/backup_utils/mssql/schema_snapshot/snapshot_processor.py +754 -0
- al_data_utils/backup_utils/mssql/scripter/__init__.py +1 -0
- al_data_utils/config_utils/__init__.py +6 -0
- al_data_utils/config_utils/config_utils.py +31 -0
- al_data_utils/files_sync_utils/__init__.py +15 -0
- al_data_utils/files_sync_utils/files_sync_utils.py +567 -0
- al_data_utils/geo_utils/__init__.py +0 -0
- al_data_utils/git_utils/__init__.py +27 -0
- al_data_utils/git_utils/git_utils.py +142 -0
- al_data_utils/logging_utils/__init__.py +3 -0
- al_data_utils/logging_utils/logging_utils.py +39 -0
- al_data_utils/ones_api_utils/__init__.py +3 -0
- al_data_utils/ones_api_utils/ones_api_utils.py +61 -0
- al_data_utils/sql_utils/__init__.py +3 -0
- al_data_utils/sql_utils/sql_utils.py +158 -0
- al_data_utils/tg_utils/__init__.py +3 -0
- al_data_utils/tg_utils/tg_utils.py +45 -0
- al_data_utils-0.3.1.dist-info/METADATA +157 -0
- al_data_utils-0.3.1.dist-info/RECORD +60 -0
- al_data_utils-0.3.1.dist-info/WHEEL +5 -0
- al_data_utils-0.3.1.dist-info/licenses/LICENSE +21 -0
- al_data_utils-0.3.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import os
|
|
3
|
+
import subprocess
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Iterable, Optional
|
|
6
|
+
|
|
7
|
+
from al_data_utils import logging_utils
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class GitError(RuntimeError):
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _run_git(args: Iterable[str], cwd: Optional[os.PathLike[str] | str] = None) -> str:
|
|
15
|
+
cmd = ['git', *args]
|
|
16
|
+
|
|
17
|
+
# nice readable command for logs
|
|
18
|
+
cmd_str = ' '.join(cmd)
|
|
19
|
+
cwd_str = str(cwd) if cwd else os.getcwd()
|
|
20
|
+
|
|
21
|
+
logging_utils.info(f'[git] run: {cmd_str} (cwd={cwd_str})')
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
res = subprocess.run(
|
|
25
|
+
cmd,
|
|
26
|
+
cwd=cwd,
|
|
27
|
+
check=True,
|
|
28
|
+
stdout=subprocess.PIPE,
|
|
29
|
+
stderr=subprocess.PIPE,
|
|
30
|
+
text=True,
|
|
31
|
+
)
|
|
32
|
+
out = (res.stdout or '').strip()
|
|
33
|
+
err = (res.stderr or '').strip()
|
|
34
|
+
|
|
35
|
+
# log only if anything interesting
|
|
36
|
+
if out:
|
|
37
|
+
logging_utils.info(f'[git] ok: {cmd_str} | stdout: {out}')
|
|
38
|
+
elif err:
|
|
39
|
+
# some git commands write non-error info to stderr
|
|
40
|
+
logging_utils.info(f'[git] ok: {cmd_str} | stderr: {err}')
|
|
41
|
+
else:
|
|
42
|
+
logging_utils.info(f'[git] ok: {cmd_str}')
|
|
43
|
+
|
|
44
|
+
return out
|
|
45
|
+
|
|
46
|
+
except subprocess.CalledProcessError as exc:
|
|
47
|
+
out = (exc.stdout or '').strip()
|
|
48
|
+
err = (exc.stderr or '').strip()
|
|
49
|
+
msg = err or out or 'git command failed'
|
|
50
|
+
logging_utils.error(f'[git] fail: {cmd_str} | {msg}')
|
|
51
|
+
raise GitError(msg) from exc
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _is_nothing_to_commit(msg: str) -> bool:
|
|
55
|
+
m = (msg or '').lower()
|
|
56
|
+
return (
|
|
57
|
+
'nothing to commit' in m
|
|
58
|
+
or 'working tree clean' in m
|
|
59
|
+
or 'no changes added to commit' in m
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def commit_all_changes(path: os.PathLike[str] | str, message: str) -> bool:
|
|
64
|
+
"""
|
|
65
|
+
Stage all changes and commit.
|
|
66
|
+
Returns True if a commit was created, False if there was nothing to commit.
|
|
67
|
+
"""
|
|
68
|
+
_run_git(['add', '--all'], cwd=path)
|
|
69
|
+
|
|
70
|
+
# Fast check: if clean after add => nothing to commit
|
|
71
|
+
if is_clean(path):
|
|
72
|
+
logging_utils.info('[git] nothing to commit (repo is clean)')
|
|
73
|
+
return False
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
_run_git(['commit', '-m', message], cwd=path)
|
|
77
|
+
return True
|
|
78
|
+
except GitError as e:
|
|
79
|
+
# Extra safety: treat "nothing to commit" as non-fatal
|
|
80
|
+
if _is_nothing_to_commit(str(e)):
|
|
81
|
+
logging_utils.info('[git] nothing to commit (commit skipped)')
|
|
82
|
+
return False
|
|
83
|
+
raise
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def push_changes(
|
|
87
|
+
path: os.PathLike[str] | str, remote: str = 'origin', branch: Optional[str] = None
|
|
88
|
+
) -> None:
|
|
89
|
+
args = ['push', remote]
|
|
90
|
+
if branch:
|
|
91
|
+
args.append(branch)
|
|
92
|
+
_run_git(args, cwd=path)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def is_git_repo(path: os.PathLike[str] | str) -> bool:
|
|
96
|
+
try:
|
|
97
|
+
_run_git(['rev-parse', '--is-inside-work-tree'], cwd=path)
|
|
98
|
+
return True
|
|
99
|
+
except GitError:
|
|
100
|
+
return False
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def repo_root(path: os.PathLike[str] | str) -> Path:
|
|
104
|
+
out = _run_git(['rev-parse', '--show-toplevel'], cwd=path)
|
|
105
|
+
return Path(out)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def current_branch(path: os.PathLike[str] | str) -> str:
|
|
109
|
+
return _run_git(['rev-parse', '--abbrev-ref', 'HEAD'], cwd=path)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def current_commit(path: os.PathLike[str] | str, short: bool = True) -> str:
|
|
113
|
+
args = ['rev-parse']
|
|
114
|
+
if short:
|
|
115
|
+
args.append('--short')
|
|
116
|
+
args.append('HEAD')
|
|
117
|
+
return _run_git(args, cwd=path)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def is_clean(path: os.PathLike[str] | str) -> bool:
|
|
121
|
+
return _run_git(['status', '--porcelain'], cwd=path) == ''
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def list_tags(path: os.PathLike[str] | str) -> list[str]:
|
|
125
|
+
out = _run_git(['tag', '--list'], cwd=path)
|
|
126
|
+
return [t for t in out.splitlines() if t]
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def list_branches(path: os.PathLike[str] | str, remote: bool = False) -> list[str]:
|
|
130
|
+
args = ['branch', '--list']
|
|
131
|
+
if remote:
|
|
132
|
+
args = ['branch', '--remote', '--list']
|
|
133
|
+
out = _run_git(args, cwd=path)
|
|
134
|
+
return [b.strip().lstrip('* ').strip() for b in out.splitlines() if b.strip()]
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def file_last_commit(
|
|
138
|
+
path: os.PathLike[str] | str, file_path: os.PathLike[str] | str
|
|
139
|
+
) -> str:
|
|
140
|
+
return _run_git(
|
|
141
|
+
['log', '-n', '1', '--pretty=format:%H', '--', str(file_path)], cwd=path
|
|
142
|
+
)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
module encapsulating logging config
|
|
3
|
+
|
|
4
|
+
stand-alone module makes sure the config is called first
|
|
5
|
+
and shared across modules
|
|
6
|
+
|
|
7
|
+
expected usage:
|
|
8
|
+
|
|
9
|
+
from al_data_utils.logging_utils import info, warning, error
|
|
10
|
+
|
|
11
|
+
info("info message")
|
|
12
|
+
warning("warning message")
|
|
13
|
+
error("error message")
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import logging
|
|
17
|
+
from logging import error, info, warning # re-exported
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def configure_logging(level: int = logging.INFO) -> None:
|
|
21
|
+
"""
|
|
22
|
+
Configure root logging once.
|
|
23
|
+
In environments like Airflow, their logging may already be configured.
|
|
24
|
+
"""
|
|
25
|
+
root = logging.getLogger()
|
|
26
|
+
if root.handlers:
|
|
27
|
+
# already configured by something else (or by earlier import)
|
|
28
|
+
return
|
|
29
|
+
|
|
30
|
+
logging.basicConfig(
|
|
31
|
+
level=level,
|
|
32
|
+
format="[{asctime}] {{{name}}} {levelname} - {message}",
|
|
33
|
+
style="{",
|
|
34
|
+
datefmt="%Y-%m-%d %H:%M:%S %z",
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# Configure on import (keeps your "stand-alone module" behavior)
|
|
39
|
+
configure_logging()
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
from requests.auth import HTTPBasicAuth
|
|
3
|
+
import traceback
|
|
4
|
+
|
|
5
|
+
from al_data_utils import logging_utils
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class OneSAPIClient:
|
|
9
|
+
def __init__(self, login, password, base_url):
|
|
10
|
+
self.base_url = base_url
|
|
11
|
+
|
|
12
|
+
self._login = login
|
|
13
|
+
self._password = password
|
|
14
|
+
|
|
15
|
+
def get_data_from_1S_api(self, payload, path):
|
|
16
|
+
"""Retrieve all objects from the API."""
|
|
17
|
+
|
|
18
|
+
url = f'{self.base_url}{path}'
|
|
19
|
+
headers = self._browser_headers()
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
response = requests.post(
|
|
23
|
+
url,
|
|
24
|
+
auth=HTTPBasicAuth(self._login, self._password),
|
|
25
|
+
headers=headers,
|
|
26
|
+
json=payload,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
if response.status_code == 200:
|
|
30
|
+
logging_utils.info(':) Successfully retrieved data.')
|
|
31
|
+
data = response.json()
|
|
32
|
+
response = {
|
|
33
|
+
'run_result': 200,
|
|
34
|
+
'run_response': {'success': True, 'errors': []},
|
|
35
|
+
'data': data,
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
else:
|
|
39
|
+
logging_utils.error(
|
|
40
|
+
f'Failed to retrieve data. Status code: {response.status_code}'
|
|
41
|
+
)
|
|
42
|
+
content = response.content.decode('utf-8')
|
|
43
|
+
logging_utils.error(content)
|
|
44
|
+
response = {
|
|
45
|
+
'run_result': 400,
|
|
46
|
+
'run_response': {'success': False, 'errors': content},
|
|
47
|
+
'rows_loaded': 0,
|
|
48
|
+
}
|
|
49
|
+
except Exception as e:
|
|
50
|
+
response = {
|
|
51
|
+
'run_result': 400,
|
|
52
|
+
'run_response': {'success': False, 'errors': str(e)},
|
|
53
|
+
'rows_loaded': 0,
|
|
54
|
+
}
|
|
55
|
+
traceback.print_exc()
|
|
56
|
+
|
|
57
|
+
return response
|
|
58
|
+
|
|
59
|
+
def _browser_headers(self):
|
|
60
|
+
"""Return headers mimicking a browser request."""
|
|
61
|
+
return {'Authorization': 'Basic', 'Content-Type': 'application/json'}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import pandas as pd
|
|
2
|
+
from sqlalchemy import create_engine, event, text
|
|
3
|
+
from urllib.parse import quote_plus
|
|
4
|
+
from enum import Flag, auto
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class SQLDialect(Flag):
|
|
8
|
+
POSTGRESQL = auto()
|
|
9
|
+
MYSQL = auto()
|
|
10
|
+
MSSQL = auto()
|
|
11
|
+
SNOWFLAKE = auto()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class UnifiedSQLClient:
|
|
15
|
+
def __init__(self, dialect: SQLDialect | str, params: dict) -> None:
|
|
16
|
+
"""
|
|
17
|
+
Initialize the Unified SQL client with support for PostgreSQL, MySQL, MSSQL, and Snowflake.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
dialect (SQLDialect): Enum value indicating which SQL dialect to use ('postgresql', 'mysql', 'mssql', 'snowflake').
|
|
21
|
+
|
|
22
|
+
params (dict): Dictionary of connection parameters. Required keys vary by dialect:
|
|
23
|
+
|
|
24
|
+
Common:
|
|
25
|
+
- host (str): Database host (not required for Snowflake).
|
|
26
|
+
- port (int or str): Database port.
|
|
27
|
+
- database (str): Database name.
|
|
28
|
+
- username (str): Database username.
|
|
29
|
+
- password (str): Database password or key passphrase (if using Snowflake private key).
|
|
30
|
+
|
|
31
|
+
MSSQL-specific:
|
|
32
|
+
- driver (str): ODBC driver name (e.g., 'ODBC Driver 17 for SQL Server').
|
|
33
|
+
|
|
34
|
+
Snowflake-specific:
|
|
35
|
+
- account (str): Snowflake account identifier (e.g., 'abc-xy12345').
|
|
36
|
+
- warehouse (str): Snowflake warehouse name.
|
|
37
|
+
- private_key (str, optional): PEM-encoded private key string for key-pair authentication.
|
|
38
|
+
- passphrase (str, optional): Passphrase to decrypt the private key.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
self.dialect = self.parse_dialect(dialect)
|
|
42
|
+
|
|
43
|
+
host = quote_plus(params.get('host', params.get('server', '')))
|
|
44
|
+
database = quote_plus(params.get('database', ''))
|
|
45
|
+
username = quote_plus(params.get('username', params.get('user', '')))
|
|
46
|
+
password = quote_plus(params.get('password', ''))
|
|
47
|
+
driver = quote_plus(params.get('driver', ''))
|
|
48
|
+
if 'driver' not in params:
|
|
49
|
+
from al_data_utils import airflow_utils
|
|
50
|
+
|
|
51
|
+
driver = airflow_utils.get_variable_value('core_mssql_odbc_driver')
|
|
52
|
+
|
|
53
|
+
connect_args = {}
|
|
54
|
+
|
|
55
|
+
if self.dialect == SQLDialect.POSTGRESQL:
|
|
56
|
+
port = params.get('port', 5432)
|
|
57
|
+
self.connection_string = (
|
|
58
|
+
f'postgresql+psycopg2://{username}:{password}@{host}:{port}/{database}'
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
elif self.dialect == SQLDialect.MYSQL:
|
|
62
|
+
port = params.get('port', 3306)
|
|
63
|
+
charset = params.get('charset', 'utf8mb4')
|
|
64
|
+
self.connection_string = f'mysql+pymysql://{username}:{password}@{host}:{port}/{database}?charset={charset}'
|
|
65
|
+
|
|
66
|
+
elif self.dialect == SQLDialect.MSSQL:
|
|
67
|
+
port = params.get('port', 1433)
|
|
68
|
+
encrypt = params.get('encrypt', 'yes')
|
|
69
|
+
self.connection_string = f'mssql+pyodbc://{username}:{password}@{host}:{port}/{database}?driver={driver.replace(" ", "+")}&TrustServerCertificate=yes&Encrypt={encrypt}'
|
|
70
|
+
elif self.dialect == SQLDialect.SNOWFLAKE:
|
|
71
|
+
warehouse = params['warehouse']
|
|
72
|
+
account = params['account']
|
|
73
|
+
if 'private_key' in params:
|
|
74
|
+
from modules import key_auth_utils
|
|
75
|
+
|
|
76
|
+
private_key_bytes = key_auth_utils.get_private_key(
|
|
77
|
+
params['private_key'], params['key_password'], result_format='DER'
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
self.connection_string = (
|
|
81
|
+
f'snowflake://{username}@{account}?warehouse={warehouse}/{database}'
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
connect_args = {'private_key': private_key_bytes}
|
|
85
|
+
|
|
86
|
+
else:
|
|
87
|
+
self.connection_string = f'snowflake://{username}:{password}@{account}?warehouse={warehouse}/{database}'
|
|
88
|
+
|
|
89
|
+
else:
|
|
90
|
+
raise ValueError(f'Unsupported dialect: {dialect}')
|
|
91
|
+
|
|
92
|
+
self.engine = create_engine(self.connection_string, connect_args=connect_args)
|
|
93
|
+
|
|
94
|
+
@staticmethod
|
|
95
|
+
def parse_dialect(value: str | SQLDialect) -> SQLDialect:
|
|
96
|
+
if isinstance(value, SQLDialect):
|
|
97
|
+
return value
|
|
98
|
+
try:
|
|
99
|
+
return SQLDialect[value.upper()]
|
|
100
|
+
except KeyError as e:
|
|
101
|
+
raise ValueError(f'Unknown dialect: {value}') from e
|
|
102
|
+
|
|
103
|
+
def read_sql_to_df(self, query: str, params: dict | None = None) -> pd.DataFrame:
|
|
104
|
+
"""Run a query and return a pandas DataFrame."""
|
|
105
|
+
with self.engine.connect() as conn:
|
|
106
|
+
df = pd.read_sql(query, conn, params=params)
|
|
107
|
+
return df
|
|
108
|
+
|
|
109
|
+
def write_df_to_sql(
|
|
110
|
+
self,
|
|
111
|
+
df: pd.DataFrame,
|
|
112
|
+
table_name: str,
|
|
113
|
+
schema_name: str = 'dbo',
|
|
114
|
+
if_exists: str = 'append',
|
|
115
|
+
index: bool = False,
|
|
116
|
+
):
|
|
117
|
+
"""Write a pandas DataFrame to a SQL table."""
|
|
118
|
+
with self.engine.begin() as conn:
|
|
119
|
+
df.to_sql(
|
|
120
|
+
table_name,
|
|
121
|
+
schema=schema_name,
|
|
122
|
+
con=conn,
|
|
123
|
+
if_exists=if_exists,
|
|
124
|
+
index=index,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
def execute_command(self, command: str, params: dict | None = None) -> None:
|
|
128
|
+
"""Execute a raw SQL command (e.g., INSERT, UPDATE, or stored procedure)."""
|
|
129
|
+
with self.engine.begin() as conn:
|
|
130
|
+
conn.execute(text(command), params)
|
|
131
|
+
|
|
132
|
+
def execute_command_with_result_to_df(
|
|
133
|
+
self, query: str, params: dict | None = None
|
|
134
|
+
) -> pd.DataFrame:
|
|
135
|
+
"""Run a query and return a pandas DataFrame."""
|
|
136
|
+
with self.engine.begin() as conn:
|
|
137
|
+
df = pd.read_sql(query, conn, params=params)
|
|
138
|
+
return df
|
|
139
|
+
|
|
140
|
+
def execute_command_with_result_to_dicts(
|
|
141
|
+
self, query: str, params: dict[str, str] | None = None
|
|
142
|
+
) -> list[dict[str, str]]:
|
|
143
|
+
"""Run a query and return a list of dicts (one per row)."""
|
|
144
|
+
with self.engine.begin() as conn:
|
|
145
|
+
result = conn.execute(text(query), params or {})
|
|
146
|
+
if result.returns_rows:
|
|
147
|
+
dc = [dict(row) for row in result.mappings()]
|
|
148
|
+
else:
|
|
149
|
+
dc = []
|
|
150
|
+
return dc
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def tsql_nstr(s) -> str:
|
|
154
|
+
"""Make a safe NVARCHAR literal (N'...') or NULL."""
|
|
155
|
+
if s is None:
|
|
156
|
+
return 'NULL'
|
|
157
|
+
# double single quotes inside the literal
|
|
158
|
+
return "N'{}'".format(str(s).replace("'", "''"))
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from typing import Union
|
|
2
|
+
import requests
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class TelegramBot:
|
|
6
|
+
"""Telegram Bot API wrapper
|
|
7
|
+
|
|
8
|
+
Examples:
|
|
9
|
+
>>> TelegramBot('YOuRAwEsomeBOtToKen', '@myprettydebugchat').send_message('Hi, darling')
|
|
10
|
+
>>> TelegramBot('YOuRAwEsomeBOtToKen').send_message('Hi, darling', chat_id=-1762374628374)
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
API_ENDPOINT = 'https://api.telegram.org/bot{}/'
|
|
14
|
+
|
|
15
|
+
def __init__(self, token: str, default_chat_id: Union[int, str] = None):
|
|
16
|
+
self.token = token
|
|
17
|
+
self._base_url = TelegramBot.API_ENDPOINT.format(token)
|
|
18
|
+
self.default_chat_id = default_chat_id
|
|
19
|
+
|
|
20
|
+
def send_message(self, message: str, chat_id: Union[int, str] = None):
|
|
21
|
+
method = 'sendMessage'
|
|
22
|
+
|
|
23
|
+
response = requests.post(
|
|
24
|
+
url='https://api.telegram.org/bot{0}/{1}'.format(self.token, method),
|
|
25
|
+
data={'chat_id': chat_id or self.default_chat_id, 'text': message},
|
|
26
|
+
).json()
|
|
27
|
+
|
|
28
|
+
if not response.get('ok'):
|
|
29
|
+
raise TelegramBotException(response)
|
|
30
|
+
|
|
31
|
+
def on_fail_callback(self, context, chat_id: Union[int, str] = None):
|
|
32
|
+
message = f"""
|
|
33
|
+
😡 Task has failed.
|
|
34
|
+
Task: {context.get('task_instance').task_id}
|
|
35
|
+
DAG: {context.get('task_instance').dag_id}
|
|
36
|
+
Execution Time: {context.get('execution_date')}
|
|
37
|
+
Exception: {str(context.get('exception'))[:255]}
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
self.send_message(message=message, chat_id=chat_id or self.default_chat_id)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class TelegramBotException(Exception):
|
|
44
|
+
def __init__(self, *args, **kwargs):
|
|
45
|
+
super().__init__((args, kwargs))
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: al-data-utils
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Reusable utilities for data engineering (Airflow, Azure, Snowflake, SQL, config).
|
|
5
|
+
Author-email: AL Data Labs <pypi@aldatalabs.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://aldatalabs.com
|
|
8
|
+
Keywords: airflow,data-engineering,azure,snowflake,sql
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# al-data-utils
|
|
18
|
+
|
|
19
|
+
Reusable Python utilities for data and automation workflows.
|
|
20
|
+
|
|
21
|
+
This repository contains independent utility modules for sync operations, git automation, SQL work, logging helpers, and MSSQL schema snapshot generation.
|
|
22
|
+
|
|
23
|
+
## Python and Setup
|
|
24
|
+
|
|
25
|
+
- Python: 3.10+
|
|
26
|
+
- Packaging: setuptools (src layout)
|
|
27
|
+
|
|
28
|
+
Install locally in editable mode:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install -e .
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Project Structure
|
|
35
|
+
|
|
36
|
+
- src/al_data_utils
|
|
37
|
+
- examples
|
|
38
|
+
- development_guides
|
|
39
|
+
- tools
|
|
40
|
+
|
|
41
|
+
Main utility groups under src/al_data_utils:
|
|
42
|
+
|
|
43
|
+
- airflow_utils
|
|
44
|
+
- backup_utils
|
|
45
|
+
- config_utils
|
|
46
|
+
- files_sync_utils
|
|
47
|
+
- geo_utils
|
|
48
|
+
- git_utils
|
|
49
|
+
- logging_utils
|
|
50
|
+
- ones_api_utils
|
|
51
|
+
- sql_utils
|
|
52
|
+
- tg_utils
|
|
53
|
+
|
|
54
|
+
## Examples
|
|
55
|
+
|
|
56
|
+
Runnable examples are available in the `examples` folder for the main utility groups.
|
|
57
|
+
Use them as reference implementations for local scripts and automation jobs.
|
|
58
|
+
|
|
59
|
+
## MSSQL Schema Snapshot
|
|
60
|
+
|
|
61
|
+
The snapshot tool exports SQL objects from MSSQL into a filesystem layout and syncs staging output to destination folders.
|
|
62
|
+
|
|
63
|
+
Current behavior highlights:
|
|
64
|
+
|
|
65
|
+
- Supports single DB and multi-DB runs in one execution.
|
|
66
|
+
- Always writes outputs into per-DB folders.
|
|
67
|
+
- Uses staging first, then syncs to final output.
|
|
68
|
+
- Supports cleanup of staging folders.
|
|
69
|
+
- Supports sync log retention cleanup.
|
|
70
|
+
- Supports include and exclude filters per object type.
|
|
71
|
+
- Supports DataGrip template override files.
|
|
72
|
+
|
|
73
|
+
Output shape:
|
|
74
|
+
|
|
75
|
+
```text
|
|
76
|
+
<MSSQL_OUT_DIR>/<DatabaseName>/schemas/<schema>/
|
|
77
|
+
tables/
|
|
78
|
+
views/
|
|
79
|
+
routines/
|
|
80
|
+
sequences/
|
|
81
|
+
synonyms/
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### Snapshot Environment Variables
|
|
85
|
+
|
|
86
|
+
Core connection and output:
|
|
87
|
+
|
|
88
|
+
- MSSQL_OUT_DIR
|
|
89
|
+
- MSSQL_SERVER
|
|
90
|
+
- MSSQL_DATABASE
|
|
91
|
+
- MSSQL_DATABASES
|
|
92
|
+
- MSSQL_AUTH_MODE
|
|
93
|
+
- MSSQL_USER
|
|
94
|
+
- MSSQL_PASSWORD
|
|
95
|
+
- MSSQL_DRIVER
|
|
96
|
+
|
|
97
|
+
Snapshot behavior:
|
|
98
|
+
|
|
99
|
+
- MSSQL_ENABLE_TABLE_DEFINITIONS
|
|
100
|
+
- MSSQL_ENABLE_INDEXES
|
|
101
|
+
- MSSQL_ENABLE_FOREIGN_KEYS
|
|
102
|
+
- MSSQL_ENABLE_VIEWS
|
|
103
|
+
- MSSQL_ENABLE_STORED_PROCEDURES
|
|
104
|
+
- MSSQL_ENABLE_FUNCTIONS
|
|
105
|
+
- MSSQL_ENABLE_SEQUENCES
|
|
106
|
+
- MSSQL_ENABLE_SYNONYMS
|
|
107
|
+
- MSSQL_CLEANUP_STAGING
|
|
108
|
+
- MSSQL_SYNC_LOG_RETENTION_DAYS
|
|
109
|
+
|
|
110
|
+
Filters:
|
|
111
|
+
|
|
112
|
+
- MSSQL_INCLUDE_SCHEMAS
|
|
113
|
+
- MSSQL_EXCLUDE_SCHEMAS
|
|
114
|
+
- MSSQL_INCLUDE_TABLES
|
|
115
|
+
- MSSQL_EXCLUDE_TABLES
|
|
116
|
+
- MSSQL_INCLUDE_VIEWS
|
|
117
|
+
- MSSQL_EXCLUDE_VIEWS
|
|
118
|
+
- MSSQL_INCLUDE_STORED_PROCEDURES
|
|
119
|
+
- MSSQL_EXCLUDE_STORED_PROCEDURES
|
|
120
|
+
- MSSQL_INCLUDE_FUNCTIONS
|
|
121
|
+
- MSSQL_EXCLUDE_FUNCTIONS
|
|
122
|
+
- MSSQL_INCLUDE_SEQUENCES
|
|
123
|
+
- MSSQL_EXCLUDE_SEQUENCES
|
|
124
|
+
- MSSQL_INCLUDE_SYNONYMS
|
|
125
|
+
- MSSQL_EXCLUDE_SYNONYMS
|
|
126
|
+
|
|
127
|
+
Notes:
|
|
128
|
+
|
|
129
|
+
- If MSSQL_DATABASES is set, it is treated as a comma-separated list.
|
|
130
|
+
- If MSSQL_DATABASES is empty, MSSQL_DATABASE is used.
|
|
131
|
+
- If MSSQL_DATABASE contains commas, it is also treated as a list for backward compatibility.
|
|
132
|
+
|
|
133
|
+
## File Sync Utilities
|
|
134
|
+
|
|
135
|
+
`files_sync_utils` provides:
|
|
136
|
+
|
|
137
|
+
- Inventory building
|
|
138
|
+
- Plan generation
|
|
139
|
+
- Copy and update execution
|
|
140
|
+
- Extra destination cleanup rules
|
|
141
|
+
- Activity CSV logging
|
|
142
|
+
- Log retention cleanup
|
|
143
|
+
|
|
144
|
+
## Git Utilities
|
|
145
|
+
|
|
146
|
+
`git_utils` wraps common git automation operations with consistent behavior and error handling.
|
|
147
|
+
|
|
148
|
+
## Development Guides
|
|
149
|
+
|
|
150
|
+
- development_guides/dev_guide.txt
|
|
151
|
+
- development_guides/auth_key_guide.txt
|
|
152
|
+
|
|
153
|
+
These guides contain release flow notes and auth-related instructions.
|
|
154
|
+
|
|
155
|
+
## Changelog
|
|
156
|
+
|
|
157
|
+
See CHANGELOG.md for release notes.
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
al_data_utils/__init__.py,sha256=l5Mdphl5Gh_pIo8qYMTyUxTQAdkvuiGkCQwXPfkea9o,180
|
|
2
|
+
al_data_utils/airflow_utils/__init__.py,sha256=Yq-Yy9rNmm4NTC2Qo0XLT-j9-b4h4pKrkoXY6l1uOcI,283
|
|
3
|
+
al_data_utils/airflow_utils/airflow_utils.py,sha256=fcCRzVogkTDBnnRovX8_3Qv4pnkPnoBRiOVNq-B-Dac,2470
|
|
4
|
+
al_data_utils/azure_as_utils/__init__.py,sha256=AYWuh37QwiPdStM7uHvW2NFzjR1cIBxScSOAh8F9WpM,789
|
|
5
|
+
al_data_utils/azure_as_utils/auth.py,sha256=2NBJN6TOV5jUpahcj5uqw3LEQHA3iWyiwQyeDKBsvI8,2825
|
|
6
|
+
al_data_utils/azure_as_utils/azure_as_utils.py,sha256=mtkw47RDHCQ9ppFafdhn-p9rqL8HvHAvkCcCQGML-8U,8875
|
|
7
|
+
al_data_utils/azure_as_utils/client.py,sha256=GJGAb6lj5AuyKRAyVIAZ4kdAlfI5A7VKZea0MqorB6A,6162
|
|
8
|
+
al_data_utils/azure_as_utils/errors.py,sha256=IYU20hNhGtvoM_6LB9bYl1LOsh6Jn9TB_3hJP10TgYg,2513
|
|
9
|
+
al_data_utils/azure_as_utils/models.py,sha256=5Q9IvWFh5hgAbhs2INg9dI7NFdOccbh0AT_btJedIw0,10299
|
|
10
|
+
al_data_utils/azure_as_utils/poller.py,sha256=-JQSb5PEgk-AsbQTGX0zpU8iRUo6Wdpfcri9XnSSBdA,4626
|
|
11
|
+
al_data_utils/backup_utils/__init__.py,sha256=ivVRrdutosWUQ6az7qYaPKsOy1UNS9TLE9ixbaV4PjI,381
|
|
12
|
+
al_data_utils/backup_utils/mssql/dacpac/__init__.py,sha256=_tuqsMspDnkc8vM2BLIjyBe0dOwVonQcyYSlwupLihY,77
|
|
13
|
+
al_data_utils/backup_utils/mssql/dacpac/sqlpackage.py,sha256=NTTZjRBn_oL2HStOeGexTzx0ianDYvAAC4Ch6FEwWjg,6227
|
|
14
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/__init__.py,sha256=ek95VFS_psM3kpiC3g3oIJTBBebXSk4uPpeABN-Vgfg,215
|
|
15
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/config.py,sha256=HUGKDiz690qR1IRywmwR3S6zU1XBJYOL7ZqRShghU6M,5451
|
|
16
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/snapshot_processor.py,sha256=9lE7Naq2hl6PA0HUftlljx86VxKbZ7ZoW1Ql_2E68aE,26635
|
|
17
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/emit/__init__.py,sha256=qlHd-Kndr0IDI8iJNBBp_lshNkeGdVGDx3pBPdXUbKo,288
|
|
18
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/emit/foreign_keys.py,sha256=k2IQp4ByXavacvOvYXGc2ZXG64TROuHSUV_qVVv93to,1056
|
|
19
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/emit/indexes.py,sha256=gF8gEY09jsaVlpEt2ro6qikBaxLbqicHlQC_tVGf3Qw,1069
|
|
20
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/emit/sql_objects.py,sha256=47QVpNg2epyanB3EXT_BiAl_5Phy9pi2N0LRtbXrw_o,434
|
|
21
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/emit/tables.py,sha256=0S6V_1SvSKcA4nOr09do_6te1_hIGhTSvjDnSpcbDhA,4168
|
|
22
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/introspect/__init__.py,sha256=d6Yvh-6JO9QrxAI8N_WFLRd-a0XSmILnVNUNEGsVMBU,501
|
|
23
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/introspect/foreign_keys.py,sha256=-EyeL5YRPDuuWoG6caZfGS7IF3j0QhTQ84rv4NTgu2g,6149
|
|
24
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/introspect/functions.py,sha256=nZK9bqBwjM4uBSaKt_Gs8daj-NOmGLRRDUwSH-ygFrw,4194
|
|
25
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/introspect/indexes.py,sha256=MUs728kqTaGCb4l7fIl0lSGNNwpPa0Bbgjt0k-dqMR0,6534
|
|
26
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/introspect/sequences.py,sha256=Ps22XFiMUts2Ye7yV1vxzy6TxbGRUUnM2MPdS83mQpI,5847
|
|
27
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/introspect/stored_procedures.py,sha256=shdi_OZeOMXtUwwGso1msqMGGoI8gXjalw8ZK570PDU,10667
|
|
28
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/introspect/synonyms.py,sha256=rcyT82ddwvQd7RaFVZbnhkeaQPScRDwRLTbmAkV7fgY,4320
|
|
29
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/introspect/tables.py,sha256=5H7QgAyiUWTdP2DVhCtzLYETR_V6ReFitSdmkJxEfzM,8926
|
|
30
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/introspect/views.py,sha256=O3B6FQWmRr1BuYrNq7bE1PK69FjUlPbLqbLtH5ECi3g,8087
|
|
31
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/io/__init__.py,sha256=om0ki99fKptoalzw-It_bHUqGjAaJRe9jJL44io5qPU,275
|
|
32
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/io/layout.py,sha256=52u6V-vwg-PDBQxiLyTDb7G2dtQqE1bfsM38gvmDId8,624
|
|
33
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/io/writer.py,sha256=BiESwNLW_yZ9V6TceUcV0DOwV7ShesctfL3f77K190E,1071
|
|
34
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/models/__init__.py,sha256=Y6aIhnMlhjpuAQPFDe4buJxyfSBWRZbMCNvwHSIOvN0,353
|
|
35
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/models/foreign_key_def.py,sha256=tFB5_66UQg0EI6xqSaCNw90aNoDlhDS7DUAJD2ga5Q4,487
|
|
36
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/models/index_def.py,sha256=hjnKyacS3XCynXXh3lLhWv0aG6aPQNzF0UK2MIhIto4,518
|
|
37
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/models/sql_object_def.py,sha256=ehsn5Y5ZBK6i1FfiwXdU_HtMGiVUOYEiFOU1FnNkuds,257
|
|
38
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/models/table_column_def.py,sha256=WH9nipBAXwjWz3yjbz12Hh0dn0xzckrEmHuRkuPyRrU,603
|
|
39
|
+
al_data_utils/backup_utils/mssql/schema_snapshot/models/table_def.py,sha256=OPZ5ngP7XRAituFNQa0jaW6jVz_wtUziPfiZ9zJrl2k,453
|
|
40
|
+
al_data_utils/backup_utils/mssql/scripter/__init__.py,sha256=RkwDreYTHuYloJr2wpjPY_Ggp2LmhLrkanUtCKFwyv8,52
|
|
41
|
+
al_data_utils/config_utils/__init__.py,sha256=HrzEu11bJhikSvP6seI3dR1ZJnm4bqtNg8G930MLWLo,109
|
|
42
|
+
al_data_utils/config_utils/config_utils.py,sha256=DMiIK7HMdLUmLAUrrd36iYks0WUHXZsB7idxjyFKLhA,1005
|
|
43
|
+
al_data_utils/files_sync_utils/__init__.py,sha256=WzoysPTOXc6s8VZzDRnr95XZ5hj8vPULQsLr08momiU,268
|
|
44
|
+
al_data_utils/files_sync_utils/files_sync_utils.py,sha256=ehor1N634BqPLTZKxCPeM9a_tJNpjHicHq9q4n_dGTQ,19724
|
|
45
|
+
al_data_utils/geo_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
46
|
+
al_data_utils/git_utils/__init__.py,sha256=T26XLhi4BTpFyFpg0NNF4H7sbE5P6MGJ647mpWJROxs,487
|
|
47
|
+
al_data_utils/git_utils/git_utils.py,sha256=ovfCy9dxvA1pMe5e4sVNzjK8U0j8UHvNKgoeqnI1hF0,4213
|
|
48
|
+
al_data_utils/logging_utils/__init__.py,sha256=HsgS_gpsmsF4QOVacINzIsVHxVuAGpVWyXGI6gn5st0,129
|
|
49
|
+
al_data_utils/logging_utils/logging_utils.py,sha256=d-HB6fRDYqs3r7faFoJ9qt4yO_xEbS2H9ulE6K4JcwQ,992
|
|
50
|
+
al_data_utils/ones_api_utils/__init__.py,sha256=qSm5LIycuuyVs2q8AMM82dJbksLhLnIlfZYsYZPxYVI,72
|
|
51
|
+
al_data_utils/ones_api_utils/ones_api_utils.py,sha256=20CW5YdrWzObL8oFcdkvMnDQ8CN0gOcCAoeKd56zLwk,1984
|
|
52
|
+
al_data_utils/sql_utils/__init__.py,sha256=UF73sf9UD7hU5DIGEuY7x2GhqDYVIo9mpyb45vVUJqY,125
|
|
53
|
+
al_data_utils/sql_utils/sql_utils.py,sha256=wAH-BbFLb6x8uL5B6xE3xzM2giUt6qHcv1buV0tlbcQ,6375
|
|
54
|
+
al_data_utils/tg_utils/__init__.py,sha256=GqloghgtS6OgL6IRnqXTvYFs7FgxnL6vMcSMGcmyEtw,110
|
|
55
|
+
al_data_utils/tg_utils/tg_utils.py,sha256=eyk9v__Mh5pSQJp_9GHfmOQw1i9C7mdKYISI6-7I0TM,1551
|
|
56
|
+
al_data_utils-0.3.1.dist-info/licenses/LICENSE,sha256=FSoDw09XQtrZEbEgid_0dBku6vuuPj3r7XPxcTklOfo,1090
|
|
57
|
+
al_data_utils-0.3.1.dist-info/METADATA,sha256=k8E6oLadDmhS7EQ140u7cxsAZqn-xN5W7rbvK2GktXA,3827
|
|
58
|
+
al_data_utils-0.3.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
59
|
+
al_data_utils-0.3.1.dist-info/top_level.txt,sha256=Ade88LWXWws-LCIeYmUHHRRN6Eq8MTM0Di5BXxA6ir0,14
|
|
60
|
+
al_data_utils-0.3.1.dist-info/RECORD,,
|