al-data-core 0.6.2__py3-none-any.whl

al_data_core/__init__.py

@@ -0,0 +1,5 @@
+from al_data_core.config_processor import DataCoreConfig
+from al_data_core.tasks_processor import TaskProcessor
+from al_data_core.external_handler_base import ExternalHandlerBase, discover_handlers
+
+__all__ = ['DataCoreConfig', 'TaskProcessor', 'ExternalHandlerBase', 'discover_handlers']

al_data_core/__main__.py

@@ -0,0 +1,4 @@
+from al_data_core.dags_installer import main
+
+if __name__ == '__main__':
+    main()

al_data_core/azure_as_processor.py

@@ -0,0 +1,34 @@
+from al_data_utils import airflow_utils
+from al_data_utils.logging_utils import info
+
+
+def run_azure_as_refresh(azure_as_cfg: dict, model_cfg: dict) -> None:
+    """
+    Refresh a single Azure Analysis Services semantic model.
+
+    Parameters
+    ----------
+    azure_as_cfg : dict
+        Must contain ``connection_variable`` — the Airflow variable name that
+        holds a dict with keys: tenant_id, client_id, client_secret, rollout,
+        server_name.
+    model_cfg : dict
+        Must contain ``name`` (model name) and ``refresh_request`` (the XMLA
+        refresh payload dict passed to SemanticModels.refresh_models).
+    """
+    from al_data_utils.azure_as_utils import SemanticModels
+
+    conn = airflow_utils.get_variable_value_dict(azure_as_cfg['connection_variable'])
+
+    sm = SemanticModels(
+        tenant_id=conn['tenant_id'],
+        client_id=conn['client_id'],
+        client_secret=conn['client_secret'],
+        rollout=conn['rollout'],
+        server_name=conn['server_name'],
+    )
+
+    model_name = model_cfg['name']
+    info(f'Starting Azure AS refresh for model: {model_name}')
+    sm.refresh_models([(model_name, model_cfg['refresh_request'])])
+    info(f'Azure AS refresh completed for model: {model_name}')

al_data_core/config_processor.py

@@ -0,0 +1,359 @@
+from pathlib import Path
+import os
+import json
+
+from al_data_utils import config_utils
+from al_data_utils.logging_utils import info, warning
+from al_data_utils import airflow_utils
+from al_data_utils.sql_utils import UnifiedSQLClient, SQLDialect
+
+
+class DataCoreConfig:
+    """
+    Loads project-specific data_core_config.yaml and manages ETL queue,
+    task execution parameters, and logging via the config DB.
+
+    Parameters
+    ----------
+    config_file_path : str
+        Absolute path to data_core_config.yaml.
+    sql_queries_path : str
+        Absolute path to the root of the sql_queries folder
+        (i.e. the folder that contains scope sub-folders such as ods_wms/, dw_crm/).
+    """
+
+    def __init__(self, config_file_path: str, sql_queries_path: str):
+        self.config = {}
+        self.etl_queue = {}
+        self.config_file_path = config_file_path
+        self.sql_queries_global_path = sql_queries_path
+
+        self.get_config_yaml()
+
+        config_connection_variable = self.config['CONFIG']['config_connection_variable']
+        self.conn_config = airflow_utils.get_variable_value_dict(config_connection_variable)
+
+        mappings = self.config['CONFIG']['mappings']
+        self.map_config_get_tables_list = mappings['config_get_tables_list']
+        self.map_source_config_parameters = mappings['config_parameters']
+        self.map_source_config_fn_get_param_values = mappings['config_fn_get_param_values']
+        self.map_source_config_calc_date_ranges = mappings['config_calc_date_ranges']
+        self.map_source_config_update_param_values = mappings['config_update_param_values']
+
+        self.conn = UnifiedSQLClient(dialect=SQLDialect.MSSQL, params=self.conn_config)
+        info(
+            f'DataCoreConfig initialized: config_file_path={self.config_file_path}, '
+            f'sql_queries_path={self.sql_queries_global_path}, '
+            f'config_connection_variable={config_connection_variable}'
+        )
+
+    # ------------------------------------------------------------------
+    # Config helpers
+    # ------------------------------------------------------------------
+
+    def get_config_yaml(self):
+        info(f'Loading config yaml from {self.config_file_path}')
+        self.config = config_utils.load_config(self.config_file_path)
+
+    # ------------------------------------------------------------------
+    # ETL queue
+    # ------------------------------------------------------------------
+
+    def get_etl_queue(self, scope_codes: list, param_id: int = -1) -> list[dict]:
+        """Return rows from the config DB that describe which tasks to run."""
+        where_condition = (
+            f' tbls.param_id = {param_id}'
+            if param_id != -1
+            else f""" tbls.scope_code IN ({', '.join(repr(code) for code in scope_codes)})
+              AND tbls.enabled = 1"""
+        )
+
+        sql_query = f"""
+select param_id,
+       scope_code,
+       seq_code,
+       param_code,
+       enabled,
+       destination_table,
+       select_query,
+       param_value_1,
+       param_value_2
+from {self.map_config_get_tables_list} as tbls
+where
+    {where_condition}
+order by seq_code, scope_code, param_code
+        """
+
+        result = self.conn.execute_command_with_result_to_dicts(sql_query)
+        info(
+            f'get_etl_queue(scope_codes={scope_codes}, param_id={param_id}) '
+            f'returned {len(result)} row(s)'
+        )
+        return result
+
+    def get_task_execution_params(self, param_id: int) -> list[dict]:
+        sql_query = f"""
+select params.param_id as param_id,
+       rng.param1      as param_value_1,
+       rng.param2      as param_value_2
+from {self.map_source_config_parameters} (nolock) as params
+         cross apply {self.map_source_config_fn_get_param_values}(params.scope_code,
+                                                             params.param_code,
+                                                             null, 0) as params_value
+         cross apply {self.map_source_config_calc_date_ranges}(params_value.param_value_1,
+                                                           params_value.param_value_2,
+                                                           params.date_range_limit) as rng
+where params.param_id = {param_id}
+order by rng.param1
+"""
+        result = self.conn.execute_command_with_result_to_dicts(sql_query)
+        info(f'get_task_execution_params(param_id={param_id}) returned {len(result)} row(s)')
+        return result
+
+    # ------------------------------------------------------------------
+    # SQL file helpers
+    # ------------------------------------------------------------------
+
+    def get_file_content(self, scope_code: str, param_code: str) -> str:
+        """Read the .sql file for a given scope/param from the sql_queries folder."""
+        file_path = os.path.join(
+            self.sql_queries_global_path, scope_code.lower(), f'{param_code}.sql'
+        )
+
+        if os.path.exists(file_path):
+            info(f'Loading SQL file: {file_path}')
+            with open(file_path, 'r', encoding='utf-8') as f:
+                return f.read()
+
+        # fallback: try exact case
+        file_path = os.path.join(
+            self.sql_queries_global_path, scope_code, f'{param_code}.sql'
+        )
+        info(f'Lowercased scope path not found, falling back to exact case: {file_path}')
+        with open(file_path, 'r', encoding='utf-8') as f:
+            return f.read()
+
+    def get_task_detail_from_config(self, scope_code: str, param_code: str) -> dict:
+        info(f'Getting task detail for scope_code={scope_code}, param_code={param_code}')
+        self.get_config_yaml()
+        task_details = dict(self.config.get(scope_code, {}))
+        task_details['command'] = self.get_file_content(scope_code, param_code)
+        return task_details
+
+    def get_scope_config(self, scope_code: str) -> dict:
+        """
+        Return the top-level YAML section for scope_code (e.g. enabled,
+        handler, source_type), without loading a .sql command file. Used
+        for dispatch decisions that apply before a SQL file is relevant,
+        such as TaskProcessor checking for a handler folder.
+        """
+        self.get_config_yaml()
+        return dict(self.config.get(scope_code, {}))
+
+    def get_source_connection_variable(self, scope_code: str) -> str:
+        return self.config[scope_code]['source_connection_variable']
+
+    def get_destination_connection_variable(self, scope_code: str) -> str:
+        return self.config[scope_code]['destination_connection_variable']
+
+    def get_sources_config(self):
+        """
+        Populate self.etl_queue for every enabled source in the YAML that
+        declares a sql_source_folder. Used by projects that scan all sources
+        at once rather than driving tasks from the config DB queue.
+        """
+        self.get_config_yaml()
+        for k, v in self.config.items():
+            if not v.get('enabled', ''):
+                info(f'Source {k} is not enabled, skipping')
+                continue
+            self.etl_queue[k] = {}
+            if 'sql_source_folder' in v:
+                self.etl_queue[k]['source_connection_variable'] = v.get(
+                    'source_connection_variable', ''
+                )
+                self.etl_queue[k]['destination_connection_variable'] = v.get(
+                    'destination_connection_variable', ''
+                )
+                self.scan_for_queries(v['sql_source_folder'], source_name=k)
+            else:
+                info(f'Source {k} has no sql_source_folder configured, skipping query scan')
+
+    def scan_for_queries(
+        self,
+        sql_source_folder: str,
+        source_name: str,
+        include_subfolders: bool = True,
+        root_key: str = '',
+    ):
+        """
+        Walk sql_queries_global_path/sql_source_folder and load all .sql files
+        (excluding files prefixed with _disabled) into self.etl_queue[source_name]['queries'].
+        """
+        sql_path = os.path.join(self.sql_queries_global_path, sql_source_folder)
+        base = Path(sql_path)
+        pattern = '**/*.sql' if include_subfolders else '*.sql'
+
+        self.etl_queue.setdefault(source_name, {}).setdefault('queries', {})
+
+        loaded, skipped = 0, 0
+        for p in sorted(base.glob(pattern)):
+            if not p.is_file():
+                continue
+            name = p.stem
+            if not name.startswith('_disabled'):
+                with p.open('r', encoding='utf-8') as f:
+                    self.etl_queue[source_name]['queries'][name] = f.read()
+                loaded += 1
+            else:
+                skipped += 1
+
+        info(
+            f'scan_for_queries({sql_path}) for source_name={source_name}: '
+            f'loaded {loaded} query file(s), skipped {skipped} disabled file(s)'
+        )
+
+    # ------------------------------------------------------------------
+    # Tasks config file (pre-generation for DAG structure)
+    # ------------------------------------------------------------------
+
+    def run_generate_dag_tasks_config_file(self, destination_config_file: str):
+        """
+        Query the config DB and write a YAML file describing all DAG tasks.
+        Call this once (e.g. via a setup task or CLI) before the DAG runs,
+        so the DAG file can build its task graph without hitting the DB at
+        Airflow scheduler parse time.
+        """
+        dags_list = self.config.get('DAGS', {})
+        config_dict = {}
+
+        if not dags_list:
+            warning(f'No DAGS section found in {self.config_file_path}, nothing to generate')
+
+        info(f'Generating tasks config for {len(dags_list)} DAG(s): {list(dags_list.keys())}')
+
+        for dag_name, dag_details in dags_list.items():
+            if not dag_details:
+                warning(
+                    f'DAG {dag_name}: empty configuration under DAGS in '
+                    f'{self.config_file_path} (check YAML indentation/content under this key), '
+                    'skipping'
+                )
+                continue
+
+            scope_codes = dag_details.get('scopes', [])
+            schedule_interval = dag_details.get('schedule_interval', '')
+            tags = dag_details.get('tags', [])
+
+            if not scope_codes:
+                warning(f'DAG {dag_name}: no "scopes" defined, this DAG will never get any tasks')
+
+            unknown_scopes = [code for code in scope_codes if code not in self.config]
+            if unknown_scopes:
+                warning(
+                    f'DAG {dag_name}: scope_codes {unknown_scopes} have no matching top-level '
+                    f'section in {self.config_file_path} — check for typos'
+                )
+
+            if not schedule_interval:
+                warning(f'DAG {dag_name}: no "schedule_interval" defined')
+
+            dag_node = config_dict.setdefault(dag_name, {})
+            azure_as = dag_details.get('azure_as', None)
+            dag_node['params'] = {'schedule_interval': schedule_interval, 'tags': tags, 'azure_as': azure_as}
+            dag_tasks = dag_node.setdefault('tasks', {})
+
+            info(
+                f'DAG {dag_name}: scope_codes={scope_codes}, '
+                f'schedule_interval={schedule_interval}, tags={tags}, azure_as={azure_as}'
+            )
+
+            task_count = 0
+            for t in self.get_etl_queue(scope_codes=scope_codes):
+                seq_code = t['seq_code']
+                scope_code = t['scope_code']
+                param_code = t['param_code']
+                param_id = t['param_id']
+
+                task_name = f'{scope_code}__{param_code}'
+                record = {'param_code': param_code, 'param_id': param_id}
+
+                dag_tasks.setdefault(seq_code, {}).setdefault(scope_code, {})[
+                    task_name
+                ] = record
+                task_count += 1
+
+            info(f'DAG {dag_name}: added {task_count} task(s)')
+            if task_count == 0:
+                warning(
+                    f'DAG {dag_name}: no tasks found for scope_codes={scope_codes} — '
+                    'check that the config DB has enabled rows for these scopes'
+                )
+
+        import yaml as _yaml
+        os.makedirs(os.path.dirname(destination_config_file), exist_ok=True)
+        with open(destination_config_file, 'w', encoding='utf-8') as f:
+            _yaml.dump(config_dict, f, default_flow_style=False, allow_unicode=True)
+
+        print(f'Tasks config written to: {destination_config_file}')
+        info(
+            f'Generated DAG tasks config file:\n\n{_yaml.dump(config_dict, default_flow_style=False, allow_unicode=True)}\n'
+        )
+
+    # ------------------------------------------------------------------
+    # Logging
+    # ------------------------------------------------------------------
+
+    def log_insert_record(self, scope_code: str, event_name: str) -> int:
+        sql_query = f"""declare @log_id int;
+        exec log.upsert_events @operation = 'I', @scope_code = '{scope_code}', @event_name = '{event_name}', @log_id = @log_id Output;
+            select @log_id as log_id;
+        """
+
+        result = self.conn.execute_command_with_result_to_dicts(sql_query)
+        info(
+            f'Log record created for scope_code={scope_code}, event_name={event_name}, '
+            f'log_id={result[0].get("log_id", 0) if result else -1}'
+        )
+
+        if result:
+            return int(result[0].get('log_id', 0))
+        return -1
+
+    def log_update_record(self, results: dict, param_id: int = -1) -> None:
+        sql_query = f"""
+        exec log.upsert_events
+            @operation = 'U',
+            @log_id = {results.get('log_id', 0)},
+            @inserted = {results.get('ins_rc', 0)},
+            @updated = {results.get('upd_rc', 0)},
+            @deleted = {results.get('del_rc', 0)},
+            @event_desc = '{results.get('event_desc', '')}',
+            @error_code = {results.get('error_code', 0)},
+            @error_message = '{results.get('error_message', '')}';
+        """
+
+        self.conn.execute_command(sql_query)
+        info(f'Log updated for log_id={results.get("log_id", 0)}')
+
+        max_param_value = results.get('max_param_value', None)
+        if max_param_value is not None and param_id != -1:
+            sql_update_param = f"""
+declare @error_code int;
+declare @error_message nvarchar(4000);
+declare @upd_date_to datetime = null;
+declare @upd_date_fr datetime = null;
+declare @result int;
+
+exec @result = {self.map_source_config_update_param_values}
+            {param_id},
+            @upd_date_to,
+            @upd_date_fr,
+            '{max_param_value}',
+            @error_code output,
+            @error_message output;
+"""
+            self.conn.execute_command(sql_update_param)
+            info(
+                f'Parameter last_param_value updated for param_id={param_id} to {max_param_value}'
+            )

al_data_core/dags_installer.py

@@ -0,0 +1,131 @@
+"""
+al-data-core DAGs installer CLI
+
+Usage:
+    al-data-core copy-dags <target_dir> [--version <ver>] [--force]
+    al-data-core list-versions
+    python -m al_data_core copy-dags <target_dir> [--version <ver>]
+
+Commands:
+    copy-dags       Copy data_core_dag.py and data_core_config_engine_dag.py
+                    to the specified Airflow dags folder.
+                    Existing files are overwritten if contents differ.
+                    Use --force to overwrite regardless of content.
+    list-versions   List all available template versions.
+"""
+
+import argparse
+import filecmp
+import shutil
+from pathlib import Path
+
+
+def _templates_dir() -> Path:
+    return Path(__file__).parent / 'templates'
+
+
+def _version_key(name: str) -> tuple:
+    stripped = name.lstrip('v')
+    try:
+        return tuple(int(x) for x in stripped.split('.'))
+    except ValueError:
+        return (0, 0, 0)
+
+
+def _available_versions() -> list[str]:
+    base = _templates_dir()
+    versions = [p.name for p in base.iterdir() if p.is_dir()]
+    return sorted(versions, key=_version_key)
+
+
+def _latest_version() -> str:
+    versions = _available_versions()
+    if not versions:
+        raise RuntimeError('No template versions found in the package.')
+    return versions[-1]
+
+
+def _copy_dags(target: str, version: str | None, force: bool = False) -> None:
+    ver = version or _latest_version()
+    src_dir = _templates_dir() / ver
+
+    if not src_dir.exists():
+        available = ', '.join(_available_versions()) or 'none'
+        raise SystemExit(
+            f"Version '{ver}' not found. Available versions: {available}"
+        )
+
+    dest_dir = Path(target)
+    dest_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f'Copying templates version {ver} -> {dest_dir}\n')
+    copied = updated = skipped = 0
+    for src in sorted(src_dir.glob('*.py')):
+        dest = dest_dir / src.name
+        if dest.exists() and not force:
+            if filecmp.cmp(src, dest, shallow=False):
+                print(f'  skip    {src.name}  (identical)')
+                skipped += 1
+            else:
+                shutil.copy2(src, dest)
+                print(f'  update  {src.name}  -> {dest}')
+                updated += 1
+        else:
+            shutil.copy2(src, dest)
+            print(f'  copy    {src.name}  -> {dest}')
+            copied += 1
+
+    print(f'\nDone. {copied} copied, {updated} updated, {skipped} skipped.')
+
+
+def _list_versions() -> None:
+    versions = _available_versions()
+    if not versions:
+        print('No template versions found.')
+        return
+    latest = versions[-1]
+    for v in versions:
+        marker = '  (latest)' if v == latest else ''
+        print(f'  {v}{marker}')
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog='al-data-core',
+        description='al-data-core DAGs installer',
+    )
+    sub = parser.add_subparsers(dest='command', required=True)
+
+    copy_cmd = sub.add_parser(
+        'copy-dags',
+        help='Copy template DAG files to the Airflow dags folder',
+    )
+    copy_cmd.add_argument(
+        'target',
+        nargs='?',
+        default='/opt/airflow/dags',
+        help='Destination directory (default: /opt/airflow/dags)',
+    )
+    copy_cmd.add_argument(
+        '--version',
+        default=None,
+        help='Template version to copy (default: latest)',
+    )
+    copy_cmd.add_argument(
+        '--force',
+        action='store_true',
+        help='Overwrite existing files even if contents are identical',
+    )
+
+    sub.add_parser('list-versions', help='List available template versions')
+
+    args = parser.parse_args()
+
+    if args.command == 'copy-dags':
+        _copy_dags(args.target, args.version, args.force)
+    elif args.command == 'list-versions':
+        _list_versions()
+
+
+if __name__ == '__main__':
+    main()

al_data_core/external_handler_base.py

@@ -0,0 +1,97 @@
+import importlib.util
+import inspect
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class HandlerContext:
+    """Everything a handler needs, passed in by the engine at run time.
+
+    Handlers must not hardcode connection names, schemas, paths, etc. — read
+    them from ``self.context`` (populated by TaskProcessor before ``proceed()``).
+    """
+
+    scope_code: str = ''
+    param_code: str = ''
+    scope_config: dict = field(default_factory=dict)
+    config_file_path: str = ''
+    params_details: dict = field(default_factory=dict)
+    debug: bool = False
+
+
+class ExternalHandlerBase(ABC):
+    """
+    Base class for project-specific task handlers (any scope whose logic is
+    just "run this project-supplied code").
+
+    Subclass this in your project, in the folder named by the scope's
+    ``handler`` value in data_core_config.yaml (e.g.
+    ``handler: data_core_handler_api/britix24``). Two layouts are supported:
+
+    * one file per param_code (file name == param_code) — one task, one file;
+    * a single generic handler serving every param_code in the scope — when the
+      folder holds exactly one handler file, it is used for all of the scope's
+      tasks and distinguishes them via ``self.context.param_code``.
+
+    Before calling ``proceed()``, the engine constructs the handler with no
+    arguments and sets ``self.context`` (a HandlerContext). Read run-time
+    settings from there rather than hardcoding them::
+
+        class BitrixIngestHandler(ExternalHandlerBase):
+            def proceed(self) -> None:
+                scope = self.context.scope_config
+                entity = scope['entities'][self.context.param_code]
+                ...
+
+    discover_handlers() picks the class up automatically — no manual
+    registration. The constructor must work with no arguments.
+    """
+
+    # Set by TaskProcessor after construction, before proceed().
+    context: 'HandlerContext | None' = None
+
+    @abstractmethod
+    def proceed(self) -> None: ...
+
+
+def discover_handlers(folder_path: str) -> dict[str, type[ExternalHandlerBase]]:
+    """
+    Scan folder_path for `<param_code>.py` modules and return
+    {param_code: HandlerClass} for use as TaskProcessor's handler_map.
+
+    Each module must define exactly one ExternalHandlerBase subclass. Files
+    whose name starts with '_' (e.g. __init__.py) are skipped. Missing
+    folder_path is not an error — handlers are optional per project.
+    """
+    handler_map: dict[str, type[ExternalHandlerBase]] = {}
+    folder = Path(folder_path)
+    if not folder.is_dir():
+        return handler_map
+
+    for file_path in sorted(folder.glob('*.py')):
+        if file_path.stem.startswith('_'):
+            continue
+
+        spec = importlib.util.spec_from_file_location(
+            f'al_data_core._external_handlers.{file_path.stem}', file_path
+        )
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+        handlers = [
+            obj
+            for _, obj in inspect.getmembers(module, inspect.isclass)
+            if issubclass(obj, ExternalHandlerBase) and obj is not ExternalHandlerBase
+        ]
+
+        if len(handlers) != 1:
+            raise ValueError(
+                f'{file_path} must define exactly one ExternalHandlerBase subclass, '
+                f'found {len(handlers)}'
+            )
+
+        handler_map[file_path.stem] = handlers[0]
+
+    return handler_map