al-data-core 0.6.2__tar.gz

al_data_core-0.6.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 AL Data Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

al_data_core-0.6.2/PKG-INFO

@@ -0,0 +1,167 @@
+Metadata-Version: 2.4
+Name: al-data-core
+Version: 0.6.2
+Summary: Generic data pipeline engine for Airflow (config, processor, DAG factory).
+Author-email: AL Data Labs <pypi@aldatalabs.com>
+License-Expression: MIT
+Project-URL: Homepage, https://aldatalabs.com
+Keywords: airflow,data-engineering,etl,dag
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: al-data-utils>=0.3.1
+Requires-Dist: pandas
+Dynamic: license-file
+
+# al-data-core
+
+Generic Airflow data pipeline engine. Install once, configure per project.
+
+## Python and Setup
+
+- Python: 3.10+
+- Packaging: setuptools (src layout)
+
+Install locally in editable mode:
+
+```bash
+pip install -e .
+```
+
+Requires `al-data-utils` to be installed first.
+
+## Project Structure
+
+- src/al_data_core
+
+Main modules under `src/al_data_core`:
+
+- `config_processor` — `DataCoreConfig`
+- `tasks_processor` — `TaskProcessor`
+- `azure_as_processor` — `run_azure_as_refresh`
+- `external_handler_base` — `ExternalHandlerBase`, `discover_handlers` (used for XLS and API handlers)
+- `dags_installer` — CLI for copying DAG templates
+
+## DAG Templates
+
+After installing, copy the ready-to-use DAG templates to your Airflow dags folder:
+
+```bash
+al-data-core copy-dags                          # latest → /opt/airflow/dags
+al-data-core copy-dags /my/dags                 # custom path
+al-data-core copy-dags --version v2.0.0         # specific Airflow version
+al-data-core list-versions                      # show available versions
+```
+
+Available versions:
+
+- `v2.0.0` — Airflow 2.x
+- `v3.0.0` — Airflow 3.x
+
+Two files are copied:
+
+- `data_core_dag.py` — main DAG file, reads `data_core_tasks_config.yaml` and registers all project DAGs
+- `data_core_config_engine_dag.py` — runs every 15 min, queries config DB and YAML, writes `data_core_tasks_config.yaml`
+
+## Project Layout (per-project repo)
+
+```text
+dags/
+├── data_core_dag.py                    <- copied from template
+├── data_core_config_engine_dag.py      <- copied from template
+└── data_core/
+    ├── data_core_config.yaml           <- project config
+    ├── .configs/data_core/data_core_tasks_config.yaml  <- generated by config engine dag
+    ├── data_core_handler_sql/
+    │   └── sql_queries/
+    │       └── <scope>/
+    │           └── <param_code>.sql
+    ├── data_core_handler_xls/          <- optional, project-specific
+    └── data_core_handler_api/          <- optional, project-specific
+```
+
+## data_core_config.yaml
+
+```yaml
+CONFIG:
+  config_connection_variable: my_mssql_secret   # Airflow variable name
+  mappings:
+    config_get_tables_list:          integrations_db.config.get_tables_list
+    config_parameters:               integrations_db.config.parameters
+    config_fn_get_param_values:      integrations_db.config.fn_get_param_values
+    config_calc_date_ranges:         integrations_db.config.calc_date_ranges
+    config_update_param_values:      integrations_db.config.update_param_values
+
+DAGS:
+  my_project_ods_dag:
+    schedule_interval: '5 */1 * * *'
+    scopes: ['ODS_SOURCE']
+    tags: ['my_project']
+  my_project_dw_dag:
+    schedule_interval: '30 */1 * * *'
+    scopes: ['DW_TARGET']
+    tags: ['my_project']
+    azure_as:                         # optional: refresh semantic model after ETL
+      connection_variable: my_azure_as_secret
+      models:
+        - name: MyModel
+          refresh_request: {Type: Full, CommitMode: transactional, MaxParallelism: 2, RetryCount: 2, Objects: []}
+
+ODS_SOURCE:
+  enabled: true
+  source_type: mssql
+  source_connection_variable: source_mssql_secret
+  destination_connection_variable: dest_mssql_secret
+
+DW_TARGET:
+  enabled: true
+  destination_connection_variable: dest_mssql_secret
+
+XLS:
+  enabled: true
+  handler: data_core_handler_xls   # opts this scope into handler-based dispatch instead of seq_code-based copy/merge
+
+ods_britix_api:
+  enabled: true
+  handler: data_core_handler_api/britix24
+```
+
+## Handlers
+
+A scope opts into handler-based dispatch by setting `handler` to its handlers folder (relative to `data_core_config.yaml`, or an absolute path) — the value *is* the path, so `TaskProcessor` doesn't hardcode scope_code names and adding a new handler scope needs only config, never a template change. Two layouts are supported:
+
+* **One file per param_code** — file name matches the task's `param_code` exactly.
+* **One generic handler per scope** — when the folder holds a single handler file, it serves every param_code in the scope and distinguishes tasks via `self.context.param_code`.
+
+```python
+# dags/data_core/data_core_handler_xls/fact_sales_plan.py
+from al_data_core.external_handler_base import ExternalHandlerBase
+
+class MySalesHandler(ExternalHandlerBase):
+    def proceed(self) -> None:
+        ...
+```
+
+```python
+# dags/data_core/data_core_handler_api/britix24/bitrix_ingest.py
+from al_data_core.external_handler_base import ExternalHandlerBase
+
+class BitrixIngestHandler(ExternalHandlerBase):
+    def proceed(self) -> None:
+        scope = self.context.scope_config
+        entity = scope['entities'][self.context.param_code]
+        ...
+```
+
+`TaskProcessor` constructs the handler with no arguments, sets `handler.context` to a `HandlerContext` (scope_code, param_code, scope_config, config_file_path, params_details, debug), then calls `proceed()` — handlers read run-time settings from `self.context` instead of hardcoding them. Discovery happens at task-run time, not DAG-parse time, so a broken handler file can't fail the whole DAG file's parse and handler modules aren't re-imported every scheduler cycle. Each file must define exactly one `ExternalHandlerBase` subclass.
+
+## Development Guides
+
+- `RELEASING.md` — release flow and Azure Blob publish steps
+
+## Changelog
+
+See `CHANGELOG.md` for release notes.

al_data_core-0.6.2/README.md

@@ -0,0 +1,149 @@
+# al-data-core
+
+Generic Airflow data pipeline engine. Install once, configure per project.
+
+## Python and Setup
+
+- Python: 3.10+
+- Packaging: setuptools (src layout)
+
+Install locally in editable mode:
+
+```bash
+pip install -e .
+```
+
+Requires `al-data-utils` to be installed first.
+
+## Project Structure
+
+- src/al_data_core
+
+Main modules under `src/al_data_core`:
+
+- `config_processor` — `DataCoreConfig`
+- `tasks_processor` — `TaskProcessor`
+- `azure_as_processor` — `run_azure_as_refresh`
+- `external_handler_base` — `ExternalHandlerBase`, `discover_handlers` (used for XLS and API handlers)
+- `dags_installer` — CLI for copying DAG templates
+
+## DAG Templates
+
+After installing, copy the ready-to-use DAG templates to your Airflow dags folder:
+
+```bash
+al-data-core copy-dags                          # latest → /opt/airflow/dags
+al-data-core copy-dags /my/dags                 # custom path
+al-data-core copy-dags --version v2.0.0         # specific Airflow version
+al-data-core list-versions                      # show available versions
+```
+
+Available versions:
+
+- `v2.0.0` — Airflow 2.x
+- `v3.0.0` — Airflow 3.x
+
+Two files are copied:
+
+- `data_core_dag.py` — main DAG file, reads `data_core_tasks_config.yaml` and registers all project DAGs
+- `data_core_config_engine_dag.py` — runs every 15 min, queries config DB and YAML, writes `data_core_tasks_config.yaml`
+
+## Project Layout (per-project repo)
+
+```text
+dags/
+├── data_core_dag.py                    <- copied from template
+├── data_core_config_engine_dag.py      <- copied from template
+└── data_core/
+    ├── data_core_config.yaml           <- project config
+    ├── .configs/data_core/data_core_tasks_config.yaml  <- generated by config engine dag
+    ├── data_core_handler_sql/
+    │   └── sql_queries/
+    │       └── <scope>/
+    │           └── <param_code>.sql
+    ├── data_core_handler_xls/          <- optional, project-specific
+    └── data_core_handler_api/          <- optional, project-specific
+```
+
+## data_core_config.yaml
+
+```yaml
+CONFIG:
+  config_connection_variable: my_mssql_secret   # Airflow variable name
+  mappings:
+    config_get_tables_list:          integrations_db.config.get_tables_list
+    config_parameters:               integrations_db.config.parameters
+    config_fn_get_param_values:      integrations_db.config.fn_get_param_values
+    config_calc_date_ranges:         integrations_db.config.calc_date_ranges
+    config_update_param_values:      integrations_db.config.update_param_values
+
+DAGS:
+  my_project_ods_dag:
+    schedule_interval: '5 */1 * * *'
+    scopes: ['ODS_SOURCE']
+    tags: ['my_project']
+  my_project_dw_dag:
+    schedule_interval: '30 */1 * * *'
+    scopes: ['DW_TARGET']
+    tags: ['my_project']
+    azure_as:                         # optional: refresh semantic model after ETL
+      connection_variable: my_azure_as_secret
+      models:
+        - name: MyModel
+          refresh_request: {Type: Full, CommitMode: transactional, MaxParallelism: 2, RetryCount: 2, Objects: []}
+
+ODS_SOURCE:
+  enabled: true
+  source_type: mssql
+  source_connection_variable: source_mssql_secret
+  destination_connection_variable: dest_mssql_secret
+
+DW_TARGET:
+  enabled: true
+  destination_connection_variable: dest_mssql_secret
+
+XLS:
+  enabled: true
+  handler: data_core_handler_xls   # opts this scope into handler-based dispatch instead of seq_code-based copy/merge
+
+ods_britix_api:
+  enabled: true
+  handler: data_core_handler_api/britix24
+```
+
+## Handlers
+
+A scope opts into handler-based dispatch by setting `handler` to its handlers folder (relative to `data_core_config.yaml`, or an absolute path) — the value *is* the path, so `TaskProcessor` doesn't hardcode scope_code names and adding a new handler scope needs only config, never a template change. Two layouts are supported:
+
+* **One file per param_code** — file name matches the task's `param_code` exactly.
+* **One generic handler per scope** — when the folder holds a single handler file, it serves every param_code in the scope and distinguishes tasks via `self.context.param_code`.
+
+```python
+# dags/data_core/data_core_handler_xls/fact_sales_plan.py
+from al_data_core.external_handler_base import ExternalHandlerBase
+
+class MySalesHandler(ExternalHandlerBase):
+    def proceed(self) -> None:
+        ...
+```
+
+```python
+# dags/data_core/data_core_handler_api/britix24/bitrix_ingest.py
+from al_data_core.external_handler_base import ExternalHandlerBase
+
+class BitrixIngestHandler(ExternalHandlerBase):
+    def proceed(self) -> None:
+        scope = self.context.scope_config
+        entity = scope['entities'][self.context.param_code]
+        ...
+```
+
+`TaskProcessor` constructs the handler with no arguments, sets `handler.context` to a `HandlerContext` (scope_code, param_code, scope_config, config_file_path, params_details, debug), then calls `proceed()` — handlers read run-time settings from `self.context` instead of hardcoding them. Discovery happens at task-run time, not DAG-parse time, so a broken handler file can't fail the whole DAG file's parse and handler modules aren't re-imported every scheduler cycle. Each file must define exactly one `ExternalHandlerBase` subclass.
+
+## Development Guides
+
+- `RELEASING.md` — release flow and Azure Blob publish steps
+
+## Changelog
+
+See `CHANGELOG.md` for release notes.

al_data_core-0.6.2/pyproject.toml

@@ -0,0 +1,58 @@
+[project]
+name = "al-data-core"
+version = "0.6.2"
+description = "Generic data pipeline engine for Airflow (config, processor, DAG factory)."
+authors = [{ name = "AL Data Labs", email = "pypi@aldatalabs.com" }]
+license = "MIT"
+license-files = ["LICENSE"]
+keywords = ["airflow", "data-engineering", "etl", "dag"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+    "Topic :: Software Development :: Libraries",
+]
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "al-data-utils>=0.3.1",
+    "pandas",
+]
+
+[project.urls]
+Homepage = "https://aldatalabs.com"
+
+
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project.scripts]
+al-data-core = "al_data_core.dags_installer:main"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+al_data_core = ["templates/**/*.py"]
+
+# Ruff configuration
+[tool.ruff]
+exclude = [
+    ".git",
+    ".venv",
+]
+
+target-version = "py313"
+
+line-length = 88
+indent-width = 4
+
+
+[tool.ruff.format]
+quote-style = "single"
+indent-style = "space"
+skip-magic-trailing-comma = true
+line-ending = "auto"

al_data_core-0.6.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

al_data_core-0.6.2/src/al_data_core/__init__.py

@@ -0,0 +1,5 @@
+from al_data_core.config_processor import DataCoreConfig
+from al_data_core.tasks_processor import TaskProcessor
+from al_data_core.external_handler_base import ExternalHandlerBase, discover_handlers
+
+__all__ = ['DataCoreConfig', 'TaskProcessor', 'ExternalHandlerBase', 'discover_handlers']

al_data_core-0.6.2/src/al_data_core/__main__.py

@@ -0,0 +1,4 @@
+from al_data_core.dags_installer import main
+
+if __name__ == '__main__':
+    main()

al_data_core-0.6.2/src/al_data_core/azure_as_processor.py

@@ -0,0 +1,34 @@
+from al_data_utils import airflow_utils
+from al_data_utils.logging_utils import info
+
+
+def run_azure_as_refresh(azure_as_cfg: dict, model_cfg: dict) -> None:
+    """
+    Refresh a single Azure Analysis Services semantic model.
+
+    Parameters
+    ----------
+    azure_as_cfg : dict
+        Must contain ``connection_variable`` — the Airflow variable name that
+        holds a dict with keys: tenant_id, client_id, client_secret, rollout,
+        server_name.
+    model_cfg : dict
+        Must contain ``name`` (model name) and ``refresh_request`` (the XMLA
+        refresh payload dict passed to SemanticModels.refresh_models).
+    """
+    from al_data_utils.azure_as_utils import SemanticModels
+
+    conn = airflow_utils.get_variable_value_dict(azure_as_cfg['connection_variable'])
+
+    sm = SemanticModels(
+        tenant_id=conn['tenant_id'],
+        client_id=conn['client_id'],
+        client_secret=conn['client_secret'],
+        rollout=conn['rollout'],
+        server_name=conn['server_name'],
+    )
+
+    model_name = model_cfg['name']
+    info(f'Starting Azure AS refresh for model: {model_name}')
+    sm.refresh_models([(model_name, model_cfg['refresh_request'])])
+    info(f'Azure AS refresh completed for model: {model_name}')