odibi 2.5.0__py3-none-any.whl

odibi/orchestration/airflow.py

@@ -0,0 +1,90 @@
+import re
+
+from jinja2 import Template
+
+from odibi.config import ProjectConfig
+
+AIRFLOW_DAG_TEMPLATE = """
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+from datetime import datetime, timedelta
+
+# Generated by Odibi
+# Project: {{ project_name }}
+# Pipeline: {{ pipeline_name }}
+
+default_args = {
+    'owner': '{{ owner }}',
+    'depends_on_past': False,
+    'start_date': datetime(2023, 1, 1),
+    'email_on_failure': False,
+    'email_on_retry': False,
+    'retries': {{ retries }},
+    'retry_delay': timedelta(minutes=5),
+}
+
+with DAG(
+    '{{ dag_id }}',
+    default_args=default_args,
+    description='{{ description }}',
+    schedule_interval=None, # Set schedule manually or via config
+    catchup=False,
+    tags=['odibi', '{{ layer }}'],
+) as dag:
+
+    # --- Nodes ---
+{% for node in nodes %}
+    {{ node.safe_name }} = BashOperator(
+        task_id='{{ node.name }}',
+        bash_command='odibi run --pipeline {{ pipeline_name }} --node {{ node.name }}',
+    )
+{% endfor %}
+
+    # --- Dependencies ---
+{% for node in nodes %}
+    {% if node.upstream_vars %}
+    # {{ node.name }} depends on {{ node.depends_on }}
+    [{{ node.upstream_vars|join(', ') }}] >> {{ node.safe_name }}
+    {% endif %}
+{% endfor %}
+"""
+
+
+class AirflowExporter:
+    def __init__(self, config: ProjectConfig):
+        self.config = config
+
+    def _sanitize(self, name: str) -> str:
+        return re.sub(r"[^a-zA-Z0-9_]", "_", name)
+
+    def generate_code(self, pipeline_name: str) -> str:
+        # Find pipeline
+        pipeline = next((p for p in self.config.pipelines if p.pipeline == pipeline_name), None)
+        if not pipeline:
+            raise ValueError(f"Pipeline '{pipeline_name}' not found in config")
+
+        nodes_ctx = []
+        for node in pipeline.nodes:
+            safe_name = self._sanitize(node.name)
+            upstream_vars = [self._sanitize(dep) for dep in node.depends_on]
+
+            nodes_ctx.append(
+                {
+                    "name": node.name,
+                    "safe_name": safe_name,
+                    "depends_on": node.depends_on,
+                    "upstream_vars": upstream_vars,
+                }
+            )
+
+        template = Template(AIRFLOW_DAG_TEMPLATE)
+        return template.render(
+            project_name=self.config.project,
+            pipeline_name=pipeline.pipeline,
+            dag_id=f"odibi_{pipeline.pipeline}",
+            owner=self.config.owner or "airflow",
+            description=pipeline.description or "Odibi Pipeline",
+            layer=pipeline.layer or "default",
+            retries=self.config.retry.max_attempts if self.config.retry.enabled else 0,
+            nodes=nodes_ctx,
+        )

odibi/orchestration/dagster.py

@@ -0,0 +1,77 @@
+from odibi.config import ProjectConfig
+
+try:
+    from dagster import (
+        AssetExecutionContext,
+        Definitions,
+        asset,
+    )
+except ImportError:
+    # Dagster is optional
+    pass
+
+
+class DagsterFactory:
+    """
+    Factory to create Dagster definitions from Odibi configuration.
+
+    Usage in definitions.py:
+        from odibi.config import load_config
+        from odibi.orchestration.dagster import DagsterFactory
+
+        config = load_config("odibi.yaml")
+        defs = DagsterFactory(config).create_definitions()
+    """
+
+    def __init__(self, config: ProjectConfig):
+        self.config = config
+
+    def create_definitions(self) -> "Definitions":
+        if "dagster" not in globals():
+            raise ImportError("Dagster not installed. Run 'pip install dagster'")
+
+        all_assets = []
+
+        for pipeline in self.config.pipelines:
+            for node in pipeline.nodes:
+                # Create an asset for each node
+                # We use dynamic function creation to bind specific node/pipeline
+
+                asset_name = node.name.replace("-", "_")
+                deps = [dep.replace("-", "_") for dep in node.depends_on]
+                group = pipeline.pipeline
+
+                # Define the asset function
+                def make_asset_fn(p_name, n_name):
+                    @asset(
+                        name=asset_name,
+                        deps=deps,
+                        group_name=group,
+                        description=node.description,
+                        compute_kind="odibi",
+                        op_tags={"odibi/pipeline": p_name, "odibi/node": n_name},
+                    )
+                    def _asset_fn(context: AssetExecutionContext):
+                        # Run Odibi Node
+                        # We shell out to CLI to ensure clean environment,
+                        # OR import runner. Shell out is safer for isolation.
+                        import subprocess
+
+                        context.log.info(f"Running Odibi node: {n_name} in pipeline {p_name}")
+                        cmd = ["odibi", "run", "--pipeline", p_name, "--node", n_name]
+
+                        result = subprocess.run(cmd, capture_output=True, text=True)
+
+                        if result.stdout:
+                            context.log.info(result.stdout)
+                        if result.stderr:
+                            context.log.error(result.stderr)
+
+                        if result.returncode != 0:
+                            raise Exception(f"Odibi execution failed: {result.stderr}")
+
+                    return _asset_fn
+
+                all_assets.append(make_asset_fn(pipeline.pipeline, node.name))
+
+        return Definitions(assets=all_assets)

odibi/patterns/__init__.py

@@ -0,0 +1,24 @@
+from typing import Dict, Type
+
+from odibi.patterns.aggregation import AggregationPattern
+from odibi.patterns.base import Pattern
+from odibi.patterns.date_dimension import DateDimensionPattern
+from odibi.patterns.dimension import DimensionPattern
+from odibi.patterns.fact import FactPattern
+from odibi.patterns.merge import MergePattern
+from odibi.patterns.scd2 import SCD2Pattern
+
+_PATTERNS: Dict[str, Type[Pattern]] = {
+    "scd2": SCD2Pattern,
+    "merge": MergePattern,
+    "dimension": DimensionPattern,
+    "date_dimension": DateDimensionPattern,
+    "aggregation": AggregationPattern,
+    "fact": FactPattern,
+}
+
+
+def get_pattern_class(name: str) -> Type[Pattern]:
+    if name not in _PATTERNS:
+        raise ValueError(f"Unknown pattern: '{name}'. Available: {list(_PATTERNS.keys())}")
+    return _PATTERNS[name]