openmetadata-managed-apis 1.6.10.0__py3-none-any.whl → 1.10.12.0__py3-none-any.whl

openmetadata_managed_apis/operations/last_dag_logs.py

@@ -1,8 +1,8 @@
-#  Copyright 2022 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -11,36 +11,67 @@
 """
 Module containing the logic to retrieve all logs from the tasks of a last DAG run
 """
-from functools import partial
+import os
+from functools import lru_cache, partial
 from io import StringIO
-from typing import List, Optional
+from typing import List, Optional, Tuple
 
 from airflow.models import DagModel, TaskInstance
 from airflow.utils.log.log_reader import TaskLogReader
 from flask import Response
 from openmetadata_managed_apis.api.response import ApiResponse
+from openmetadata_managed_apis.utils.logger import operations_logger
+
+logger = operations_logger()
 
 LOG_METADATA = {
     "download_logs": False,
 }
-# Make chunks of 2M characters
 CHUNK_SIZE = 2_000_000
+DOT_STR = "_DOT_"
 
 
-def last_dag_logs(dag_id: str, task_id: str, after: Optional[int] = None) -> Response:
-    """Validate that the DAG is registered by Airflow and have at least one Run.
-
-    If exists, returns all logs for each task instance of the last DAG run.
+@lru_cache(maxsize=10)
+def get_log_file_info(log_file_path: str, mtime: int) -> Tuple[int, int]:
+    """
+    Get total size and number of chunks for a log file.
+    :param log_file_path: Path to log file
+    :param mtime: File modification time in seconds (used as cache key)
+    :return: Tuple of (file_size_bytes, total_chunks)
+    """
+    file_size = os.path.getsize(log_file_path)
+    total_chunks = (file_size + CHUNK_SIZE - 1) // CHUNK_SIZE
+    return file_size, total_chunks
 
-    Args:
-        dag_id (str): DAG to look for
-        task_id (str): Task to fetch logs from
-        after (int): log stream cursor
 
-    Return:
-        Response with log and pagination
+def read_log_chunk_from_file(file_path: str, chunk_index: int) -> Optional[str]:
+    """
+    Read a specific chunk from a log file without loading entire file.
+    :param file_path: Path to the log file
+    :param chunk_index: 0-based chunk index to read
+    :return: Log chunk content or None if error
     """
+    try:
+        offset = chunk_index * CHUNK_SIZE
+        with open(file_path, "r", encoding="utf-8", errors="replace") as f:
+            f.seek(offset)
+            chunk = f.read(CHUNK_SIZE)
+        return chunk
+    except Exception as exc:
+        logger.warning(f"Failed to read log chunk from {file_path}: {exc}")
+        return None
+
 
+def last_dag_logs(dag_id: str, task_id: str, after: Optional[int] = None) -> Response:
+    """
+    Validate that the DAG is registered by Airflow and have at least one Run.
+    If exists, returns all logs for each task instance of the last DAG run.
+    Uses file streaming to avoid loading entire log file into memory.
+        :param dag_id: DAG to look for
+        :param task_id: Task to fetch logs from
+        :param after: log stream cursor
+        :return: Response with log and pagination
+    """
     dag_model = DagModel.get_dagmodel(dag_id=dag_id)
 
     if not dag_model:
@@ -58,32 +89,116 @@ def last_dag_logs(dag_id: str, task_id: str, after: Optional[int] = None) -> Res
             f"Cannot find any task instance for the last DagRun of {dag_id}."
         )
 
-    raw_logs_str = None
-
+    target_task_instance = None
     for task_instance in task_instances:
-        # Only fetch the required logs
         if task_instance.task_id == task_id:
-            # Pick up the _try_number, otherwise they are adding 1
-            try_number = task_instance._try_number  # pylint: disable=protected-access
+            target_task_instance = task_instance
+            break
+
+    if not target_task_instance:
+        return ApiResponse.bad_request(f"Task {task_id} not found in DAG {dag_id}.")
+
+    try_number = target_task_instance._try_number  # pylint: disable=protected-access
+
+    task_log_reader = TaskLogReader()
+    if not task_log_reader.supports_read:
+        return ApiResponse.server_error("Task Log Reader does not support read logs.")
+
+    # Try to use file streaming for better performance
+    try:
+
+        from airflow.configuration import (  # pylint: disable=import-outside-toplevel
+            conf,
+        )
 
-            task_log_reader = TaskLogReader()
-            if not task_log_reader.supports_read:
-                return ApiResponse.server_error(
-                    "Task Log Reader does not support read logs."
+        base_log_folder = conf.get("logging", "base_log_folder")
+        # dag_id and task_id are already sanitized at route level
+        # Only dots are replaced for Airflow log path compatibility
+        dag_id_safe = dag_id.replace(".", DOT_STR)
+        task_id_safe = task_id.replace(".", DOT_STR)
+
+        log_relative_path = f"dag_id={dag_id_safe}/run_id={last_dag_run.run_id}/task_id={task_id_safe}/attempt={try_number}.log"
+        log_file_path = os.path.join(base_log_folder, log_relative_path)
+
+        # Security: Validate the resolved path stays within base_log_folder
+        # to prevent directory traversal attacks. This provides defense-in-depth
+        # even though dag_id and task_id are already sanitized at the route level.
+        log_file_path_real = os.path.realpath(log_file_path)
+        base_log_folder_real = os.path.realpath(base_log_folder)
+
+        if not log_file_path_real.startswith(base_log_folder_real + os.sep):
+            logger.warning(
+                f"Path traversal attempt detected: {log_file_path} is outside {base_log_folder}"
+            )
+            return ApiResponse.bad_request(
+                f"Invalid log path for DAG {dag_id} and Task {task_id}."
+            )
+
+        if os.path.exists(log_file_path_real):
+            stat_info = os.stat(log_file_path_real)
+            file_mtime = int(stat_info.st_mtime)
+
+            _, total_chunks = get_log_file_info(log_file_path_real, file_mtime)
+
+            after_idx = int(after) if after is not None else 0
+
+            if after_idx >= total_chunks:
+                return ApiResponse.bad_request(
+                    f"After index {after} is out of bounds. Total pagination is {total_chunks} for DAG {dag_id} and Task {task_id}."
                 )
 
-            # Even when generating a ton of logs, we just get a single element.
-            # Same happens when trying to call task_log_reader.read_log_chunks
-            # We'll create our own chunk size and paginate based on that
-            raw_logs_str = "".join(
-                list(
-                    task_log_reader.read_log_stream(
-                        ti=task_instance,
-                        try_number=try_number,
-                        metadata=LOG_METADATA,
-                    )
+            chunk_content = read_log_chunk_from_file(log_file_path_real, after_idx)
+
+            if chunk_content is not None:
+                return ApiResponse.success(
+                    {
+                        task_id: chunk_content,
+                        "total": total_chunks,
+                        **(
+                            {"after": after_idx + 1}
+                            if after_idx < total_chunks - 1
+                            else {}
+                        ),
+                    }
                 )
+    except Exception as exc:
+        logger.debug(
+            f"File streaming failed for DAG {dag_id}, falling back to TaskLogReader: {exc}"
+        )
+
+    # Fallback to TaskLogReader if streaming fails
+    return _last_dag_logs_fallback(
+        dag_id, task_id, after, target_task_instance, task_log_reader, try_number
+    )
+
+
+def _last_dag_logs_fallback(
+    dag_id: str,
+    task_id: str,
+    after: Optional[int],
+    task_instance: TaskInstance,
+    task_log_reader: TaskLogReader,
+    try_number: int,
+) -> Response:
+    """
+    Fallback to reading entire log file into memory (old behavior).
+    :param dag_id: DAG to look for
+    :param task_id: Task to fetch logs from
+    :param after: log stream cursor
+    :param task_instance: Task instance to fetch logs from
+    :param task_log_reader: TaskLogReader instance
+    :param try_number: Task attempt number
+    :return: API Response
+    """
+    raw_logs_str = "".join(
+        list(
+            task_log_reader.read_log_stream(
+                ti=task_instance,
+                try_number=try_number,
+                metadata=LOG_METADATA,
             )
+        )
+    )
 
     if not raw_logs_str:
         return ApiResponse.bad_request(

openmetadata_managed_apis/operations/state.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

openmetadata_managed_apis/operations/status.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

openmetadata_managed_apis/operations/trigger.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,17 +17,20 @@ try:
     from airflow.api.common.trigger_dag import trigger_dag
 except ImportError:
     from airflow.api.common.experimental.trigger_dag import trigger_dag
+
 from airflow.utils import timezone
 from flask import Response
 from openmetadata_managed_apis.api.response import ApiResponse
 
 
-def trigger(dag_id: str, run_id: Optional[str]) -> Response:
+def trigger(
+    dag_id: str, run_id: Optional[str], conf: Optional[dict] = None
+) -> Response:
     dag_run = trigger_dag(
         dag_id=dag_id,
         run_id=run_id,
-        conf=None,
         execution_date=timezone.utcnow(),
+        conf=conf,
     )
     return ApiResponse.success(
         {"message": f"Workflow [{dag_id}] has been triggered {dag_run}"}

openmetadata_managed_apis/plugin.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

openmetadata_managed_apis/utils/parser.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

openmetadata_managed_apis/views/rest_api.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

openmetadata_managed_apis/workflows/config.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

openmetadata_managed_apis/workflows/ingestion/application.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,6 +18,7 @@ from openmetadata_managed_apis.utils.logger import set_operator_logger
 from openmetadata_managed_apis.workflows.ingestion.common import (
     build_dag,
     build_workflow_config_property,
+    execute_workflow,
 )
 
 from metadata.generated.schema.entity.applications.configuration.applicationConfig import (
@@ -36,7 +37,7 @@ from metadata.generated.schema.metadataIngestion.applicationPipeline import (
 from metadata.workflow.application import ApplicationWorkflow
 
 
-def application_workflow(workflow_config: OpenMetadataApplicationConfig):
+def application_workflow(workflow_config: OpenMetadataApplicationConfig, **context):
     """
     Task that creates and runs the ingestion workflow.
 
@@ -48,13 +49,17 @@ def application_workflow(workflow_config: OpenMetadataApplicationConfig):
 
     set_operator_logger(workflow_config)
 
-    config = json.loads(workflow_config.model_dump_json(exclude_defaults=False))
+    # set overridden app config
+    config = json.loads(
+        workflow_config.model_dump_json(exclude_defaults=False, mask_secrets=False)
+    )
+    params = context.get("params") or {}
+    config["appConfig"] = {
+        **(config.get("appConfig") or {}),
+        **(params.get("appConfigOverride") or {}),
+    }
     workflow = ApplicationWorkflow.create(config)
-
-    workflow.execute()
-    workflow.raise_from_status()
-    workflow.print_status()
-    workflow.stop()
+    execute_workflow(workflow, workflow_config)
 
 
 def build_application_workflow_config(
@@ -84,6 +89,7 @@ def build_application_workflow_config(
         else None,
         workflowConfig=build_workflow_config_property(ingestion_pipeline),
         ingestionPipelineFQN=ingestion_pipeline.fullyQualifiedName.root,
+        enableStreamableLogs=ingestion_pipeline.enableStreamableLogs,
     )
 
     return application_workflow_config
@@ -99,6 +105,9 @@ def build_application_dag(ingestion_pipeline: IngestionPipeline) -> DAG:
         ingestion_pipeline=ingestion_pipeline,
         workflow_config=application_workflow_config,
         workflow_fn=application_workflow,
+        params={
+            "appConfigOverride": None  # Default to None, will be overridden by trigger conf
+        },
     )
 
     return dag

openmetadata_managed_apis/workflows/ingestion/auto_classification.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,7 +15,11 @@ import json
 
 from airflow import DAG
 from openmetadata_managed_apis.utils.logger import set_operator_logger
-from openmetadata_managed_apis.workflows.ingestion.common import build_dag, build_source
+from openmetadata_managed_apis.workflows.ingestion.common import (
+    build_dag,
+    build_source,
+    execute_workflow,
+)
 
 from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
     IngestionPipeline,
@@ -30,7 +34,9 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.workflow.classification import AutoClassificationWorkflow
 
 
-def auto_classification_workflow(workflow_config: OpenMetadataWorkflowConfig):
+def auto_classification_workflow(
+    workflow_config: OpenMetadataWorkflowConfig,
+):
     """
     Task that creates and runs the auto classification workflow.
 
@@ -42,13 +48,11 @@ def auto_classification_workflow(workflow_config: OpenMetadataWorkflowConfig):
 
     set_operator_logger(workflow_config)
 
-    config = json.loads(workflow_config.model_dump_json(exclude_defaults=False))
+    config = json.loads(
+        workflow_config.model_dump_json(exclude_defaults=False, mask_secrets=False)
+    )
     workflow = AutoClassificationWorkflow.create(config)
-
-    workflow.execute()
-    workflow.raise_from_status()
-    workflow.print_status()
-    workflow.stop()
+    execute_workflow(workflow, workflow_config)
 
 
 def build_auto_classification_workflow_config(
@@ -72,6 +76,7 @@ def build_auto_classification_workflow_config(
             openMetadataServerConfig=ingestion_pipeline.openMetadataServerConnection,
         ),
         ingestionPipelineFQN=ingestion_pipeline.fullyQualifiedName.root,
+        enableStreamableLogs=ingestion_pipeline.enableStreamableLogs,
     )
 
     return workflow_config

openmetadata_managed_apis/workflows/ingestion/common.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -37,6 +37,7 @@ from metadata.generated.schema.metadataIngestion.application import (
 from metadata.generated.schema.type.basic import Timestamp, Uuid
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils import fqn
+from metadata.workflow.base import BaseWorkflow
 
 # pylint: disable=ungrouped-imports
 try:
@@ -44,6 +45,7 @@ try:
 except ModuleNotFoundError:
     from airflow.operators.python_operator import PythonOperator
 
+from croniter import croniter
 from openmetadata_managed_apis.utils.logger import set_operator_logger, workflow_logger
 from openmetadata_managed_apis.utils.parser import (
     parse_service_connection,
@@ -191,6 +193,18 @@ def build_source(ingestion_pipeline: IngestionPipeline) -> WorkflowSource:
     )
 
 
+def execute_workflow(
+    workflow: BaseWorkflow, workflow_config: OpenMetadataWorkflowConfig
+) -> None:
+    """
+    Execute the workflow and handle the status
+    """
+    workflow.execute()
+    workflow.stop()
+    if workflow_config.workflowConfig.raiseOnError:
+        workflow.raise_from_status()
+
+
 def metadata_ingestion_workflow(workflow_config: OpenMetadataWorkflowConfig):
     """
     Task that creates and runs the ingestion workflow.
@@ -203,13 +217,11 @@ def metadata_ingestion_workflow(workflow_config: OpenMetadataWorkflowConfig):
 
     set_operator_logger(workflow_config)
 
-    config = json.loads(workflow_config.model_dump_json(exclude_defaults=False))
+    config = json.loads(
+        workflow_config.model_dump_json(exclude_defaults=False, mask_secrets=False)
+    )
     workflow = MetadataWorkflow.create(config)
-
-    workflow.execute()
-    workflow.raise_from_status()
-    workflow.print_status()
-    workflow.stop()
+    execute_workflow(workflow, workflow_config)
 
 
 def build_workflow_config_property(
@@ -222,6 +234,7 @@ def build_workflow_config_property(
     """
     return WorkflowConfig(
         loggerLevel=ingestion_pipeline.loggerLevel or LogLevels.INFO,
+        raiseOnError=ingestion_pipeline.raiseOnError,
         openMetadataServerConfig=ingestion_pipeline.openMetadataServerConnection,
     )
 
@@ -247,11 +260,19 @@ def build_dag_configs(ingestion_pipeline: IngestionPipeline) -> dict:
     :param ingestion_pipeline: pipeline configs
     :return: dict to use as kwargs
     """
-
-    if ingestion_pipeline.airflowConfig.startDate:
-        start_date = ingestion_pipeline.airflowConfig.startDate.root
+    # Determine start_date based on schedule_interval using croniter
+    schedule_interval = ingestion_pipeline.airflowConfig.scheduleInterval
+    now = datetime.now()
+
+    if schedule_interval is None:
+        # On-demand DAG, set start_date to now
+        start_date = now
+    elif croniter.is_valid(schedule_interval):
+        cron = croniter(schedule_interval, now)
+        start_date = cron.get_prev(datetime)
     else:
-        start_date = datetime.now() - timedelta(days=1)
+        # Handle invalid cron expressions if necessary
+        start_date = now
 
     return {
         "dag_id": clean_dag_id(ingestion_pipeline.name.root),
@@ -272,7 +293,7 @@ def build_dag_configs(ingestion_pipeline: IngestionPipeline) -> dict:
         "is_paused_upon_creation": ingestion_pipeline.airflowConfig.pausePipeline
         or False,
         "catchup": ingestion_pipeline.airflowConfig.pipelineCatchup or False,
-        "schedule_interval": ingestion_pipeline.airflowConfig.scheduleInterval,
+        "schedule_interval": schedule_interval,
         "tags": [
             "OpenMetadata",
             clean_name_tag(ingestion_pipeline.displayName)
@@ -347,9 +368,16 @@ def build_dag(
     ingestion_pipeline: IngestionPipeline,
     workflow_config: Union[OpenMetadataWorkflowConfig, OpenMetadataApplicationConfig],
     workflow_fn: Callable,
+    params: Optional[dict] = None,
 ) -> DAG:
     """
     Build a simple metadata workflow DAG
+    :param task_name: Name of the task
+    :param ingestion_pipeline: Pipeline configs
+    :param workflow_config: Workflow configurations
+    :param workflow_fn: Function to be executed
+    :param params: Optional parameters to pass to the operator
+    :return: DAG
     """
 
     with DAG(**build_dag_configs(ingestion_pipeline)) as dag:
@@ -360,7 +388,9 @@ def build_dag(
         CustomPythonOperator(
             task_id=task_name,
             python_callable=workflow_fn,
-            op_kwargs={"workflow_config": workflow_config},
+            op_kwargs={
+                "workflow_config": workflow_config,
+            },
             # There's no need to retry if we have had an error. Wait until the next schedule or manual rerun.
             retries=ingestion_pipeline.airflowConfig.retries or 0,
             # each DAG will call its own OpenMetadataWorkflowConfig
@@ -369,6 +399,7 @@ def build_dag(
             owner=ingestion_pipeline.owners.root[0].name
             if (ingestion_pipeline.owners and ingestion_pipeline.owners.root)
             else "openmetadata",
+            params=params,
         )
 
         return dag

openmetadata_managed_apis/workflows/ingestion/dbt.py

@@ -1,8 +1,8 @@
 #  Copyright 2022 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -42,7 +42,7 @@ def build_dbt_workflow_config(
     """
 
     source = build_source(ingestion_pipeline)
-    source.type = f"dbt"  # Mark the source as dbt
+    source.type = "dbt"  # Mark the source as dbt
 
     workflow_config = OpenMetadataWorkflowConfig(
         source=source,
@@ -52,6 +52,7 @@ def build_dbt_workflow_config(
         ),
         workflowConfig=build_workflow_config_property(ingestion_pipeline),
         ingestionPipelineFQN=ingestion_pipeline.fullyQualifiedName.root,
+        enableStreamableLogs=ingestion_pipeline.enableStreamableLogs,
     )
 
     return workflow_config

openmetadata_managed_apis/workflows/ingestion/elasticsearch_sink.py

@@ -1,8 +1,8 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
+#  Copyright 2025 Collate
+#  Licensed under the Collate Community License, Version 1.0 (the "License");
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
+#  https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.