openmetadata-managed-apis 1.7.0.1__tar.gz → 1.7.1.1__tar.gz

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openmetadata_managed_apis
-Version: 1.7.0.1
+Version: 1.7.1.1
 Summary: Airflow REST APIs to create and manage DAGS
 Author: OpenMetadata Committers
 License:                                  Apache License
@@ -229,11 +229,15 @@ OpenMetadata workflow definition and manage DAGS and tasks.
 
 ## Development
 
-You can run `make branch=issue-3659-v2 test_up` and specify any branch from OpenMetadata that you'd
-need to test the changes in the APIs. This will prepare a separated airflow container.
+The file [`development/airflow/airflow.cfg`](./development/airflow/airflow.cfg) contains configuration which runs based on
+the airflow server deployed by the quick-start and development compose files.
 
-The command will build the image by downloading the branch changes inside the container. This helps us
-test the REST APIs using some ongoing changes on OpenMetadata as well.
+You ca run the following command to start the development environment:
+
+```bash
+export AIRFLOW_HOME=$(pwd)/openmetadata-airflow-managed-api/development/airflow
+airflow webserver
+```
 
 ## Requirements
 

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/README.md

@@ -5,11 +5,15 @@ OpenMetadata workflow definition and manage DAGS and tasks.
 
 ## Development
 
-You can run `make branch=issue-3659-v2 test_up` and specify any branch from OpenMetadata that you'd
-need to test the changes in the APIs. This will prepare a separated airflow container.
+The file [`development/airflow/airflow.cfg`](./development/airflow/airflow.cfg) contains configuration which runs based on
+the airflow server deployed by the quick-start and development compose files.
 
-The command will build the image by downloading the branch changes inside the container. This helps us
-test the REST APIs using some ongoing changes on OpenMetadata as well.
+You ca run the following command to start the development environment:
+
+```bash
+export AIRFLOW_HOME=$(pwd)/openmetadata-airflow-managed-api/development/airflow
+airflow webserver
+```
 
 ## Requirements
 

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis/api/routes/trigger.py

@@ -16,7 +16,11 @@ from typing import Callable
 
 from flask import Blueprint, Response, request
 from openmetadata_managed_apis.api.response import ApiResponse
-from openmetadata_managed_apis.api.utils import get_request_arg, get_request_dag_id
+from openmetadata_managed_apis.api.utils import (
+    get_request_arg,
+    get_request_conf,
+    get_request_dag_id,
+)
 from openmetadata_managed_apis.operations.trigger import trigger
 from openmetadata_managed_apis.utils.logger import routes_logger
 
@@ -41,13 +45,14 @@ def get_fn(blueprint: Blueprint) -> Callable:
     @security.requires_access([(permissions.ACTION_CAN_EDIT, permissions.RESOURCE_DAG)])
     def trigger_dag() -> Response:
         """
-        Trigger a dag run
+        Trigger a dag run with optional configuration
         """
         dag_id = get_request_dag_id()
 
         try:
             run_id = get_request_arg(request, "run_id", raise_missing=False)
-            response = trigger(dag_id, run_id)
+            conf = get_request_conf()
+            response = trigger(dag_id, run_id, conf=conf)
 
             return response
 

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis/api/utils.py

@@ -99,6 +99,16 @@ def get_request_dag_id() -> Optional[str]:
     return clean_dag_id(raw_dag_id)
 
 
+def get_request_conf() -> Optional[dict]:
+    """
+    Try to fetch the conf from the JSON request. Return None if no conf is provided.
+    """
+    try:
+        return request.get_json().get("conf")
+    except Exception:
+        return None
+
+
 def get_dagbag():
     """
     Load the dagbag from Airflow settings

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis/operations/deploy.py

@@ -97,6 +97,8 @@ class DagDeployer:
         Store the airflow pipeline config in a JSON file and
         return the path for the Jinja rendering.
         """
+        # Create directory if it doesn't exist
+        dag_config_file_path.parent.mkdir(parents=True, exist_ok=True)
 
         logger.info(f"Saving file to {dag_config_file_path}")
         with open(dag_config_file_path, "w") as outfile:

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis/operations/trigger.py

@@ -17,17 +17,20 @@ try:
     from airflow.api.common.trigger_dag import trigger_dag
 except ImportError:
     from airflow.api.common.experimental.trigger_dag import trigger_dag
+
 from airflow.utils import timezone
 from flask import Response
 from openmetadata_managed_apis.api.response import ApiResponse
 
 
-def trigger(dag_id: str, run_id: Optional[str]) -> Response:
+def trigger(
+    dag_id: str, run_id: Optional[str], conf: Optional[dict] = None
+) -> Response:
     dag_run = trigger_dag(
         dag_id=dag_id,
         run_id=run_id,
-        conf=None,
         execution_date=timezone.utcnow(),
+        conf=conf,
     )
     return ApiResponse.success(
         {"message": f"Workflow [{dag_id}] has been triggered {dag_run}"}

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis/workflows/ingestion/application.py

@@ -18,6 +18,7 @@ from openmetadata_managed_apis.utils.logger import set_operator_logger
 from openmetadata_managed_apis.workflows.ingestion.common import (
     build_dag,
     build_workflow_config_property,
+    execute_workflow,
 )
 
 from metadata.generated.schema.entity.applications.configuration.applicationConfig import (
@@ -36,7 +37,7 @@ from metadata.generated.schema.metadataIngestion.applicationPipeline import (
 from metadata.workflow.application import ApplicationWorkflow
 
 
-def application_workflow(workflow_config: OpenMetadataApplicationConfig):
+def application_workflow(workflow_config: OpenMetadataApplicationConfig, **context):
     """
     Task that creates and runs the ingestion workflow.
 
@@ -48,15 +49,17 @@ def application_workflow(workflow_config: OpenMetadataApplicationConfig):
 
     set_operator_logger(workflow_config)
 
+    # set overridden app config
     config = json.loads(
         workflow_config.model_dump_json(exclude_defaults=False, mask_secrets=False)
     )
+    params = context.get("params") or {}
+    config["appConfig"] = {
+        **(config.get("appConfig") or {}),
+        **(params.get("appConfigOverride") or {}),
+    }
     workflow = ApplicationWorkflow.create(config)
-
-    workflow.execute()
-    workflow.raise_from_status()
-    workflow.print_status()
-    workflow.stop()
+    execute_workflow(workflow, workflow_config)
 
 
 def build_application_workflow_config(
@@ -101,6 +104,9 @@ def build_application_dag(ingestion_pipeline: IngestionPipeline) -> DAG:
         ingestion_pipeline=ingestion_pipeline,
         workflow_config=application_workflow_config,
         workflow_fn=application_workflow,
+        params={
+            "appConfigOverride": None  # Default to None, will be overridden by trigger conf
+        },
     )
 
     return dag

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis/workflows/ingestion/auto_classification.py

@@ -15,7 +15,11 @@ import json
 
 from airflow import DAG
 from openmetadata_managed_apis.utils.logger import set_operator_logger
-from openmetadata_managed_apis.workflows.ingestion.common import build_dag, build_source
+from openmetadata_managed_apis.workflows.ingestion.common import (
+    build_dag,
+    build_source,
+    execute_workflow,
+)
 
 from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
     IngestionPipeline,
@@ -30,7 +34,9 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.workflow.classification import AutoClassificationWorkflow
 
 
-def auto_classification_workflow(workflow_config: OpenMetadataWorkflowConfig):
+def auto_classification_workflow(
+    workflow_config: OpenMetadataWorkflowConfig,
+):
     """
     Task that creates and runs the auto classification workflow.
 
@@ -46,11 +52,7 @@ def auto_classification_workflow(workflow_config: OpenMetadataWorkflowConfig):
         workflow_config.model_dump_json(exclude_defaults=False, mask_secrets=False)
     )
     workflow = AutoClassificationWorkflow.create(config)
-
-    workflow.execute()
-    workflow.raise_from_status()
-    workflow.print_status()
-    workflow.stop()
+    execute_workflow(workflow, workflow_config)
 
 
 def build_auto_classification_workflow_config(

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis/workflows/ingestion/common.py

@@ -37,6 +37,7 @@ from metadata.generated.schema.metadataIngestion.application import (
 from metadata.generated.schema.type.basic import Timestamp, Uuid
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils import fqn
+from metadata.workflow.base import BaseWorkflow
 
 # pylint: disable=ungrouped-imports
 try:
@@ -192,6 +193,19 @@ def build_source(ingestion_pipeline: IngestionPipeline) -> WorkflowSource:
     )
 
 
+def execute_workflow(
+    workflow: BaseWorkflow, workflow_config: OpenMetadataWorkflowConfig
+) -> None:
+    """
+    Execute the workflow and handle the status
+    """
+    workflow.execute()
+    workflow.print_status()
+    workflow.stop()
+    if workflow_config.workflowConfig.raiseOnError:
+        workflow.raise_from_status()
+
+
 def metadata_ingestion_workflow(workflow_config: OpenMetadataWorkflowConfig):
     """
     Task that creates and runs the ingestion workflow.
@@ -208,11 +222,7 @@ def metadata_ingestion_workflow(workflow_config: OpenMetadataWorkflowConfig):
         workflow_config.model_dump_json(exclude_defaults=False, mask_secrets=False)
     )
     workflow = MetadataWorkflow.create(config)
-
-    workflow.execute()
-    workflow.raise_from_status()
-    workflow.print_status()
-    workflow.stop()
+    execute_workflow(workflow, workflow_config)
 
 
 def build_workflow_config_property(
@@ -225,6 +235,7 @@ def build_workflow_config_property(
     """
     return WorkflowConfig(
         loggerLevel=ingestion_pipeline.loggerLevel or LogLevels.INFO,
+        raiseOnError=ingestion_pipeline.raiseOnError,
         openMetadataServerConfig=ingestion_pipeline.openMetadataServerConnection,
     )
 
@@ -358,9 +369,16 @@ def build_dag(
     ingestion_pipeline: IngestionPipeline,
     workflow_config: Union[OpenMetadataWorkflowConfig, OpenMetadataApplicationConfig],
     workflow_fn: Callable,
+    params: Optional[dict] = None,
 ) -> DAG:
     """
     Build a simple metadata workflow DAG
+    :param task_name: Name of the task
+    :param ingestion_pipeline: Pipeline configs
+    :param workflow_config: Workflow configurations
+    :param workflow_fn: Function to be executed
+    :param params: Optional parameters to pass to the operator
+    :return: DAG
     """
 
     with DAG(**build_dag_configs(ingestion_pipeline)) as dag:
@@ -371,7 +389,9 @@ def build_dag(
         CustomPythonOperator(
             task_id=task_name,
             python_callable=workflow_fn,
-            op_kwargs={"workflow_config": workflow_config},
+            op_kwargs={
+                "workflow_config": workflow_config,
+            },
             # There's no need to retry if we have had an error. Wait until the next schedule or manual rerun.
             retries=ingestion_pipeline.airflowConfig.retries or 0,
             # each DAG will call its own OpenMetadataWorkflowConfig
@@ -380,6 +400,7 @@ def build_dag(
             owner=ingestion_pipeline.owners.root[0].name
             if (ingestion_pipeline.owners and ingestion_pipeline.owners.root)
             else "openmetadata",
+            params=params,
         )
 
         return dag

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis/workflows/ingestion/dbt.py

@@ -42,7 +42,7 @@ def build_dbt_workflow_config(
     """
 
     source = build_source(ingestion_pipeline)
-    source.type = f"dbt"  # Mark the source as dbt
+    source.type = "dbt"  # Mark the source as dbt
 
     workflow_config = OpenMetadataWorkflowConfig(
         source=source,

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis/workflows/ingestion/profiler.py

@@ -15,7 +15,11 @@ import json
 
 from airflow import DAG
 from openmetadata_managed_apis.utils.logger import set_operator_logger
-from openmetadata_managed_apis.workflows.ingestion.common import build_dag, build_source
+from openmetadata_managed_apis.workflows.ingestion.common import (
+    build_dag,
+    build_source,
+    execute_workflow,
+)
 
 from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
     IngestionPipeline,
@@ -30,7 +34,9 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.workflow.profiler import ProfilerWorkflow
 
 
-def profiler_workflow(workflow_config: OpenMetadataWorkflowConfig):
+def profiler_workflow(
+    workflow_config: OpenMetadataWorkflowConfig,
+):
     """
     Task that creates and runs the profiler workflow.
 
@@ -46,11 +52,7 @@ def profiler_workflow(workflow_config: OpenMetadataWorkflowConfig):
         workflow_config.model_dump_json(exclude_defaults=False, mask_secrets=False)
     )
     workflow = ProfilerWorkflow.create(config)
-
-    workflow.execute()
-    workflow.raise_from_status()
-    workflow.print_status()
-    workflow.stop()
+    execute_workflow(workflow, workflow_config)
 
 
 def build_profiler_workflow_config(

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis/workflows/ingestion/test_suite.py

@@ -15,7 +15,11 @@ import json
 
 from airflow import DAG
 from openmetadata_managed_apis.utils.logger import set_operator_logger
-from openmetadata_managed_apis.workflows.ingestion.common import build_dag, build_source
+from openmetadata_managed_apis.workflows.ingestion.common import (
+    build_dag,
+    build_source,
+    execute_workflow,
+)
 
 from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
     IngestionPipeline,
@@ -30,7 +34,9 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.workflow.data_quality import TestSuiteWorkflow
 
 
-def test_suite_workflow(workflow_config: OpenMetadataWorkflowConfig):
+def test_suite_workflow(
+    workflow_config: OpenMetadataWorkflowConfig,
+):
     """
     Task that creates and runs the test suite workflow.
 
@@ -46,11 +52,7 @@ def test_suite_workflow(workflow_config: OpenMetadataWorkflowConfig):
         workflow_config.model_dump_json(exclude_defaults=False, mask_secrets=False)
     )
     workflow = TestSuiteWorkflow.create(config)
-
-    workflow.execute()
-    workflow.raise_from_status()
-    workflow.print_status()
-    workflow.stop()
+    execute_workflow(workflow, workflow_config)
 
 
 def build_test_suite_workflow_config(

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis/workflows/ingestion/usage.py

@@ -20,6 +20,7 @@ from openmetadata_managed_apis.workflows.ingestion.common import (
     build_dag,
     build_source,
     build_workflow_config_property,
+    execute_workflow,
 )
 
 from metadata.generated.schema.entity.services.ingestionPipelines.ingestionPipeline import (
@@ -34,7 +35,9 @@ from metadata.generated.schema.metadataIngestion.workflow import (
 from metadata.workflow.usage import UsageWorkflow
 
 
-def usage_workflow(workflow_config: OpenMetadataWorkflowConfig):
+def usage_workflow(
+    workflow_config: OpenMetadataWorkflowConfig,
+):
     """
     Task that creates and runs the ingestion workflow.
 
@@ -50,11 +53,7 @@ def usage_workflow(workflow_config: OpenMetadataWorkflowConfig):
         workflow_config.model_dump_json(exclude_defaults=False, mask_secrets=False)
     )
     workflow = UsageWorkflow.create(config)
-
-    workflow.execute()
-    workflow.raise_from_status()
-    workflow.print_status()
-    workflow.stop()
+    execute_workflow(workflow, workflow_config)
 
 
 def build_usage_config_from_file(

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/openmetadata_managed_apis.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openmetadata_managed_apis
-Version: 1.7.0.1
+Version: 1.7.1.1
 Summary: Airflow REST APIs to create and manage DAGS
 Author: OpenMetadata Committers
 License:                                  Apache License
@@ -229,11 +229,15 @@ OpenMetadata workflow definition and manage DAGS and tasks.
 
 ## Development
 
-You can run `make branch=issue-3659-v2 test_up` and specify any branch from OpenMetadata that you'd
-need to test the changes in the APIs. This will prepare a separated airflow container.
+The file [`development/airflow/airflow.cfg`](./development/airflow/airflow.cfg) contains configuration which runs based on
+the airflow server deployed by the quick-start and development compose files.
 
-The command will build the image by downloading the branch changes inside the container. This helps us
-test the REST APIs using some ongoing changes on OpenMetadata as well.
+You ca run the following command to start the development environment:
+
+```bash
+export AIRFLOW_HOME=$(pwd)/openmetadata-airflow-managed-api/development/airflow
+airflow webserver
+```
 
 ## Requirements
 

{openmetadata_managed_apis-1.7.0.1 → openmetadata_managed_apis-1.7.1.1}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 # since it helps us organize and isolate version management
 [project]
 name = "openmetadata_managed_apis"
-version = "1.7.0.1"
+version = "1.7.1.1"
 readme = "README.md"
 authors = [
     {name = "OpenMetadata Committers"}