openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl

metadata/examples/workflows/databricks_usage.yaml

@@ -3,7 +3,13 @@ source:
   serviceName: local_databricks
   serviceConnection:
     config:
-      token: <databricks token>
+      authType:
+        token: <databricks token>
+        # clientId: databricks service principal client id
+        # clientSecret: databricks service principal client secret
+        # azureClientSecret: azure client secret
+        # azureClientId: azure client id
+        # azureTenantId: azure tenant id
       hostPort: localhost:443
       connectionArguments:
         http_path: <http path of databricks cluster>

metadata/examples/workflows/exasol.yaml

@@ -0,0 +1,25 @@
+source:
+  type: exasol
+  serviceName: local_exasol
+  serviceConnection:
+    config:
+      type: Exasol
+      username: openmetadata_user
+      password: openmetadata_password
+      hostPort: localhost:8563
+      tls: disable-tls
+      connectionOptions: {}
+      connectionArguments: {}
+  sourceConfig:
+    config:
+      type: DatabaseMetadata
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+#  loggerLevel: INFO # DEBUG, INFO, WARN or ERROR
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

metadata/examples/workflows/grafana.yaml

@@ -0,0 +1,115 @@
+source:
+  type: grafana
+  serviceName: local_grafana
+  serviceConnection:
+    config:
+      type: Grafana
+      # Grafana instance URL
+      hostPort: http://localhost:3000
+      # Service Account Token - requires Admin role for full access
+      # Create at: Configuration -> Service accounts -> Add service account
+      apiKey: ${GRAFANA_API_KEY}  # e.g., glsa_xxxxxxxxxxxxxxxxxxxx
+      # SSL verification for HTTPS connections
+      verifySSL: true
+      # Items per API page (for large instances)
+      pageSize: 100
+  sourceConfig:
+    config:
+      type: DashboardMetadata
+      # Optional: Filter dashboards by name pattern
+      # dashboardFilterPattern:
+      #   includes:
+      #     - ".*production.*"
+      #     - ".*analytics.*"
+      #   excludes:
+      #     - ".*test.*"
+      #     - ".*dev.*"
+      
+      # Optional: Filter charts/panels by name pattern
+      # chartFilterPattern:
+      #   excludes:
+      #     - ".*debug.*"
+      
+      # Include Grafana tags as OpenMetadata tags
+      includeTags: true
+      
+      # Optional: Database service name for lineage
+      # If panels contain SQL queries, this will create lineage to tables
+      # dbServiceName: my_postgres_service
+      
+      # Enable/disable specific metadata
+      includeOwners: true
+      includeDataModels: false
+      includeDraftDashboard: false
+      
+      # Limit the ingestion to specific projects/folders
+      # projectFilterPattern:
+      #   includes:
+      #     - ".*Marketing.*"
+      #     - ".*Sales.*"
+
+sink:
+  type: metadata-rest
+  config:
+    api_endpoint: http://localhost:8585/api
+
+workflowConfig:
+  loggerLevel: INFO
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: ${OM_JWT_TOKEN}
+
+# Advanced Configuration Examples:
+
+# 1. Production Environment with Filters
+# source:
+#   type: grafana
+#   serviceName: grafana_prod
+#   serviceConnection:
+#     config:
+#       type: Grafana
+#       hostPort: https://grafana.company.com
+#       apiKey: ${GRAFANA_PROD_API_KEY}
+#       verifySSL: true
+#   sourceConfig:
+#     config:
+#       type: DashboardMetadata
+#       dashboardFilterPattern:
+#         includes:
+#           - "^(Production|Prod).*"
+#       includeTags: true
+#       dbServiceName: production_postgres
+
+# 2. Development Environment with Self-Signed Cert
+# source:
+#   type: grafana
+#   serviceName: grafana_dev
+#   serviceConnection:
+#     config:
+#       type: Grafana
+#       hostPort: https://grafana-dev.internal.com
+#       apiKey: ${GRAFANA_DEV_API_KEY}
+#       verifySSL: false  # For self-signed certificates
+#   sourceConfig:
+#     config:
+#       type: DashboardMetadata
+#       includeDraftDashboard: true
+
+# 3. Multi-Database Lineage Setup
+# source:
+#   type: grafana
+#   serviceName: grafana_analytics
+#   serviceConnection:
+#     config:
+#       type: Grafana
+#       hostPort: https://analytics.grafana.com
+#       apiKey: ${GRAFANA_ANALYTICS_KEY}
+#   sourceConfig:
+#     config:
+#       type: DashboardMetadata
+#       # Use dbServiceName pattern to map different folders to different databases
+#       # Folder "MySQL Dashboards" -> mysql_service
+#       # Folder "PostgreSQL Dashboards" -> postgres_service
+#       dbServiceName: "{folder_name}_service"

metadata/examples/workflows/kinesisfirehose.yaml

@@ -0,0 +1,99 @@
+# AWS Kinesis Firehose Workflow Configuration
+#
+# This workflow extracts metadata from AWS Kinesis Firehose delivery streams
+# and creates lineage between sources (DynamoDB) and destinations
+# (S3, Redshift, OpenSearch, Snowflake)
+#
+# Supported Features:
+# - Pipeline metadata extraction
+# - Entity-level lineage (DynamoDB → Firehose → Destination)
+# - Column-level lineage (automatic column matching by name)
+#
+# Required IAM Permissions:
+# - firehose:ListDeliveryStreams
+# - firehose:DescribeDeliveryStream
+
+source:
+  type: kinesisfirehose
+  serviceName: aws_firehose_production
+  serviceConnection:
+    config:
+      type: KinesisFirehose
+      awsConfig:
+        # AWS Region where Firehose delivery streams are located
+        awsRegion: us-east-1
+
+        # Authentication Option 1: Access Key & Secret (for development/testing)
+        awsAccessKeyId: AKIAIOSFODNN7EXAMPLE
+        awsSecretAccessKey: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
+        # Authentication Option 2: Temporary Session Token (for STS)
+        # awsSessionToken: "temporary-session-token"
+
+        # Authentication Option 3: AWS Profile (uses ~/.aws/credentials)
+        # profileName: "openmetadata-profile"
+
+        # Authentication Option 4: Assume Role (for cross-account access)
+        # assumeRoleArn: "arn:aws:iam::123456789012:role/OpenMetadataFirehoseRole"
+        # assumeRoleSessionName: "OpenMetadataFirehoseSession"
+        # assumeRoleSourceIdentity: "arn:aws:iam::987654321098:user/admin"
+
+        # Optional: Custom endpoint for LocalStack or VPC endpoints
+        # endPointURL: "https://firehose.us-east-1.amazonaws.com/"
+
+  sourceConfig:
+    config:
+      type: PipelineMetadata
+
+      # Optional: Filter delivery streams by name pattern
+      # pipelineFilterPattern:
+      #   includes:
+      #     - "prod-.*"
+      #     - "cdc-.*"
+      #   excludes:
+      #     - "test-.*"
+      #     - "dev-.*"
+
+      # Optional: Enable lineage extraction (default: true)
+      # includeLineage: true
+
+      # Optional: Mark deleted pipelines (delivery streams no longer in AWS)
+      # markDeletedPipelines: true
+
+sink:
+  type: metadata-rest
+  config: {}
+
+workflowConfig:
+  # Optional: Set log level (DEBUG, INFO, WARN, ERROR)
+  # loggerLevel: INFO
+
+  openMetadataServerConfig:
+    hostPort: "http://localhost:8585/api"
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
+
+# Lineage Support:
+#
+# This connector automatically creates lineage when it finds:
+#
+# 1. DynamoDB → Firehose → S3
+#    - Source: DynamoDB table (from KinesisStreamSourceConfiguration)
+#    - Destination: S3 container (from S3DestinationDescription)
+#
+# 2. DynamoDB → Firehose → Redshift
+#    - Source: DynamoDB table
+#    - Destination: Redshift table (from RedshiftDestinationDescription)
+#
+# 3. DynamoDB → Firehose → OpenSearch
+#    - Source: DynamoDB table
+#    - Destination: OpenSearch index (from AmazonopensearchserviceDestinationDescription)
+#
+# 4. DynamoDB → Firehose → Snowflake
+#    - Source: DynamoDB table
+#    - Destination: Snowflake table (from SnowflakeDestinationDescription)
+#
+# Note: Both source and destination entities must exist in OpenMetadata
+# for lineage to be created. Column-level lineage is automatically
+# generated by matching column names (case-insensitive).

metadata/examples/workflows/metabase.yaml

@@ -7,6 +7,7 @@ source:
       username: username
       password: password
       hostPort: http://hostPort
+      # apiKey: api_key
   sourceConfig:
     config:
       type: DashboardMetadata

metadata/examples/workflows/my_confluent_cloud.yaml

@@ -0,0 +1,49 @@
+source:
+  type: kafkaconnect
+  serviceName: confluent_cloud_cdc
+  serviceConnection:
+    config:
+      type: KafkaConnect
+      # IMPORTANT: Use the Kafka Connect API endpoint, NOT the Kafka broker endpoint
+      # Format: https://api.confluent.cloud/connect/v1/environments/{ENV_ID}/clusters/{CLUSTER_ID}
+      # Example: https://api.confluent.cloud/connect/v1/environments/env-abc123/clusters/lkc-xyz789
+      hostPort: https://api.confluent.cloud/connect/v1/environments/YOUR_ENV_ID/clusters/YOUR_CONNECT_CLUSTER_ID
+
+      # Use Kafka Connect API Key (NOT Kafka broker API key)
+      KafkaConnectConfig:
+        username: YOUR_KAFKA_CONNECT_API_KEY
+        password: YOUR_KAFKA_CONNECT_API_SECRET
+
+      verifySSL: true
+
+      # REQUIRED for Confluent Cloud: Specify your Kafka messaging service
+      # This should match the service name you used when ingesting Kafka topics
+      messagingServiceName: confluent_kafka_cloud
+
+      # Optional: Filter specific connectors
+      # pipelineFilterPattern:
+      #   includes:
+      #     - "mysql-cdc-.*"
+      #     - "postgres-sink-.*"
+
+  sourceConfig:
+    config:
+      type: PipelineMetadata
+      # Specify the database services where your tables are ingested
+      lineageInformation:
+        dbServiceNames:
+          - local_mysql        # Replace with your MySQL service name
+          - local_postgres     # Replace with your Postgres service name
+        storageServiceNames: []  # Add S3/GCS service names if using sink connectors to storage
+
+sink:
+  type: metadata-rest
+  config: {}
+
+workflowConfig:
+  loggerLevel: DEBUG  # Use DEBUG to see detailed connection info
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg

metadata/examples/workflows/unity_catalog.yaml

@@ -6,7 +6,13 @@ source:
       type: UnityCatalog
       catalog: hive_metastore
       databaseSchema: default
-      token: <databricks token>
+      authType:
+        token: <databricks token>
+        # clientId: databricks service principal client id
+        # clientSecret: databricks service principal client secret
+        # azureClientSecret: azure client secret
+        # azureClientId: azure client id
+        # azureTenantId: azure tenant id
       hostPort: localhost:443
       connectionTimeout: 120
       connectionArguments:

metadata/examples/workflows/unity_catalog_usage.yaml

@@ -6,7 +6,13 @@ source:
       type: UnityCatalog
       catalog: hive_metastore
       databaseSchema: default
-      token: <databricks token>
+      authType:
+        token: <databricks token>
+        # clientId: databricks service principal client id
+        # clientSecret: databricks service principal client secret
+        # azureClientSecret: azure client secret
+        # azureClientId: azure client id
+        # azureTenantId: azure tenant id
       hostPort: localhost:443
       connectionTimeout: 120
       connectionArguments: