openmetadata-ingestion 1.8.10.0__py3-none-any.whl → 1.9.1.0__py3-none-any.whl

metadata/__init__.py

@@ -13,6 +13,7 @@ OpenMetadata package initialization.
 """
 from typing import Type
 
+from metadata.profiler.api.models import ProfilerProcessorConfig
 from metadata.profiler.metrics.registry import Metrics
 from metadata.profiler.registry import MetricRegistry
 from metadata.profiler.source.database.base.profiler_resolver import (
@@ -29,3 +30,4 @@ container = DependencyContainer()
 container.register(SourceLoader, DefaultSourceLoader)
 container.register(Type[MetricRegistry], lambda: Metrics)
 container.register(Type[ProfilerResolver], lambda: DefaultProfilerResolver)
+container.register(Type[ProfilerProcessorConfig], lambda: ProfilerProcessorConfig)

metadata/data_quality/validations/models.py

@@ -1,6 +1,6 @@
 """Models for the TableDiff test case"""
 
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from pydantic import BaseModel
 
@@ -12,7 +12,7 @@ from metadata.ingestion.models.custom_pydantic import CustomSecretStr
 
 
 class TableParameter(BaseModel):
-    serviceUrl: str
+    serviceUrl: Union[str, dict]
     path: str
     columns: List[Column]
     database_service_type: DatabaseServiceType

metadata/data_quality/validations/runtime_param_setter/base_diff_params_setter.py

@@ -1,15 +1,52 @@
 """Base class for param setter logic for table data diff"""
 
-from typing import List, Optional, Set
+from typing import List, Optional, Set, Type, Union
 from urllib.parse import urlparse
 
 from metadata.data_quality.validations.models import Column, TableParameter
 from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
+from metadata.generated.schema.entity.services.serviceType import ServiceType
+from metadata.ingestion.connections.connection import BaseConnection
 from metadata.ingestion.source.connections import get_connection
 from metadata.profiler.orm.registry import Dialects
 from metadata.utils import fqn
 from metadata.utils.collections import CaseInsensitiveList
+from metadata.utils.importer import get_module_dir, import_from_module
+
+
+# TODO: Refactor to avoid the circular import that makes us unable to use the BaseSpec class and the helper methods.
+# Using the specs class method causes circular import as TestSuiteInterface
+# imports RuntimeParameterSetter
+class ServiceSpecPatch:
+    def __init__(self, service_type: ServiceType, source_type: str):
+        self.service_type = service_type
+        self.source_type = source_type
+        self._service_spec = None
+
+    @property
+    def service_spec(self):
+        if self._service_spec is None:
+            self._service_spec = self.get_for_source()
+        return self._service_spec
+
+    def get_for_source(self):
+        return import_from_module(
+            "metadata.{}.source.{}.{}.{}.ServiceSpec".format(  # pylint: disable=C0209
+                "ingestion",
+                self.service_type.name.lower(),
+                get_module_dir(self.source_type),
+                "service_spec",
+            )
+        )
+
+    def get_data_diff_class(self) -> Type["BaseTableParameter"]:
+        return import_from_module(self.service_spec.data_diff)
+
+    def get_connection_class(self) -> Optional[Type[BaseConnection]]:
+        if self.service_spec.connection_class:
+            return import_from_module(self.service_spec.connection_class)
+        return None
 
 
 class BaseTableParameter:
@@ -22,7 +59,7 @@ class BaseTableParameter:
         key_columns,
         extra_columns,
         case_sensitive_columns,
-        service_url: Optional[str],
+        service_url: Optional[Union[str, dict]],
     ) -> TableParameter:
         """Getter table parameter for the table diff test.
 
@@ -62,10 +99,40 @@ class BaseTableParameter:
             "___SERVICE___", "__DATABASE__", schema, table
         ).replace("___SERVICE___.__DATABASE__.", "")
 
+    @staticmethod
+    def _get_service_connection_config(
+        service_connection_config,
+    ) -> Optional[Union[str, dict]]:
+        """
+        Get the connection dictionary for the service.
+        """
+        service_spec_patch = ServiceSpecPatch(
+            ServiceType.Database, service_connection_config.type.value.lower()
+        )
+
+        try:
+            connection_class = service_spec_patch.get_connection_class()
+            if not connection_class:
+                return (
+                    str(get_connection(service_connection_config).url)
+                    if service_connection_config
+                    else None
+                )
+            connection = connection_class(service_connection_config)
+            return connection.get_connection_dict()
+        except (ValueError, AttributeError, NotImplementedError):
+            return (
+                str(get_connection(service_connection_config).url)
+                if service_connection_config
+                else None
+            )
+
     @staticmethod
     def get_data_diff_url(
-        db_service: DatabaseService, table_fqn, override_url: Optional[str] = None
-    ) -> str:
+        db_service: DatabaseService,
+        table_fqn,
+        override_url: Optional[Union[str, dict]] = None,
+    ) -> Union[str, dict]:
         """Get the url for the data diff service.
 
         Args:
@@ -77,10 +144,16 @@ class BaseTableParameter:
             str: The url for the data diff service
         """
         source_url = (
-            str(get_connection(db_service.connection.config).url)
+            BaseTableParameter._get_service_connection_config(
+                db_service.connection.config
+            )
             if not override_url
             else override_url
         )
+        if isinstance(source_url, dict):
+            source_url["driver"] = source_url["driver"].split("+")[0]
+            return source_url
+
         url = urlparse(source_url)
         # remove the driver name from the url because table-diff doesn't support it
         kwargs = {"scheme": url.scheme.split("+")[0]}

metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py

@@ -11,10 +11,13 @@
 """Module that defines the TableDiffParamsSetter class."""
 from ast import literal_eval
 from typing import List, Optional, Set
-from urllib.parse import urlparse
 
 from metadata.data_quality.validations import utils
 from metadata.data_quality.validations.models import Column, TableDiffRuntimeParameters
+from metadata.data_quality.validations.runtime_param_setter.base_diff_params_setter import (
+    BaseTableParameter,
+    ServiceSpecPatch,
+)
 from metadata.data_quality.validations.runtime_param_setter.param_setter import (
     RuntimeParameterSetter,
 )
@@ -22,24 +25,8 @@ from metadata.generated.schema.entity.data.table import Constraint, Table
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.serviceType import ServiceType
 from metadata.generated.schema.tests.testCase import TestCase
-from metadata.ingestion.source.connections import get_connection
-from metadata.profiler.orm.registry import Dialects
 from metadata.utils import fqn
 from metadata.utils.collections import CaseInsensitiveList
-from metadata.utils.importer import get_module_dir, import_from_module
-
-
-def get_for_source(
-    service_type: ServiceType, source_type: str, from_: str = "ingestion"
-):
-    return import_from_module(
-        "metadata.{}.source.{}.{}.{}.ServiceSpec".format(  # pylint: disable=C0209
-            from_,
-            service_type.name.lower(),
-            get_module_dir(source_type),
-            "service_spec",
-        )
-    )
 
 
 class TableDiffParamsSetter(RuntimeParameterSetter):
@@ -62,22 +49,17 @@ class TableDiffParamsSetter(RuntimeParameterSetter):
         }
 
     def get_parameters(self, test_case) -> TableDiffRuntimeParameters:
-        # Using the specs class method causes circular import as TestSuiteInterface
-        # imports RuntimeParameterSetter
-        cls_path = get_for_source(
-            ServiceType.Database,
-            source_type=self.service_connection_config.type.value.lower(),
-        ).data_diff
-        cls = import_from_module(cls_path)()
+        service_spec_patch = ServiceSpecPatch(
+            ServiceType.Database, self.service_connection_config.type.value.lower()
+        )
+        cls = service_spec_patch.get_data_diff_class()()
 
         service1: DatabaseService = self.ometa_client.get_by_id(
             DatabaseService, self.table_entity.service.id, nullable=False
         )
 
-        service1_url = (
-            str(get_connection(self.service_connection_config).url)
-            if self.service_connection_config
-            else None
+        service1_url = BaseTableParameter._get_service_connection_config(
+            self.service_connection_config
         )
 
         table2_fqn = self.get_parameter(test_case, "table2")
@@ -209,41 +191,6 @@ class TableDiffParamsSetter(RuntimeParameterSetter):
             (p.value for p in test_case.parameterValues if p.name == key), default
         )
 
-    @staticmethod
-    def get_data_diff_url(
-        db_service: DatabaseService, table_fqn, override_url: Optional[str] = None
-    ) -> str:
-        """Get the url for the data diff service.
-
-        Args:
-            db_service (DatabaseService): The database service entity
-            table_fqn (str): The fully qualified name of the table
-            override_url (Optional[str], optional): Override the url. Defaults to None.
-
-        Returns:
-            str: The url for the data diff service
-        """
-        source_url = (
-            str(get_connection(db_service.connection.config).url)
-            if not override_url
-            else override_url
-        )
-        url = urlparse(source_url)
-        # remove the driver name from the url because table-diff doesn't support it
-        kwargs = {"scheme": url.scheme.split("+")[0]}
-        service, database, schema, table = fqn.split(  # pylint: disable=unused-variable
-            table_fqn
-        )
-        # path needs to include the database AND schema in some of the connectors
-        if hasattr(db_service.connection.config, "supportsDatabase"):
-            kwargs["path"] = f"/{database}"
-        # this can be found by going to:
-        # https://github.com/open-metadata/collate-data-diff/blob/main/data_diff/databases/<connector>.py
-        # and looking at the `CONNECT_URI_HELPER` variable
-        if kwargs["scheme"] in {Dialects.MSSQL, Dialects.Snowflake, Dialects.Trino}:
-            kwargs["path"] = f"/{database}/{schema}"
-        return url._replace(**kwargs).geturl()
-
     @staticmethod
     def get_data_diff_table_path(table_fqn: str) -> str:
         service, database, schema, table = fqn.split(  # pylint: disable=unused-variable

metadata/data_quality/validations/table/sqlalchemy/tableDiff.py

@@ -508,7 +508,10 @@ class TableDiffValidator(BaseTestValidator, SQAValidatorMixin):
             ("table1.serviceUrl", self.runtime_params.table1.serviceUrl),
             ("table2.serviceUrl", self.runtime_params.table2.serviceUrl),
         ]:
-            dialect = urlparse(param).scheme
+            if isinstance(param, dict):
+                dialect = param.get("driver")
+            else:
+                dialect = urlparse(param).scheme
             if dialect not in SUPPORTED_DIALECTS:
                 raise UnsupportedDialectError(name, dialect)
 

metadata/examples/workflows/grafana.yaml

@@ -0,0 +1,115 @@
+source:
+  type: grafana
+  serviceName: local_grafana
+  serviceConnection:
+    config:
+      type: Grafana
+      # Grafana instance URL
+      hostPort: http://localhost:3000
+      # Service Account Token - requires Admin role for full access
+      # Create at: Configuration -> Service accounts -> Add service account
+      apiKey: ${GRAFANA_API_KEY}  # e.g., glsa_xxxxxxxxxxxxxxxxxxxx
+      # SSL verification for HTTPS connections
+      verifySSL: true
+      # Items per API page (for large instances)
+      pageSize: 100
+  sourceConfig:
+    config:
+      type: DashboardMetadata
+      # Optional: Filter dashboards by name pattern
+      # dashboardFilterPattern:
+      #   includes:
+      #     - ".*production.*"
+      #     - ".*analytics.*"
+      #   excludes:
+      #     - ".*test.*"
+      #     - ".*dev.*"
+      
+      # Optional: Filter charts/panels by name pattern
+      # chartFilterPattern:
+      #   excludes:
+      #     - ".*debug.*"
+      
+      # Include Grafana tags as OpenMetadata tags
+      includeTags: true
+      
+      # Optional: Database service name for lineage
+      # If panels contain SQL queries, this will create lineage to tables
+      # dbServiceName: my_postgres_service
+      
+      # Enable/disable specific metadata
+      includeOwners: true
+      includeDataModels: false
+      includeDraftDashboard: false
+      
+      # Limit the ingestion to specific projects/folders
+      # projectFilterPattern:
+      #   includes:
+      #     - ".*Marketing.*"
+      #     - ".*Sales.*"
+
+sink:
+  type: metadata-rest
+  config:
+    api_endpoint: http://localhost:8585/api
+
+workflowConfig:
+  loggerLevel: INFO
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: ${OM_JWT_TOKEN}
+
+# Advanced Configuration Examples:
+
+# 1. Production Environment with Filters
+# source:
+#   type: grafana
+#   serviceName: grafana_prod
+#   serviceConnection:
+#     config:
+#       type: Grafana
+#       hostPort: https://grafana.company.com
+#       apiKey: ${GRAFANA_PROD_API_KEY}
+#       verifySSL: true
+#   sourceConfig:
+#     config:
+#       type: DashboardMetadata
+#       dashboardFilterPattern:
+#         includes:
+#           - "^(Production|Prod).*"
+#       includeTags: true
+#       dbServiceName: production_postgres
+
+# 2. Development Environment with Self-Signed Cert
+# source:
+#   type: grafana
+#   serviceName: grafana_dev
+#   serviceConnection:
+#     config:
+#       type: Grafana
+#       hostPort: https://grafana-dev.internal.com
+#       apiKey: ${GRAFANA_DEV_API_KEY}
+#       verifySSL: false  # For self-signed certificates
+#   sourceConfig:
+#     config:
+#       type: DashboardMetadata
+#       includeDraftDashboard: true
+
+# 3. Multi-Database Lineage Setup
+# source:
+#   type: grafana
+#   serviceName: grafana_analytics
+#   serviceConnection:
+#     config:
+#       type: Grafana
+#       hostPort: https://analytics.grafana.com
+#       apiKey: ${GRAFANA_ANALYTICS_KEY}
+#   sourceConfig:
+#     config:
+#       type: DashboardMetadata
+#       # Use dbServiceName pattern to map different folders to different databases
+#       # Folder "MySQL Dashboards" -> mysql_service
+#       # Folder "PostgreSQL Dashboards" -> postgres_service
+#       dbServiceName: "{folder_name}_service"

metadata/generated/schema/analytics/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00

metadata/generated/schema/analytics/basic.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/basic.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportData.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00

metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/aggregatedCostAnalysisReportData.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/entityReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/entityReportData.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/rawCostAnalysisReportData.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/webAnalyticEntityViewReportData.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/webAnalyticUserActivityReportData.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/webAnalyticEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEvent.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 
@@ -11,7 +11,7 @@ from typing_extensions import Annotated
 
 from metadata.ingestion.models.custom_pydantic import BaseModel
 
-from ..type import basic, entityHistory, entityReference, entityReferenceList
+from ..type import basic, entityHistory, entityReferenceList
 from . import basic as basic_1
 
 
@@ -83,10 +83,10 @@ class WebAnalyticEvent(BaseModel):
             description='Weather the event is enable (i.e. data is being collected)',
         ),
     ]
-    domain: Annotated[
-        Optional[entityReference.EntityReference],
+    domains: Annotated[
+        Optional[entityReferenceList.EntityReferenceList],
         Field(
             None,
-            description='Domain the asset belongs to. When not set, the asset inherits the domain from the parent it belongs to.',
+            description='Domains the asset belongs to. When not set, the asset inherits the domain from the parent it belongs to.',
         ),
     ]

metadata/generated/schema/analytics/webAnalyticEventData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEventData.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/webAnalyticEventType/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00

metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEventType/customEvent.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEventType/pageViewEvent.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00

metadata/generated/schema/api/addGlossaryToAssetsRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/addGlossaryToAssetsRequest.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/addTagToAssetsRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/addTagToAssetsRequest.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/analytics/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00

metadata/generated/schema/api/analytics/createWebAnalyticEvent.py

@@ -1,10 +1,10 @@
 # generated by datamodel-codegen:
 #   filename:  api/analytics/createWebAnalyticEvent.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 
-from typing import Optional
+from typing import List, Optional
 
 from pydantic import ConfigDict, Field
 from typing_extensions import Annotated
@@ -38,9 +38,10 @@ class CreateWebAnalyticEvent(BaseModel):
         Optional[entityReferenceList.EntityReferenceList],
         Field(None, description='Owners of this report definition'),
     ]
-    domain: Annotated[
-        Optional[str],
+    domains: Annotated[
+        Optional[List[str]],
         Field(
-            None, description='Fully qualified name of the domain the Table belongs to.'
+            None,
+            description='Fully qualified names of the domains the Web Analytic Event belongs to.',
         ),
     ]

metadata/generated/schema/api/automations/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00

metadata/generated/schema/api/automations/createWorkflow.py

@@ -1,10 +1,10 @@
 # generated by datamodel-codegen:
 #   filename:  api/automations/createWorkflow.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 from pydantic import ConfigDict, Field
 from typing_extensions import Annotated
@@ -62,9 +62,10 @@ class CreateWorkflowRequest(BaseModel):
         Optional[entityReferenceList.EntityReferenceList],
         Field(None, description='Owners of this workflow.'),
     ]
-    domain: Annotated[
-        Optional[str],
+    domains: Annotated[
+        Optional[List[str]],
         Field(
-            None, description='Fully qualified name of the domain the Table belongs to.'
+            None,
+            description='Fully qualified names of the domains the Workflow belongs to.',
         ),
     ]

metadata/generated/schema/api/bulkAssets.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/bulkAssets.json
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/classification/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2025-08-05T07:46:01+00:00
+#   timestamp: 2025-08-14T07:03:44+00:00