openmetadata-ingestion 1.9.0.0__py3-none-any.whl → 1.10.14.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow_provider_openmetadata/lineage/backend.py +4 -2
- airflow_provider_openmetadata/lineage/config/loader.py +8 -1
- metadata/cli/common.py +0 -1
- metadata/clients/aws_client.py +4 -0
- metadata/data_quality/interface/test_suite_interface.py +9 -3
- metadata/data_quality/validations/models.py +11 -1
- metadata/data_quality/validations/runtime_param_setter/base_diff_params_setter.py +39 -13
- metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +9 -0
- metadata/data_quality/validations/runtime_param_setter/table_custom_sql_query_params_setter.py +31 -0
- metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +20 -13
- metadata/data_quality/validations/table/base/tableCustomSQLQuery.py +132 -3
- metadata/data_quality/validations/table/pandas/tableCustomSQLQuery.py +38 -0
- metadata/data_quality/validations/table/sqlalchemy/tableCustomSQLQuery.py +320 -0
- metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +4 -0
- metadata/examples/workflows/airbyte.yaml +8 -2
- metadata/examples/workflows/bigquery.yaml +1 -0
- metadata/examples/workflows/confluent_cdc.yaml +70 -0
- metadata/examples/workflows/confluent_cdc_cloud.yaml +86 -0
- metadata/examples/workflows/confluent_cdc_local.yaml +120 -0
- metadata/examples/workflows/databricks.yaml +9 -1
- metadata/examples/workflows/databricks_usage.yaml +7 -1
- metadata/examples/workflows/exasol.yaml +25 -0
- metadata/examples/workflows/grafana.yaml +115 -0
- metadata/examples/workflows/kinesisfirehose.yaml +99 -0
- metadata/examples/workflows/metabase.yaml +1 -0
- metadata/examples/workflows/my_confluent_cloud.yaml +49 -0
- metadata/examples/workflows/unity_catalog.yaml +7 -1
- metadata/examples/workflows/unity_catalog_usage.yaml +7 -1
- metadata/generated/antlr/EntityLinkLexer.py +419 -353
- metadata/generated/schema/analytics/__init__.py +1 -1
- metadata/generated/schema/analytics/basic.py +1 -1
- metadata/generated/schema/analytics/reportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
- metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
- metadata/generated/schema/api/__init__.py +1 -1
- metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
- metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
- metadata/generated/schema/api/analytics/__init__.py +1 -1
- metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
- metadata/generated/schema/api/automations/__init__.py +1 -1
- metadata/generated/schema/api/automations/createWorkflow.py +1 -1
- metadata/generated/schema/api/bulkAssets.py +1 -1
- metadata/generated/schema/api/classification/__init__.py +1 -1
- metadata/generated/schema/api/classification/createClassification.py +52 -1
- metadata/generated/schema/api/classification/createTag.py +29 -2
- metadata/generated/schema/api/classification/createTagWithRecognizers.py +67 -0
- metadata/generated/schema/api/classification/loadTags.py +1 -1
- metadata/generated/schema/api/configuration/__init__.py +3 -0
- metadata/generated/schema/api/configuration/rdfConfiguration.py +47 -0
- metadata/generated/schema/api/createBot.py +1 -1
- metadata/generated/schema/api/createEventPublisherJob.py +1 -1
- metadata/generated/schema/api/createType.py +1 -1
- metadata/generated/schema/api/data/__init__.py +1 -1
- metadata/generated/schema/api/data/createAPICollection.py +1 -1
- metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
- metadata/generated/schema/api/data/createChart.py +1 -1
- metadata/generated/schema/api/data/createContainer.py +1 -1
- metadata/generated/schema/api/data/createCustomProperty.py +1 -1
- metadata/generated/schema/api/data/createDashboard.py +1 -1
- metadata/generated/schema/api/data/createDashboardDataModel.py +5 -1
- metadata/generated/schema/api/data/createDataContract.py +33 -3
- metadata/generated/schema/api/data/createDatabase.py +1 -1
- metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
- metadata/generated/schema/api/data/createDirectory.py +1 -1
- metadata/generated/schema/api/data/createEntityProfile.py +36 -0
- metadata/generated/schema/api/data/createFile.py +1 -1
- metadata/generated/schema/api/data/createGlossary.py +1 -1
- metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
- metadata/generated/schema/api/data/createMetric.py +12 -1
- metadata/generated/schema/api/data/createMlModel.py +1 -1
- metadata/generated/schema/api/data/createPipeline.py +1 -1
- metadata/generated/schema/api/data/createQuery.py +1 -1
- metadata/generated/schema/api/data/createQueryCostRecord.py +1 -1
- metadata/generated/schema/api/data/createSearchIndex.py +1 -1
- metadata/generated/schema/api/data/createSpreadsheet.py +9 -1
- metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
- metadata/generated/schema/api/data/createTable.py +1 -1
- metadata/generated/schema/api/data/createTableProfile.py +1 -1
- metadata/generated/schema/api/data/createTopic.py +1 -1
- metadata/generated/schema/api/data/createWorksheet.py +1 -1
- metadata/generated/schema/api/data/loadGlossary.py +1 -1
- metadata/generated/schema/api/data/restoreEntity.py +1 -1
- metadata/generated/schema/api/data/updateColumn.py +1 -1
- metadata/generated/schema/api/dataInsight/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
- metadata/generated/schema/api/docStore/__init__.py +1 -1
- metadata/generated/schema/api/docStore/createDocument.py +1 -1
- metadata/generated/schema/api/domains/__init__.py +1 -1
- metadata/generated/schema/api/domains/createDataProduct.py +7 -1
- metadata/generated/schema/api/domains/createDomain.py +1 -1
- metadata/generated/schema/api/entityRelationship/__init__.py +1 -1
- metadata/generated/schema/api/entityRelationship/entityRelationshipDirection.py +1 -1
- metadata/generated/schema/api/entityRelationship/esEntityRelationshipData.py +1 -1
- metadata/generated/schema/api/entityRelationship/relationshipRef.py +1 -1
- metadata/generated/schema/api/entityRelationship/searchEntityRelationshipRequest.py +1 -1
- metadata/generated/schema/api/entityRelationship/searchEntityRelationshipResult.py +1 -1
- metadata/generated/schema/api/entityRelationship/searchSchemaEntityRelationshipResult.py +1 -1
- metadata/generated/schema/api/events/__init__.py +3 -0
- metadata/generated/schema/api/events/createNotificationTemplate.py +61 -0
- metadata/generated/schema/api/events/notificationTemplateValidationRequest.py +30 -0
- metadata/generated/schema/api/events/notificationTemplateValidationResponse.py +36 -0
- metadata/generated/schema/api/feed/__init__.py +1 -1
- metadata/generated/schema/api/feed/closeTask.py +1 -1
- metadata/generated/schema/api/feed/createPost.py +1 -1
- metadata/generated/schema/api/feed/createSuggestion.py +1 -1
- metadata/generated/schema/api/feed/createThread.py +1 -1
- metadata/generated/schema/api/feed/resolveTask.py +1 -1
- metadata/generated/schema/api/feed/threadCount.py +1 -1
- metadata/generated/schema/api/governance/__init__.py +1 -1
- metadata/generated/schema/api/governance/createWorkflowDefinition.py +20 -5
- metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
- metadata/generated/schema/api/lineage/__init__.py +1 -1
- metadata/generated/schema/api/lineage/addLineage.py +1 -1
- metadata/generated/schema/api/lineage/entityCountLineageRequest.py +66 -0
- metadata/generated/schema/api/lineage/esLineageData.py +1 -1
- metadata/generated/schema/api/lineage/lineageDirection.py +1 -1
- metadata/generated/schema/api/lineage/lineagePaginationInfo.py +57 -0
- metadata/generated/schema/api/lineage/nodeInformation.py +10 -2
- metadata/generated/schema/api/lineage/searchLineageRequest.py +1 -1
- metadata/generated/schema/api/lineage/searchLineageResult.py +1 -1
- metadata/generated/schema/api/mcp/__init__.py +1 -1
- metadata/generated/schema/api/mcp/mcpSearchResponse.py +118 -0
- metadata/generated/schema/api/mcp/mcpToolDefinition.py +1 -1
- metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
- metadata/generated/schema/api/policies/__init__.py +1 -1
- metadata/generated/schema/api/policies/createPolicy.py +1 -1
- metadata/generated/schema/api/rdf/__init__.py +3 -0
- metadata/generated/schema/api/rdf/sparqlQuery.py +57 -0
- metadata/generated/schema/api/rdf/sparqlResponse.py +56 -0
- metadata/generated/schema/api/scim/__init__.py +1 -1
- metadata/generated/schema/api/scim/scimGroup.py +1 -1
- metadata/generated/schema/api/scim/scimPatchOp.py +1 -1
- metadata/generated/schema/api/scim/scimUser.py +1 -1
- metadata/generated/schema/api/search/__init__.py +1 -1
- metadata/generated/schema/api/search/previewSearchRequest.py +1 -1
- metadata/generated/schema/api/services/__init__.py +1 -1
- metadata/generated/schema/api/services/createApiService.py +1 -1
- metadata/generated/schema/api/services/createDashboardService.py +1 -1
- metadata/generated/schema/api/services/createDatabaseService.py +1 -1
- metadata/generated/schema/api/services/createDriveService.py +9 -2
- metadata/generated/schema/api/services/createMessagingService.py +1 -1
- metadata/generated/schema/api/services/createMetadataService.py +1 -1
- metadata/generated/schema/api/services/createMlModelService.py +1 -1
- metadata/generated/schema/api/services/createPipelineService.py +1 -1
- metadata/generated/schema/api/services/createSearchService.py +1 -1
- metadata/generated/schema/api/services/createSecurityService.py +1 -1
- metadata/generated/schema/api/services/createStorageService.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +8 -1
- metadata/generated/schema/api/setOwner.py +1 -1
- metadata/generated/schema/api/teams/__init__.py +1 -1
- metadata/generated/schema/api/teams/createPersona.py +1 -1
- metadata/generated/schema/api/teams/createRole.py +1 -1
- metadata/generated/schema/api/teams/createTeam.py +1 -1
- metadata/generated/schema/api/teams/createUser.py +1 -1
- metadata/generated/schema/api/tests/__init__.py +1 -1
- metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
- metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
- metadata/generated/schema/api/tests/createTestCase.py +5 -1
- metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
- metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
- metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
- metadata/generated/schema/api/tests/createTestSuite.py +5 -1
- metadata/generated/schema/api/tests/moveGlossaryTermRequest.py +1 -1
- metadata/generated/schema/api/validateGlossaryTagsRequest.py +1 -1
- metadata/generated/schema/api/voteRequest.py +1 -1
- metadata/generated/schema/auth/__init__.py +1 -1
- metadata/generated/schema/auth/basicAuth.py +1 -1
- metadata/generated/schema/auth/basicLoginRequest.py +1 -1
- metadata/generated/schema/auth/changePasswordRequest.py +1 -1
- metadata/generated/schema/auth/createPersonalToken.py +1 -1
- metadata/generated/schema/auth/emailRequest.py +1 -1
- metadata/generated/schema/auth/emailVerificationToken.py +1 -1
- metadata/generated/schema/auth/generateToken.py +1 -1
- metadata/generated/schema/auth/jwtAuth.py +1 -1
- metadata/generated/schema/auth/loginRequest.py +1 -1
- metadata/generated/schema/auth/logoutRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetToken.py +1 -1
- metadata/generated/schema/auth/personalAccessToken.py +1 -1
- metadata/generated/schema/auth/refreshToken.py +1 -1
- metadata/generated/schema/auth/registrationRequest.py +1 -1
- metadata/generated/schema/auth/revokePersonalToken.py +1 -1
- metadata/generated/schema/auth/revokeToken.py +1 -1
- metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
- metadata/generated/schema/auth/ssoAuth.py +1 -1
- metadata/generated/schema/auth/supportToken.py +1 -1
- metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
- metadata/generated/schema/configuration/__init__.py +1 -1
- metadata/generated/schema/configuration/aiPlatformConfiguration.py +1 -1
- metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
- metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
- metadata/generated/schema/configuration/authConfig.py +1 -1
- metadata/generated/schema/configuration/authenticationConfiguration.py +49 -13
- metadata/generated/schema/configuration/authorizerConfiguration.py +39 -10
- metadata/generated/schema/configuration/cacheConfiguration.py +1 -1
- metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
- metadata/generated/schema/configuration/elasticSearchConfiguration.py +19 -1
- metadata/generated/schema/configuration/entityRulesSettings.py +1 -1
- metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
- metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
- metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
- metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
- metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
- metadata/generated/schema/configuration/lineageSettings.py +9 -1
- metadata/generated/schema/configuration/logStorageConfiguration.py +105 -0
- metadata/generated/schema/configuration/loginConfiguration.py +1 -1
- metadata/generated/schema/configuration/logoConfiguration.py +1 -1
- metadata/generated/schema/configuration/openMetadataBaseUrlConfiguration.py +1 -1
- metadata/generated/schema/configuration/opertionalConfiguration.py +1 -1
- metadata/generated/schema/configuration/opsConfig.py +1 -1
- metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +9 -2
- metadata/generated/schema/configuration/profilerConfiguration.py +2 -1
- metadata/generated/schema/configuration/searchSettings.py +7 -2
- metadata/generated/schema/configuration/securityConfiguration.py +26 -0
- metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
- metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/themeConfiguration.py +1 -1
- metadata/generated/schema/configuration/uiThemePreference.py +1 -1
- metadata/generated/schema/configuration/workflowSettings.py +15 -1
- metadata/generated/schema/dataInsight/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
- metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
- metadata/generated/schema/dataInsight/custom/lineChart.py +1 -1
- metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
- metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
- metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
- metadata/generated/schema/dataInsight/type/__init__.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
- metadata/generated/schema/email/__init__.py +1 -1
- metadata/generated/schema/email/emailRequest.py +1 -1
- metadata/generated/schema/email/emailTemplate.py +1 -1
- metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
- metadata/generated/schema/email/smtpSettings.py +1 -1
- metadata/generated/schema/email/templateValidationReponse.py +1 -1
- metadata/generated/schema/entity/__init__.py +1 -1
- metadata/generated/schema/entity/applications/__init__.py +1 -1
- metadata/generated/schema/entity/applications/app.py +15 -1
- metadata/generated/schema/entity/applications/appExtension.py +1 -1
- metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
- metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/applicationConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addCustomProperties.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDataProductAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +7 -2
- metadata/generated/schema/entity/applications/configuration/external/automator/addTermsAction.py +48 -0
- metadata/generated/schema/entity/applications/configuration/external/automator/addTestCaseAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/propagationStopConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDataProductAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +6 -2
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTermsAction.py +59 -0
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTestCaseAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +7 -1
- metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterAppConfig.py +103 -0
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/__init__.py +3 -0
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/bigQueryConnection.py +90 -0
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/databricksConnection.py +101 -0
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/redshiftConnection.py +76 -0
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/snowflakeConnection.py +115 -0
- metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/autoPilotAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/collateAITierAgentAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataRetentionConfiguration.py +15 -1
- metadata/generated/schema/entity/applications/configuration/internal/helloPipelinesConfiguration.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/internal/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/internal/collateAITierAgentAppPrivateConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/limits.py +1 -1
- metadata/generated/schema/entity/applications/createAppRequest.py +9 -2
- metadata/generated/schema/entity/applications/jobStatus.py +1 -1
- metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +8 -1
- metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +8 -1
- metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
- metadata/generated/schema/entity/automations/__init__.py +1 -1
- metadata/generated/schema/entity/automations/queryRunnerRequest.py +46 -0
- metadata/generated/schema/entity/automations/response/__init__.py +3 -0
- metadata/generated/schema/entity/automations/response/queryRunnerResponse.py +51 -0
- metadata/generated/schema/entity/automations/testServiceConnection.py +3 -1
- metadata/generated/schema/entity/automations/testSparkEngineConnection.py +31 -0
- metadata/generated/schema/entity/automations/workflow.py +8 -2
- metadata/generated/schema/entity/bot.py +1 -1
- metadata/generated/schema/entity/classification/__init__.py +1 -1
- metadata/generated/schema/entity/classification/classification.py +57 -2
- metadata/generated/schema/entity/classification/tag.py +41 -3
- metadata/generated/schema/entity/data/__init__.py +1 -1
- metadata/generated/schema/entity/data/apiCollection.py +8 -1
- metadata/generated/schema/entity/data/apiEndpoint.py +8 -1
- metadata/generated/schema/entity/data/chart.py +12 -1
- metadata/generated/schema/entity/data/container.py +7 -1
- metadata/generated/schema/entity/data/dashboard.py +6 -1
- metadata/generated/schema/entity/data/dashboardDataModel.py +13 -1
- metadata/generated/schema/entity/data/dataContract.py +238 -2
- metadata/generated/schema/entity/data/database.py +6 -1
- metadata/generated/schema/entity/data/databaseSchema.py +8 -1
- metadata/generated/schema/entity/data/directory.py +6 -1
- metadata/generated/schema/entity/data/file.py +6 -1
- metadata/generated/schema/entity/data/glossary.py +6 -1
- metadata/generated/schema/entity/data/glossaryTerm.py +8 -12
- metadata/generated/schema/entity/data/metric.py +18 -1
- metadata/generated/schema/entity/data/mlmodel.py +6 -1
- metadata/generated/schema/entity/data/pipeline.py +136 -2
- metadata/generated/schema/entity/data/query.py +6 -1
- metadata/generated/schema/entity/data/queryCostRecord.py +1 -1
- metadata/generated/schema/entity/data/queryCostSearchResult.py +1 -1
- metadata/generated/schema/entity/data/report.py +6 -1
- metadata/generated/schema/entity/data/searchIndex.py +8 -1
- metadata/generated/schema/entity/data/spreadsheet.py +10 -4
- metadata/generated/schema/entity/data/storedProcedure.py +9 -1
- metadata/generated/schema/entity/data/table.py +102 -1
- metadata/generated/schema/entity/data/topic.py +6 -1
- metadata/generated/schema/entity/data/worksheet.py +6 -1
- metadata/generated/schema/entity/datacontract/__init__.py +1 -1
- metadata/generated/schema/entity/datacontract/dataContractResult.py +1 -1
- metadata/generated/schema/entity/datacontract/qualityValidation.py +1 -1
- metadata/generated/schema/entity/datacontract/schemaValidation.py +1 -1
- metadata/generated/schema/entity/datacontract/semanticsValidation.py +1 -1
- metadata/generated/schema/entity/datacontract/slaValidation.py +1 -1
- metadata/generated/schema/entity/docStore/__init__.py +1 -1
- metadata/generated/schema/entity/docStore/document.py +1 -1
- metadata/generated/schema/entity/domains/__init__.py +1 -1
- metadata/generated/schema/entity/domains/dataProduct.py +155 -2
- metadata/generated/schema/entity/domains/domain.py +1 -1
- metadata/generated/schema/entity/events/__init__.py +1 -1
- metadata/generated/schema/entity/events/notificationTemplate.py +111 -0
- metadata/generated/schema/entity/events/webhook.py +1 -1
- metadata/generated/schema/entity/feed/__init__.py +1 -1
- metadata/generated/schema/entity/feed/assets.py +1 -1
- metadata/generated/schema/entity/feed/customProperty.py +1 -1
- metadata/generated/schema/entity/feed/description.py +1 -1
- metadata/generated/schema/entity/feed/domain.py +1 -1
- metadata/generated/schema/entity/feed/entityInfo.py +1 -1
- metadata/generated/schema/entity/feed/owner.py +1 -1
- metadata/generated/schema/entity/feed/suggestion.py +1 -1
- metadata/generated/schema/entity/feed/tag.py +1 -1
- metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
- metadata/generated/schema/entity/feed/thread.py +1 -1
- metadata/generated/schema/entity/policies/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
- metadata/generated/schema/entity/policies/filters.py +1 -1
- metadata/generated/schema/entity/policies/policy.py +1 -1
- metadata/generated/schema/entity/services/__init__.py +1 -1
- metadata/generated/schema/entity/services/apiService.py +1 -1
- metadata/generated/schema/entity/services/connections/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/api/restConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/grafanaConnection.py +77 -0
- metadata/generated/schema/entity/services/connections/dashboard/hexConnection.py +76 -0
- metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +13 -2
- metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +17 -4
- metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +9 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/thoughtSpotConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/athenaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +9 -1
- metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/cassandra/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/cassandra/cloudConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/cassandraConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/cockroachConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
- metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/databricks/__init__.py +3 -0
- metadata/generated/schema/entity/services/connections/database/databricks/azureAdSetup.py +37 -0
- metadata/generated/schema/entity/services/connections/database/databricks/databricksOAuth.py +30 -0
- metadata/generated/schema/entity/services/connections/database/databricks/personalAccessToken.py +23 -0
- metadata/generated/schema/entity/services/connections/database/databricksConnection.py +14 -6
- metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/db2Connection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/druidConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/epicConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/exasolConnection.py +4 -1
- metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/hiveConnection.py +16 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/impalaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/oracleConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/postgresConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +25 -6
- metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/serviceNowConnection.py +90 -0
- metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +10 -2
- metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/ssasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/teradataConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/timescaleConnection.py +162 -0
- metadata/generated/schema/entity/services/connections/database/trinoConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +14 -6
- metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/drive/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/drive/customDriveConnection.py +35 -2
- metadata/generated/schema/entity/services/connections/drive/googleDriveConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/drive/sharePointConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/collibraConnection.py +73 -0
- metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airbyte/__init__.py +3 -0
- metadata/generated/schema/entity/services/connections/pipeline/airbyte/basicAuth.py +26 -0
- metadata/generated/schema/entity/services/connections/pipeline/airbyte/oauthClientAuth.py +30 -0
- metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +16 -10
- metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +11 -1
- metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/kinesisFirehoseConnection.py +58 -0
- metadata/generated/schema/entity/services/connections/pipeline/matillion/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/matillion/matillionETL.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifi/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifi/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/snowplowConnection.py +93 -0
- metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/ssisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/wherescapeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/security/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/security/ranger/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/security/ranger/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/security/rangerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/s3Connection.py +10 -2
- metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
- metadata/generated/schema/entity/services/dashboardService.py +7 -1
- metadata/generated/schema/entity/services/databaseService.py +7 -1
- metadata/generated/schema/entity/services/driveService.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +8 -1
- metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/reverseIngestionResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
- metadata/generated/schema/entity/services/messagingService.py +1 -1
- metadata/generated/schema/entity/services/metadataService.py +4 -1
- metadata/generated/schema/entity/services/mlmodelService.py +1 -1
- metadata/generated/schema/entity/services/pipelineService.py +7 -1
- metadata/generated/schema/entity/services/searchService.py +1 -1
- metadata/generated/schema/entity/services/securityService.py +1 -1
- metadata/generated/schema/entity/services/serviceType.py +1 -1
- metadata/generated/schema/entity/services/storageService.py +1 -1
- metadata/generated/schema/entity/teams/__init__.py +1 -1
- metadata/generated/schema/entity/teams/persona.py +1 -1
- metadata/generated/schema/entity/teams/role.py +1 -1
- metadata/generated/schema/entity/teams/team.py +1 -1
- metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
- metadata/generated/schema/entity/teams/user.py +1 -1
- metadata/generated/schema/entity/type.py +1 -1
- metadata/generated/schema/entity/utils/__init__.py +1 -1
- metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
- metadata/generated/schema/entity/utils/servicesCount.py +1 -1
- metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
- metadata/generated/schema/events/__init__.py +1 -1
- metadata/generated/schema/events/alertMetrics.py +1 -1
- metadata/generated/schema/events/api/__init__.py +1 -1
- metadata/generated/schema/events/api/createEventSubscription.py +1 -1
- metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
- metadata/generated/schema/events/api/eventsRecord.py +1 -1
- metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
- metadata/generated/schema/events/api/typedEvent.py +1 -1
- metadata/generated/schema/events/emailAlertConfig.py +1 -1
- metadata/generated/schema/events/eventFilterRule.py +1 -1
- metadata/generated/schema/events/eventSubscription.py +16 -1
- metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
- metadata/generated/schema/events/failedEvent.py +1 -1
- metadata/generated/schema/events/failedEventResponse.py +1 -1
- metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
- metadata/generated/schema/events/statusContext.py +1 -1
- metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
- metadata/generated/schema/events/subscriptionStatus.py +1 -1
- metadata/generated/schema/events/testDestinationStatus.py +1 -1
- metadata/generated/schema/governance/workflows/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/edge.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodeSubType.py +4 -1
- metadata/generated/schema/governance/workflows/elements/nodeType.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipelineTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/dataCompletenessTask.py +124 -0
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/rollbackEntityTask.py +54 -0
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityAttributeTask.py +66 -0
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +11 -12
- metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/gateway/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/gateway/parallelGateway.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +22 -1
- metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +90 -7
- metadata/generated/schema/governance/workflows/elements/triggers/noOpTrigger.py +1 -1
- metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +80 -7
- metadata/generated/schema/governance/workflows/workflowDefinition.py +1 -1
- metadata/generated/schema/governance/workflows/workflowInstance.py +1 -1
- metadata/generated/schema/governance/workflows/workflowInstanceState.py +1 -1
- metadata/generated/schema/jobs/__init__.py +1 -1
- metadata/generated/schema/jobs/backgroundJob.py +1 -1
- metadata/generated/schema/jobs/enumCleanupArgs.py +1 -1
- metadata/generated/schema/metadataIngestion/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/application.py +8 -1
- metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/driveServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/engine/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/engine/nativeEngineConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/engine/sparkEngineConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +29 -2
- metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/reverseIngestionPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/ownerConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/tagsConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/securityServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +17 -1
- metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/workflow.py +8 -1
- metadata/generated/schema/monitoring/__init__.py +1 -1
- metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
- metadata/generated/schema/scim/__init__.py +1 -1
- metadata/generated/schema/scim/scimConfiguration.py +1 -1
- metadata/generated/schema/search/__init__.py +1 -1
- metadata/generated/schema/search/aggregationRequest.py +1 -1
- metadata/generated/schema/search/searchRequest.py +8 -1
- metadata/generated/schema/security/__init__.py +1 -1
- metadata/generated/schema/security/client/__init__.py +1 -1
- metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/oidcClientConfig.py +6 -10
- metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
- metadata/generated/schema/security/client/samlSSOClientConfig.py +6 -6
- metadata/generated/schema/security/credentials/__init__.py +1 -1
- metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
- metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
- metadata/generated/schema/security/credentials/basicAuth.py +1 -1
- metadata/generated/schema/security/credentials/bitbucketCredentials.py +10 -2
- metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
- metadata/generated/schema/security/credentials/gcpValues.py +1 -1
- metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
- metadata/generated/schema/security/credentials/githubCredentials.py +10 -2
- metadata/generated/schema/security/credentials/gitlabCredentials.py +10 -2
- metadata/generated/schema/security/credentials/kubernetesCredentials.py +1 -1
- metadata/generated/schema/security/sasl/__init__.py +1 -1
- metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
- metadata/generated/schema/security/secrets/__init__.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
- metadata/generated/schema/security/securityConfiguration.py +1 -1
- metadata/generated/schema/security/ssl/__init__.py +1 -1
- metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
- metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
- metadata/generated/schema/settings/__init__.py +1 -1
- metadata/generated/schema/settings/settings.py +5 -2
- metadata/generated/schema/system/__init__.py +1 -1
- metadata/generated/schema/system/entityError.py +1 -1
- metadata/generated/schema/system/eventPublisherJob.py +1 -1
- metadata/generated/schema/system/indexingError.py +1 -1
- metadata/generated/schema/system/limitsResponse.py +1 -1
- metadata/generated/schema/system/securityValidationResponse.py +62 -0
- metadata/generated/schema/system/ui/__init__.py +1 -1
- metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
- metadata/generated/schema/system/ui/navigationItem.py +1 -1
- metadata/generated/schema/system/ui/page.py +8 -1
- metadata/generated/schema/system/ui/tab.py +1 -1
- metadata/generated/schema/system/ui/uiCustomization.py +1 -1
- metadata/generated/schema/system/validationResponse.py +8 -1
- metadata/generated/schema/tests/__init__.py +1 -1
- metadata/generated/schema/tests/assigned.py +1 -1
- metadata/generated/schema/tests/basic.py +20 -20
- metadata/generated/schema/tests/customMetric.py +1 -1
- metadata/generated/schema/tests/dataQualityReport.py +1 -1
- metadata/generated/schema/tests/resolved.py +1 -1
- metadata/generated/schema/tests/testCase.py +26 -2
- metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
- metadata/generated/schema/tests/testDefinition.py +1 -1
- metadata/generated/schema/tests/testSuite.py +12 -1
- metadata/generated/schema/type/__init__.py +1 -1
- metadata/generated/schema/type/apiSchema.py +1 -1
- metadata/generated/schema/type/assetCertification.py +1 -1
- metadata/generated/schema/type/auditLog.py +1 -1
- metadata/generated/schema/type/basic.py +9 -1
- metadata/generated/schema/type/bulkOperationResult.py +1 -1
- metadata/generated/schema/type/changeEvent.py +1 -1
- metadata/generated/schema/type/changeEventType.py +1 -1
- metadata/generated/schema/type/changeSummaryMap.py +1 -1
- metadata/generated/schema/type/collectionDescriptor.py +1 -1
- metadata/generated/schema/type/contextRecognizer.py +50 -0
- metadata/generated/schema/type/contractExecutionStatus.py +1 -1
- metadata/generated/schema/type/csvDocumentation.py +1 -1
- metadata/generated/schema/type/csvErrorType.py +1 -1
- metadata/generated/schema/type/csvFile.py +1 -1
- metadata/generated/schema/type/csvImportResult.py +1 -1
- metadata/generated/schema/type/customProperties/__init__.py +1 -1
- metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
- metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
- metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
- metadata/generated/schema/type/customProperty.py +1 -1
- metadata/generated/schema/type/customRecognizer.py +39 -0
- metadata/generated/schema/type/dailyCount.py +1 -1
- metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
- metadata/generated/schema/type/denyListRecognizer.py +37 -0
- metadata/generated/schema/type/entityHierarchy.py +1 -1
- metadata/generated/schema/type/entityHistory.py +1 -1
- metadata/generated/schema/type/entityLineage.py +1 -1
- metadata/generated/schema/type/entityProfile.py +45 -0
- metadata/generated/schema/type/entityReference.py +1 -1
- metadata/generated/schema/type/entityReferenceList.py +1 -1
- metadata/generated/schema/type/entityRelationship/__init__.py +1 -1
- metadata/generated/schema/type/entityRelationship/nodeInformation.py +1 -1
- metadata/generated/schema/type/entityUsage.py +1 -1
- metadata/generated/schema/type/filterPattern.py +1 -1
- metadata/generated/schema/type/function.py +1 -1
- metadata/generated/schema/type/include.py +1 -1
- metadata/generated/schema/type/jdbcConnection.py +1 -1
- metadata/generated/schema/type/layerPaging.py +1 -1
- metadata/generated/schema/type/lifeCycle.py +1 -1
- metadata/generated/schema/type/paging.py +1 -1
- metadata/generated/schema/type/patternRecognizer.py +39 -0
- metadata/generated/schema/type/personaPreferences.py +1 -1
- metadata/generated/schema/type/piiEntity.py +48 -0
- metadata/generated/schema/type/predefinedRecognizer.py +85 -0
- metadata/generated/schema/type/profile.py +1 -1
- metadata/generated/schema/type/queryParserData.py +1 -1
- metadata/generated/schema/type/reaction.py +1 -1
- metadata/generated/schema/type/recognizer.py +159 -0
- metadata/generated/schema/type/recognizerFeedback.py +120 -0
- metadata/generated/schema/type/recognizers/__init__.py +3 -0
- metadata/generated/schema/type/recognizers/patterns.py +29 -0
- metadata/generated/schema/type/recognizers/regexFlags.py +29 -0
- metadata/generated/schema/type/schedule.py +1 -1
- metadata/generated/schema/type/schema.py +1 -1
- metadata/generated/schema/type/status.py +16 -0
- metadata/generated/schema/type/tableQuery.py +1 -1
- metadata/generated/schema/type/tableUsageCount.py +1 -1
- metadata/generated/schema/type/tagLabel.py +8 -1
- metadata/generated/schema/type/usageDetails.py +1 -1
- metadata/generated/schema/type/usageRequest.py +1 -1
- metadata/generated/schema/type/votes.py +1 -1
- metadata/great_expectations/action1xx.py +2 -1
- metadata/ingestion/api/status.py +6 -1
- metadata/ingestion/api/step.py +3 -1
- metadata/ingestion/bulksink/metadata_usage.py +19 -2
- metadata/ingestion/connections/builders.py +5 -1
- metadata/ingestion/lineage/masker.py +5 -0
- metadata/ingestion/lineage/models.py +4 -0
- metadata/ingestion/lineage/parser.py +56 -4
- metadata/ingestion/lineage/sql_lineage.py +243 -103
- metadata/ingestion/models/custom_basemodel_validation.py +169 -43
- metadata/ingestion/models/custom_pydantic.py +6 -19
- metadata/ingestion/models/patch_request.py +2 -0
- metadata/ingestion/ometa/client.py +2 -2
- metadata/ingestion/ometa/mixins/csv_mixin.py +180 -0
- metadata/ingestion/ometa/mixins/domain_mixin.py +48 -0
- metadata/ingestion/ometa/mixins/es_mixin.py +7 -1
- metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py +26 -0
- metadata/ingestion/ometa/mixins/lineage_mixin.py +1 -1
- metadata/ingestion/ometa/mixins/logs_mixin.py +316 -0
- metadata/ingestion/ometa/mixins/patch_mixin.py +23 -9
- metadata/ingestion/ometa/mixins/profile_mixin.py +77 -0
- metadata/ingestion/ometa/mixins/tag_glossary_mixin.py +68 -0
- metadata/ingestion/ometa/mixins/tests_mixin.py +1 -1
- metadata/ingestion/ometa/mixins/user_mixin.py +57 -16
- metadata/ingestion/ometa/ometa_api.py +24 -1
- metadata/ingestion/ometa/routes.py +5 -0
- metadata/ingestion/source/api/rest/connection.py +12 -6
- metadata/ingestion/source/api/rest/metadata.py +227 -37
- metadata/ingestion/source/api/rest/parser.py +103 -0
- metadata/ingestion/source/dashboard/domodashboard/metadata.py +19 -9
- metadata/ingestion/source/dashboard/grafana/__init__.py +10 -0
- metadata/ingestion/source/dashboard/grafana/client.py +223 -0
- metadata/ingestion/source/dashboard/grafana/connection.py +66 -0
- metadata/ingestion/source/dashboard/grafana/metadata.py +476 -0
- metadata/ingestion/source/dashboard/grafana/models.py +159 -0
- metadata/ingestion/source/dashboard/grafana/service_spec.py +17 -0
- metadata/ingestion/source/dashboard/hex/client.py +104 -0
- metadata/ingestion/source/dashboard/hex/connection.py +60 -0
- metadata/ingestion/source/dashboard/hex/metadata.py +363 -0
- metadata/ingestion/source/dashboard/hex/models.py +143 -0
- metadata/ingestion/source/dashboard/hex/query_fetcher.py +503 -0
- metadata/ingestion/source/dashboard/hex/service_spec.py +18 -0
- metadata/ingestion/source/dashboard/hex/warehouse_queries.py +225 -0
- metadata/ingestion/source/dashboard/looker/bulk_parser.py +31 -14
- metadata/ingestion/source/dashboard/looker/metadata.py +131 -32
- metadata/ingestion/source/dashboard/looker/models.py +3 -0
- metadata/ingestion/source/dashboard/looker/utils.py +18 -3
- metadata/ingestion/source/dashboard/metabase/client.py +33 -11
- metadata/ingestion/source/dashboard/metabase/metadata.py +4 -0
- metadata/ingestion/source/dashboard/metabase/models.py +49 -1
- metadata/ingestion/source/dashboard/microstrategy/metadata.py +1 -1
- metadata/ingestion/source/dashboard/powerbi/client.py +18 -1
- metadata/ingestion/source/dashboard/powerbi/databricks_parser.py +108 -0
- metadata/ingestion/source/dashboard/powerbi/metadata.py +412 -100
- metadata/ingestion/source/dashboard/powerbi/models.py +20 -1
- metadata/ingestion/source/dashboard/redash/metadata.py +4 -0
- metadata/ingestion/source/dashboard/sigma/metadata.py +4 -0
- metadata/ingestion/source/dashboard/superset/api_source.py +2 -0
- metadata/ingestion/source/dashboard/superset/db_source.py +2 -0
- metadata/ingestion/source/dashboard/superset/mixin.py +2 -0
- metadata/ingestion/source/dashboard/tableau/client.py +29 -14
- metadata/ingestion/source/dashboard/tableau/metadata.py +7 -1
- metadata/ingestion/source/database/athena/utils.py +1 -1
- metadata/ingestion/source/database/bigquery/connection.py +33 -13
- metadata/ingestion/source/database/bigquery/lineage.py +5 -10
- metadata/ingestion/source/database/bigquery/metadata.py +39 -34
- metadata/ingestion/source/database/bigquery/queries.py +6 -1
- metadata/ingestion/source/database/column_type_parser.py +1 -1
- metadata/ingestion/source/database/common/data_diff/databricks_base.py +42 -0
- metadata/ingestion/source/database/common_pg_mappings.py +7 -0
- metadata/ingestion/source/database/databricks/auth.py +99 -0
- metadata/ingestion/source/database/databricks/client.py +201 -51
- metadata/ingestion/source/database/databricks/connection.py +99 -13
- metadata/ingestion/source/database/databricks/data_diff/__init__.py +1 -0
- metadata/ingestion/source/database/databricks/data_diff/data_diff.py +12 -0
- metadata/ingestion/source/database/databricks/queries.py +86 -16
- metadata/ingestion/source/database/databricks/service_spec.py +4 -0
- metadata/ingestion/source/database/dbt/constants.py +0 -1
- metadata/ingestion/source/database/dbt/metadata.py +47 -21
- metadata/ingestion/source/database/dbt/models.py +1 -0
- metadata/ingestion/source/database/exasol/connection.py +6 -0
- metadata/ingestion/source/database/exasol/lineage.py +24 -0
- metadata/ingestion/source/database/exasol/metadata.py +6 -0
- metadata/ingestion/source/database/exasol/queries.py +22 -0
- metadata/ingestion/source/database/exasol/query_parser.py +33 -0
- metadata/ingestion/source/database/exasol/service_spec.py +5 -1
- metadata/ingestion/source/database/glue/metadata.py +2 -2
- metadata/ingestion/source/database/hive/connection.py +41 -2
- metadata/ingestion/source/database/hive/custom_hive_connection.py +195 -0
- metadata/ingestion/source/database/iceberg/metadata.py +44 -1
- metadata/ingestion/source/database/iceberg/models.py +22 -13
- metadata/ingestion/source/database/lineage_processors.py +386 -0
- metadata/ingestion/source/database/lineage_source.py +314 -248
- metadata/ingestion/source/database/mariadb/metadata.py +86 -1
- metadata/ingestion/source/database/mariadb/models.py +36 -0
- metadata/ingestion/source/database/mariadb/queries.py +43 -0
- metadata/ingestion/source/database/mssql/lineage.py +3 -27
- metadata/ingestion/source/database/oracle/connection.py +158 -114
- metadata/ingestion/source/database/oracle/lineage.py +3 -10
- metadata/ingestion/source/database/oracle/queries.py +43 -6
- metadata/ingestion/source/database/oracle/service_spec.py +2 -0
- metadata/ingestion/source/database/postgres/connection.py +6 -0
- metadata/ingestion/source/database/postgres/queries.py +22 -0
- metadata/ingestion/source/database/redshift/lineage.py +4 -10
- metadata/ingestion/source/database/redshift/utils.py +2 -1
- metadata/ingestion/source/database/salesforce/connection.py +14 -6
- metadata/ingestion/source/database/salesforce/metadata.py +14 -16
- metadata/ingestion/source/database/sample_data.py +24 -8
- metadata/ingestion/source/database/saphana/cdata_parser.py +327 -117
- metadata/ingestion/source/database/saphana/lineage.py +15 -2
- metadata/ingestion/source/database/saphana/queries.py +7 -0
- metadata/ingestion/source/database/snowflake/connection.py +7 -0
- metadata/ingestion/source/database/snowflake/constants.py +21 -0
- metadata/ingestion/source/database/snowflake/lineage.py +48 -30
- metadata/ingestion/source/database/snowflake/metadata.py +176 -22
- metadata/ingestion/source/database/snowflake/queries.py +15 -1
- metadata/ingestion/source/database/snowflake/query_parser.py +12 -2
- metadata/ingestion/source/database/snowflake/usage.py +97 -0
- metadata/ingestion/source/database/sql_column_handler.py +11 -3
- metadata/ingestion/source/database/stored_procedures_mixin.py +66 -221
- metadata/ingestion/source/database/timescale/__init__.py +3 -0
- metadata/ingestion/source/database/timescale/connection.py +103 -0
- metadata/ingestion/source/database/timescale/lineage.py +124 -0
- metadata/ingestion/source/database/timescale/metadata.py +228 -0
- metadata/ingestion/source/database/timescale/models.py +60 -0
- metadata/ingestion/source/database/timescale/queries.py +96 -0
- metadata/ingestion/source/database/timescale/query_parser.py +106 -0
- metadata/ingestion/source/database/timescale/service_spec.py +12 -0
- metadata/ingestion/source/database/timescale/usage.py +43 -0
- metadata/ingestion/source/database/timescale/utils.py +545 -0
- metadata/ingestion/source/database/trino/metadata.py +43 -4
- metadata/ingestion/source/database/trino/queries.py +4 -0
- metadata/ingestion/source/database/unitycatalog/client.py +35 -1
- metadata/ingestion/source/database/unitycatalog/connection.py +43 -12
- metadata/ingestion/source/database/unitycatalog/data_diff/__init__.py +1 -0
- metadata/ingestion/source/database/unitycatalog/data_diff/data_diff.py +12 -0
- metadata/ingestion/source/database/unitycatalog/lineage.py +213 -24
- metadata/ingestion/source/database/unitycatalog/metadata.py +5 -4
- metadata/ingestion/source/database/unitycatalog/models.py +18 -2
- metadata/ingestion/source/database/unitycatalog/service_spec.py +4 -0
- metadata/ingestion/source/messaging/kafka/connection.py +13 -4
- metadata/ingestion/source/metadata/amundsen/client.py +6 -6
- metadata/ingestion/source/mlmodel/mlmodel_service.py +75 -1
- metadata/ingestion/source/pipeline/airbyte/client.py +142 -4
- metadata/ingestion/source/pipeline/airbyte/connection.py +18 -5
- metadata/ingestion/source/pipeline/airbyte/metadata.py +102 -2
- metadata/ingestion/source/pipeline/airflow/connection.py +8 -6
- metadata/ingestion/source/pipeline/airflow/metadata.py +5 -1
- metadata/ingestion/source/pipeline/databrickspipeline/connection.py +1 -0
- metadata/ingestion/source/pipeline/databrickspipeline/kafka_parser.py +603 -0
- metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +1019 -90
- metadata/ingestion/source/pipeline/databrickspipeline/models.py +19 -3
- metadata/ingestion/source/pipeline/dbtcloud/client.py +85 -76
- metadata/ingestion/source/pipeline/dbtcloud/metadata.py +135 -84
- metadata/ingestion/source/pipeline/dbtcloud/models.py +4 -0
- metadata/ingestion/source/pipeline/dbtcloud/queries.py +7 -11
- metadata/ingestion/source/pipeline/fivetran/client.py +7 -3
- metadata/ingestion/source/pipeline/fivetran/metadata.py +122 -90
- metadata/ingestion/source/pipeline/kafkaconnect/client.py +388 -107
- metadata/ingestion/source/pipeline/kafkaconnect/metadata.py +1678 -72
- metadata/ingestion/source/pipeline/kafkaconnect/models.py +51 -14
- metadata/ingestion/source/pipeline/nifi/metadata.py +62 -5
- metadata/ingestion/source/pipeline/openlineage/metadata.py +6 -2
- metadata/ingestion/source/search/elasticsearch/connection.py +1 -1
- metadata/ingestion/source/storage/gcs/connection.py +15 -5
- metadata/ingestion/source/storage/gcs/metadata.py +285 -26
- metadata/ingestion/source/storage/gcs/models.py +12 -6
- metadata/ingestion/source/storage/s3/metadata.py +46 -11
- metadata/ingestion/source/storage/storage_service.py +2 -0
- metadata/parsers/json_schema_parser.py +41 -1
- metadata/pii/algorithms/classifiers.py +63 -1
- metadata/pii/algorithms/feature_extraction.py +13 -2
- metadata/pii/algorithms/patterns.py +171 -0
- metadata/pii/algorithms/preprocessing.py +19 -13
- metadata/pii/algorithms/presidio_recognizer_factory.py +302 -0
- metadata/pii/algorithms/presidio_utils.py +205 -24
- metadata/pii/algorithms/utils.py +4 -0
- metadata/pii/base_processor.py +2 -1
- metadata/pii/processor.py +8 -5
- metadata/pii/processor_factory.py +28 -0
- metadata/pii/scanners/custom_ner_scanner.py +290 -0
- metadata/pii/tag_analyzer.py +124 -0
- metadata/pii/tag_processor.py +100 -0
- metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +17 -0
- metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +24 -0
- metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +16 -2
- metadata/profiler/metrics/hybrid/cardinality_distribution.py +232 -0
- metadata/profiler/metrics/hybrid/histogram.py +1 -1
- metadata/profiler/metrics/registry.py +4 -0
- metadata/profiler/metrics/system/bigquery/system.py +4 -1
- metadata/profiler/metrics/system/databricks/system.py +1 -1
- metadata/profiler/orm/converter/converter_registry.py +2 -0
- metadata/profiler/orm/converter/trino/__init__.py +14 -0
- metadata/profiler/orm/converter/trino/converter.py +39 -0
- metadata/profiler/orm/functions/conn_test.py +5 -0
- metadata/profiler/orm/functions/table_metric_computer.py +5 -18
- metadata/profiler/orm/registry.py +7 -0
- metadata/profiler/orm/types/__init__.py +10 -0
- metadata/profiler/orm/types/trino.py +70 -0
- metadata/profiler/processor/core.py +1 -1
- metadata/profiler/processor/default.py +1 -0
- metadata/readers/dataframe/mf4.py +75 -0
- metadata/readers/dataframe/reader_factory.py +3 -0
- metadata/sampler/sqlalchemy/bigquery/sampler.py +17 -0
- metadata/sampler/sqlalchemy/databricks/sampler.py +13 -1
- metadata/sampler/sqlalchemy/unitycatalog/sampler.py +6 -39
- metadata/sdk/__init__.py +188 -0
- metadata/sdk/api/__init__.py +7 -0
- metadata/sdk/api/lineage.py +421 -0
- metadata/sdk/api/search.py +382 -0
- metadata/sdk/client.py +91 -0
- metadata/sdk/config.py +121 -0
- metadata/sdk/entities/__init__.py +64 -0
- metadata/sdk/entities/apicollections.py +19 -0
- metadata/sdk/entities/apiendpoints.py +19 -0
- metadata/sdk/entities/base.py +487 -0
- metadata/sdk/entities/charts.py +17 -0
- metadata/sdk/entities/classifications.py +21 -0
- metadata/sdk/entities/containers.py +17 -0
- metadata/sdk/entities/custom_properties.py +144 -0
- metadata/sdk/entities/dashboard_services.py +18 -0
- metadata/sdk/entities/dashboarddatamodels.py +21 -0
- metadata/sdk/entities/dashboards.py +17 -0
- metadata/sdk/entities/database_services.py +18 -0
- metadata/sdk/entities/databases.py +17 -0
- metadata/sdk/entities/databaseschemas.py +19 -0
- metadata/sdk/entities/datacontracts.py +19 -0
- metadata/sdk/entities/dataproducts.py +19 -0
- metadata/sdk/entities/domains.py +17 -0
- metadata/sdk/entities/glossaries.py +17 -0
- metadata/sdk/entities/glossary_terms.py +18 -0
- metadata/sdk/entities/glossaryterms.py +19 -0
- metadata/sdk/entities/metrics.py +49 -0
- metadata/sdk/entities/mlmodels.py +17 -0
- metadata/sdk/entities/pipelines.py +17 -0
- metadata/sdk/entities/queries.py +17 -0
- metadata/sdk/entities/searchindexes.py +19 -0
- metadata/sdk/entities/storedprocedures.py +19 -0
- metadata/sdk/entities/tables.py +68 -0
- metadata/sdk/entities/tags.py +17 -0
- metadata/sdk/entities/teams.py +17 -0
- metadata/sdk/entities/testcases.py +17 -0
- metadata/sdk/entities/testdefinitions.py +19 -0
- metadata/sdk/entities/testsuites.py +17 -0
- metadata/sdk/entities/users.py +17 -0
- metadata/sdk/examples/builder_end_to_end.py +387 -0
- metadata/sdk/types.py +15 -0
- metadata/utils/bigquery_utils.py +7 -4
- metadata/utils/datalake/datalake_utils.py +121 -0
- metadata/utils/db_utils.py +20 -6
- metadata/utils/deprecation.py +9 -6
- metadata/utils/dispatch.py +6 -3
- metadata/utils/elasticsearch.py +3 -0
- metadata/utils/fqn.py +104 -7
- metadata/utils/helpers.py +18 -1
- metadata/utils/importer.py +3 -2
- metadata/utils/logger.py +12 -0
- metadata/utils/sqa_utils.py +3 -0
- metadata/utils/ssl_manager.py +93 -5
- metadata/utils/streamable_logger.py +569 -0
- metadata/workflow/base.py +49 -2
- metadata/workflow/classification.py +2 -2
- metadata/workflow/workflow_resource_metrics.py +111 -0
- metadata/workflow/workflow_status_mixin.py +1 -1
- {openmetadata_ingestion-1.9.0.0.dist-info → openmetadata_ingestion-1.10.14.4.dist-info}/METADATA +559 -540
- {openmetadata_ingestion-1.9.0.0.dist-info → openmetadata_ingestion-1.10.14.4.dist-info}/RECORD +1115 -950
- {openmetadata_ingestion-1.9.0.0.dist-info → openmetadata_ingestion-1.10.14.4.dist-info}/LICENSE +0 -0
- {openmetadata_ingestion-1.9.0.0.dist-info → openmetadata_ingestion-1.10.14.4.dist-info}/WHEEL +0 -0
- {openmetadata_ingestion-1.9.0.0.dist-info → openmetadata_ingestion-1.10.14.4.dist-info}/entry_points.txt +0 -0
- {openmetadata_ingestion-1.9.0.0.dist-info → openmetadata_ingestion-1.10.14.4.dist-info}/top_level.txt +0 -0
|
@@ -11,21 +11,44 @@
|
|
|
11
11
|
"""
|
|
12
12
|
MariaDB source module
|
|
13
13
|
"""
|
|
14
|
-
|
|
14
|
+
import traceback
|
|
15
|
+
from typing import Iterable, Optional
|
|
15
16
|
|
|
16
17
|
from sqlalchemy.dialects.mysql.base import ischema_names
|
|
17
18
|
from sqlalchemy.dialects.mysql.reflection import MySQLTableDefinitionParser
|
|
18
19
|
|
|
20
|
+
from metadata.generated.schema.api.data.createStoredProcedure import (
|
|
21
|
+
CreateStoredProcedureRequest,
|
|
22
|
+
)
|
|
23
|
+
from metadata.generated.schema.entity.data.databaseSchema import DatabaseSchema
|
|
24
|
+
from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureCode
|
|
19
25
|
from metadata.generated.schema.entity.services.connections.database.mariaDBConnection import (
|
|
20
26
|
MariaDBConnection,
|
|
21
27
|
)
|
|
28
|
+
from metadata.generated.schema.entity.services.ingestionPipelines.status import (
|
|
29
|
+
StackTraceError,
|
|
30
|
+
)
|
|
22
31
|
from metadata.generated.schema.metadataIngestion.workflow import (
|
|
23
32
|
Source as WorkflowSource,
|
|
24
33
|
)
|
|
34
|
+
from metadata.generated.schema.type.basic import EntityName, Markdown
|
|
35
|
+
from metadata.ingestion.api.models import Either
|
|
25
36
|
from metadata.ingestion.api.steps import InvalidSourceException
|
|
26
37
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
27
38
|
from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
|
|
39
|
+
from metadata.ingestion.source.database.mariadb.models import (
|
|
40
|
+
ROUTINE_TYPE_MAP,
|
|
41
|
+
MariaDBStoredProcedure,
|
|
42
|
+
)
|
|
43
|
+
from metadata.ingestion.source.database.mariadb.queries import (
|
|
44
|
+
MARIADB_GET_FUNCTIONS,
|
|
45
|
+
MARIADB_GET_STORED_PROCEDURES,
|
|
46
|
+
)
|
|
28
47
|
from metadata.ingestion.source.database.mysql.utils import col_type_map, parse_column
|
|
48
|
+
from metadata.utils import fqn
|
|
49
|
+
from metadata.utils.logger import ingestion_logger
|
|
50
|
+
|
|
51
|
+
logger = ingestion_logger()
|
|
29
52
|
|
|
30
53
|
ischema_names.update(col_type_map)
|
|
31
54
|
|
|
@@ -52,3 +75,65 @@ class MariadbSource(CommonDbSourceService):
|
|
|
52
75
|
f"Expected MariaDBConnection, but got {connection}"
|
|
53
76
|
)
|
|
54
77
|
return cls(config, metadata)
|
|
78
|
+
|
|
79
|
+
def _get_stored_procedures_internal(
|
|
80
|
+
self, query: str
|
|
81
|
+
) -> Iterable[MariaDBStoredProcedure]:
|
|
82
|
+
results = self.engine.execute(query).all()
|
|
83
|
+
for row in results:
|
|
84
|
+
try:
|
|
85
|
+
yield MariaDBStoredProcedure.model_validate(dict(row._mapping))
|
|
86
|
+
except Exception as exc:
|
|
87
|
+
error = f"Error parsing Stored Procedure payload: {exc}"
|
|
88
|
+
logger.error(error)
|
|
89
|
+
self.status.failed(
|
|
90
|
+
error=StackTraceError(
|
|
91
|
+
name=dict(row).get("procedure_name", "UNKNOWN"),
|
|
92
|
+
error=error,
|
|
93
|
+
stackTrace=traceback.format_exc(),
|
|
94
|
+
)
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
def get_stored_procedures(self) -> Iterable[MariaDBStoredProcedure]:
|
|
98
|
+
"""List stored procedures"""
|
|
99
|
+
if self.source_config.includeStoredProcedures:
|
|
100
|
+
for query in (MARIADB_GET_STORED_PROCEDURES, MARIADB_GET_FUNCTIONS):
|
|
101
|
+
yield from self._get_stored_procedures_internal(
|
|
102
|
+
query.format(schema_name=self.context.get().database_schema)
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
def yield_stored_procedure(
|
|
106
|
+
self, stored_procedure
|
|
107
|
+
) -> Iterable[Either[CreateStoredProcedureRequest]]:
|
|
108
|
+
"""Prepare the stored procedure payload"""
|
|
109
|
+
try:
|
|
110
|
+
stored_procedure_request = CreateStoredProcedureRequest(
|
|
111
|
+
name=EntityName(stored_procedure.name),
|
|
112
|
+
description=(
|
|
113
|
+
Markdown(stored_procedure.description)
|
|
114
|
+
if stored_procedure.description
|
|
115
|
+
else None
|
|
116
|
+
),
|
|
117
|
+
storedProcedureCode=StoredProcedureCode(
|
|
118
|
+
language=stored_procedure.language, code=stored_procedure.definition
|
|
119
|
+
),
|
|
120
|
+
databaseSchema=fqn.build(
|
|
121
|
+
metadata=self.metadata,
|
|
122
|
+
entity_type=DatabaseSchema,
|
|
123
|
+
service_name=self.context.get().database_service,
|
|
124
|
+
database_name=self.context.get().database,
|
|
125
|
+
schema_name=self.context.get().database_schema,
|
|
126
|
+
),
|
|
127
|
+
storedProcedureType=ROUTINE_TYPE_MAP[stored_procedure.procedure_type],
|
|
128
|
+
)
|
|
129
|
+
yield Either(right=stored_procedure_request)
|
|
130
|
+
self.register_record_stored_proc_request(stored_procedure_request)
|
|
131
|
+
|
|
132
|
+
except Exception as exc:
|
|
133
|
+
yield Either(
|
|
134
|
+
left=StackTraceError(
|
|
135
|
+
name=stored_procedure.name,
|
|
136
|
+
error=f"Error yielding Stored Procedure [{stored_procedure.name}] due to [{exc}]",
|
|
137
|
+
stackTrace=traceback.format_exc(),
|
|
138
|
+
)
|
|
139
|
+
)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Copyright 2025 Collate
|
|
2
|
+
# Licensed under the Collate Community License, Version 1.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
|
|
6
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
7
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
8
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
9
|
+
# See the License for the specific language governing permissions and
|
|
10
|
+
# limitations under the License.
|
|
11
|
+
"""
|
|
12
|
+
MariaDB models
|
|
13
|
+
"""
|
|
14
|
+
from typing import Optional
|
|
15
|
+
|
|
16
|
+
from pydantic import BaseModel, Field
|
|
17
|
+
|
|
18
|
+
from metadata.generated.schema.entity.data.storedProcedure import StoredProcedureType
|
|
19
|
+
|
|
20
|
+
ROUTINE_TYPE_MAP = {
|
|
21
|
+
"PROCEDURE": StoredProcedureType.StoredProcedure,
|
|
22
|
+
"FUNCTION": StoredProcedureType.Function,
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class MariaDBStoredProcedure(BaseModel):
|
|
27
|
+
"""
|
|
28
|
+
MariaDB stored procedure list query results
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
name: str = Field(alias="procedure_name")
|
|
32
|
+
schema_name: str
|
|
33
|
+
definition: str
|
|
34
|
+
language: Optional[str]
|
|
35
|
+
procedure_type: Optional[str]
|
|
36
|
+
description: Optional[str]
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# Copyright 2025 Collate
|
|
2
|
+
# Licensed under the Collate Community License, Version 1.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
|
|
6
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
7
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
8
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
9
|
+
# See the License for the specific language governing permissions and
|
|
10
|
+
# limitations under the License.
|
|
11
|
+
"""
|
|
12
|
+
SQL Queries used during MariaDB ingestion
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
MARIADB_GET_STORED_PROCEDURES = """
|
|
16
|
+
SELECT
|
|
17
|
+
ROUTINE_NAME AS procedure_name,
|
|
18
|
+
ROUTINE_SCHEMA AS schema_name,
|
|
19
|
+
ROUTINE_DEFINITION AS definition,
|
|
20
|
+
ROUTINE_TYPE AS procedure_type,
|
|
21
|
+
ROUTINE_COMMENT AS description,
|
|
22
|
+
EXTERNAL_LANGUAGE AS language
|
|
23
|
+
FROM
|
|
24
|
+
information_schema.ROUTINES
|
|
25
|
+
WHERE
|
|
26
|
+
ROUTINE_SCHEMA = '{schema_name}'
|
|
27
|
+
AND ROUTINE_TYPE = 'PROCEDURE'
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
MARIADB_GET_FUNCTIONS = """
|
|
31
|
+
SELECT
|
|
32
|
+
ROUTINE_NAME AS procedure_name,
|
|
33
|
+
ROUTINE_SCHEMA AS schema_name,
|
|
34
|
+
ROUTINE_DEFINITION AS definition,
|
|
35
|
+
ROUTINE_TYPE AS procedure_type,
|
|
36
|
+
ROUTINE_COMMENT AS description,
|
|
37
|
+
EXTERNAL_LANGUAGE AS language
|
|
38
|
+
FROM
|
|
39
|
+
information_schema.ROUTINES
|
|
40
|
+
WHERE
|
|
41
|
+
ROUTINE_SCHEMA = '{schema_name}'
|
|
42
|
+
AND ROUTINE_TYPE = 'FUNCTION'
|
|
43
|
+
"""
|
|
@@ -11,13 +11,8 @@
|
|
|
11
11
|
"""
|
|
12
12
|
MSSQL lineage module
|
|
13
13
|
"""
|
|
14
|
-
import traceback
|
|
15
14
|
from datetime import datetime
|
|
16
|
-
from typing import Dict, List
|
|
17
15
|
|
|
18
|
-
from metadata.generated.schema.entity.services.ingestionPipelines.status import (
|
|
19
|
-
StackTraceError,
|
|
20
|
-
)
|
|
21
16
|
from metadata.ingestion.source.database.lineage_source import LineageSource
|
|
22
17
|
from metadata.ingestion.source.database.mssql.constants import (
|
|
23
18
|
DEFAULT_DATETIME_FORMAT,
|
|
@@ -32,13 +27,9 @@ from metadata.ingestion.source.database.mssql.utils import (
|
|
|
32
27
|
get_sqlalchemy_engine_dateformat,
|
|
33
28
|
)
|
|
34
29
|
from metadata.ingestion.source.database.stored_procedures_mixin import (
|
|
35
|
-
QueryByProcedure,
|
|
36
30
|
StoredProcedureLineageMixin,
|
|
37
31
|
)
|
|
38
32
|
from metadata.utils.helpers import get_start_and_end
|
|
39
|
-
from metadata.utils.logger import ingestion_logger
|
|
40
|
-
|
|
41
|
-
logger = ingestion_logger()
|
|
42
33
|
|
|
43
34
|
|
|
44
35
|
class MssqlLineageSource(
|
|
@@ -74,10 +65,9 @@ class MssqlLineageSource(
|
|
|
74
65
|
result_limit=self.source_config.resultLimit,
|
|
75
66
|
)
|
|
76
67
|
|
|
77
|
-
def
|
|
68
|
+
def get_stored_procedure_sql_statement(self) -> str:
|
|
78
69
|
"""
|
|
79
|
-
Return the
|
|
80
|
-
queries they triggered
|
|
70
|
+
Return the SQL statement to get the stored procedure queries
|
|
81
71
|
"""
|
|
82
72
|
start, _ = get_start_and_end(self.source_config.queryLogDuration)
|
|
83
73
|
server_date_format = get_sqlalchemy_engine_dateformat(self.engine)
|
|
@@ -88,19 +78,5 @@ class MssqlLineageSource(
|
|
|
88
78
|
query = MSSQL_GET_STORED_PROCEDURE_QUERIES.format(
|
|
89
79
|
start_date=start,
|
|
90
80
|
)
|
|
91
|
-
try:
|
|
92
|
-
queries_dict = self.procedure_queries_dict(
|
|
93
|
-
query=query,
|
|
94
|
-
)
|
|
95
|
-
except Exception as ex: # pylint: disable=broad-except
|
|
96
|
-
logger.debug(f"Error runnning query:\n{query}")
|
|
97
|
-
self.status.failed(
|
|
98
|
-
StackTraceError(
|
|
99
|
-
name="Stored Procedure",
|
|
100
|
-
error=f"Error trying to get stored procedure queries: {ex}",
|
|
101
|
-
stackTrace=traceback.format_exc(),
|
|
102
|
-
)
|
|
103
|
-
)
|
|
104
|
-
return {}
|
|
105
81
|
|
|
106
|
-
return
|
|
82
|
+
return query
|
|
@@ -14,6 +14,7 @@ Source connection handler
|
|
|
14
14
|
"""
|
|
15
15
|
import os
|
|
16
16
|
import sys
|
|
17
|
+
from copy import deepcopy
|
|
17
18
|
from typing import Optional
|
|
18
19
|
from urllib.parse import quote_plus
|
|
19
20
|
|
|
@@ -26,7 +27,9 @@ from metadata.generated.schema.entity.automations.workflow import (
|
|
|
26
27
|
Workflow as AutomationWorkflow,
|
|
27
28
|
)
|
|
28
29
|
from metadata.generated.schema.entity.services.connections.database.oracleConnection import (
|
|
29
|
-
OracleConnection,
|
|
30
|
+
OracleConnection as OracleConnectionConfig,
|
|
31
|
+
)
|
|
32
|
+
from metadata.generated.schema.entity.services.connections.database.oracleConnection import (
|
|
30
33
|
OracleDatabaseSchema,
|
|
31
34
|
OracleServiceName,
|
|
32
35
|
OracleTNSConnection,
|
|
@@ -39,12 +42,15 @@ from metadata.ingestion.connections.builders import (
|
|
|
39
42
|
get_connection_args_common,
|
|
40
43
|
get_connection_options_dict,
|
|
41
44
|
)
|
|
45
|
+
from metadata.ingestion.connections.connection import BaseConnection
|
|
46
|
+
from metadata.ingestion.connections.secrets import connection_with_options_secrets
|
|
42
47
|
from metadata.ingestion.connections.test_connections import test_connection_db_common
|
|
43
48
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
44
49
|
from metadata.ingestion.source.database.oracle.queries import (
|
|
45
50
|
CHECK_ACCESS_TO_ALL,
|
|
46
|
-
|
|
47
|
-
|
|
51
|
+
TEST_MATERIALIZED_VIEWS,
|
|
52
|
+
TEST_ORACLE_GET_STORED_PACKAGES,
|
|
53
|
+
TEST_QUERY_HISTORY,
|
|
48
54
|
)
|
|
49
55
|
from metadata.utils.constants import THREE_MIN
|
|
50
56
|
from metadata.utils.logger import ingestion_logger
|
|
@@ -55,123 +61,161 @@ LD_LIB_ENV = "LD_LIBRARY_PATH"
|
|
|
55
61
|
logger = ingestion_logger()
|
|
56
62
|
|
|
57
63
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
"""
|
|
62
|
-
|
|
63
|
-
oracledb.version = CX_ORACLE_LIB_VERSION
|
|
64
|
-
sys.modules["cx_Oracle"] = oracledb
|
|
64
|
+
class OracleConnection(BaseConnection[OracleConnectionConfig, Engine]):
|
|
65
|
+
def __init__(self, connection: OracleConnectionConfig):
|
|
66
|
+
super().__init__(connection)
|
|
65
67
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
68
|
+
def _get_client(self) -> Engine:
|
|
69
|
+
"""
|
|
70
|
+
Create connection
|
|
71
|
+
"""
|
|
72
|
+
try:
|
|
73
|
+
if self.service_connection.instantClientDirectory:
|
|
74
|
+
logger.info(
|
|
75
|
+
f"Initializing Oracle thick client at {self.service_connection.instantClientDirectory}"
|
|
76
|
+
)
|
|
77
|
+
os.environ[LD_LIB_ENV] = self.service_connection.instantClientDirectory
|
|
78
|
+
oracledb.init_oracle_client(
|
|
79
|
+
lib_dir=self.service_connection.instantClientDirectory
|
|
80
|
+
)
|
|
81
|
+
except DatabaseError as err:
|
|
82
|
+
logger.info(f"Could not initialize Oracle thick client: {err}")
|
|
83
|
+
|
|
84
|
+
return create_generic_db_connection(
|
|
85
|
+
connection=self.service_connection,
|
|
86
|
+
get_connection_url_fn=self.get_connection_url,
|
|
87
|
+
get_connection_args_fn=get_connection_args_common,
|
|
80
88
|
)
|
|
81
|
-
if isinstance(connection.oracleConnectionType, OracleServiceName):
|
|
82
|
-
url = f"{url}&{params}"
|
|
83
|
-
else:
|
|
84
|
-
url = f"{url}?{params}"
|
|
85
|
-
|
|
86
|
-
return url
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def _handle_connection_type(url: str, connection: OracleConnection) -> str:
|
|
90
|
-
"""
|
|
91
|
-
Depending on the oracle connection type, we need to handle the URL differently
|
|
92
|
-
"""
|
|
93
|
-
|
|
94
|
-
if isinstance(connection.oracleConnectionType, OracleTNSConnection):
|
|
95
|
-
# ref https://stackoverflow.com/questions/14140902/using-oracle-service-names-with-sqlalchemy
|
|
96
|
-
url += connection.oracleConnectionType.oracleTNSConnection
|
|
97
|
-
return url
|
|
98
|
-
|
|
99
|
-
# If not TNS, we add the hostPort
|
|
100
|
-
url += connection.hostPort
|
|
101
89
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
90
|
+
def test_connection(
|
|
91
|
+
self,
|
|
92
|
+
metadata: OpenMetadata,
|
|
93
|
+
automation_workflow: Optional[AutomationWorkflow] = None,
|
|
94
|
+
timeout_seconds: Optional[int] = THREE_MIN,
|
|
95
|
+
) -> TestConnectionResult:
|
|
96
|
+
"""
|
|
97
|
+
Test connection. This can be executed either as part
|
|
98
|
+
of a metadata workflow or during an Automation Workflow
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
test_conn_queries = {
|
|
102
|
+
"CheckAccess": CHECK_ACCESS_TO_ALL,
|
|
103
|
+
"PackageAccess": TEST_ORACLE_GET_STORED_PACKAGES,
|
|
104
|
+
"GetMaterializedViews": TEST_MATERIALIZED_VIEWS,
|
|
105
|
+
"GetQueryHistory": TEST_QUERY_HISTORY,
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return test_connection_db_common(
|
|
109
|
+
metadata=metadata,
|
|
110
|
+
engine=self.client,
|
|
111
|
+
service_connection=self.service_connection,
|
|
112
|
+
automation_workflow=automation_workflow,
|
|
113
|
+
queries=test_conn_queries,
|
|
114
|
+
timeout_seconds=timeout_seconds,
|
|
107
115
|
)
|
|
108
|
-
return url
|
|
109
116
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
117
|
+
def get_connection_dict(self) -> dict:
|
|
118
|
+
"""
|
|
119
|
+
Return the connection dictionary for this service.
|
|
120
|
+
"""
|
|
121
|
+
url = self.client.url
|
|
122
|
+
connection_copy = deepcopy(self.service_connection)
|
|
123
|
+
|
|
124
|
+
connection_dict = {
|
|
125
|
+
"driver": url.drivername,
|
|
126
|
+
"host": f"{url.host}:{url.port}", # This is the format expected by data-diff. If we start using this for something else, we need to change it and modify the data-diff code.
|
|
127
|
+
"user": url.username,
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
# Add password if present in the connection
|
|
131
|
+
if connection_copy.password:
|
|
132
|
+
connection_dict["password"] = connection_copy.password.get_secret_value()
|
|
133
|
+
|
|
134
|
+
# Add connection type specific information
|
|
135
|
+
if isinstance(connection_copy.oracleConnectionType, OracleDatabaseSchema):
|
|
136
|
+
connection_dict[
|
|
137
|
+
"database"
|
|
138
|
+
] = connection_copy.oracleConnectionType.databaseSchema
|
|
139
|
+
elif isinstance(connection_copy.oracleConnectionType, OracleServiceName):
|
|
140
|
+
connection_dict[
|
|
141
|
+
"database"
|
|
142
|
+
] = connection_copy.oracleConnectionType.oracleServiceName
|
|
143
|
+
elif isinstance(connection_copy.oracleConnectionType, OracleTNSConnection):
|
|
144
|
+
connection_dict[
|
|
145
|
+
"host"
|
|
146
|
+
] = connection_copy.oracleConnectionType.oracleTNSConnection
|
|
147
|
+
|
|
148
|
+
# Add connection options if present
|
|
149
|
+
if connection_copy.connectionOptions and connection_copy.connectionOptions.root:
|
|
150
|
+
connection_with_options_secrets(lambda: connection_copy)
|
|
151
|
+
connection_dict.update(connection_copy.connectionOptions.root)
|
|
152
|
+
|
|
153
|
+
# Add connection arguments if present
|
|
154
|
+
if (
|
|
155
|
+
connection_copy.connectionArguments
|
|
156
|
+
and connection_copy.connectionArguments.root
|
|
157
|
+
):
|
|
158
|
+
connection_dict.update(get_connection_args_common(connection_copy))
|
|
159
|
+
|
|
160
|
+
return connection_dict
|
|
161
|
+
|
|
162
|
+
@staticmethod
|
|
163
|
+
def get_connection_url(connection: OracleConnectionConfig) -> str:
|
|
164
|
+
"""
|
|
165
|
+
Build the URL and handle driver version at system level
|
|
166
|
+
"""
|
|
167
|
+
|
|
168
|
+
oracledb.version = CX_ORACLE_LIB_VERSION
|
|
169
|
+
sys.modules["cx_Oracle"] = oracledb
|
|
170
|
+
|
|
171
|
+
url = f"{connection.scheme.value}://"
|
|
172
|
+
if connection.username:
|
|
173
|
+
url += f"{quote_plus(connection.username)}"
|
|
174
|
+
if not connection.password:
|
|
175
|
+
connection.password = SecretStr("")
|
|
176
|
+
url += f":{quote_plus(connection.password.get_secret_value())}"
|
|
177
|
+
url += "@"
|
|
178
|
+
|
|
179
|
+
url = OracleConnection._handle_connection_type(url=url, connection=connection)
|
|
180
|
+
|
|
181
|
+
options = get_connection_options_dict(connection)
|
|
182
|
+
if options:
|
|
183
|
+
params = "&".join(
|
|
184
|
+
f"{key}={quote_plus(value)}"
|
|
185
|
+
for (key, value) in options.items()
|
|
186
|
+
if value
|
|
187
|
+
)
|
|
188
|
+
if isinstance(connection.oracleConnectionType, OracleServiceName):
|
|
189
|
+
url = f"{url}&{params}"
|
|
190
|
+
else:
|
|
191
|
+
url = f"{url}?{params}"
|
|
115
192
|
|
|
193
|
+
return url
|
|
116
194
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
class OraclePackageAccessError(Exception):
|
|
139
|
-
"""
|
|
140
|
-
Raised when unable to access Oracle stored packages
|
|
141
|
-
"""
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
def test_connection(
|
|
145
|
-
metadata: OpenMetadata,
|
|
146
|
-
engine: Engine,
|
|
147
|
-
service_connection: OracleConnection,
|
|
148
|
-
automation_workflow: Optional[AutomationWorkflow] = None,
|
|
149
|
-
timeout_seconds: Optional[int] = THREE_MIN,
|
|
150
|
-
) -> TestConnectionResult:
|
|
151
|
-
"""
|
|
152
|
-
Test connection. This can be executed either as part
|
|
153
|
-
of a metadata workflow or during an Automation Workflow
|
|
154
|
-
"""
|
|
155
|
-
|
|
156
|
-
def test_oracle_package_access(engine):
|
|
157
|
-
try:
|
|
158
|
-
schema_name = engine.execute(ORACLE_GET_SCHEMA).scalar()
|
|
159
|
-
return ORACLE_GET_STORED_PACKAGES.format(schema=schema_name)
|
|
160
|
-
except Exception as e:
|
|
161
|
-
raise OraclePackageAccessError(
|
|
162
|
-
f"Failed to access Oracle stored packages: {e}"
|
|
195
|
+
@staticmethod
|
|
196
|
+
def _handle_connection_type(url: str, connection: OracleConnectionConfig) -> str:
|
|
197
|
+
"""
|
|
198
|
+
Depending on the oracle connection type, we need to handle the URL differently
|
|
199
|
+
"""
|
|
200
|
+
|
|
201
|
+
if isinstance(connection.oracleConnectionType, OracleTNSConnection):
|
|
202
|
+
# ref https://stackoverflow.com/questions/14140902/using-oracle-service-names-with-sqlalchemy
|
|
203
|
+
url += connection.oracleConnectionType.oracleTNSConnection
|
|
204
|
+
return url
|
|
205
|
+
|
|
206
|
+
# If not TNS, we add the hostPort
|
|
207
|
+
url += connection.hostPort
|
|
208
|
+
|
|
209
|
+
if isinstance(connection.oracleConnectionType, OracleDatabaseSchema):
|
|
210
|
+
url += (
|
|
211
|
+
f"/{connection.oracleConnectionType.databaseSchema}"
|
|
212
|
+
if connection.oracleConnectionType.databaseSchema
|
|
213
|
+
else ""
|
|
163
214
|
)
|
|
215
|
+
return url
|
|
216
|
+
|
|
217
|
+
if isinstance(connection.oracleConnectionType, OracleServiceName):
|
|
218
|
+
url = f"{url}/?service_name={connection.oracleConnectionType.oracleServiceName}"
|
|
219
|
+
return url
|
|
164
220
|
|
|
165
|
-
|
|
166
|
-
"CheckAccess": CHECK_ACCESS_TO_ALL,
|
|
167
|
-
"PackageAccess": test_oracle_package_access(engine),
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
return test_connection_db_common(
|
|
171
|
-
metadata=metadata,
|
|
172
|
-
engine=engine,
|
|
173
|
-
service_connection=service_connection,
|
|
174
|
-
automation_workflow=automation_workflow,
|
|
175
|
-
queries=test_conn_queries,
|
|
176
|
-
timeout_seconds=timeout_seconds,
|
|
177
|
-
)
|
|
221
|
+
raise ValueError(f"Unknown connection type {connection.oracleConnectionType}")
|
|
@@ -30,8 +30,6 @@ workflowConfig:
|
|
|
30
30
|
jwtToken: "token"
|
|
31
31
|
"""
|
|
32
32
|
|
|
33
|
-
from typing import Dict, List
|
|
34
|
-
|
|
35
33
|
from metadata.ingestion.source.database.lineage_source import LineageSource
|
|
36
34
|
from metadata.ingestion.source.database.oracle.queries import (
|
|
37
35
|
ORACLE_GET_STORED_PROCEDURE_QUERIES,
|
|
@@ -41,7 +39,6 @@ from metadata.ingestion.source.database.oracle.query_parser import (
|
|
|
41
39
|
OracleQueryParserSource,
|
|
42
40
|
)
|
|
43
41
|
from metadata.ingestion.source.database.stored_procedures_mixin import (
|
|
44
|
-
QueryByProcedure,
|
|
45
42
|
StoredProcedureLineageMixin,
|
|
46
43
|
)
|
|
47
44
|
from metadata.utils.helpers import get_start_and_end
|
|
@@ -63,17 +60,13 @@ class OracleLineageSource(
|
|
|
63
60
|
|
|
64
61
|
stored_procedure_query = ORACLE_GET_STORED_PROCEDURE_QUERIES
|
|
65
62
|
|
|
66
|
-
def
|
|
63
|
+
def get_stored_procedure_sql_statement(self) -> str:
|
|
67
64
|
"""
|
|
68
|
-
Return the
|
|
69
|
-
queries they triggered
|
|
65
|
+
Return the SQL statement to get the stored procedure queries
|
|
70
66
|
"""
|
|
71
67
|
start, _ = get_start_and_end(self.source_config.queryLogDuration)
|
|
72
68
|
query = self.stored_procedure_query.format(
|
|
73
69
|
start_date=start,
|
|
74
70
|
)
|
|
75
|
-
queries_dict = self.procedure_queries_dict(
|
|
76
|
-
query=query,
|
|
77
|
-
)
|
|
78
71
|
|
|
79
|
-
return
|
|
72
|
+
return query
|
|
@@ -95,12 +95,7 @@ WHERE
|
|
|
95
95
|
type = 'PROCEDURE' and owner = '{schema}'
|
|
96
96
|
"""
|
|
97
97
|
)
|
|
98
|
-
|
|
99
|
-
SELECT USERNAME AS SCHEMA_NAME
|
|
100
|
-
FROM ALL_USERS
|
|
101
|
-
WHERE ROWNUM = 1
|
|
102
|
-
ORDER BY USERNAME
|
|
103
|
-
"""
|
|
98
|
+
|
|
104
99
|
ORACLE_GET_STORED_PACKAGES = textwrap.dedent(
|
|
105
100
|
"""
|
|
106
101
|
SELECT
|
|
@@ -115,7 +110,49 @@ FROM
|
|
|
115
110
|
WHERE TYPE IN ('PACKAGE', 'PACKAGE BODY') AND owner = '{schema}'
|
|
116
111
|
"""
|
|
117
112
|
)
|
|
113
|
+
|
|
114
|
+
TEST_ORACLE_GET_STORED_PACKAGES = textwrap.dedent(
|
|
115
|
+
"""
|
|
116
|
+
SELECT
|
|
117
|
+
OWNER,
|
|
118
|
+
NAME,
|
|
119
|
+
LINE,
|
|
120
|
+
TEXT,
|
|
121
|
+
'StoredPackage' as procedure_type
|
|
122
|
+
FROM
|
|
123
|
+
DBA_SOURCE
|
|
124
|
+
WHERE
|
|
125
|
+
TYPE IN ('PACKAGE', 'PACKAGE BODY')
|
|
126
|
+
AND owner = (
|
|
127
|
+
SELECT
|
|
128
|
+
USERNAME
|
|
129
|
+
FROM
|
|
130
|
+
ALL_USERS
|
|
131
|
+
WHERE
|
|
132
|
+
ROWNUM = 1
|
|
133
|
+
)
|
|
134
|
+
"""
|
|
135
|
+
)
|
|
136
|
+
|
|
118
137
|
CHECK_ACCESS_TO_ALL = "SELECT table_name FROM DBA_TABLES where ROWNUM < 2"
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
TEST_MATERIALIZED_VIEWS = textwrap.dedent(
|
|
141
|
+
"""
|
|
142
|
+
SELECT COUNT(*) as count
|
|
143
|
+
FROM DBA_MVIEWS
|
|
144
|
+
WHERE ROWNUM = 1
|
|
145
|
+
"""
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
TEST_QUERY_HISTORY = textwrap.dedent(
|
|
149
|
+
"""
|
|
150
|
+
SELECT COUNT(*) as count
|
|
151
|
+
FROM gv$sql
|
|
152
|
+
WHERE ROWNUM = 1
|
|
153
|
+
"""
|
|
154
|
+
)
|
|
155
|
+
|
|
119
156
|
ORACLE_GET_STORED_PROCEDURE_QUERIES = textwrap.dedent(
|
|
120
157
|
"""
|
|
121
158
|
WITH SP_HISTORY AS (SELECT
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
from metadata.ingestion.source.database.oracle.connection import OracleConnection
|
|
1
2
|
from metadata.ingestion.source.database.oracle.lineage import OracleLineageSource
|
|
2
3
|
from metadata.ingestion.source.database.oracle.metadata import OracleSource
|
|
3
4
|
from metadata.ingestion.source.database.oracle.usage import OracleUsageSource
|
|
@@ -7,4 +8,5 @@ ServiceSpec = DefaultDatabaseSpec(
|
|
|
7
8
|
metadata_source_class=OracleSource,
|
|
8
9
|
lineage_source_class=OracleLineageSource,
|
|
9
10
|
usage_source_class=OracleUsageSource,
|
|
11
|
+
connection_class=OracleConnection,
|
|
10
12
|
)
|