openmetadata-ingestion 1.5.13.0__py3-none-any.whl → 1.6.0.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openmetadata-ingestion might be problematic. Click here for more details.
- _openmetadata_testutils/dict.py +23 -0
- _openmetadata_testutils/pydantic/test_utils.py +22 -2
- _openmetadata_testutils/pytest_openmetadata/plugin.py +86 -0
- _openmetadata_testutils/pytest_openmetadata/test_utils.py +99 -0
- metadata/cli/app.py +1 -0
- metadata/cli/classify.py +52 -0
- metadata/cli/dataquality.py +4 -1
- metadata/cli/ingest.py +2 -1
- metadata/cli/lineage.py +2 -1
- metadata/cli/profile.py +4 -1
- metadata/cli/usage.py +2 -1
- metadata/cmd.py +9 -0
- metadata/config/common.py +3 -1
- metadata/data_quality/api/models.py +2 -1
- metadata/data_quality/builders/{i_validator_builder.py → validator_builder.py} +45 -31
- metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +25 -24
- metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +23 -49
- metadata/data_quality/interface/test_suite_interface.py +36 -52
- metadata/data_quality/processor/test_case_runner.py +9 -9
- metadata/data_quality/runner/base_test_suite_source.py +52 -26
- metadata/data_quality/source/test_suite.py +1 -1
- metadata/data_quality/validations/base_test_handler.py +13 -8
- metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +4 -2
- metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeAtExpectedLocation.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +2 -8
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +1 -1
- metadata/data_quality/validations/models.py +6 -1
- metadata/data_quality/validations/runtime_param_setter/param_setter.py +2 -3
- metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +45 -17
- metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +26 -16
- metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +192 -17
- metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py +2 -2
- metadata/examples/workflows/bigquery_classifier.yaml +56 -0
- metadata/examples/workflows/bigquery_profiler.yaml +1 -2
- metadata/examples/workflows/datafactory.yaml +29 -0
- metadata/examples/workflows/db2_profiler.yaml +1 -2
- metadata/examples/workflows/dbtcloud.yaml +2 -1
- metadata/examples/workflows/matillion.yaml +31 -0
- metadata/examples/workflows/{mstr.yaml → microstrategy.yaml} +3 -2
- metadata/examples/workflows/mysql_profiler.yaml +0 -1
- metadata/examples/workflows/redshift_classifier.yaml +38 -0
- metadata/examples/workflows/redshift_profiler.yaml +2 -3
- metadata/examples/workflows/rest.yaml +20 -0
- metadata/examples/workflows/sigma.yaml +25 -0
- metadata/examples/workflows/trino_profiler.yaml +20 -0
- metadata/generated/antlr/EntityLinkLexer.py +70 -99
- metadata/generated/schema/analytics/__init__.py +1 -1
- metadata/generated/schema/analytics/basic.py +1 -1
- metadata/generated/schema/analytics/reportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
- metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
- metadata/generated/schema/api/__init__.py +1 -1
- metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
- metadata/generated/schema/api/addTagToAssetsRequest.py +43 -0
- metadata/generated/schema/api/analytics/__init__.py +1 -1
- metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
- metadata/generated/schema/api/automations/__init__.py +1 -1
- metadata/generated/schema/api/automations/createWorkflow.py +1 -1
- metadata/generated/schema/api/bulkAssets.py +1 -1
- metadata/generated/schema/api/classification/__init__.py +1 -1
- metadata/generated/schema/api/classification/createClassification.py +1 -1
- metadata/generated/schema/api/classification/createTag.py +1 -1
- metadata/generated/schema/api/classification/loadTags.py +1 -1
- metadata/generated/schema/api/createBot.py +1 -1
- metadata/generated/schema/api/createEventPublisherJob.py +1 -1
- metadata/generated/schema/api/createType.py +1 -1
- metadata/generated/schema/api/data/__init__.py +1 -1
- metadata/generated/schema/api/data/createAPICollection.py +1 -1
- metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
- metadata/generated/schema/api/data/createChart.py +1 -1
- metadata/generated/schema/api/data/createContainer.py +1 -1
- metadata/generated/schema/api/data/createCustomProperty.py +1 -1
- metadata/generated/schema/api/data/createDashboard.py +1 -1
- metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
- metadata/generated/schema/api/data/createDatabase.py +1 -1
- metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
- metadata/generated/schema/api/data/createGlossary.py +1 -1
- metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
- metadata/generated/schema/api/data/createMetric.py +83 -0
- metadata/generated/schema/api/data/createMlModel.py +1 -1
- metadata/generated/schema/api/data/createPipeline.py +1 -1
- metadata/generated/schema/api/data/createQuery.py +1 -1
- metadata/generated/schema/api/data/createSearchIndex.py +8 -1
- metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
- metadata/generated/schema/api/data/createTable.py +1 -1
- metadata/generated/schema/api/data/createTableProfile.py +1 -1
- metadata/generated/schema/api/data/createTopic.py +1 -1
- metadata/generated/schema/api/data/loadGlossary.py +1 -1
- metadata/generated/schema/api/data/restoreEntity.py +1 -1
- metadata/generated/schema/api/dataInsight/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
- metadata/generated/schema/api/docStore/__init__.py +1 -1
- metadata/generated/schema/api/docStore/createDocument.py +1 -1
- metadata/generated/schema/api/domains/__init__.py +1 -1
- metadata/generated/schema/api/domains/createDataProduct.py +1 -1
- metadata/generated/schema/api/domains/createDomain.py +1 -1
- metadata/generated/schema/api/feed/__init__.py +1 -1
- metadata/generated/schema/api/feed/closeTask.py +1 -1
- metadata/generated/schema/api/feed/createPost.py +1 -1
- metadata/generated/schema/api/feed/createSuggestion.py +1 -1
- metadata/generated/schema/api/feed/createThread.py +1 -1
- metadata/generated/schema/api/feed/resolveTask.py +1 -1
- metadata/generated/schema/api/feed/threadCount.py +1 -1
- metadata/generated/schema/api/governance/__init__.py +3 -0
- metadata/generated/schema/api/governance/createWorkflowDefinition.py +83 -0
- metadata/generated/schema/api/governance/createWorkflowInstanceState.py +22 -0
- metadata/generated/schema/api/lineage/__init__.py +1 -1
- metadata/generated/schema/api/lineage/addLineage.py +1 -1
- metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
- metadata/generated/schema/api/policies/__init__.py +1 -1
- metadata/generated/schema/api/policies/createPolicy.py +1 -1
- metadata/generated/schema/api/services/__init__.py +1 -1
- metadata/generated/schema/api/services/createApiService.py +2 -2
- metadata/generated/schema/api/services/createDashboardService.py +1 -1
- metadata/generated/schema/api/services/createDatabaseService.py +1 -1
- metadata/generated/schema/api/services/createMessagingService.py +1 -1
- metadata/generated/schema/api/services/createMetadataService.py +1 -1
- metadata/generated/schema/api/services/createMlModelService.py +1 -1
- metadata/generated/schema/api/services/createPipelineService.py +1 -1
- metadata/generated/schema/api/services/createSearchService.py +1 -1
- metadata/generated/schema/api/services/createStorageService.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
- metadata/generated/schema/api/setOwner.py +1 -1
- metadata/generated/schema/api/teams/__init__.py +1 -1
- metadata/generated/schema/api/teams/createPersona.py +1 -1
- metadata/generated/schema/api/teams/createRole.py +1 -1
- metadata/generated/schema/api/teams/createTeam.py +1 -1
- metadata/generated/schema/api/teams/createUser.py +1 -1
- metadata/generated/schema/api/tests/__init__.py +1 -1
- metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
- metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
- metadata/generated/schema/api/tests/createTestCase.py +1 -1
- metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
- metadata/generated/schema/api/tests/createTestCaseResult.py +71 -0
- metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
- metadata/generated/schema/api/tests/createTestSuite.py +1 -1
- metadata/generated/schema/api/voteRequest.py +1 -1
- metadata/generated/schema/auth/__init__.py +1 -1
- metadata/generated/schema/auth/basicAuth.py +1 -1
- metadata/generated/schema/auth/basicLoginRequest.py +1 -1
- metadata/generated/schema/auth/changePasswordRequest.py +1 -1
- metadata/generated/schema/auth/createPersonalToken.py +1 -1
- metadata/generated/schema/auth/emailRequest.py +1 -1
- metadata/generated/schema/auth/emailVerificationToken.py +1 -1
- metadata/generated/schema/auth/generateToken.py +1 -1
- metadata/generated/schema/auth/jwtAuth.py +1 -1
- metadata/generated/schema/auth/loginRequest.py +1 -1
- metadata/generated/schema/auth/logoutRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetToken.py +1 -1
- metadata/generated/schema/auth/personalAccessToken.py +1 -1
- metadata/generated/schema/auth/refreshToken.py +1 -1
- metadata/generated/schema/auth/registrationRequest.py +1 -1
- metadata/generated/schema/auth/revokePersonalToken.py +1 -1
- metadata/generated/schema/auth/revokeToken.py +1 -1
- metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
- metadata/generated/schema/auth/ssoAuth.py +1 -1
- metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
- metadata/generated/schema/configuration/__init__.py +1 -1
- metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
- metadata/generated/schema/configuration/assetCertificationSettings.py +22 -0
- metadata/generated/schema/configuration/authConfig.py +1 -1
- metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
- metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
- metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
- metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
- metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
- metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
- metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
- metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
- metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
- metadata/generated/schema/configuration/lineageSettings.py +1 -1
- metadata/generated/schema/configuration/loginConfiguration.py +1 -1
- metadata/generated/schema/configuration/logoConfiguration.py +1 -1
- metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
- metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
- metadata/generated/schema/configuration/searchSettings.py +1 -1
- metadata/generated/schema/configuration/slackAppConfiguration.py +1 -14
- metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/themeConfiguration.py +1 -1
- metadata/generated/schema/configuration/uiThemePreference.py +1 -1
- metadata/generated/schema/dataInsight/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +9 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
- metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
- metadata/generated/schema/dataInsight/custom/lineChart.py +23 -8
- metadata/generated/schema/dataInsight/custom/summaryCard.py +20 -9
- metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
- metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
- metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
- metadata/generated/schema/dataInsight/type/__init__.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
- metadata/generated/schema/email/__init__.py +1 -1
- metadata/generated/schema/email/emailRequest.py +1 -1
- metadata/generated/schema/email/emailTemplate.py +1 -1
- metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
- metadata/generated/schema/email/smtpSettings.py +1 -1
- metadata/generated/schema/email/templateValidationReponse.py +1 -1
- metadata/generated/schema/entity/__init__.py +1 -1
- metadata/generated/schema/entity/applications/__init__.py +1 -1
- metadata/generated/schema/entity/applications/app.py +9 -1
- metadata/generated/schema/entity/applications/appExtension.py +1 -1
- metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
- metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/applicationConfig.py +4 -2
- metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +54 -2
- metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +22 -0
- metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
- metadata/generated/schema/entity/applications/createAppRequest.py +8 -1
- metadata/generated/schema/entity/applications/jobStatus.py +1 -1
- metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +8 -1
- metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +8 -1
- metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
- metadata/generated/schema/entity/automations/__init__.py +1 -1
- metadata/generated/schema/entity/automations/testServiceConnection.py +3 -1
- metadata/generated/schema/entity/automations/workflow.py +1 -1
- metadata/generated/schema/entity/bot.py +1 -1
- metadata/generated/schema/entity/classification/__init__.py +1 -1
- metadata/generated/schema/entity/classification/classification.py +2 -2
- metadata/generated/schema/entity/classification/tag.py +2 -2
- metadata/generated/schema/entity/data/__init__.py +1 -1
- metadata/generated/schema/entity/data/apiCollection.py +4 -2
- metadata/generated/schema/entity/data/apiEndpoint.py +4 -2
- metadata/generated/schema/entity/data/chart.py +4 -3
- metadata/generated/schema/entity/data/container.py +4 -2
- metadata/generated/schema/entity/data/dashboard.py +4 -2
- metadata/generated/schema/entity/data/dashboardDataModel.py +5 -2
- metadata/generated/schema/entity/data/database.py +7 -7
- metadata/generated/schema/entity/data/databaseSchema.py +5 -2
- metadata/generated/schema/entity/data/glossary.py +2 -2
- metadata/generated/schema/entity/data/glossaryTerm.py +3 -2
- metadata/generated/schema/entity/data/metric.py +196 -0
- metadata/generated/schema/entity/data/mlmodel.py +4 -2
- metadata/generated/schema/entity/data/pipeline.py +4 -3
- metadata/generated/schema/entity/data/query.py +3 -5
- metadata/generated/schema/entity/data/report.py +1 -1
- metadata/generated/schema/entity/data/searchIndex.py +16 -2
- metadata/generated/schema/entity/data/storedProcedure.py +4 -2
- metadata/generated/schema/entity/data/table.py +36 -3
- metadata/generated/schema/entity/data/topic.py +4 -3
- metadata/generated/schema/entity/docStore/__init__.py +1 -1
- metadata/generated/schema/entity/docStore/document.py +1 -1
- metadata/generated/schema/entity/domains/__init__.py +1 -1
- metadata/generated/schema/entity/domains/dataProduct.py +1 -1
- metadata/generated/schema/entity/domains/domain.py +1 -1
- metadata/generated/schema/entity/events/__init__.py +1 -1
- metadata/generated/schema/entity/events/webhook.py +19 -2
- metadata/generated/schema/entity/feed/__init__.py +1 -1
- metadata/generated/schema/entity/feed/assets.py +1 -1
- metadata/generated/schema/entity/feed/customProperty.py +1 -1
- metadata/generated/schema/entity/feed/description.py +1 -1
- metadata/generated/schema/entity/feed/domain.py +1 -1
- metadata/generated/schema/entity/feed/entityInfo.py +1 -1
- metadata/generated/schema/entity/feed/owner.py +1 -1
- metadata/generated/schema/entity/feed/suggestion.py +1 -1
- metadata/generated/schema/entity/feed/tag.py +1 -1
- metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
- metadata/generated/schema/entity/feed/thread.py +1 -1
- metadata/generated/schema/entity/policies/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +4 -1
- metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
- metadata/generated/schema/entity/policies/filters.py +1 -1
- metadata/generated/schema/entity/policies/policy.py +1 -1
- metadata/generated/schema/entity/services/__init__.py +1 -1
- metadata/generated/schema/entity/services/apiService.py +7 -7
- metadata/generated/schema/entity/services/connections/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/{apiService → api}/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/{apiService → api}/restConnection.py +15 -7
- metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/connectionBasicType.py +29 -1
- metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +67 -0
- metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +41 -0
- metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/athenaConnection.py +5 -1
- metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +9 -1
- metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +11 -0
- metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/db2Connection.py +4 -1
- metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/druidConnection.py +4 -1
- metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/exasolConnection.py +84 -0
- metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +4 -1
- metadata/generated/schema/entity/services/connections/database/hiveConnection.py +4 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/impalaConnection.py +4 -1
- metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +4 -1
- metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +3 -2
- metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -1
- metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +11 -7
- metadata/generated/schema/entity/services/connections/database/oracleConnection.py +5 -1
- metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +4 -1
- metadata/generated/schema/entity/services/connections/database/postgresConnection.py +5 -1
- metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +9 -1
- metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +5 -1
- metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +4 -1
- metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +9 -1
- metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +4 -1
- metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/teradataConnection.py +4 -1
- metadata/generated/schema/entity/services/connections/database/trinoConnection.py +7 -2
- metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +69 -0
- metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +17 -5
- metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +65 -0
- metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +43 -0
- metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/serviceConnection.py +3 -3
- metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
- metadata/generated/schema/entity/services/dashboardService.py +8 -5
- metadata/generated/schema/entity/services/databaseService.py +5 -2
- metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +2 -1
- metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
- metadata/generated/schema/entity/services/messagingService.py +2 -2
- metadata/generated/schema/entity/services/metadataService.py +2 -2
- metadata/generated/schema/entity/services/mlmodelService.py +2 -2
- metadata/generated/schema/entity/services/pipelineService.py +11 -2
- metadata/generated/schema/entity/services/searchService.py +2 -2
- metadata/generated/schema/entity/services/serviceType.py +2 -2
- metadata/generated/schema/entity/services/storageService.py +2 -2
- metadata/generated/schema/entity/teams/__init__.py +1 -1
- metadata/generated/schema/entity/teams/persona.py +1 -1
- metadata/generated/schema/entity/teams/role.py +1 -1
- metadata/generated/schema/entity/teams/team.py +1 -1
- metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
- metadata/generated/schema/entity/teams/user.py +1 -1
- metadata/generated/schema/entity/type.py +1 -1
- metadata/generated/schema/entity/utils/__init__.py +1 -1
- metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
- metadata/generated/schema/entity/utils/servicesCount.py +1 -1
- metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
- metadata/generated/schema/events/__init__.py +1 -1
- metadata/generated/schema/events/alertMetrics.py +1 -1
- metadata/generated/schema/events/api/__init__.py +1 -1
- metadata/generated/schema/events/api/createEventSubscription.py +1 -1
- metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +69 -0
- metadata/generated/schema/events/api/eventsRecord.py +37 -0
- metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -4
- metadata/generated/schema/events/api/typedEvent.py +46 -0
- metadata/generated/schema/events/emailAlertConfig.py +1 -1
- metadata/generated/schema/events/eventFilterRule.py +1 -1
- metadata/generated/schema/events/eventSubscription.py +15 -46
- metadata/generated/schema/events/eventSubscriptionOffset.py +7 -2
- metadata/generated/schema/events/failedEvent.py +1 -1
- metadata/generated/schema/events/failedEventResponse.py +38 -0
- metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
- metadata/generated/schema/events/statusContext.py +43 -0
- metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
- metadata/generated/schema/events/subscriptionStatus.py +78 -0
- metadata/generated/schema/events/testDestinationStatus.py +88 -0
- metadata/generated/schema/governance/workflows/__init__.py +3 -0
- metadata/generated/schema/governance/workflows/elements/__init__.py +5 -0
- metadata/generated/schema/governance/workflows/elements/edge.py +29 -0
- metadata/generated/schema/governance/workflows/elements/nodeSubType.py +16 -0
- metadata/generated/schema/governance/workflows/elements/nodeType.py +14 -0
- metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +3 -0
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +5 -0
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +44 -0
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +49 -0
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +42 -0
- metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +3 -0
- metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +30 -0
- metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +3 -0
- metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +30 -0
- metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +3 -0
- metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +57 -0
- metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +3 -0
- metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +43 -0
- metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +44 -0
- metadata/generated/schema/{entity/data/metrics.py → governance/workflows/workflowDefinition.py} +36 -49
- metadata/generated/schema/governance/workflows/workflowInstance.py +50 -0
- metadata/generated/schema/governance/workflows/workflowInstanceState.py +73 -0
- metadata/generated/schema/metadataIngestion/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +53 -0
- metadata/generated/schema/metadataIngestion/application.py +1 -1
- metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +127 -0
- metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -9
- metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +28 -17
- metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +42 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +9 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +9 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +9 -1
- metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/testSuitePipeline.py +4 -1
- metadata/generated/schema/metadataIngestion/workflow.py +15 -1
- metadata/generated/schema/monitoring/__init__.py +1 -1
- metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
- metadata/generated/schema/security/__init__.py +1 -1
- metadata/generated/schema/security/client/__init__.py +1 -1
- metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
- metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
- metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
- metadata/generated/schema/security/credentials/__init__.py +1 -1
- metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
- metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
- metadata/generated/schema/security/credentials/basicAuth.py +1 -1
- metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
- metadata/generated/schema/security/credentials/gcpValues.py +1 -1
- metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
- metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
- metadata/generated/schema/security/sasl/__init__.py +1 -1
- metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
- metadata/generated/schema/security/secrets/__init__.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
- metadata/generated/schema/security/securityConfiguration.py +1 -1
- metadata/generated/schema/security/ssl/__init__.py +1 -1
- metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
- metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
- metadata/generated/schema/settings/__init__.py +1 -1
- metadata/generated/schema/settings/settings.py +7 -1
- metadata/generated/schema/system/__init__.py +1 -1
- metadata/generated/schema/system/entityError.py +1 -1
- metadata/generated/schema/system/eventPublisherJob.py +2 -1
- metadata/generated/schema/system/indexingError.py +1 -1
- metadata/generated/schema/system/limitsResponse.py +1 -1
- metadata/generated/schema/system/ui/__init__.py +1 -1
- metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
- metadata/generated/schema/system/ui/page.py +1 -1
- metadata/generated/schema/system/validationResponse.py +1 -1
- metadata/generated/schema/tests/__init__.py +1 -1
- metadata/generated/schema/tests/assigned.py +1 -1
- metadata/generated/schema/tests/basic.py +19 -4
- metadata/generated/schema/tests/customMetric.py +1 -1
- metadata/generated/schema/tests/dataQualityReport.py +1 -1
- metadata/generated/schema/tests/resolved.py +1 -1
- metadata/generated/schema/tests/testCase.py +6 -2
- metadata/generated/schema/tests/testCaseResolutionStatus.py +17 -2
- metadata/generated/schema/tests/testDefinition.py +1 -1
- metadata/generated/schema/tests/testSuite.py +9 -2
- metadata/generated/schema/type/__init__.py +1 -1
- metadata/generated/schema/type/apiSchema.py +1 -1
- metadata/generated/schema/type/assetCertification.py +26 -0
- metadata/generated/schema/type/auditLog.py +1 -1
- metadata/generated/schema/type/basic.py +1 -1
- metadata/generated/schema/type/bulkOperationResult.py +1 -1
- metadata/generated/schema/type/changeEvent.py +1 -1
- metadata/generated/schema/type/changeEventType.py +1 -1
- metadata/generated/schema/type/collectionDescriptor.py +1 -1
- metadata/generated/schema/type/csvDocumentation.py +1 -1
- metadata/generated/schema/type/csvErrorType.py +1 -1
- metadata/generated/schema/type/csvFile.py +1 -1
- metadata/generated/schema/type/csvImportResult.py +1 -1
- metadata/generated/schema/type/customProperties/__init__.py +1 -1
- metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
- metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
- metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
- metadata/generated/schema/type/customProperty.py +8 -1
- metadata/generated/schema/type/dailyCount.py +1 -1
- metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
- metadata/generated/schema/type/entityHierarchy.py +1 -1
- metadata/generated/schema/type/entityHistory.py +1 -1
- metadata/generated/schema/type/entityLineage.py +1 -1
- metadata/generated/schema/type/entityReference.py +1 -1
- metadata/generated/schema/type/entityReferenceList.py +1 -1
- metadata/generated/schema/type/entityRelationship.py +1 -1
- metadata/generated/schema/type/entityUsage.py +1 -1
- metadata/generated/schema/type/filterPattern.py +1 -1
- metadata/generated/schema/type/function.py +1 -1
- metadata/generated/schema/type/include.py +1 -1
- metadata/generated/schema/type/jdbcConnection.py +1 -1
- metadata/generated/schema/type/lifeCycle.py +1 -1
- metadata/generated/schema/type/paging.py +1 -1
- metadata/generated/schema/type/profile.py +1 -1
- metadata/generated/schema/type/queryParserData.py +1 -1
- metadata/generated/schema/type/reaction.py +1 -1
- metadata/generated/schema/type/schedule.py +1 -1
- metadata/generated/schema/type/schema.py +1 -1
- metadata/generated/schema/type/tableQuery.py +1 -1
- metadata/generated/schema/type/tableUsageCount.py +1 -1
- metadata/generated/schema/type/tagLabel.py +1 -1
- metadata/generated/schema/type/usageDetails.py +1 -1
- metadata/generated/schema/type/usageRequest.py +1 -1
- metadata/generated/schema/type/votes.py +1 -1
- metadata/ingestion/api/models.py +9 -5
- metadata/ingestion/api/parser.py +14 -0
- metadata/ingestion/lineage/parser.py +4 -6
- metadata/ingestion/lineage/sql_lineage.py +29 -185
- metadata/ingestion/models/custom_basemodel_validation.py +80 -0
- metadata/ingestion/models/custom_pydantic.py +33 -1
- metadata/ingestion/models/custom_types.py +2 -0
- metadata/ingestion/models/entity_interface.py +71 -0
- metadata/ingestion/models/patch_request.py +0 -1
- metadata/ingestion/ometa/mixins/es_mixin.py +66 -8
- metadata/ingestion/ometa/models.py +1 -0
- metadata/ingestion/ometa/ometa_api.py +6 -3
- metadata/ingestion/ometa/routes.py +4 -2
- metadata/ingestion/processor/query_parser.py +2 -2
- metadata/ingestion/sink/metadata_rest.py +23 -18
- metadata/ingestion/source/api/api_service.py +212 -0
- metadata/ingestion/source/api/rest/connection.py +93 -0
- metadata/ingestion/source/api/rest/metadata.py +278 -0
- metadata/ingestion/source/api/rest/models.py +41 -0
- metadata/ingestion/source/api/rest/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/domodashboard/service_spec.py +6 -0
- metadata/ingestion/source/dashboard/lightdash/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/looker/metadata.py +33 -4
- metadata/ingestion/source/dashboard/looker/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/metabase/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/microstrategy/__init__.py +0 -0
- metadata/ingestion/source/dashboard/{mstr → microstrategy}/client.py +27 -32
- metadata/ingestion/source/dashboard/{mstr → microstrategy}/connection.py +7 -10
- metadata/ingestion/source/dashboard/{mstr → microstrategy}/metadata.py +18 -14
- metadata/ingestion/source/dashboard/{mstr → microstrategy}/models.py +1 -1
- metadata/ingestion/source/dashboard/microstrategy/service_spec.py +6 -0
- metadata/ingestion/source/dashboard/mode/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/powerbi/client.py +1 -2
- metadata/ingestion/source/dashboard/powerbi/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/qlikcloud/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/qliksense/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/quicksight/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/redash/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/sigma/__init__.py +0 -0
- metadata/ingestion/source/dashboard/sigma/client.py +186 -0
- metadata/ingestion/source/dashboard/sigma/connection.py +67 -0
- metadata/ingestion/source/dashboard/sigma/metadata.py +331 -0
- metadata/ingestion/source/dashboard/sigma/models.py +97 -0
- metadata/ingestion/source/dashboard/sigma/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/superset/service_spec.py +4 -0
- metadata/ingestion/source/dashboard/tableau/service_spec.py +4 -0
- metadata/ingestion/source/database/athena/service_spec.py +10 -0
- metadata/ingestion/source/database/azuresql/service_spec.py +12 -0
- metadata/ingestion/source/database/bigquery/connection.py +28 -2
- metadata/ingestion/source/database/bigquery/lineage.py +29 -2
- metadata/ingestion/source/database/bigquery/metadata.py +1 -25
- metadata/ingestion/source/database/bigquery/profiler/__init__.py +0 -0
- metadata/ingestion/source/database/bigquery/profiler/profiler.py +30 -0
- metadata/ingestion/source/database/bigquery/profiler/system.py +165 -0
- metadata/ingestion/source/database/bigquery/queries.py +69 -1
- metadata/ingestion/source/database/bigquery/service_spec.py +16 -0
- metadata/ingestion/source/database/bigtable/service_spec.py +4 -0
- metadata/ingestion/source/database/clickhouse/service_spec.py +12 -0
- metadata/ingestion/source/database/common_db_source.py +150 -147
- metadata/ingestion/source/database/common_nosql_source.py +0 -14
- metadata/ingestion/source/database/couchbase/metadata.py +62 -2
- metadata/{data_quality/builders/sqa_validator_builder.py → ingestion/source/database/couchbase/models.py} +15 -9
- metadata/ingestion/source/database/couchbase/queries.py +6 -1
- metadata/ingestion/source/database/couchbase/service_spec.py +4 -0
- metadata/ingestion/source/database/database_service.py +15 -22
- metadata/ingestion/source/database/databricks/metadata.py +12 -1
- metadata/ingestion/source/database/databricks/service_spec.py +20 -0
- metadata/ingestion/source/database/datalake/clients/azure_blob.py +0 -3
- metadata/ingestion/source/database/datalake/clients/base.py +10 -1
- metadata/ingestion/source/database/datalake/clients/gcs.py +10 -6
- metadata/ingestion/source/database/datalake/clients/s3.py +0 -3
- metadata/ingestion/source/database/datalake/metadata.py +2 -13
- metadata/ingestion/source/database/datalake/service_spec.py +16 -0
- metadata/ingestion/source/database/db2/lineage.py +46 -0
- metadata/ingestion/source/database/db2/service_spec.py +9 -0
- metadata/ingestion/source/database/dbt/constants.py +4 -0
- metadata/ingestion/source/database/dbt/dbt_config.py +19 -1
- metadata/ingestion/source/database/dbt/dbt_service.py +8 -1
- metadata/ingestion/source/database/dbt/dbt_utils.py +29 -0
- metadata/ingestion/source/database/dbt/metadata.py +73 -7
- metadata/ingestion/source/database/dbt/models.py +2 -0
- metadata/ingestion/source/database/dbt/service_spec.py +4 -0
- metadata/ingestion/source/database/deltalake/metadata.py +1 -12
- metadata/ingestion/source/database/deltalake/service_spec.py +4 -0
- metadata/ingestion/source/database/domodatabase/metadata.py +1 -12
- metadata/ingestion/source/database/domodatabase/service_spec.py +4 -0
- metadata/ingestion/source/database/doris/metadata.py +1 -1
- metadata/ingestion/source/database/doris/service_spec.py +4 -0
- metadata/ingestion/source/database/druid/lineage.py +46 -0
- metadata/ingestion/source/database/druid/service_spec.py +4 -0
- metadata/ingestion/source/database/dynamodb/service_spec.py +10 -0
- metadata/ingestion/source/database/exasol/__init__.py +0 -0
- metadata/ingestion/source/database/exasol/connection.py +87 -0
- metadata/ingestion/source/database/exasol/metadata.py +27 -0
- metadata/ingestion/source/database/exasol/service_spec.py +4 -0
- metadata/ingestion/source/database/glue/metadata.py +49 -19
- metadata/ingestion/source/database/glue/models.py +8 -0
- metadata/ingestion/source/database/glue/service_spec.py +4 -0
- metadata/ingestion/source/database/greenplum/lineage.py +46 -0
- metadata/ingestion/source/database/greenplum/service_spec.py +4 -0
- metadata/ingestion/source/database/hive/lineage.py +46 -0
- metadata/ingestion/source/database/hive/service_spec.py +4 -0
- metadata/ingestion/source/database/iceberg/metadata.py +1 -16
- metadata/ingestion/source/database/iceberg/service_spec.py +4 -0
- metadata/ingestion/source/database/impala/lineage.py +46 -0
- metadata/ingestion/source/database/impala/service_spec.py +4 -0
- metadata/ingestion/source/database/life_cycle_query_mixin.py +3 -5
- metadata/ingestion/source/database/lineage_source.py +124 -32
- metadata/ingestion/source/database/mariadb/lineage.py +46 -0
- metadata/ingestion/source/database/mariadb/service_spec.py +4 -0
- metadata/ingestion/source/database/mongodb/service_spec.py +10 -0
- metadata/ingestion/source/database/mssql/lineage.py +51 -2
- metadata/ingestion/source/database/mssql/metadata.py +2 -44
- metadata/ingestion/source/database/mssql/service_spec.py +10 -0
- metadata/ingestion/source/database/mysql/lineage.py +0 -2
- metadata/ingestion/source/database/mysql/queries.py +0 -2
- metadata/ingestion/source/database/mysql/service_spec.py +10 -0
- metadata/ingestion/source/database/mysql/usage.py +0 -2
- metadata/ingestion/source/database/oracle/lineage.py +28 -1
- metadata/ingestion/source/database/oracle/metadata.py +2 -23
- metadata/ingestion/source/database/oracle/service_spec.py +10 -0
- metadata/ingestion/source/database/pinotdb/lineage.py +46 -0
- metadata/ingestion/source/database/pinotdb/service_spec.py +4 -0
- metadata/ingestion/source/database/postgres/metadata.py +4 -13
- metadata/ingestion/source/database/postgres/service_spec.py +10 -0
- metadata/ingestion/source/database/postgres/utils.py +26 -0
- metadata/ingestion/source/database/presto/service_spec.py +4 -0
- metadata/ingestion/source/database/query/service_spec.py +9 -0
- metadata/ingestion/source/database/query_parser_source.py +1 -0
- metadata/ingestion/source/database/redshift/connection.py +9 -2
- metadata/ingestion/source/database/redshift/lineage.py +27 -3
- metadata/ingestion/source/database/redshift/metadata.py +2 -27
- metadata/ingestion/source/database/redshift/profiler/__init__.py +0 -0
- metadata/ingestion/source/database/redshift/profiler/profiler.py +14 -0
- metadata/ingestion/source/database/redshift/profiler/system.py +148 -0
- metadata/ingestion/source/database/redshift/queries.py +99 -8
- metadata/ingestion/source/database/redshift/service_spec.py +14 -0
- metadata/ingestion/source/database/redshift/utils.py +3 -3
- metadata/ingestion/source/database/salesforce/metadata.py +1 -12
- metadata/ingestion/source/database/salesforce/service_spec.py +4 -0
- metadata/ingestion/source/database/sample_data.py +129 -34
- metadata/ingestion/source/database/saperp/service_spec.py +4 -0
- metadata/ingestion/source/database/saphana/service_spec.py +7 -0
- metadata/ingestion/source/database/sas/metadata.py +1 -10
- metadata/ingestion/source/database/sas/service_spec.py +4 -0
- metadata/ingestion/source/database/singlestore/lineage.py +46 -0
- metadata/ingestion/source/database/singlestore/service_spec.py +9 -0
- metadata/ingestion/source/database/snowflake/connection.py +6 -1
- metadata/ingestion/source/database/snowflake/lineage.py +31 -2
- metadata/ingestion/source/database/snowflake/metadata.py +20 -62
- metadata/ingestion/source/database/snowflake/models.py +50 -2
- metadata/ingestion/source/database/snowflake/profiler/__init__.py +0 -0
- metadata/ingestion/source/database/snowflake/profiler/profiler.py +26 -0
- metadata/ingestion/source/database/snowflake/profiler/system.py +436 -0
- metadata/ingestion/source/database/snowflake/queries.py +28 -22
- metadata/ingestion/source/database/snowflake/service_spec.py +20 -0
- metadata/ingestion/source/database/sql_column_handler.py +26 -0
- metadata/ingestion/source/database/sqlite/lineage.py +46 -0
- metadata/ingestion/source/database/sqlite/service_spec.py +4 -0
- metadata/ingestion/source/database/stored_procedures_mixin.py +78 -52
- metadata/ingestion/source/database/teradata/lineage.py +46 -0
- metadata/ingestion/source/database/teradata/service_spec.py +4 -0
- metadata/ingestion/source/database/trino/connection.py +60 -23
- metadata/ingestion/source/database/trino/profiler/__init__.py +0 -0
- metadata/ingestion/source/database/trino/profiler/system_tables_profiler.py +167 -0
- metadata/ingestion/source/database/trino/service_spec.py +16 -0
- metadata/ingestion/source/database/unitycatalog/metadata.py +3 -24
- metadata/ingestion/source/database/unitycatalog/service_spec.py +22 -0
- metadata/ingestion/source/database/usage_source.py +4 -1
- metadata/ingestion/source/database/vertica/service_spec.py +10 -0
- metadata/ingestion/source/messaging/kafka/service_spec.py +4 -0
- metadata/ingestion/source/messaging/kinesis/service_spec.py +4 -0
- metadata/ingestion/source/messaging/redpanda/service_spec.py +4 -0
- metadata/ingestion/source/metadata/alationsink/service_spec.py +4 -0
- metadata/ingestion/source/metadata/amundsen/service_spec.py +4 -0
- metadata/ingestion/source/metadata/atlas/service_spec.py +4 -0
- metadata/ingestion/source/mlmodel/mlflow/service_spec.py +4 -0
- metadata/ingestion/source/mlmodel/sagemaker/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/airbyte/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/airflow/connection.py +45 -2
- metadata/ingestion/source/pipeline/airflow/metadata.py +71 -27
- metadata/ingestion/source/pipeline/airflow/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/dagster/metadata.py +3 -1
- metadata/ingestion/source/pipeline/dagster/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/databrickspipeline/service_spec.py +6 -0
- metadata/ingestion/source/pipeline/dbtcloud/client.py +67 -28
- metadata/ingestion/source/pipeline/dbtcloud/connection.py +1 -3
- metadata/ingestion/source/pipeline/dbtcloud/models.py +1 -1
- metadata/ingestion/source/pipeline/dbtcloud/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/domopipeline/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/fivetran/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/flink/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/gluepipeline/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/kafkaconnect/client.py +1 -1
- metadata/ingestion/source/pipeline/kafkaconnect/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/nifi/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/openlineage/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/pipeline_service.py +3 -1
- metadata/ingestion/source/pipeline/spline/service_spec.py +4 -0
- metadata/ingestion/source/search/elasticsearch/metadata.py +53 -0
- metadata/ingestion/source/search/elasticsearch/service_spec.py +4 -0
- metadata/ingestion/source/search/search_service.py +44 -1
- metadata/ingestion/source/storage/gcs/metadata.py +2 -2
- metadata/ingestion/source/storage/gcs/service_spec.py +4 -0
- metadata/ingestion/source/storage/s3/service_spec.py +4 -0
- metadata/mixins/pandas/pandas_mixin.py +2 -31
- metadata/mixins/sqalchemy/sqa_mixin.py +18 -17
- metadata/parsers/avro_parser.py +51 -16
- metadata/parsers/json_schema_parser.py +45 -16
- metadata/pii/processor.py +10 -9
- metadata/pii/scanners/ner_scanner.py +2 -1
- metadata/profiler/api/models.py +4 -75
- metadata/profiler/config.py +39 -0
- metadata/profiler/interface/nosql/profiler_interface.py +1 -26
- metadata/profiler/interface/pandas/profiler_interface.py +37 -77
- metadata/profiler/interface/profiler_interface.py +25 -279
- metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +0 -19
- metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +2 -17
- metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +1 -1
- metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +7 -5
- metadata/profiler/interface/sqlalchemy/profiler_interface.py +73 -125
- metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +5 -5
- metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +1 -1
- metadata/profiler/interface/sqlalchemy/stored_statistics_profiler.py +145 -0
- metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +9 -4
- metadata/profiler/metrics/system/system.py +69 -374
- metadata/profiler/orm/converter/base.py +23 -13
- metadata/profiler/orm/converter/bigquery/converter.py +1 -1
- metadata/profiler/orm/functions/count.py +9 -0
- metadata/profiler/orm/functions/md5.py +39 -0
- metadata/{data_quality/builders/pandas_validator_builder.py → profiler/orm/functions/substr.py} +13 -10
- metadata/profiler/orm/functions/table_metric_computer.py +6 -5
- metadata/profiler/orm/functions/unique_count.py +6 -0
- metadata/profiler/orm/registry.py +27 -2
- metadata/profiler/processor/core.py +13 -62
- metadata/profiler/processor/handle_partition.py +0 -48
- metadata/profiler/processor/processor.py +1 -1
- metadata/profiler/processor/runner.py +111 -35
- metadata/profiler/processor/sample_data_handler.py +7 -4
- metadata/profiler/source/database/base/profiler_source.py +215 -0
- metadata/profiler/source/{bigquery → database/bigquery}/profiler_source.py +4 -4
- metadata/profiler/source/{databricks → database/databricks}/profiler_source.py +6 -7
- metadata/profiler/source/{mariadb → database/mariadb}/metrics/window/first_quartile.py +1 -1
- metadata/profiler/source/{mariadb → database/mariadb}/metrics/window/median.py +1 -1
- metadata/profiler/source/{mariadb → database/mariadb}/metrics/window/third_quartile.py +1 -1
- metadata/profiler/source/{single_store → database/single_store}/metrics/window/first_quartile.py +3 -1
- metadata/profiler/source/{single_store → database/single_store}/metrics/window/median.py +3 -1
- metadata/profiler/source/{single_store → database/single_store}/metrics/window/third_quartile.py +3 -1
- metadata/profiler/source/fetcher/config.py +44 -0
- metadata/profiler/source/fetcher/entity_fetcher.py +62 -0
- metadata/profiler/source/fetcher/fetcher_strategy.py +299 -0
- metadata/profiler/source/fetcher/profiler_source_factory.py +89 -0
- metadata/profiler/source/metadata.py +16 -201
- metadata/profiler/source/metadata_ext.py +23 -18
- metadata/profiler/source/model.py +29 -0
- metadata/sampler/config.py +237 -0
- metadata/sampler/models.py +106 -0
- metadata/{profiler/processor/sampler → sampler}/nosql/sampler.py +44 -12
- metadata/sampler/pandas/sampler.py +239 -0
- metadata/{utils → sampler}/partition.py +66 -51
- metadata/sampler/processor.py +158 -0
- metadata/sampler/sampler_interface.py +251 -0
- metadata/{profiler/processor/sampler → sampler}/sqlalchemy/azuresql/sampler.py +1 -1
- metadata/{profiler/processor/sampler → sampler}/sqlalchemy/bigquery/sampler.py +55 -26
- metadata/sampler/sqlalchemy/postgres/sampler.py +91 -0
- metadata/{profiler/processor/sampler → sampler}/sqlalchemy/sampler.py +90 -98
- metadata/sampler/sqlalchemy/snowflake/sampler.py +95 -0
- metadata/{profiler/processor/sampler → sampler}/sqlalchemy/trino/sampler.py +5 -3
- metadata/utils/class_helper.py +5 -0
- metadata/utils/collaborative_super.py +28 -0
- metadata/utils/constants.py +65 -0
- metadata/utils/constraints.py +65 -0
- metadata/utils/dict.py +8 -0
- metadata/utils/dispatch.py +1 -53
- metadata/utils/entity_link.py +8 -2
- metadata/utils/entity_utils.py +93 -0
- metadata/utils/filters.py +15 -0
- metadata/utils/fqn.py +15 -0
- metadata/utils/helpers.py +3 -2
- metadata/utils/importer.py +12 -24
- metadata/utils/logger.py +31 -1
- metadata/utils/lru_cache.py +76 -15
- metadata/utils/profiler_utils.py +43 -9
- metadata/utils/service_spec/__init__.py +5 -0
- metadata/utils/service_spec/default.py +21 -0
- metadata/utils/service_spec/service_spec.py +146 -0
- metadata/utils/ssl_manager.py +33 -0
- metadata/utils/time_utils.py +33 -5
- metadata/workflow/application.py +7 -36
- metadata/workflow/base.py +47 -17
- metadata/workflow/classification.py +53 -0
- metadata/workflow/ingestion.py +6 -38
- metadata/workflow/profiler.py +1 -17
- metadata/workflow/workflow_output_handler.py +11 -8
- metadata/workflow/workflow_status_mixin.py +0 -2
- openmetadata_ingestion-1.6.0.0rc2.dist-info/LICENSE +114 -0
- {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/METADATA +473 -548
- {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/RECORD +995 -809
- {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/entry_points.txt +3 -0
- metadata/data_quality/interface/test_suite_interface_factory.py +0 -128
- metadata/data_quality/runner/test_suite_source_factory.py +0 -38
- metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +0 -54
- metadata/profiler/interface/profiler_interface_factory.py +0 -107
- metadata/profiler/metrics/system/queries/bigquery.py +0 -54
- metadata/profiler/metrics/system/queries/redshift.py +0 -103
- metadata/profiler/metrics/system/queries/snowflake.py +0 -191
- metadata/profiler/processor/sampler/pandas/sampler.py +0 -170
- metadata/profiler/processor/sampler/sampler_factory.py +0 -100
- metadata/profiler/processor/sampler/sampler_interface.py +0 -74
- metadata/profiler/processor/sampler/sqlalchemy/snowflake/sampler.py +0 -60
- metadata/profiler/source/base/profiler_source.py +0 -299
- metadata/profiler/source/profiler_source_factory.py +0 -54
- openmetadata_ingestion-1.5.13.0.dist-info/LICENSE +0 -201
- {metadata/ingestion/source/dashboard/mstr → _openmetadata_testutils/pytest_openmetadata}/__init__.py +0 -0
- /metadata/profiler/source/{bigquery → database/bigquery}/type_mapper.py +0 -0
- /metadata/profiler/source/{mariadb → database/mariadb}/functions/median.py +0 -0
- /metadata/profiler/source/{single_store → database/single_store}/functions/median.py +0 -0
- {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/WHEEL +0 -0
- {openmetadata_ingestion-1.5.13.0.dist-info → openmetadata_ingestion-1.6.0.0rc2.dist-info}/top_level.txt +0 -0
|
@@ -16,10 +16,10 @@ supporting sqlalchemy abstraction layer
|
|
|
16
16
|
from typing import List
|
|
17
17
|
|
|
18
18
|
from pyhive.sqlalchemy_hive import HiveCompiler
|
|
19
|
-
from sqlalchemy import Column
|
|
19
|
+
from sqlalchemy import Column
|
|
20
20
|
|
|
21
21
|
from metadata.generated.schema.entity.data.table import Column as OMColumn
|
|
22
|
-
from metadata.generated.schema.entity.data.table import ColumnName, DataType
|
|
22
|
+
from metadata.generated.schema.entity.data.table import ColumnName, DataType
|
|
23
23
|
from metadata.generated.schema.entity.services.databaseService import (
|
|
24
24
|
DatabaseServiceType,
|
|
25
25
|
)
|
|
@@ -97,18 +97,3 @@ class DatabricksProfilerInterface(SQAProfilerInterface):
|
|
|
97
97
|
)
|
|
98
98
|
columns.append(col)
|
|
99
99
|
return columns
|
|
100
|
-
|
|
101
|
-
def fetch_sample_data(self, table, columns) -> TableData:
|
|
102
|
-
"""Fetch sample data from database
|
|
103
|
-
|
|
104
|
-
Args:
|
|
105
|
-
table: ORM declarative table
|
|
106
|
-
|
|
107
|
-
Returns:
|
|
108
|
-
TableData: sample table data
|
|
109
|
-
"""
|
|
110
|
-
sampler = self._get_sampler(
|
|
111
|
-
table=table,
|
|
112
|
-
)
|
|
113
|
-
|
|
114
|
-
return sampler.fetch_sample_data(list(inspect(self.table).c))
|
|
@@ -32,7 +32,7 @@ class DB2ProfilerInterface(SQAProfilerInterface):
|
|
|
32
32
|
# pylint: disable=protected-access
|
|
33
33
|
if exc.orig and "overflow" in exc.orig._message:
|
|
34
34
|
logger.info(
|
|
35
|
-
f"Computing metrics without sum for {runner.
|
|
35
|
+
f"Computing metrics without sum for {runner.table_name}.{column.name}"
|
|
36
36
|
)
|
|
37
37
|
return self._compute_static_metrics_wo_sum(metrics, runner, session, column)
|
|
38
38
|
return None
|
|
@@ -24,11 +24,13 @@ from metadata.profiler.interface.sqlalchemy.profiler_interface import (
|
|
|
24
24
|
)
|
|
25
25
|
from metadata.profiler.metrics.registry import Metrics
|
|
26
26
|
from metadata.profiler.processor.runner import QueryRunner
|
|
27
|
-
from metadata.profiler.source.mariadb.metrics.window.first_quartile import (
|
|
27
|
+
from metadata.profiler.source.database.mariadb.metrics.window.first_quartile import (
|
|
28
28
|
MariaDBFirstQuartile,
|
|
29
29
|
)
|
|
30
|
-
from metadata.profiler.source.mariadb.metrics.window.median import
|
|
31
|
-
|
|
30
|
+
from metadata.profiler.source.database.mariadb.metrics.window.median import (
|
|
31
|
+
MariaDBMedian,
|
|
32
|
+
)
|
|
33
|
+
from metadata.profiler.source.database.mariadb.metrics.window.third_quartile import (
|
|
32
34
|
MariaDBThirdQuartile,
|
|
33
35
|
)
|
|
34
36
|
from metadata.utils.logger import profiler_interface_registry_logger
|
|
@@ -75,11 +77,11 @@ class MariaDBProfilerInterface(SQAProfilerInterface):
|
|
|
75
77
|
return dict(row)
|
|
76
78
|
except ProgrammingError:
|
|
77
79
|
logger.info(
|
|
78
|
-
f"Skipping window metrics for {runner.
|
|
80
|
+
f"Skipping window metrics for {runner.table_name}.{column.name} due to overflow"
|
|
79
81
|
)
|
|
80
82
|
return None
|
|
81
83
|
|
|
82
84
|
except Exception as exc:
|
|
83
|
-
msg = f"Error trying to compute profile for {runner.
|
|
85
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
84
86
|
handle_query_exception(msg, exc, session)
|
|
85
87
|
return None
|
|
@@ -21,28 +21,41 @@ import threading
|
|
|
21
21
|
import traceback
|
|
22
22
|
from collections import defaultdict
|
|
23
23
|
from datetime import datetime
|
|
24
|
-
from typing import Any, Dict, List, Optional
|
|
24
|
+
from typing import Any, Dict, List, Optional, Type, Union
|
|
25
25
|
|
|
26
26
|
from sqlalchemy import Column, inspect, text
|
|
27
27
|
from sqlalchemy.exc import DBAPIError, ProgrammingError, ResourceClosedError
|
|
28
28
|
from sqlalchemy.orm import scoped_session
|
|
29
29
|
|
|
30
|
-
from metadata.generated.schema.entity.data.table import
|
|
30
|
+
from metadata.generated.schema.entity.data.table import (
|
|
31
|
+
CustomMetricProfile,
|
|
32
|
+
SystemProfile,
|
|
33
|
+
Table,
|
|
34
|
+
)
|
|
35
|
+
from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
|
|
36
|
+
DatalakeConnection,
|
|
37
|
+
)
|
|
38
|
+
from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
|
|
39
|
+
from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline import (
|
|
40
|
+
DatabaseServiceProfilerPipeline,
|
|
41
|
+
)
|
|
31
42
|
from metadata.generated.schema.tests.customMetric import CustomMetric
|
|
32
43
|
from metadata.ingestion.connections.session import create_and_bind_thread_safe_session
|
|
44
|
+
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
33
45
|
from metadata.mixins.sqalchemy.sqa_mixin import SQAInterfaceMixin
|
|
34
46
|
from metadata.profiler.api.models import ThreadPoolMetrics
|
|
35
47
|
from metadata.profiler.interface.profiler_interface import ProfilerInterface
|
|
36
|
-
from metadata.profiler.metrics.core import MetricTypes
|
|
48
|
+
from metadata.profiler.metrics.core import HybridMetric, MetricTypes
|
|
37
49
|
from metadata.profiler.metrics.registry import Metrics
|
|
38
50
|
from metadata.profiler.metrics.static.mean import Mean
|
|
39
51
|
from metadata.profiler.metrics.static.stddev import StdDev
|
|
40
52
|
from metadata.profiler.metrics.static.sum import Sum
|
|
53
|
+
from metadata.profiler.metrics.system.system import System, SystemMetricsComputer
|
|
41
54
|
from metadata.profiler.orm.functions.table_metric_computer import TableMetricComputer
|
|
42
55
|
from metadata.profiler.orm.registry import Dialects
|
|
43
56
|
from metadata.profiler.processor.metric_filter import MetricFilter
|
|
44
57
|
from metadata.profiler.processor.runner import QueryRunner
|
|
45
|
-
from metadata.
|
|
58
|
+
from metadata.sampler.sampler_interface import SamplerInterface
|
|
46
59
|
from metadata.utils.custom_thread_pool import CustomThreadPoolExecutor
|
|
47
60
|
from metadata.utils.helpers import is_safe_sql_query
|
|
48
61
|
from metadata.utils.logger import profiler_interface_registry_logger
|
|
@@ -70,41 +83,40 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
70
83
|
"""
|
|
71
84
|
|
|
72
85
|
# pylint: disable=too-many-arguments
|
|
73
|
-
|
|
74
86
|
def __init__(
|
|
75
87
|
self,
|
|
76
|
-
service_connection_config,
|
|
77
|
-
ometa_client,
|
|
78
|
-
entity,
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
source_config,
|
|
82
|
-
sample_query,
|
|
83
|
-
table_partition_config,
|
|
88
|
+
service_connection_config: Union[DatabaseConnection, DatalakeConnection],
|
|
89
|
+
ometa_client: OpenMetadata,
|
|
90
|
+
entity: Table,
|
|
91
|
+
source_config: DatabaseServiceProfilerPipeline,
|
|
92
|
+
sampler: SamplerInterface,
|
|
84
93
|
thread_count: int = 5,
|
|
85
94
|
timeout_seconds: int = 43200,
|
|
86
|
-
sqa_metadata=None,
|
|
87
|
-
sample_data_count: Optional[int] = SAMPLE_DATA_DEFAULT_COUNT,
|
|
88
95
|
**kwargs,
|
|
89
96
|
):
|
|
90
97
|
"""Instantiate SQA Interface object"""
|
|
98
|
+
self.session_factory = None
|
|
99
|
+
self.session = None
|
|
91
100
|
|
|
92
101
|
super().__init__(
|
|
93
|
-
service_connection_config,
|
|
94
|
-
ometa_client,
|
|
95
|
-
entity,
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
table_partition_config,
|
|
101
|
-
thread_count,
|
|
102
|
-
timeout_seconds,
|
|
103
|
-
sample_data_count,
|
|
102
|
+
service_connection_config=service_connection_config,
|
|
103
|
+
ometa_client=ometa_client,
|
|
104
|
+
entity=entity,
|
|
105
|
+
source_config=source_config,
|
|
106
|
+
sampler=sampler,
|
|
107
|
+
thread_count=thread_count,
|
|
108
|
+
timeout_seconds=timeout_seconds,
|
|
104
109
|
)
|
|
105
110
|
|
|
106
|
-
self._table = self.
|
|
111
|
+
self._table = self.sampler.raw_dataset
|
|
107
112
|
self.create_session()
|
|
113
|
+
self.system_metrics_computer = self.initialize_system_metrics_computer()
|
|
114
|
+
|
|
115
|
+
def initialize_system_metrics_computer(self) -> SystemMetricsComputer:
|
|
116
|
+
"""Initialize system metrics computer. Override this if you want to use a metric source with
|
|
117
|
+
state or other dependencies.
|
|
118
|
+
"""
|
|
119
|
+
return SystemMetricsComputer()
|
|
108
120
|
|
|
109
121
|
def create_session(self):
|
|
110
122
|
self.session_factory = self._session_factory()
|
|
@@ -114,25 +126,6 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
114
126
|
def table(self):
|
|
115
127
|
return self._table
|
|
116
128
|
|
|
117
|
-
def _get_sampler(self, **kwargs):
|
|
118
|
-
"""get sampler object"""
|
|
119
|
-
from metadata.profiler.processor.sampler.sampler_factory import ( # pylint: disable=import-outside-toplevel
|
|
120
|
-
sampler_factory_,
|
|
121
|
-
)
|
|
122
|
-
|
|
123
|
-
session = kwargs.get("session")
|
|
124
|
-
table = kwargs["table"]
|
|
125
|
-
|
|
126
|
-
return sampler_factory_.create(
|
|
127
|
-
self.service_connection_config.__class__.__name__,
|
|
128
|
-
client=session or self.session,
|
|
129
|
-
table=table,
|
|
130
|
-
profile_sample_config=self.profile_sample_config,
|
|
131
|
-
partition_details=self.partition_details,
|
|
132
|
-
profile_sample_query=self.profile_query,
|
|
133
|
-
sample_data_count=self.sample_data_count,
|
|
134
|
-
)
|
|
135
|
-
|
|
136
129
|
def _session_factory(self) -> scoped_session:
|
|
137
130
|
"""Create thread safe session that will be automatically
|
|
138
131
|
garbage collected once the application thread ends
|
|
@@ -163,7 +156,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
163
156
|
)
|
|
164
157
|
return dict(row)
|
|
165
158
|
except Exception as exc:
|
|
166
|
-
msg = f"Error trying to compute profile for {runner.
|
|
159
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
167
160
|
handle_query_exception(msg, exc, session)
|
|
168
161
|
return None
|
|
169
162
|
|
|
@@ -174,7 +167,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
174
167
|
session,
|
|
175
168
|
*args,
|
|
176
169
|
**kwargs,
|
|
177
|
-
):
|
|
170
|
+
) -> Optional[Dict[str, Any]]:
|
|
178
171
|
"""Given a list of metrics, compute the given results
|
|
179
172
|
and returns the values
|
|
180
173
|
|
|
@@ -201,7 +194,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
201
194
|
except Exception as exc:
|
|
202
195
|
logger.debug(traceback.format_exc())
|
|
203
196
|
logger.warning(
|
|
204
|
-
f"Error trying to compute profile for {runner.
|
|
197
|
+
f"Error trying to compute profile for {runner.table_name}: {exc}" # type: ignore
|
|
205
198
|
)
|
|
206
199
|
session.rollback()
|
|
207
200
|
raise RuntimeError(exc)
|
|
@@ -238,7 +231,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
238
231
|
runner, column, exc, session, metrics
|
|
239
232
|
)
|
|
240
233
|
except Exception as exc:
|
|
241
|
-
msg = f"Error trying to compute profile for {runner.
|
|
234
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
242
235
|
handle_query_exception(msg, exc, session)
|
|
243
236
|
return None
|
|
244
237
|
|
|
@@ -281,10 +274,10 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
281
274
|
runner._session.get_bind().dialect.name
|
|
282
275
|
!= Dialects.Druid
|
|
283
276
|
):
|
|
284
|
-
msg = f"Error trying to compute profile for {runner.
|
|
277
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
285
278
|
handle_query_exception(msg, exc, session)
|
|
286
279
|
except Exception as exc:
|
|
287
|
-
msg = f"Error trying to compute profile for {runner.
|
|
280
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
288
281
|
handle_query_exception(msg, exc, session)
|
|
289
282
|
return None
|
|
290
283
|
|
|
@@ -317,10 +310,10 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
317
310
|
return dict(row)
|
|
318
311
|
except ProgrammingError as exc:
|
|
319
312
|
logger.info(
|
|
320
|
-
f"Skipping metrics for {runner.
|
|
313
|
+
f"Skipping metrics for {runner.table_name}.{column.name} due to {exc}"
|
|
321
314
|
)
|
|
322
315
|
except Exception as exc:
|
|
323
|
-
msg = f"Error trying to compute profile for {runner.
|
|
316
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
324
317
|
handle_query_exception(msg, exc, session)
|
|
325
318
|
return None
|
|
326
319
|
|
|
@@ -354,7 +347,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
354
347
|
)
|
|
355
348
|
|
|
356
349
|
except Exception as exc:
|
|
357
|
-
msg = f"Error trying to compute profile for {runner.
|
|
350
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{metric.columnName}: {exc}"
|
|
358
351
|
logger.debug(traceback.format_exc())
|
|
359
352
|
logger.warning(msg)
|
|
360
353
|
if custom_metrics:
|
|
@@ -363,12 +356,11 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
363
356
|
|
|
364
357
|
def _compute_system_metrics(
|
|
365
358
|
self,
|
|
366
|
-
metrics:
|
|
359
|
+
metrics: Type[System],
|
|
367
360
|
runner: QueryRunner,
|
|
368
|
-
session,
|
|
369
361
|
*args,
|
|
370
362
|
**kwargs,
|
|
371
|
-
):
|
|
363
|
+
) -> List[SystemProfile]:
|
|
372
364
|
"""Get system metric for tables
|
|
373
365
|
|
|
374
366
|
Args:
|
|
@@ -379,44 +371,21 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
379
371
|
Returns:
|
|
380
372
|
dictionnary of results
|
|
381
373
|
"""
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
return rows
|
|
385
|
-
except Exception as exc:
|
|
386
|
-
msg = f"Error trying to compute profile for {runner.table.__tablename__}: {exc}"
|
|
387
|
-
handle_query_exception(msg, exc, session)
|
|
388
|
-
return None
|
|
374
|
+
logger.debug(f"Computing system metrics for {runner.table_name}")
|
|
375
|
+
return self.system_metrics_computer.get_system_metrics(table=runner.dataset)
|
|
389
376
|
|
|
390
|
-
def
|
|
391
|
-
self,
|
|
392
|
-
session,
|
|
393
|
-
table,
|
|
394
|
-
):
|
|
395
|
-
"""Create thread safe runner"""
|
|
396
|
-
if not hasattr(thread_local, "sampler"):
|
|
397
|
-
thread_local.sampler = self._get_sampler(
|
|
398
|
-
table=table,
|
|
399
|
-
session=session,
|
|
400
|
-
)
|
|
401
|
-
return thread_local.sampler
|
|
402
|
-
|
|
403
|
-
def _create_thread_safe_runner(
|
|
404
|
-
self,
|
|
405
|
-
session,
|
|
406
|
-
table,
|
|
407
|
-
sample,
|
|
408
|
-
):
|
|
377
|
+
def _create_thread_safe_runner(self, session, column=None):
|
|
409
378
|
"""Create thread safe runner"""
|
|
410
379
|
if not hasattr(thread_local, "runner"):
|
|
411
380
|
thread_local.runner = QueryRunner(
|
|
412
381
|
session=session,
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
partition_details=self.partition_details,
|
|
416
|
-
profile_sample_query=self.
|
|
382
|
+
dataset=self.sampler.get_dataset(column=column),
|
|
383
|
+
raw_dataset=self.sampler.raw_dataset,
|
|
384
|
+
partition_details=self.sampler.partition_details,
|
|
385
|
+
profile_sample_query=self.sampler.sample_query,
|
|
417
386
|
)
|
|
418
387
|
return thread_local.runner
|
|
419
|
-
thread_local.runner.
|
|
388
|
+
thread_local.runner.dataset = self.sampler.get_dataset(column=column)
|
|
420
389
|
return thread_local.runner
|
|
421
390
|
|
|
422
391
|
def compute_metrics_in_thread(
|
|
@@ -431,31 +400,19 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
431
400
|
with Session() as session:
|
|
432
401
|
self.set_session_tag(session)
|
|
433
402
|
self.set_catalog(session)
|
|
434
|
-
|
|
435
|
-
session,
|
|
436
|
-
metric_func.table,
|
|
437
|
-
)
|
|
438
|
-
sample = sampler.random_sample(metric_func.column)
|
|
439
|
-
runner = self._create_thread_safe_runner(
|
|
440
|
-
session,
|
|
441
|
-
metric_func.table,
|
|
442
|
-
sample,
|
|
443
|
-
)
|
|
403
|
+
runner = self._create_thread_safe_runner(session, metric_func.column)
|
|
444
404
|
row = None
|
|
445
|
-
|
|
446
405
|
try:
|
|
447
406
|
row = self._get_metric_fn[metric_func.metric_type.value](
|
|
448
407
|
metric_func.metrics,
|
|
449
408
|
runner=runner,
|
|
450
409
|
session=session,
|
|
451
410
|
column=metric_func.column,
|
|
452
|
-
sample=
|
|
411
|
+
sample=runner.dataset,
|
|
453
412
|
)
|
|
454
|
-
if
|
|
413
|
+
if isinstance(row, dict):
|
|
455
414
|
row = self._validate_nulls(row)
|
|
456
|
-
|
|
457
|
-
# System metrics return a list of dictionaries, with UPDATE, INSERT or DELETE ops results
|
|
458
|
-
if row and metric_func.metric_type == MetricTypes.System:
|
|
415
|
+
if isinstance(row, list):
|
|
459
416
|
row = [
|
|
460
417
|
self._validate_nulls(r) if isinstance(r, dict) else r
|
|
461
418
|
for r in row
|
|
@@ -537,24 +494,12 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
537
494
|
logger.debug(traceback.format_exc())
|
|
538
495
|
logger.error(f"Operation was cancelled due to TimeoutError - {exc}")
|
|
539
496
|
raise concurrent.futures.TimeoutError
|
|
497
|
+
except KeyboardInterrupt:
|
|
498
|
+
pool.shutdown39(wait=True, cancel_futures=True)
|
|
499
|
+
raise
|
|
540
500
|
|
|
541
501
|
return profile_results
|
|
542
502
|
|
|
543
|
-
def fetch_sample_data(self, table, columns) -> TableData:
|
|
544
|
-
"""Fetch sample data from database
|
|
545
|
-
|
|
546
|
-
Args:
|
|
547
|
-
table: ORM declarative table
|
|
548
|
-
|
|
549
|
-
Returns:
|
|
550
|
-
TableData: sample table data
|
|
551
|
-
"""
|
|
552
|
-
sampler = self._get_sampler(
|
|
553
|
-
table=table,
|
|
554
|
-
)
|
|
555
|
-
|
|
556
|
-
return sampler.fetch_sample_data(columns)
|
|
557
|
-
|
|
558
503
|
def get_composed_metrics(
|
|
559
504
|
self, column: Column, metric: Metrics, column_results: Dict
|
|
560
505
|
):
|
|
@@ -576,21 +521,24 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
576
521
|
return None
|
|
577
522
|
|
|
578
523
|
def get_hybrid_metrics(
|
|
579
|
-
self,
|
|
524
|
+
self,
|
|
525
|
+
column: Column,
|
|
526
|
+
metric: Type[HybridMetric],
|
|
527
|
+
column_results: Dict[str, Any],
|
|
580
528
|
):
|
|
581
529
|
"""Given a list of metrics, compute the given results
|
|
582
530
|
and returns the values
|
|
583
531
|
|
|
584
532
|
Args:
|
|
585
533
|
column: the column to compute the metrics against
|
|
586
|
-
|
|
534
|
+
metric: metric to compute
|
|
535
|
+
column_results: results of the column
|
|
587
536
|
Returns:
|
|
588
537
|
dictionnary of results
|
|
589
538
|
"""
|
|
590
|
-
|
|
591
|
-
sample = sampler.random_sample(column)
|
|
539
|
+
dataset = self.sampler.get_dataset(column=column)
|
|
592
540
|
try:
|
|
593
|
-
return metric(column).fn(
|
|
541
|
+
return metric(column).fn(dataset, column_results, self.session)
|
|
594
542
|
except Exception as exc:
|
|
595
543
|
logger.debug(traceback.format_exc())
|
|
596
544
|
logger.warning(f"Unexpected exception computing metrics: {exc}")
|
|
@@ -24,13 +24,13 @@ from metadata.profiler.interface.sqlalchemy.profiler_interface import (
|
|
|
24
24
|
)
|
|
25
25
|
from metadata.profiler.metrics.registry import Metrics
|
|
26
26
|
from metadata.profiler.processor.runner import QueryRunner
|
|
27
|
-
from metadata.profiler.source.single_store.metrics.window.first_quartile import (
|
|
27
|
+
from metadata.profiler.source.database.single_store.metrics.window.first_quartile import (
|
|
28
28
|
SingleStoreFirstQuartile,
|
|
29
29
|
)
|
|
30
|
-
from metadata.profiler.source.single_store.metrics.window.median import (
|
|
30
|
+
from metadata.profiler.source.database.single_store.metrics.window.median import (
|
|
31
31
|
SingleStoreMedian,
|
|
32
32
|
)
|
|
33
|
-
from metadata.profiler.source.single_store.metrics.window.third_quartile import (
|
|
33
|
+
from metadata.profiler.source.database.single_store.metrics.window.third_quartile import (
|
|
34
34
|
SingleStoreThirdQuartile,
|
|
35
35
|
)
|
|
36
36
|
from metadata.utils.logger import profiler_interface_registry_logger
|
|
@@ -76,11 +76,11 @@ class SingleStoreProfilerInterface(SQAProfilerInterface):
|
|
|
76
76
|
return dict(row)
|
|
77
77
|
except ProgrammingError:
|
|
78
78
|
logger.info(
|
|
79
|
-
f"Skipping window metrics for {runner.
|
|
79
|
+
f"Skipping window metrics for {runner.table_name}.{column.name} due to overflow"
|
|
80
80
|
)
|
|
81
81
|
return None
|
|
82
82
|
|
|
83
83
|
except Exception as exc:
|
|
84
|
-
msg = f"Error trying to compute profile for {runner.
|
|
84
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
85
85
|
handle_query_exception(msg, exc, session)
|
|
86
86
|
return None
|
|
@@ -41,7 +41,7 @@ class SnowflakeProfilerInterface(SQAProfilerInterface):
|
|
|
41
41
|
session.bind.dialect.name
|
|
42
42
|
):
|
|
43
43
|
logger.info(
|
|
44
|
-
f"Computing metrics without sum for {runner.
|
|
44
|
+
f"Computing metrics without sum for {runner.table_name}.{column.name}"
|
|
45
45
|
)
|
|
46
46
|
return self._compute_static_metrics_wo_sum(metrics, runner, session, column)
|
|
47
47
|
return None
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# Copyright 2021 Collate
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
7
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
8
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
9
|
+
# See the License for the specific language governing permissions and
|
|
10
|
+
# limitations under the License.
|
|
11
|
+
|
|
12
|
+
"""
|
|
13
|
+
Interfaces with database for all database engine
|
|
14
|
+
supporting sqlalchemy abstraction layer
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import threading
|
|
18
|
+
from typing import Any, Dict, List, Set
|
|
19
|
+
|
|
20
|
+
from more_itertools import partition
|
|
21
|
+
from sqlalchemy import Column
|
|
22
|
+
|
|
23
|
+
from metadata.generated.schema.entity.data.table import Table
|
|
24
|
+
from metadata.mixins.sqalchemy.sqa_mixin import Root
|
|
25
|
+
from metadata.profiler.interface.sqlalchemy.profiler_interface import (
|
|
26
|
+
SQAProfilerInterface,
|
|
27
|
+
)
|
|
28
|
+
from metadata.profiler.metrics.core import Metric
|
|
29
|
+
from metadata.profiler.metrics.registry import Metrics
|
|
30
|
+
from metadata.profiler.processor.runner import QueryRunner
|
|
31
|
+
from metadata.utils.logger import profiler_interface_registry_logger
|
|
32
|
+
|
|
33
|
+
logger = profiler_interface_registry_logger()
|
|
34
|
+
thread_local = threading.local()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class StoredStatisticsSource(Root):
|
|
38
|
+
def get_statistics_metrics(self) -> Set[Metrics]:
|
|
39
|
+
"""Statistic metrics that are found in system tables. Different for each database."""
|
|
40
|
+
return set()
|
|
41
|
+
|
|
42
|
+
def get_column_statistics(
|
|
43
|
+
self, metric: List[Metrics], schema: str, table_name: Table, column: str
|
|
44
|
+
) -> dict:
|
|
45
|
+
raise NotImplementedError(
|
|
46
|
+
"You used a connector that does not support using statistics tables."
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
def get_table_statistics(
|
|
50
|
+
self, metric: List[Metrics], schema: str, table_name: Table
|
|
51
|
+
) -> dict:
|
|
52
|
+
raise NotImplementedError(
|
|
53
|
+
"You used a connector that does not support using statistics tables."
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class ProfilerWithStatistics(SQAProfilerInterface, StoredStatisticsSource):
|
|
58
|
+
"""
|
|
59
|
+
Interface to interact with registry supporting
|
|
60
|
+
sqlalchemy.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def __init__(self, *args, **kwargs):
|
|
64
|
+
"""Collaborative constructor"""
|
|
65
|
+
super().__init__(*args, **kwargs)
|
|
66
|
+
|
|
67
|
+
def _compute_static_metrics(
|
|
68
|
+
self,
|
|
69
|
+
metrics: List[Metrics],
|
|
70
|
+
runner: QueryRunner,
|
|
71
|
+
column,
|
|
72
|
+
session,
|
|
73
|
+
*args,
|
|
74
|
+
**kwargs,
|
|
75
|
+
) -> dict:
|
|
76
|
+
result = {}
|
|
77
|
+
if self.source_config.useStatistics:
|
|
78
|
+
metrics, stat_metrics = map(
|
|
79
|
+
list,
|
|
80
|
+
partition(self.is_statistic_metric, metrics),
|
|
81
|
+
)
|
|
82
|
+
schema = runner.schema_name
|
|
83
|
+
table_name = runner.table_name
|
|
84
|
+
logger.debug(
|
|
85
|
+
"Getting statistics for column: %s.%s.%s",
|
|
86
|
+
schema,
|
|
87
|
+
table_name,
|
|
88
|
+
column.name,
|
|
89
|
+
)
|
|
90
|
+
result.update(
|
|
91
|
+
super().get_column_statistics(
|
|
92
|
+
stat_metrics, schema, table_name, column.name
|
|
93
|
+
)
|
|
94
|
+
)
|
|
95
|
+
result.update(
|
|
96
|
+
super()._compute_static_metrics(
|
|
97
|
+
metrics,
|
|
98
|
+
runner,
|
|
99
|
+
column,
|
|
100
|
+
session,
|
|
101
|
+
*args,
|
|
102
|
+
**kwargs,
|
|
103
|
+
)
|
|
104
|
+
)
|
|
105
|
+
return result
|
|
106
|
+
|
|
107
|
+
def _compute_table_metrics(
|
|
108
|
+
self,
|
|
109
|
+
metrics: List[Metric],
|
|
110
|
+
runner: QueryRunner,
|
|
111
|
+
session,
|
|
112
|
+
*args,
|
|
113
|
+
**kwargs,
|
|
114
|
+
) -> Dict[str, Any]:
|
|
115
|
+
result = {}
|
|
116
|
+
if self.source_config.useStatistics:
|
|
117
|
+
metrics, stat_metrics = map(
|
|
118
|
+
list,
|
|
119
|
+
partition(self.is_statistic_metric, metrics),
|
|
120
|
+
)
|
|
121
|
+
schema = runner.schema_name
|
|
122
|
+
table_name = runner.table_name
|
|
123
|
+
logger.debug("Geting statistics for table: %s.%s", schema, table_name)
|
|
124
|
+
result.update(
|
|
125
|
+
super().get_table_statistics(stat_metrics, schema, table_name)
|
|
126
|
+
)
|
|
127
|
+
super_table_metrics = super()._compute_table_metrics(
|
|
128
|
+
metrics,
|
|
129
|
+
runner,
|
|
130
|
+
session,
|
|
131
|
+
*args,
|
|
132
|
+
**kwargs,
|
|
133
|
+
)
|
|
134
|
+
if super_table_metrics is not None:
|
|
135
|
+
result.update(super_table_metrics)
|
|
136
|
+
return result
|
|
137
|
+
|
|
138
|
+
def get_hybrid_metrics(self, column: Column, metric: Metric, column_results: Dict):
|
|
139
|
+
# this metrics might have been computed in a previous step
|
|
140
|
+
return column_results.get(metric.name()) or super().get_hybrid_metrics(
|
|
141
|
+
column, metric, column_results
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
def is_statistic_metric(self, metric: Metric) -> bool:
|
|
145
|
+
return metric.name() in {m.name for m in super().get_statistics_metrics()}
|
|
@@ -19,10 +19,15 @@ from typing import List
|
|
|
19
19
|
from sqlalchemy import func
|
|
20
20
|
from sqlalchemy.exc import ProgrammingError
|
|
21
21
|
|
|
22
|
+
from metadata.ingestion.source.database.trino.profiler.system_tables_profiler import (
|
|
23
|
+
TrinoStoredStatisticsSource,
|
|
24
|
+
)
|
|
22
25
|
from metadata.profiler.interface.sqlalchemy.profiler_interface import (
|
|
23
|
-
SQAProfilerInterface,
|
|
24
26
|
handle_query_exception,
|
|
25
27
|
)
|
|
28
|
+
from metadata.profiler.interface.sqlalchemy.stored_statistics_profiler import (
|
|
29
|
+
ProfilerWithStatistics,
|
|
30
|
+
)
|
|
26
31
|
from metadata.profiler.metrics.registry import Metrics
|
|
27
32
|
from metadata.profiler.orm.registry import FLOAT_SET
|
|
28
33
|
from metadata.profiler.processor.runner import QueryRunner
|
|
@@ -31,7 +36,7 @@ from metadata.utils.logger import profiler_interface_registry_logger
|
|
|
31
36
|
logger = profiler_interface_registry_logger()
|
|
32
37
|
|
|
33
38
|
|
|
34
|
-
class TrinoProfilerInterface(
|
|
39
|
+
class TrinoProfilerInterface(ProfilerWithStatistics, TrinoStoredStatisticsSource):
|
|
35
40
|
"""
|
|
36
41
|
Interface to interact with registry supporting
|
|
37
42
|
sqlalchemy.
|
|
@@ -71,11 +76,11 @@ class TrinoProfilerInterface(SQAProfilerInterface):
|
|
|
71
76
|
return dict(row)
|
|
72
77
|
except ProgrammingError as err:
|
|
73
78
|
logger.info(
|
|
74
|
-
f"Skipping window metrics for {runner.
|
|
79
|
+
f"Skipping window metrics for {runner.table_name}.{column.name} due to {err}"
|
|
75
80
|
)
|
|
76
81
|
return None
|
|
77
82
|
|
|
78
83
|
except Exception as exc:
|
|
79
|
-
msg = f"Error trying to compute profile for {runner.
|
|
84
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
80
85
|
handle_query_exception(msg, exc, session)
|
|
81
86
|
return None
|