openmetadata-ingestion 1.6.0.0rc1__py3-none-any.whl → 1.6.0.0rc3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openmetadata-ingestion might be problematic. Click here for more details.
- metadata/cli/classify.py +52 -0
- metadata/clients/aws_client.py +2 -2
- metadata/cmd.py +9 -0
- metadata/data_quality/builders/{i_validator_builder.py → validator_builder.py} +40 -29
- metadata/data_quality/interface/pandas/pandas_test_suite_interface.py +25 -25
- metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +23 -50
- metadata/data_quality/interface/test_suite_interface.py +45 -56
- metadata/data_quality/processor/test_case_runner.py +9 -9
- metadata/data_quality/runner/base_test_suite_source.py +52 -26
- metadata/data_quality/validations/base_test_handler.py +14 -9
- metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValueMaxToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValueMeanToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValueMedianToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValueMinToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValueStdDevToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesMissingCount.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesSumToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeAtExpectedLocation.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeUnique.py +2 -8
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +1 -1
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +1 -1
- metadata/data_quality/validations/runtime_param_setter/param_setter.py +2 -3
- metadata/data_quality/validations/runtime_param_setter/param_setter_factory.py +45 -17
- metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py +21 -16
- metadata/data_quality/validations/table/sqlalchemy/tableCustomSQLQuery.py +7 -1
- metadata/data_quality/validations/table/sqlalchemy/tableDiff.py +21 -18
- metadata/data_quality/validations/table/sqlalchemy/tableRowInsertedCountToBeBetween.py +2 -2
- metadata/examples/workflows/bigquery_classifier.yaml +56 -0
- metadata/examples/workflows/bigquery_profiler.yaml +1 -2
- metadata/examples/workflows/db2_profiler.yaml +1 -2
- metadata/examples/workflows/dbtcloud.yaml +2 -1
- metadata/examples/workflows/{mstr.yaml → microstrategy.yaml} +3 -2
- metadata/examples/workflows/mongodb.yaml +4 -0
- metadata/examples/workflows/mysql_profiler.yaml +0 -1
- metadata/examples/workflows/redshift_classifier.yaml +38 -0
- metadata/examples/workflows/redshift_profiler.yaml +2 -3
- metadata/generated/antlr/FqnLexer.py +15 -15
- metadata/generated/schema/analytics/__init__.py +1 -1
- metadata/generated/schema/analytics/basic.py +1 -1
- metadata/generated/schema/analytics/reportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
- metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
- metadata/generated/schema/api/__init__.py +1 -1
- metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
- metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
- metadata/generated/schema/api/analytics/__init__.py +1 -1
- metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
- metadata/generated/schema/api/automations/__init__.py +1 -1
- metadata/generated/schema/api/automations/createWorkflow.py +1 -1
- metadata/generated/schema/api/bulkAssets.py +1 -1
- metadata/generated/schema/api/classification/__init__.py +1 -1
- metadata/generated/schema/api/classification/createClassification.py +1 -1
- metadata/generated/schema/api/classification/createTag.py +1 -1
- metadata/generated/schema/api/classification/loadTags.py +1 -1
- metadata/generated/schema/api/createBot.py +1 -1
- metadata/generated/schema/api/createEventPublisherJob.py +1 -1
- metadata/generated/schema/api/createType.py +1 -1
- metadata/generated/schema/api/data/__init__.py +1 -1
- metadata/generated/schema/api/data/createAPICollection.py +1 -1
- metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
- metadata/generated/schema/api/data/createChart.py +1 -1
- metadata/generated/schema/api/data/createContainer.py +1 -1
- metadata/generated/schema/api/data/createCustomProperty.py +1 -1
- metadata/generated/schema/api/data/createDashboard.py +1 -1
- metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
- metadata/generated/schema/api/data/createDatabase.py +1 -1
- metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
- metadata/generated/schema/api/data/createGlossary.py +1 -1
- metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
- metadata/generated/schema/api/data/createMetric.py +1 -1
- metadata/generated/schema/api/data/createMlModel.py +1 -1
- metadata/generated/schema/api/data/createPipeline.py +1 -1
- metadata/generated/schema/api/data/createQuery.py +1 -1
- metadata/generated/schema/api/data/createSearchIndex.py +8 -1
- metadata/generated/schema/api/data/createStoredProcedure.py +8 -1
- metadata/generated/schema/api/data/createTable.py +1 -1
- metadata/generated/schema/api/data/createTableProfile.py +1 -1
- metadata/generated/schema/api/data/createTopic.py +1 -1
- metadata/generated/schema/api/data/loadGlossary.py +1 -1
- metadata/generated/schema/api/data/restoreEntity.py +1 -1
- metadata/generated/schema/api/dataInsight/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
- metadata/generated/schema/api/docStore/__init__.py +1 -1
- metadata/generated/schema/api/docStore/createDocument.py +1 -1
- metadata/generated/schema/api/domains/__init__.py +1 -1
- metadata/generated/schema/api/domains/createDataProduct.py +1 -1
- metadata/generated/schema/api/domains/createDomain.py +1 -1
- metadata/generated/schema/api/feed/__init__.py +1 -1
- metadata/generated/schema/api/feed/closeTask.py +1 -1
- metadata/generated/schema/api/feed/createPost.py +1 -1
- metadata/generated/schema/api/feed/createSuggestion.py +1 -1
- metadata/generated/schema/api/feed/createThread.py +1 -1
- metadata/generated/schema/api/feed/resolveTask.py +1 -1
- metadata/generated/schema/api/feed/threadCount.py +1 -1
- metadata/generated/schema/api/governance/__init__.py +1 -1
- metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
- metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
- metadata/generated/schema/api/lineage/__init__.py +1 -1
- metadata/generated/schema/api/lineage/addLineage.py +1 -1
- metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
- metadata/generated/schema/api/policies/__init__.py +1 -1
- metadata/generated/schema/api/policies/createPolicy.py +1 -1
- metadata/generated/schema/api/services/__init__.py +1 -1
- metadata/generated/schema/api/services/createApiService.py +1 -1
- metadata/generated/schema/api/services/createDashboardService.py +1 -1
- metadata/generated/schema/api/services/createDatabaseService.py +1 -1
- metadata/generated/schema/api/services/createMessagingService.py +1 -1
- metadata/generated/schema/api/services/createMetadataService.py +1 -1
- metadata/generated/schema/api/services/createMlModelService.py +1 -1
- metadata/generated/schema/api/services/createPipelineService.py +1 -1
- metadata/generated/schema/api/services/createSearchService.py +1 -1
- metadata/generated/schema/api/services/createStorageService.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
- metadata/generated/schema/api/setOwner.py +1 -1
- metadata/generated/schema/api/teams/__init__.py +1 -1
- metadata/generated/schema/api/teams/createPersona.py +1 -1
- metadata/generated/schema/api/teams/createRole.py +1 -1
- metadata/generated/schema/api/teams/createTeam.py +1 -1
- metadata/generated/schema/api/teams/createUser.py +1 -1
- metadata/generated/schema/api/tests/__init__.py +1 -1
- metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
- metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
- metadata/generated/schema/api/tests/createTestCase.py +1 -1
- metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
- metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
- metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
- metadata/generated/schema/api/tests/createTestSuite.py +1 -1
- metadata/generated/schema/api/voteRequest.py +1 -1
- metadata/generated/schema/auth/__init__.py +1 -1
- metadata/generated/schema/auth/basicAuth.py +1 -1
- metadata/generated/schema/auth/basicLoginRequest.py +1 -1
- metadata/generated/schema/auth/changePasswordRequest.py +1 -1
- metadata/generated/schema/auth/createPersonalToken.py +1 -1
- metadata/generated/schema/auth/emailRequest.py +1 -1
- metadata/generated/schema/auth/emailVerificationToken.py +1 -1
- metadata/generated/schema/auth/generateToken.py +1 -1
- metadata/generated/schema/auth/jwtAuth.py +1 -1
- metadata/generated/schema/auth/loginRequest.py +1 -1
- metadata/generated/schema/auth/logoutRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetToken.py +1 -1
- metadata/generated/schema/auth/personalAccessToken.py +1 -1
- metadata/generated/schema/auth/refreshToken.py +1 -1
- metadata/generated/schema/auth/registrationRequest.py +1 -1
- metadata/generated/schema/auth/revokePersonalToken.py +1 -1
- metadata/generated/schema/auth/revokeToken.py +1 -1
- metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
- metadata/generated/schema/auth/ssoAuth.py +1 -1
- metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
- metadata/generated/schema/configuration/__init__.py +1 -1
- metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
- metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
- metadata/generated/schema/configuration/authConfig.py +1 -1
- metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
- metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
- metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
- metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
- metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
- metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
- metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
- metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
- metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
- metadata/generated/schema/configuration/lineageSettings.py +1 -1
- metadata/generated/schema/configuration/loginConfiguration.py +1 -1
- metadata/generated/schema/configuration/logoConfiguration.py +1 -1
- metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
- metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
- metadata/generated/schema/configuration/searchSettings.py +1 -1
- metadata/generated/schema/configuration/slackAppConfiguration.py +1 -14
- metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/themeConfiguration.py +1 -1
- metadata/generated/schema/configuration/uiThemePreference.py +1 -1
- metadata/generated/schema/dataInsight/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
- metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
- metadata/generated/schema/dataInsight/custom/lineChart.py +1 -1
- metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
- metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
- metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
- metadata/generated/schema/dataInsight/type/__init__.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
- metadata/generated/schema/email/__init__.py +1 -1
- metadata/generated/schema/email/emailRequest.py +1 -1
- metadata/generated/schema/email/emailTemplate.py +1 -1
- metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
- metadata/generated/schema/email/smtpSettings.py +1 -1
- metadata/generated/schema/email/templateValidationReponse.py +1 -1
- metadata/generated/schema/entity/__init__.py +1 -1
- metadata/generated/schema/entity/applications/__init__.py +1 -1
- metadata/generated/schema/entity/applications/app.py +2 -1
- metadata/generated/schema/entity/applications/appExtension.py +1 -1
- metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
- metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/applicationConfig.py +4 -2
- metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +22 -0
- metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
- metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
- metadata/generated/schema/entity/applications/jobStatus.py +1 -1
- metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +1 -1
- metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
- metadata/generated/schema/entity/automations/__init__.py +1 -1
- metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
- metadata/generated/schema/entity/automations/workflow.py +1 -1
- metadata/generated/schema/entity/bot.py +1 -1
- metadata/generated/schema/entity/classification/__init__.py +1 -1
- metadata/generated/schema/entity/classification/classification.py +1 -1
- metadata/generated/schema/entity/classification/tag.py +1 -1
- metadata/generated/schema/entity/data/__init__.py +1 -1
- metadata/generated/schema/entity/data/apiCollection.py +1 -1
- metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
- metadata/generated/schema/entity/data/chart.py +1 -1
- metadata/generated/schema/entity/data/container.py +1 -1
- metadata/generated/schema/entity/data/dashboard.py +1 -1
- metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
- metadata/generated/schema/entity/data/database.py +1 -1
- metadata/generated/schema/entity/data/databaseSchema.py +1 -1
- metadata/generated/schema/entity/data/glossary.py +1 -1
- metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
- metadata/generated/schema/entity/data/metric.py +1 -1
- metadata/generated/schema/entity/data/mlmodel.py +1 -1
- metadata/generated/schema/entity/data/pipeline.py +1 -1
- metadata/generated/schema/entity/data/query.py +1 -1
- metadata/generated/schema/entity/data/report.py +1 -1
- metadata/generated/schema/entity/data/searchIndex.py +13 -1
- metadata/generated/schema/entity/data/storedProcedure.py +13 -1
- metadata/generated/schema/entity/data/table.py +1 -1
- metadata/generated/schema/entity/data/topic.py +1 -1
- metadata/generated/schema/entity/docStore/__init__.py +1 -1
- metadata/generated/schema/entity/docStore/document.py +1 -1
- metadata/generated/schema/entity/domains/__init__.py +1 -1
- metadata/generated/schema/entity/domains/dataProduct.py +1 -1
- metadata/generated/schema/entity/domains/domain.py +1 -1
- metadata/generated/schema/entity/events/__init__.py +1 -1
- metadata/generated/schema/entity/events/webhook.py +1 -1
- metadata/generated/schema/entity/feed/__init__.py +1 -1
- metadata/generated/schema/entity/feed/assets.py +1 -1
- metadata/generated/schema/entity/feed/customProperty.py +1 -1
- metadata/generated/schema/entity/feed/description.py +1 -1
- metadata/generated/schema/entity/feed/domain.py +1 -1
- metadata/generated/schema/entity/feed/entityInfo.py +1 -1
- metadata/generated/schema/entity/feed/owner.py +1 -1
- metadata/generated/schema/entity/feed/suggestion.py +1 -1
- metadata/generated/schema/entity/feed/tag.py +1 -1
- metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
- metadata/generated/schema/entity/feed/thread.py +1 -1
- metadata/generated/schema/entity/policies/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
- metadata/generated/schema/entity/policies/filters.py +1 -1
- metadata/generated/schema/entity/policies/policy.py +1 -1
- metadata/generated/schema/entity/services/__init__.py +1 -1
- metadata/generated/schema/entity/services/apiService.py +1 -1
- metadata/generated/schema/entity/services/connections/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/api/restConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +67 -0
- metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/athenaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
- metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/db2Connection.py +13 -1
- metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/druidConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/exasolConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/hiveConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/impalaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +6 -2
- metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +6 -3
- metadata/generated/schema/entity/services/connections/database/oracleConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/postgresConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/teradataConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/trinoConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +17 -5
- metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
- metadata/generated/schema/entity/services/dashboardService.py +4 -4
- metadata/generated/schema/entity/services/databaseService.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +2 -1
- metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
- metadata/generated/schema/entity/services/messagingService.py +1 -1
- metadata/generated/schema/entity/services/metadataService.py +1 -1
- metadata/generated/schema/entity/services/mlmodelService.py +1 -1
- metadata/generated/schema/entity/services/pipelineService.py +1 -1
- metadata/generated/schema/entity/services/searchService.py +1 -1
- metadata/generated/schema/entity/services/serviceType.py +1 -1
- metadata/generated/schema/entity/services/storageService.py +1 -1
- metadata/generated/schema/entity/teams/__init__.py +1 -1
- metadata/generated/schema/entity/teams/persona.py +1 -1
- metadata/generated/schema/entity/teams/role.py +1 -1
- metadata/generated/schema/entity/teams/team.py +1 -1
- metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
- metadata/generated/schema/entity/teams/user.py +1 -1
- metadata/generated/schema/entity/type.py +1 -1
- metadata/generated/schema/entity/utils/__init__.py +1 -1
- metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
- metadata/generated/schema/entity/utils/servicesCount.py +1 -1
- metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
- metadata/generated/schema/events/__init__.py +1 -1
- metadata/generated/schema/events/alertMetrics.py +1 -1
- metadata/generated/schema/events/api/__init__.py +1 -1
- metadata/generated/schema/events/api/createEventSubscription.py +1 -1
- metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
- metadata/generated/schema/events/api/eventsRecord.py +1 -1
- metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
- metadata/generated/schema/events/api/typedEvent.py +1 -1
- metadata/generated/schema/events/emailAlertConfig.py +1 -1
- metadata/generated/schema/events/eventFilterRule.py +1 -1
- metadata/generated/schema/events/eventSubscription.py +1 -1
- metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
- metadata/generated/schema/events/failedEvent.py +1 -1
- metadata/generated/schema/events/failedEventResponse.py +1 -1
- metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
- metadata/generated/schema/events/statusContext.py +1 -1
- metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
- metadata/generated/schema/events/subscriptionStatus.py +1 -1
- metadata/generated/schema/events/testDestinationStatus.py +1 -1
- metadata/generated/schema/governance/workflows/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/edge.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodeSubType.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodeType.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +1 -1
- metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +2 -2
- metadata/generated/schema/governance/workflows/workflowDefinition.py +1 -1
- metadata/generated/schema/governance/workflows/workflowInstance.py +8 -1
- metadata/generated/schema/governance/workflows/workflowInstanceState.py +8 -1
- metadata/generated/schema/metadataIngestion/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/application.py +1 -1
- metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +111 -0
- metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +1 -33
- metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +9 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +9 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +9 -1
- metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/workflow.py +3 -1
- metadata/generated/schema/monitoring/__init__.py +1 -1
- metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
- metadata/generated/schema/security/__init__.py +1 -1
- metadata/generated/schema/security/client/__init__.py +1 -1
- metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
- metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
- metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
- metadata/generated/schema/security/credentials/__init__.py +1 -1
- metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
- metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
- metadata/generated/schema/security/credentials/basicAuth.py +1 -1
- metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
- metadata/generated/schema/security/credentials/gcpValues.py +1 -1
- metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
- metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
- metadata/generated/schema/security/sasl/__init__.py +1 -1
- metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
- metadata/generated/schema/security/secrets/__init__.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
- metadata/generated/schema/security/securityConfiguration.py +1 -1
- metadata/generated/schema/security/ssl/__init__.py +1 -1
- metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
- metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
- metadata/generated/schema/settings/__init__.py +1 -1
- metadata/generated/schema/settings/settings.py +1 -1
- metadata/generated/schema/system/__init__.py +1 -1
- metadata/generated/schema/system/entityError.py +1 -1
- metadata/generated/schema/system/eventPublisherJob.py +2 -1
- metadata/generated/schema/system/indexingError.py +1 -1
- metadata/generated/schema/system/limitsResponse.py +1 -1
- metadata/generated/schema/system/ui/__init__.py +1 -1
- metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
- metadata/generated/schema/system/ui/page.py +1 -1
- metadata/generated/schema/system/validationResponse.py +1 -1
- metadata/generated/schema/tests/__init__.py +1 -1
- metadata/generated/schema/tests/assigned.py +1 -1
- metadata/generated/schema/tests/basic.py +1 -1
- metadata/generated/schema/tests/customMetric.py +1 -1
- metadata/generated/schema/tests/dataQualityReport.py +1 -1
- metadata/generated/schema/tests/resolved.py +1 -1
- metadata/generated/schema/tests/testCase.py +1 -1
- metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
- metadata/generated/schema/tests/testDefinition.py +1 -1
- metadata/generated/schema/tests/testSuite.py +1 -1
- metadata/generated/schema/type/__init__.py +1 -1
- metadata/generated/schema/type/apiSchema.py +1 -1
- metadata/generated/schema/type/assetCertification.py +1 -1
- metadata/generated/schema/type/auditLog.py +1 -1
- metadata/generated/schema/type/basic.py +1 -1
- metadata/generated/schema/type/bulkOperationResult.py +1 -1
- metadata/generated/schema/type/changeEvent.py +1 -1
- metadata/generated/schema/type/changeEventType.py +1 -1
- metadata/generated/schema/type/collectionDescriptor.py +1 -1
- metadata/generated/schema/type/csvDocumentation.py +1 -1
- metadata/generated/schema/type/csvErrorType.py +1 -1
- metadata/generated/schema/type/csvFile.py +1 -1
- metadata/generated/schema/type/csvImportResult.py +1 -1
- metadata/generated/schema/type/customProperties/__init__.py +1 -1
- metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
- metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
- metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
- metadata/generated/schema/type/customProperty.py +1 -1
- metadata/generated/schema/type/dailyCount.py +1 -1
- metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
- metadata/generated/schema/type/entityHierarchy.py +1 -1
- metadata/generated/schema/type/entityHistory.py +1 -1
- metadata/generated/schema/type/entityLineage.py +1 -1
- metadata/generated/schema/type/entityReference.py +1 -1
- metadata/generated/schema/type/entityReferenceList.py +1 -1
- metadata/generated/schema/type/entityRelationship.py +2 -1
- metadata/generated/schema/type/entityUsage.py +1 -1
- metadata/generated/schema/type/filterPattern.py +1 -1
- metadata/generated/schema/type/function.py +1 -1
- metadata/generated/schema/type/include.py +1 -1
- metadata/generated/schema/type/jdbcConnection.py +1 -1
- metadata/generated/schema/type/lifeCycle.py +1 -1
- metadata/generated/schema/type/paging.py +1 -1
- metadata/generated/schema/type/profile.py +1 -1
- metadata/generated/schema/type/queryParserData.py +1 -1
- metadata/generated/schema/type/reaction.py +1 -1
- metadata/generated/schema/type/schedule.py +1 -1
- metadata/generated/schema/type/schema.py +1 -1
- metadata/generated/schema/type/tableQuery.py +1 -1
- metadata/generated/schema/type/tableUsageCount.py +1 -1
- metadata/generated/schema/type/tagLabel.py +1 -1
- metadata/generated/schema/type/usageDetails.py +1 -1
- metadata/generated/schema/type/usageRequest.py +1 -1
- metadata/generated/schema/type/votes.py +1 -1
- metadata/ingestion/api/models.py +9 -5
- metadata/ingestion/lineage/parser.py +6 -4
- metadata/ingestion/lineage/sql_lineage.py +258 -29
- metadata/ingestion/models/patch_request.py +2 -0
- metadata/ingestion/ometa/client.py +13 -8
- metadata/ingestion/ometa/mixins/data_insight_mixin.py +3 -2
- metadata/ingestion/ometa/mixins/ingestion_pipeline_mixin.py +5 -3
- metadata/ingestion/ometa/mixins/suggestions_mixin.py +3 -3
- metadata/ingestion/ometa/mixins/table_mixin.py +1 -1
- metadata/ingestion/ometa/models.py +1 -0
- metadata/ingestion/ometa/ometa_api.py +6 -3
- metadata/ingestion/sink/metadata_rest.py +23 -18
- metadata/ingestion/source/dashboard/dashboard_service.py +2 -0
- metadata/ingestion/source/dashboard/microstrategy/client.py +268 -0
- metadata/ingestion/source/dashboard/{mstr → microstrategy}/connection.py +11 -11
- metadata/ingestion/source/dashboard/{mstr → microstrategy}/metadata.py +74 -51
- metadata/ingestion/source/dashboard/{mstr → microstrategy}/models.py +9 -3
- metadata/ingestion/source/dashboard/microstrategy/service_spec.py +6 -0
- metadata/ingestion/source/dashboard/powerbi/client.py +1 -2
- metadata/ingestion/source/dashboard/superset/db_source.py +1 -1
- metadata/ingestion/source/dashboard/tableau/metadata.py +29 -12
- metadata/ingestion/source/dashboard/tableau/models.py +10 -0
- metadata/ingestion/source/database/athena/metadata.py +8 -0
- metadata/ingestion/source/database/azuresql/service_spec.py +2 -0
- metadata/ingestion/source/database/bigquery/profiler/profiler.py +1 -1
- metadata/ingestion/source/database/bigquery/profiler/system.py +5 -4
- metadata/ingestion/source/database/bigquery/service_spec.py +2 -0
- metadata/ingestion/source/database/common_nosql_source.py +7 -0
- metadata/ingestion/source/database/databricks/metadata.py +1 -3
- metadata/ingestion/source/database/databricks/service_spec.py +4 -0
- metadata/ingestion/source/database/datalake/clients/azure_blob.py +0 -3
- metadata/ingestion/source/database/datalake/clients/base.py +10 -1
- metadata/ingestion/source/database/datalake/clients/gcs.py +4 -2
- metadata/ingestion/source/database/datalake/clients/s3.py +0 -3
- metadata/ingestion/source/database/datalake/service_spec.py +8 -1
- metadata/ingestion/source/database/db2/connection.py +14 -1
- metadata/ingestion/source/database/dbt/constants.py +12 -0
- metadata/ingestion/source/database/dbt/dbt_config.py +19 -1
- metadata/ingestion/source/database/dbt/dbt_service.py +25 -1
- metadata/ingestion/source/database/dbt/dbt_utils.py +29 -0
- metadata/ingestion/source/database/dbt/metadata.py +70 -1
- metadata/ingestion/source/database/dbt/models.py +2 -0
- metadata/ingestion/source/database/dynamodb/service_spec.py +4 -1
- metadata/ingestion/source/database/mongodb/service_spec.py +4 -1
- metadata/ingestion/source/database/mysql/connection.py +5 -0
- metadata/ingestion/source/database/mysql/lineage.py +12 -30
- metadata/ingestion/source/database/mysql/queries.py +43 -0
- metadata/ingestion/source/database/mysql/query_parser.py +46 -0
- metadata/ingestion/source/database/mysql/service_spec.py +7 -1
- metadata/{data_quality/builders/pandas_validator_builder.py → ingestion/source/database/mysql/usage.py} +9 -14
- metadata/ingestion/source/database/redshift/profiler/system.py +5 -4
- metadata/ingestion/source/database/sample_data.py +4 -12
- metadata/ingestion/source/database/snowflake/connection.py +8 -3
- metadata/ingestion/source/database/snowflake/lineage.py +35 -1
- metadata/ingestion/source/database/snowflake/metadata.py +38 -19
- metadata/ingestion/source/database/snowflake/models.py +1 -0
- metadata/ingestion/source/database/snowflake/profiler/system.py +5 -4
- metadata/ingestion/source/database/snowflake/queries.py +22 -1
- metadata/ingestion/source/database/snowflake/service_spec.py +6 -0
- metadata/ingestion/source/database/stored_procedures_mixin.py +20 -8
- metadata/ingestion/source/database/trino/service_spec.py +2 -0
- metadata/ingestion/source/database/unitycatalog/service_spec.py +4 -0
- metadata/ingestion/source/pipeline/airflow/connection.py +45 -2
- metadata/ingestion/source/pipeline/airflow/metadata.py +35 -25
- metadata/ingestion/source/pipeline/dbtcloud/client.py +67 -28
- metadata/ingestion/source/pipeline/dbtcloud/connection.py +1 -3
- metadata/ingestion/source/pipeline/dbtcloud/models.py +1 -1
- metadata/ingestion/source/pipeline/fivetran/metadata.py +1 -1
- metadata/ingestion/source/pipeline/kafkaconnect/client.py +1 -1
- metadata/ingestion/source/search/elasticsearch/metadata.py +53 -0
- metadata/ingestion/source/search/search_service.py +44 -1
- metadata/mixins/pandas/pandas_mixin.py +2 -31
- metadata/mixins/sqalchemy/sqa_mixin.py +16 -16
- metadata/pii/processor.py +10 -9
- metadata/profiler/api/models.py +3 -79
- metadata/profiler/config.py +39 -0
- metadata/profiler/interface/nosql/profiler_interface.py +1 -26
- metadata/profiler/interface/pandas/profiler_interface.py +37 -77
- metadata/profiler/interface/profiler_interface.py +10 -282
- metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +0 -19
- metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +2 -17
- metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +1 -1
- metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +2 -2
- metadata/profiler/interface/sqlalchemy/profiler_interface.py +46 -109
- metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
- metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +1 -1
- metadata/profiler/interface/sqlalchemy/stored_statistics_profiler.py +6 -8
- metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
- metadata/profiler/orm/converter/base.py +21 -12
- metadata/profiler/orm/functions/table_metric_computer.py +9 -8
- metadata/profiler/processor/core.py +5 -58
- metadata/profiler/processor/handle_partition.py +0 -48
- metadata/profiler/processor/runner.py +116 -35
- metadata/profiler/processor/sample_data_handler.py +7 -4
- metadata/profiler/source/database/base/profiler_source.py +56 -138
- metadata/profiler/source/database/bigquery/profiler_source.py +3 -3
- metadata/profiler/source/database/databricks/profiler_source.py +2 -3
- metadata/profiler/source/fetcher/config.py +44 -0
- metadata/profiler/source/fetcher/fetcher_strategy.py +2 -4
- metadata/sampler/config.py +237 -0
- metadata/sampler/models.py +106 -0
- metadata/{profiler/processor/sampler → sampler}/nosql/sampler.py +44 -12
- metadata/sampler/pandas/sampler.py +239 -0
- metadata/{utils → sampler}/partition.py +66 -51
- metadata/sampler/processor.py +154 -0
- metadata/sampler/sampler_interface.py +251 -0
- metadata/{profiler/processor/sampler → sampler}/sqlalchemy/azuresql/sampler.py +1 -1
- metadata/{profiler/processor/sampler → sampler}/sqlalchemy/bigquery/sampler.py +55 -26
- metadata/sampler/sqlalchemy/postgres/sampler.py +91 -0
- metadata/{profiler/processor/sampler → sampler}/sqlalchemy/sampler.py +93 -98
- metadata/sampler/sqlalchemy/snowflake/sampler.py +95 -0
- metadata/{profiler/processor/sampler → sampler}/sqlalchemy/trino/sampler.py +5 -3
- metadata/utils/class_helper.py +7 -0
- metadata/utils/constants.py +63 -0
- metadata/utils/helpers.py +1 -2
- metadata/utils/importer.py +1 -1
- metadata/utils/logger.py +9 -0
- metadata/utils/profiler_utils.py +42 -0
- metadata/utils/service_spec/default.py +6 -0
- metadata/utils/service_spec/service_spec.py +47 -0
- metadata/utils/sqlalchemy_utils.py +1 -1
- metadata/utils/ssl_manager.py +39 -1
- metadata/workflow/classification.py +53 -0
- metadata/workflow/data_quality.py +5 -1
- metadata/workflow/profiler.py +1 -17
- {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/METADATA +373 -373
- {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/RECORD +805 -795
- metadata/data_quality/builders/sqa_validator_builder.py +0 -25
- metadata/data_quality/interface/test_suite_interface_factory.py +0 -158
- metadata/data_quality/runner/test_suite_source_factory.py +0 -38
- metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +0 -54
- metadata/ingestion/source/dashboard/mstr/client.py +0 -209
- metadata/ingestion/source/dashboard/mstr/service_spec.py +0 -4
- metadata/profiler/processor/sampler/pandas/sampler.py +0 -170
- metadata/profiler/processor/sampler/sampler_factory.py +0 -100
- metadata/profiler/processor/sampler/sampler_interface.py +0 -74
- metadata/profiler/processor/sampler/sqlalchemy/snowflake/sampler.py +0 -83
- /metadata/ingestion/source/dashboard/{mstr → microstrategy}/__init__.py +0 -0
- {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/LICENSE +0 -0
- {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/WHEEL +0 -0
- {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/entry_points.txt +0 -0
- {openmetadata_ingestion-1.6.0.0rc1.dist-info → openmetadata_ingestion-1.6.0.0rc3.dist-info}/top_level.txt +0 -0
|
@@ -77,11 +77,11 @@ class MariaDBProfilerInterface(SQAProfilerInterface):
|
|
|
77
77
|
return dict(row)
|
|
78
78
|
except ProgrammingError:
|
|
79
79
|
logger.info(
|
|
80
|
-
f"Skipping window metrics for {runner.
|
|
80
|
+
f"Skipping window metrics for {runner.table_name}.{column.name} due to overflow"
|
|
81
81
|
)
|
|
82
82
|
return None
|
|
83
83
|
|
|
84
84
|
except Exception as exc:
|
|
85
|
-
msg = f"Error trying to compute profile for {runner.
|
|
85
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
86
86
|
handle_query_exception(msg, exc, session)
|
|
87
87
|
return None
|
|
@@ -21,7 +21,7 @@ import threading
|
|
|
21
21
|
import traceback
|
|
22
22
|
from collections import defaultdict
|
|
23
23
|
from datetime import datetime
|
|
24
|
-
from typing import Any, Dict, List, Optional, Type
|
|
24
|
+
from typing import Any, Dict, List, Optional, Type, Union
|
|
25
25
|
|
|
26
26
|
from sqlalchemy import Column, inspect, text
|
|
27
27
|
from sqlalchemy.exc import DBAPIError, ProgrammingError, ResourceClosedError
|
|
@@ -30,10 +30,18 @@ from sqlalchemy.orm import scoped_session
|
|
|
30
30
|
from metadata.generated.schema.entity.data.table import (
|
|
31
31
|
CustomMetricProfile,
|
|
32
32
|
SystemProfile,
|
|
33
|
-
|
|
33
|
+
Table,
|
|
34
|
+
)
|
|
35
|
+
from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
|
|
36
|
+
DatalakeConnection,
|
|
37
|
+
)
|
|
38
|
+
from metadata.generated.schema.entity.services.databaseService import DatabaseConnection
|
|
39
|
+
from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline import (
|
|
40
|
+
DatabaseServiceProfilerPipeline,
|
|
34
41
|
)
|
|
35
42
|
from metadata.generated.schema.tests.customMetric import CustomMetric
|
|
36
43
|
from metadata.ingestion.connections.session import create_and_bind_thread_safe_session
|
|
44
|
+
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
37
45
|
from metadata.mixins.sqalchemy.sqa_mixin import SQAInterfaceMixin
|
|
38
46
|
from metadata.profiler.api.models import ThreadPoolMetrics
|
|
39
47
|
from metadata.profiler.interface.profiler_interface import ProfilerInterface
|
|
@@ -47,7 +55,7 @@ from metadata.profiler.orm.functions.table_metric_computer import TableMetricCom
|
|
|
47
55
|
from metadata.profiler.orm.registry import Dialects
|
|
48
56
|
from metadata.profiler.processor.metric_filter import MetricFilter
|
|
49
57
|
from metadata.profiler.processor.runner import QueryRunner
|
|
50
|
-
from metadata.
|
|
58
|
+
from metadata.sampler.sampler_interface import SamplerInterface
|
|
51
59
|
from metadata.utils.custom_thread_pool import CustomThreadPoolExecutor
|
|
52
60
|
from metadata.utils.helpers import is_safe_sql_query
|
|
53
61
|
from metadata.utils.logger import profiler_interface_registry_logger
|
|
@@ -75,40 +83,32 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
75
83
|
"""
|
|
76
84
|
|
|
77
85
|
# pylint: disable=too-many-arguments
|
|
78
|
-
|
|
79
86
|
def __init__(
|
|
80
87
|
self,
|
|
81
|
-
service_connection_config,
|
|
82
|
-
ometa_client,
|
|
83
|
-
entity,
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
source_config,
|
|
87
|
-
sample_query,
|
|
88
|
-
table_partition_config,
|
|
88
|
+
service_connection_config: Union[DatabaseConnection, DatalakeConnection],
|
|
89
|
+
ometa_client: OpenMetadata,
|
|
90
|
+
entity: Table,
|
|
91
|
+
source_config: DatabaseServiceProfilerPipeline,
|
|
92
|
+
sampler: SamplerInterface,
|
|
89
93
|
thread_count: int = 5,
|
|
90
94
|
timeout_seconds: int = 43200,
|
|
91
|
-
sqa_metadata=None,
|
|
92
|
-
sample_data_count: Optional[int] = SAMPLE_DATA_DEFAULT_COUNT,
|
|
93
95
|
**kwargs,
|
|
94
96
|
):
|
|
95
97
|
"""Instantiate SQA Interface object"""
|
|
98
|
+
self.session_factory = None
|
|
99
|
+
self.session = None
|
|
96
100
|
|
|
97
101
|
super().__init__(
|
|
98
|
-
service_connection_config,
|
|
99
|
-
ometa_client,
|
|
100
|
-
entity,
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
table_partition_config,
|
|
106
|
-
thread_count,
|
|
107
|
-
timeout_seconds,
|
|
108
|
-
sample_data_count,
|
|
102
|
+
service_connection_config=service_connection_config,
|
|
103
|
+
ometa_client=ometa_client,
|
|
104
|
+
entity=entity,
|
|
105
|
+
source_config=source_config,
|
|
106
|
+
sampler=sampler,
|
|
107
|
+
thread_count=thread_count,
|
|
108
|
+
timeout_seconds=timeout_seconds,
|
|
109
109
|
)
|
|
110
110
|
|
|
111
|
-
self._table = self.
|
|
111
|
+
self._table = self.sampler.raw_dataset
|
|
112
112
|
self.create_session()
|
|
113
113
|
self.system_metrics_computer = self.initialize_system_metrics_computer()
|
|
114
114
|
|
|
@@ -126,25 +126,6 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
126
126
|
def table(self):
|
|
127
127
|
return self._table
|
|
128
128
|
|
|
129
|
-
def _get_sampler(self, **kwargs):
|
|
130
|
-
"""get sampler object"""
|
|
131
|
-
from metadata.profiler.processor.sampler.sampler_factory import ( # pylint: disable=import-outside-toplevel
|
|
132
|
-
sampler_factory_,
|
|
133
|
-
)
|
|
134
|
-
|
|
135
|
-
session = kwargs.get("session")
|
|
136
|
-
table = kwargs["table"]
|
|
137
|
-
|
|
138
|
-
return sampler_factory_.create(
|
|
139
|
-
self.service_connection_config.__class__.__name__,
|
|
140
|
-
client=session or self.session,
|
|
141
|
-
table=table,
|
|
142
|
-
profile_sample_config=self.profile_sample_config,
|
|
143
|
-
partition_details=self.partition_details,
|
|
144
|
-
profile_sample_query=self.profile_query,
|
|
145
|
-
sample_data_count=self.sample_data_count,
|
|
146
|
-
)
|
|
147
|
-
|
|
148
129
|
def _session_factory(self) -> scoped_session:
|
|
149
130
|
"""Create thread safe session that will be automatically
|
|
150
131
|
garbage collected once the application thread ends
|
|
@@ -175,7 +156,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
175
156
|
)
|
|
176
157
|
return dict(row)
|
|
177
158
|
except Exception as exc:
|
|
178
|
-
msg = f"Error trying to compute profile for {runner.
|
|
159
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
179
160
|
handle_query_exception(msg, exc, session)
|
|
180
161
|
return None
|
|
181
162
|
|
|
@@ -213,7 +194,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
213
194
|
except Exception as exc:
|
|
214
195
|
logger.debug(traceback.format_exc())
|
|
215
196
|
logger.warning(
|
|
216
|
-
f"Error trying to compute profile for {runner.
|
|
197
|
+
f"Error trying to compute profile for {runner.table_name}: {exc}" # type: ignore
|
|
217
198
|
)
|
|
218
199
|
session.rollback()
|
|
219
200
|
raise RuntimeError(exc)
|
|
@@ -250,7 +231,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
250
231
|
runner, column, exc, session, metrics
|
|
251
232
|
)
|
|
252
233
|
except Exception as exc:
|
|
253
|
-
msg = f"Error trying to compute profile for {runner.
|
|
234
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
254
235
|
handle_query_exception(msg, exc, session)
|
|
255
236
|
return None
|
|
256
237
|
|
|
@@ -293,10 +274,10 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
293
274
|
runner._session.get_bind().dialect.name
|
|
294
275
|
!= Dialects.Druid
|
|
295
276
|
):
|
|
296
|
-
msg = f"Error trying to compute profile for {runner.
|
|
277
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
297
278
|
handle_query_exception(msg, exc, session)
|
|
298
279
|
except Exception as exc:
|
|
299
|
-
msg = f"Error trying to compute profile for {runner.
|
|
280
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
300
281
|
handle_query_exception(msg, exc, session)
|
|
301
282
|
return None
|
|
302
283
|
|
|
@@ -329,10 +310,10 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
329
310
|
return dict(row)
|
|
330
311
|
except ProgrammingError as exc:
|
|
331
312
|
logger.info(
|
|
332
|
-
f"Skipping metrics for {runner.
|
|
313
|
+
f"Skipping metrics for {runner.table_name}.{column.name} due to {exc}"
|
|
333
314
|
)
|
|
334
315
|
except Exception as exc:
|
|
335
|
-
msg = f"Error trying to compute profile for {runner.
|
|
316
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
336
317
|
handle_query_exception(msg, exc, session)
|
|
337
318
|
return None
|
|
338
319
|
|
|
@@ -366,7 +347,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
366
347
|
)
|
|
367
348
|
|
|
368
349
|
except Exception as exc:
|
|
369
|
-
msg = f"Error trying to compute profile for {runner.
|
|
350
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{metric.columnName}: {exc}"
|
|
370
351
|
logger.debug(traceback.format_exc())
|
|
371
352
|
logger.warning(msg)
|
|
372
353
|
if custom_metrics:
|
|
@@ -390,39 +371,21 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
390
371
|
Returns:
|
|
391
372
|
dictionnary of results
|
|
392
373
|
"""
|
|
393
|
-
logger.debug(f"Computing system metrics for {runner.
|
|
394
|
-
return self.system_metrics_computer.get_system_metrics(
|
|
374
|
+
logger.debug(f"Computing system metrics for {runner.table_name}")
|
|
375
|
+
return self.system_metrics_computer.get_system_metrics(runner=runner)
|
|
395
376
|
|
|
396
|
-
def
|
|
397
|
-
self,
|
|
398
|
-
session,
|
|
399
|
-
table,
|
|
400
|
-
):
|
|
401
|
-
"""Create thread safe runner"""
|
|
402
|
-
if not hasattr(thread_local, "sampler"):
|
|
403
|
-
thread_local.sampler = self._get_sampler(
|
|
404
|
-
table=table,
|
|
405
|
-
session=session,
|
|
406
|
-
)
|
|
407
|
-
return thread_local.sampler
|
|
408
|
-
|
|
409
|
-
def _create_thread_safe_runner(
|
|
410
|
-
self,
|
|
411
|
-
session,
|
|
412
|
-
table,
|
|
413
|
-
sample,
|
|
414
|
-
):
|
|
377
|
+
def _create_thread_safe_runner(self, session, column=None):
|
|
415
378
|
"""Create thread safe runner"""
|
|
416
379
|
if not hasattr(thread_local, "runner"):
|
|
417
380
|
thread_local.runner = QueryRunner(
|
|
418
381
|
session=session,
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
partition_details=self.partition_details,
|
|
422
|
-
profile_sample_query=self.
|
|
382
|
+
dataset=self.sampler.get_dataset(column=column),
|
|
383
|
+
raw_dataset=self.sampler.raw_dataset,
|
|
384
|
+
partition_details=self.sampler.partition_details,
|
|
385
|
+
profile_sample_query=self.sampler.sample_query,
|
|
423
386
|
)
|
|
424
387
|
return thread_local.runner
|
|
425
|
-
thread_local.runner.
|
|
388
|
+
thread_local.runner.dataset = self.sampler.get_dataset(column=column)
|
|
426
389
|
return thread_local.runner
|
|
427
390
|
|
|
428
391
|
def compute_metrics_in_thread(
|
|
@@ -437,16 +400,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
437
400
|
with Session() as session:
|
|
438
401
|
self.set_session_tag(session)
|
|
439
402
|
self.set_catalog(session)
|
|
440
|
-
|
|
441
|
-
session,
|
|
442
|
-
metric_func.table,
|
|
443
|
-
)
|
|
444
|
-
sample = sampler.random_sample(metric_func.column)
|
|
445
|
-
runner = self._create_thread_safe_runner(
|
|
446
|
-
session,
|
|
447
|
-
metric_func.table,
|
|
448
|
-
sample,
|
|
449
|
-
)
|
|
403
|
+
runner = self._create_thread_safe_runner(session, metric_func.column)
|
|
450
404
|
row = None
|
|
451
405
|
try:
|
|
452
406
|
row = self._get_metric_fn[metric_func.metric_type.value](
|
|
@@ -454,7 +408,7 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
454
408
|
runner=runner,
|
|
455
409
|
session=session,
|
|
456
410
|
column=metric_func.column,
|
|
457
|
-
sample=
|
|
411
|
+
sample=runner.dataset,
|
|
458
412
|
)
|
|
459
413
|
if isinstance(row, dict):
|
|
460
414
|
row = self._validate_nulls(row)
|
|
@@ -546,21 +500,6 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
546
500
|
|
|
547
501
|
return profile_results
|
|
548
502
|
|
|
549
|
-
def fetch_sample_data(self, table, columns) -> TableData:
|
|
550
|
-
"""Fetch sample data from database
|
|
551
|
-
|
|
552
|
-
Args:
|
|
553
|
-
table: ORM declarative table
|
|
554
|
-
|
|
555
|
-
Returns:
|
|
556
|
-
TableData: sample table data
|
|
557
|
-
"""
|
|
558
|
-
sampler = self._get_sampler(
|
|
559
|
-
table=table,
|
|
560
|
-
)
|
|
561
|
-
|
|
562
|
-
return sampler.fetch_sample_data(columns)
|
|
563
|
-
|
|
564
503
|
def get_composed_metrics(
|
|
565
504
|
self, column: Column, metric: Metrics, column_results: Dict
|
|
566
505
|
):
|
|
@@ -586,7 +525,6 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
586
525
|
column: Column,
|
|
587
526
|
metric: Type[HybridMetric],
|
|
588
527
|
column_results: Dict[str, Any],
|
|
589
|
-
**kwargs,
|
|
590
528
|
):
|
|
591
529
|
"""Given a list of metrics, compute the given results
|
|
592
530
|
and returns the values
|
|
@@ -598,10 +536,9 @@ class SQAProfilerInterface(ProfilerInterface, SQAInterfaceMixin):
|
|
|
598
536
|
Returns:
|
|
599
537
|
dictionnary of results
|
|
600
538
|
"""
|
|
601
|
-
|
|
602
|
-
sample = sampler.random_sample(column)
|
|
539
|
+
dataset = self.sampler.get_dataset(column=column)
|
|
603
540
|
try:
|
|
604
|
-
return metric(column).fn(
|
|
541
|
+
return metric(column).fn(dataset, column_results, self.session)
|
|
605
542
|
except Exception as exc:
|
|
606
543
|
logger.debug(traceback.format_exc())
|
|
607
544
|
logger.warning(f"Unexpected exception computing metrics: {exc}")
|
|
@@ -76,11 +76,11 @@ class SingleStoreProfilerInterface(SQAProfilerInterface):
|
|
|
76
76
|
return dict(row)
|
|
77
77
|
except ProgrammingError:
|
|
78
78
|
logger.info(
|
|
79
|
-
f"Skipping window metrics for {runner.
|
|
79
|
+
f"Skipping window metrics for {runner.table_name}.{column.name} due to overflow"
|
|
80
80
|
)
|
|
81
81
|
return None
|
|
82
82
|
|
|
83
83
|
except Exception as exc:
|
|
84
|
-
msg = f"Error trying to compute profile for {runner.
|
|
84
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
85
85
|
handle_query_exception(msg, exc, session)
|
|
86
86
|
return None
|
|
@@ -41,7 +41,7 @@ class SnowflakeProfilerInterface(SQAProfilerInterface):
|
|
|
41
41
|
session.bind.dialect.name
|
|
42
42
|
):
|
|
43
43
|
logger.info(
|
|
44
|
-
f"Computing metrics without sum for {runner.
|
|
44
|
+
f"Computing metrics without sum for {runner.table_name}.{column.name}"
|
|
45
45
|
)
|
|
46
46
|
return self._compute_static_metrics_wo_sum(metrics, runner, session, column)
|
|
47
47
|
return None
|
|
@@ -79,8 +79,8 @@ class ProfilerWithStatistics(SQAProfilerInterface, StoredStatisticsSource):
|
|
|
79
79
|
list,
|
|
80
80
|
partition(self.is_statistic_metric, metrics),
|
|
81
81
|
)
|
|
82
|
-
schema = runner.
|
|
83
|
-
table_name = runner.
|
|
82
|
+
schema = runner.schema_name
|
|
83
|
+
table_name = runner.table_name
|
|
84
84
|
logger.debug(
|
|
85
85
|
"Getting statistics for column: %s.%s.%s",
|
|
86
86
|
schema,
|
|
@@ -118,8 +118,8 @@ class ProfilerWithStatistics(SQAProfilerInterface, StoredStatisticsSource):
|
|
|
118
118
|
list,
|
|
119
119
|
partition(self.is_statistic_metric, metrics),
|
|
120
120
|
)
|
|
121
|
-
schema = runner.
|
|
122
|
-
table_name = runner.
|
|
121
|
+
schema = runner.schema_name
|
|
122
|
+
table_name = runner.table_name
|
|
123
123
|
logger.debug("Geting statistics for table: %s.%s", schema, table_name)
|
|
124
124
|
result.update(
|
|
125
125
|
super().get_table_statistics(stat_metrics, schema, table_name)
|
|
@@ -135,12 +135,10 @@ class ProfilerWithStatistics(SQAProfilerInterface, StoredStatisticsSource):
|
|
|
135
135
|
result.update(super_table_metrics)
|
|
136
136
|
return result
|
|
137
137
|
|
|
138
|
-
def get_hybrid_metrics(
|
|
139
|
-
self, column: Column, metric: Metric, column_results: Dict, **kwargs
|
|
140
|
-
):
|
|
138
|
+
def get_hybrid_metrics(self, column: Column, metric: Metric, column_results: Dict):
|
|
141
139
|
# this metrics might have been computed in a previous step
|
|
142
140
|
return column_results.get(metric.name()) or super().get_hybrid_metrics(
|
|
143
|
-
column, metric, column_results
|
|
141
|
+
column, metric, column_results
|
|
144
142
|
)
|
|
145
143
|
|
|
146
144
|
def is_statistic_metric(self, metric: Metric) -> bool:
|
|
@@ -76,11 +76,11 @@ class TrinoProfilerInterface(ProfilerWithStatistics, TrinoStoredStatisticsSource
|
|
|
76
76
|
return dict(row)
|
|
77
77
|
except ProgrammingError as err:
|
|
78
78
|
logger.info(
|
|
79
|
-
f"Skipping window metrics for {runner.
|
|
79
|
+
f"Skipping window metrics for {runner.table_name}.{column.name} due to {err}"
|
|
80
80
|
)
|
|
81
81
|
return None
|
|
82
82
|
|
|
83
83
|
except Exception as exc:
|
|
84
|
-
msg = f"Error trying to compute profile for {runner.
|
|
84
|
+
msg = f"Error trying to compute profile for {runner.table_name}.{column.name}: {exc}"
|
|
85
85
|
handle_query_exception(msg, exc, session)
|
|
86
86
|
return None
|
|
@@ -109,10 +109,29 @@ def ometa_to_sqa_orm(
|
|
|
109
109
|
We are building the class dynamically using
|
|
110
110
|
`type` and passing SQLAlchemy `Base` class
|
|
111
111
|
as the bases tuple for inheritance.
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
table (Table): OpenMetadata Table instance
|
|
115
|
+
metadata (OpenMetadata): OpenMetadata connection
|
|
116
|
+
sqa_metadata_obj (MetaData): For advanced use cases, you can pass a custom MetaData object. For most cases, this
|
|
117
|
+
can be left as None so that the global_metadata object is used.
|
|
112
118
|
"""
|
|
119
|
+
_metadata = sqa_metadata_obj or Base.metadata
|
|
113
120
|
table.serviceType = cast(
|
|
114
121
|
databaseService.DatabaseServiceType, table.serviceType
|
|
115
122
|
) # satisfy mypy
|
|
123
|
+
|
|
124
|
+
orm_database_name = get_orm_database(table, metadata)
|
|
125
|
+
# SQLite does not support schemas
|
|
126
|
+
orm_schema_name = (
|
|
127
|
+
get_orm_schema(table, metadata)
|
|
128
|
+
if table.serviceType != databaseService.DatabaseServiceType.SQLite
|
|
129
|
+
else None
|
|
130
|
+
)
|
|
131
|
+
orm_name = f"{orm_database_name}_{orm_schema_name}_{table.name.root}".replace(
|
|
132
|
+
".", "_"
|
|
133
|
+
)
|
|
134
|
+
|
|
116
135
|
cols = {
|
|
117
136
|
(
|
|
118
137
|
col.name.root + "_"
|
|
@@ -122,12 +141,6 @@ def ometa_to_sqa_orm(
|
|
|
122
141
|
for idx, col in enumerate(table.columns)
|
|
123
142
|
}
|
|
124
143
|
|
|
125
|
-
orm_database_name = get_orm_database(table, metadata)
|
|
126
|
-
orm_schema_name = get_orm_schema(table, metadata)
|
|
127
|
-
orm_name = f"{orm_database_name}_{orm_schema_name}_{table.name.root}".replace(
|
|
128
|
-
".", "_"
|
|
129
|
-
)
|
|
130
|
-
|
|
131
144
|
# Type takes positional arguments in the form of (name, bases, dict)
|
|
132
145
|
orm = type(
|
|
133
146
|
orm_name, # Output class name
|
|
@@ -135,10 +148,7 @@ def ometa_to_sqa_orm(
|
|
|
135
148
|
{
|
|
136
149
|
"__tablename__": str(table.name.root),
|
|
137
150
|
"__table_args__": {
|
|
138
|
-
|
|
139
|
-
"schema": orm_schema_name
|
|
140
|
-
if table.serviceType != databaseService.DatabaseServiceType.SQLite
|
|
141
|
-
else None,
|
|
151
|
+
"schema": orm_schema_name,
|
|
142
152
|
"extend_existing": True, # Recreates the table ORM object if it already exists. Useful for testing
|
|
143
153
|
"quote": check_snowflake_case_sensitive(
|
|
144
154
|
table.serviceType, table.name.root
|
|
@@ -146,13 +156,12 @@ def ometa_to_sqa_orm(
|
|
|
146
156
|
or None,
|
|
147
157
|
},
|
|
148
158
|
**cols,
|
|
149
|
-
"metadata":
|
|
159
|
+
"metadata": _metadata,
|
|
150
160
|
},
|
|
151
161
|
)
|
|
152
162
|
|
|
153
163
|
if not isinstance(orm, DeclarativeMeta):
|
|
154
164
|
raise ValueError("OMeta to ORM did not create a DeclarativeMeta")
|
|
155
|
-
|
|
156
165
|
return orm
|
|
157
166
|
|
|
158
167
|
|
|
@@ -53,7 +53,7 @@ class AbstractTableMetricComputer(ABC):
|
|
|
53
53
|
self._metrics = metrics
|
|
54
54
|
self._conn_config = conn_config
|
|
55
55
|
self._database = self._runner._session.get_bind().url.database
|
|
56
|
-
self._table = self._runner.
|
|
56
|
+
self._table = self._runner.dataset
|
|
57
57
|
self._entity = entity
|
|
58
58
|
|
|
59
59
|
@property
|
|
@@ -91,8 +91,8 @@ class AbstractTableMetricComputer(ABC):
|
|
|
91
91
|
table (DeclarativeMeta): _description_
|
|
92
92
|
"""
|
|
93
93
|
try:
|
|
94
|
-
self._schema_name = self.
|
|
95
|
-
self._table_name = self.
|
|
94
|
+
self._schema_name = self.runner.schema_name
|
|
95
|
+
self._table_name = self.runner.table_name
|
|
96
96
|
except AttributeError:
|
|
97
97
|
raise AttributeError(ERROR_MSG)
|
|
98
98
|
|
|
@@ -119,10 +119,10 @@ class AbstractTableMetricComputer(ABC):
|
|
|
119
119
|
Returns:
|
|
120
120
|
Tuple[str, int]
|
|
121
121
|
"""
|
|
122
|
-
col_names = literal(
|
|
123
|
-
|
|
124
|
-
)
|
|
125
|
-
col_count = literal(len(inspect(self.
|
|
122
|
+
col_names = literal(
|
|
123
|
+
",".join(inspect(self.runner.raw_dataset).c.keys()), type_=String
|
|
124
|
+
).label(COLUMN_NAMES)
|
|
125
|
+
col_count = literal(len(inspect(self.runner.raw_dataset).c)).label(COLUMN_COUNT)
|
|
126
126
|
return col_names, col_count
|
|
127
127
|
|
|
128
128
|
def _build_query(
|
|
@@ -147,7 +147,8 @@ class BaseTableMetricComputer(AbstractTableMetricComputer):
|
|
|
147
147
|
"""Base table computer"""
|
|
148
148
|
|
|
149
149
|
def compute(self):
|
|
150
|
-
"""Default compute behavior for table metrics
|
|
150
|
+
"""Default compute behavior for table metrics. This method will use the raw table
|
|
151
|
+
to compute metrics and omit any sampling or partitioning logic."""
|
|
151
152
|
return self.runner.select_first_from_table(
|
|
152
153
|
*[metric().fn() for metric in self.metrics]
|
|
153
154
|
)
|
|
@@ -40,7 +40,7 @@ from metadata.generated.schema.tests.customMetric import (
|
|
|
40
40
|
CustomMetric as CustomMetricEntity,
|
|
41
41
|
)
|
|
42
42
|
from metadata.generated.schema.type.basic import Timestamp
|
|
43
|
-
from metadata.profiler.api.models import ProfilerResponse,
|
|
43
|
+
from metadata.profiler.api.models import ProfilerResponse, ThreadPoolMetrics
|
|
44
44
|
from metadata.profiler.interface.profiler_interface import ProfilerInterface
|
|
45
45
|
from metadata.profiler.metrics.core import (
|
|
46
46
|
ComposedMetric,
|
|
@@ -54,9 +54,6 @@ from metadata.profiler.metrics.static.row_count import RowCount
|
|
|
54
54
|
from metadata.profiler.orm.functions.table_metric_computer import CREATE_DATETIME
|
|
55
55
|
from metadata.profiler.orm.registry import NOT_COMPUTE
|
|
56
56
|
from metadata.profiler.processor.metric_filter import MetricFilter
|
|
57
|
-
from metadata.profiler.processor.sample_data_handler import upload_sample_data
|
|
58
|
-
from metadata.utils.constants import SAMPLE_DATA_DEFAULT_COUNT
|
|
59
|
-
from metadata.utils.execution_time_tracker import calculate_execution_time
|
|
60
57
|
from metadata.utils.logger import profiler_logger
|
|
61
58
|
|
|
62
59
|
logger = profiler_logger()
|
|
@@ -107,7 +104,6 @@ class Profiler(Generic[TMetric]):
|
|
|
107
104
|
self.exclude_columns = exclude_columns
|
|
108
105
|
self._metrics = metrics
|
|
109
106
|
self._profile_ts = Timestamp(int(datetime.now().timestamp() * 1000))
|
|
110
|
-
self.profile_sample_config = self.profiler_interface.profile_sample_config
|
|
111
107
|
|
|
112
108
|
self.metric_filter = MetricFilter(
|
|
113
109
|
metrics=self.metrics,
|
|
@@ -249,11 +245,6 @@ class Profiler(Generic[TMetric]):
|
|
|
249
245
|
return column.customMetrics or None
|
|
250
246
|
return None
|
|
251
247
|
|
|
252
|
-
@property
|
|
253
|
-
def sample(self):
|
|
254
|
-
"""Return the sample used for the profiler"""
|
|
255
|
-
return self.profiler_interface.sample
|
|
256
|
-
|
|
257
248
|
def validate_composed_metric(self) -> None:
|
|
258
249
|
"""
|
|
259
250
|
Make sure that all composed metrics have
|
|
@@ -317,7 +308,6 @@ class Profiler(Generic[TMetric]):
|
|
|
317
308
|
col,
|
|
318
309
|
metric,
|
|
319
310
|
current_col_results,
|
|
320
|
-
table=self.table,
|
|
321
311
|
)
|
|
322
312
|
|
|
323
313
|
def _prepare_table_metrics(self) -> List:
|
|
@@ -495,16 +485,6 @@ class Profiler(Generic[TMetric]):
|
|
|
495
485
|
)
|
|
496
486
|
self.compute_metrics()
|
|
497
487
|
|
|
498
|
-
# We need the sample data for Sample Data or PII Sensitive processing.
|
|
499
|
-
# We'll nullify the Sample Data after the PII processing so that it's not stored.
|
|
500
|
-
if (
|
|
501
|
-
self.source_config.generateSampleData
|
|
502
|
-
or self.source_config.processPiiSensitive
|
|
503
|
-
):
|
|
504
|
-
sample_data = self.generate_sample_data()
|
|
505
|
-
else:
|
|
506
|
-
sample_data = None
|
|
507
|
-
|
|
508
488
|
profile = self.get_profile()
|
|
509
489
|
if self.source_config.computeMetrics:
|
|
510
490
|
self._check_profile_and_handle(profile)
|
|
@@ -512,43 +492,10 @@ class Profiler(Generic[TMetric]):
|
|
|
512
492
|
table_profile = ProfilerResponse(
|
|
513
493
|
table=self.profiler_interface.table_entity,
|
|
514
494
|
profile=profile,
|
|
515
|
-
sample_data=sample_data,
|
|
516
495
|
)
|
|
517
496
|
|
|
518
497
|
return table_profile
|
|
519
498
|
|
|
520
|
-
@calculate_execution_time(store=False)
|
|
521
|
-
def generate_sample_data(self) -> Optional[SampleData]:
|
|
522
|
-
"""Fetch and ingest sample data
|
|
523
|
-
|
|
524
|
-
Returns:
|
|
525
|
-
TableData: sample data
|
|
526
|
-
"""
|
|
527
|
-
try:
|
|
528
|
-
logger.debug(
|
|
529
|
-
"Fetching sample data for "
|
|
530
|
-
f"{self.profiler_interface.table_entity.fullyQualifiedName.root}..." # type: ignore
|
|
531
|
-
)
|
|
532
|
-
table_data = self.profiler_interface.fetch_sample_data(
|
|
533
|
-
self.table, self.columns
|
|
534
|
-
)
|
|
535
|
-
upload_sample_data(
|
|
536
|
-
data=table_data, profiler_interface=self.profiler_interface
|
|
537
|
-
)
|
|
538
|
-
table_data.rows = table_data.rows[
|
|
539
|
-
: min(
|
|
540
|
-
SAMPLE_DATA_DEFAULT_COUNT, self.profiler_interface.sample_data_count
|
|
541
|
-
)
|
|
542
|
-
]
|
|
543
|
-
return SampleData(
|
|
544
|
-
data=table_data, store=self.source_config.generateSampleData
|
|
545
|
-
)
|
|
546
|
-
|
|
547
|
-
except Exception as err:
|
|
548
|
-
logger.debug(traceback.format_exc())
|
|
549
|
-
logger.warning(f"Error fetching sample data: {err}")
|
|
550
|
-
return None
|
|
551
|
-
|
|
552
499
|
def get_profile(self) -> CreateTableProfileRequest:
|
|
553
500
|
"""
|
|
554
501
|
After executing the profiler, get all results
|
|
@@ -603,13 +550,13 @@ class Profiler(Generic[TMetric]):
|
|
|
603
550
|
createDateTime=raw_create_date,
|
|
604
551
|
sizeInByte=self._table_results.get("sizeInBytes"),
|
|
605
552
|
profileSample=(
|
|
606
|
-
self.
|
|
607
|
-
if self.
|
|
553
|
+
self.profiler_interface.sampler.sample_config.profile_sample
|
|
554
|
+
if self.profiler_interface.sampler.sample_config
|
|
608
555
|
else None
|
|
609
556
|
),
|
|
610
557
|
profileSampleType=(
|
|
611
|
-
self.
|
|
612
|
-
if self.
|
|
558
|
+
self.profiler_interface.sampler.sample_config.profile_sample_type
|
|
559
|
+
if self.profiler_interface.sampler.sample_config
|
|
613
560
|
else None
|
|
614
561
|
),
|
|
615
562
|
customMetrics=self._table_results.get("customMetrics"),
|