openmetadata-ingestion 1.2.4.0__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openmetadata-ingestion might be problematic. Click here for more details.
- airflow_provider_openmetadata/lineage/runner.py +44 -5
- airflow_provider_openmetadata/lineage/status.py +1 -1
- metadata/antlr/split_listener.py +1 -4
- metadata/applications/auto_tagger.py +212 -0
- metadata/cli/app.py +47 -0
- metadata/cli/db_dump.py +7 -3
- metadata/clients/aws_client.py +4 -0
- metadata/clients/azure_client.py +85 -0
- metadata/clients/domo_client.py +0 -2
- metadata/cmd.py +24 -13
- metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py +40 -48
- metadata/data_insight/processor/reports/data_processor.py +1 -0
- metadata/data_insight/processor/reports/entity_report_data_processor.py +8 -2
- metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +4 -0
- metadata/data_insight/producer/cost_analysis_producer.py +78 -14
- metadata/data_insight/producer/entity_producer.py +1 -1
- metadata/data_insight/producer/producer_interface.py +1 -1
- metadata/data_insight/producer/web_analytics_producer.py +1 -1
- metadata/data_insight/source/metadata.py +21 -5
- metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +29 -0
- metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +29 -0
- metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +6 -6
- metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +35 -0
- metadata/data_quality/interface/test_suite_interface_factory.py +40 -0
- metadata/data_quality/processor/test_case_runner.py +17 -8
- metadata/data_quality/source/test_suite.py +10 -4
- metadata/data_quality/validations/base_test_handler.py +21 -17
- metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +38 -1
- metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +36 -1
- metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +15 -0
- metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +27 -0
- metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +23 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +23 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +27 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +23 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +11 -0
- metadata/data_quality/validations/mixins/pandas_validator_mixin.py +13 -2
- metadata/data_quality/validations/mixins/sqa_validator_mixin.py +40 -0
- metadata/data_quality/validations/table/base/tableColumnToMatchSet.py +2 -1
- metadata/data_quality/validations/table/pandas/tableColumnToMatchSet.py +2 -1
- metadata/data_quality/validations/table/sqlalchemy/tableColumnToMatchSet.py +7 -2
- metadata/examples/workflows/bigtable.yaml +32 -0
- metadata/examples/workflows/databricks.yaml +0 -1
- metadata/examples/workflows/datalake_azure_default.yaml +29 -0
- metadata/examples/workflows/mongodb.yaml +4 -2
- metadata/examples/workflows/mstr.yaml +24 -0
- metadata/examples/workflows/sas.yaml +28 -0
- metadata/examples/workflows/unity_catalog.yaml +27 -0
- metadata/examples/workflows/unity_catalog_lineage.yaml +18 -0
- metadata/examples/workflows/unity_catalog_usage.yaml +35 -0
- metadata/generated/antlr/EntityLinkLexer.py +425 -292
- metadata/generated/antlr/EntityLinkListener.py +12 -12
- metadata/generated/antlr/EntityLinkParser.py +228 -103
- metadata/generated/schema/analytics/__init__.py +1 -1
- metadata/generated/schema/analytics/basic.py +1 -1
- metadata/generated/schema/analytics/reportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
- metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
- metadata/generated/schema/api/__init__.py +1 -1
- metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
- metadata/generated/schema/api/analytics/__init__.py +1 -1
- metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
- metadata/generated/schema/api/automations/__init__.py +1 -1
- metadata/generated/schema/api/automations/createWorkflow.py +1 -1
- metadata/generated/schema/api/bulkAssets.py +1 -1
- metadata/generated/schema/api/classification/__init__.py +1 -1
- metadata/generated/schema/api/classification/createClassification.py +1 -1
- metadata/generated/schema/api/classification/createTag.py +1 -1
- metadata/generated/schema/api/classification/loadTags.py +1 -1
- metadata/generated/schema/api/createBot.py +2 -3
- metadata/generated/schema/api/createEventPublisherJob.py +2 -2
- metadata/generated/schema/api/createType.py +1 -1
- metadata/generated/schema/api/data/__init__.py +1 -1
- metadata/generated/schema/api/data/createChart.py +1 -1
- metadata/generated/schema/api/data/createContainer.py +2 -2
- metadata/generated/schema/api/data/createCustomProperty.py +7 -12
- metadata/generated/schema/api/data/createDashboard.py +1 -1
- metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
- metadata/generated/schema/api/data/createDatabase.py +2 -3
- metadata/generated/schema/api/data/createDatabaseSchema.py +2 -3
- metadata/generated/schema/api/data/createGlossary.py +2 -3
- metadata/generated/schema/api/data/createGlossaryTerm.py +2 -3
- metadata/generated/schema/api/data/createMlModel.py +1 -1
- metadata/generated/schema/api/data/createPipeline.py +2 -2
- metadata/generated/schema/api/data/createQuery.py +2 -2
- metadata/generated/schema/api/data/createSearchIndex.py +1 -1
- metadata/generated/schema/api/data/createStoredProcedure.py +2 -4
- metadata/generated/schema/api/data/createTable.py +2 -2
- metadata/generated/schema/api/data/createTableProfile.py +1 -1
- metadata/generated/schema/api/data/createTopic.py +1 -1
- metadata/generated/schema/api/data/loadGlossary.py +1 -1
- metadata/generated/schema/api/data/restoreEntity.py +1 -1
- metadata/generated/schema/api/dataInsight/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
- metadata/generated/schema/api/docStore/__init__.py +1 -1
- metadata/generated/schema/api/docStore/createDocument.py +1 -1
- metadata/generated/schema/api/domains/__init__.py +1 -1
- metadata/generated/schema/api/domains/createDataProduct.py +1 -1
- metadata/generated/schema/api/domains/createDomain.py +1 -1
- metadata/generated/schema/api/feed/__init__.py +1 -1
- metadata/generated/schema/api/feed/closeTask.py +8 -1
- metadata/generated/schema/api/feed/createPost.py +1 -1
- metadata/generated/schema/api/feed/createSuggestion.py +30 -0
- metadata/generated/schema/api/feed/createThread.py +5 -1
- metadata/generated/schema/api/feed/resolveTask.py +12 -1
- metadata/generated/schema/api/feed/threadCount.py +18 -14
- metadata/generated/schema/api/lineage/__init__.py +1 -1
- metadata/generated/schema/api/lineage/addLineage.py +1 -1
- metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
- metadata/generated/schema/api/policies/__init__.py +1 -1
- metadata/generated/schema/api/policies/createPolicy.py +1 -1
- metadata/generated/schema/api/services/__init__.py +1 -1
- metadata/generated/schema/api/services/createDashboardService.py +1 -1
- metadata/generated/schema/api/services/createDatabaseService.py +1 -1
- metadata/generated/schema/api/services/createMessagingService.py +1 -1
- metadata/generated/schema/api/services/createMetadataService.py +1 -1
- metadata/generated/schema/api/services/createMlModelService.py +1 -1
- metadata/generated/schema/api/services/createPipelineService.py +1 -1
- metadata/generated/schema/api/services/createSearchService.py +1 -1
- metadata/generated/schema/api/services/createStorageService.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
- metadata/generated/schema/api/setOwner.py +1 -1
- metadata/generated/schema/api/teams/__init__.py +1 -1
- metadata/generated/schema/api/teams/createPersona.py +1 -1
- metadata/generated/schema/api/teams/createRole.py +2 -3
- metadata/generated/schema/api/teams/createTeam.py +1 -1
- metadata/generated/schema/api/teams/createUser.py +2 -2
- metadata/generated/schema/api/tests/__init__.py +1 -1
- metadata/generated/schema/api/tests/createCustomMetric.py +4 -2
- metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
- metadata/generated/schema/api/tests/createTestCase.py +4 -1
- metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +31 -0
- metadata/generated/schema/api/tests/createTestDefinition.py +2 -1
- metadata/generated/schema/api/tests/createTestSuite.py +1 -1
- metadata/generated/schema/api/voteRequest.py +1 -1
- metadata/generated/schema/auth/__init__.py +1 -1
- metadata/generated/schema/auth/basicAuth.py +1 -1
- metadata/generated/schema/auth/basicLoginRequest.py +1 -1
- metadata/generated/schema/auth/changePasswordRequest.py +1 -1
- metadata/generated/schema/auth/createPersonalToken.py +1 -1
- metadata/generated/schema/auth/emailRequest.py +1 -1
- metadata/generated/schema/auth/emailVerificationToken.py +1 -1
- metadata/generated/schema/auth/generateToken.py +1 -1
- metadata/generated/schema/auth/jwtAuth.py +1 -1
- metadata/generated/schema/auth/loginRequest.py +1 -1
- metadata/generated/schema/auth/logoutRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetRequest.py +3 -3
- metadata/generated/schema/auth/passwordResetToken.py +1 -1
- metadata/generated/schema/auth/personalAccessToken.py +1 -1
- metadata/generated/schema/auth/refreshToken.py +1 -1
- metadata/generated/schema/auth/registrationRequest.py +2 -2
- metadata/generated/schema/auth/revokePersonalToken.py +1 -1
- metadata/generated/schema/auth/revokeToken.py +1 -1
- metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
- metadata/generated/schema/auth/ssoAuth.py +1 -1
- metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
- metadata/generated/schema/configuration/__init__.py +1 -1
- metadata/generated/schema/configuration/appsPrivateConfiguration.py +32 -0
- metadata/generated/schema/configuration/authConfig.py +1 -1
- metadata/generated/schema/configuration/authenticationConfiguration.py +13 -2
- metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
- metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/dataQualityConfiguration.py +16 -0
- metadata/generated/schema/configuration/elasticSearchConfiguration.py +7 -1
- metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
- metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
- metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
- metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapConfiguration.py +28 -2
- metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
- metadata/generated/schema/configuration/loginConfiguration.py +1 -1
- metadata/generated/schema/configuration/logoConfiguration.py +1 -1
- metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
- metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
- metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
- metadata/generated/schema/dataInsight/__init__.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
- metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
- metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
- metadata/generated/schema/dataInsight/type/__init__.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +1 -1
- metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +1 -1
- metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +1 -1
- metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
- metadata/generated/schema/email/__init__.py +1 -1
- metadata/generated/schema/email/emailRequest.py +1 -1
- metadata/generated/schema/email/smtpSettings.py +1 -1
- metadata/generated/schema/entity/__init__.py +1 -1
- metadata/generated/schema/entity/applications/__init__.py +1 -1
- metadata/generated/schema/entity/applications/app.py +24 -5
- metadata/generated/schema/entity/applications/appRunRecord.py +2 -7
- metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/applicationConfig.py +30 -0
- metadata/generated/schema/entity/{services/connections/database/mongoDB → applications/configuration/external}/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/autoTaggerAppConfig.py +28 -0
- metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +46 -0
- metadata/generated/schema/entity/applications/configuration/internal/__init__.py +3 -0
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +14 -0
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +27 -0
- metadata/generated/schema/entity/applications/configuration/{searchIndexingApp.py → internal/searchIndexingAppConfig.py} +14 -4
- metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +3 -0
- metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +26 -0
- metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
- metadata/generated/schema/entity/applications/jobStatus.py +3 -3
- metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +12 -3
- metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +12 -3
- metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
- metadata/generated/schema/entity/automations/__init__.py +1 -1
- metadata/generated/schema/entity/automations/testServiceConnection.py +2 -2
- metadata/generated/schema/entity/automations/workflow.py +1 -1
- metadata/generated/schema/entity/bot.py +2 -3
- metadata/generated/schema/entity/classification/__init__.py +1 -1
- metadata/generated/schema/entity/classification/classification.py +1 -1
- metadata/generated/schema/entity/classification/tag.py +2 -2
- metadata/generated/schema/entity/data/__init__.py +1 -1
- metadata/generated/schema/entity/data/chart.py +1 -1
- metadata/generated/schema/entity/data/container.py +4 -9
- metadata/generated/schema/entity/data/dashboard.py +1 -1
- metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
- metadata/generated/schema/entity/data/database.py +4 -8
- metadata/generated/schema/entity/data/databaseSchema.py +2 -8
- metadata/generated/schema/entity/data/glossary.py +1 -1
- metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
- metadata/generated/schema/entity/data/metrics.py +1 -1
- metadata/generated/schema/entity/data/mlmodel.py +2 -2
- metadata/generated/schema/entity/data/pipeline.py +6 -9
- metadata/generated/schema/entity/data/query.py +2 -2
- metadata/generated/schema/entity/data/report.py +1 -1
- metadata/generated/schema/entity/data/searchIndex.py +1 -1
- metadata/generated/schema/entity/data/storedProcedure.py +3 -9
- metadata/generated/schema/entity/data/table.py +18 -12
- metadata/generated/schema/entity/data/topic.py +1 -1
- metadata/generated/schema/entity/docStore/__init__.py +1 -1
- metadata/generated/schema/entity/docStore/document.py +1 -1
- metadata/generated/schema/entity/domains/__init__.py +1 -1
- metadata/generated/schema/entity/domains/dataProduct.py +1 -1
- metadata/generated/schema/entity/domains/domain.py +1 -1
- metadata/generated/schema/entity/events/__init__.py +1 -1
- metadata/generated/schema/entity/events/webhook.py +6 -3
- metadata/generated/schema/entity/feed/__init__.py +1 -1
- metadata/generated/schema/entity/feed/suggestion.py +61 -0
- metadata/generated/schema/entity/feed/thread.py +17 -1
- metadata/generated/schema/entity/policies/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
- metadata/generated/schema/entity/policies/filters.py +2 -5
- metadata/generated/schema/entity/policies/policy.py +1 -1
- metadata/generated/schema/entity/services/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +46 -0
- metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/athenaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +35 -2
- metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +41 -0
- metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +20 -0
- metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +6 -1
- metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricksConnection.py +2 -7
- metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/db2Connection.py +3 -2
- metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dorisConnection.py +86 -0
- metadata/generated/schema/entity/services/connections/database/druidConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/hiveConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +3 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +17 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +16 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +23 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +43 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +20 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +69 -0
- metadata/generated/schema/entity/services/connections/database/icebergConnection.py +35 -0
- metadata/generated/schema/entity/services/connections/database/impalaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +24 -14
- metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -2
- metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +8 -4
- metadata/generated/schema/entity/services/connections/database/oracleConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +9 -7
- metadata/generated/schema/entity/services/connections/database/postgresConnection.py +8 -4
- metadata/generated/schema/entity/services/connections/database/prestoConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +3 -0
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +20 -0
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +36 -0
- metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +7 -41
- metadata/generated/schema/entity/services/connections/database/sasConnection.py +48 -0
- metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/trinoConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +90 -0
- metadata/generated/schema/entity/services/connections/database/verticaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +14 -2
- metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +14 -2
- metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +5 -20
- metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +2 -8
- metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +23 -0
- metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/{adlsConection.py → adlsConnection.py} +2 -2
- metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
- metadata/generated/schema/entity/services/dashboardService.py +4 -1
- metadata/generated/schema/entity/services/databaseService.py +16 -1
- metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +27 -22
- metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/status.py +56 -0
- metadata/generated/schema/entity/services/messagingService.py +1 -1
- metadata/generated/schema/entity/services/metadataService.py +1 -1
- metadata/generated/schema/entity/services/mlmodelService.py +1 -1
- metadata/generated/schema/entity/services/pipelineService.py +4 -1
- metadata/generated/schema/entity/services/searchService.py +1 -1
- metadata/generated/schema/entity/services/serviceType.py +1 -1
- metadata/generated/schema/entity/services/storageService.py +10 -3
- metadata/generated/schema/entity/teams/__init__.py +1 -1
- metadata/generated/schema/entity/teams/persona.py +1 -1
- metadata/generated/schema/entity/teams/role.py +2 -6
- metadata/generated/schema/entity/teams/team.py +1 -1
- metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
- metadata/generated/schema/entity/teams/user.py +3 -10
- metadata/generated/schema/entity/type.py +4 -19
- metadata/generated/schema/entity/utils/__init__.py +1 -1
- metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
- metadata/generated/schema/entity/utils/servicesCount.py +1 -1
- metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
- metadata/generated/schema/events/__init__.py +1 -1
- metadata/generated/schema/events/alertMetrics.py +27 -0
- metadata/generated/schema/events/api/__init__.py +1 -1
- metadata/generated/schema/events/api/createEventSubscription.py +14 -12
- metadata/generated/schema/events/emailAlertConfig.py +1 -1
- metadata/generated/schema/events/eventFilterRule.py +13 -2
- metadata/generated/schema/events/eventSubscription.py +91 -49
- metadata/generated/schema/events/eventSubscriptionOffset.py +21 -0
- metadata/generated/schema/events/failedEvent.py +26 -0
- metadata/generated/schema/events/filterResourceDescriptor.py +27 -0
- metadata/generated/schema/events/subscriptionResourceDescriptor.py +11 -10
- metadata/generated/schema/metadataIngestion/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/application.py +40 -0
- metadata/generated/schema/metadataIngestion/applicationPipeline.py +15 -6
- metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +6 -1
- metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +11 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +6 -3
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +6 -3
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +6 -3
- metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +2 -2
- metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +24 -0
- metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +8 -1
- metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/workflow.py +1 -1
- metadata/generated/schema/monitoring/__init__.py +1 -1
- metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
- metadata/generated/schema/security/__init__.py +1 -1
- metadata/generated/schema/security/client/__init__.py +1 -1
- metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/oidcClientConfig.py +46 -0
- metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
- metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
- metadata/generated/schema/security/credentials/__init__.py +1 -1
- metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
- metadata/generated/schema/security/credentials/azureCredentials.py +21 -9
- metadata/generated/schema/security/credentials/basicAuth.py +1 -1
- metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpCredentials.py +7 -3
- metadata/generated/schema/security/credentials/gcpExternalAccount.py +37 -0
- metadata/generated/schema/security/credentials/gcpValues.py +2 -2
- metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
- metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
- metadata/generated/schema/security/secrets/__init__.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +14 -4
- metadata/generated/schema/security/secrets/secretsManagerProvider.py +4 -2
- metadata/generated/schema/security/securityConfiguration.py +1 -1
- metadata/generated/schema/security/ssl/__init__.py +1 -1
- metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
- metadata/generated/schema/security/ssl/verifySSLConfig.py +4 -1
- metadata/generated/schema/settings/__init__.py +1 -1
- metadata/generated/schema/settings/settings.py +1 -1
- metadata/generated/schema/system/__init__.py +1 -1
- metadata/generated/schema/system/entityError.py +17 -0
- metadata/generated/schema/system/eventPublisherJob.py +5 -19
- metadata/generated/schema/system/indexingError.py +34 -0
- metadata/generated/schema/system/ui/__init__.py +1 -1
- metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
- metadata/generated/schema/system/ui/page.py +1 -1
- metadata/generated/schema/system/validationResponse.py +43 -0
- metadata/generated/schema/tests/__init__.py +1 -1
- metadata/generated/schema/tests/assigned.py +18 -0
- metadata/generated/schema/tests/basic.py +17 -39
- metadata/generated/schema/tests/customMetric.py +4 -2
- metadata/generated/schema/tests/resolved.py +35 -0
- metadata/generated/schema/tests/testCase.py +11 -2
- metadata/generated/schema/tests/testCaseResolutionStatus.py +63 -0
- metadata/generated/schema/tests/testDefinition.py +6 -1
- metadata/generated/schema/tests/testSuite.py +1 -1
- metadata/generated/schema/type/__init__.py +1 -1
- metadata/generated/schema/type/auditLog.py +1 -1
- metadata/generated/schema/type/basic.py +10 -6
- metadata/generated/schema/type/bulkOperationResult.py +1 -1
- metadata/generated/schema/type/changeEvent.py +5 -75
- metadata/generated/schema/type/changeEventType.py +29 -0
- metadata/generated/schema/type/collectionDescriptor.py +1 -1
- metadata/generated/schema/type/csvDocumentation.py +1 -1
- metadata/generated/schema/type/csvErrorType.py +1 -1
- metadata/generated/schema/type/csvFile.py +1 -1
- metadata/generated/schema/type/csvImportResult.py +1 -1
- metadata/generated/schema/type/customProperties/__init__.py +3 -0
- metadata/generated/schema/type/customProperties/enumConfig.py +17 -0
- metadata/generated/schema/type/customProperty.py +52 -0
- metadata/generated/schema/type/dailyCount.py +1 -1
- metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
- metadata/generated/schema/type/entityHistory.py +1 -1
- metadata/generated/schema/type/entityLineage.py +2 -1
- metadata/generated/schema/type/entityReference.py +5 -1
- metadata/generated/schema/type/entityReferenceList.py +1 -1
- metadata/generated/schema/type/entityRelationship.py +1 -1
- metadata/generated/schema/type/entityUsage.py +1 -1
- metadata/generated/schema/type/filterPattern.py +1 -1
- metadata/generated/schema/type/function.py +7 -6
- metadata/generated/schema/type/include.py +1 -1
- metadata/generated/schema/type/jdbcConnection.py +1 -1
- metadata/generated/schema/type/lifeCycle.py +1 -1
- metadata/generated/schema/type/paging.py +1 -1
- metadata/generated/schema/type/profile.py +1 -1
- metadata/generated/schema/type/queryParserData.py +2 -2
- metadata/generated/schema/type/reaction.py +1 -1
- metadata/generated/schema/type/schedule.py +1 -1
- metadata/generated/schema/type/schema.py +1 -1
- metadata/generated/schema/type/tableQuery.py +2 -2
- metadata/generated/schema/type/tableUsageCount.py +1 -1
- metadata/generated/schema/type/tagLabel.py +1 -1
- metadata/generated/schema/type/usageDetails.py +1 -1
- metadata/generated/schema/type/usageRequest.py +1 -1
- metadata/generated/schema/type/votes.py +1 -1
- metadata/great_expectations/action.py +12 -20
- metadata/ingestion/api/delete.py +5 -2
- metadata/ingestion/api/models.py +4 -10
- metadata/ingestion/api/parser.py +3 -37
- metadata/ingestion/api/status.py +15 -4
- metadata/ingestion/api/step.py +39 -6
- metadata/ingestion/api/steps.py +34 -1
- metadata/ingestion/api/topology_runner.py +117 -168
- metadata/ingestion/bulksink/metadata_usage.py +11 -5
- metadata/ingestion/lineage/models.py +4 -0
- metadata/ingestion/lineage/parser.py +28 -12
- metadata/ingestion/lineage/sql_lineage.py +9 -17
- metadata/ingestion/models/custom_properties.py +0 -1
- metadata/ingestion/models/custom_pydantic.py +4 -2
- metadata/ingestion/models/lf_tags_model.py +33 -0
- metadata/ingestion/models/patch_request.py +160 -2
- metadata/ingestion/models/tests_data.py +9 -0
- metadata/ingestion/models/topology.py +177 -60
- metadata/ingestion/ometa/auth_provider.py +0 -349
- metadata/ingestion/ometa/client.py +12 -3
- metadata/ingestion/ometa/mixins/custom_property_mixin.py +11 -11
- metadata/ingestion/ometa/mixins/es_mixin.py +17 -0
- metadata/ingestion/ometa/mixins/patch_mixin.py +21 -72
- metadata/ingestion/ometa/mixins/suggestions_mixin.py +41 -0
- metadata/ingestion/ometa/mixins/table_mixin.py +18 -0
- metadata/ingestion/ometa/mixins/tests_mixin.py +24 -3
- metadata/ingestion/ometa/mixins/user_mixin.py +117 -22
- metadata/ingestion/ometa/ometa_api.py +19 -26
- metadata/ingestion/ometa/routes.py +7 -0
- metadata/ingestion/processor/query_parser.py +9 -2
- metadata/ingestion/sink/metadata_rest.py +29 -8
- metadata/ingestion/source/dashboard/dashboard_service.py +38 -61
- metadata/ingestion/source/dashboard/domodashboard/metadata.py +13 -14
- metadata/ingestion/source/dashboard/lightdash/metadata.py +2 -1
- metadata/ingestion/source/dashboard/looker/metadata.py +41 -29
- metadata/ingestion/source/dashboard/looker/utils.py +4 -2
- metadata/ingestion/source/dashboard/metabase/client.py +4 -0
- metadata/ingestion/source/dashboard/metabase/metadata.py +28 -10
- metadata/ingestion/source/dashboard/metabase/models.py +2 -2
- metadata/ingestion/source/dashboard/mode/client.py +10 -23
- metadata/ingestion/source/dashboard/mode/connection.py +6 -1
- metadata/ingestion/source/dashboard/mode/metadata.py +8 -4
- metadata/ingestion/source/dashboard/mstr/client.py +208 -0
- metadata/ingestion/source/dashboard/mstr/connection.py +53 -0
- metadata/ingestion/source/dashboard/mstr/metadata.py +182 -0
- metadata/ingestion/source/dashboard/mstr/models.py +144 -0
- metadata/ingestion/source/dashboard/powerbi/client.py +4 -1
- metadata/ingestion/source/dashboard/powerbi/metadata.py +15 -10
- metadata/ingestion/source/dashboard/qliksense/metadata.py +11 -7
- metadata/ingestion/source/dashboard/quicksight/metadata.py +9 -5
- metadata/ingestion/source/dashboard/redash/metadata.py +23 -14
- metadata/ingestion/source/dashboard/superset/api_source.py +11 -8
- metadata/ingestion/source/dashboard/superset/client.py +16 -9
- metadata/ingestion/source/dashboard/superset/connection.py +3 -3
- metadata/ingestion/source/dashboard/superset/db_source.py +14 -11
- metadata/ingestion/source/dashboard/superset/mixin.py +22 -18
- metadata/ingestion/source/dashboard/superset/queries.py +1 -1
- metadata/ingestion/source/dashboard/tableau/client.py +91 -11
- metadata/ingestion/source/dashboard/tableau/connection.py +10 -1
- metadata/ingestion/source/dashboard/tableau/metadata.py +76 -70
- metadata/ingestion/source/dashboard/tableau/models.py +0 -8
- metadata/ingestion/source/dashboard/tableau/queries.py +5 -5
- metadata/ingestion/source/database/athena/client.py +80 -0
- metadata/ingestion/source/database/athena/connection.py +7 -0
- metadata/ingestion/source/database/athena/metadata.py +161 -19
- metadata/ingestion/source/database/azuresql/connection.py +21 -3
- metadata/ingestion/source/database/azuresql/metadata.py +0 -1
- metadata/ingestion/source/database/bigquery/connection.py +24 -3
- metadata/ingestion/source/database/bigquery/helper.py +68 -1
- metadata/ingestion/source/database/bigquery/metadata.py +49 -28
- metadata/ingestion/source/database/bigquery/queries.py +33 -4
- metadata/ingestion/source/database/bigquery/query_parser.py +13 -0
- metadata/ingestion/source/database/bigquery/usage.py +1 -3
- metadata/ingestion/source/database/bigtable/client.py +62 -0
- metadata/ingestion/source/database/bigtable/connection.py +116 -0
- metadata/ingestion/source/database/bigtable/metadata.py +224 -0
- metadata/ingestion/source/database/bigtable/models.py +60 -0
- metadata/ingestion/source/database/column_helpers.py +0 -10
- metadata/ingestion/source/database/column_type_parser.py +11 -5
- metadata/ingestion/source/database/common_db_source.py +33 -8
- metadata/ingestion/source/database/common_nosql_source.py +27 -6
- metadata/ingestion/source/database/database_service.py +89 -7
- metadata/ingestion/source/database/databricks/client.py +55 -103
- metadata/ingestion/source/database/databricks/connection.py +16 -55
- metadata/ingestion/source/database/databricks/lineage.py +29 -26
- metadata/ingestion/source/database/databricks/metadata.py +534 -11
- metadata/ingestion/source/database/databricks/queries.py +27 -0
- metadata/ingestion/source/database/databricks/query_parser.py +5 -1
- metadata/ingestion/source/database/databricks/usage.py +3 -3
- metadata/ingestion/source/database/datalake/connection.py +33 -18
- metadata/ingestion/source/database/datalake/metadata.py +113 -27
- metadata/ingestion/source/database/dbt/dbt_config.py +7 -14
- metadata/ingestion/source/database/dbt/dbt_service.py +10 -14
- metadata/ingestion/source/database/dbt/dbt_utils.py +3 -1
- metadata/ingestion/source/database/dbt/metadata.py +27 -53
- metadata/ingestion/source/database/deltalake/metadata.py +6 -3
- metadata/ingestion/source/database/domodatabase/metadata.py +7 -6
- metadata/ingestion/source/database/doris/connection.py +72 -0
- metadata/ingestion/source/database/doris/metadata.py +315 -0
- metadata/ingestion/source/database/doris/queries.py +54 -0
- metadata/ingestion/source/database/doris/utils.py +64 -0
- metadata/ingestion/source/database/extended_sample_data.py +532 -0
- metadata/ingestion/source/database/glue/metadata.py +8 -5
- metadata/ingestion/source/database/hive/connection.py +0 -2
- metadata/ingestion/source/database/hive/utils.py +3 -0
- metadata/ingestion/source/database/iceberg/catalog/__init__.py +65 -0
- metadata/ingestion/source/database/iceberg/catalog/base.py +40 -0
- metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +102 -0
- metadata/ingestion/source/database/iceberg/catalog/glue.py +88 -0
- metadata/ingestion/source/database/iceberg/catalog/hive.py +51 -0
- metadata/ingestion/source/database/iceberg/catalog/rest.py +84 -0
- metadata/ingestion/source/database/iceberg/connection.py +68 -0
- metadata/ingestion/source/database/iceberg/fs/__init__.py +52 -0
- metadata/ingestion/source/database/iceberg/fs/azure.py +44 -0
- metadata/ingestion/source/database/iceberg/fs/base.py +30 -0
- metadata/ingestion/source/database/iceberg/fs/s3.py +77 -0
- metadata/ingestion/source/database/iceberg/helper.py +124 -0
- metadata/ingestion/source/database/iceberg/metadata.py +311 -0
- metadata/ingestion/source/database/iceberg/models.py +66 -0
- metadata/ingestion/source/database/life_cycle_query_mixin.py +72 -3
- metadata/ingestion/source/database/mongodb/connection.py +1 -5
- metadata/ingestion/source/database/mssql/lineage.py +3 -0
- metadata/ingestion/source/database/mssql/metadata.py +108 -4
- metadata/ingestion/source/database/mssql/models.py +30 -0
- metadata/ingestion/source/database/mssql/queries.py +178 -1
- metadata/ingestion/source/database/mssql/usage.py +5 -1
- metadata/ingestion/source/database/mssql/utils.py +207 -4
- metadata/ingestion/source/database/mysql/connection.py +14 -0
- metadata/ingestion/source/database/mysql/metadata.py +0 -2
- metadata/ingestion/source/database/oracle/metadata.py +108 -2
- metadata/ingestion/source/database/oracle/models.py +30 -0
- metadata/ingestion/source/database/oracle/queries.py +99 -18
- metadata/ingestion/source/database/oracle/utils.py +0 -1
- metadata/ingestion/source/database/postgres/connection.py +15 -0
- metadata/ingestion/source/database/postgres/lineage.py +32 -14
- metadata/ingestion/source/database/postgres/metadata.py +15 -7
- metadata/ingestion/source/database/postgres/pgspider/lineage.py +0 -1
- metadata/ingestion/source/database/postgres/queries.py +4 -2
- metadata/ingestion/source/database/postgres/query_parser.py +4 -72
- metadata/ingestion/source/database/postgres/usage.py +41 -0
- metadata/ingestion/source/database/postgres/utils.py +34 -0
- metadata/ingestion/source/database/query_parser_source.py +8 -2
- metadata/ingestion/source/database/redshift/metadata.py +14 -4
- metadata/ingestion/source/database/redshift/queries.py +10 -4
- metadata/ingestion/source/database/redshift/query_parser.py +16 -0
- metadata/ingestion/source/database/redshift/usage.py +0 -2
- metadata/ingestion/source/database/salesforce/metadata.py +32 -3
- metadata/ingestion/source/database/sample_data.py +120 -6
- metadata/ingestion/source/database/sas/client.py +184 -0
- metadata/ingestion/source/database/sas/connection.py +47 -0
- metadata/ingestion/source/database/sas/extension_attr.py +103 -0
- metadata/ingestion/source/database/sas/metadata.py +914 -0
- metadata/ingestion/source/database/snowflake/metadata.py +34 -48
- metadata/ingestion/source/database/snowflake/models.py +6 -1
- metadata/ingestion/source/database/snowflake/queries.py +6 -4
- metadata/ingestion/source/database/snowflake/query_parser.py +5 -20
- metadata/ingestion/source/database/snowflake/utils.py +23 -8
- metadata/ingestion/source/database/stored_procedures_mixin.py +12 -8
- metadata/ingestion/source/database/unitycatalog/__init__.py +0 -0
- metadata/ingestion/source/database/unitycatalog/client.py +87 -0
- metadata/ingestion/source/database/unitycatalog/connection.py +97 -0
- metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/lineage.py +11 -11
- metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/metadata.py +42 -49
- metadata/ingestion/source/database/unitycatalog/query_parser.py +60 -0
- metadata/ingestion/source/database/unitycatalog/usage.py +31 -0
- metadata/ingestion/source/database/usage_source.py +3 -2
- metadata/ingestion/source/messaging/common_broker_source.py +15 -11
- metadata/ingestion/source/messaging/kafka/connection.py +45 -4
- metadata/ingestion/source/messaging/kinesis/metadata.py +6 -3
- metadata/ingestion/source/messaging/messaging_service.py +6 -2
- metadata/ingestion/source/metadata/amundsen/metadata.py +10 -7
- metadata/ingestion/source/metadata/atlas/metadata.py +5 -5
- metadata/ingestion/source/mlmodel/mlflow/metadata.py +5 -2
- metadata/ingestion/source/mlmodel/mlmodel_service.py +6 -2
- metadata/ingestion/source/mlmodel/sagemaker/metadata.py +20 -8
- metadata/ingestion/source/pipeline/airflow/connection.py +0 -12
- metadata/ingestion/source/pipeline/airflow/lineage_parser.py +12 -6
- metadata/ingestion/source/pipeline/airflow/metadata.py +63 -34
- metadata/ingestion/source/pipeline/airflow/models.py +5 -4
- metadata/ingestion/source/pipeline/dagster/metadata.py +7 -4
- metadata/ingestion/source/pipeline/databrickspipeline/connection.py +1 -5
- metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +14 -11
- metadata/ingestion/source/pipeline/domopipeline/metadata.py +7 -4
- metadata/ingestion/source/pipeline/gluepipeline/metadata.py +5 -2
- metadata/ingestion/source/pipeline/pipeline_service.py +6 -2
- metadata/ingestion/source/pipeline/spline/metadata.py +0 -1
- metadata/ingestion/source/search/elasticsearch/connection.py +4 -1
- metadata/ingestion/source/search/elasticsearch/metadata.py +1 -2
- metadata/ingestion/source/search/search_service.py +6 -2
- metadata/ingestion/source/storage/s3/metadata.py +22 -17
- metadata/ingestion/source/storage/storage_service.py +53 -11
- metadata/ingestion/stage/table_usage.py +9 -2
- metadata/mixins/sqalchemy/sqa_mixin.py +14 -7
- metadata/parsers/protobuf_parser.py +29 -11
- metadata/pii/processor.py +9 -2
- metadata/pii/scanners/ner_scanner.py +5 -3
- metadata/profiler/api/models.py +19 -1
- metadata/profiler/interface/pandas/profiler_interface.py +59 -18
- metadata/profiler/interface/profiler_interface.py +14 -4
- metadata/profiler/interface/profiler_interface_factory.py +49 -14
- metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +3 -0
- metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +26 -0
- metadata/profiler/interface/sqlalchemy/db2/__init__.py +0 -0
- metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +38 -0
- metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +85 -0
- metadata/profiler/interface/sqlalchemy/profiler_interface.py +77 -34
- metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
- metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +7 -0
- metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
- metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +33 -0
- metadata/profiler/metrics/composed/null_ratio.py +1 -1
- metadata/profiler/metrics/hybrid/histogram.py +1 -0
- metadata/profiler/metrics/static/max.py +4 -1
- metadata/profiler/metrics/static/min.py +4 -1
- metadata/profiler/metrics/system/queries/snowflake.py +89 -17
- metadata/profiler/metrics/system/system.py +62 -20
- metadata/profiler/orm/functions/conn_test.py +1 -0
- metadata/profiler/orm/functions/length.py +1 -0
- metadata/profiler/orm/functions/median.py +9 -1
- metadata/profiler/orm/functions/sum.py +1 -0
- metadata/profiler/orm/functions/table_metric_computer.py +462 -0
- metadata/profiler/orm/registry.py +2 -0
- metadata/profiler/processor/core.py +129 -62
- metadata/profiler/processor/default.py +14 -3
- metadata/profiler/processor/handle_partition.py +2 -2
- metadata/profiler/processor/processor.py +10 -7
- metadata/profiler/processor/sample_data_handler.py +6 -2
- metadata/profiler/processor/sampler/sqlalchemy/bigquery/sampler.py +31 -3
- metadata/profiler/processor/sampler/sqlalchemy/sampler.py +29 -6
- metadata/profiler/processor/sampler/sqlalchemy/trino/sampler.py +10 -4
- metadata/profiler/source/base/profiler_source.py +5 -2
- metadata/profiler/source/bigquery/type_mapper.py +0 -1
- metadata/profiler/source/databricks/profiler_source.py +36 -0
- metadata/profiler/source/mariadb/functions/median.py +20 -0
- metadata/profiler/source/mariadb/metrics/window/first_quartile.py +10 -0
- metadata/profiler/source/mariadb/metrics/window/median.py +10 -0
- metadata/profiler/source/mariadb/metrics/window/third_quartile.py +10 -0
- metadata/profiler/source/metadata.py +49 -10
- metadata/profiler/source/metadata_ext.py +16 -50
- metadata/profiler/source/profiler_source_factory.py +8 -0
- metadata/py.typed +0 -0
- metadata/readers/dataframe/json.py +5 -4
- metadata/readers/file/api_reader.py +0 -1
- metadata/utils/constants.py +5 -0
- metadata/utils/credentials.py +36 -19
- metadata/utils/datalake/datalake_utils.py +369 -129
- metadata/utils/entity_link.py +26 -6
- metadata/utils/execution_time_tracker.py +199 -0
- metadata/utils/filters.py +4 -0
- metadata/utils/fqn.py +20 -0
- metadata/utils/helpers.py +34 -39
- metadata/utils/importer.py +2 -3
- metadata/utils/life_cycle_utils.py +4 -4
- metadata/utils/logger.py +13 -2
- metadata/utils/partition.py +10 -5
- metadata/utils/secrets/aws_based_secrets_manager.py +67 -4
- metadata/utils/secrets/aws_secrets_manager.py +7 -2
- metadata/utils/secrets/aws_ssm_secrets_manager.py +7 -2
- metadata/utils/secrets/azure_kv_secrets_manager.py +132 -0
- metadata/utils/secrets/{noop_secrets_manager.py → db_secrets_manager.py} +4 -3
- metadata/utils/secrets/external_secrets_manager.py +25 -3
- metadata/utils/secrets/secrets_manager_factory.py +15 -33
- metadata/utils/{source_hash_utils.py → source_hash.py} +10 -1
- metadata/utils/sqlalchemy_utils.py +21 -0
- metadata/utils/storage_metadata_config.py +42 -14
- metadata/utils/tag_utils.py +5 -2
- metadata/workflow/application.py +154 -0
- metadata/workflow/application_output_handler.py +34 -0
- metadata/workflow/base.py +88 -153
- metadata/workflow/data_insight.py +8 -7
- metadata/workflow/data_quality.py +3 -2
- metadata/workflow/ingestion.py +203 -0
- metadata/workflow/metadata.py +2 -3
- metadata/workflow/output_handler.py +226 -0
- metadata/workflow/profiler.py +2 -2
- metadata/workflow/usage.py +3 -4
- metadata/workflow/workflow_output_handler.py +15 -255
- metadata/workflow/workflow_status_mixin.py +44 -52
- openmetadata_ingestion-1.3.2.0rc1.dist-info/METADATA +758 -0
- {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/RECORD +870 -762
- {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/WHEEL +1 -1
- metadata/generated/schema/entity/applications/appConfig.py +0 -21
- metadata/generated/schema/entity/applications/configuration/dataInsightsApp.py +0 -17
- metadata/generated/schema/entity/applications/configuration/externalAppIngestionConfig.py +0 -38
- metadata/generated/schema/entity/services/connections/database/mongoDB/mongoDBValues.py +0 -44
- metadata/generated/schema/events/dataInsightAlertConfig.py +0 -17
- metadata/generated/schema/events/entitySpelFilters.py +0 -19
- metadata/ingestion/models/es_documents.py +0 -339
- metadata/ingestion/ometa/mixins/glossary_mixin.py +0 -501
- metadata/ingestion/ometa/provider_registry.py +0 -144
- metadata/ingestion/source/database/databricks/legacy/lineage.py +0 -51
- metadata/ingestion/source/database/databricks/legacy/metadata.py +0 -339
- metadata/ingestion/source/metadata/metadata_elasticsearch/metadata.py +0 -144
- metadata/profiler/orm/functions/table_metric_construct.py +0 -365
- metadata/utils/secrets/client/loader.py +0 -78
- openmetadata_ingestion-1.2.4.0.dist-info/METADATA +0 -429
- /metadata/examples/workflows/{datalake_azure.yaml → datalake_azure_client_secret.yaml} +0 -0
- /metadata/ingestion/source/{database/databricks/legacy → dashboard/mstr}/__init__.py +0 -0
- /metadata/ingestion/source/database/{databricks/unity_catalog → bigtable}/__init__.py +0 -0
- /metadata/ingestion/source/{metadata/metadata_elasticsearch → database/doris}/__init__.py +0 -0
- /metadata/{utils/secrets/client → ingestion/source/database/sas}/__init__.py +0 -0
- /metadata/ingestion/source/database/{databricks → unitycatalog}/models.py +0 -0
- {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/LICENSE +0 -0
- {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/entry_points.txt +0 -0
- {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -15,7 +15,6 @@ Superset source module
|
|
|
15
15
|
import traceback
|
|
16
16
|
from typing import Iterable, Optional
|
|
17
17
|
|
|
18
|
-
from sqlalchemy import sql
|
|
19
18
|
from sqlalchemy.engine import Engine
|
|
20
19
|
from sqlalchemy.engine.url import make_url
|
|
21
20
|
|
|
@@ -28,10 +27,13 @@ from metadata.generated.schema.entity.data.chart import Chart
|
|
|
28
27
|
from metadata.generated.schema.entity.data.dashboardDataModel import DataModelType
|
|
29
28
|
from metadata.generated.schema.entity.data.table import Table
|
|
30
29
|
from metadata.generated.schema.entity.services.databaseService import DatabaseService
|
|
30
|
+
from metadata.generated.schema.entity.services.ingestionPipelines.status import (
|
|
31
|
+
StackTraceError,
|
|
32
|
+
)
|
|
31
33
|
from metadata.generated.schema.metadataIngestion.workflow import (
|
|
32
34
|
Source as WorkflowSource,
|
|
33
35
|
)
|
|
34
|
-
from metadata.ingestion.api.models import Either
|
|
36
|
+
from metadata.ingestion.api.models import Either
|
|
35
37
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
36
38
|
from metadata.ingestion.source.dashboard.superset.mixin import SupersetSourceMixin
|
|
37
39
|
from metadata.ingestion.source.dashboard.superset.models import (
|
|
@@ -83,8 +85,9 @@ class SupersetDBSource(SupersetSourceMixin):
|
|
|
83
85
|
def get_column_list(self, table_name: str) -> Iterable[FetchChart]:
|
|
84
86
|
try:
|
|
85
87
|
if table_name:
|
|
86
|
-
|
|
87
|
-
|
|
88
|
+
col_list = self.engine.execute(
|
|
89
|
+
FETCH_COLUMN, table_name=table_name.lower()
|
|
90
|
+
)
|
|
88
91
|
return [FetchColumn(**col) for col in col_list]
|
|
89
92
|
except Exception as err:
|
|
90
93
|
logger.debug(traceback.format_exc())
|
|
@@ -117,9 +120,10 @@ class SupersetDBSource(SupersetSourceMixin):
|
|
|
117
120
|
service_name=self.context.dashboard_service,
|
|
118
121
|
chart_name=chart,
|
|
119
122
|
)
|
|
120
|
-
for chart in self.context.charts
|
|
123
|
+
for chart in self.context.charts or []
|
|
121
124
|
],
|
|
122
125
|
service=self.context.dashboard_service,
|
|
126
|
+
owner=self.get_owner_ref(dashboard_details=dashboard_details),
|
|
123
127
|
)
|
|
124
128
|
yield Either(right=dashboard_request)
|
|
125
129
|
self.register_record(dashboard_request=dashboard_request)
|
|
@@ -131,7 +135,7 @@ class SupersetDBSource(SupersetSourceMixin):
|
|
|
131
135
|
f"Error yielding Dashboard [{dashboard_details.id} "
|
|
132
136
|
f"- {dashboard_details.dashboard_title}]: {exc}"
|
|
133
137
|
),
|
|
134
|
-
|
|
138
|
+
stackTrace=traceback.format_exc(),
|
|
135
139
|
)
|
|
136
140
|
)
|
|
137
141
|
|
|
@@ -173,7 +177,7 @@ class SupersetDBSource(SupersetSourceMixin):
|
|
|
173
177
|
left=StackTraceError(
|
|
174
178
|
name=chart_json.id,
|
|
175
179
|
error=f"Error yielding Chart [{chart_json.id} - {chart_json.slice_name}]: {exc}",
|
|
176
|
-
|
|
180
|
+
stackTrace=traceback.format_exc(),
|
|
177
181
|
)
|
|
178
182
|
)
|
|
179
183
|
|
|
@@ -211,11 +215,10 @@ class SupersetDBSource(SupersetSourceMixin):
|
|
|
211
215
|
def yield_datamodel(
|
|
212
216
|
self, dashboard_details: FetchDashboard
|
|
213
217
|
) -> Iterable[Either[CreateDashboardDataModelRequest]]:
|
|
214
|
-
|
|
215
218
|
if self.source_config.includeDataModels:
|
|
216
219
|
for chart_id in self._get_charts_of_dashboard(dashboard_details):
|
|
217
220
|
chart_json = self.all_charts.get(chart_id)
|
|
218
|
-
if not chart_json:
|
|
221
|
+
if not chart_json or not chart_json.datasource_id:
|
|
219
222
|
logger.warning(
|
|
220
223
|
f"chart details for id: {chart_id} not found, skipped"
|
|
221
224
|
)
|
|
@@ -236,13 +239,13 @@ class SupersetDBSource(SupersetSourceMixin):
|
|
|
236
239
|
dataModelType=DataModelType.SupersetDataModel.value,
|
|
237
240
|
)
|
|
238
241
|
yield Either(right=data_model_request)
|
|
239
|
-
self.register_record_datamodel(
|
|
242
|
+
self.register_record_datamodel(datamodel_request=data_model_request)
|
|
240
243
|
|
|
241
244
|
except Exception as exc:
|
|
242
245
|
yield Either(
|
|
243
246
|
left=StackTraceError(
|
|
244
247
|
name=chart_json.table_name,
|
|
245
248
|
error=f"Error yielding Data Model [{chart_json.table_name}]: {exc}",
|
|
246
|
-
|
|
249
|
+
stackTrace=traceback.format_exc(),
|
|
247
250
|
)
|
|
248
251
|
)
|
|
@@ -28,11 +28,14 @@ from metadata.generated.schema.entity.services.dashboardService import (
|
|
|
28
28
|
DashboardServiceType,
|
|
29
29
|
)
|
|
30
30
|
from metadata.generated.schema.entity.services.databaseService import DatabaseService
|
|
31
|
+
from metadata.generated.schema.entity.services.ingestionPipelines.status import (
|
|
32
|
+
StackTraceError,
|
|
33
|
+
)
|
|
31
34
|
from metadata.generated.schema.metadataIngestion.workflow import (
|
|
32
35
|
Source as WorkflowSource,
|
|
33
36
|
)
|
|
34
37
|
from metadata.generated.schema.type.entityReference import EntityReference
|
|
35
|
-
from metadata.ingestion.api.models import Either
|
|
38
|
+
from metadata.ingestion.api.models import Either
|
|
36
39
|
from metadata.ingestion.api.steps import InvalidSourceException
|
|
37
40
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
38
41
|
from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
|
|
@@ -94,25 +97,26 @@ class SupersetSourceMixin(DashboardServiceSource):
|
|
|
94
97
|
|
|
95
98
|
def _get_user_by_email(self, email: Optional[str]) -> Optional[EntityReference]:
|
|
96
99
|
if email:
|
|
97
|
-
|
|
98
|
-
if user:
|
|
99
|
-
return EntityReference(id=user.id.__root__, type="user")
|
|
100
|
-
|
|
100
|
+
return self.metadata.get_reference_by_email(email)
|
|
101
101
|
return None
|
|
102
102
|
|
|
103
|
-
def
|
|
103
|
+
def get_owner_ref(
|
|
104
104
|
self, dashboard_details: Union[DashboardResult, FetchDashboard]
|
|
105
105
|
) -> EntityReference:
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
106
|
+
try:
|
|
107
|
+
if hasattr(dashboard_details, "owners"):
|
|
108
|
+
for owner in dashboard_details.owners or []:
|
|
109
|
+
if owner.email:
|
|
110
|
+
user = self._get_user_by_email(owner.email)
|
|
111
|
+
if user:
|
|
112
|
+
return user
|
|
113
|
+
if dashboard_details.email:
|
|
114
|
+
user = self._get_user_by_email(dashboard_details.email)
|
|
115
|
+
if user:
|
|
116
|
+
return user
|
|
117
|
+
except Exception as err:
|
|
118
|
+
logger.debug(traceback.format_exc())
|
|
119
|
+
logger.warning(f"Could not fetch owner data due to {err}")
|
|
116
120
|
return None
|
|
117
121
|
|
|
118
122
|
def _get_charts_of_dashboard(
|
|
@@ -185,7 +189,7 @@ class SupersetSourceMixin(DashboardServiceSource):
|
|
|
185
189
|
"Error to yield dashboard lineage details for DB "
|
|
186
190
|
f"service name [{db_service_name}]: {exc}"
|
|
187
191
|
),
|
|
188
|
-
|
|
192
|
+
stackTrace=traceback.format_exc(),
|
|
189
193
|
)
|
|
190
194
|
)
|
|
191
195
|
|
|
@@ -230,7 +234,7 @@ class SupersetSourceMixin(DashboardServiceSource):
|
|
|
230
234
|
name=field.id,
|
|
231
235
|
displayName=field.column_name,
|
|
232
236
|
description=field.description,
|
|
233
|
-
dataLength=col_parse.get("dataLength", 0),
|
|
237
|
+
dataLength=int(col_parse.get("dataLength", 0)),
|
|
234
238
|
)
|
|
235
239
|
datasource_columns.append(parsed_fields)
|
|
236
240
|
except Exception as exc:
|
|
@@ -15,7 +15,9 @@ import math
|
|
|
15
15
|
import traceback
|
|
16
16
|
from typing import Any, Callable, Dict, List, Optional
|
|
17
17
|
|
|
18
|
+
import validators
|
|
18
19
|
from cached_property import cached_property
|
|
20
|
+
from packaging import version
|
|
19
21
|
from tableau_api_lib import TableauServerConnection
|
|
20
22
|
from tableau_api_lib.utils import extract_pages
|
|
21
23
|
|
|
@@ -60,20 +62,22 @@ class TableauClient:
|
|
|
60
62
|
|
|
61
63
|
def __init__(
|
|
62
64
|
self,
|
|
63
|
-
|
|
65
|
+
tableau_server_config: Dict[str, Dict[str, Any]],
|
|
66
|
+
config,
|
|
64
67
|
env: str,
|
|
65
68
|
ssl_verify: bool,
|
|
66
69
|
pagination_limit: int,
|
|
67
70
|
):
|
|
68
71
|
# ssl_verify is typed as a `bool` in TableauServerConnection
|
|
69
72
|
# However, it is passed as `verify=self.ssl_verify` in each `requests` call.
|
|
70
|
-
# In requests (https://requests.readthedocs.io/en/latest/user/advanced
|
|
73
|
+
# In requests (https://requests.readthedocs.io/en/latest/user/advanced.html?highlight=ssl#ssl-cert-verification)
|
|
71
74
|
# the param can be None, False to ignore HTTPS certs or a string with the path to the cert.
|
|
72
75
|
self._client = TableauServerConnection(
|
|
73
|
-
config_json=
|
|
76
|
+
config_json=tableau_server_config,
|
|
74
77
|
env=env,
|
|
75
78
|
ssl_verify=ssl_verify,
|
|
76
79
|
)
|
|
80
|
+
self.config = config
|
|
77
81
|
self._client.sign_in().json()
|
|
78
82
|
self.pagination_limit = pagination_limit
|
|
79
83
|
|
|
@@ -81,6 +85,10 @@ class TableauClient:
|
|
|
81
85
|
def server_info(self) -> Callable:
|
|
82
86
|
return self._client.server_info
|
|
83
87
|
|
|
88
|
+
@property
|
|
89
|
+
def server_api_version(self) -> str:
|
|
90
|
+
return self.server_info().json()["serverInfo"]["restApiVersion"]
|
|
91
|
+
|
|
84
92
|
@property
|
|
85
93
|
def site_id(self) -> str:
|
|
86
94
|
return self._client.site_id
|
|
@@ -122,11 +130,79 @@ class TableauClient:
|
|
|
122
130
|
)
|
|
123
131
|
]
|
|
124
132
|
|
|
133
|
+
def get_workbook_charts(self, dashboard_id: str) -> Optional[List[TableauChart]]:
|
|
134
|
+
"""
|
|
135
|
+
Get the charts for a workbook
|
|
136
|
+
"""
|
|
137
|
+
try:
|
|
138
|
+
return [
|
|
139
|
+
TableauChart(**chart)
|
|
140
|
+
for chart in self._client.query_views_for_workbook(
|
|
141
|
+
workbook_id=dashboard_id,
|
|
142
|
+
parameter_dict=TABLEAU_GET_VIEWS_PARAM_DICT,
|
|
143
|
+
).json()["views"]["view"]
|
|
144
|
+
]
|
|
145
|
+
except Exception as exc:
|
|
146
|
+
logger.debug(traceback.format_exc())
|
|
147
|
+
logger.warning(
|
|
148
|
+
f"Error processing charts for dashboard [{dashboard_id}]: {exc}"
|
|
149
|
+
)
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
def test_api_version(self):
|
|
153
|
+
"""
|
|
154
|
+
Method to validate the declared v/s server tableau rest api version
|
|
155
|
+
"""
|
|
156
|
+
server_api_version = version.parse(self.server_api_version)
|
|
157
|
+
declared_api_version = version.parse(self.config.apiVersion)
|
|
158
|
+
if declared_api_version > server_api_version:
|
|
159
|
+
raise ValueError(
|
|
160
|
+
f"""
|
|
161
|
+
Your API version of '{declared_api_version}' is too damn high!
|
|
162
|
+
The server you are establishing a connection with is using REST API version '{server_api_version}'.
|
|
163
|
+
"""
|
|
164
|
+
)
|
|
165
|
+
if declared_api_version < server_api_version:
|
|
166
|
+
raise ValueError(
|
|
167
|
+
f"""
|
|
168
|
+
The Tableau Server REST API version you specified is lower than the version your server uses.
|
|
169
|
+
Your Tableau Server is on REST API version {server_api_version}.
|
|
170
|
+
The REST API version you specified is {declared_api_version}.
|
|
171
|
+
For optimal results, please change the 'api_version' config variable to {server_api_version}.
|
|
172
|
+
"""
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
def test_site_url(self):
|
|
176
|
+
"""
|
|
177
|
+
Method to test the site url and site name fields
|
|
178
|
+
"""
|
|
179
|
+
validation = validators.url(self.config.siteUrl)
|
|
180
|
+
if validation:
|
|
181
|
+
raise ValueError(
|
|
182
|
+
f"""
|
|
183
|
+
The site url "{self.config.siteUrl}" is in incorrect format.
|
|
184
|
+
If "https://xxx.tableau.com/#/site/MarketingTeam/home" represents the homepage url for your tableau site,
|
|
185
|
+
the "MarketingTeam" from the url should be entered in the Site Name and Site Url fields.
|
|
186
|
+
"""
|
|
187
|
+
)
|
|
188
|
+
return True
|
|
189
|
+
|
|
125
190
|
def test_get_datamodels(self):
|
|
126
191
|
"""
|
|
127
192
|
Method to test the datamodels
|
|
128
193
|
"""
|
|
129
|
-
|
|
194
|
+
workbooks = self.get_workbooks()
|
|
195
|
+
|
|
196
|
+
if len(workbooks) == 0:
|
|
197
|
+
raise TableauDataModelsException(
|
|
198
|
+
"Unable to get any workbooks to fetch tableau data sources"
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
# Take the 1st workbook's id and pass to the graphql query
|
|
202
|
+
test_workbook = workbooks[0]
|
|
203
|
+
data = self._query_datasources(
|
|
204
|
+
dashboard_id=test_workbook.id, entities_per_page=1, offset=0
|
|
205
|
+
)
|
|
130
206
|
if data:
|
|
131
207
|
return data
|
|
132
208
|
raise TableauDataModelsException(
|
|
@@ -138,7 +214,7 @@ class TableauClient:
|
|
|
138
214
|
)
|
|
139
215
|
|
|
140
216
|
def _query_datasources(
|
|
141
|
-
self, entities_per_page: int, offset: int
|
|
217
|
+
self, dashboard_id: str, entities_per_page: int, offset: int
|
|
142
218
|
) -> Optional[TableauDatasources]:
|
|
143
219
|
"""
|
|
144
220
|
Method to query the graphql endpoint to get data sources
|
|
@@ -146,14 +222,14 @@ class TableauClient:
|
|
|
146
222
|
try:
|
|
147
223
|
datasources_graphql_result = self._client.metadata_graphql_query(
|
|
148
224
|
query=TABLEAU_DATASOURCES_QUERY.format(
|
|
149
|
-
first=entities_per_page, offset=offset
|
|
225
|
+
workbook_id=dashboard_id, first=entities_per_page, offset=offset
|
|
150
226
|
)
|
|
151
227
|
)
|
|
152
228
|
if datasources_graphql_result:
|
|
153
229
|
resp = datasources_graphql_result.json()
|
|
154
230
|
if resp and resp.get("data"):
|
|
155
231
|
tableau_datasource_connection = TableauDatasourcesConnection(
|
|
156
|
-
**resp
|
|
232
|
+
**resp["data"]["workbooks"][0]
|
|
157
233
|
)
|
|
158
234
|
return tableau_datasource_connection.embeddedDatasourcesConnection
|
|
159
235
|
except Exception:
|
|
@@ -167,13 +243,15 @@ class TableauClient:
|
|
|
167
243
|
)
|
|
168
244
|
return None
|
|
169
245
|
|
|
170
|
-
def get_datasources(self) -> Optional[List[DataSource]]:
|
|
246
|
+
def get_datasources(self, dashboard_id: str) -> Optional[List[DataSource]]:
|
|
171
247
|
"""
|
|
172
|
-
Paginate and get the list of all data sources
|
|
248
|
+
Paginate and get the list of all data sources of the workbook
|
|
173
249
|
"""
|
|
174
250
|
try:
|
|
175
251
|
# Query the graphql endpoint once to get total count of data sources
|
|
176
|
-
tableau_datasource = self._query_datasources(
|
|
252
|
+
tableau_datasource = self._query_datasources(
|
|
253
|
+
dashboard_id=dashboard_id, entities_per_page=1, offset=1
|
|
254
|
+
)
|
|
177
255
|
entities_per_page = min(50, self.pagination_limit)
|
|
178
256
|
indexes = math.ceil(tableau_datasource.totalCount / entities_per_page)
|
|
179
257
|
|
|
@@ -182,7 +260,9 @@ class TableauClient:
|
|
|
182
260
|
for index in range(indexes):
|
|
183
261
|
offset = index * entities_per_page
|
|
184
262
|
tableau_datasource = self._query_datasources(
|
|
185
|
-
|
|
263
|
+
dashboard_id=dashboard_id,
|
|
264
|
+
entities_per_page=entities_per_page,
|
|
265
|
+
offset=offset,
|
|
186
266
|
)
|
|
187
267
|
if tableau_datasource:
|
|
188
268
|
data_sources.extend(tableau_datasource.nodes)
|
|
@@ -52,7 +52,8 @@ def get_connection(connection: TableauConnection) -> TableauClient:
|
|
|
52
52
|
get_verify_ssl = get_verify_ssl_fn(connection.verifySSL)
|
|
53
53
|
try:
|
|
54
54
|
return TableauClient(
|
|
55
|
-
|
|
55
|
+
tableau_server_config=tableau_server_config,
|
|
56
|
+
config=connection,
|
|
56
57
|
env=connection.env,
|
|
57
58
|
ssl_verify=get_verify_ssl(connection.sslConfig),
|
|
58
59
|
pagination_limit=connection.paginationLimit,
|
|
@@ -77,6 +78,14 @@ def test_connection(
|
|
|
77
78
|
|
|
78
79
|
test_fn = {
|
|
79
80
|
"ServerInfo": client.server_info,
|
|
81
|
+
# The Tableau server_info API doesn't provide direct access to the API version.
|
|
82
|
+
# This is due to the "api_version" being a mandatory field for the tableau library's connection class.
|
|
83
|
+
# Without this information, requests to the Tableau server cannot be made,
|
|
84
|
+
# including fetching the server info containing the "api_version".
|
|
85
|
+
# Consequently, we'll compare the declared api_version with the server's api_version during the test connection
|
|
86
|
+
# once the tableau library's connection class is initialized.
|
|
87
|
+
"ValidateApiVersion": client.test_api_version,
|
|
88
|
+
"ValidateSiteUrl": client.test_site_url,
|
|
80
89
|
"GetWorkbooks": partial(
|
|
81
90
|
extract_pages,
|
|
82
91
|
query_func=client.query_workbooks_for_site,
|
|
@@ -14,6 +14,8 @@ Tableau source module
|
|
|
14
14
|
import traceback
|
|
15
15
|
from typing import Any, Iterable, List, Optional, Set
|
|
16
16
|
|
|
17
|
+
from requests.utils import urlparse
|
|
18
|
+
|
|
17
19
|
from metadata.generated.schema.api.data.createChart import CreateChartRequest
|
|
18
20
|
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
|
|
19
21
|
from metadata.generated.schema.api.data.createDashboardDataModel import (
|
|
@@ -39,11 +41,14 @@ from metadata.generated.schema.entity.services.dashboardService import (
|
|
|
39
41
|
DashboardServiceType,
|
|
40
42
|
)
|
|
41
43
|
from metadata.generated.schema.entity.services.databaseService import DatabaseService
|
|
44
|
+
from metadata.generated.schema.entity.services.ingestionPipelines.status import (
|
|
45
|
+
StackTraceError,
|
|
46
|
+
)
|
|
42
47
|
from metadata.generated.schema.metadataIngestion.workflow import (
|
|
43
48
|
Source as WorkflowSource,
|
|
44
49
|
)
|
|
45
50
|
from metadata.generated.schema.type.entityReference import EntityReference
|
|
46
|
-
from metadata.ingestion.api.models import Either
|
|
51
|
+
from metadata.ingestion.api.models import Either
|
|
47
52
|
from metadata.ingestion.api.steps import InvalidSourceException
|
|
48
53
|
from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
|
|
49
54
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
@@ -82,51 +87,6 @@ class TableauSource(DashboardServiceSource):
|
|
|
82
87
|
metadata_config: OpenMetadataConnection
|
|
83
88
|
client: TableauClient
|
|
84
89
|
|
|
85
|
-
def __init__(
|
|
86
|
-
self,
|
|
87
|
-
config: WorkflowSource,
|
|
88
|
-
metadata: OpenMetadata,
|
|
89
|
-
):
|
|
90
|
-
super().__init__(config, metadata)
|
|
91
|
-
self.workbooks: List[
|
|
92
|
-
TableauDashboard
|
|
93
|
-
] = [] # We will populate this in `prepare`
|
|
94
|
-
self.tags: Set[TableauTag] = set()
|
|
95
|
-
|
|
96
|
-
def prepare(self):
|
|
97
|
-
"""Restructure the API response"""
|
|
98
|
-
try:
|
|
99
|
-
# get workbooks which are considered Dashboards in OM
|
|
100
|
-
self.workbooks = self.client.get_workbooks()
|
|
101
|
-
|
|
102
|
-
# get views which are considered charts in OM
|
|
103
|
-
charts = self.client.get_charts()
|
|
104
|
-
|
|
105
|
-
# get datasources which are considered as datamodels in OM
|
|
106
|
-
data_models = self.client.get_datasources()
|
|
107
|
-
|
|
108
|
-
# add all the charts (views) and datasources from the API to each workbook
|
|
109
|
-
for workbook in self.workbooks:
|
|
110
|
-
workbook.charts = [
|
|
111
|
-
chart for chart in charts if chart.workbook.id == workbook.id
|
|
112
|
-
]
|
|
113
|
-
|
|
114
|
-
for data_model in data_models or []:
|
|
115
|
-
if data_model.workbook and data_model.workbook.luid == workbook.id:
|
|
116
|
-
workbook.dataModels.append(data_model)
|
|
117
|
-
|
|
118
|
-
# collect all the tags from charts and workbooks before yielding final entities
|
|
119
|
-
if self.source_config.includeTags:
|
|
120
|
-
for container in [self.workbooks, charts]:
|
|
121
|
-
for elem in container:
|
|
122
|
-
self.tags.update(elem.tags)
|
|
123
|
-
|
|
124
|
-
except Exception:
|
|
125
|
-
logger.debug(traceback.format_exc())
|
|
126
|
-
logger.error("Error in fetching the Tableau Workbook metadata")
|
|
127
|
-
|
|
128
|
-
return super().prepare()
|
|
129
|
-
|
|
130
90
|
@classmethod
|
|
131
91
|
def create(cls, config_dict: dict, metadata: OpenMetadata):
|
|
132
92
|
config: WorkflowSource = WorkflowSource.parse_obj(config_dict)
|
|
@@ -138,36 +98,76 @@ class TableauSource(DashboardServiceSource):
|
|
|
138
98
|
return cls(config, metadata)
|
|
139
99
|
|
|
140
100
|
def get_dashboards_list(self) -> Optional[List[TableauDashboard]]:
|
|
141
|
-
return self.
|
|
101
|
+
return self.client.get_workbooks()
|
|
142
102
|
|
|
143
103
|
def get_dashboard_name(self, dashboard: TableauDashboard) -> str:
|
|
144
104
|
return dashboard.name
|
|
145
105
|
|
|
146
106
|
def get_dashboard_details(self, dashboard: TableauDashboard) -> TableauDashboard:
|
|
147
107
|
"""
|
|
148
|
-
Get Dashboard Details
|
|
149
|
-
during the `prepare` stage
|
|
108
|
+
Get Dashboard Details including the dashboard charts and datamodels
|
|
150
109
|
"""
|
|
110
|
+
|
|
111
|
+
# Get the tableau views/sheets
|
|
112
|
+
dashboard.charts = self.client.get_workbook_charts(dashboard_id=dashboard.id)
|
|
113
|
+
|
|
114
|
+
# Get the tableau data sources
|
|
115
|
+
dashboard.dataModels = self.client.get_datasources(dashboard_id=dashboard.id)
|
|
116
|
+
|
|
151
117
|
return dashboard
|
|
152
118
|
|
|
153
|
-
def
|
|
119
|
+
def get_owner_ref(
|
|
154
120
|
self, dashboard_details: TableauDashboard
|
|
155
121
|
) -> Optional[EntityReference]:
|
|
156
|
-
"""
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
122
|
+
"""
|
|
123
|
+
Get dashboard owner from email
|
|
124
|
+
"""
|
|
125
|
+
try:
|
|
126
|
+
if dashboard_details.owner and dashboard_details.owner.email:
|
|
127
|
+
return self.metadata.get_reference_by_email(
|
|
128
|
+
dashboard_details.owner.email
|
|
129
|
+
)
|
|
130
|
+
except Exception as err:
|
|
131
|
+
logger.debug(traceback.format_exc())
|
|
132
|
+
logger.warning(f"Could not fetch owner data due to {err}")
|
|
161
133
|
return None
|
|
162
134
|
|
|
163
|
-
def
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
135
|
+
def yield_tags(
|
|
136
|
+
self, dashboard_details: TableauDashboard
|
|
137
|
+
) -> Iterable[Either[OMetaTagAndClassification]]:
|
|
138
|
+
"""
|
|
139
|
+
Method to yield tags related to specific dashboards
|
|
140
|
+
"""
|
|
141
|
+
if self.source_config.includeTags:
|
|
142
|
+
tags: Set[TableauTag] = set()
|
|
143
|
+
for container in [[dashboard_details], dashboard_details.charts]:
|
|
144
|
+
for elem in container:
|
|
145
|
+
tags.update(elem.tags)
|
|
146
|
+
|
|
147
|
+
yield from get_ometa_tag_and_classification(
|
|
148
|
+
tags=[tag.label for tag in tags],
|
|
149
|
+
classification_name=TABLEAU_TAG_CATEGORY,
|
|
150
|
+
tag_description="Tableau Tag",
|
|
151
|
+
classification_description="Tags associated with tableau entities",
|
|
152
|
+
include_tags=self.source_config.includeTags,
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
def _get_datamodel_sql_query(self, data_model: DataSource) -> Optional[str]:
|
|
156
|
+
"""
|
|
157
|
+
Method to fetch the custom sql query from the tableau datamodels
|
|
158
|
+
"""
|
|
159
|
+
try:
|
|
160
|
+
sql_queries = []
|
|
161
|
+
for table in data_model.upstreamTables or []:
|
|
162
|
+
for referenced_query in table.referencedByQueries or []:
|
|
163
|
+
sql_queries.append(referenced_query.query)
|
|
164
|
+
return "\n\n".join(sql_queries) or None
|
|
165
|
+
except Exception as exc:
|
|
166
|
+
logger.debug(traceback.format_exc())
|
|
167
|
+
logger.warning(
|
|
168
|
+
f"Error processing queries for datamodel [{data_model.id}]: {exc}"
|
|
169
|
+
)
|
|
170
|
+
return None
|
|
171
171
|
|
|
172
172
|
def yield_datamodel(
|
|
173
173
|
self, dashboard_details: TableauDashboard
|
|
@@ -188,16 +188,17 @@ class TableauSource(DashboardServiceSource):
|
|
|
188
188
|
dataModelType=DataModelType.TableauDataModel.value,
|
|
189
189
|
serviceType=DashboardServiceType.Tableau.value,
|
|
190
190
|
columns=self.get_column_info(data_model),
|
|
191
|
+
sql=self._get_datamodel_sql_query(data_model=data_model),
|
|
191
192
|
)
|
|
192
193
|
yield Either(right=data_model_request)
|
|
193
|
-
self.register_record_datamodel(
|
|
194
|
+
self.register_record_datamodel(datamodel_request=data_model_request)
|
|
194
195
|
|
|
195
196
|
except Exception as exc:
|
|
196
197
|
yield Either(
|
|
197
198
|
left=StackTraceError(
|
|
198
199
|
name=data_model_name,
|
|
199
200
|
error=f"Error yielding Data Model [{data_model_name}]: {exc}",
|
|
200
|
-
|
|
201
|
+
stackTrace=traceback.format_exc(),
|
|
201
202
|
)
|
|
202
203
|
)
|
|
203
204
|
|
|
@@ -214,6 +215,10 @@ class TableauSource(DashboardServiceSource):
|
|
|
214
215
|
topology. And they are cleared after processing each Dashboard because of the 'clear_cache' option.
|
|
215
216
|
"""
|
|
216
217
|
try:
|
|
218
|
+
dashboard_url = (
|
|
219
|
+
f"{clean_uri(str(self.config.serviceConnection.__root__.config.hostPort))}"
|
|
220
|
+
f"/#{urlparse(dashboard_details.webpageUrl).fragment}/views"
|
|
221
|
+
)
|
|
217
222
|
dashboard_request = CreateDashboardRequest(
|
|
218
223
|
name=dashboard_details.id,
|
|
219
224
|
displayName=dashboard_details.name,
|
|
@@ -226,7 +231,7 @@ class TableauSource(DashboardServiceSource):
|
|
|
226
231
|
service_name=self.context.dashboard_service,
|
|
227
232
|
chart_name=chart,
|
|
228
233
|
)
|
|
229
|
-
for chart in self.context.charts
|
|
234
|
+
for chart in self.context.charts or []
|
|
230
235
|
],
|
|
231
236
|
dataModels=[
|
|
232
237
|
fqn.build(
|
|
@@ -243,8 +248,9 @@ class TableauSource(DashboardServiceSource):
|
|
|
243
248
|
classification_name=TABLEAU_TAG_CATEGORY,
|
|
244
249
|
include_tags=self.source_config.includeTags,
|
|
245
250
|
),
|
|
246
|
-
sourceUrl=
|
|
251
|
+
sourceUrl=dashboard_url,
|
|
247
252
|
service=self.context.dashboard_service,
|
|
253
|
+
owner=self.get_owner_ref(dashboard_details=dashboard_details),
|
|
248
254
|
)
|
|
249
255
|
yield Either(right=dashboard_request)
|
|
250
256
|
self.register_record(dashboard_request=dashboard_request)
|
|
@@ -253,7 +259,7 @@ class TableauSource(DashboardServiceSource):
|
|
|
253
259
|
left=StackTraceError(
|
|
254
260
|
name=dashboard_details.id,
|
|
255
261
|
error=f"Error to yield dashboard for {dashboard_details}: {exc}",
|
|
256
|
-
|
|
262
|
+
stackTrace=traceback.format_exc(),
|
|
257
263
|
)
|
|
258
264
|
)
|
|
259
265
|
|
|
@@ -294,7 +300,7 @@ class TableauSource(DashboardServiceSource):
|
|
|
294
300
|
"Error to yield dashboard lineage details for DB "
|
|
295
301
|
f"service name [{db_service_name}]: {err}"
|
|
296
302
|
),
|
|
297
|
-
|
|
303
|
+
stackTrace=traceback.format_exc(),
|
|
298
304
|
)
|
|
299
305
|
)
|
|
300
306
|
|
|
@@ -342,7 +348,7 @@ class TableauSource(DashboardServiceSource):
|
|
|
342
348
|
left=StackTraceError(
|
|
343
349
|
name="Chart",
|
|
344
350
|
error=f"Error to yield dashboard chart [{chart}]: {exc}",
|
|
345
|
-
|
|
351
|
+
stackTrace=traceback.format_exc(),
|
|
346
352
|
)
|
|
347
353
|
)
|
|
348
354
|
|
|
@@ -99,12 +99,6 @@ class DatasourceField(BaseModel):
|
|
|
99
99
|
description: Optional[str]
|
|
100
100
|
|
|
101
101
|
|
|
102
|
-
class Workbook(BaseModel):
|
|
103
|
-
id: str
|
|
104
|
-
luid: str
|
|
105
|
-
name: Optional[str]
|
|
106
|
-
|
|
107
|
-
|
|
108
102
|
class UpstreamTableColumn(BaseModel):
|
|
109
103
|
id: str
|
|
110
104
|
name: Optional[str]
|
|
@@ -130,7 +124,6 @@ class DataSource(BaseModel):
|
|
|
130
124
|
id: str
|
|
131
125
|
name: Optional[str]
|
|
132
126
|
fields: Optional[List[DatasourceField]]
|
|
133
|
-
workbook: Optional[Workbook]
|
|
134
127
|
upstreamTables: Optional[List[UpstreamTable]]
|
|
135
128
|
|
|
136
129
|
|
|
@@ -148,7 +141,6 @@ class TableauChart(TableauBaseModel):
|
|
|
148
141
|
Aux class for Chart object of the tableau_api_lib response
|
|
149
142
|
"""
|
|
150
143
|
|
|
151
|
-
workbook: TableauBaseModel
|
|
152
144
|
owner: Optional[TableauOwner]
|
|
153
145
|
tags: Optional[List[TableauTag]] = []
|
|
154
146
|
_extract_tags = validator("tags", pre=True, allow_reuse=True)(transform_tags)
|