openmetadata-ingestion 1.6.7.1__py3-none-any.whl → 1.7.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openmetadata-ingestion might be problematic. Click here for more details.
- metadata/automations/execute_runner.py +49 -0
- metadata/automations/extended_runner.py +13 -0
- metadata/automations/runner.py +34 -35
- metadata/data_quality/validations/table/base/tableCustomSQLQuery.py +11 -1
- metadata/examples/workflows/cassandra.yaml +35 -0
- metadata/examples/workflows/cockroach.yaml +24 -0
- metadata/examples/workflows/nifi.yaml +51 -0
- metadata/examples/workflows/opensearch.yaml +20 -0
- metadata/generated/schema/analytics/__init__.py +1 -1
- metadata/generated/schema/analytics/basic.py +1 -1
- metadata/generated/schema/analytics/reportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
- metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
- metadata/generated/schema/api/__init__.py +1 -1
- metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
- metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
- metadata/generated/schema/api/analytics/__init__.py +1 -1
- metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
- metadata/generated/schema/api/automations/__init__.py +1 -1
- metadata/generated/schema/api/automations/createWorkflow.py +14 -4
- metadata/generated/schema/api/bulkAssets.py +1 -1
- metadata/generated/schema/api/classification/__init__.py +1 -1
- metadata/generated/schema/api/classification/createClassification.py +2 -2
- metadata/generated/schema/api/classification/createTag.py +2 -2
- metadata/generated/schema/api/classification/loadTags.py +1 -1
- metadata/generated/schema/api/createBot.py +1 -1
- metadata/generated/schema/api/createEventPublisherJob.py +1 -1
- metadata/generated/schema/api/createType.py +1 -1
- metadata/generated/schema/api/data/__init__.py +1 -1
- metadata/generated/schema/api/data/createAPICollection.py +1 -1
- metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
- metadata/generated/schema/api/data/createChart.py +1 -1
- metadata/generated/schema/api/data/createContainer.py +1 -1
- metadata/generated/schema/api/data/createCustomProperty.py +1 -1
- metadata/generated/schema/api/data/createDashboard.py +1 -1
- metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
- metadata/generated/schema/api/data/createDatabase.py +1 -1
- metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
- metadata/generated/schema/api/data/createGlossary.py +1 -1
- metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
- metadata/generated/schema/api/data/createMetric.py +1 -1
- metadata/generated/schema/api/data/createMlModel.py +1 -1
- metadata/generated/schema/api/data/createPipeline.py +5 -1
- metadata/generated/schema/api/data/createQuery.py +1 -1
- metadata/generated/schema/api/data/createQueryCostRecord.py +35 -0
- metadata/generated/schema/api/data/createSearchIndex.py +1 -1
- metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
- metadata/generated/schema/api/data/createTable.py +1 -1
- metadata/generated/schema/api/data/createTableProfile.py +1 -1
- metadata/generated/schema/api/data/createTopic.py +1 -1
- metadata/generated/schema/api/data/loadGlossary.py +1 -1
- metadata/generated/schema/api/data/restoreEntity.py +1 -1
- metadata/generated/schema/api/dataInsight/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
- metadata/generated/schema/api/docStore/__init__.py +1 -1
- metadata/generated/schema/api/docStore/createDocument.py +1 -1
- metadata/generated/schema/api/domains/__init__.py +1 -1
- metadata/generated/schema/api/domains/createDataProduct.py +1 -1
- metadata/generated/schema/api/domains/createDomain.py +8 -1
- metadata/generated/schema/api/feed/__init__.py +1 -1
- metadata/generated/schema/api/feed/closeTask.py +1 -1
- metadata/generated/schema/api/feed/createPost.py +1 -1
- metadata/generated/schema/api/feed/createSuggestion.py +1 -1
- metadata/generated/schema/api/feed/createThread.py +1 -1
- metadata/generated/schema/api/feed/resolveTask.py +1 -1
- metadata/generated/schema/api/feed/threadCount.py +25 -1
- metadata/generated/schema/api/governance/__init__.py +1 -1
- metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
- metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
- metadata/generated/schema/api/lineage/__init__.py +1 -1
- metadata/generated/schema/api/lineage/addLineage.py +1 -1
- metadata/generated/schema/api/lineage/esLineageData.py +85 -0
- metadata/generated/schema/api/lineage/lineageDirection.py +12 -0
- metadata/generated/schema/api/lineage/nodeInformation.py +19 -0
- metadata/generated/schema/api/lineage/searchLineageRequest.py +45 -0
- metadata/generated/schema/api/lineage/searchLineageResult.py +46 -0
- metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
- metadata/generated/schema/api/policies/__init__.py +1 -1
- metadata/generated/schema/api/policies/createPolicy.py +1 -1
- metadata/generated/schema/api/search/__init__.py +3 -0
- metadata/generated/schema/api/search/previewSearchRequest.py +47 -0
- metadata/generated/schema/api/services/__init__.py +1 -1
- metadata/generated/schema/api/services/createApiService.py +9 -2
- metadata/generated/schema/api/services/createDashboardService.py +9 -2
- metadata/generated/schema/api/services/createDatabaseService.py +9 -2
- metadata/generated/schema/api/services/createMessagingService.py +9 -2
- metadata/generated/schema/api/services/createMetadataService.py +9 -2
- metadata/generated/schema/api/services/createMlModelService.py +9 -2
- metadata/generated/schema/api/services/createPipelineService.py +9 -2
- metadata/generated/schema/api/services/createSearchService.py +9 -2
- metadata/generated/schema/api/services/createStorageService.py +9 -2
- metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
- metadata/generated/schema/api/setOwner.py +1 -1
- metadata/generated/schema/api/teams/__init__.py +1 -1
- metadata/generated/schema/api/teams/createPersona.py +1 -1
- metadata/generated/schema/api/teams/createRole.py +1 -1
- metadata/generated/schema/api/teams/createTeam.py +1 -1
- metadata/generated/schema/api/teams/createUser.py +1 -1
- metadata/generated/schema/api/tests/__init__.py +1 -1
- metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
- metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
- metadata/generated/schema/api/tests/createTestCase.py +1 -1
- metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
- metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
- metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
- metadata/generated/schema/api/tests/createTestSuite.py +1 -1
- metadata/generated/schema/api/voteRequest.py +1 -1
- metadata/generated/schema/auth/__init__.py +1 -1
- metadata/generated/schema/auth/basicAuth.py +1 -1
- metadata/generated/schema/auth/basicLoginRequest.py +1 -1
- metadata/generated/schema/auth/changePasswordRequest.py +1 -1
- metadata/generated/schema/auth/createPersonalToken.py +1 -1
- metadata/generated/schema/auth/emailRequest.py +1 -1
- metadata/generated/schema/auth/emailVerificationToken.py +1 -1
- metadata/generated/schema/auth/generateToken.py +1 -1
- metadata/generated/schema/auth/jwtAuth.py +1 -1
- metadata/generated/schema/auth/loginRequest.py +1 -1
- metadata/generated/schema/auth/logoutRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetToken.py +1 -1
- metadata/generated/schema/auth/personalAccessToken.py +1 -1
- metadata/generated/schema/auth/refreshToken.py +1 -1
- metadata/generated/schema/auth/registrationRequest.py +1 -1
- metadata/generated/schema/auth/revokePersonalToken.py +1 -1
- metadata/generated/schema/auth/revokeToken.py +1 -1
- metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
- metadata/generated/schema/auth/ssoAuth.py +1 -1
- metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
- metadata/generated/schema/configuration/__init__.py +1 -1
- metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
- metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
- metadata/generated/schema/configuration/authConfig.py +1 -1
- metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
- metadata/generated/schema/configuration/authorizerConfiguration.py +4 -1
- metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
- metadata/generated/schema/configuration/elasticSearchConfiguration.py +55 -1
- metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
- metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
- metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
- metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
- metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
- metadata/generated/schema/configuration/lineageSettings.py +1 -1
- metadata/generated/schema/configuration/loginConfiguration.py +1 -1
- metadata/generated/schema/configuration/logoConfiguration.py +1 -1
- metadata/generated/schema/configuration/openMetadataBaseUrlConfiguration.py +21 -0
- metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
- metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
- metadata/generated/schema/configuration/searchSettings.py +428 -4
- metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
- metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/themeConfiguration.py +1 -1
- metadata/generated/schema/configuration/uiThemePreference.py +1 -1
- metadata/generated/schema/configuration/workflowSettings.py +8 -1
- metadata/generated/schema/dataInsight/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
- metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
- metadata/generated/schema/dataInsight/custom/lineChart.py +15 -1
- metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
- metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
- metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
- metadata/generated/schema/dataInsight/type/__init__.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
- metadata/generated/schema/email/__init__.py +1 -1
- metadata/generated/schema/email/emailRequest.py +1 -1
- metadata/generated/schema/email/emailTemplate.py +1 -1
- metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
- metadata/generated/schema/email/smtpSettings.py +1 -2
- metadata/generated/schema/email/templateValidationReponse.py +1 -1
- metadata/generated/schema/entity/__init__.py +1 -1
- metadata/generated/schema/entity/applications/__init__.py +1 -1
- metadata/generated/schema/entity/applications/app.py +14 -1
- metadata/generated/schema/entity/applications/appExtension.py +1 -1
- metadata/generated/schema/entity/applications/appRunRecord.py +23 -4
- metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/applicationConfig.py +9 -3
- metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addCustomProperties.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDataProductAction.py +41 -0
- metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTestCaseAction.py +71 -0
- metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDataProductAction.py +33 -0
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTestCaseAction.py +45 -0
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +13 -1
- metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.py +46 -0
- metadata/generated/schema/entity/applications/configuration/internal/collateAITierAgentAppConfig.py +46 -0
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +109 -2
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataRetentionConfiguration.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dayOneExperienceAppConfig.py +48 -0
- metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/__init__.py +3 -0
- metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +3 -22
- metadata/generated/schema/entity/applications/configuration/private/internal/__init__.py +3 -0
- metadata/generated/schema/entity/applications/configuration/private/internal/collateAITierAgentAppPrivateConfig.py +24 -0
- metadata/generated/schema/entity/applications/configuration/private/limits.py +33 -0
- metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
- metadata/generated/schema/entity/applications/jobStatus.py +1 -1
- metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +13 -1
- metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +13 -1
- metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
- metadata/generated/schema/entity/automations/__init__.py +1 -1
- metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
- metadata/generated/schema/entity/automations/workflow.py +15 -4
- metadata/generated/schema/entity/bot.py +1 -1
- metadata/generated/schema/entity/classification/__init__.py +1 -1
- metadata/generated/schema/entity/classification/classification.py +1 -1
- metadata/generated/schema/entity/classification/tag.py +1 -1
- metadata/generated/schema/entity/data/__init__.py +1 -1
- metadata/generated/schema/entity/data/apiCollection.py +1 -1
- metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
- metadata/generated/schema/entity/data/chart.py +1 -1
- metadata/generated/schema/entity/data/container.py +1 -1
- metadata/generated/schema/entity/data/dashboard.py +1 -1
- metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
- metadata/generated/schema/entity/data/database.py +5 -1
- metadata/generated/schema/entity/data/databaseSchema.py +7 -5
- metadata/generated/schema/entity/data/glossary.py +1 -1
- metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
- metadata/generated/schema/entity/data/metric.py +1 -1
- metadata/generated/schema/entity/data/mlmodel.py +1 -1
- metadata/generated/schema/entity/data/pipeline.py +14 -1
- metadata/generated/schema/entity/data/query.py +1 -1
- metadata/generated/schema/entity/data/queryCostRecord.py +47 -0
- metadata/generated/schema/entity/data/queryCostSearchResult.py +56 -0
- metadata/generated/schema/entity/data/report.py +1 -1
- metadata/generated/schema/entity/data/searchIndex.py +1 -1
- metadata/generated/schema/entity/data/storedProcedure.py +3 -1
- metadata/generated/schema/entity/data/table.py +6 -1
- metadata/generated/schema/entity/data/topic.py +1 -1
- metadata/generated/schema/entity/docStore/__init__.py +1 -1
- metadata/generated/schema/entity/docStore/document.py +1 -1
- metadata/generated/schema/entity/domains/__init__.py +1 -1
- metadata/generated/schema/entity/domains/dataProduct.py +1 -1
- metadata/generated/schema/entity/domains/domain.py +8 -1
- metadata/generated/schema/entity/events/__init__.py +1 -1
- metadata/generated/schema/entity/events/webhook.py +1 -1
- metadata/generated/schema/entity/feed/__init__.py +1 -1
- metadata/generated/schema/entity/feed/assets.py +1 -1
- metadata/generated/schema/entity/feed/customProperty.py +1 -1
- metadata/generated/schema/entity/feed/description.py +1 -1
- metadata/generated/schema/entity/feed/domain.py +1 -1
- metadata/generated/schema/entity/feed/entityInfo.py +1 -1
- metadata/generated/schema/entity/feed/owner.py +1 -1
- metadata/generated/schema/entity/feed/suggestion.py +1 -1
- metadata/generated/schema/entity/feed/tag.py +1 -1
- metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
- metadata/generated/schema/entity/feed/thread.py +1 -1
- metadata/generated/schema/entity/policies/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
- metadata/generated/schema/entity/policies/filters.py +1 -1
- metadata/generated/schema/entity/policies/policy.py +1 -1
- metadata/generated/schema/entity/services/__init__.py +1 -1
- metadata/generated/schema/entity/services/apiService.py +8 -1
- metadata/generated/schema/entity/services/connections/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/api/restConnection.py +11 -1
- metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/connectionBasicType.py +7 -1
- metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +50 -2
- metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +35 -1
- metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +34 -1
- metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/athenaConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +36 -1
- metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/cassandra/__init__.py +3 -0
- metadata/generated/schema/entity/services/connections/database/cassandra/cloudConfig.py +61 -0
- metadata/generated/schema/entity/services/connections/database/cassandraConnection.py +106 -0
- metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/cockroachConnection.py +136 -0
- metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
- metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/databricksConnection.py +37 -2
- metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/db2Connection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/dorisConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/druidConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/exasolConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/glueConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/hiveConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/icebergConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/impalaConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +42 -2
- metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +32 -2
- metadata/generated/schema/entity/services/connections/database/oracleConnection.py +32 -2
- metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/postgresConnection.py +37 -2
- metadata/generated/schema/entity/services/connections/database/prestoConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +41 -2
- metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/sasConnection.py +27 -1
- metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +58 -3
- metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/synapseConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/teradataConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/trinoConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +37 -2
- metadata/generated/schema/entity/services/connections/database/verticaConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +26 -1
- metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +26 -2
- metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +11 -1
- metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +11 -1
- metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +11 -1
- metadata/generated/schema/entity/services/connections/pipeline/matillion/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/matillion/matillionETL.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifi/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifi/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +11 -1
- metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +37 -29
- metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +10 -1
- metadata/generated/schema/entity/services/connections/storage/s3Connection.py +10 -1
- metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
- metadata/generated/schema/entity/services/dashboardService.py +8 -1
- metadata/generated/schema/entity/services/databaseService.py +14 -1
- metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +8 -1
- metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/reverseIngestionResponse.py +47 -0
- metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
- metadata/generated/schema/entity/services/messagingService.py +8 -1
- metadata/generated/schema/entity/services/metadataService.py +8 -1
- metadata/generated/schema/entity/services/mlmodelService.py +8 -1
- metadata/generated/schema/entity/services/pipelineService.py +8 -1
- metadata/generated/schema/entity/services/searchService.py +9 -2
- metadata/generated/schema/entity/services/serviceType.py +1 -1
- metadata/generated/schema/entity/services/storageService.py +8 -1
- metadata/generated/schema/entity/teams/__init__.py +1 -1
- metadata/generated/schema/entity/teams/persona.py +5 -1
- metadata/generated/schema/entity/teams/role.py +1 -1
- metadata/generated/schema/entity/teams/team.py +1 -1
- metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
- metadata/generated/schema/entity/teams/user.py +5 -1
- metadata/generated/schema/entity/type.py +1 -1
- metadata/generated/schema/entity/utils/__init__.py +1 -1
- metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
- metadata/generated/schema/entity/utils/servicesCount.py +1 -1
- metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
- metadata/generated/schema/events/__init__.py +1 -1
- metadata/generated/schema/events/alertMetrics.py +1 -1
- metadata/generated/schema/events/api/__init__.py +1 -1
- metadata/generated/schema/events/api/createEventSubscription.py +9 -1
- metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
- metadata/generated/schema/events/api/eventsRecord.py +1 -1
- metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
- metadata/generated/schema/events/api/typedEvent.py +1 -1
- metadata/generated/schema/events/emailAlertConfig.py +1 -1
- metadata/generated/schema/events/eventFilterRule.py +1 -1
- metadata/generated/schema/events/eventSubscription.py +9 -2
- metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
- metadata/generated/schema/events/failedEvent.py +1 -1
- metadata/generated/schema/events/failedEventResponse.py +1 -1
- metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
- metadata/generated/schema/events/statusContext.py +1 -1
- metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
- metadata/generated/schema/events/subscriptionStatus.py +1 -1
- metadata/generated/schema/events/testDestinationStatus.py +1 -1
- metadata/generated/schema/governance/workflows/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/edge.py +2 -2
- metadata/generated/schema/governance/workflows/elements/nodeSubType.py +4 -1
- metadata/generated/schema/governance/workflows/elements/nodeType.py +2 -1
- metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +28 -8
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipelineTask.py +86 -0
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.py +73 -0
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +27 -6
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +28 -7
- metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +9 -4
- metadata/generated/schema/governance/workflows/elements/nodes/gateway/__init__.py +3 -0
- metadata/generated/schema/governance/workflows/elements/nodes/gateway/parallelGateway.py +30 -0
- metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +9 -4
- metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +24 -7
- metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +8 -4
- metadata/generated/schema/governance/workflows/elements/triggers/noOpTrigger.py +20 -0
- metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +18 -5
- metadata/generated/schema/governance/workflows/workflowDefinition.py +30 -7
- metadata/generated/schema/governance/workflows/workflowInstance.py +11 -8
- metadata/generated/schema/governance/workflows/workflowInstanceState.py +4 -8
- metadata/generated/schema/jobs/__init__.py +1 -1
- metadata/generated/schema/jobs/backgroundJob.py +1 -1
- metadata/generated/schema/jobs/enumCleanupArgs.py +1 -1
- metadata/generated/schema/metadataIngestion/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/application.py +1 -1
- metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +4 -4
- metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +5 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +18 -2
- metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +9 -1
- metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/reverseIngestionPipeline.py +64 -0
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/__init__.py +3 -0
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.py +24 -0
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/ownerConfig.py +28 -0
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/tagsConfig.py +28 -0
- metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/workflow.py +3 -1
- metadata/generated/schema/monitoring/__init__.py +1 -1
- metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
- metadata/generated/schema/search/__init__.py +3 -0
- metadata/generated/schema/search/searchRequest.py +106 -0
- metadata/generated/schema/security/__init__.py +1 -1
- metadata/generated/schema/security/client/__init__.py +1 -1
- metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
- metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
- metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
- metadata/generated/schema/security/credentials/__init__.py +1 -1
- metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
- metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
- metadata/generated/schema/security/credentials/basicAuth.py +1 -1
- metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpCredentials.py +42 -22
- metadata/generated/schema/security/credentials/gcpExternalAccount.py +2 -2
- metadata/generated/schema/security/credentials/gcpValues.py +1 -1
- metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
- metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
- metadata/generated/schema/security/sasl/__init__.py +1 -1
- metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
- metadata/generated/schema/security/secrets/__init__.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
- metadata/generated/schema/security/securityConfiguration.py +1 -1
- metadata/generated/schema/security/ssl/__init__.py +1 -1
- metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
- metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
- metadata/generated/schema/settings/__init__.py +1 -1
- metadata/generated/schema/settings/settings.py +4 -1
- metadata/generated/schema/system/__init__.py +1 -1
- metadata/generated/schema/system/entityError.py +1 -1
- metadata/generated/schema/system/eventPublisherJob.py +1 -1
- metadata/generated/schema/system/indexingError.py +1 -1
- metadata/generated/schema/system/limitsResponse.py +1 -1
- metadata/generated/schema/system/ui/__init__.py +1 -1
- metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
- metadata/generated/schema/system/ui/navigationItem.py +38 -0
- metadata/generated/schema/system/ui/page.py +23 -15
- metadata/generated/schema/system/ui/tab.py +36 -0
- metadata/generated/schema/system/ui/uiCustomization.py +45 -0
- metadata/generated/schema/system/validationResponse.py +1 -1
- metadata/generated/schema/tests/__init__.py +1 -1
- metadata/generated/schema/tests/assigned.py +1 -1
- metadata/generated/schema/tests/basic.py +1 -1
- metadata/generated/schema/tests/customMetric.py +1 -1
- metadata/generated/schema/tests/dataQualityReport.py +1 -1
- metadata/generated/schema/tests/resolved.py +1 -1
- metadata/generated/schema/tests/testCase.py +4 -1
- metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
- metadata/generated/schema/tests/testDefinition.py +1 -1
- metadata/generated/schema/tests/testSuite.py +1 -1
- metadata/generated/schema/type/__init__.py +1 -1
- metadata/generated/schema/type/apiSchema.py +1 -1
- metadata/generated/schema/type/assetCertification.py +1 -1
- metadata/generated/schema/type/auditLog.py +1 -1
- metadata/generated/schema/type/basic.py +1 -1
- metadata/generated/schema/type/bulkOperationResult.py +1 -1
- metadata/generated/schema/type/changeEvent.py +1 -1
- metadata/generated/schema/type/changeEventType.py +1 -1
- metadata/generated/schema/type/changeSummaryMap.py +37 -0
- metadata/generated/schema/type/collectionDescriptor.py +1 -1
- metadata/generated/schema/type/csvDocumentation.py +1 -1
- metadata/generated/schema/type/csvErrorType.py +1 -1
- metadata/generated/schema/type/csvFile.py +1 -1
- metadata/generated/schema/type/csvImportResult.py +1 -1
- metadata/generated/schema/type/customProperties/__init__.py +1 -1
- metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
- metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
- metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
- metadata/generated/schema/type/customProperty.py +1 -1
- metadata/generated/schema/type/dailyCount.py +1 -1
- metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
- metadata/generated/schema/type/entityHierarchy.py +1 -1
- metadata/generated/schema/type/entityHistory.py +6 -3
- metadata/generated/schema/type/entityLineage.py +27 -1
- metadata/generated/schema/type/entityReference.py +1 -1
- metadata/generated/schema/type/entityReferenceList.py +1 -1
- metadata/generated/schema/type/entityRelationship.py +1 -1
- metadata/generated/schema/type/entityUsage.py +1 -1
- metadata/generated/schema/type/filterPattern.py +1 -1
- metadata/generated/schema/type/function.py +1 -1
- metadata/generated/schema/type/include.py +1 -1
- metadata/generated/schema/type/jdbcConnection.py +1 -1
- metadata/generated/schema/type/lifeCycle.py +1 -1
- metadata/generated/schema/type/paging.py +1 -1
- metadata/generated/schema/type/profile.py +1 -1
- metadata/generated/schema/type/queryParserData.py +4 -1
- metadata/generated/schema/type/reaction.py +1 -1
- metadata/generated/schema/type/schedule.py +1 -1
- metadata/generated/schema/type/schema.py +1 -1
- metadata/generated/schema/type/tableQuery.py +4 -1
- metadata/generated/schema/type/tableUsageCount.py +18 -1
- metadata/generated/schema/type/tagLabel.py +1 -1
- metadata/generated/schema/type/usageDetails.py +1 -1
- metadata/generated/schema/type/usageRequest.py +1 -1
- metadata/generated/schema/type/votes.py +1 -1
- metadata/ingestion/api/topology_runner.py +3 -3
- metadata/ingestion/bulksink/metadata_usage.py +27 -6
- metadata/ingestion/lineage/masker.py +13 -2
- metadata/ingestion/models/custom_pydantic.py +44 -6
- metadata/ingestion/ometa/client.py +20 -0
- metadata/ingestion/ometa/mixins/domain_mixin.py +80 -0
- metadata/ingestion/ometa/mixins/es_mixin.py +42 -2
- metadata/ingestion/ometa/mixins/lineage_mixin.py +7 -33
- metadata/ingestion/ometa/mixins/patch_mixin.py +15 -5
- metadata/ingestion/ometa/mixins/pipeline_mixin.py +18 -0
- metadata/ingestion/ometa/mixins/query_mixin.py +43 -0
- metadata/ingestion/ometa/ometa_api.py +28 -3
- metadata/ingestion/ometa/routes.py +7 -0
- metadata/ingestion/ometa/utils.py +13 -0
- metadata/ingestion/processor/query_parser.py +1 -0
- metadata/ingestion/sink/metadata_rest.py +21 -1
- metadata/ingestion/source/dashboard/dashboard_service.py +9 -1
- metadata/ingestion/source/dashboard/domodashboard/metadata.py +3 -1
- metadata/ingestion/source/dashboard/lightdash/metadata.py +1 -1
- metadata/ingestion/source/dashboard/looker/metadata.py +8 -2
- metadata/ingestion/source/dashboard/metabase/client.py +65 -6
- metadata/ingestion/source/dashboard/metabase/metadata.py +88 -37
- metadata/ingestion/source/dashboard/metabase/models.py +2 -1
- metadata/ingestion/source/dashboard/microstrategy/metadata.py +3 -1
- metadata/ingestion/source/dashboard/mode/metadata.py +16 -9
- metadata/ingestion/source/dashboard/powerbi/client.py +3 -14
- metadata/ingestion/source/dashboard/powerbi/metadata.py +6 -54
- metadata/ingestion/source/dashboard/powerbi/models.py +0 -12
- metadata/ingestion/source/dashboard/qlikcloud/client.py +25 -2
- metadata/ingestion/source/dashboard/qlikcloud/metadata.py +69 -15
- metadata/ingestion/source/dashboard/qlikcloud/models.py +44 -1
- metadata/ingestion/source/dashboard/qliksense/metadata.py +26 -18
- metadata/ingestion/source/dashboard/quicksight/metadata.py +32 -29
- metadata/ingestion/source/dashboard/redash/metadata.py +12 -10
- metadata/ingestion/source/dashboard/sigma/metadata.py +12 -14
- metadata/ingestion/source/dashboard/superset/api_source.py +25 -26
- metadata/ingestion/source/dashboard/superset/db_source.py +22 -20
- metadata/ingestion/source/dashboard/superset/mixin.py +207 -53
- metadata/ingestion/source/dashboard/superset/models.py +3 -0
- metadata/ingestion/source/dashboard/superset/queries.py +7 -4
- metadata/ingestion/source/dashboard/superset/utils.py +36 -0
- metadata/ingestion/source/dashboard/tableau/client.py +56 -1
- metadata/ingestion/source/dashboard/tableau/metadata.py +243 -128
- metadata/ingestion/source/database/bigquery/helper.py +9 -1
- metadata/ingestion/source/database/bigquery/metadata.py +82 -10
- metadata/ingestion/source/database/bigquery/queries.py +2 -1
- metadata/ingestion/source/database/bigquery/query_parser.py +1 -0
- metadata/ingestion/source/database/bigtable/metadata.py +11 -3
- metadata/ingestion/source/database/cassandra/__init__.py +0 -0
- metadata/ingestion/source/database/cassandra/connection.py +138 -0
- metadata/ingestion/source/database/cassandra/helpers.py +100 -0
- metadata/ingestion/source/database/cassandra/metadata.py +138 -0
- metadata/ingestion/source/database/cassandra/queries.py +45 -0
- metadata/ingestion/source/database/cassandra/service_spec.py +10 -0
- metadata/ingestion/source/database/cockroach/__init__.py +0 -0
- metadata/ingestion/source/database/cockroach/connection.py +71 -0
- metadata/ingestion/source/database/cockroach/metadata.py +206 -0
- metadata/ingestion/source/database/cockroach/queries.py +90 -0
- metadata/ingestion/source/database/cockroach/service_spec.py +4 -0
- metadata/ingestion/source/database/common_db_source.py +9 -0
- metadata/ingestion/source/database/common_nosql_source.py +78 -29
- metadata/ingestion/source/database/common_pg_mappings.py +49 -0
- metadata/ingestion/source/database/couchbase/metadata.py +8 -2
- metadata/ingestion/source/database/databricks/metadata.py +8 -0
- metadata/ingestion/source/database/datalake/metadata.py +15 -9
- metadata/ingestion/source/database/db2/service_spec.py +4 -1
- metadata/ingestion/source/database/dbt/dbt_config.py +1 -0
- metadata/ingestion/source/database/dbt/dbt_utils.py +1 -1
- metadata/ingestion/source/database/dbt/metadata.py +5 -1
- metadata/ingestion/source/database/doris/metadata.py +7 -2
- metadata/ingestion/source/database/druid/service_spec.py +5 -1
- metadata/ingestion/source/database/dynamodb/metadata.py +6 -3
- metadata/ingestion/source/database/greenplum/metadata.py +6 -37
- metadata/ingestion/source/database/greenplum/service_spec.py +5 -1
- metadata/ingestion/source/database/hive/metastore_dialects/mysql/dialect.py +35 -13
- metadata/ingestion/source/database/hive/metastore_dialects/postgres/dialect.py +38 -14
- metadata/ingestion/source/database/hive/service_spec.py +5 -1
- metadata/ingestion/source/database/impala/service_spec.py +5 -1
- metadata/ingestion/source/database/life_cycle_query_mixin.py +11 -7
- metadata/ingestion/source/database/lineage_source.py +75 -2
- metadata/ingestion/source/database/mariadb/service_spec.py +9 -1
- metadata/ingestion/source/database/mongodb/metadata.py +8 -3
- metadata/ingestion/source/database/mssql/metadata.py +68 -2
- metadata/ingestion/source/database/mssql/queries.py +43 -0
- metadata/ingestion/source/database/mysql/utils.py +5 -3
- metadata/ingestion/source/database/oracle/connection.py +24 -2
- metadata/ingestion/source/database/oracle/metadata.py +36 -19
- metadata/ingestion/source/database/oracle/models.py +5 -4
- metadata/ingestion/source/database/oracle/queries.py +22 -1
- metadata/ingestion/source/database/pinotdb/service_spec.py +5 -1
- metadata/ingestion/source/database/postgres/lineage.py +1 -2
- metadata/ingestion/source/database/postgres/metadata.py +32 -56
- metadata/ingestion/source/database/postgres/models.py +1 -0
- metadata/ingestion/source/database/postgres/queries.py +18 -1
- metadata/ingestion/source/database/query_parser_source.py +11 -6
- metadata/ingestion/source/database/sample_data.py +11 -0
- metadata/ingestion/source/database/sample_usage.py +2 -0
- metadata/ingestion/source/database/saphana/lineage.py +1 -1
- metadata/ingestion/source/database/singlestore/service_spec.py +6 -1
- metadata/ingestion/source/database/snowflake/connection.py +6 -0
- metadata/ingestion/source/database/snowflake/constants.py +48 -0
- metadata/ingestion/source/database/snowflake/metadata.py +114 -3
- metadata/ingestion/source/database/snowflake/models.py +3 -3
- metadata/ingestion/source/database/snowflake/queries.py +27 -1
- metadata/ingestion/source/database/snowflake/query_parser.py +2 -0
- metadata/ingestion/source/database/snowflake/utils.py +110 -26
- metadata/ingestion/source/database/sql_column_handler.py +95 -60
- metadata/ingestion/source/database/sqlite/service_spec.py +5 -1
- metadata/ingestion/source/database/teradata/service_spec.py +5 -1
- metadata/ingestion/source/database/trino/lineage.py +95 -0
- metadata/ingestion/source/database/unitycatalog/connection.py +23 -0
- metadata/ingestion/source/database/unitycatalog/lineage.py +2 -0
- metadata/ingestion/source/database/unitycatalog/metadata.py +2 -0
- metadata/ingestion/source/database/usage_source.py +2 -0
- metadata/ingestion/source/messaging/messaging_service.py +2 -0
- metadata/ingestion/source/metadata/alationsink/metadata.py +32 -18
- metadata/ingestion/source/metadata/amundsen/metadata.py +7 -4
- metadata/ingestion/source/metadata/atlas/metadata.py +2 -1
- metadata/ingestion/source/mlmodel/mlmodel_service.py +2 -0
- metadata/ingestion/source/pipeline/airflow/metadata.py +25 -0
- metadata/ingestion/source/pipeline/airflow/models.py +1 -0
- metadata/ingestion/source/pipeline/nifi/client.py +68 -74
- metadata/ingestion/source/pipeline/nifi/connection.py +1 -18
- metadata/ingestion/source/pipeline/openlineage/metadata.py +1 -0
- metadata/ingestion/source/pipeline/pipeline_service.py +121 -2
- metadata/ingestion/source/search/opensearch/connection.py +215 -0
- metadata/ingestion/source/search/opensearch/metadata.py +270 -0
- metadata/ingestion/source/search/opensearch/parser.py +71 -0
- metadata/ingestion/source/search/opensearch/service_spec.py +4 -0
- metadata/ingestion/source/search/search_service.py +2 -0
- metadata/ingestion/source/storage/storage_service.py +2 -0
- metadata/ingestion/stage/table_usage.py +53 -1
- metadata/profiler/interface/sqlalchemy/unity_catalog/sampler_interface.py +3 -4
- metadata/profiler/orm/converter/azuresql/converter.py +39 -0
- metadata/profiler/orm/converter/converter_registry.py +2 -0
- metadata/profiler/orm/functions/length.py +2 -0
- metadata/profiler/orm/functions/median.py +19 -0
- metadata/profiler/orm/functions/modulo.py +1 -0
- metadata/profiler/orm/functions/random_num.py +1 -0
- metadata/profiler/orm/functions/sum.py +1 -0
- metadata/profiler/orm/registry.py +2 -0
- metadata/profiler/processor/core.py +1 -0
- metadata/profiler/processor/metric_filter.py +1 -1
- metadata/profiler/source/database/base/profiler_source.py +1 -0
- metadata/profiler/source/fetcher/fetcher_strategy.py +4 -0
- metadata/sampler/config.py +1 -0
- metadata/sampler/models.py +3 -0
- metadata/sampler/nosql/sampler.py +4 -0
- metadata/sampler/pandas/sampler.py +1 -0
- metadata/sampler/partition.py +9 -2
- metadata/sampler/sampler_interface.py +0 -1
- metadata/sampler/sqlalchemy/azuresql/sampler.py +30 -3
- metadata/sampler/sqlalchemy/mssql/sampler.py +2 -1
- metadata/sampler/sqlalchemy/sampler.py +51 -42
- metadata/sampler/sqlalchemy/snowflake/sampler.py +2 -1
- metadata/sampler/sqlalchemy/trino/sampler.py +9 -8
- metadata/utils/constants.py +1 -0
- metadata/utils/credentials.py +25 -4
- metadata/utils/db_utils.py +1 -11
- metadata/utils/elasticsearch.py +4 -0
- metadata/utils/helpers.py +91 -0
- metadata/utils/logger.py +9 -0
- metadata/utils/ssl_manager.py +35 -0
- metadata/utils/tag_utils.py +72 -27
- metadata/workflow/classification.py +3 -0
- metadata/workflow/profiler.py +13 -5
- metadata/workflow/workflow_status_mixin.py +6 -0
- {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/METADATA +433 -370
- {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/RECORD +867 -798
- {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/LICENSE +0 -0
- {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/WHEEL +0 -0
- {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/entry_points.txt +0 -0
- {openmetadata_ingestion-1.6.7.1.dist-info → openmetadata_ingestion-1.7.0.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -13,7 +13,7 @@ REST Auth & Client for Metabase
|
|
|
13
13
|
"""
|
|
14
14
|
import json
|
|
15
15
|
import traceback
|
|
16
|
-
from typing import List, Optional
|
|
16
|
+
from typing import Dict, List, Optional
|
|
17
17
|
|
|
18
18
|
import requests
|
|
19
19
|
|
|
@@ -23,6 +23,7 @@ from metadata.generated.schema.entity.services.connections.dashboard.metabaseCon
|
|
|
23
23
|
from metadata.ingestion.connections.test_connections import SourceConnectionException
|
|
24
24
|
from metadata.ingestion.ometa.client import REST, ClientConfig
|
|
25
25
|
from metadata.ingestion.source.dashboard.metabase.models import (
|
|
26
|
+
MetabaseChart,
|
|
26
27
|
MetabaseCollection,
|
|
27
28
|
MetabaseCollectionList,
|
|
28
29
|
MetabaseDashboard,
|
|
@@ -32,7 +33,11 @@ from metadata.ingestion.source.dashboard.metabase.models import (
|
|
|
32
33
|
MetabaseTable,
|
|
33
34
|
MetabaseUser,
|
|
34
35
|
)
|
|
35
|
-
from metadata.utils.constants import
|
|
36
|
+
from metadata.utils.constants import (
|
|
37
|
+
AUTHORIZATION_HEADER,
|
|
38
|
+
DEFAULT_DASHBAORD,
|
|
39
|
+
NO_ACCESS_TOKEN,
|
|
40
|
+
)
|
|
36
41
|
from metadata.utils.helpers import clean_uri
|
|
37
42
|
from metadata.utils.logger import ingestion_logger
|
|
38
43
|
|
|
@@ -146,22 +151,76 @@ class MetabaseClient:
|
|
|
146
151
|
logger.warning("Failed to fetch the collections list")
|
|
147
152
|
return []
|
|
148
153
|
|
|
154
|
+
def get_charts_dict(self) -> Dict:
|
|
155
|
+
charts_dict = {}
|
|
156
|
+
try:
|
|
157
|
+
resp_charts = self.client.get("/card")
|
|
158
|
+
if resp_charts:
|
|
159
|
+
for chart_data in resp_charts:
|
|
160
|
+
chart = MetabaseChart.model_validate(chart_data)
|
|
161
|
+
charts_dict[chart.id] = chart
|
|
162
|
+
return charts_dict
|
|
163
|
+
except Exception:
|
|
164
|
+
logger.debug(traceback.format_exc())
|
|
165
|
+
logger.warning("Failed to fetch the cards")
|
|
166
|
+
return {}
|
|
167
|
+
|
|
168
|
+
def _create_default_dashboard_details(
|
|
169
|
+
self, orphan_charts_id: List
|
|
170
|
+
) -> MetabaseDashboardDetails:
|
|
171
|
+
"""
|
|
172
|
+
Returns:
|
|
173
|
+
MetabaseDashboardDetails object representing the default dashboard containing orphaned charts
|
|
174
|
+
"""
|
|
175
|
+
return MetabaseDashboardDetails(
|
|
176
|
+
id=DEFAULT_DASHBAORD,
|
|
177
|
+
card_ids=orphan_charts_id,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
def _process_dashboard_response(
|
|
181
|
+
self, resp_dashboard: Dict, charts_dict: Dict, dashboard_id: str
|
|
182
|
+
) -> MetabaseDashboardDetails:
|
|
183
|
+
"""
|
|
184
|
+
Process dashboard response and create MetabaseDashboardDetails object
|
|
185
|
+
"""
|
|
186
|
+
if "ordered_cards" in resp_dashboard:
|
|
187
|
+
resp_dashboard["dashcards"] = resp_dashboard["ordered_cards"]
|
|
188
|
+
|
|
189
|
+
card_ids = []
|
|
190
|
+
for card in resp_dashboard.get("dashcards", []):
|
|
191
|
+
if card.get("card") and card["card"].get("id"):
|
|
192
|
+
card_id = str(card["card"]["id"])
|
|
193
|
+
card_ids.append(card_id)
|
|
194
|
+
if card_id in charts_dict:
|
|
195
|
+
charts_dict[card_id].dashboard_ids.append(dashboard_id)
|
|
196
|
+
|
|
197
|
+
return MetabaseDashboardDetails(
|
|
198
|
+
description=resp_dashboard.get("description"),
|
|
199
|
+
name=resp_dashboard.get("name"),
|
|
200
|
+
id=resp_dashboard.get("id"),
|
|
201
|
+
creator_id=resp_dashboard.get("creator_id"),
|
|
202
|
+
collection_id=resp_dashboard.get("collection_id"),
|
|
203
|
+
card_ids=card_ids,
|
|
204
|
+
)
|
|
205
|
+
|
|
149
206
|
def get_dashboard_details(
|
|
150
|
-
self, dashboard_id: str
|
|
207
|
+
self, dashboard_id: str, charts_dict: Dict, orphan_charts_id: List
|
|
151
208
|
) -> Optional[MetabaseDashboardDetails]:
|
|
152
209
|
"""
|
|
153
210
|
Get Dashboard Details
|
|
154
211
|
"""
|
|
155
212
|
if not dashboard_id:
|
|
156
213
|
return None # don't call api if dashboard_id is None
|
|
214
|
+
if dashboard_id == DEFAULT_DASHBAORD:
|
|
215
|
+
return self._create_default_dashboard_details(orphan_charts_id)
|
|
157
216
|
try:
|
|
158
217
|
resp_dashboard = self.client.get(f"/dashboard/{dashboard_id}")
|
|
159
218
|
if resp_dashboard:
|
|
160
219
|
# Small hack needed to support Metabase versions older than 0.48
|
|
161
220
|
# https://www.metabase.com/releases/metabase-48#fyi--breaking-changes
|
|
162
|
-
|
|
163
|
-
resp_dashboard
|
|
164
|
-
|
|
221
|
+
return self._process_dashboard_response(
|
|
222
|
+
resp_dashboard, charts_dict, dashboard_id
|
|
223
|
+
)
|
|
165
224
|
except Exception:
|
|
166
225
|
logger.debug(traceback.format_exc())
|
|
167
226
|
logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}")
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"""Metabase source module"""
|
|
12
12
|
|
|
13
13
|
import traceback
|
|
14
|
-
from typing import Any, Iterable, List, Optional
|
|
14
|
+
from typing import Any, Dict, Iterable, List, Optional
|
|
15
15
|
|
|
16
16
|
from metadata.generated.schema.api.data.createChart import CreateChartRequest
|
|
17
17
|
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
|
|
@@ -20,6 +20,7 @@ from metadata.generated.schema.entity.data.chart import Chart
|
|
|
20
20
|
from metadata.generated.schema.entity.data.dashboard import (
|
|
21
21
|
Dashboard as LineageDashboard,
|
|
22
22
|
)
|
|
23
|
+
from metadata.generated.schema.entity.data.table import Table
|
|
23
24
|
from metadata.generated.schema.entity.services.connections.dashboard.metabaseConnection import (
|
|
24
25
|
MetabaseConnection,
|
|
25
26
|
)
|
|
@@ -42,9 +43,8 @@ from metadata.generated.schema.type.basic import (
|
|
|
42
43
|
from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
|
|
43
44
|
from metadata.ingestion.api.models import Either
|
|
44
45
|
from metadata.ingestion.api.steps import InvalidSourceException
|
|
45
|
-
from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper
|
|
46
|
+
from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
|
|
46
47
|
from metadata.ingestion.lineage.parser import LineageParser
|
|
47
|
-
from metadata.ingestion.lineage.sql_lineage import search_table_entities
|
|
48
48
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
49
49
|
from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
|
|
50
50
|
from metadata.ingestion.source.dashboard.metabase.models import (
|
|
@@ -54,7 +54,9 @@ from metadata.ingestion.source.dashboard.metabase.models import (
|
|
|
54
54
|
MetabaseDashboardDetails,
|
|
55
55
|
)
|
|
56
56
|
from metadata.utils import fqn
|
|
57
|
+
from metadata.utils.constants import DEFAULT_DASHBAORD
|
|
57
58
|
from metadata.utils.filters import filter_by_chart
|
|
59
|
+
from metadata.utils.fqn import build_es_fqn_search_string
|
|
58
60
|
from metadata.utils.helpers import (
|
|
59
61
|
clean_uri,
|
|
60
62
|
get_standard_chart_type,
|
|
@@ -92,16 +94,22 @@ class MetabaseSource(DashboardServiceSource):
|
|
|
92
94
|
):
|
|
93
95
|
super().__init__(config, metadata)
|
|
94
96
|
self.collections: List[MetabaseCollection] = []
|
|
97
|
+
self.dashboards_list: List[MetabaseDashboard] = []
|
|
98
|
+
self.charts_dict: Dict[str] = {}
|
|
99
|
+
self.orphan_charts_id: List[str] = []
|
|
100
|
+
self._default_dashboard_added = False
|
|
95
101
|
|
|
96
102
|
def prepare(self):
|
|
97
103
|
self.collections = self.client.get_collections_list()
|
|
104
|
+
self.charts_dict = self.client.get_charts_dict()
|
|
98
105
|
return super().prepare()
|
|
99
106
|
|
|
100
107
|
def get_dashboards_list(self) -> Optional[List[MetabaseDashboard]]:
|
|
101
108
|
"""
|
|
102
109
|
Get List of all dashboards
|
|
103
110
|
"""
|
|
104
|
-
|
|
111
|
+
self.dashboards_list = self.client.get_dashboards_list(self.collections)
|
|
112
|
+
return self.dashboards_list
|
|
105
113
|
|
|
106
114
|
def get_dashboard_name(self, dashboard: MetabaseDashboard) -> str:
|
|
107
115
|
"""
|
|
@@ -115,13 +123,39 @@ class MetabaseSource(DashboardServiceSource):
|
|
|
115
123
|
"""
|
|
116
124
|
Get Dashboard Details
|
|
117
125
|
"""
|
|
118
|
-
|
|
126
|
+
retrieved_dashboards = self.client.get_dashboard_details(
|
|
127
|
+
dashboard.id, self.charts_dict, self.orphan_charts_id
|
|
128
|
+
)
|
|
129
|
+
if (
|
|
130
|
+
retrieved_dashboards
|
|
131
|
+
and dashboard == self.dashboards_list[-1]
|
|
132
|
+
and not self._default_dashboard_added
|
|
133
|
+
):
|
|
134
|
+
# If processing the last dashboard, identify any orphaned charts (not associated with dashboards)
|
|
135
|
+
# and create a default dashboard to maintain visibility of these charts
|
|
136
|
+
self.orphan_charts_id = [
|
|
137
|
+
chart_id
|
|
138
|
+
for chart_id, chart in self.charts_dict.items()
|
|
139
|
+
if not chart.dashboard_ids
|
|
140
|
+
]
|
|
141
|
+
if self.orphan_charts_id:
|
|
142
|
+
# add the default dashboard to the dashboards list
|
|
143
|
+
default_dashboard = MetabaseDashboard(
|
|
144
|
+
name=DEFAULT_DASHBAORD,
|
|
145
|
+
id=DEFAULT_DASHBAORD,
|
|
146
|
+
)
|
|
147
|
+
self.dashboards_list.append(default_dashboard)
|
|
148
|
+
self._default_dashboard_added = True
|
|
149
|
+
return retrieved_dashboards
|
|
119
150
|
|
|
120
151
|
def get_project_name(self, dashboard_details: Any) -> Optional[str]:
|
|
121
152
|
"""
|
|
122
153
|
Method to get the project name by searching the dataset using id in the workspace dict
|
|
123
154
|
"""
|
|
124
155
|
try:
|
|
156
|
+
# Return default for the default dashboard containing orphaned charts
|
|
157
|
+
if dashboard_details.id == DEFAULT_DASHBAORD:
|
|
158
|
+
return DEFAULT_DASHBAORD
|
|
125
159
|
if dashboard_details.collection_id:
|
|
126
160
|
collection_name = next(
|
|
127
161
|
(
|
|
@@ -164,10 +198,14 @@ class MetabaseSource(DashboardServiceSource):
|
|
|
164
198
|
Method to Get Dashboard Entity
|
|
165
199
|
"""
|
|
166
200
|
try:
|
|
201
|
+
# dashboard_url to be empty for default dashboard
|
|
167
202
|
dashboard_url = (
|
|
168
|
-
|
|
203
|
+
""
|
|
204
|
+
if dashboard_details.id == DEFAULT_DASHBAORD
|
|
205
|
+
else f"{clean_uri(self.service_connection.hostPort)}/dashboard/{dashboard_details.id}-"
|
|
169
206
|
f"{replace_special_with(raw=dashboard_details.name.lower(), replacement='-')}"
|
|
170
207
|
)
|
|
208
|
+
|
|
171
209
|
dashboard_request = CreateDashboardRequest(
|
|
172
210
|
name=EntityName(str(dashboard_details.id)),
|
|
173
211
|
sourceUrl=SourceUrl(dashboard_url),
|
|
@@ -211,10 +249,10 @@ class MetabaseSource(DashboardServiceSource):
|
|
|
211
249
|
Returns:
|
|
212
250
|
Iterable[CreateChartRequest]
|
|
213
251
|
"""
|
|
214
|
-
|
|
215
|
-
for
|
|
252
|
+
chart_ids = dashboard_details.card_ids
|
|
253
|
+
for chart_id in chart_ids:
|
|
216
254
|
try:
|
|
217
|
-
chart_details =
|
|
255
|
+
chart_details = self.charts_dict[chart_id]
|
|
218
256
|
if not chart_details.id or not chart_details.name:
|
|
219
257
|
continue
|
|
220
258
|
chart_url = (
|
|
@@ -240,7 +278,7 @@ class MetabaseSource(DashboardServiceSource):
|
|
|
240
278
|
yield Either(
|
|
241
279
|
left=StackTraceError(
|
|
242
280
|
name="Chart",
|
|
243
|
-
error=f"Error creating chart [{
|
|
281
|
+
error=f"Error creating chart [{self.charts_dict[chart_id]}]: {exc}",
|
|
244
282
|
stackTrace=traceback.format_exc(),
|
|
245
283
|
)
|
|
246
284
|
)
|
|
@@ -248,22 +286,20 @@ class MetabaseSource(DashboardServiceSource):
|
|
|
248
286
|
def yield_dashboard_lineage_details(
|
|
249
287
|
self,
|
|
250
288
|
dashboard_details: MetabaseDashboardDetails,
|
|
251
|
-
db_service_name: Optional[str],
|
|
289
|
+
db_service_name: Optional[str] = None,
|
|
252
290
|
) -> Iterable[Either[AddLineageRequest]]:
|
|
253
291
|
"""Get lineage method
|
|
254
292
|
|
|
255
293
|
Args:
|
|
256
294
|
dashboard_details
|
|
257
295
|
"""
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
chart_list, dashboard_name = (
|
|
261
|
-
dashboard_details.dashcards,
|
|
296
|
+
chart_ids, dashboard_name = (
|
|
297
|
+
dashboard_details.card_ids,
|
|
262
298
|
str(dashboard_details.id),
|
|
263
299
|
)
|
|
264
|
-
for
|
|
300
|
+
for chart_id in chart_ids:
|
|
265
301
|
try:
|
|
266
|
-
chart_details =
|
|
302
|
+
chart_details = self.charts_dict[chart_id]
|
|
267
303
|
if (
|
|
268
304
|
chart_details.dataset_query is None
|
|
269
305
|
or chart_details.dataset_query.type is None
|
|
@@ -296,11 +332,17 @@ class MetabaseSource(DashboardServiceSource):
|
|
|
296
332
|
)
|
|
297
333
|
)
|
|
298
334
|
|
|
299
|
-
def _get_database_service(self, db_service_name: str):
|
|
335
|
+
def _get_database_service(self, db_service_name: Optional[str]):
|
|
336
|
+
if not db_service_name:
|
|
337
|
+
return None
|
|
300
338
|
return self.metadata.get_by_name(DatabaseService, db_service_name)
|
|
301
339
|
|
|
340
|
+
# pylint: disable=too-many-locals
|
|
302
341
|
def _yield_lineage_from_query(
|
|
303
|
-
self,
|
|
342
|
+
self,
|
|
343
|
+
chart_details: MetabaseChart,
|
|
344
|
+
db_service_name: Optional[str],
|
|
345
|
+
dashboard_name: str,
|
|
304
346
|
) -> Iterable[Either[AddLineageRequest]]:
|
|
305
347
|
database = self.client.get_database(chart_details.database_id)
|
|
306
348
|
|
|
@@ -323,20 +365,23 @@ class MetabaseSource(DashboardServiceSource):
|
|
|
323
365
|
query,
|
|
324
366
|
ConnectionTypeDialectMapper.dialect_of(db_service.serviceType.value)
|
|
325
367
|
if db_service
|
|
326
|
-
else
|
|
368
|
+
else Dialect.ANSI,
|
|
327
369
|
)
|
|
328
370
|
|
|
329
371
|
for table in lineage_parser.source_tables:
|
|
330
372
|
database_schema_name, table = fqn.split(str(table))[-2:]
|
|
331
373
|
database_schema_name = self.check_database_schema_name(database_schema_name)
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
service_name=db_service_name,
|
|
336
|
-
|
|
337
|
-
|
|
374
|
+
fqn_search_string = build_es_fqn_search_string(
|
|
375
|
+
database_name=database_name,
|
|
376
|
+
schema_name=database_schema_name,
|
|
377
|
+
service_name=db_service_name or "*",
|
|
378
|
+
table_name=table,
|
|
379
|
+
)
|
|
380
|
+
from_entities = self.metadata.search_in_any_service(
|
|
381
|
+
entity_type=Table,
|
|
382
|
+
fqn_search_string=fqn_search_string,
|
|
383
|
+
fetch_multiple_entities=True,
|
|
338
384
|
)
|
|
339
|
-
|
|
340
385
|
to_fqn = fqn.build(
|
|
341
386
|
self.metadata,
|
|
342
387
|
entity_type=LineageDashboard,
|
|
@@ -348,13 +393,16 @@ class MetabaseSource(DashboardServiceSource):
|
|
|
348
393
|
fqn=to_fqn,
|
|
349
394
|
)
|
|
350
395
|
|
|
351
|
-
for from_entity in from_entities:
|
|
396
|
+
for from_entity in from_entities or []:
|
|
352
397
|
yield self._get_add_lineage_request(
|
|
353
398
|
to_entity=to_entity, from_entity=from_entity
|
|
354
399
|
)
|
|
355
400
|
|
|
356
401
|
def _yield_lineage_from_api(
|
|
357
|
-
self,
|
|
402
|
+
self,
|
|
403
|
+
chart_details: MetabaseChart,
|
|
404
|
+
db_service_name: Optional[str],
|
|
405
|
+
dashboard_name: str,
|
|
358
406
|
) -> Iterable[Either[AddLineageRequest]]:
|
|
359
407
|
table = self.client.get_table(chart_details.table_id)
|
|
360
408
|
table_name = table.name or table.display_name
|
|
@@ -363,14 +411,17 @@ class MetabaseSource(DashboardServiceSource):
|
|
|
363
411
|
return
|
|
364
412
|
|
|
365
413
|
database_name = table.db.details.db if table.db and table.db.details else None
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
service_name=db_service_name,
|
|
370
|
-
|
|
371
|
-
|
|
414
|
+
fqn_search_string = build_es_fqn_search_string(
|
|
415
|
+
database_name=database_name,
|
|
416
|
+
schema_name=table.table_schema,
|
|
417
|
+
service_name=db_service_name or "*",
|
|
418
|
+
table_name=table_name,
|
|
419
|
+
)
|
|
420
|
+
from_entities = self.metadata.search_in_any_service(
|
|
421
|
+
entity_type=Table,
|
|
422
|
+
fqn_search_string=fqn_search_string,
|
|
423
|
+
fetch_multiple_entities=True,
|
|
372
424
|
)
|
|
373
|
-
|
|
374
425
|
to_fqn = fqn.build(
|
|
375
426
|
self.metadata,
|
|
376
427
|
entity_type=LineageDashboard,
|
|
@@ -383,7 +434,7 @@ class MetabaseSource(DashboardServiceSource):
|
|
|
383
434
|
fqn=to_fqn,
|
|
384
435
|
)
|
|
385
436
|
|
|
386
|
-
for from_entity in from_entities:
|
|
437
|
+
for from_entity in from_entities or []:
|
|
387
438
|
yield self._get_add_lineage_request(
|
|
388
439
|
to_entity=to_entity, from_entity=from_entity
|
|
389
440
|
)
|
|
@@ -82,6 +82,7 @@ class MetabaseChart(BaseModel):
|
|
|
82
82
|
dataset_query: Optional[DatasetQuery] = None
|
|
83
83
|
id: Optional[MetabaseStrId] = None
|
|
84
84
|
display: Optional[str] = None
|
|
85
|
+
dashboard_ids: List[str] = []
|
|
85
86
|
|
|
86
87
|
|
|
87
88
|
class DashCard(BaseModel):
|
|
@@ -94,7 +95,7 @@ class MetabaseDashboardDetails(BaseModel):
|
|
|
94
95
|
"""
|
|
95
96
|
|
|
96
97
|
description: Optional[str] = None
|
|
97
|
-
|
|
98
|
+
card_ids: List[str] = []
|
|
98
99
|
name: Optional[str] = None
|
|
99
100
|
id: MetabaseStrId
|
|
100
101
|
creator_id: Optional[MetabaseStrId] = None
|
|
@@ -159,7 +159,9 @@ class MicrostrategySource(DashboardServiceSource):
|
|
|
159
159
|
)
|
|
160
160
|
|
|
161
161
|
def yield_dashboard_lineage_details(
|
|
162
|
-
self,
|
|
162
|
+
self,
|
|
163
|
+
dashboard_details: MstrDashboardDetails,
|
|
164
|
+
db_service_name: Optional[str] = None,
|
|
163
165
|
) -> Optional[Iterable[AddLineageRequest]]:
|
|
164
166
|
"""Not Implemented"""
|
|
165
167
|
|
|
@@ -20,6 +20,7 @@ from metadata.generated.schema.entity.data.chart import Chart, ChartType
|
|
|
20
20
|
from metadata.generated.schema.entity.data.dashboard import (
|
|
21
21
|
Dashboard as Lineage_Dashboard,
|
|
22
22
|
)
|
|
23
|
+
from metadata.generated.schema.entity.data.table import Table
|
|
23
24
|
from metadata.generated.schema.entity.services.connections.dashboard.modeConnection import (
|
|
24
25
|
ModeConnection,
|
|
25
26
|
)
|
|
@@ -38,12 +39,12 @@ from metadata.generated.schema.type.basic import (
|
|
|
38
39
|
from metadata.ingestion.api.models import Either
|
|
39
40
|
from metadata.ingestion.api.steps import InvalidSourceException
|
|
40
41
|
from metadata.ingestion.lineage.parser import LineageParser
|
|
41
|
-
from metadata.ingestion.lineage.sql_lineage import search_table_entities
|
|
42
42
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
43
43
|
from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
|
|
44
44
|
from metadata.ingestion.source.dashboard.mode import client
|
|
45
45
|
from metadata.utils import fqn
|
|
46
46
|
from metadata.utils.filters import filter_by_chart
|
|
47
|
+
from metadata.utils.fqn import build_es_fqn_search_string
|
|
47
48
|
from metadata.utils.helpers import clean_uri
|
|
48
49
|
from metadata.utils.logger import ingestion_logger
|
|
49
50
|
|
|
@@ -130,7 +131,9 @@ class ModeSource(DashboardServiceSource):
|
|
|
130
131
|
self.register_record(dashboard_request=dashboard_request)
|
|
131
132
|
|
|
132
133
|
def yield_dashboard_lineage_details(
|
|
133
|
-
self,
|
|
134
|
+
self,
|
|
135
|
+
dashboard_details: dict,
|
|
136
|
+
db_service_name: Optional[str] = None,
|
|
134
137
|
) -> Iterable[Either[AddLineageRequest]]:
|
|
135
138
|
"""Get lineage method"""
|
|
136
139
|
try:
|
|
@@ -151,14 +154,18 @@ class ModeSource(DashboardServiceSource):
|
|
|
151
154
|
database_schema_name = self.check_database_schema_name(
|
|
152
155
|
database_schema_name
|
|
153
156
|
)
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
service_name=db_service_name,
|
|
158
|
-
|
|
159
|
-
|
|
157
|
+
fqn_search_string = build_es_fqn_search_string(
|
|
158
|
+
database_name=data_source.get(client.DATABASE),
|
|
159
|
+
schema_name=database_schema_name,
|
|
160
|
+
service_name=db_service_name or "*",
|
|
161
|
+
table_name=table,
|
|
162
|
+
)
|
|
163
|
+
from_entities = self.metadata.search_in_any_service(
|
|
164
|
+
entity_type=Table,
|
|
165
|
+
fqn_search_string=fqn_search_string,
|
|
166
|
+
fetch_multiple_entities=True,
|
|
160
167
|
)
|
|
161
|
-
for from_entity in from_entities:
|
|
168
|
+
for from_entity in from_entities or []:
|
|
162
169
|
to_entity = self.metadata.get_by_name(
|
|
163
170
|
entity=Lineage_Dashboard,
|
|
164
171
|
fqn=fqn.build(
|
|
@@ -14,7 +14,6 @@ REST Auth & Client for PowerBi
|
|
|
14
14
|
import json
|
|
15
15
|
import math
|
|
16
16
|
import traceback
|
|
17
|
-
from copy import deepcopy
|
|
18
17
|
from time import sleep
|
|
19
18
|
from typing import List, Optional, Tuple
|
|
20
19
|
|
|
@@ -350,13 +349,7 @@ class PowerBiApiClient:
|
|
|
350
349
|
api_url = f"/myorg/{admin}groups"
|
|
351
350
|
entities_per_page = self.pagination_entity_per_page
|
|
352
351
|
failed_indexes = []
|
|
353
|
-
|
|
354
|
-
if filter_pattern:
|
|
355
|
-
parsed_filter_query = self.create_filter_query(filter_pattern)
|
|
356
|
-
logger.debug(f"Filter query applied = {parsed_filter_query}")
|
|
357
|
-
params_data = deepcopy(GETGROUPS_DEFAULT_PARAMS)
|
|
358
|
-
if parsed_filter_query:
|
|
359
|
-
params_data["$filter"] = parsed_filter_query
|
|
352
|
+
params_data = GETGROUPS_DEFAULT_PARAMS
|
|
360
353
|
response = self.client.get(api_url, data=params_data)
|
|
361
354
|
if (
|
|
362
355
|
not response
|
|
@@ -379,18 +372,14 @@ class PowerBiApiClient:
|
|
|
379
372
|
logger.warning(f"Error processing GetGroups response: {exc}")
|
|
380
373
|
count = 0
|
|
381
374
|
indexes = math.ceil(count / entities_per_page)
|
|
382
|
-
logger.debug(
|
|
383
|
-
f"Total {count} workspaces found, Will run {indexes} iterations fetching"
|
|
384
|
-
f" maximum {entities_per_page} workspaces in a single iteration"
|
|
385
|
-
)
|
|
386
375
|
workspaces = []
|
|
387
376
|
for index in range(indexes):
|
|
388
377
|
params_data = {
|
|
389
378
|
"$top": str(entities_per_page),
|
|
390
379
|
"$skip": str(index * entities_per_page),
|
|
391
380
|
}
|
|
392
|
-
if
|
|
393
|
-
params_data["$filter"] =
|
|
381
|
+
if filter_pattern:
|
|
382
|
+
params_data["$filter"] = self.create_filter_query(filter_pattern)
|
|
394
383
|
|
|
395
384
|
response = self.client.get(api_url, data=params_data)
|
|
396
385
|
if (
|
|
@@ -14,9 +14,6 @@ import re
|
|
|
14
14
|
import traceback
|
|
15
15
|
from typing import Any, Iterable, List, Optional, Union
|
|
16
16
|
|
|
17
|
-
from pydantic import EmailStr
|
|
18
|
-
from pydantic_core import PydanticCustomError
|
|
19
|
-
|
|
20
17
|
from metadata.generated.schema.api.data.createChart import CreateChartRequest
|
|
21
18
|
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
|
|
22
19
|
from metadata.generated.schema.api.data.createDashboardDataModel import (
|
|
@@ -51,7 +48,6 @@ from metadata.generated.schema.type.basic import (
|
|
|
51
48
|
Markdown,
|
|
52
49
|
SourceUrl,
|
|
53
50
|
)
|
|
54
|
-
from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
|
|
55
51
|
from metadata.ingestion.api.models import Either
|
|
56
52
|
from metadata.ingestion.api.steps import InvalidSourceException
|
|
57
53
|
from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
|
|
@@ -73,6 +69,7 @@ from metadata.utils.filters import (
|
|
|
73
69
|
filter_by_dashboard,
|
|
74
70
|
filter_by_datamodel,
|
|
75
71
|
)
|
|
72
|
+
from metadata.utils.fqn import build_es_fqn_search_string
|
|
76
73
|
from metadata.utils.helpers import clean_uri
|
|
77
74
|
from metadata.utils.logger import ingestion_logger
|
|
78
75
|
|
|
@@ -491,7 +488,6 @@ class PowerbiSource(DashboardServiceSource):
|
|
|
491
488
|
"name": table.name,
|
|
492
489
|
"displayName": table.name,
|
|
493
490
|
"description": table.description,
|
|
494
|
-
"children": [],
|
|
495
491
|
}
|
|
496
492
|
child_columns = self._get_child_columns(table=table)
|
|
497
493
|
child_measures = self._get_child_measures(table=table)
|
|
@@ -538,7 +534,6 @@ class PowerbiSource(DashboardServiceSource):
|
|
|
538
534
|
serviceType=DashboardServiceType.PowerBI.value,
|
|
539
535
|
columns=self._get_column_info(dataset),
|
|
540
536
|
project=self.get_project_name(dashboard_details=dataset),
|
|
541
|
-
owners=self.get_owner_ref(dashboard_details=dataset),
|
|
542
537
|
)
|
|
543
538
|
yield Either(right=data_model_request)
|
|
544
539
|
self.register_record_datamodel(datamodel_request=data_model_request)
|
|
@@ -768,17 +763,15 @@ class PowerbiSource(DashboardServiceSource):
|
|
|
768
763
|
"""
|
|
769
764
|
try:
|
|
770
765
|
table_info = self._parse_table_info_from_source_exp(table)
|
|
771
|
-
|
|
772
|
-
self.metadata,
|
|
773
|
-
entity_type=Table,
|
|
774
|
-
service_name=db_service_name or "*",
|
|
766
|
+
fqn_search_string = build_es_fqn_search_string(
|
|
775
767
|
database_name=table_info.get("database"),
|
|
776
768
|
schema_name=table_info.get("schema"),
|
|
769
|
+
service_name=db_service_name or "*",
|
|
777
770
|
table_name=table_info.get("table") or table.name,
|
|
778
771
|
)
|
|
779
|
-
table_entity = self.metadata.
|
|
780
|
-
|
|
781
|
-
|
|
772
|
+
table_entity = self.metadata.search_in_any_service(
|
|
773
|
+
entity_type=Table,
|
|
774
|
+
fqn_search_string=fqn_search_string,
|
|
782
775
|
)
|
|
783
776
|
if table_entity and datamodel_entity:
|
|
784
777
|
columns_list = [column.name for column in table.columns]
|
|
@@ -946,44 +939,3 @@ class PowerbiSource(DashboardServiceSource):
|
|
|
946
939
|
f"Error fetching project name for {dashboard_details.id}: {exc}"
|
|
947
940
|
)
|
|
948
941
|
return None
|
|
949
|
-
|
|
950
|
-
def get_owner_ref( # pylint: disable=unused-argument, useless-return
|
|
951
|
-
self, dashboard_details: Any
|
|
952
|
-
) -> Optional[EntityReferenceList]:
|
|
953
|
-
"""
|
|
954
|
-
Method to process the dashboard owners
|
|
955
|
-
"""
|
|
956
|
-
try:
|
|
957
|
-
owner_ref_list = [] # to assign multiple owners to entity if they exist
|
|
958
|
-
for owner in dashboard_details.users or []:
|
|
959
|
-
if owner.email:
|
|
960
|
-
owner_ref = None
|
|
961
|
-
try:
|
|
962
|
-
owner_email = EmailStr._validate(owner.email)
|
|
963
|
-
owner_ref = self.metadata.get_reference_by_email(
|
|
964
|
-
owner_email.lower()
|
|
965
|
-
)
|
|
966
|
-
except PydanticCustomError:
|
|
967
|
-
logger.warning(
|
|
968
|
-
f"Could not fetch owner data for email: {owner.email}"
|
|
969
|
-
)
|
|
970
|
-
if owner.displayName:
|
|
971
|
-
owner_ref = self.metadata.get_reference_by_name(
|
|
972
|
-
name=owner.displayName
|
|
973
|
-
)
|
|
974
|
-
except Exception as err:
|
|
975
|
-
logger.warning(
|
|
976
|
-
f"Error processing current owner data in {dashboard_details.id}: {err}"
|
|
977
|
-
)
|
|
978
|
-
if owner_ref:
|
|
979
|
-
owner_ref_list.append(owner_ref.root[0])
|
|
980
|
-
if len(owner_ref_list) > 0:
|
|
981
|
-
logger.debug(
|
|
982
|
-
f"Successfully fetched owners data for {dashboard_details.id}"
|
|
983
|
-
)
|
|
984
|
-
return EntityReferenceList(root=owner_ref_list)
|
|
985
|
-
return None
|
|
986
|
-
except Exception as err:
|
|
987
|
-
logger.debug(traceback.format_exc())
|
|
988
|
-
logger.warning(f"Could not fetch owner data due to {err}")
|
|
989
|
-
return None
|
|
@@ -32,15 +32,6 @@ class Tile(BaseModel):
|
|
|
32
32
|
reportId: Optional[str] = None
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
class PowerBIUser(BaseModel):
|
|
36
|
-
"""
|
|
37
|
-
PowerBI User Model
|
|
38
|
-
"""
|
|
39
|
-
|
|
40
|
-
displayName: Optional[str] = None
|
|
41
|
-
email: Optional[str] = Field(alias="emailAddress", default=None)
|
|
42
|
-
|
|
43
|
-
|
|
44
35
|
class PowerBIDashboard(BaseModel):
|
|
45
36
|
"""
|
|
46
37
|
PowerBI PowerBIDashboard Model
|
|
@@ -52,7 +43,6 @@ class PowerBIDashboard(BaseModel):
|
|
|
52
43
|
webUrl: Optional[str] = None
|
|
53
44
|
embedUrl: Optional[str] = None
|
|
54
45
|
tiles: Optional[List[Tile]] = []
|
|
55
|
-
users: Optional[List[PowerBIUser]] = []
|
|
56
46
|
|
|
57
47
|
|
|
58
48
|
class PowerBIReport(BaseModel):
|
|
@@ -64,7 +54,6 @@ class PowerBIReport(BaseModel):
|
|
|
64
54
|
id: str
|
|
65
55
|
name: str
|
|
66
56
|
datasetId: Optional[str] = None
|
|
67
|
-
users: Optional[List[PowerBIUser]] = []
|
|
68
57
|
|
|
69
58
|
|
|
70
59
|
class DashboardsResponse(BaseModel):
|
|
@@ -173,7 +162,6 @@ class Dataset(BaseModel):
|
|
|
173
162
|
name: str
|
|
174
163
|
tables: Optional[List[PowerBiTable]] = []
|
|
175
164
|
description: Optional[str] = None
|
|
176
|
-
users: Optional[List[PowerBIUser]] = []
|
|
177
165
|
|
|
178
166
|
|
|
179
167
|
class DatasetResponse(BaseModel):
|