openmetadata-ingestion 1.2.5.1__py3-none-any.whl → 1.3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openmetadata-ingestion might be problematic. Click here for more details.
- airflow_provider_openmetadata/lineage/runner.py +44 -5
- airflow_provider_openmetadata/lineage/status.py +1 -1
- metadata/antlr/split_listener.py +1 -4
- metadata/applications/auto_tagger.py +212 -0
- metadata/cli/app.py +47 -0
- metadata/cli/db_dump.py +6 -3
- metadata/clients/aws_client.py +4 -0
- metadata/clients/domo_client.py +0 -2
- metadata/cmd.py +24 -13
- metadata/data_insight/processor/reports/entity_report_data_processor.py +8 -2
- metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +4 -0
- metadata/data_insight/source/metadata.py +11 -4
- metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +29 -0
- metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +29 -0
- metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +6 -6
- metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +35 -0
- metadata/data_quality/interface/test_suite_interface_factory.py +40 -0
- metadata/data_quality/processor/test_case_runner.py +17 -8
- metadata/data_quality/source/test_suite.py +8 -2
- metadata/data_quality/validations/base_test_handler.py +21 -17
- metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +38 -1
- metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +36 -1
- metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +15 -0
- metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +27 -0
- metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +23 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +23 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +27 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +23 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +11 -0
- metadata/data_quality/validations/mixins/pandas_validator_mixin.py +13 -2
- metadata/data_quality/validations/mixins/sqa_validator_mixin.py +40 -0
- metadata/examples/workflows/databricks.yaml +0 -1
- metadata/examples/workflows/mongodb.yaml +4 -2
- metadata/examples/workflows/mstr.yaml +24 -0
- metadata/examples/workflows/sas.yaml +28 -0
- metadata/examples/workflows/unity_catalog.yaml +27 -0
- metadata/examples/workflows/unity_catalog_lineage.yaml +18 -0
- metadata/examples/workflows/unity_catalog_usage.yaml +35 -0
- metadata/generated/antlr/EntityLinkLexer.py +391 -292
- metadata/generated/antlr/EntityLinkListener.py +12 -12
- metadata/generated/antlr/EntityLinkParser.py +228 -103
- metadata/generated/schema/analytics/__init__.py +1 -1
- metadata/generated/schema/analytics/basic.py +1 -1
- metadata/generated/schema/analytics/reportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
- metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
- metadata/generated/schema/api/__init__.py +1 -1
- metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
- metadata/generated/schema/api/analytics/__init__.py +1 -1
- metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
- metadata/generated/schema/api/automations/__init__.py +1 -1
- metadata/generated/schema/api/automations/createWorkflow.py +1 -1
- metadata/generated/schema/api/bulkAssets.py +1 -1
- metadata/generated/schema/api/classification/__init__.py +1 -1
- metadata/generated/schema/api/classification/createClassification.py +1 -1
- metadata/generated/schema/api/classification/createTag.py +1 -1
- metadata/generated/schema/api/classification/loadTags.py +1 -1
- metadata/generated/schema/api/createBot.py +2 -3
- metadata/generated/schema/api/createEventPublisherJob.py +1 -1
- metadata/generated/schema/api/createType.py +1 -1
- metadata/generated/schema/api/data/__init__.py +1 -1
- metadata/generated/schema/api/data/createChart.py +1 -1
- metadata/generated/schema/api/data/createContainer.py +2 -2
- metadata/generated/schema/api/data/createCustomProperty.py +2 -3
- metadata/generated/schema/api/data/createDashboard.py +1 -1
- metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
- metadata/generated/schema/api/data/createDatabase.py +2 -3
- metadata/generated/schema/api/data/createDatabaseSchema.py +2 -3
- metadata/generated/schema/api/data/createGlossary.py +2 -3
- metadata/generated/schema/api/data/createGlossaryTerm.py +2 -3
- metadata/generated/schema/api/data/createMlModel.py +1 -1
- metadata/generated/schema/api/data/createPipeline.py +2 -2
- metadata/generated/schema/api/data/createQuery.py +2 -2
- metadata/generated/schema/api/data/createSearchIndex.py +1 -1
- metadata/generated/schema/api/data/createStoredProcedure.py +2 -4
- metadata/generated/schema/api/data/createTable.py +2 -2
- metadata/generated/schema/api/data/createTableProfile.py +1 -1
- metadata/generated/schema/api/data/createTopic.py +1 -1
- metadata/generated/schema/api/data/loadGlossary.py +1 -1
- metadata/generated/schema/api/data/restoreEntity.py +1 -1
- metadata/generated/schema/api/dataInsight/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
- metadata/generated/schema/api/docStore/__init__.py +1 -1
- metadata/generated/schema/api/docStore/createDocument.py +1 -1
- metadata/generated/schema/api/domains/__init__.py +1 -1
- metadata/generated/schema/api/domains/createDataProduct.py +1 -1
- metadata/generated/schema/api/domains/createDomain.py +1 -1
- metadata/generated/schema/api/feed/__init__.py +1 -1
- metadata/generated/schema/api/feed/closeTask.py +8 -1
- metadata/generated/schema/api/feed/createPost.py +1 -1
- metadata/generated/schema/api/feed/createSuggestion.py +30 -0
- metadata/generated/schema/api/feed/createThread.py +1 -1
- metadata/generated/schema/api/feed/resolveTask.py +12 -1
- metadata/generated/schema/api/feed/threadCount.py +18 -14
- metadata/generated/schema/api/lineage/__init__.py +1 -1
- metadata/generated/schema/api/lineage/addLineage.py +1 -1
- metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
- metadata/generated/schema/api/policies/__init__.py +1 -1
- metadata/generated/schema/api/policies/createPolicy.py +1 -1
- metadata/generated/schema/api/services/__init__.py +1 -1
- metadata/generated/schema/api/services/createDashboardService.py +1 -1
- metadata/generated/schema/api/services/createDatabaseService.py +1 -1
- metadata/generated/schema/api/services/createMessagingService.py +1 -1
- metadata/generated/schema/api/services/createMetadataService.py +1 -1
- metadata/generated/schema/api/services/createMlModelService.py +1 -1
- metadata/generated/schema/api/services/createPipelineService.py +1 -1
- metadata/generated/schema/api/services/createSearchService.py +1 -1
- metadata/generated/schema/api/services/createStorageService.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
- metadata/generated/schema/api/setOwner.py +1 -1
- metadata/generated/schema/api/teams/__init__.py +1 -1
- metadata/generated/schema/api/teams/createPersona.py +1 -1
- metadata/generated/schema/api/teams/createRole.py +2 -3
- metadata/generated/schema/api/teams/createTeam.py +1 -1
- metadata/generated/schema/api/teams/createUser.py +2 -2
- metadata/generated/schema/api/tests/__init__.py +1 -1
- metadata/generated/schema/api/tests/createCustomMetric.py +4 -2
- metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
- metadata/generated/schema/api/tests/createTestCase.py +4 -1
- metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +31 -0
- metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
- metadata/generated/schema/api/tests/createTestSuite.py +1 -1
- metadata/generated/schema/api/voteRequest.py +1 -1
- metadata/generated/schema/auth/__init__.py +1 -1
- metadata/generated/schema/auth/basicAuth.py +1 -1
- metadata/generated/schema/auth/basicLoginRequest.py +1 -1
- metadata/generated/schema/auth/changePasswordRequest.py +1 -1
- metadata/generated/schema/auth/createPersonalToken.py +1 -1
- metadata/generated/schema/auth/emailRequest.py +1 -1
- metadata/generated/schema/auth/emailVerificationToken.py +1 -1
- metadata/generated/schema/auth/generateToken.py +1 -1
- metadata/generated/schema/auth/jwtAuth.py +1 -1
- metadata/generated/schema/auth/loginRequest.py +1 -1
- metadata/generated/schema/auth/logoutRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetRequest.py +3 -3
- metadata/generated/schema/auth/passwordResetToken.py +1 -1
- metadata/generated/schema/auth/personalAccessToken.py +1 -1
- metadata/generated/schema/auth/refreshToken.py +1 -1
- metadata/generated/schema/auth/registrationRequest.py +2 -2
- metadata/generated/schema/auth/revokePersonalToken.py +1 -1
- metadata/generated/schema/auth/revokeToken.py +1 -1
- metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
- metadata/generated/schema/auth/ssoAuth.py +1 -1
- metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
- metadata/generated/schema/configuration/__init__.py +1 -1
- metadata/generated/schema/configuration/appsPrivateConfiguration.py +28 -0
- metadata/generated/schema/configuration/authConfig.py +1 -1
- metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
- metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
- metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/elasticSearchConfiguration.py +7 -1
- metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
- metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
- metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
- metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapConfiguration.py +28 -2
- metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
- metadata/generated/schema/configuration/loginConfiguration.py +1 -1
- metadata/generated/schema/configuration/logoConfiguration.py +1 -1
- metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
- metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
- metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
- metadata/generated/schema/dataInsight/__init__.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
- metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
- metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
- metadata/generated/schema/dataInsight/type/__init__.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +1 -1
- metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +1 -1
- metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +1 -1
- metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
- metadata/generated/schema/email/__init__.py +1 -1
- metadata/generated/schema/email/emailRequest.py +1 -1
- metadata/generated/schema/email/smtpSettings.py +1 -1
- metadata/generated/schema/entity/__init__.py +1 -1
- metadata/generated/schema/entity/applications/__init__.py +1 -1
- metadata/generated/schema/entity/applications/app.py +18 -4
- metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
- metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/applicationConfig.py +30 -0
- metadata/generated/schema/entity/{services/connections/database/mongoDB → applications/configuration/external}/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/autoTaggerAppConfig.py +28 -0
- metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +46 -0
- metadata/generated/schema/entity/applications/configuration/internal/__init__.py +3 -0
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +14 -0
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +27 -0
- metadata/generated/schema/entity/applications/configuration/{searchIndexingApp.py → internal/searchIndexingAppConfig.py} +13 -3
- metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +3 -0
- metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +26 -0
- metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
- metadata/generated/schema/entity/applications/jobStatus.py +3 -3
- metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +12 -3
- metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +12 -3
- metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
- metadata/generated/schema/entity/automations/__init__.py +1 -1
- metadata/generated/schema/entity/automations/testServiceConnection.py +2 -2
- metadata/generated/schema/entity/automations/workflow.py +1 -1
- metadata/generated/schema/entity/bot.py +2 -3
- metadata/generated/schema/entity/classification/__init__.py +1 -1
- metadata/generated/schema/entity/classification/classification.py +1 -1
- metadata/generated/schema/entity/classification/tag.py +2 -2
- metadata/generated/schema/entity/data/__init__.py +1 -1
- metadata/generated/schema/entity/data/chart.py +1 -1
- metadata/generated/schema/entity/data/container.py +4 -9
- metadata/generated/schema/entity/data/dashboard.py +1 -1
- metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
- metadata/generated/schema/entity/data/database.py +4 -8
- metadata/generated/schema/entity/data/databaseSchema.py +2 -8
- metadata/generated/schema/entity/data/glossary.py +1 -1
- metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
- metadata/generated/schema/entity/data/metrics.py +1 -1
- metadata/generated/schema/entity/data/mlmodel.py +1 -1
- metadata/generated/schema/entity/data/pipeline.py +6 -9
- metadata/generated/schema/entity/data/query.py +2 -2
- metadata/generated/schema/entity/data/report.py +1 -1
- metadata/generated/schema/entity/data/searchIndex.py +1 -1
- metadata/generated/schema/entity/data/storedProcedure.py +3 -9
- metadata/generated/schema/entity/data/table.py +18 -12
- metadata/generated/schema/entity/data/topic.py +1 -1
- metadata/generated/schema/entity/docStore/__init__.py +1 -1
- metadata/generated/schema/entity/docStore/document.py +1 -1
- metadata/generated/schema/entity/domains/__init__.py +1 -1
- metadata/generated/schema/entity/domains/dataProduct.py +1 -1
- metadata/generated/schema/entity/domains/domain.py +1 -1
- metadata/generated/schema/entity/events/__init__.py +1 -1
- metadata/generated/schema/entity/events/webhook.py +6 -3
- metadata/generated/schema/entity/feed/__init__.py +1 -1
- metadata/generated/schema/entity/feed/suggestion.py +61 -0
- metadata/generated/schema/entity/feed/thread.py +6 -1
- metadata/generated/schema/entity/policies/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
- metadata/generated/schema/entity/policies/filters.py +2 -5
- metadata/generated/schema/entity/policies/policy.py +1 -1
- metadata/generated/schema/entity/services/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +46 -0
- metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/athenaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +6 -1
- metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricksConnection.py +2 -7
- metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/db2Connection.py +3 -2
- metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dorisConnection.py +86 -0
- metadata/generated/schema/entity/services/connections/database/druidConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/hiveConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +3 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +17 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +16 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +23 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +43 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +20 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +69 -0
- metadata/generated/schema/entity/services/connections/database/icebergConnection.py +35 -0
- metadata/generated/schema/entity/services/connections/database/impalaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +24 -14
- metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -2
- metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/oracleConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/postgresConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/prestoConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/sasConnection.py +48 -0
- metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/trinoConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +90 -0
- metadata/generated/schema/entity/services/connections/database/verticaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +5 -20
- metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +2 -8
- metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +23 -0
- metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/{adlsConection.py → adlsConnection.py} +2 -2
- metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
- metadata/generated/schema/entity/services/dashboardService.py +4 -1
- metadata/generated/schema/entity/services/databaseService.py +13 -1
- metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +27 -22
- metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/status.py +56 -0
- metadata/generated/schema/entity/services/messagingService.py +1 -1
- metadata/generated/schema/entity/services/metadataService.py +1 -1
- metadata/generated/schema/entity/services/mlmodelService.py +1 -1
- metadata/generated/schema/entity/services/pipelineService.py +4 -1
- metadata/generated/schema/entity/services/searchService.py +1 -1
- metadata/generated/schema/entity/services/serviceType.py +1 -1
- metadata/generated/schema/entity/services/storageService.py +10 -3
- metadata/generated/schema/entity/teams/__init__.py +1 -1
- metadata/generated/schema/entity/teams/persona.py +1 -1
- metadata/generated/schema/entity/teams/role.py +2 -6
- metadata/generated/schema/entity/teams/team.py +1 -1
- metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
- metadata/generated/schema/entity/teams/user.py +3 -10
- metadata/generated/schema/entity/type.py +2 -2
- metadata/generated/schema/entity/utils/__init__.py +1 -1
- metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
- metadata/generated/schema/entity/utils/servicesCount.py +1 -1
- metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
- metadata/generated/schema/events/__init__.py +1 -1
- metadata/generated/schema/events/alertMetrics.py +27 -0
- metadata/generated/schema/events/api/__init__.py +1 -1
- metadata/generated/schema/events/api/createEventSubscription.py +14 -12
- metadata/generated/schema/events/emailAlertConfig.py +1 -1
- metadata/generated/schema/events/eventFilterRule.py +13 -2
- metadata/generated/schema/events/eventSubscription.py +89 -49
- metadata/generated/schema/events/eventSubscriptionOffset.py +21 -0
- metadata/generated/schema/events/failedEvent.py +26 -0
- metadata/generated/schema/events/filterResourceDescriptor.py +27 -0
- metadata/generated/schema/events/subscriptionResourceDescriptor.py +11 -10
- metadata/generated/schema/metadataIngestion/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/application.py +40 -0
- metadata/generated/schema/metadataIngestion/applicationPipeline.py +15 -6
- metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +6 -1
- metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +6 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +6 -3
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +6 -3
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +6 -3
- metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +2 -2
- metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +24 -0
- metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +8 -1
- metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/workflow.py +1 -1
- metadata/generated/schema/monitoring/__init__.py +1 -1
- metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
- metadata/generated/schema/security/__init__.py +1 -1
- metadata/generated/schema/security/client/__init__.py +1 -1
- metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
- metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
- metadata/generated/schema/security/credentials/__init__.py +1 -1
- metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
- metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
- metadata/generated/schema/security/credentials/basicAuth.py +1 -1
- metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpValues.py +1 -1
- metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
- metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
- metadata/generated/schema/security/secrets/__init__.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +14 -4
- metadata/generated/schema/security/secrets/secretsManagerProvider.py +2 -2
- metadata/generated/schema/security/securityConfiguration.py +1 -1
- metadata/generated/schema/security/ssl/__init__.py +1 -1
- metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
- metadata/generated/schema/security/ssl/verifySSLConfig.py +4 -1
- metadata/generated/schema/settings/__init__.py +1 -1
- metadata/generated/schema/settings/settings.py +1 -1
- metadata/generated/schema/system/__init__.py +1 -1
- metadata/generated/schema/system/entityError.py +17 -0
- metadata/generated/schema/system/eventPublisherJob.py +5 -19
- metadata/generated/schema/system/indexingError.py +34 -0
- metadata/generated/schema/system/ui/__init__.py +1 -1
- metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
- metadata/generated/schema/system/ui/page.py +1 -1
- metadata/generated/schema/tests/__init__.py +1 -1
- metadata/generated/schema/tests/assigned.py +18 -0
- metadata/generated/schema/tests/basic.py +17 -39
- metadata/generated/schema/tests/customMetric.py +4 -2
- metadata/generated/schema/tests/resolved.py +35 -0
- metadata/generated/schema/tests/testCase.py +11 -2
- metadata/generated/schema/tests/testCaseResolutionStatus.py +63 -0
- metadata/generated/schema/tests/testDefinition.py +5 -1
- metadata/generated/schema/tests/testSuite.py +1 -1
- metadata/generated/schema/type/__init__.py +1 -1
- metadata/generated/schema/type/auditLog.py +1 -1
- metadata/generated/schema/type/basic.py +5 -5
- metadata/generated/schema/type/bulkOperationResult.py +1 -1
- metadata/generated/schema/type/changeEvent.py +5 -75
- metadata/generated/schema/type/changeEventType.py +29 -0
- metadata/generated/schema/type/collectionDescriptor.py +1 -1
- metadata/generated/schema/type/csvDocumentation.py +1 -1
- metadata/generated/schema/type/csvErrorType.py +1 -1
- metadata/generated/schema/type/csvFile.py +1 -1
- metadata/generated/schema/type/csvImportResult.py +1 -1
- metadata/generated/schema/type/dailyCount.py +1 -1
- metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
- metadata/generated/schema/type/entityHistory.py +1 -1
- metadata/generated/schema/type/entityLineage.py +2 -1
- metadata/generated/schema/type/entityReference.py +5 -1
- metadata/generated/schema/type/entityReferenceList.py +1 -1
- metadata/generated/schema/type/entityRelationship.py +1 -1
- metadata/generated/schema/type/entityUsage.py +1 -1
- metadata/generated/schema/type/filterPattern.py +1 -1
- metadata/generated/schema/type/function.py +7 -6
- metadata/generated/schema/type/include.py +1 -1
- metadata/generated/schema/type/jdbcConnection.py +1 -1
- metadata/generated/schema/type/lifeCycle.py +1 -1
- metadata/generated/schema/type/paging.py +1 -1
- metadata/generated/schema/type/profile.py +1 -1
- metadata/generated/schema/type/queryParserData.py +2 -2
- metadata/generated/schema/type/reaction.py +1 -1
- metadata/generated/schema/type/schedule.py +1 -1
- metadata/generated/schema/type/schema.py +1 -1
- metadata/generated/schema/type/tableQuery.py +2 -2
- metadata/generated/schema/type/tableUsageCount.py +1 -1
- metadata/generated/schema/type/tagLabel.py +1 -1
- metadata/generated/schema/type/usageDetails.py +1 -1
- metadata/generated/schema/type/usageRequest.py +1 -1
- metadata/generated/schema/type/votes.py +1 -1
- metadata/great_expectations/action.py +3 -2
- metadata/ingestion/api/delete.py +5 -2
- metadata/ingestion/api/models.py +4 -10
- metadata/ingestion/api/parser.py +3 -37
- metadata/ingestion/api/status.py +15 -4
- metadata/ingestion/api/step.py +39 -6
- metadata/ingestion/api/steps.py +20 -0
- metadata/ingestion/api/topology_runner.py +114 -168
- metadata/ingestion/bulksink/metadata_usage.py +11 -5
- metadata/ingestion/lineage/models.py +4 -0
- metadata/ingestion/lineage/parser.py +4 -5
- metadata/ingestion/lineage/sql_lineage.py +9 -17
- metadata/ingestion/models/custom_pydantic.py +4 -2
- metadata/ingestion/models/lf_tags_model.py +33 -0
- metadata/ingestion/models/patch_request.py +108 -2
- metadata/ingestion/models/tests_data.py +9 -0
- metadata/ingestion/models/topology.py +177 -60
- metadata/ingestion/ometa/auth_provider.py +0 -349
- metadata/ingestion/ometa/mixins/es_mixin.py +17 -0
- metadata/ingestion/ometa/mixins/patch_mixin.py +17 -68
- metadata/ingestion/ometa/mixins/suggestions_mixin.py +41 -0
- metadata/ingestion/ometa/mixins/table_mixin.py +18 -0
- metadata/ingestion/ometa/mixins/tests_mixin.py +24 -3
- metadata/ingestion/ometa/mixins/user_mixin.py +117 -22
- metadata/ingestion/ometa/ometa_api.py +18 -25
- metadata/ingestion/ometa/routes.py +7 -0
- metadata/ingestion/processor/query_parser.py +9 -2
- metadata/ingestion/sink/metadata_rest.py +25 -6
- metadata/ingestion/source/dashboard/dashboard_service.py +38 -61
- metadata/ingestion/source/dashboard/domodashboard/metadata.py +13 -14
- metadata/ingestion/source/dashboard/lightdash/metadata.py +2 -1
- metadata/ingestion/source/dashboard/looker/metadata.py +38 -25
- metadata/ingestion/source/dashboard/metabase/metadata.py +23 -6
- metadata/ingestion/source/dashboard/mode/client.py +10 -23
- metadata/ingestion/source/dashboard/mode/connection.py +6 -1
- metadata/ingestion/source/dashboard/mode/metadata.py +8 -4
- metadata/ingestion/source/dashboard/mstr/client.py +208 -0
- metadata/ingestion/source/dashboard/mstr/connection.py +53 -0
- metadata/ingestion/source/dashboard/mstr/metadata.py +182 -0
- metadata/ingestion/source/dashboard/mstr/models.py +144 -0
- metadata/ingestion/source/dashboard/powerbi/metadata.py +15 -10
- metadata/ingestion/source/dashboard/qliksense/metadata.py +11 -7
- metadata/ingestion/source/dashboard/quicksight/metadata.py +9 -5
- metadata/ingestion/source/dashboard/redash/metadata.py +23 -14
- metadata/ingestion/source/dashboard/superset/api_source.py +11 -8
- metadata/ingestion/source/dashboard/superset/client.py +16 -9
- metadata/ingestion/source/dashboard/superset/connection.py +3 -3
- metadata/ingestion/source/dashboard/superset/db_source.py +14 -10
- metadata/ingestion/source/dashboard/superset/mixin.py +22 -18
- metadata/ingestion/source/dashboard/superset/queries.py +1 -1
- metadata/ingestion/source/dashboard/tableau/client.py +91 -11
- metadata/ingestion/source/dashboard/tableau/connection.py +10 -1
- metadata/ingestion/source/dashboard/tableau/metadata.py +58 -70
- metadata/ingestion/source/dashboard/tableau/models.py +0 -8
- metadata/ingestion/source/dashboard/tableau/queries.py +5 -5
- metadata/ingestion/source/database/athena/client.py +80 -0
- metadata/ingestion/source/database/athena/connection.py +7 -0
- metadata/ingestion/source/database/athena/metadata.py +161 -19
- metadata/ingestion/source/database/azuresql/metadata.py +0 -1
- metadata/ingestion/source/database/bigquery/connection.py +24 -3
- metadata/ingestion/source/database/bigquery/metadata.py +39 -27
- metadata/ingestion/source/database/bigquery/queries.py +11 -4
- metadata/ingestion/source/database/bigquery/query_parser.py +13 -0
- metadata/ingestion/source/database/bigquery/usage.py +1 -3
- metadata/ingestion/source/database/column_helpers.py +0 -10
- metadata/ingestion/source/database/column_type_parser.py +11 -5
- metadata/ingestion/source/database/common_db_source.py +8 -2
- metadata/ingestion/source/database/common_nosql_source.py +8 -4
- metadata/ingestion/source/database/database_service.py +89 -7
- metadata/ingestion/source/database/databricks/client.py +5 -10
- metadata/ingestion/source/database/databricks/connection.py +16 -55
- metadata/ingestion/source/database/databricks/lineage.py +29 -26
- metadata/ingestion/source/database/databricks/metadata.py +448 -11
- metadata/ingestion/source/database/databricks/queries.py +28 -0
- metadata/ingestion/source/database/databricks/query_parser.py +5 -1
- metadata/ingestion/source/database/databricks/usage.py +2 -2
- metadata/ingestion/source/database/datalake/connection.py +31 -4
- metadata/ingestion/source/database/datalake/metadata.py +113 -27
- metadata/ingestion/source/database/dbt/dbt_config.py +5 -0
- metadata/ingestion/source/database/dbt/dbt_service.py +10 -14
- metadata/ingestion/source/database/dbt/dbt_utils.py +3 -1
- metadata/ingestion/source/database/dbt/metadata.py +11 -25
- metadata/ingestion/source/database/deltalake/metadata.py +6 -3
- metadata/ingestion/source/database/domodatabase/metadata.py +7 -6
- metadata/ingestion/source/database/doris/connection.py +72 -0
- metadata/ingestion/source/database/doris/metadata.py +315 -0
- metadata/ingestion/source/database/doris/queries.py +54 -0
- metadata/ingestion/source/database/doris/utils.py +64 -0
- metadata/ingestion/source/database/extended_sample_data.py +532 -0
- metadata/ingestion/source/database/glue/metadata.py +8 -5
- metadata/ingestion/source/database/hive/connection.py +0 -2
- metadata/ingestion/source/database/hive/utils.py +3 -0
- metadata/ingestion/source/database/iceberg/catalog/__init__.py +65 -0
- metadata/ingestion/source/database/iceberg/catalog/base.py +40 -0
- metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +102 -0
- metadata/ingestion/source/database/iceberg/catalog/glue.py +88 -0
- metadata/ingestion/source/database/iceberg/catalog/hive.py +51 -0
- metadata/ingestion/source/database/iceberg/catalog/rest.py +84 -0
- metadata/ingestion/source/database/iceberg/connection.py +68 -0
- metadata/ingestion/source/database/iceberg/fs/__init__.py +52 -0
- metadata/ingestion/source/database/iceberg/fs/azure.py +44 -0
- metadata/ingestion/source/database/iceberg/fs/base.py +30 -0
- metadata/ingestion/source/database/iceberg/fs/s3.py +77 -0
- metadata/ingestion/source/database/iceberg/helper.py +124 -0
- metadata/ingestion/source/database/iceberg/metadata.py +311 -0
- metadata/ingestion/source/database/iceberg/models.py +66 -0
- metadata/ingestion/source/database/life_cycle_query_mixin.py +72 -3
- metadata/ingestion/source/database/mongodb/connection.py +1 -5
- metadata/ingestion/source/database/mssql/lineage.py +2 -0
- metadata/ingestion/source/database/mssql/metadata.py +108 -4
- metadata/ingestion/source/database/mssql/models.py +30 -0
- metadata/ingestion/source/database/mssql/queries.py +179 -1
- metadata/ingestion/source/database/mssql/utils.py +207 -4
- metadata/ingestion/source/database/mysql/metadata.py +0 -2
- metadata/ingestion/source/database/oracle/metadata.py +108 -2
- metadata/ingestion/source/database/oracle/models.py +30 -0
- metadata/ingestion/source/database/oracle/queries.py +98 -17
- metadata/ingestion/source/database/oracle/utils.py +0 -1
- metadata/ingestion/source/database/postgres/lineage.py +32 -14
- metadata/ingestion/source/database/postgres/metadata.py +15 -7
- metadata/ingestion/source/database/postgres/pgspider/lineage.py +0 -1
- metadata/ingestion/source/database/postgres/queries.py +4 -2
- metadata/ingestion/source/database/postgres/query_parser.py +4 -72
- metadata/ingestion/source/database/postgres/usage.py +41 -0
- metadata/ingestion/source/database/postgres/utils.py +34 -0
- metadata/ingestion/source/database/query_parser_source.py +8 -2
- metadata/ingestion/source/database/redshift/metadata.py +14 -4
- metadata/ingestion/source/database/redshift/queries.py +10 -4
- metadata/ingestion/source/database/redshift/query_parser.py +16 -0
- metadata/ingestion/source/database/redshift/usage.py +0 -2
- metadata/ingestion/source/database/salesforce/metadata.py +32 -3
- metadata/ingestion/source/database/sample_data.py +120 -6
- metadata/ingestion/source/database/sas/client.py +184 -0
- metadata/ingestion/source/database/sas/connection.py +47 -0
- metadata/ingestion/source/database/sas/extension_attr.py +103 -0
- metadata/ingestion/source/database/sas/metadata.py +914 -0
- metadata/ingestion/source/database/snowflake/metadata.py +8 -51
- metadata/ingestion/source/database/snowflake/models.py +6 -1
- metadata/ingestion/source/database/snowflake/queries.py +0 -4
- metadata/ingestion/source/database/snowflake/query_parser.py +5 -20
- metadata/ingestion/source/database/snowflake/utils.py +2 -6
- metadata/ingestion/source/database/stored_procedures_mixin.py +12 -8
- metadata/ingestion/source/database/unitycatalog/__init__.py +0 -0
- metadata/ingestion/source/database/unitycatalog/client.py +87 -0
- metadata/ingestion/source/database/unitycatalog/connection.py +97 -0
- metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/lineage.py +11 -11
- metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/metadata.py +42 -49
- metadata/ingestion/source/database/unitycatalog/query_parser.py +60 -0
- metadata/ingestion/source/database/unitycatalog/usage.py +31 -0
- metadata/ingestion/source/database/usage_source.py +3 -2
- metadata/ingestion/source/messaging/common_broker_source.py +6 -4
- metadata/ingestion/source/messaging/kinesis/metadata.py +6 -3
- metadata/ingestion/source/messaging/messaging_service.py +6 -2
- metadata/ingestion/source/metadata/amundsen/metadata.py +10 -7
- metadata/ingestion/source/metadata/atlas/metadata.py +5 -5
- metadata/ingestion/source/mlmodel/mlflow/metadata.py +5 -2
- metadata/ingestion/source/mlmodel/mlmodel_service.py +6 -2
- metadata/ingestion/source/pipeline/airflow/connection.py +0 -12
- metadata/ingestion/source/pipeline/airflow/lineage_parser.py +12 -6
- metadata/ingestion/source/pipeline/airflow/metadata.py +63 -34
- metadata/ingestion/source/pipeline/airflow/models.py +5 -4
- metadata/ingestion/source/pipeline/dagster/metadata.py +7 -4
- metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +12 -9
- metadata/ingestion/source/pipeline/domopipeline/metadata.py +7 -4
- metadata/ingestion/source/pipeline/gluepipeline/metadata.py +5 -2
- metadata/ingestion/source/pipeline/pipeline_service.py +6 -2
- metadata/ingestion/source/pipeline/spline/metadata.py +0 -1
- metadata/ingestion/source/search/elasticsearch/connection.py +4 -1
- metadata/ingestion/source/search/elasticsearch/metadata.py +1 -2
- metadata/ingestion/source/search/search_service.py +6 -2
- metadata/ingestion/source/storage/s3/metadata.py +22 -17
- metadata/ingestion/source/storage/storage_service.py +53 -11
- metadata/ingestion/stage/table_usage.py +9 -2
- metadata/mixins/sqalchemy/sqa_mixin.py +14 -7
- metadata/parsers/protobuf_parser.py +29 -11
- metadata/pii/processor.py +9 -2
- metadata/profiler/api/models.py +19 -1
- metadata/profiler/interface/pandas/profiler_interface.py +59 -18
- metadata/profiler/interface/profiler_interface.py +13 -2
- metadata/profiler/interface/profiler_interface_factory.py +49 -14
- metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +3 -0
- metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +26 -0
- metadata/profiler/interface/sqlalchemy/db2/__init__.py +0 -0
- metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +38 -0
- metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +85 -0
- metadata/profiler/interface/sqlalchemy/profiler_interface.py +77 -34
- metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
- metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +7 -0
- metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
- metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +33 -0
- metadata/profiler/metrics/composed/null_ratio.py +1 -1
- metadata/profiler/metrics/hybrid/histogram.py +1 -0
- metadata/profiler/metrics/static/max.py +4 -1
- metadata/profiler/metrics/static/min.py +4 -1
- metadata/profiler/metrics/system/queries/snowflake.py +89 -17
- metadata/profiler/metrics/system/system.py +62 -20
- metadata/profiler/orm/functions/length.py +1 -0
- metadata/profiler/orm/functions/median.py +7 -0
- metadata/profiler/orm/functions/table_metric_computer.py +462 -0
- metadata/profiler/orm/registry.py +1 -0
- metadata/profiler/processor/core.py +116 -52
- metadata/profiler/processor/default.py +14 -3
- metadata/profiler/processor/handle_partition.py +2 -2
- metadata/profiler/processor/processor.py +9 -4
- metadata/profiler/processor/sampler/sqlalchemy/bigquery/sampler.py +31 -3
- metadata/profiler/processor/sampler/sqlalchemy/sampler.py +29 -6
- metadata/profiler/processor/sampler/sqlalchemy/trino/sampler.py +10 -4
- metadata/profiler/source/base/profiler_source.py +5 -2
- metadata/profiler/source/bigquery/type_mapper.py +0 -1
- metadata/profiler/source/mariadb/functions/median.py +20 -0
- metadata/profiler/source/mariadb/metrics/window/first_quartile.py +10 -0
- metadata/profiler/source/mariadb/metrics/window/median.py +10 -0
- metadata/profiler/source/mariadb/metrics/window/third_quartile.py +10 -0
- metadata/profiler/source/metadata.py +43 -10
- metadata/profiler/source/metadata_ext.py +16 -50
- metadata/py.typed +0 -0
- metadata/readers/dataframe/json.py +5 -4
- metadata/readers/file/api_reader.py +0 -1
- metadata/utils/constants.py +5 -0
- metadata/utils/datalake/datalake_utils.py +363 -129
- metadata/utils/entity_link.py +26 -6
- metadata/utils/fqn.py +20 -0
- metadata/utils/helpers.py +55 -9
- metadata/utils/importer.py +2 -3
- metadata/utils/life_cycle_utils.py +4 -4
- metadata/utils/logger.py +13 -2
- metadata/utils/partition.py +10 -5
- metadata/utils/secrets/client/loader.py +0 -1
- metadata/utils/secrets/noop_secrets_manager.py +4 -3
- metadata/utils/secrets/secrets_manager_factory.py +3 -4
- metadata/utils/{source_hash_utils.py → source_hash.py} +10 -1
- metadata/utils/sqlalchemy_utils.py +21 -0
- metadata/utils/storage_metadata_config.py +42 -1
- metadata/utils/tag_utils.py +5 -2
- metadata/workflow/application.py +154 -0
- metadata/workflow/application_output_handler.py +34 -0
- metadata/workflow/base.py +84 -153
- metadata/workflow/data_insight.py +8 -7
- metadata/workflow/data_quality.py +3 -2
- metadata/workflow/ingestion.py +203 -0
- metadata/workflow/metadata.py +2 -3
- metadata/workflow/output_handler.py +204 -0
- metadata/workflow/profiler.py +2 -2
- metadata/workflow/usage.py +3 -4
- metadata/workflow/workflow_output_handler.py +15 -255
- metadata/workflow/workflow_status_mixin.py +44 -52
- openmetadata_ingestion-1.3.0.0.dist-info/METADATA +749 -0
- {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/RECORD +812 -725
- {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/WHEEL +1 -1
- metadata/generated/schema/entity/applications/appConfig.py +0 -21
- metadata/generated/schema/entity/applications/configuration/dataInsightsApp.py +0 -17
- metadata/generated/schema/entity/applications/configuration/externalAppIngestionConfig.py +0 -38
- metadata/generated/schema/entity/services/connections/database/mongoDB/mongoDBValues.py +0 -44
- metadata/generated/schema/events/dataInsightAlertConfig.py +0 -17
- metadata/generated/schema/events/entitySpelFilters.py +0 -19
- metadata/ingestion/models/es_documents.py +0 -339
- metadata/ingestion/ometa/mixins/glossary_mixin.py +0 -501
- metadata/ingestion/ometa/provider_registry.py +0 -144
- metadata/ingestion/source/database/databricks/legacy/lineage.py +0 -51
- metadata/ingestion/source/database/databricks/legacy/metadata.py +0 -339
- metadata/ingestion/source/metadata/metadata_elasticsearch/metadata.py +0 -144
- metadata/profiler/orm/functions/table_metric_construct.py +0 -365
- openmetadata_ingestion-1.2.5.1.dist-info/METADATA +0 -426
- /metadata/ingestion/source/{database/databricks/legacy → dashboard/mstr}/__init__.py +0 -0
- /metadata/ingestion/source/database/{databricks/unity_catalog → doris}/__init__.py +0 -0
- /metadata/ingestion/source/{metadata/metadata_elasticsearch → database/sas}/__init__.py +0 -0
- /metadata/ingestion/source/database/{databricks → unitycatalog}/models.py +0 -0
- {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/LICENSE +0 -0
- {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/entry_points.txt +0 -0
- {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/top_level.txt +0 -0
|
@@ -13,12 +13,15 @@ Mixin class containing User specific methods
|
|
|
13
13
|
|
|
14
14
|
To be used by OpenMetadata class
|
|
15
15
|
"""
|
|
16
|
-
import traceback
|
|
17
16
|
from functools import lru_cache
|
|
18
|
-
from typing import Optional
|
|
17
|
+
from typing import Optional, Type
|
|
19
18
|
|
|
19
|
+
from metadata.generated.schema.entity.teams.team import Team
|
|
20
20
|
from metadata.generated.schema.entity.teams.user import User
|
|
21
|
+
from metadata.generated.schema.type.entityReference import EntityReference
|
|
22
|
+
from metadata.ingestion.api.common import T
|
|
21
23
|
from metadata.ingestion.ometa.client import REST
|
|
24
|
+
from metadata.utils.constants import ENTITY_REFERENCE_TYPE_MAP
|
|
22
25
|
from metadata.utils.elasticsearch import ES_INDEX_MAP
|
|
23
26
|
from metadata.utils.logger import ometa_logger
|
|
24
27
|
|
|
@@ -34,42 +37,134 @@ class OMetaUserMixin:
|
|
|
34
37
|
|
|
35
38
|
client: REST
|
|
36
39
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
40
|
+
@staticmethod
|
|
41
|
+
def email_search_query_es(entity: Type[T]) -> str:
|
|
42
|
+
return (
|
|
43
|
+
"/search/query?q=email.keyword:{email}&from={from_}&size={size}&index="
|
|
44
|
+
+ ES_INDEX_MAP[entity.__name__]
|
|
45
|
+
)
|
|
41
46
|
|
|
42
|
-
@
|
|
43
|
-
def
|
|
47
|
+
@staticmethod
|
|
48
|
+
def name_search_query_es(entity: Type[T]) -> str:
|
|
49
|
+
"""
|
|
50
|
+
Allow for more flexible lookup following what the UI is doing when searching users.
|
|
51
|
+
|
|
52
|
+
We don't want to stick to `q=name:{name}` since in case a user is named `random.user`
|
|
53
|
+
but looked as `Random User`, we want to find this match.
|
|
54
|
+
"""
|
|
55
|
+
return (
|
|
56
|
+
"/search/query?q={name}&from={from_}&size={size}&index="
|
|
57
|
+
+ ES_INDEX_MAP[entity.__name__]
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
def _search_by_email(
|
|
44
61
|
self,
|
|
62
|
+
entity: Type[T],
|
|
45
63
|
email: Optional[str],
|
|
46
64
|
from_count: int = 0,
|
|
47
65
|
size: int = 1,
|
|
48
66
|
fields: Optional[list] = None,
|
|
49
|
-
) -> Optional[
|
|
67
|
+
) -> Optional[T]:
|
|
50
68
|
"""
|
|
51
|
-
GET user entity by
|
|
69
|
+
GET user or team entity by mail
|
|
52
70
|
|
|
53
71
|
Args:
|
|
54
72
|
email: user email to search
|
|
55
73
|
from_count: records to expect
|
|
56
74
|
size: number of records
|
|
75
|
+
fields: Optional field list to pass to ES request
|
|
57
76
|
"""
|
|
58
77
|
if email:
|
|
59
|
-
query_string = self.
|
|
78
|
+
query_string = self.email_search_query_es(entity=entity).format(
|
|
60
79
|
email=email, from_=from_count, size=size
|
|
61
80
|
)
|
|
81
|
+
return self._get_entity_from_es(
|
|
82
|
+
entity=entity, query_string=query_string, fields=fields
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
def _search_by_name(
|
|
88
|
+
self,
|
|
89
|
+
entity: Type[T],
|
|
90
|
+
name: Optional[str],
|
|
91
|
+
from_count: int = 0,
|
|
92
|
+
size: int = 1,
|
|
93
|
+
fields: Optional[list] = None,
|
|
94
|
+
) -> Optional[T]:
|
|
95
|
+
"""
|
|
96
|
+
GET entity by name
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
name: user name to search
|
|
100
|
+
from_count: records to expect
|
|
101
|
+
size: number of records
|
|
102
|
+
fields: Optional field list to pass to ES request
|
|
103
|
+
"""
|
|
104
|
+
if name:
|
|
105
|
+
query_string = self.name_search_query_es(entity=entity).format(
|
|
106
|
+
name=name, from_=from_count, size=size
|
|
107
|
+
)
|
|
108
|
+
return self._get_entity_from_es(
|
|
109
|
+
entity=entity, query_string=query_string, fields=fields
|
|
110
|
+
)
|
|
62
111
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
@lru_cache(maxsize=None)
|
|
115
|
+
def get_reference_by_email(
|
|
116
|
+
self,
|
|
117
|
+
email: Optional[str],
|
|
118
|
+
from_count: int = 0,
|
|
119
|
+
size: int = 1,
|
|
120
|
+
fields: Optional[list] = None,
|
|
121
|
+
) -> Optional[EntityReference]:
|
|
122
|
+
"""
|
|
123
|
+
Get a User or Team Entity Reference by searching by its mail
|
|
124
|
+
"""
|
|
125
|
+
maybe_user = self._search_by_email(
|
|
126
|
+
entity=User, email=email, from_count=from_count, size=size, fields=fields
|
|
127
|
+
)
|
|
128
|
+
if maybe_user:
|
|
129
|
+
return EntityReference(
|
|
130
|
+
id=maybe_user.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[User.__name__]
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
maybe_team = self._search_by_email(
|
|
134
|
+
entity=Team, email=email, from_count=from_count, size=size, fields=fields
|
|
135
|
+
)
|
|
136
|
+
if maybe_team:
|
|
137
|
+
return EntityReference(
|
|
138
|
+
id=maybe_team.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[Team.__name__]
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
return None
|
|
142
|
+
|
|
143
|
+
@lru_cache(maxsize=None)
|
|
144
|
+
def get_reference_by_name(
|
|
145
|
+
self,
|
|
146
|
+
name: Optional[str],
|
|
147
|
+
from_count: int = 0,
|
|
148
|
+
size: int = 1,
|
|
149
|
+
fields: Optional[list] = None,
|
|
150
|
+
) -> Optional[EntityReference]:
|
|
151
|
+
"""
|
|
152
|
+
Get a User or Team Entity Reference by searching by its name
|
|
153
|
+
"""
|
|
154
|
+
maybe_user = self._search_by_name(
|
|
155
|
+
entity=User, name=name, from_count=from_count, size=size, fields=fields
|
|
156
|
+
)
|
|
157
|
+
if maybe_user:
|
|
158
|
+
return EntityReference(
|
|
159
|
+
id=maybe_user.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[User.__name__]
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
maybe_team = self._search_by_name(
|
|
163
|
+
entity=Team, name=name, from_count=from_count, size=size, fields=fields
|
|
164
|
+
)
|
|
165
|
+
if maybe_team:
|
|
166
|
+
return EntityReference(
|
|
167
|
+
id=maybe_team.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[Team.__name__]
|
|
168
|
+
)
|
|
74
169
|
|
|
75
170
|
return None
|
|
@@ -31,7 +31,7 @@ from metadata.generated.schema.type.basic import FullyQualifiedEntityName
|
|
|
31
31
|
from metadata.generated.schema.type.entityHistory import EntityVersionHistory
|
|
32
32
|
from metadata.generated.schema.type.entityReference import EntityReference
|
|
33
33
|
from metadata.ingestion.models.encoders import show_secrets_encoder
|
|
34
|
-
from metadata.ingestion.ometa.auth_provider import
|
|
34
|
+
from metadata.ingestion.ometa.auth_provider import OpenMetadataAuthenticationProvider
|
|
35
35
|
from metadata.ingestion.ometa.client import REST, APIError, ClientConfig
|
|
36
36
|
from metadata.ingestion.ometa.mixins.custom_property_mixin import (
|
|
37
37
|
OMetaCustomPropertyMixin,
|
|
@@ -39,7 +39,6 @@ from metadata.ingestion.ometa.mixins.custom_property_mixin import (
|
|
|
39
39
|
from metadata.ingestion.ometa.mixins.dashboard_mixin import OMetaDashboardMixin
|
|
40
40
|
from metadata.ingestion.ometa.mixins.data_insight_mixin import DataInsightMixin
|
|
41
41
|
from metadata.ingestion.ometa.mixins.es_mixin import ESMixin
|
|
42
|
-
from metadata.ingestion.ometa.mixins.glossary_mixin import GlossaryMixin
|
|
43
42
|
from metadata.ingestion.ometa.mixins.ingestion_pipeline_mixin import (
|
|
44
43
|
OMetaIngestionPipelineMixin,
|
|
45
44
|
)
|
|
@@ -51,16 +50,13 @@ from metadata.ingestion.ometa.mixins.role_policy_mixin import OMetaRolePolicyMix
|
|
|
51
50
|
from metadata.ingestion.ometa.mixins.search_index_mixin import OMetaSearchIndexMixin
|
|
52
51
|
from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin
|
|
53
52
|
from metadata.ingestion.ometa.mixins.service_mixin import OMetaServiceMixin
|
|
53
|
+
from metadata.ingestion.ometa.mixins.suggestions_mixin import OMetaSuggestionsMixin
|
|
54
54
|
from metadata.ingestion.ometa.mixins.table_mixin import OMetaTableMixin
|
|
55
55
|
from metadata.ingestion.ometa.mixins.tests_mixin import OMetaTestsMixin
|
|
56
56
|
from metadata.ingestion.ometa.mixins.topic_mixin import OMetaTopicMixin
|
|
57
57
|
from metadata.ingestion.ometa.mixins.user_mixin import OMetaUserMixin
|
|
58
58
|
from metadata.ingestion.ometa.mixins.version_mixin import OMetaVersionMixin
|
|
59
59
|
from metadata.ingestion.ometa.models import EntityList
|
|
60
|
-
from metadata.ingestion.ometa.provider_registry import (
|
|
61
|
-
InvalidAuthProviderException,
|
|
62
|
-
auth_provider_registry,
|
|
63
|
-
)
|
|
64
60
|
from metadata.ingestion.ometa.routes import ROUTES
|
|
65
61
|
from metadata.ingestion.ometa.utils import get_entity_type, model_str
|
|
66
62
|
from metadata.utils.logger import ometa_logger
|
|
@@ -100,7 +96,6 @@ class OpenMetadata(
|
|
|
100
96
|
OMetaTableMixin,
|
|
101
97
|
OMetaTopicMixin,
|
|
102
98
|
OMetaVersionMixin,
|
|
103
|
-
GlossaryMixin,
|
|
104
99
|
OMetaServiceMixin,
|
|
105
100
|
ESMixin,
|
|
106
101
|
OMetaServerMixin,
|
|
@@ -114,6 +109,7 @@ class OpenMetadata(
|
|
|
114
109
|
OMetaRolePolicyMixin,
|
|
115
110
|
OMetaSearchIndexMixin,
|
|
116
111
|
OMetaCustomPropertyMixin,
|
|
112
|
+
OMetaSuggestionsMixin,
|
|
117
113
|
Generic[T, C],
|
|
118
114
|
):
|
|
119
115
|
"""
|
|
@@ -125,7 +121,7 @@ class OpenMetadata(
|
|
|
125
121
|
"""
|
|
126
122
|
|
|
127
123
|
client: REST
|
|
128
|
-
_auth_provider:
|
|
124
|
+
_auth_provider: OpenMetadataAuthenticationProvider
|
|
129
125
|
config: OpenMetadataConnection
|
|
130
126
|
|
|
131
127
|
class_root = ".".join(["metadata", "generated", "schema"])
|
|
@@ -146,16 +142,7 @@ class OpenMetadata(
|
|
|
146
142
|
config.secretsManagerLoader,
|
|
147
143
|
).get_secrets_manager()
|
|
148
144
|
|
|
149
|
-
|
|
150
|
-
auth_provider_fn = auth_provider_registry.registry.get(
|
|
151
|
-
self.config.authProvider.value
|
|
152
|
-
)
|
|
153
|
-
if not auth_provider_fn:
|
|
154
|
-
raise InvalidAuthProviderException(
|
|
155
|
-
f"Cannot find {self.config.authProvider.value} in {auth_provider_registry.registry}"
|
|
156
|
-
)
|
|
157
|
-
|
|
158
|
-
self._auth_provider = auth_provider_fn(self.config)
|
|
145
|
+
self._auth_provider = OpenMetadataAuthenticationProvider.create(self.config)
|
|
159
146
|
|
|
160
147
|
get_verify_ssl = get_verify_ssl_fn(self.config.verifySSL)
|
|
161
148
|
|
|
@@ -261,11 +248,9 @@ class OpenMetadata(
|
|
|
261
248
|
)
|
|
262
249
|
return entity_class
|
|
263
250
|
|
|
264
|
-
def
|
|
251
|
+
def _create(self, data: C, method: str) -> T:
|
|
265
252
|
"""
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
We PUT to the endpoint and return the Entity generated result
|
|
253
|
+
Internal logic to run POST vs. PUT
|
|
269
254
|
"""
|
|
270
255
|
entity = data.__class__
|
|
271
256
|
is_create = "create" in data.__class__.__name__.lower()
|
|
@@ -277,15 +262,23 @@ class OpenMetadata(
|
|
|
277
262
|
raise InvalidEntityException(
|
|
278
263
|
f"PUT operations need a CreateEntity, not {entity}"
|
|
279
264
|
)
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
)
|
|
265
|
+
|
|
266
|
+
fn = getattr(self.client, method)
|
|
267
|
+
resp = fn(self.get_suffix(entity), data=data.json(encoder=show_secrets_encoder))
|
|
283
268
|
if not resp:
|
|
284
269
|
raise EmptyPayloadException(
|
|
285
270
|
f"Got an empty response when trying to PUT to {self.get_suffix(entity)}, {data.json()}"
|
|
286
271
|
)
|
|
287
272
|
return entity_class(**resp)
|
|
288
273
|
|
|
274
|
+
def create_or_update(self, data: C) -> T:
|
|
275
|
+
"""Run a PUT requesting via create request C"""
|
|
276
|
+
return self._create(data=data, method="put")
|
|
277
|
+
|
|
278
|
+
def create(self, data: C) -> T:
|
|
279
|
+
"""Run a POST requesting via create request C"""
|
|
280
|
+
return self._create(data=data, method="post")
|
|
281
|
+
|
|
289
282
|
def get_by_name(
|
|
290
283
|
self,
|
|
291
284
|
entity: Type[T],
|
|
@@ -50,6 +50,7 @@ from metadata.generated.schema.api.domains.createDataProduct import (
|
|
|
50
50
|
CreateDataProductRequest,
|
|
51
51
|
)
|
|
52
52
|
from metadata.generated.schema.api.domains.createDomain import CreateDomainRequest
|
|
53
|
+
from metadata.generated.schema.api.feed.createSuggestion import CreateSuggestionRequest
|
|
53
54
|
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
|
|
54
55
|
from metadata.generated.schema.api.policies.createPolicy import CreatePolicyRequest
|
|
55
56
|
from metadata.generated.schema.api.services.createDashboardService import (
|
|
@@ -90,6 +91,7 @@ from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteR
|
|
|
90
91
|
from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart
|
|
91
92
|
from metadata.generated.schema.dataInsight.kpi.kpi import Kpi
|
|
92
93
|
from metadata.generated.schema.entity.automations.workflow import Workflow
|
|
94
|
+
from metadata.generated.schema.entity.bot import Bot
|
|
93
95
|
from metadata.generated.schema.entity.classification.classification import (
|
|
94
96
|
Classification,
|
|
95
97
|
)
|
|
@@ -113,6 +115,7 @@ from metadata.generated.schema.entity.data.table import Table
|
|
|
113
115
|
from metadata.generated.schema.entity.data.topic import Topic
|
|
114
116
|
from metadata.generated.schema.entity.domains.dataProduct import DataProduct
|
|
115
117
|
from metadata.generated.schema.entity.domains.domain import Domain
|
|
118
|
+
from metadata.generated.schema.entity.feed.suggestion import Suggestion
|
|
116
119
|
from metadata.generated.schema.entity.policies.policy import Policy
|
|
117
120
|
from metadata.generated.schema.entity.services.connections.testConnectionDefinition import (
|
|
118
121
|
TestConnectionDefinition,
|
|
@@ -181,6 +184,7 @@ ROUTES = {
|
|
|
181
184
|
User.__name__: "/users",
|
|
182
185
|
CreateUserRequest.__name__: "/users",
|
|
183
186
|
AuthenticationMechanism.__name__: "/users/auth-mechanism",
|
|
187
|
+
Bot.__name__: "/bots", # We won't allow bot creation from the client
|
|
184
188
|
# Roles
|
|
185
189
|
Role.__name__: "/roles",
|
|
186
190
|
CreateRoleRequest.__name__: "/roles",
|
|
@@ -225,4 +229,7 @@ ROUTES = {
|
|
|
225
229
|
CreateDomainRequest.__name__: "/domains",
|
|
226
230
|
DataProduct.__name__: "/dataProducts",
|
|
227
231
|
CreateDataProductRequest.__name__: "/dataProducts",
|
|
232
|
+
# Suggestions
|
|
233
|
+
Suggestion.__name__: "/suggestions",
|
|
234
|
+
CreateSuggestionRequest.__name__: "/suggestions",
|
|
228
235
|
}
|
|
@@ -17,10 +17,13 @@ import traceback
|
|
|
17
17
|
from typing import Optional
|
|
18
18
|
|
|
19
19
|
from metadata.config.common import ConfigModel
|
|
20
|
+
from metadata.generated.schema.entity.services.ingestionPipelines.status import (
|
|
21
|
+
StackTraceError,
|
|
22
|
+
)
|
|
20
23
|
from metadata.generated.schema.type.basic import DateTime
|
|
21
24
|
from metadata.generated.schema.type.queryParserData import ParsedData, QueryParserData
|
|
22
25
|
from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery
|
|
23
|
-
from metadata.ingestion.api.models import Either
|
|
26
|
+
from metadata.ingestion.api.models import Either
|
|
24
27
|
from metadata.ingestion.api.steps import Processor
|
|
25
28
|
from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
|
|
26
29
|
from metadata.ingestion.lineage.parser import LineageParser
|
|
@@ -84,6 +87,10 @@ class QueryParserProcessor(Processor):
|
|
|
84
87
|
self.metadata = metadata
|
|
85
88
|
self.connection_type = connection_type
|
|
86
89
|
|
|
90
|
+
@property
|
|
91
|
+
def name(self) -> str:
|
|
92
|
+
return "Query Parser"
|
|
93
|
+
|
|
87
94
|
@classmethod
|
|
88
95
|
def create(cls, config_dict: dict, metadata: OpenMetadata, **kwargs):
|
|
89
96
|
config = ConfigModel.parse_obj(config_dict)
|
|
@@ -106,7 +113,7 @@ class QueryParserProcessor(Processor):
|
|
|
106
113
|
left=StackTraceError(
|
|
107
114
|
name="Query",
|
|
108
115
|
error=f"Error processing query [{table_query.query}]: {exc}",
|
|
109
|
-
|
|
116
|
+
stackTrace=traceback.format_exc(),
|
|
110
117
|
)
|
|
111
118
|
)
|
|
112
119
|
|
|
@@ -47,6 +47,9 @@ from metadata.generated.schema.entity.teams.team import Team
|
|
|
47
47
|
from metadata.generated.schema.entity.teams.user import User
|
|
48
48
|
from metadata.generated.schema.tests.basic import TestCaseResult
|
|
49
49
|
from metadata.generated.schema.tests.testCase import TestCase
|
|
50
|
+
from metadata.generated.schema.tests.testCaseResolutionStatus import (
|
|
51
|
+
TestCaseResolutionStatus,
|
|
52
|
+
)
|
|
50
53
|
from metadata.generated.schema.tests.testSuite import TestSuite
|
|
51
54
|
from metadata.generated.schema.type.schema import Topic
|
|
52
55
|
from metadata.ingestion.api.models import Either, Entity, StackTraceError
|
|
@@ -60,6 +63,7 @@ from metadata.ingestion.models.ometa_topic_data import OMetaTopicSampleData
|
|
|
60
63
|
from metadata.ingestion.models.patch_request import (
|
|
61
64
|
ALLOWED_COMMON_PATCH_FIELDS,
|
|
62
65
|
RESTRICT_UPDATE_LIST,
|
|
66
|
+
PatchedEntity,
|
|
63
67
|
PatchRequest,
|
|
64
68
|
)
|
|
65
69
|
from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
|
|
@@ -67,6 +71,7 @@ from metadata.ingestion.models.profile_data import OMetaTableProfileSampleData
|
|
|
67
71
|
from metadata.ingestion.models.search_index_data import OMetaIndexSampleData
|
|
68
72
|
from metadata.ingestion.models.tests_data import (
|
|
69
73
|
OMetaLogicalTestSuiteSample,
|
|
74
|
+
OMetaTestCaseResolutionStatus,
|
|
70
75
|
OMetaTestCaseResultsSample,
|
|
71
76
|
OMetaTestCaseSample,
|
|
72
77
|
OMetaTestSuiteSample,
|
|
@@ -115,6 +120,10 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
115
120
|
config = MetadataRestSinkConfig.parse_obj(config_dict)
|
|
116
121
|
return cls(config, metadata)
|
|
117
122
|
|
|
123
|
+
@property
|
|
124
|
+
def name(self) -> str:
|
|
125
|
+
return "OpenMetadata"
|
|
126
|
+
|
|
118
127
|
@singledispatchmethod
|
|
119
128
|
def _run_dispatch(self, record: Entity) -> Either[Any]:
|
|
120
129
|
logger.debug(f"Processing Create request {type(record)}")
|
|
@@ -132,14 +141,14 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
132
141
|
error = f"Failed to ingest {log} due to api request failure: {err}"
|
|
133
142
|
return Either(
|
|
134
143
|
left=StackTraceError(
|
|
135
|
-
name=log, error=error,
|
|
144
|
+
name=log, error=error, stackTrace=traceback.format_exc()
|
|
136
145
|
)
|
|
137
146
|
)
|
|
138
147
|
except Exception as exc:
|
|
139
148
|
error = f"Failed to ingest {log}: {exc}"
|
|
140
149
|
return Either(
|
|
141
150
|
left=StackTraceError(
|
|
142
|
-
name=log, error=error,
|
|
151
|
+
name=log, error=error, stackTrace=traceback.format_exc()
|
|
143
152
|
)
|
|
144
153
|
)
|
|
145
154
|
|
|
@@ -155,7 +164,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
155
164
|
error = f"Failed to ingest {type(entity_request).__name__}"
|
|
156
165
|
return Either(
|
|
157
166
|
left=StackTraceError(
|
|
158
|
-
name=type(entity_request).__name__, error=error,
|
|
167
|
+
name=type(entity_request).__name__, error=error, stackTrace=None
|
|
159
168
|
)
|
|
160
169
|
)
|
|
161
170
|
|
|
@@ -171,7 +180,8 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
171
180
|
allowed_fields=ALLOWED_COMMON_PATCH_FIELDS,
|
|
172
181
|
restrict_update_fields=RESTRICT_UPDATE_LIST,
|
|
173
182
|
)
|
|
174
|
-
|
|
183
|
+
patched_entity = PatchedEntity(new_entity=entity) if entity else None
|
|
184
|
+
return Either(right=patched_entity)
|
|
175
185
|
|
|
176
186
|
@_run_dispatch.register
|
|
177
187
|
def write_custom_properties(self, record: OMetaCustomProperties) -> Either[Dict]:
|
|
@@ -200,7 +210,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
200
210
|
left=StackTraceError(
|
|
201
211
|
name="Data Model",
|
|
202
212
|
error="Sink did not receive a table. We cannot ingest the data model.",
|
|
203
|
-
|
|
213
|
+
stackTrace=None,
|
|
204
214
|
)
|
|
205
215
|
)
|
|
206
216
|
|
|
@@ -275,7 +285,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
275
285
|
for role in record.roles:
|
|
276
286
|
try:
|
|
277
287
|
role_entity = self.metadata.get_by_name(
|
|
278
|
-
entity=Role, fqn=str(role.name.__root__
|
|
288
|
+
entity=Role, fqn=str(role.name.__root__)
|
|
279
289
|
)
|
|
280
290
|
except APIError:
|
|
281
291
|
role_entity = self._create_role(role)
|
|
@@ -411,6 +421,15 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
411
421
|
)
|
|
412
422
|
return Either(right=res)
|
|
413
423
|
|
|
424
|
+
@_run_dispatch.register
|
|
425
|
+
def write_test_case_resolution_status(
|
|
426
|
+
self, record: OMetaTestCaseResolutionStatus
|
|
427
|
+
) -> TestCaseResolutionStatus:
|
|
428
|
+
"""For sample data"""
|
|
429
|
+
res = self.metadata.create_test_case_resolution(record.test_case_resolution)
|
|
430
|
+
|
|
431
|
+
return Either(right=res)
|
|
432
|
+
|
|
414
433
|
@_run_dispatch.register
|
|
415
434
|
def write_data_insight_sample(
|
|
416
435
|
self, record: OMetaDataInsightSample
|
|
@@ -34,7 +34,6 @@ from metadata.generated.schema.entity.services.dashboardService import (
|
|
|
34
34
|
DashboardConnection,
|
|
35
35
|
DashboardService,
|
|
36
36
|
)
|
|
37
|
-
from metadata.generated.schema.entity.teams.user import User
|
|
38
37
|
from metadata.generated.schema.metadataIngestion.dashboardServiceMetadataPipeline import (
|
|
39
38
|
DashboardServiceMetadataPipeline,
|
|
40
39
|
)
|
|
@@ -55,8 +54,8 @@ from metadata.ingestion.models.patch_request import PatchRequest
|
|
|
55
54
|
from metadata.ingestion.models.topology import (
|
|
56
55
|
NodeStage,
|
|
57
56
|
ServiceTopology,
|
|
57
|
+
TopologyContext,
|
|
58
58
|
TopologyNode,
|
|
59
|
-
create_source_context,
|
|
60
59
|
)
|
|
61
60
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
62
61
|
from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
|
|
@@ -105,7 +104,7 @@ class DashboardServiceTopology(ServiceTopology):
|
|
|
105
104
|
),
|
|
106
105
|
NodeStage(
|
|
107
106
|
type_=OMetaTagAndClassification,
|
|
108
|
-
processor="
|
|
107
|
+
processor="yield_bulk_tags",
|
|
109
108
|
nullable=True,
|
|
110
109
|
),
|
|
111
110
|
],
|
|
@@ -133,14 +132,19 @@ class DashboardServiceTopology(ServiceTopology):
|
|
|
133
132
|
dashboard = TopologyNode(
|
|
134
133
|
producer="get_dashboard",
|
|
135
134
|
stages=[
|
|
135
|
+
NodeStage(
|
|
136
|
+
type_=OMetaTagAndClassification,
|
|
137
|
+
processor="yield_tags",
|
|
138
|
+
nullable=True,
|
|
139
|
+
),
|
|
136
140
|
NodeStage(
|
|
137
141
|
type_=Chart,
|
|
138
142
|
context="charts",
|
|
139
143
|
processor="yield_dashboard_chart",
|
|
140
144
|
consumer=["dashboard_service"],
|
|
141
145
|
nullable=True,
|
|
142
|
-
|
|
143
|
-
|
|
146
|
+
store_all_in_context=True,
|
|
147
|
+
clear_context=True,
|
|
144
148
|
use_cache=True,
|
|
145
149
|
),
|
|
146
150
|
NodeStage(
|
|
@@ -149,8 +153,8 @@ class DashboardServiceTopology(ServiceTopology):
|
|
|
149
153
|
processor="yield_datamodel",
|
|
150
154
|
consumer=["dashboard_service"],
|
|
151
155
|
nullable=True,
|
|
152
|
-
|
|
153
|
-
|
|
156
|
+
store_all_in_context=True,
|
|
157
|
+
clear_context=True,
|
|
154
158
|
use_cache=True,
|
|
155
159
|
),
|
|
156
160
|
NodeStage(
|
|
@@ -160,12 +164,6 @@ class DashboardServiceTopology(ServiceTopology):
|
|
|
160
164
|
consumer=["dashboard_service"],
|
|
161
165
|
use_cache=True,
|
|
162
166
|
),
|
|
163
|
-
NodeStage(
|
|
164
|
-
type_=User,
|
|
165
|
-
context="owner",
|
|
166
|
-
processor="process_owner",
|
|
167
|
-
consumer=["dashboard_service"],
|
|
168
|
-
),
|
|
169
167
|
NodeStage(
|
|
170
168
|
type_=AddLineageRequest,
|
|
171
169
|
processor="yield_dashboard_lineage",
|
|
@@ -196,7 +194,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
|
|
|
196
194
|
service_connection: DashboardConnection.__fields__["config"].type_
|
|
197
195
|
|
|
198
196
|
topology = DashboardServiceTopology()
|
|
199
|
-
context =
|
|
197
|
+
context = TopologyContext.create(topology)
|
|
200
198
|
dashboard_source_state: Set = set()
|
|
201
199
|
datamodel_source_state: Set = set()
|
|
202
200
|
|
|
@@ -218,6 +216,10 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
|
|
|
218
216
|
self.connection_obj = self.client
|
|
219
217
|
self.test_connection()
|
|
220
218
|
|
|
219
|
+
@property
|
|
220
|
+
def name(self) -> str:
|
|
221
|
+
return self.service_connection.type.name
|
|
222
|
+
|
|
221
223
|
@abstractmethod
|
|
222
224
|
def yield_dashboard(
|
|
223
225
|
self, dashboard_details: Any
|
|
@@ -333,7 +335,16 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
|
|
|
333
335
|
dashboard_details, db_service_name
|
|
334
336
|
) or []
|
|
335
337
|
|
|
336
|
-
def
|
|
338
|
+
def yield_bulk_tags(
|
|
339
|
+
self, *args, **kwargs
|
|
340
|
+
) -> Iterable[Either[OMetaTagAndClassification]]:
|
|
341
|
+
"""
|
|
342
|
+
Method to bulk fetch dashboard tags
|
|
343
|
+
"""
|
|
344
|
+
|
|
345
|
+
def yield_tags(
|
|
346
|
+
self, dashboard_details
|
|
347
|
+
) -> Iterable[Either[OMetaTagAndClassification]]:
|
|
337
348
|
"""
|
|
338
349
|
Method to fetch dashboard tags
|
|
339
350
|
"""
|
|
@@ -386,34 +397,16 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
|
|
|
386
397
|
params={"service": self.context.dashboard_service},
|
|
387
398
|
)
|
|
388
399
|
|
|
389
|
-
def
|
|
400
|
+
def get_owner_ref( # pylint: disable=unused-argument, useless-return
|
|
401
|
+
self, dashboard_details
|
|
402
|
+
) -> Optional[EntityReference]:
|
|
390
403
|
"""
|
|
391
|
-
Method to process the dashboard
|
|
404
|
+
Method to process the dashboard owners
|
|
392
405
|
"""
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
)
|
|
398
|
-
if owner:
|
|
399
|
-
dashboard_fqn = fqn.build(
|
|
400
|
-
self.metadata,
|
|
401
|
-
entity_type=Dashboard,
|
|
402
|
-
service_name=self.context.dashboard_service,
|
|
403
|
-
dashboard_name=self.context.dashboard,
|
|
404
|
-
)
|
|
405
|
-
dashboard_entity = self.metadata.get_by_name(
|
|
406
|
-
entity=Dashboard, fqn=dashboard_fqn
|
|
407
|
-
)
|
|
408
|
-
self.metadata.patch_owner(
|
|
409
|
-
entity=Dashboard,
|
|
410
|
-
source=dashboard_entity,
|
|
411
|
-
owner=owner,
|
|
412
|
-
force=False,
|
|
413
|
-
)
|
|
414
|
-
except Exception as exc:
|
|
415
|
-
logger.debug(traceback.format_exc())
|
|
416
|
-
logger.warning(f"Error processing owner for {dashboard_details}: {exc}")
|
|
406
|
+
logger.debug(
|
|
407
|
+
f"Processing ownership is not supported for {self.service_connection.type.name}"
|
|
408
|
+
)
|
|
409
|
+
return None
|
|
417
410
|
|
|
418
411
|
def register_record(self, dashboard_request: CreateDashboardRequest) -> None:
|
|
419
412
|
"""
|
|
@@ -429,7 +422,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
|
|
|
429
422
|
self.dashboard_source_state.add(dashboard_fqn)
|
|
430
423
|
|
|
431
424
|
def register_record_datamodel(
|
|
432
|
-
self,
|
|
425
|
+
self, datamodel_request: CreateDashboardDataModelRequest
|
|
433
426
|
) -> None:
|
|
434
427
|
"""
|
|
435
428
|
Mark the datamodel record as scanned and update the datamodel_source_state
|
|
@@ -437,27 +430,12 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
|
|
|
437
430
|
datamodel_fqn = fqn.build(
|
|
438
431
|
self.metadata,
|
|
439
432
|
entity_type=DashboardDataModel,
|
|
440
|
-
service_name=
|
|
441
|
-
data_model_name=
|
|
433
|
+
service_name=datamodel_request.service.__root__,
|
|
434
|
+
data_model_name=datamodel_request.name.__root__,
|
|
442
435
|
)
|
|
443
436
|
|
|
444
437
|
self.datamodel_source_state.add(datamodel_fqn)
|
|
445
438
|
|
|
446
|
-
def get_owner_details( # pylint: disable=useless-return
|
|
447
|
-
self, dashboard_details # pylint: disable=unused-argument
|
|
448
|
-
) -> Optional[EntityReference]:
|
|
449
|
-
"""Get dashboard owner
|
|
450
|
-
|
|
451
|
-
Args:
|
|
452
|
-
dashboard_details:
|
|
453
|
-
Returns:
|
|
454
|
-
Optional[EntityReference]
|
|
455
|
-
"""
|
|
456
|
-
logger.debug(
|
|
457
|
-
f"Processing ownership is not supported for {self.service_connection.type.name}"
|
|
458
|
-
)
|
|
459
|
-
return None
|
|
460
|
-
|
|
461
439
|
@staticmethod
|
|
462
440
|
def _get_add_lineage_request(
|
|
463
441
|
to_entity: Union[Dashboard, DashboardDataModel],
|
|
@@ -505,7 +483,7 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
|
|
|
505
483
|
self.context.project_name = ( # pylint: disable=assignment-from-none
|
|
506
484
|
self.get_project_name(dashboard_details=dashboard_details)
|
|
507
485
|
)
|
|
508
|
-
if
|
|
486
|
+
if filter_by_project(
|
|
509
487
|
self.source_config.projectFilterPattern,
|
|
510
488
|
self.context.project_name,
|
|
511
489
|
):
|
|
@@ -553,7 +531,6 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
|
|
|
553
531
|
)
|
|
554
532
|
|
|
555
533
|
def check_database_schema_name(self, database_schema_name: str):
|
|
556
|
-
|
|
557
534
|
"""
|
|
558
535
|
Check if the input database schema name is equal to "<default>" and return the input name if it is not.
|
|
559
536
|
|