openmetadata-ingestion 1.2.4.0__py3-none-any.whl → 1.3.2.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openmetadata-ingestion might be problematic. Click here for more details.
- airflow_provider_openmetadata/lineage/runner.py +44 -5
- airflow_provider_openmetadata/lineage/status.py +1 -1
- metadata/antlr/split_listener.py +1 -4
- metadata/applications/auto_tagger.py +212 -0
- metadata/cli/app.py +47 -0
- metadata/cli/db_dump.py +7 -3
- metadata/clients/aws_client.py +4 -0
- metadata/clients/azure_client.py +85 -0
- metadata/clients/domo_client.py +0 -2
- metadata/cmd.py +24 -13
- metadata/data_insight/processor/reports/cost_analysis_report_data_processor.py +40 -48
- metadata/data_insight/processor/reports/data_processor.py +1 -0
- metadata/data_insight/processor/reports/entity_report_data_processor.py +8 -2
- metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +4 -0
- metadata/data_insight/producer/cost_analysis_producer.py +78 -14
- metadata/data_insight/producer/entity_producer.py +1 -1
- metadata/data_insight/producer/producer_interface.py +1 -1
- metadata/data_insight/producer/web_analytics_producer.py +1 -1
- metadata/data_insight/source/metadata.py +21 -5
- metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +29 -0
- metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +29 -0
- metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +6 -6
- metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +35 -0
- metadata/data_quality/interface/test_suite_interface_factory.py +40 -0
- metadata/data_quality/processor/test_case_runner.py +17 -8
- metadata/data_quality/source/test_suite.py +10 -4
- metadata/data_quality/validations/base_test_handler.py +21 -17
- metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +38 -1
- metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +36 -1
- metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +15 -0
- metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +27 -0
- metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +23 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +23 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +27 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +23 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +11 -0
- metadata/data_quality/validations/mixins/pandas_validator_mixin.py +13 -2
- metadata/data_quality/validations/mixins/sqa_validator_mixin.py +40 -0
- metadata/data_quality/validations/table/base/tableColumnToMatchSet.py +2 -1
- metadata/data_quality/validations/table/pandas/tableColumnToMatchSet.py +2 -1
- metadata/data_quality/validations/table/sqlalchemy/tableColumnToMatchSet.py +7 -2
- metadata/examples/workflows/bigtable.yaml +32 -0
- metadata/examples/workflows/databricks.yaml +0 -1
- metadata/examples/workflows/datalake_azure_default.yaml +29 -0
- metadata/examples/workflows/mongodb.yaml +4 -2
- metadata/examples/workflows/mstr.yaml +24 -0
- metadata/examples/workflows/sas.yaml +28 -0
- metadata/examples/workflows/unity_catalog.yaml +27 -0
- metadata/examples/workflows/unity_catalog_lineage.yaml +18 -0
- metadata/examples/workflows/unity_catalog_usage.yaml +35 -0
- metadata/generated/antlr/EntityLinkLexer.py +425 -292
- metadata/generated/antlr/EntityLinkListener.py +12 -12
- metadata/generated/antlr/EntityLinkParser.py +228 -103
- metadata/generated/schema/analytics/__init__.py +1 -1
- metadata/generated/schema/analytics/basic.py +1 -1
- metadata/generated/schema/analytics/reportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
- metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
- metadata/generated/schema/api/__init__.py +1 -1
- metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
- metadata/generated/schema/api/analytics/__init__.py +1 -1
- metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
- metadata/generated/schema/api/automations/__init__.py +1 -1
- metadata/generated/schema/api/automations/createWorkflow.py +1 -1
- metadata/generated/schema/api/bulkAssets.py +1 -1
- metadata/generated/schema/api/classification/__init__.py +1 -1
- metadata/generated/schema/api/classification/createClassification.py +1 -1
- metadata/generated/schema/api/classification/createTag.py +1 -1
- metadata/generated/schema/api/classification/loadTags.py +1 -1
- metadata/generated/schema/api/createBot.py +2 -3
- metadata/generated/schema/api/createEventPublisherJob.py +2 -2
- metadata/generated/schema/api/createType.py +1 -1
- metadata/generated/schema/api/data/__init__.py +1 -1
- metadata/generated/schema/api/data/createChart.py +1 -1
- metadata/generated/schema/api/data/createContainer.py +2 -2
- metadata/generated/schema/api/data/createCustomProperty.py +7 -12
- metadata/generated/schema/api/data/createDashboard.py +1 -1
- metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
- metadata/generated/schema/api/data/createDatabase.py +2 -3
- metadata/generated/schema/api/data/createDatabaseSchema.py +2 -3
- metadata/generated/schema/api/data/createGlossary.py +2 -3
- metadata/generated/schema/api/data/createGlossaryTerm.py +2 -3
- metadata/generated/schema/api/data/createMlModel.py +1 -1
- metadata/generated/schema/api/data/createPipeline.py +2 -2
- metadata/generated/schema/api/data/createQuery.py +2 -2
- metadata/generated/schema/api/data/createSearchIndex.py +1 -1
- metadata/generated/schema/api/data/createStoredProcedure.py +2 -4
- metadata/generated/schema/api/data/createTable.py +2 -2
- metadata/generated/schema/api/data/createTableProfile.py +1 -1
- metadata/generated/schema/api/data/createTopic.py +1 -1
- metadata/generated/schema/api/data/loadGlossary.py +1 -1
- metadata/generated/schema/api/data/restoreEntity.py +1 -1
- metadata/generated/schema/api/dataInsight/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
- metadata/generated/schema/api/docStore/__init__.py +1 -1
- metadata/generated/schema/api/docStore/createDocument.py +1 -1
- metadata/generated/schema/api/domains/__init__.py +1 -1
- metadata/generated/schema/api/domains/createDataProduct.py +1 -1
- metadata/generated/schema/api/domains/createDomain.py +1 -1
- metadata/generated/schema/api/feed/__init__.py +1 -1
- metadata/generated/schema/api/feed/closeTask.py +8 -1
- metadata/generated/schema/api/feed/createPost.py +1 -1
- metadata/generated/schema/api/feed/createSuggestion.py +30 -0
- metadata/generated/schema/api/feed/createThread.py +5 -1
- metadata/generated/schema/api/feed/resolveTask.py +12 -1
- metadata/generated/schema/api/feed/threadCount.py +18 -14
- metadata/generated/schema/api/lineage/__init__.py +1 -1
- metadata/generated/schema/api/lineage/addLineage.py +1 -1
- metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
- metadata/generated/schema/api/policies/__init__.py +1 -1
- metadata/generated/schema/api/policies/createPolicy.py +1 -1
- metadata/generated/schema/api/services/__init__.py +1 -1
- metadata/generated/schema/api/services/createDashboardService.py +1 -1
- metadata/generated/schema/api/services/createDatabaseService.py +1 -1
- metadata/generated/schema/api/services/createMessagingService.py +1 -1
- metadata/generated/schema/api/services/createMetadataService.py +1 -1
- metadata/generated/schema/api/services/createMlModelService.py +1 -1
- metadata/generated/schema/api/services/createPipelineService.py +1 -1
- metadata/generated/schema/api/services/createSearchService.py +1 -1
- metadata/generated/schema/api/services/createStorageService.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
- metadata/generated/schema/api/setOwner.py +1 -1
- metadata/generated/schema/api/teams/__init__.py +1 -1
- metadata/generated/schema/api/teams/createPersona.py +1 -1
- metadata/generated/schema/api/teams/createRole.py +2 -3
- metadata/generated/schema/api/teams/createTeam.py +1 -1
- metadata/generated/schema/api/teams/createUser.py +2 -2
- metadata/generated/schema/api/tests/__init__.py +1 -1
- metadata/generated/schema/api/tests/createCustomMetric.py +4 -2
- metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
- metadata/generated/schema/api/tests/createTestCase.py +4 -1
- metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +31 -0
- metadata/generated/schema/api/tests/createTestDefinition.py +2 -1
- metadata/generated/schema/api/tests/createTestSuite.py +1 -1
- metadata/generated/schema/api/voteRequest.py +1 -1
- metadata/generated/schema/auth/__init__.py +1 -1
- metadata/generated/schema/auth/basicAuth.py +1 -1
- metadata/generated/schema/auth/basicLoginRequest.py +1 -1
- metadata/generated/schema/auth/changePasswordRequest.py +1 -1
- metadata/generated/schema/auth/createPersonalToken.py +1 -1
- metadata/generated/schema/auth/emailRequest.py +1 -1
- metadata/generated/schema/auth/emailVerificationToken.py +1 -1
- metadata/generated/schema/auth/generateToken.py +1 -1
- metadata/generated/schema/auth/jwtAuth.py +1 -1
- metadata/generated/schema/auth/loginRequest.py +1 -1
- metadata/generated/schema/auth/logoutRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetRequest.py +3 -3
- metadata/generated/schema/auth/passwordResetToken.py +1 -1
- metadata/generated/schema/auth/personalAccessToken.py +1 -1
- metadata/generated/schema/auth/refreshToken.py +1 -1
- metadata/generated/schema/auth/registrationRequest.py +2 -2
- metadata/generated/schema/auth/revokePersonalToken.py +1 -1
- metadata/generated/schema/auth/revokeToken.py +1 -1
- metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
- metadata/generated/schema/auth/ssoAuth.py +1 -1
- metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
- metadata/generated/schema/configuration/__init__.py +1 -1
- metadata/generated/schema/configuration/appsPrivateConfiguration.py +32 -0
- metadata/generated/schema/configuration/authConfig.py +1 -1
- metadata/generated/schema/configuration/authenticationConfiguration.py +13 -2
- metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
- metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/dataQualityConfiguration.py +16 -0
- metadata/generated/schema/configuration/elasticSearchConfiguration.py +7 -1
- metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
- metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
- metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
- metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapConfiguration.py +28 -2
- metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
- metadata/generated/schema/configuration/loginConfiguration.py +1 -1
- metadata/generated/schema/configuration/logoConfiguration.py +1 -1
- metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
- metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
- metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
- metadata/generated/schema/dataInsight/__init__.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
- metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
- metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
- metadata/generated/schema/dataInsight/type/__init__.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +1 -1
- metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +1 -1
- metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +1 -1
- metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
- metadata/generated/schema/email/__init__.py +1 -1
- metadata/generated/schema/email/emailRequest.py +1 -1
- metadata/generated/schema/email/smtpSettings.py +1 -1
- metadata/generated/schema/entity/__init__.py +1 -1
- metadata/generated/schema/entity/applications/__init__.py +1 -1
- metadata/generated/schema/entity/applications/app.py +24 -5
- metadata/generated/schema/entity/applications/appRunRecord.py +2 -7
- metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/applicationConfig.py +30 -0
- metadata/generated/schema/entity/{services/connections/database/mongoDB → applications/configuration/external}/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/autoTaggerAppConfig.py +28 -0
- metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +46 -0
- metadata/generated/schema/entity/applications/configuration/internal/__init__.py +3 -0
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +14 -0
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +27 -0
- metadata/generated/schema/entity/applications/configuration/{searchIndexingApp.py → internal/searchIndexingAppConfig.py} +14 -4
- metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +3 -0
- metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +26 -0
- metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
- metadata/generated/schema/entity/applications/jobStatus.py +3 -3
- metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +12 -3
- metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +12 -3
- metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
- metadata/generated/schema/entity/automations/__init__.py +1 -1
- metadata/generated/schema/entity/automations/testServiceConnection.py +2 -2
- metadata/generated/schema/entity/automations/workflow.py +1 -1
- metadata/generated/schema/entity/bot.py +2 -3
- metadata/generated/schema/entity/classification/__init__.py +1 -1
- metadata/generated/schema/entity/classification/classification.py +1 -1
- metadata/generated/schema/entity/classification/tag.py +2 -2
- metadata/generated/schema/entity/data/__init__.py +1 -1
- metadata/generated/schema/entity/data/chart.py +1 -1
- metadata/generated/schema/entity/data/container.py +4 -9
- metadata/generated/schema/entity/data/dashboard.py +1 -1
- metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
- metadata/generated/schema/entity/data/database.py +4 -8
- metadata/generated/schema/entity/data/databaseSchema.py +2 -8
- metadata/generated/schema/entity/data/glossary.py +1 -1
- metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
- metadata/generated/schema/entity/data/metrics.py +1 -1
- metadata/generated/schema/entity/data/mlmodel.py +2 -2
- metadata/generated/schema/entity/data/pipeline.py +6 -9
- metadata/generated/schema/entity/data/query.py +2 -2
- metadata/generated/schema/entity/data/report.py +1 -1
- metadata/generated/schema/entity/data/searchIndex.py +1 -1
- metadata/generated/schema/entity/data/storedProcedure.py +3 -9
- metadata/generated/schema/entity/data/table.py +18 -12
- metadata/generated/schema/entity/data/topic.py +1 -1
- metadata/generated/schema/entity/docStore/__init__.py +1 -1
- metadata/generated/schema/entity/docStore/document.py +1 -1
- metadata/generated/schema/entity/domains/__init__.py +1 -1
- metadata/generated/schema/entity/domains/dataProduct.py +1 -1
- metadata/generated/schema/entity/domains/domain.py +1 -1
- metadata/generated/schema/entity/events/__init__.py +1 -1
- metadata/generated/schema/entity/events/webhook.py +6 -3
- metadata/generated/schema/entity/feed/__init__.py +1 -1
- metadata/generated/schema/entity/feed/suggestion.py +61 -0
- metadata/generated/schema/entity/feed/thread.py +17 -1
- metadata/generated/schema/entity/policies/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
- metadata/generated/schema/entity/policies/filters.py +2 -5
- metadata/generated/schema/entity/policies/policy.py +1 -1
- metadata/generated/schema/entity/services/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +46 -0
- metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/athenaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +35 -2
- metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +41 -0
- metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +20 -0
- metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +6 -1
- metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricksConnection.py +2 -7
- metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/db2Connection.py +3 -2
- metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dorisConnection.py +86 -0
- metadata/generated/schema/entity/services/connections/database/druidConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/hiveConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +3 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +17 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +16 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +23 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +43 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +20 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +69 -0
- metadata/generated/schema/entity/services/connections/database/icebergConnection.py +35 -0
- metadata/generated/schema/entity/services/connections/database/impalaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +24 -14
- metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -2
- metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +8 -4
- metadata/generated/schema/entity/services/connections/database/oracleConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +9 -7
- metadata/generated/schema/entity/services/connections/database/postgresConnection.py +8 -4
- metadata/generated/schema/entity/services/connections/database/prestoConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +3 -0
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +20 -0
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +36 -0
- metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +7 -41
- metadata/generated/schema/entity/services/connections/database/sasConnection.py +48 -0
- metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/trinoConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +90 -0
- metadata/generated/schema/entity/services/connections/database/verticaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +14 -2
- metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +14 -2
- metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +5 -20
- metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +2 -8
- metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +23 -0
- metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/{adlsConection.py → adlsConnection.py} +2 -2
- metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
- metadata/generated/schema/entity/services/dashboardService.py +4 -1
- metadata/generated/schema/entity/services/databaseService.py +16 -1
- metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +27 -22
- metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/status.py +56 -0
- metadata/generated/schema/entity/services/messagingService.py +1 -1
- metadata/generated/schema/entity/services/metadataService.py +1 -1
- metadata/generated/schema/entity/services/mlmodelService.py +1 -1
- metadata/generated/schema/entity/services/pipelineService.py +4 -1
- metadata/generated/schema/entity/services/searchService.py +1 -1
- metadata/generated/schema/entity/services/serviceType.py +1 -1
- metadata/generated/schema/entity/services/storageService.py +10 -3
- metadata/generated/schema/entity/teams/__init__.py +1 -1
- metadata/generated/schema/entity/teams/persona.py +1 -1
- metadata/generated/schema/entity/teams/role.py +2 -6
- metadata/generated/schema/entity/teams/team.py +1 -1
- metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
- metadata/generated/schema/entity/teams/user.py +3 -10
- metadata/generated/schema/entity/type.py +4 -19
- metadata/generated/schema/entity/utils/__init__.py +1 -1
- metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
- metadata/generated/schema/entity/utils/servicesCount.py +1 -1
- metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
- metadata/generated/schema/events/__init__.py +1 -1
- metadata/generated/schema/events/alertMetrics.py +27 -0
- metadata/generated/schema/events/api/__init__.py +1 -1
- metadata/generated/schema/events/api/createEventSubscription.py +14 -12
- metadata/generated/schema/events/emailAlertConfig.py +1 -1
- metadata/generated/schema/events/eventFilterRule.py +13 -2
- metadata/generated/schema/events/eventSubscription.py +91 -49
- metadata/generated/schema/events/eventSubscriptionOffset.py +21 -0
- metadata/generated/schema/events/failedEvent.py +26 -0
- metadata/generated/schema/events/filterResourceDescriptor.py +27 -0
- metadata/generated/schema/events/subscriptionResourceDescriptor.py +11 -10
- metadata/generated/schema/metadataIngestion/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/application.py +40 -0
- metadata/generated/schema/metadataIngestion/applicationPipeline.py +15 -6
- metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +6 -1
- metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +11 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +6 -3
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +6 -3
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +6 -3
- metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +2 -2
- metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +24 -0
- metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +8 -1
- metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/workflow.py +1 -1
- metadata/generated/schema/monitoring/__init__.py +1 -1
- metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
- metadata/generated/schema/security/__init__.py +1 -1
- metadata/generated/schema/security/client/__init__.py +1 -1
- metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/oidcClientConfig.py +46 -0
- metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
- metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
- metadata/generated/schema/security/credentials/__init__.py +1 -1
- metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
- metadata/generated/schema/security/credentials/azureCredentials.py +21 -9
- metadata/generated/schema/security/credentials/basicAuth.py +1 -1
- metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpCredentials.py +7 -3
- metadata/generated/schema/security/credentials/gcpExternalAccount.py +37 -0
- metadata/generated/schema/security/credentials/gcpValues.py +2 -2
- metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
- metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
- metadata/generated/schema/security/secrets/__init__.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +14 -4
- metadata/generated/schema/security/secrets/secretsManagerProvider.py +4 -2
- metadata/generated/schema/security/securityConfiguration.py +1 -1
- metadata/generated/schema/security/ssl/__init__.py +1 -1
- metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
- metadata/generated/schema/security/ssl/verifySSLConfig.py +4 -1
- metadata/generated/schema/settings/__init__.py +1 -1
- metadata/generated/schema/settings/settings.py +1 -1
- metadata/generated/schema/system/__init__.py +1 -1
- metadata/generated/schema/system/entityError.py +17 -0
- metadata/generated/schema/system/eventPublisherJob.py +5 -19
- metadata/generated/schema/system/indexingError.py +34 -0
- metadata/generated/schema/system/ui/__init__.py +1 -1
- metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
- metadata/generated/schema/system/ui/page.py +1 -1
- metadata/generated/schema/system/validationResponse.py +43 -0
- metadata/generated/schema/tests/__init__.py +1 -1
- metadata/generated/schema/tests/assigned.py +18 -0
- metadata/generated/schema/tests/basic.py +17 -39
- metadata/generated/schema/tests/customMetric.py +4 -2
- metadata/generated/schema/tests/resolved.py +35 -0
- metadata/generated/schema/tests/testCase.py +11 -2
- metadata/generated/schema/tests/testCaseResolutionStatus.py +63 -0
- metadata/generated/schema/tests/testDefinition.py +6 -1
- metadata/generated/schema/tests/testSuite.py +1 -1
- metadata/generated/schema/type/__init__.py +1 -1
- metadata/generated/schema/type/auditLog.py +1 -1
- metadata/generated/schema/type/basic.py +10 -6
- metadata/generated/schema/type/bulkOperationResult.py +1 -1
- metadata/generated/schema/type/changeEvent.py +5 -75
- metadata/generated/schema/type/changeEventType.py +29 -0
- metadata/generated/schema/type/collectionDescriptor.py +1 -1
- metadata/generated/schema/type/csvDocumentation.py +1 -1
- metadata/generated/schema/type/csvErrorType.py +1 -1
- metadata/generated/schema/type/csvFile.py +1 -1
- metadata/generated/schema/type/csvImportResult.py +1 -1
- metadata/generated/schema/type/customProperties/__init__.py +3 -0
- metadata/generated/schema/type/customProperties/enumConfig.py +17 -0
- metadata/generated/schema/type/customProperty.py +52 -0
- metadata/generated/schema/type/dailyCount.py +1 -1
- metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
- metadata/generated/schema/type/entityHistory.py +1 -1
- metadata/generated/schema/type/entityLineage.py +2 -1
- metadata/generated/schema/type/entityReference.py +5 -1
- metadata/generated/schema/type/entityReferenceList.py +1 -1
- metadata/generated/schema/type/entityRelationship.py +1 -1
- metadata/generated/schema/type/entityUsage.py +1 -1
- metadata/generated/schema/type/filterPattern.py +1 -1
- metadata/generated/schema/type/function.py +7 -6
- metadata/generated/schema/type/include.py +1 -1
- metadata/generated/schema/type/jdbcConnection.py +1 -1
- metadata/generated/schema/type/lifeCycle.py +1 -1
- metadata/generated/schema/type/paging.py +1 -1
- metadata/generated/schema/type/profile.py +1 -1
- metadata/generated/schema/type/queryParserData.py +2 -2
- metadata/generated/schema/type/reaction.py +1 -1
- metadata/generated/schema/type/schedule.py +1 -1
- metadata/generated/schema/type/schema.py +1 -1
- metadata/generated/schema/type/tableQuery.py +2 -2
- metadata/generated/schema/type/tableUsageCount.py +1 -1
- metadata/generated/schema/type/tagLabel.py +1 -1
- metadata/generated/schema/type/usageDetails.py +1 -1
- metadata/generated/schema/type/usageRequest.py +1 -1
- metadata/generated/schema/type/votes.py +1 -1
- metadata/great_expectations/action.py +12 -20
- metadata/ingestion/api/delete.py +5 -2
- metadata/ingestion/api/models.py +4 -10
- metadata/ingestion/api/parser.py +3 -37
- metadata/ingestion/api/status.py +15 -4
- metadata/ingestion/api/step.py +39 -6
- metadata/ingestion/api/steps.py +34 -1
- metadata/ingestion/api/topology_runner.py +117 -168
- metadata/ingestion/bulksink/metadata_usage.py +11 -5
- metadata/ingestion/lineage/models.py +4 -0
- metadata/ingestion/lineage/parser.py +28 -12
- metadata/ingestion/lineage/sql_lineage.py +9 -17
- metadata/ingestion/models/custom_properties.py +0 -1
- metadata/ingestion/models/custom_pydantic.py +4 -2
- metadata/ingestion/models/lf_tags_model.py +33 -0
- metadata/ingestion/models/patch_request.py +160 -2
- metadata/ingestion/models/tests_data.py +9 -0
- metadata/ingestion/models/topology.py +177 -60
- metadata/ingestion/ometa/auth_provider.py +0 -349
- metadata/ingestion/ometa/client.py +12 -3
- metadata/ingestion/ometa/mixins/custom_property_mixin.py +11 -11
- metadata/ingestion/ometa/mixins/es_mixin.py +17 -0
- metadata/ingestion/ometa/mixins/patch_mixin.py +21 -72
- metadata/ingestion/ometa/mixins/suggestions_mixin.py +41 -0
- metadata/ingestion/ometa/mixins/table_mixin.py +18 -0
- metadata/ingestion/ometa/mixins/tests_mixin.py +24 -3
- metadata/ingestion/ometa/mixins/user_mixin.py +117 -22
- metadata/ingestion/ometa/ometa_api.py +19 -26
- metadata/ingestion/ometa/routes.py +7 -0
- metadata/ingestion/processor/query_parser.py +9 -2
- metadata/ingestion/sink/metadata_rest.py +29 -8
- metadata/ingestion/source/dashboard/dashboard_service.py +38 -61
- metadata/ingestion/source/dashboard/domodashboard/metadata.py +13 -14
- metadata/ingestion/source/dashboard/lightdash/metadata.py +2 -1
- metadata/ingestion/source/dashboard/looker/metadata.py +41 -29
- metadata/ingestion/source/dashboard/looker/utils.py +4 -2
- metadata/ingestion/source/dashboard/metabase/client.py +4 -0
- metadata/ingestion/source/dashboard/metabase/metadata.py +28 -10
- metadata/ingestion/source/dashboard/metabase/models.py +2 -2
- metadata/ingestion/source/dashboard/mode/client.py +10 -23
- metadata/ingestion/source/dashboard/mode/connection.py +6 -1
- metadata/ingestion/source/dashboard/mode/metadata.py +8 -4
- metadata/ingestion/source/dashboard/mstr/client.py +208 -0
- metadata/ingestion/source/dashboard/mstr/connection.py +53 -0
- metadata/ingestion/source/dashboard/mstr/metadata.py +182 -0
- metadata/ingestion/source/dashboard/mstr/models.py +144 -0
- metadata/ingestion/source/dashboard/powerbi/client.py +4 -1
- metadata/ingestion/source/dashboard/powerbi/metadata.py +15 -10
- metadata/ingestion/source/dashboard/qliksense/metadata.py +11 -7
- metadata/ingestion/source/dashboard/quicksight/metadata.py +9 -5
- metadata/ingestion/source/dashboard/redash/metadata.py +23 -14
- metadata/ingestion/source/dashboard/superset/api_source.py +11 -8
- metadata/ingestion/source/dashboard/superset/client.py +16 -9
- metadata/ingestion/source/dashboard/superset/connection.py +3 -3
- metadata/ingestion/source/dashboard/superset/db_source.py +14 -11
- metadata/ingestion/source/dashboard/superset/mixin.py +22 -18
- metadata/ingestion/source/dashboard/superset/queries.py +1 -1
- metadata/ingestion/source/dashboard/tableau/client.py +91 -11
- metadata/ingestion/source/dashboard/tableau/connection.py +10 -1
- metadata/ingestion/source/dashboard/tableau/metadata.py +76 -70
- metadata/ingestion/source/dashboard/tableau/models.py +0 -8
- metadata/ingestion/source/dashboard/tableau/queries.py +5 -5
- metadata/ingestion/source/database/athena/client.py +80 -0
- metadata/ingestion/source/database/athena/connection.py +7 -0
- metadata/ingestion/source/database/athena/metadata.py +161 -19
- metadata/ingestion/source/database/azuresql/connection.py +21 -3
- metadata/ingestion/source/database/azuresql/metadata.py +0 -1
- metadata/ingestion/source/database/bigquery/connection.py +24 -3
- metadata/ingestion/source/database/bigquery/helper.py +68 -1
- metadata/ingestion/source/database/bigquery/metadata.py +49 -28
- metadata/ingestion/source/database/bigquery/queries.py +33 -4
- metadata/ingestion/source/database/bigquery/query_parser.py +13 -0
- metadata/ingestion/source/database/bigquery/usage.py +1 -3
- metadata/ingestion/source/database/bigtable/client.py +62 -0
- metadata/ingestion/source/database/bigtable/connection.py +116 -0
- metadata/ingestion/source/database/bigtable/metadata.py +224 -0
- metadata/ingestion/source/database/bigtable/models.py +60 -0
- metadata/ingestion/source/database/column_helpers.py +0 -10
- metadata/ingestion/source/database/column_type_parser.py +11 -5
- metadata/ingestion/source/database/common_db_source.py +33 -8
- metadata/ingestion/source/database/common_nosql_source.py +27 -6
- metadata/ingestion/source/database/database_service.py +89 -7
- metadata/ingestion/source/database/databricks/client.py +55 -103
- metadata/ingestion/source/database/databricks/connection.py +16 -55
- metadata/ingestion/source/database/databricks/lineage.py +29 -26
- metadata/ingestion/source/database/databricks/metadata.py +534 -11
- metadata/ingestion/source/database/databricks/queries.py +27 -0
- metadata/ingestion/source/database/databricks/query_parser.py +5 -1
- metadata/ingestion/source/database/databricks/usage.py +3 -3
- metadata/ingestion/source/database/datalake/connection.py +33 -18
- metadata/ingestion/source/database/datalake/metadata.py +113 -27
- metadata/ingestion/source/database/dbt/dbt_config.py +7 -14
- metadata/ingestion/source/database/dbt/dbt_service.py +10 -14
- metadata/ingestion/source/database/dbt/dbt_utils.py +3 -1
- metadata/ingestion/source/database/dbt/metadata.py +27 -53
- metadata/ingestion/source/database/deltalake/metadata.py +6 -3
- metadata/ingestion/source/database/domodatabase/metadata.py +7 -6
- metadata/ingestion/source/database/doris/connection.py +72 -0
- metadata/ingestion/source/database/doris/metadata.py +315 -0
- metadata/ingestion/source/database/doris/queries.py +54 -0
- metadata/ingestion/source/database/doris/utils.py +64 -0
- metadata/ingestion/source/database/extended_sample_data.py +532 -0
- metadata/ingestion/source/database/glue/metadata.py +8 -5
- metadata/ingestion/source/database/hive/connection.py +0 -2
- metadata/ingestion/source/database/hive/utils.py +3 -0
- metadata/ingestion/source/database/iceberg/catalog/__init__.py +65 -0
- metadata/ingestion/source/database/iceberg/catalog/base.py +40 -0
- metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +102 -0
- metadata/ingestion/source/database/iceberg/catalog/glue.py +88 -0
- metadata/ingestion/source/database/iceberg/catalog/hive.py +51 -0
- metadata/ingestion/source/database/iceberg/catalog/rest.py +84 -0
- metadata/ingestion/source/database/iceberg/connection.py +68 -0
- metadata/ingestion/source/database/iceberg/fs/__init__.py +52 -0
- metadata/ingestion/source/database/iceberg/fs/azure.py +44 -0
- metadata/ingestion/source/database/iceberg/fs/base.py +30 -0
- metadata/ingestion/source/database/iceberg/fs/s3.py +77 -0
- metadata/ingestion/source/database/iceberg/helper.py +124 -0
- metadata/ingestion/source/database/iceberg/metadata.py +311 -0
- metadata/ingestion/source/database/iceberg/models.py +66 -0
- metadata/ingestion/source/database/life_cycle_query_mixin.py +72 -3
- metadata/ingestion/source/database/mongodb/connection.py +1 -5
- metadata/ingestion/source/database/mssql/lineage.py +3 -0
- metadata/ingestion/source/database/mssql/metadata.py +108 -4
- metadata/ingestion/source/database/mssql/models.py +30 -0
- metadata/ingestion/source/database/mssql/queries.py +178 -1
- metadata/ingestion/source/database/mssql/usage.py +5 -1
- metadata/ingestion/source/database/mssql/utils.py +207 -4
- metadata/ingestion/source/database/mysql/connection.py +14 -0
- metadata/ingestion/source/database/mysql/metadata.py +0 -2
- metadata/ingestion/source/database/oracle/metadata.py +108 -2
- metadata/ingestion/source/database/oracle/models.py +30 -0
- metadata/ingestion/source/database/oracle/queries.py +99 -18
- metadata/ingestion/source/database/oracle/utils.py +0 -1
- metadata/ingestion/source/database/postgres/connection.py +15 -0
- metadata/ingestion/source/database/postgres/lineage.py +32 -14
- metadata/ingestion/source/database/postgres/metadata.py +15 -7
- metadata/ingestion/source/database/postgres/pgspider/lineage.py +0 -1
- metadata/ingestion/source/database/postgres/queries.py +4 -2
- metadata/ingestion/source/database/postgres/query_parser.py +4 -72
- metadata/ingestion/source/database/postgres/usage.py +41 -0
- metadata/ingestion/source/database/postgres/utils.py +34 -0
- metadata/ingestion/source/database/query_parser_source.py +8 -2
- metadata/ingestion/source/database/redshift/metadata.py +14 -4
- metadata/ingestion/source/database/redshift/queries.py +10 -4
- metadata/ingestion/source/database/redshift/query_parser.py +16 -0
- metadata/ingestion/source/database/redshift/usage.py +0 -2
- metadata/ingestion/source/database/salesforce/metadata.py +32 -3
- metadata/ingestion/source/database/sample_data.py +120 -6
- metadata/ingestion/source/database/sas/client.py +184 -0
- metadata/ingestion/source/database/sas/connection.py +47 -0
- metadata/ingestion/source/database/sas/extension_attr.py +103 -0
- metadata/ingestion/source/database/sas/metadata.py +914 -0
- metadata/ingestion/source/database/snowflake/metadata.py +34 -48
- metadata/ingestion/source/database/snowflake/models.py +6 -1
- metadata/ingestion/source/database/snowflake/queries.py +6 -4
- metadata/ingestion/source/database/snowflake/query_parser.py +5 -20
- metadata/ingestion/source/database/snowflake/utils.py +23 -8
- metadata/ingestion/source/database/stored_procedures_mixin.py +12 -8
- metadata/ingestion/source/database/unitycatalog/__init__.py +0 -0
- metadata/ingestion/source/database/unitycatalog/client.py +87 -0
- metadata/ingestion/source/database/unitycatalog/connection.py +97 -0
- metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/lineage.py +11 -11
- metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/metadata.py +42 -49
- metadata/ingestion/source/database/unitycatalog/query_parser.py +60 -0
- metadata/ingestion/source/database/unitycatalog/usage.py +31 -0
- metadata/ingestion/source/database/usage_source.py +3 -2
- metadata/ingestion/source/messaging/common_broker_source.py +15 -11
- metadata/ingestion/source/messaging/kafka/connection.py +45 -4
- metadata/ingestion/source/messaging/kinesis/metadata.py +6 -3
- metadata/ingestion/source/messaging/messaging_service.py +6 -2
- metadata/ingestion/source/metadata/amundsen/metadata.py +10 -7
- metadata/ingestion/source/metadata/atlas/metadata.py +5 -5
- metadata/ingestion/source/mlmodel/mlflow/metadata.py +5 -2
- metadata/ingestion/source/mlmodel/mlmodel_service.py +6 -2
- metadata/ingestion/source/mlmodel/sagemaker/metadata.py +20 -8
- metadata/ingestion/source/pipeline/airflow/connection.py +0 -12
- metadata/ingestion/source/pipeline/airflow/lineage_parser.py +12 -6
- metadata/ingestion/source/pipeline/airflow/metadata.py +63 -34
- metadata/ingestion/source/pipeline/airflow/models.py +5 -4
- metadata/ingestion/source/pipeline/dagster/metadata.py +7 -4
- metadata/ingestion/source/pipeline/databrickspipeline/connection.py +1 -5
- metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +14 -11
- metadata/ingestion/source/pipeline/domopipeline/metadata.py +7 -4
- metadata/ingestion/source/pipeline/gluepipeline/metadata.py +5 -2
- metadata/ingestion/source/pipeline/pipeline_service.py +6 -2
- metadata/ingestion/source/pipeline/spline/metadata.py +0 -1
- metadata/ingestion/source/search/elasticsearch/connection.py +4 -1
- metadata/ingestion/source/search/elasticsearch/metadata.py +1 -2
- metadata/ingestion/source/search/search_service.py +6 -2
- metadata/ingestion/source/storage/s3/metadata.py +22 -17
- metadata/ingestion/source/storage/storage_service.py +53 -11
- metadata/ingestion/stage/table_usage.py +9 -2
- metadata/mixins/sqalchemy/sqa_mixin.py +14 -7
- metadata/parsers/protobuf_parser.py +29 -11
- metadata/pii/processor.py +9 -2
- metadata/pii/scanners/ner_scanner.py +5 -3
- metadata/profiler/api/models.py +19 -1
- metadata/profiler/interface/pandas/profiler_interface.py +59 -18
- metadata/profiler/interface/profiler_interface.py +14 -4
- metadata/profiler/interface/profiler_interface_factory.py +49 -14
- metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +3 -0
- metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +26 -0
- metadata/profiler/interface/sqlalchemy/db2/__init__.py +0 -0
- metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +38 -0
- metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +85 -0
- metadata/profiler/interface/sqlalchemy/profiler_interface.py +77 -34
- metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
- metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +7 -0
- metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
- metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +33 -0
- metadata/profiler/metrics/composed/null_ratio.py +1 -1
- metadata/profiler/metrics/hybrid/histogram.py +1 -0
- metadata/profiler/metrics/static/max.py +4 -1
- metadata/profiler/metrics/static/min.py +4 -1
- metadata/profiler/metrics/system/queries/snowflake.py +89 -17
- metadata/profiler/metrics/system/system.py +62 -20
- metadata/profiler/orm/functions/conn_test.py +1 -0
- metadata/profiler/orm/functions/length.py +1 -0
- metadata/profiler/orm/functions/median.py +9 -1
- metadata/profiler/orm/functions/sum.py +1 -0
- metadata/profiler/orm/functions/table_metric_computer.py +462 -0
- metadata/profiler/orm/registry.py +2 -0
- metadata/profiler/processor/core.py +129 -62
- metadata/profiler/processor/default.py +14 -3
- metadata/profiler/processor/handle_partition.py +2 -2
- metadata/profiler/processor/processor.py +10 -7
- metadata/profiler/processor/sample_data_handler.py +6 -2
- metadata/profiler/processor/sampler/sqlalchemy/bigquery/sampler.py +31 -3
- metadata/profiler/processor/sampler/sqlalchemy/sampler.py +29 -6
- metadata/profiler/processor/sampler/sqlalchemy/trino/sampler.py +10 -4
- metadata/profiler/source/base/profiler_source.py +5 -2
- metadata/profiler/source/bigquery/type_mapper.py +0 -1
- metadata/profiler/source/databricks/profiler_source.py +36 -0
- metadata/profiler/source/mariadb/functions/median.py +20 -0
- metadata/profiler/source/mariadb/metrics/window/first_quartile.py +10 -0
- metadata/profiler/source/mariadb/metrics/window/median.py +10 -0
- metadata/profiler/source/mariadb/metrics/window/third_quartile.py +10 -0
- metadata/profiler/source/metadata.py +49 -10
- metadata/profiler/source/metadata_ext.py +16 -50
- metadata/profiler/source/profiler_source_factory.py +8 -0
- metadata/py.typed +0 -0
- metadata/readers/dataframe/json.py +5 -4
- metadata/readers/file/api_reader.py +0 -1
- metadata/utils/constants.py +5 -0
- metadata/utils/credentials.py +36 -19
- metadata/utils/datalake/datalake_utils.py +369 -129
- metadata/utils/entity_link.py +26 -6
- metadata/utils/execution_time_tracker.py +199 -0
- metadata/utils/filters.py +4 -0
- metadata/utils/fqn.py +20 -0
- metadata/utils/helpers.py +34 -39
- metadata/utils/importer.py +2 -3
- metadata/utils/life_cycle_utils.py +4 -4
- metadata/utils/logger.py +13 -2
- metadata/utils/partition.py +10 -5
- metadata/utils/secrets/aws_based_secrets_manager.py +67 -4
- metadata/utils/secrets/aws_secrets_manager.py +7 -2
- metadata/utils/secrets/aws_ssm_secrets_manager.py +7 -2
- metadata/utils/secrets/azure_kv_secrets_manager.py +132 -0
- metadata/utils/secrets/{noop_secrets_manager.py → db_secrets_manager.py} +4 -3
- metadata/utils/secrets/external_secrets_manager.py +25 -3
- metadata/utils/secrets/secrets_manager_factory.py +15 -33
- metadata/utils/{source_hash_utils.py → source_hash.py} +10 -1
- metadata/utils/sqlalchemy_utils.py +21 -0
- metadata/utils/storage_metadata_config.py +42 -14
- metadata/utils/tag_utils.py +5 -2
- metadata/workflow/application.py +154 -0
- metadata/workflow/application_output_handler.py +34 -0
- metadata/workflow/base.py +88 -153
- metadata/workflow/data_insight.py +8 -7
- metadata/workflow/data_quality.py +3 -2
- metadata/workflow/ingestion.py +203 -0
- metadata/workflow/metadata.py +2 -3
- metadata/workflow/output_handler.py +226 -0
- metadata/workflow/profiler.py +2 -2
- metadata/workflow/usage.py +3 -4
- metadata/workflow/workflow_output_handler.py +15 -255
- metadata/workflow/workflow_status_mixin.py +44 -52
- openmetadata_ingestion-1.3.2.0rc1.dist-info/METADATA +758 -0
- {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/RECORD +870 -762
- {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/WHEEL +1 -1
- metadata/generated/schema/entity/applications/appConfig.py +0 -21
- metadata/generated/schema/entity/applications/configuration/dataInsightsApp.py +0 -17
- metadata/generated/schema/entity/applications/configuration/externalAppIngestionConfig.py +0 -38
- metadata/generated/schema/entity/services/connections/database/mongoDB/mongoDBValues.py +0 -44
- metadata/generated/schema/events/dataInsightAlertConfig.py +0 -17
- metadata/generated/schema/events/entitySpelFilters.py +0 -19
- metadata/ingestion/models/es_documents.py +0 -339
- metadata/ingestion/ometa/mixins/glossary_mixin.py +0 -501
- metadata/ingestion/ometa/provider_registry.py +0 -144
- metadata/ingestion/source/database/databricks/legacy/lineage.py +0 -51
- metadata/ingestion/source/database/databricks/legacy/metadata.py +0 -339
- metadata/ingestion/source/metadata/metadata_elasticsearch/metadata.py +0 -144
- metadata/profiler/orm/functions/table_metric_construct.py +0 -365
- metadata/utils/secrets/client/loader.py +0 -78
- openmetadata_ingestion-1.2.4.0.dist-info/METADATA +0 -429
- /metadata/examples/workflows/{datalake_azure.yaml → datalake_azure_client_secret.yaml} +0 -0
- /metadata/ingestion/source/{database/databricks/legacy → dashboard/mstr}/__init__.py +0 -0
- /metadata/ingestion/source/database/{databricks/unity_catalog → bigtable}/__init__.py +0 -0
- /metadata/ingestion/source/{metadata/metadata_elasticsearch → database/doris}/__init__.py +0 -0
- /metadata/{utils/secrets/client → ingestion/source/database/sas}/__init__.py +0 -0
- /metadata/ingestion/source/database/{databricks → unitycatalog}/models.py +0 -0
- {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/LICENSE +0 -0
- {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/entry_points.txt +0 -0
- {openmetadata_ingestion-1.2.4.0.dist-info → openmetadata_ingestion-1.3.2.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Copyright 2021 Collate
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
7
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
8
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
9
|
+
# See the License for the specific language governing permissions and
|
|
10
|
+
# limitations under the License.
|
|
11
|
+
"""
|
|
12
|
+
Mixin class containing Suggestions specific methods
|
|
13
|
+
|
|
14
|
+
To be used by OpenMetadata class
|
|
15
|
+
"""
|
|
16
|
+
from metadata.generated.schema.entity.feed.suggestion import Suggestion
|
|
17
|
+
from metadata.ingestion.ometa.client import REST
|
|
18
|
+
from metadata.utils.logger import ometa_logger
|
|
19
|
+
|
|
20
|
+
logger = ometa_logger()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class OMetaSuggestionsMixin:
|
|
24
|
+
"""
|
|
25
|
+
OpenMetadata API methods related to the Suggestion Entity
|
|
26
|
+
|
|
27
|
+
To be inherited by OpenMetadata
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
client: REST
|
|
31
|
+
|
|
32
|
+
def update_suggestion(self, suggestion: Suggestion) -> Suggestion:
|
|
33
|
+
"""
|
|
34
|
+
Update an existing Suggestion with new fields
|
|
35
|
+
"""
|
|
36
|
+
resp = self.client.put(
|
|
37
|
+
f"{self.get_suffix(Suggestion)}/{str(suggestion.id.__root__)}",
|
|
38
|
+
data=suggestion.json(),
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
return Suggestion(**resp)
|
|
@@ -22,6 +22,9 @@ from requests.utils import quote
|
|
|
22
22
|
from metadata.generated.schema.api.data.createTableProfile import (
|
|
23
23
|
CreateTableProfileRequest,
|
|
24
24
|
)
|
|
25
|
+
from metadata.generated.schema.api.tests.createCustomMetric import (
|
|
26
|
+
CreateCustomMetricRequest,
|
|
27
|
+
)
|
|
25
28
|
from metadata.generated.schema.entity.data.table import (
|
|
26
29
|
ColumnProfile,
|
|
27
30
|
DataModel,
|
|
@@ -288,3 +291,18 @@ class OMetaTableMixin:
|
|
|
288
291
|
Optional[Table]: OM table object
|
|
289
292
|
"""
|
|
290
293
|
return self._get(Table, f"{quote(model_str(fqn))}/tableProfile/latest")
|
|
294
|
+
|
|
295
|
+
def create_or_update_custom_metric(
|
|
296
|
+
self, custom_metric: CreateCustomMetricRequest, table_id: str
|
|
297
|
+
) -> Table:
|
|
298
|
+
"""Create or update custom metric. If custom metric name matches an existing
|
|
299
|
+
one then it will be updated.
|
|
300
|
+
|
|
301
|
+
Args:
|
|
302
|
+
custom_metric (CreateCustomMetricRequest): custom metric to be create or updated
|
|
303
|
+
"""
|
|
304
|
+
resp = self.client.put(
|
|
305
|
+
f"{self.get_suffix(Table)}/{table_id}/customMetric",
|
|
306
|
+
data=custom_metric.json(),
|
|
307
|
+
)
|
|
308
|
+
return Table(**resp)
|
|
@@ -23,6 +23,9 @@ from metadata.generated.schema.api.tests.createLogicalTestCases import (
|
|
|
23
23
|
CreateLogicalTestCases,
|
|
24
24
|
)
|
|
25
25
|
from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest
|
|
26
|
+
from metadata.generated.schema.api.tests.createTestCaseResolutionStatus import (
|
|
27
|
+
CreateTestCaseResolutionStatus,
|
|
28
|
+
)
|
|
26
29
|
from metadata.generated.schema.api.tests.createTestDefinition import (
|
|
27
30
|
CreateTestDefinitionRequest,
|
|
28
31
|
)
|
|
@@ -30,6 +33,9 @@ from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteR
|
|
|
30
33
|
from metadata.generated.schema.entity.data.table import Table
|
|
31
34
|
from metadata.generated.schema.tests.basic import TestCaseResult
|
|
32
35
|
from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
|
|
36
|
+
from metadata.generated.schema.tests.testCaseResolutionStatus import (
|
|
37
|
+
TestCaseResolutionStatus,
|
|
38
|
+
)
|
|
33
39
|
from metadata.generated.schema.tests.testDefinition import (
|
|
34
40
|
EntityType,
|
|
35
41
|
TestCaseParameterDefinition,
|
|
@@ -242,10 +248,9 @@ class OMetaTestsMixin:
|
|
|
242
248
|
end_ts (int): timestamp
|
|
243
249
|
"""
|
|
244
250
|
|
|
245
|
-
# timestamp should be changed to milliseconds in https://github.com/open-metadata/OpenMetadata/issues/8930
|
|
246
251
|
params = {
|
|
247
|
-
"startTs": start_ts
|
|
248
|
-
"endTs": end_ts
|
|
252
|
+
"startTs": start_ts,
|
|
253
|
+
"endTs": end_ts,
|
|
249
254
|
}
|
|
250
255
|
|
|
251
256
|
resp = self.client.get(
|
|
@@ -302,3 +307,19 @@ class OMetaTestsMixin:
|
|
|
302
307
|
"""
|
|
303
308
|
path = self.get_suffix(TestCase) + "/logicalTestCases"
|
|
304
309
|
self.client.put(path, data=data.json(encoder=show_secrets_encoder))
|
|
310
|
+
|
|
311
|
+
def create_test_case_resolution(
|
|
312
|
+
self, data: CreateTestCaseResolutionStatus
|
|
313
|
+
) -> TestCaseResolutionStatus:
|
|
314
|
+
"""Create a test case resolution
|
|
315
|
+
|
|
316
|
+
Args:
|
|
317
|
+
data (CreateTestCaseResolutionStatus): test case resolution
|
|
318
|
+
|
|
319
|
+
Returns:
|
|
320
|
+
TestCaseResolutionStatus
|
|
321
|
+
"""
|
|
322
|
+
path = self.get_suffix(TestCase) + "/testCaseIncidentStatus"
|
|
323
|
+
response = self.client.post(path, data=data.json(encoder=show_secrets_encoder))
|
|
324
|
+
|
|
325
|
+
return TestCaseResolutionStatus(**response)
|
|
@@ -13,12 +13,15 @@ Mixin class containing User specific methods
|
|
|
13
13
|
|
|
14
14
|
To be used by OpenMetadata class
|
|
15
15
|
"""
|
|
16
|
-
import traceback
|
|
17
16
|
from functools import lru_cache
|
|
18
|
-
from typing import Optional
|
|
17
|
+
from typing import Optional, Type
|
|
19
18
|
|
|
19
|
+
from metadata.generated.schema.entity.teams.team import Team
|
|
20
20
|
from metadata.generated.schema.entity.teams.user import User
|
|
21
|
+
from metadata.generated.schema.type.entityReference import EntityReference
|
|
22
|
+
from metadata.ingestion.api.common import T
|
|
21
23
|
from metadata.ingestion.ometa.client import REST
|
|
24
|
+
from metadata.utils.constants import ENTITY_REFERENCE_TYPE_MAP
|
|
22
25
|
from metadata.utils.elasticsearch import ES_INDEX_MAP
|
|
23
26
|
from metadata.utils.logger import ometa_logger
|
|
24
27
|
|
|
@@ -34,42 +37,134 @@ class OMetaUserMixin:
|
|
|
34
37
|
|
|
35
38
|
client: REST
|
|
36
39
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
40
|
+
@staticmethod
|
|
41
|
+
def email_search_query_es(entity: Type[T]) -> str:
|
|
42
|
+
return (
|
|
43
|
+
"/search/query?q=email.keyword:{email}&from={from_}&size={size}&index="
|
|
44
|
+
+ ES_INDEX_MAP[entity.__name__]
|
|
45
|
+
)
|
|
41
46
|
|
|
42
|
-
@
|
|
43
|
-
def
|
|
47
|
+
@staticmethod
|
|
48
|
+
def name_search_query_es(entity: Type[T]) -> str:
|
|
49
|
+
"""
|
|
50
|
+
Allow for more flexible lookup following what the UI is doing when searching users.
|
|
51
|
+
|
|
52
|
+
We don't want to stick to `q=name:{name}` since in case a user is named `random.user`
|
|
53
|
+
but looked as `Random User`, we want to find this match.
|
|
54
|
+
"""
|
|
55
|
+
return (
|
|
56
|
+
"/search/query?q={name}&from={from_}&size={size}&index="
|
|
57
|
+
+ ES_INDEX_MAP[entity.__name__]
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
def _search_by_email(
|
|
44
61
|
self,
|
|
62
|
+
entity: Type[T],
|
|
45
63
|
email: Optional[str],
|
|
46
64
|
from_count: int = 0,
|
|
47
65
|
size: int = 1,
|
|
48
66
|
fields: Optional[list] = None,
|
|
49
|
-
) -> Optional[
|
|
67
|
+
) -> Optional[T]:
|
|
50
68
|
"""
|
|
51
|
-
GET user entity by
|
|
69
|
+
GET user or team entity by mail
|
|
52
70
|
|
|
53
71
|
Args:
|
|
54
72
|
email: user email to search
|
|
55
73
|
from_count: records to expect
|
|
56
74
|
size: number of records
|
|
75
|
+
fields: Optional field list to pass to ES request
|
|
57
76
|
"""
|
|
58
77
|
if email:
|
|
59
|
-
query_string = self.
|
|
78
|
+
query_string = self.email_search_query_es(entity=entity).format(
|
|
60
79
|
email=email, from_=from_count, size=size
|
|
61
80
|
)
|
|
81
|
+
return self._get_entity_from_es(
|
|
82
|
+
entity=entity, query_string=query_string, fields=fields
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
def _search_by_name(
|
|
88
|
+
self,
|
|
89
|
+
entity: Type[T],
|
|
90
|
+
name: Optional[str],
|
|
91
|
+
from_count: int = 0,
|
|
92
|
+
size: int = 1,
|
|
93
|
+
fields: Optional[list] = None,
|
|
94
|
+
) -> Optional[T]:
|
|
95
|
+
"""
|
|
96
|
+
GET entity by name
|
|
97
|
+
|
|
98
|
+
Args:
|
|
99
|
+
name: user name to search
|
|
100
|
+
from_count: records to expect
|
|
101
|
+
size: number of records
|
|
102
|
+
fields: Optional field list to pass to ES request
|
|
103
|
+
"""
|
|
104
|
+
if name:
|
|
105
|
+
query_string = self.name_search_query_es(entity=entity).format(
|
|
106
|
+
name=name, from_=from_count, size=size
|
|
107
|
+
)
|
|
108
|
+
return self._get_entity_from_es(
|
|
109
|
+
entity=entity, query_string=query_string, fields=fields
|
|
110
|
+
)
|
|
62
111
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
@lru_cache(maxsize=None)
|
|
115
|
+
def get_reference_by_email(
|
|
116
|
+
self,
|
|
117
|
+
email: Optional[str],
|
|
118
|
+
from_count: int = 0,
|
|
119
|
+
size: int = 1,
|
|
120
|
+
fields: Optional[list] = None,
|
|
121
|
+
) -> Optional[EntityReference]:
|
|
122
|
+
"""
|
|
123
|
+
Get a User or Team Entity Reference by searching by its mail
|
|
124
|
+
"""
|
|
125
|
+
maybe_user = self._search_by_email(
|
|
126
|
+
entity=User, email=email, from_count=from_count, size=size, fields=fields
|
|
127
|
+
)
|
|
128
|
+
if maybe_user:
|
|
129
|
+
return EntityReference(
|
|
130
|
+
id=maybe_user.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[User.__name__]
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
maybe_team = self._search_by_email(
|
|
134
|
+
entity=Team, email=email, from_count=from_count, size=size, fields=fields
|
|
135
|
+
)
|
|
136
|
+
if maybe_team:
|
|
137
|
+
return EntityReference(
|
|
138
|
+
id=maybe_team.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[Team.__name__]
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
return None
|
|
142
|
+
|
|
143
|
+
@lru_cache(maxsize=None)
|
|
144
|
+
def get_reference_by_name(
|
|
145
|
+
self,
|
|
146
|
+
name: Optional[str],
|
|
147
|
+
from_count: int = 0,
|
|
148
|
+
size: int = 1,
|
|
149
|
+
fields: Optional[list] = None,
|
|
150
|
+
) -> Optional[EntityReference]:
|
|
151
|
+
"""
|
|
152
|
+
Get a User or Team Entity Reference by searching by its name
|
|
153
|
+
"""
|
|
154
|
+
maybe_user = self._search_by_name(
|
|
155
|
+
entity=User, name=name, from_count=from_count, size=size, fields=fields
|
|
156
|
+
)
|
|
157
|
+
if maybe_user:
|
|
158
|
+
return EntityReference(
|
|
159
|
+
id=maybe_user.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[User.__name__]
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
maybe_team = self._search_by_name(
|
|
163
|
+
entity=Team, name=name, from_count=from_count, size=size, fields=fields
|
|
164
|
+
)
|
|
165
|
+
if maybe_team:
|
|
166
|
+
return EntityReference(
|
|
167
|
+
id=maybe_team.id.__root__, type=ENTITY_REFERENCE_TYPE_MAP[Team.__name__]
|
|
168
|
+
)
|
|
74
169
|
|
|
75
170
|
return None
|
|
@@ -31,7 +31,7 @@ from metadata.generated.schema.type.basic import FullyQualifiedEntityName
|
|
|
31
31
|
from metadata.generated.schema.type.entityHistory import EntityVersionHistory
|
|
32
32
|
from metadata.generated.schema.type.entityReference import EntityReference
|
|
33
33
|
from metadata.ingestion.models.encoders import show_secrets_encoder
|
|
34
|
-
from metadata.ingestion.ometa.auth_provider import
|
|
34
|
+
from metadata.ingestion.ometa.auth_provider import OpenMetadataAuthenticationProvider
|
|
35
35
|
from metadata.ingestion.ometa.client import REST, APIError, ClientConfig
|
|
36
36
|
from metadata.ingestion.ometa.mixins.custom_property_mixin import (
|
|
37
37
|
OMetaCustomPropertyMixin,
|
|
@@ -39,7 +39,6 @@ from metadata.ingestion.ometa.mixins.custom_property_mixin import (
|
|
|
39
39
|
from metadata.ingestion.ometa.mixins.dashboard_mixin import OMetaDashboardMixin
|
|
40
40
|
from metadata.ingestion.ometa.mixins.data_insight_mixin import DataInsightMixin
|
|
41
41
|
from metadata.ingestion.ometa.mixins.es_mixin import ESMixin
|
|
42
|
-
from metadata.ingestion.ometa.mixins.glossary_mixin import GlossaryMixin
|
|
43
42
|
from metadata.ingestion.ometa.mixins.ingestion_pipeline_mixin import (
|
|
44
43
|
OMetaIngestionPipelineMixin,
|
|
45
44
|
)
|
|
@@ -51,16 +50,13 @@ from metadata.ingestion.ometa.mixins.role_policy_mixin import OMetaRolePolicyMix
|
|
|
51
50
|
from metadata.ingestion.ometa.mixins.search_index_mixin import OMetaSearchIndexMixin
|
|
52
51
|
from metadata.ingestion.ometa.mixins.server_mixin import OMetaServerMixin
|
|
53
52
|
from metadata.ingestion.ometa.mixins.service_mixin import OMetaServiceMixin
|
|
53
|
+
from metadata.ingestion.ometa.mixins.suggestions_mixin import OMetaSuggestionsMixin
|
|
54
54
|
from metadata.ingestion.ometa.mixins.table_mixin import OMetaTableMixin
|
|
55
55
|
from metadata.ingestion.ometa.mixins.tests_mixin import OMetaTestsMixin
|
|
56
56
|
from metadata.ingestion.ometa.mixins.topic_mixin import OMetaTopicMixin
|
|
57
57
|
from metadata.ingestion.ometa.mixins.user_mixin import OMetaUserMixin
|
|
58
58
|
from metadata.ingestion.ometa.mixins.version_mixin import OMetaVersionMixin
|
|
59
59
|
from metadata.ingestion.ometa.models import EntityList
|
|
60
|
-
from metadata.ingestion.ometa.provider_registry import (
|
|
61
|
-
InvalidAuthProviderException,
|
|
62
|
-
auth_provider_registry,
|
|
63
|
-
)
|
|
64
60
|
from metadata.ingestion.ometa.routes import ROUTES
|
|
65
61
|
from metadata.ingestion.ometa.utils import get_entity_type, model_str
|
|
66
62
|
from metadata.utils.logger import ometa_logger
|
|
@@ -100,7 +96,6 @@ class OpenMetadata(
|
|
|
100
96
|
OMetaTableMixin,
|
|
101
97
|
OMetaTopicMixin,
|
|
102
98
|
OMetaVersionMixin,
|
|
103
|
-
GlossaryMixin,
|
|
104
99
|
OMetaServiceMixin,
|
|
105
100
|
ESMixin,
|
|
106
101
|
OMetaServerMixin,
|
|
@@ -114,6 +109,7 @@ class OpenMetadata(
|
|
|
114
109
|
OMetaRolePolicyMixin,
|
|
115
110
|
OMetaSearchIndexMixin,
|
|
116
111
|
OMetaCustomPropertyMixin,
|
|
112
|
+
OMetaSuggestionsMixin,
|
|
117
113
|
Generic[T, C],
|
|
118
114
|
):
|
|
119
115
|
"""
|
|
@@ -125,7 +121,7 @@ class OpenMetadata(
|
|
|
125
121
|
"""
|
|
126
122
|
|
|
127
123
|
client: REST
|
|
128
|
-
_auth_provider:
|
|
124
|
+
_auth_provider: OpenMetadataAuthenticationProvider
|
|
129
125
|
config: OpenMetadataConnection
|
|
130
126
|
|
|
131
127
|
class_root = ".".join(["metadata", "generated", "schema"])
|
|
@@ -146,16 +142,7 @@ class OpenMetadata(
|
|
|
146
142
|
config.secretsManagerLoader,
|
|
147
143
|
).get_secrets_manager()
|
|
148
144
|
|
|
149
|
-
|
|
150
|
-
auth_provider_fn = auth_provider_registry.registry.get(
|
|
151
|
-
self.config.authProvider.value
|
|
152
|
-
)
|
|
153
|
-
if not auth_provider_fn:
|
|
154
|
-
raise InvalidAuthProviderException(
|
|
155
|
-
f"Cannot find {self.config.authProvider.value} in {auth_provider_registry.registry}"
|
|
156
|
-
)
|
|
157
|
-
|
|
158
|
-
self._auth_provider = auth_provider_fn(self.config)
|
|
145
|
+
self._auth_provider = OpenMetadataAuthenticationProvider.create(self.config)
|
|
159
146
|
|
|
160
147
|
get_verify_ssl = get_verify_ssl_fn(self.config.verifySSL)
|
|
161
148
|
|
|
@@ -261,11 +248,9 @@ class OpenMetadata(
|
|
|
261
248
|
)
|
|
262
249
|
return entity_class
|
|
263
250
|
|
|
264
|
-
def
|
|
251
|
+
def _create(self, data: C, method: str) -> T:
|
|
265
252
|
"""
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
We PUT to the endpoint and return the Entity generated result
|
|
253
|
+
Internal logic to run POST vs. PUT
|
|
269
254
|
"""
|
|
270
255
|
entity = data.__class__
|
|
271
256
|
is_create = "create" in data.__class__.__name__.lower()
|
|
@@ -277,15 +262,23 @@ class OpenMetadata(
|
|
|
277
262
|
raise InvalidEntityException(
|
|
278
263
|
f"PUT operations need a CreateEntity, not {entity}"
|
|
279
264
|
)
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
)
|
|
265
|
+
|
|
266
|
+
fn = getattr(self.client, method)
|
|
267
|
+
resp = fn(self.get_suffix(entity), data=data.json(encoder=show_secrets_encoder))
|
|
283
268
|
if not resp:
|
|
284
269
|
raise EmptyPayloadException(
|
|
285
270
|
f"Got an empty response when trying to PUT to {self.get_suffix(entity)}, {data.json()}"
|
|
286
271
|
)
|
|
287
272
|
return entity_class(**resp)
|
|
288
273
|
|
|
274
|
+
def create_or_update(self, data: C) -> T:
|
|
275
|
+
"""Run a PUT requesting via create request C"""
|
|
276
|
+
return self._create(data=data, method="put")
|
|
277
|
+
|
|
278
|
+
def create(self, data: C) -> T:
|
|
279
|
+
"""Run a POST requesting via create request C"""
|
|
280
|
+
return self._create(data=data, method="post")
|
|
281
|
+
|
|
289
282
|
def get_by_name(
|
|
290
283
|
self,
|
|
291
284
|
entity: Type[T],
|
|
@@ -425,7 +418,7 @@ class OpenMetadata(
|
|
|
425
418
|
self,
|
|
426
419
|
entity: Type[T],
|
|
427
420
|
fields: Optional[List[str]] = None,
|
|
428
|
-
limit: int =
|
|
421
|
+
limit: int = 100,
|
|
429
422
|
params: Optional[Dict[str, str]] = None,
|
|
430
423
|
skip_on_failure: bool = False,
|
|
431
424
|
) -> Iterable[T]:
|
|
@@ -50,6 +50,7 @@ from metadata.generated.schema.api.domains.createDataProduct import (
|
|
|
50
50
|
CreateDataProductRequest,
|
|
51
51
|
)
|
|
52
52
|
from metadata.generated.schema.api.domains.createDomain import CreateDomainRequest
|
|
53
|
+
from metadata.generated.schema.api.feed.createSuggestion import CreateSuggestionRequest
|
|
53
54
|
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
|
|
54
55
|
from metadata.generated.schema.api.policies.createPolicy import CreatePolicyRequest
|
|
55
56
|
from metadata.generated.schema.api.services.createDashboardService import (
|
|
@@ -90,6 +91,7 @@ from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteR
|
|
|
90
91
|
from metadata.generated.schema.dataInsight.dataInsightChart import DataInsightChart
|
|
91
92
|
from metadata.generated.schema.dataInsight.kpi.kpi import Kpi
|
|
92
93
|
from metadata.generated.schema.entity.automations.workflow import Workflow
|
|
94
|
+
from metadata.generated.schema.entity.bot import Bot
|
|
93
95
|
from metadata.generated.schema.entity.classification.classification import (
|
|
94
96
|
Classification,
|
|
95
97
|
)
|
|
@@ -113,6 +115,7 @@ from metadata.generated.schema.entity.data.table import Table
|
|
|
113
115
|
from metadata.generated.schema.entity.data.topic import Topic
|
|
114
116
|
from metadata.generated.schema.entity.domains.dataProduct import DataProduct
|
|
115
117
|
from metadata.generated.schema.entity.domains.domain import Domain
|
|
118
|
+
from metadata.generated.schema.entity.feed.suggestion import Suggestion
|
|
116
119
|
from metadata.generated.schema.entity.policies.policy import Policy
|
|
117
120
|
from metadata.generated.schema.entity.services.connections.testConnectionDefinition import (
|
|
118
121
|
TestConnectionDefinition,
|
|
@@ -181,6 +184,7 @@ ROUTES = {
|
|
|
181
184
|
User.__name__: "/users",
|
|
182
185
|
CreateUserRequest.__name__: "/users",
|
|
183
186
|
AuthenticationMechanism.__name__: "/users/auth-mechanism",
|
|
187
|
+
Bot.__name__: "/bots", # We won't allow bot creation from the client
|
|
184
188
|
# Roles
|
|
185
189
|
Role.__name__: "/roles",
|
|
186
190
|
CreateRoleRequest.__name__: "/roles",
|
|
@@ -225,4 +229,7 @@ ROUTES = {
|
|
|
225
229
|
CreateDomainRequest.__name__: "/domains",
|
|
226
230
|
DataProduct.__name__: "/dataProducts",
|
|
227
231
|
CreateDataProductRequest.__name__: "/dataProducts",
|
|
232
|
+
# Suggestions
|
|
233
|
+
Suggestion.__name__: "/suggestions",
|
|
234
|
+
CreateSuggestionRequest.__name__: "/suggestions",
|
|
228
235
|
}
|
|
@@ -17,10 +17,13 @@ import traceback
|
|
|
17
17
|
from typing import Optional
|
|
18
18
|
|
|
19
19
|
from metadata.config.common import ConfigModel
|
|
20
|
+
from metadata.generated.schema.entity.services.ingestionPipelines.status import (
|
|
21
|
+
StackTraceError,
|
|
22
|
+
)
|
|
20
23
|
from metadata.generated.schema.type.basic import DateTime
|
|
21
24
|
from metadata.generated.schema.type.queryParserData import ParsedData, QueryParserData
|
|
22
25
|
from metadata.generated.schema.type.tableQuery import TableQueries, TableQuery
|
|
23
|
-
from metadata.ingestion.api.models import Either
|
|
26
|
+
from metadata.ingestion.api.models import Either
|
|
24
27
|
from metadata.ingestion.api.steps import Processor
|
|
25
28
|
from metadata.ingestion.lineage.models import ConnectionTypeDialectMapper, Dialect
|
|
26
29
|
from metadata.ingestion.lineage.parser import LineageParser
|
|
@@ -84,6 +87,10 @@ class QueryParserProcessor(Processor):
|
|
|
84
87
|
self.metadata = metadata
|
|
85
88
|
self.connection_type = connection_type
|
|
86
89
|
|
|
90
|
+
@property
|
|
91
|
+
def name(self) -> str:
|
|
92
|
+
return "Query Parser"
|
|
93
|
+
|
|
87
94
|
@classmethod
|
|
88
95
|
def create(cls, config_dict: dict, metadata: OpenMetadata, **kwargs):
|
|
89
96
|
config = ConfigModel.parse_obj(config_dict)
|
|
@@ -106,7 +113,7 @@ class QueryParserProcessor(Processor):
|
|
|
106
113
|
left=StackTraceError(
|
|
107
114
|
name="Query",
|
|
108
115
|
error=f"Error processing query [{table_query.query}]: {exc}",
|
|
109
|
-
|
|
116
|
+
stackTrace=traceback.format_exc(),
|
|
110
117
|
)
|
|
111
118
|
)
|
|
112
119
|
|
|
@@ -47,6 +47,9 @@ from metadata.generated.schema.entity.teams.team import Team
|
|
|
47
47
|
from metadata.generated.schema.entity.teams.user import User
|
|
48
48
|
from metadata.generated.schema.tests.basic import TestCaseResult
|
|
49
49
|
from metadata.generated.schema.tests.testCase import TestCase
|
|
50
|
+
from metadata.generated.schema.tests.testCaseResolutionStatus import (
|
|
51
|
+
TestCaseResolutionStatus,
|
|
52
|
+
)
|
|
50
53
|
from metadata.generated.schema.tests.testSuite import TestSuite
|
|
51
54
|
from metadata.generated.schema.type.schema import Topic
|
|
52
55
|
from metadata.ingestion.api.models import Either, Entity, StackTraceError
|
|
@@ -59,7 +62,9 @@ from metadata.ingestion.models.ometa_classification import OMetaTagAndClassifica
|
|
|
59
62
|
from metadata.ingestion.models.ometa_topic_data import OMetaTopicSampleData
|
|
60
63
|
from metadata.ingestion.models.patch_request import (
|
|
61
64
|
ALLOWED_COMMON_PATCH_FIELDS,
|
|
65
|
+
ARRAY_ENTITY_FIELDS,
|
|
62
66
|
RESTRICT_UPDATE_LIST,
|
|
67
|
+
PatchedEntity,
|
|
63
68
|
PatchRequest,
|
|
64
69
|
)
|
|
65
70
|
from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
|
|
@@ -67,6 +72,7 @@ from metadata.ingestion.models.profile_data import OMetaTableProfileSampleData
|
|
|
67
72
|
from metadata.ingestion.models.search_index_data import OMetaIndexSampleData
|
|
68
73
|
from metadata.ingestion.models.tests_data import (
|
|
69
74
|
OMetaLogicalTestSuiteSample,
|
|
75
|
+
OMetaTestCaseResolutionStatus,
|
|
70
76
|
OMetaTestCaseResultsSample,
|
|
71
77
|
OMetaTestCaseSample,
|
|
72
78
|
OMetaTestSuiteSample,
|
|
@@ -77,7 +83,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
|
77
83
|
from metadata.ingestion.source.dashboard.dashboard_service import DashboardUsage
|
|
78
84
|
from metadata.ingestion.source.database.database_service import DataModelLink
|
|
79
85
|
from metadata.profiler.api.models import ProfilerResponse
|
|
80
|
-
from metadata.utils.
|
|
86
|
+
from metadata.utils.execution_time_tracker import calculate_execution_time
|
|
81
87
|
from metadata.utils.logger import get_log_name, ingestion_logger
|
|
82
88
|
|
|
83
89
|
logger = ingestion_logger()
|
|
@@ -115,12 +121,16 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
115
121
|
config = MetadataRestSinkConfig.parse_obj(config_dict)
|
|
116
122
|
return cls(config, metadata)
|
|
117
123
|
|
|
124
|
+
@property
|
|
125
|
+
def name(self) -> str:
|
|
126
|
+
return "OpenMetadata"
|
|
127
|
+
|
|
118
128
|
@singledispatchmethod
|
|
119
129
|
def _run_dispatch(self, record: Entity) -> Either[Any]:
|
|
120
130
|
logger.debug(f"Processing Create request {type(record)}")
|
|
121
131
|
return self.write_create_request(record)
|
|
122
132
|
|
|
123
|
-
@calculate_execution_time
|
|
133
|
+
@calculate_execution_time(store=False)
|
|
124
134
|
def _run(self, record: Entity, *_, **__) -> Either[Any]:
|
|
125
135
|
"""
|
|
126
136
|
Default implementation for the single dispatch
|
|
@@ -132,14 +142,14 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
132
142
|
error = f"Failed to ingest {log} due to api request failure: {err}"
|
|
133
143
|
return Either(
|
|
134
144
|
left=StackTraceError(
|
|
135
|
-
name=log, error=error,
|
|
145
|
+
name=log, error=error, stackTrace=traceback.format_exc()
|
|
136
146
|
)
|
|
137
147
|
)
|
|
138
148
|
except Exception as exc:
|
|
139
149
|
error = f"Failed to ingest {log}: {exc}"
|
|
140
150
|
return Either(
|
|
141
151
|
left=StackTraceError(
|
|
142
|
-
name=log, error=error,
|
|
152
|
+
name=log, error=error, stackTrace=traceback.format_exc()
|
|
143
153
|
)
|
|
144
154
|
)
|
|
145
155
|
|
|
@@ -155,7 +165,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
155
165
|
error = f"Failed to ingest {type(entity_request).__name__}"
|
|
156
166
|
return Either(
|
|
157
167
|
left=StackTraceError(
|
|
158
|
-
name=type(entity_request).__name__, error=error,
|
|
168
|
+
name=type(entity_request).__name__, error=error, stackTrace=None
|
|
159
169
|
)
|
|
160
170
|
)
|
|
161
171
|
|
|
@@ -170,8 +180,10 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
170
180
|
destination=record.new_entity,
|
|
171
181
|
allowed_fields=ALLOWED_COMMON_PATCH_FIELDS,
|
|
172
182
|
restrict_update_fields=RESTRICT_UPDATE_LIST,
|
|
183
|
+
array_entity_fields=ARRAY_ENTITY_FIELDS,
|
|
173
184
|
)
|
|
174
|
-
|
|
185
|
+
patched_entity = PatchedEntity(new_entity=entity) if entity else None
|
|
186
|
+
return Either(right=patched_entity)
|
|
175
187
|
|
|
176
188
|
@_run_dispatch.register
|
|
177
189
|
def write_custom_properties(self, record: OMetaCustomProperties) -> Either[Dict]:
|
|
@@ -200,7 +212,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
200
212
|
left=StackTraceError(
|
|
201
213
|
name="Data Model",
|
|
202
214
|
error="Sink did not receive a table. We cannot ingest the data model.",
|
|
203
|
-
|
|
215
|
+
stackTrace=None,
|
|
204
216
|
)
|
|
205
217
|
)
|
|
206
218
|
|
|
@@ -275,7 +287,7 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
275
287
|
for role in record.roles:
|
|
276
288
|
try:
|
|
277
289
|
role_entity = self.metadata.get_by_name(
|
|
278
|
-
entity=Role, fqn=str(role.name.__root__
|
|
290
|
+
entity=Role, fqn=str(role.name.__root__)
|
|
279
291
|
)
|
|
280
292
|
except APIError:
|
|
281
293
|
role_entity = self._create_role(role)
|
|
@@ -411,6 +423,15 @@ class MetadataRestSink(Sink): # pylint: disable=too-many-public-methods
|
|
|
411
423
|
)
|
|
412
424
|
return Either(right=res)
|
|
413
425
|
|
|
426
|
+
@_run_dispatch.register
|
|
427
|
+
def write_test_case_resolution_status(
|
|
428
|
+
self, record: OMetaTestCaseResolutionStatus
|
|
429
|
+
) -> TestCaseResolutionStatus:
|
|
430
|
+
"""For sample data"""
|
|
431
|
+
res = self.metadata.create_test_case_resolution(record.test_case_resolution)
|
|
432
|
+
|
|
433
|
+
return Either(right=res)
|
|
434
|
+
|
|
414
435
|
@_run_dispatch.register
|
|
415
436
|
def write_data_insight_sample(
|
|
416
437
|
self, record: OMetaDataInsightSample
|