openmetadata-ingestion 1.2.5.1__py3-none-any.whl → 1.3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openmetadata-ingestion might be problematic. Click here for more details.
- airflow_provider_openmetadata/lineage/runner.py +44 -5
- airflow_provider_openmetadata/lineage/status.py +1 -1
- metadata/antlr/split_listener.py +1 -4
- metadata/applications/auto_tagger.py +212 -0
- metadata/cli/app.py +47 -0
- metadata/cli/db_dump.py +6 -3
- metadata/clients/aws_client.py +4 -0
- metadata/clients/domo_client.py +0 -2
- metadata/cmd.py +24 -13
- metadata/data_insight/processor/reports/entity_report_data_processor.py +8 -2
- metadata/data_insight/processor/reports/web_analytic_report_data_processor.py +4 -0
- metadata/data_insight/source/metadata.py +11 -4
- metadata/data_quality/interface/sqlalchemy/databricks/test_suite_interface.py +29 -0
- metadata/data_quality/interface/sqlalchemy/snowflake/test_suite_interface.py +29 -0
- metadata/data_quality/interface/sqlalchemy/sqa_test_suite_interface.py +6 -6
- metadata/data_quality/interface/sqlalchemy/unity_catalog/test_suite_interface.py +35 -0
- metadata/data_quality/interface/test_suite_interface_factory.py +40 -0
- metadata/data_quality/processor/test_case_runner.py +17 -8
- metadata/data_quality/source/test_suite.py +8 -2
- metadata/data_quality/validations/base_test_handler.py +21 -17
- metadata/data_quality/validations/column/base/columnValueLengthsToBeBetween.py +38 -1
- metadata/data_quality/validations/column/base/columnValuesToBeBetween.py +36 -1
- metadata/data_quality/validations/column/base/columnValuesToBeInSet.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeNotInSet.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeNotNull.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToBeUnique.py +15 -0
- metadata/data_quality/validations/column/base/columnValuesToMatchRegex.py +27 -0
- metadata/data_quality/validations/column/base/columnValuesToNotMatchRegex.py +27 -0
- metadata/data_quality/validations/column/pandas/columnValueLengthsToBeBetween.py +23 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeBetween.py +23 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeInSet.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeNotInSet.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToBeNotNull.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToMatchRegex.py +11 -0
- metadata/data_quality/validations/column/pandas/columnValuesToNotMatchRegex.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValueLengthsToBeBetween.py +27 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeBetween.py +23 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeInSet.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotInSet.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToBeNotNull.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToMatchRegex.py +11 -0
- metadata/data_quality/validations/column/sqlalchemy/columnValuesToNotMatchRegex.py +11 -0
- metadata/data_quality/validations/mixins/pandas_validator_mixin.py +13 -2
- metadata/data_quality/validations/mixins/sqa_validator_mixin.py +40 -0
- metadata/examples/workflows/databricks.yaml +0 -1
- metadata/examples/workflows/mongodb.yaml +4 -2
- metadata/examples/workflows/mstr.yaml +24 -0
- metadata/examples/workflows/sas.yaml +28 -0
- metadata/examples/workflows/unity_catalog.yaml +27 -0
- metadata/examples/workflows/unity_catalog_lineage.yaml +18 -0
- metadata/examples/workflows/unity_catalog_usage.yaml +35 -0
- metadata/generated/antlr/EntityLinkLexer.py +391 -292
- metadata/generated/antlr/EntityLinkListener.py +12 -12
- metadata/generated/antlr/EntityLinkParser.py +228 -103
- metadata/generated/schema/analytics/__init__.py +1 -1
- metadata/generated/schema/analytics/basic.py +1 -1
- metadata/generated/schema/analytics/reportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
- metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
- metadata/generated/schema/api/__init__.py +1 -1
- metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
- metadata/generated/schema/api/analytics/__init__.py +1 -1
- metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
- metadata/generated/schema/api/automations/__init__.py +1 -1
- metadata/generated/schema/api/automations/createWorkflow.py +1 -1
- metadata/generated/schema/api/bulkAssets.py +1 -1
- metadata/generated/schema/api/classification/__init__.py +1 -1
- metadata/generated/schema/api/classification/createClassification.py +1 -1
- metadata/generated/schema/api/classification/createTag.py +1 -1
- metadata/generated/schema/api/classification/loadTags.py +1 -1
- metadata/generated/schema/api/createBot.py +2 -3
- metadata/generated/schema/api/createEventPublisherJob.py +1 -1
- metadata/generated/schema/api/createType.py +1 -1
- metadata/generated/schema/api/data/__init__.py +1 -1
- metadata/generated/schema/api/data/createChart.py +1 -1
- metadata/generated/schema/api/data/createContainer.py +2 -2
- metadata/generated/schema/api/data/createCustomProperty.py +2 -3
- metadata/generated/schema/api/data/createDashboard.py +1 -1
- metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
- metadata/generated/schema/api/data/createDatabase.py +2 -3
- metadata/generated/schema/api/data/createDatabaseSchema.py +2 -3
- metadata/generated/schema/api/data/createGlossary.py +2 -3
- metadata/generated/schema/api/data/createGlossaryTerm.py +2 -3
- metadata/generated/schema/api/data/createMlModel.py +1 -1
- metadata/generated/schema/api/data/createPipeline.py +2 -2
- metadata/generated/schema/api/data/createQuery.py +2 -2
- metadata/generated/schema/api/data/createSearchIndex.py +1 -1
- metadata/generated/schema/api/data/createStoredProcedure.py +2 -4
- metadata/generated/schema/api/data/createTable.py +2 -2
- metadata/generated/schema/api/data/createTableProfile.py +1 -1
- metadata/generated/schema/api/data/createTopic.py +1 -1
- metadata/generated/schema/api/data/loadGlossary.py +1 -1
- metadata/generated/schema/api/data/restoreEntity.py +1 -1
- metadata/generated/schema/api/dataInsight/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
- metadata/generated/schema/api/docStore/__init__.py +1 -1
- metadata/generated/schema/api/docStore/createDocument.py +1 -1
- metadata/generated/schema/api/domains/__init__.py +1 -1
- metadata/generated/schema/api/domains/createDataProduct.py +1 -1
- metadata/generated/schema/api/domains/createDomain.py +1 -1
- metadata/generated/schema/api/feed/__init__.py +1 -1
- metadata/generated/schema/api/feed/closeTask.py +8 -1
- metadata/generated/schema/api/feed/createPost.py +1 -1
- metadata/generated/schema/api/feed/createSuggestion.py +30 -0
- metadata/generated/schema/api/feed/createThread.py +1 -1
- metadata/generated/schema/api/feed/resolveTask.py +12 -1
- metadata/generated/schema/api/feed/threadCount.py +18 -14
- metadata/generated/schema/api/lineage/__init__.py +1 -1
- metadata/generated/schema/api/lineage/addLineage.py +1 -1
- metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
- metadata/generated/schema/api/policies/__init__.py +1 -1
- metadata/generated/schema/api/policies/createPolicy.py +1 -1
- metadata/generated/schema/api/services/__init__.py +1 -1
- metadata/generated/schema/api/services/createDashboardService.py +1 -1
- metadata/generated/schema/api/services/createDatabaseService.py +1 -1
- metadata/generated/schema/api/services/createMessagingService.py +1 -1
- metadata/generated/schema/api/services/createMetadataService.py +1 -1
- metadata/generated/schema/api/services/createMlModelService.py +1 -1
- metadata/generated/schema/api/services/createPipelineService.py +1 -1
- metadata/generated/schema/api/services/createSearchService.py +1 -1
- metadata/generated/schema/api/services/createStorageService.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
- metadata/generated/schema/api/setOwner.py +1 -1
- metadata/generated/schema/api/teams/__init__.py +1 -1
- metadata/generated/schema/api/teams/createPersona.py +1 -1
- metadata/generated/schema/api/teams/createRole.py +2 -3
- metadata/generated/schema/api/teams/createTeam.py +1 -1
- metadata/generated/schema/api/teams/createUser.py +2 -2
- metadata/generated/schema/api/tests/__init__.py +1 -1
- metadata/generated/schema/api/tests/createCustomMetric.py +4 -2
- metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
- metadata/generated/schema/api/tests/createTestCase.py +4 -1
- metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +31 -0
- metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
- metadata/generated/schema/api/tests/createTestSuite.py +1 -1
- metadata/generated/schema/api/voteRequest.py +1 -1
- metadata/generated/schema/auth/__init__.py +1 -1
- metadata/generated/schema/auth/basicAuth.py +1 -1
- metadata/generated/schema/auth/basicLoginRequest.py +1 -1
- metadata/generated/schema/auth/changePasswordRequest.py +1 -1
- metadata/generated/schema/auth/createPersonalToken.py +1 -1
- metadata/generated/schema/auth/emailRequest.py +1 -1
- metadata/generated/schema/auth/emailVerificationToken.py +1 -1
- metadata/generated/schema/auth/generateToken.py +1 -1
- metadata/generated/schema/auth/jwtAuth.py +1 -1
- metadata/generated/schema/auth/loginRequest.py +1 -1
- metadata/generated/schema/auth/logoutRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetRequest.py +3 -3
- metadata/generated/schema/auth/passwordResetToken.py +1 -1
- metadata/generated/schema/auth/personalAccessToken.py +1 -1
- metadata/generated/schema/auth/refreshToken.py +1 -1
- metadata/generated/schema/auth/registrationRequest.py +2 -2
- metadata/generated/schema/auth/revokePersonalToken.py +1 -1
- metadata/generated/schema/auth/revokeToken.py +1 -1
- metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
- metadata/generated/schema/auth/ssoAuth.py +1 -1
- metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
- metadata/generated/schema/configuration/__init__.py +1 -1
- metadata/generated/schema/configuration/appsPrivateConfiguration.py +28 -0
- metadata/generated/schema/configuration/authConfig.py +1 -1
- metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
- metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
- metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/elasticSearchConfiguration.py +7 -1
- metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
- metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
- metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
- metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapConfiguration.py +28 -2
- metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
- metadata/generated/schema/configuration/loginConfiguration.py +1 -1
- metadata/generated/schema/configuration/logoConfiguration.py +1 -1
- metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
- metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
- metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
- metadata/generated/schema/dataInsight/__init__.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
- metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
- metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
- metadata/generated/schema/dataInsight/type/__init__.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithDescriptionByType.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfEntitiesWithOwnerByType.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfServicesWithDescription.py +1 -1
- metadata/generated/schema/dataInsight/type/percentageOfServicesWithOwner.py +1 -1
- metadata/generated/schema/dataInsight/type/totalEntitiesByTier.py +1 -1
- metadata/generated/schema/dataInsight/type/totalEntitiesByType.py +1 -1
- metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
- metadata/generated/schema/email/__init__.py +1 -1
- metadata/generated/schema/email/emailRequest.py +1 -1
- metadata/generated/schema/email/smtpSettings.py +1 -1
- metadata/generated/schema/entity/__init__.py +1 -1
- metadata/generated/schema/entity/applications/__init__.py +1 -1
- metadata/generated/schema/entity/applications/app.py +18 -4
- metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
- metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/applicationConfig.py +30 -0
- metadata/generated/schema/entity/{services/connections/database/mongoDB → applications/configuration/external}/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/autoTaggerAppConfig.py +28 -0
- metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +46 -0
- metadata/generated/schema/entity/applications/configuration/internal/__init__.py +3 -0
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +14 -0
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +27 -0
- metadata/generated/schema/entity/applications/configuration/{searchIndexingApp.py → internal/searchIndexingAppConfig.py} +13 -3
- metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +3 -0
- metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +26 -0
- metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
- metadata/generated/schema/entity/applications/jobStatus.py +3 -3
- metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +12 -3
- metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +12 -3
- metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
- metadata/generated/schema/entity/automations/__init__.py +1 -1
- metadata/generated/schema/entity/automations/testServiceConnection.py +2 -2
- metadata/generated/schema/entity/automations/workflow.py +1 -1
- metadata/generated/schema/entity/bot.py +2 -3
- metadata/generated/schema/entity/classification/__init__.py +1 -1
- metadata/generated/schema/entity/classification/classification.py +1 -1
- metadata/generated/schema/entity/classification/tag.py +2 -2
- metadata/generated/schema/entity/data/__init__.py +1 -1
- metadata/generated/schema/entity/data/chart.py +1 -1
- metadata/generated/schema/entity/data/container.py +4 -9
- metadata/generated/schema/entity/data/dashboard.py +1 -1
- metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
- metadata/generated/schema/entity/data/database.py +4 -8
- metadata/generated/schema/entity/data/databaseSchema.py +2 -8
- metadata/generated/schema/entity/data/glossary.py +1 -1
- metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
- metadata/generated/schema/entity/data/metrics.py +1 -1
- metadata/generated/schema/entity/data/mlmodel.py +1 -1
- metadata/generated/schema/entity/data/pipeline.py +6 -9
- metadata/generated/schema/entity/data/query.py +2 -2
- metadata/generated/schema/entity/data/report.py +1 -1
- metadata/generated/schema/entity/data/searchIndex.py +1 -1
- metadata/generated/schema/entity/data/storedProcedure.py +3 -9
- metadata/generated/schema/entity/data/table.py +18 -12
- metadata/generated/schema/entity/data/topic.py +1 -1
- metadata/generated/schema/entity/docStore/__init__.py +1 -1
- metadata/generated/schema/entity/docStore/document.py +1 -1
- metadata/generated/schema/entity/domains/__init__.py +1 -1
- metadata/generated/schema/entity/domains/dataProduct.py +1 -1
- metadata/generated/schema/entity/domains/domain.py +1 -1
- metadata/generated/schema/entity/events/__init__.py +1 -1
- metadata/generated/schema/entity/events/webhook.py +6 -3
- metadata/generated/schema/entity/feed/__init__.py +1 -1
- metadata/generated/schema/entity/feed/suggestion.py +61 -0
- metadata/generated/schema/entity/feed/thread.py +6 -1
- metadata/generated/schema/entity/policies/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
- metadata/generated/schema/entity/policies/filters.py +2 -5
- metadata/generated/schema/entity/policies/policy.py +1 -1
- metadata/generated/schema/entity/services/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +46 -0
- metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/athenaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +6 -1
- metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricksConnection.py +2 -7
- metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/db2Connection.py +3 -2
- metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dorisConnection.py +86 -0
- metadata/generated/schema/entity/services/connections/database/druidConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/hiveConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +3 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +17 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +16 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +23 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +43 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +20 -0
- metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +69 -0
- metadata/generated/schema/entity/services/connections/database/icebergConnection.py +35 -0
- metadata/generated/schema/entity/services/connections/database/impalaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +24 -14
- metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +5 -2
- metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/oracleConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/postgresConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/prestoConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/sasConnection.py +48 -0
- metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/trinoConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +90 -0
- metadata/generated/schema/entity/services/connections/database/verticaConnection.py +2 -2
- metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +5 -20
- metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +2 -8
- metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +23 -0
- metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/{adlsConection.py → adlsConnection.py} +2 -2
- metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
- metadata/generated/schema/entity/services/dashboardService.py +4 -1
- metadata/generated/schema/entity/services/databaseService.py +13 -1
- metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +27 -22
- metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/status.py +56 -0
- metadata/generated/schema/entity/services/messagingService.py +1 -1
- metadata/generated/schema/entity/services/metadataService.py +1 -1
- metadata/generated/schema/entity/services/mlmodelService.py +1 -1
- metadata/generated/schema/entity/services/pipelineService.py +4 -1
- metadata/generated/schema/entity/services/searchService.py +1 -1
- metadata/generated/schema/entity/services/serviceType.py +1 -1
- metadata/generated/schema/entity/services/storageService.py +10 -3
- metadata/generated/schema/entity/teams/__init__.py +1 -1
- metadata/generated/schema/entity/teams/persona.py +1 -1
- metadata/generated/schema/entity/teams/role.py +2 -6
- metadata/generated/schema/entity/teams/team.py +1 -1
- metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
- metadata/generated/schema/entity/teams/user.py +3 -10
- metadata/generated/schema/entity/type.py +2 -2
- metadata/generated/schema/entity/utils/__init__.py +1 -1
- metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
- metadata/generated/schema/entity/utils/servicesCount.py +1 -1
- metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
- metadata/generated/schema/events/__init__.py +1 -1
- metadata/generated/schema/events/alertMetrics.py +27 -0
- metadata/generated/schema/events/api/__init__.py +1 -1
- metadata/generated/schema/events/api/createEventSubscription.py +14 -12
- metadata/generated/schema/events/emailAlertConfig.py +1 -1
- metadata/generated/schema/events/eventFilterRule.py +13 -2
- metadata/generated/schema/events/eventSubscription.py +89 -49
- metadata/generated/schema/events/eventSubscriptionOffset.py +21 -0
- metadata/generated/schema/events/failedEvent.py +26 -0
- metadata/generated/schema/events/filterResourceDescriptor.py +27 -0
- metadata/generated/schema/events/subscriptionResourceDescriptor.py +11 -10
- metadata/generated/schema/metadataIngestion/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/application.py +40 -0
- metadata/generated/schema/metadataIngestion/applicationPipeline.py +15 -6
- metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +6 -1
- metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +6 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +6 -3
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +6 -3
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +6 -3
- metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +2 -2
- metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +24 -0
- metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +8 -1
- metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/workflow.py +1 -1
- metadata/generated/schema/monitoring/__init__.py +1 -1
- metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
- metadata/generated/schema/security/__init__.py +1 -1
- metadata/generated/schema/security/client/__init__.py +1 -1
- metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
- metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
- metadata/generated/schema/security/credentials/__init__.py +1 -1
- metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
- metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
- metadata/generated/schema/security/credentials/basicAuth.py +1 -1
- metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpValues.py +1 -1
- metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
- metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
- metadata/generated/schema/security/secrets/__init__.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +14 -4
- metadata/generated/schema/security/secrets/secretsManagerProvider.py +2 -2
- metadata/generated/schema/security/securityConfiguration.py +1 -1
- metadata/generated/schema/security/ssl/__init__.py +1 -1
- metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
- metadata/generated/schema/security/ssl/verifySSLConfig.py +4 -1
- metadata/generated/schema/settings/__init__.py +1 -1
- metadata/generated/schema/settings/settings.py +1 -1
- metadata/generated/schema/system/__init__.py +1 -1
- metadata/generated/schema/system/entityError.py +17 -0
- metadata/generated/schema/system/eventPublisherJob.py +5 -19
- metadata/generated/schema/system/indexingError.py +34 -0
- metadata/generated/schema/system/ui/__init__.py +1 -1
- metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
- metadata/generated/schema/system/ui/page.py +1 -1
- metadata/generated/schema/tests/__init__.py +1 -1
- metadata/generated/schema/tests/assigned.py +18 -0
- metadata/generated/schema/tests/basic.py +17 -39
- metadata/generated/schema/tests/customMetric.py +4 -2
- metadata/generated/schema/tests/resolved.py +35 -0
- metadata/generated/schema/tests/testCase.py +11 -2
- metadata/generated/schema/tests/testCaseResolutionStatus.py +63 -0
- metadata/generated/schema/tests/testDefinition.py +5 -1
- metadata/generated/schema/tests/testSuite.py +1 -1
- metadata/generated/schema/type/__init__.py +1 -1
- metadata/generated/schema/type/auditLog.py +1 -1
- metadata/generated/schema/type/basic.py +5 -5
- metadata/generated/schema/type/bulkOperationResult.py +1 -1
- metadata/generated/schema/type/changeEvent.py +5 -75
- metadata/generated/schema/type/changeEventType.py +29 -0
- metadata/generated/schema/type/collectionDescriptor.py +1 -1
- metadata/generated/schema/type/csvDocumentation.py +1 -1
- metadata/generated/schema/type/csvErrorType.py +1 -1
- metadata/generated/schema/type/csvFile.py +1 -1
- metadata/generated/schema/type/csvImportResult.py +1 -1
- metadata/generated/schema/type/dailyCount.py +1 -1
- metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
- metadata/generated/schema/type/entityHistory.py +1 -1
- metadata/generated/schema/type/entityLineage.py +2 -1
- metadata/generated/schema/type/entityReference.py +5 -1
- metadata/generated/schema/type/entityReferenceList.py +1 -1
- metadata/generated/schema/type/entityRelationship.py +1 -1
- metadata/generated/schema/type/entityUsage.py +1 -1
- metadata/generated/schema/type/filterPattern.py +1 -1
- metadata/generated/schema/type/function.py +7 -6
- metadata/generated/schema/type/include.py +1 -1
- metadata/generated/schema/type/jdbcConnection.py +1 -1
- metadata/generated/schema/type/lifeCycle.py +1 -1
- metadata/generated/schema/type/paging.py +1 -1
- metadata/generated/schema/type/profile.py +1 -1
- metadata/generated/schema/type/queryParserData.py +2 -2
- metadata/generated/schema/type/reaction.py +1 -1
- metadata/generated/schema/type/schedule.py +1 -1
- metadata/generated/schema/type/schema.py +1 -1
- metadata/generated/schema/type/tableQuery.py +2 -2
- metadata/generated/schema/type/tableUsageCount.py +1 -1
- metadata/generated/schema/type/tagLabel.py +1 -1
- metadata/generated/schema/type/usageDetails.py +1 -1
- metadata/generated/schema/type/usageRequest.py +1 -1
- metadata/generated/schema/type/votes.py +1 -1
- metadata/great_expectations/action.py +3 -2
- metadata/ingestion/api/delete.py +5 -2
- metadata/ingestion/api/models.py +4 -10
- metadata/ingestion/api/parser.py +3 -37
- metadata/ingestion/api/status.py +15 -4
- metadata/ingestion/api/step.py +39 -6
- metadata/ingestion/api/steps.py +20 -0
- metadata/ingestion/api/topology_runner.py +114 -168
- metadata/ingestion/bulksink/metadata_usage.py +11 -5
- metadata/ingestion/lineage/models.py +4 -0
- metadata/ingestion/lineage/parser.py +4 -5
- metadata/ingestion/lineage/sql_lineage.py +9 -17
- metadata/ingestion/models/custom_pydantic.py +4 -2
- metadata/ingestion/models/lf_tags_model.py +33 -0
- metadata/ingestion/models/patch_request.py +108 -2
- metadata/ingestion/models/tests_data.py +9 -0
- metadata/ingestion/models/topology.py +177 -60
- metadata/ingestion/ometa/auth_provider.py +0 -349
- metadata/ingestion/ometa/mixins/es_mixin.py +17 -0
- metadata/ingestion/ometa/mixins/patch_mixin.py +17 -68
- metadata/ingestion/ometa/mixins/suggestions_mixin.py +41 -0
- metadata/ingestion/ometa/mixins/table_mixin.py +18 -0
- metadata/ingestion/ometa/mixins/tests_mixin.py +24 -3
- metadata/ingestion/ometa/mixins/user_mixin.py +117 -22
- metadata/ingestion/ometa/ometa_api.py +18 -25
- metadata/ingestion/ometa/routes.py +7 -0
- metadata/ingestion/processor/query_parser.py +9 -2
- metadata/ingestion/sink/metadata_rest.py +25 -6
- metadata/ingestion/source/dashboard/dashboard_service.py +38 -61
- metadata/ingestion/source/dashboard/domodashboard/metadata.py +13 -14
- metadata/ingestion/source/dashboard/lightdash/metadata.py +2 -1
- metadata/ingestion/source/dashboard/looker/metadata.py +38 -25
- metadata/ingestion/source/dashboard/metabase/metadata.py +23 -6
- metadata/ingestion/source/dashboard/mode/client.py +10 -23
- metadata/ingestion/source/dashboard/mode/connection.py +6 -1
- metadata/ingestion/source/dashboard/mode/metadata.py +8 -4
- metadata/ingestion/source/dashboard/mstr/client.py +208 -0
- metadata/ingestion/source/dashboard/mstr/connection.py +53 -0
- metadata/ingestion/source/dashboard/mstr/metadata.py +182 -0
- metadata/ingestion/source/dashboard/mstr/models.py +144 -0
- metadata/ingestion/source/dashboard/powerbi/metadata.py +15 -10
- metadata/ingestion/source/dashboard/qliksense/metadata.py +11 -7
- metadata/ingestion/source/dashboard/quicksight/metadata.py +9 -5
- metadata/ingestion/source/dashboard/redash/metadata.py +23 -14
- metadata/ingestion/source/dashboard/superset/api_source.py +11 -8
- metadata/ingestion/source/dashboard/superset/client.py +16 -9
- metadata/ingestion/source/dashboard/superset/connection.py +3 -3
- metadata/ingestion/source/dashboard/superset/db_source.py +14 -10
- metadata/ingestion/source/dashboard/superset/mixin.py +22 -18
- metadata/ingestion/source/dashboard/superset/queries.py +1 -1
- metadata/ingestion/source/dashboard/tableau/client.py +91 -11
- metadata/ingestion/source/dashboard/tableau/connection.py +10 -1
- metadata/ingestion/source/dashboard/tableau/metadata.py +58 -70
- metadata/ingestion/source/dashboard/tableau/models.py +0 -8
- metadata/ingestion/source/dashboard/tableau/queries.py +5 -5
- metadata/ingestion/source/database/athena/client.py +80 -0
- metadata/ingestion/source/database/athena/connection.py +7 -0
- metadata/ingestion/source/database/athena/metadata.py +161 -19
- metadata/ingestion/source/database/azuresql/metadata.py +0 -1
- metadata/ingestion/source/database/bigquery/connection.py +24 -3
- metadata/ingestion/source/database/bigquery/metadata.py +39 -27
- metadata/ingestion/source/database/bigquery/queries.py +11 -4
- metadata/ingestion/source/database/bigquery/query_parser.py +13 -0
- metadata/ingestion/source/database/bigquery/usage.py +1 -3
- metadata/ingestion/source/database/column_helpers.py +0 -10
- metadata/ingestion/source/database/column_type_parser.py +11 -5
- metadata/ingestion/source/database/common_db_source.py +8 -2
- metadata/ingestion/source/database/common_nosql_source.py +8 -4
- metadata/ingestion/source/database/database_service.py +89 -7
- metadata/ingestion/source/database/databricks/client.py +5 -10
- metadata/ingestion/source/database/databricks/connection.py +16 -55
- metadata/ingestion/source/database/databricks/lineage.py +29 -26
- metadata/ingestion/source/database/databricks/metadata.py +448 -11
- metadata/ingestion/source/database/databricks/queries.py +28 -0
- metadata/ingestion/source/database/databricks/query_parser.py +5 -1
- metadata/ingestion/source/database/databricks/usage.py +2 -2
- metadata/ingestion/source/database/datalake/connection.py +31 -4
- metadata/ingestion/source/database/datalake/metadata.py +113 -27
- metadata/ingestion/source/database/dbt/dbt_config.py +5 -0
- metadata/ingestion/source/database/dbt/dbt_service.py +10 -14
- metadata/ingestion/source/database/dbt/dbt_utils.py +3 -1
- metadata/ingestion/source/database/dbt/metadata.py +11 -25
- metadata/ingestion/source/database/deltalake/metadata.py +6 -3
- metadata/ingestion/source/database/domodatabase/metadata.py +7 -6
- metadata/ingestion/source/database/doris/connection.py +72 -0
- metadata/ingestion/source/database/doris/metadata.py +315 -0
- metadata/ingestion/source/database/doris/queries.py +54 -0
- metadata/ingestion/source/database/doris/utils.py +64 -0
- metadata/ingestion/source/database/extended_sample_data.py +532 -0
- metadata/ingestion/source/database/glue/metadata.py +8 -5
- metadata/ingestion/source/database/hive/connection.py +0 -2
- metadata/ingestion/source/database/hive/utils.py +3 -0
- metadata/ingestion/source/database/iceberg/catalog/__init__.py +65 -0
- metadata/ingestion/source/database/iceberg/catalog/base.py +40 -0
- metadata/ingestion/source/database/iceberg/catalog/dynamodb.py +102 -0
- metadata/ingestion/source/database/iceberg/catalog/glue.py +88 -0
- metadata/ingestion/source/database/iceberg/catalog/hive.py +51 -0
- metadata/ingestion/source/database/iceberg/catalog/rest.py +84 -0
- metadata/ingestion/source/database/iceberg/connection.py +68 -0
- metadata/ingestion/source/database/iceberg/fs/__init__.py +52 -0
- metadata/ingestion/source/database/iceberg/fs/azure.py +44 -0
- metadata/ingestion/source/database/iceberg/fs/base.py +30 -0
- metadata/ingestion/source/database/iceberg/fs/s3.py +77 -0
- metadata/ingestion/source/database/iceberg/helper.py +124 -0
- metadata/ingestion/source/database/iceberg/metadata.py +311 -0
- metadata/ingestion/source/database/iceberg/models.py +66 -0
- metadata/ingestion/source/database/life_cycle_query_mixin.py +72 -3
- metadata/ingestion/source/database/mongodb/connection.py +1 -5
- metadata/ingestion/source/database/mssql/lineage.py +2 -0
- metadata/ingestion/source/database/mssql/metadata.py +108 -4
- metadata/ingestion/source/database/mssql/models.py +30 -0
- metadata/ingestion/source/database/mssql/queries.py +179 -1
- metadata/ingestion/source/database/mssql/utils.py +207 -4
- metadata/ingestion/source/database/mysql/metadata.py +0 -2
- metadata/ingestion/source/database/oracle/metadata.py +108 -2
- metadata/ingestion/source/database/oracle/models.py +30 -0
- metadata/ingestion/source/database/oracle/queries.py +98 -17
- metadata/ingestion/source/database/oracle/utils.py +0 -1
- metadata/ingestion/source/database/postgres/lineage.py +32 -14
- metadata/ingestion/source/database/postgres/metadata.py +15 -7
- metadata/ingestion/source/database/postgres/pgspider/lineage.py +0 -1
- metadata/ingestion/source/database/postgres/queries.py +4 -2
- metadata/ingestion/source/database/postgres/query_parser.py +4 -72
- metadata/ingestion/source/database/postgres/usage.py +41 -0
- metadata/ingestion/source/database/postgres/utils.py +34 -0
- metadata/ingestion/source/database/query_parser_source.py +8 -2
- metadata/ingestion/source/database/redshift/metadata.py +14 -4
- metadata/ingestion/source/database/redshift/queries.py +10 -4
- metadata/ingestion/source/database/redshift/query_parser.py +16 -0
- metadata/ingestion/source/database/redshift/usage.py +0 -2
- metadata/ingestion/source/database/salesforce/metadata.py +32 -3
- metadata/ingestion/source/database/sample_data.py +120 -6
- metadata/ingestion/source/database/sas/client.py +184 -0
- metadata/ingestion/source/database/sas/connection.py +47 -0
- metadata/ingestion/source/database/sas/extension_attr.py +103 -0
- metadata/ingestion/source/database/sas/metadata.py +914 -0
- metadata/ingestion/source/database/snowflake/metadata.py +8 -51
- metadata/ingestion/source/database/snowflake/models.py +6 -1
- metadata/ingestion/source/database/snowflake/queries.py +0 -4
- metadata/ingestion/source/database/snowflake/query_parser.py +5 -20
- metadata/ingestion/source/database/snowflake/utils.py +2 -6
- metadata/ingestion/source/database/stored_procedures_mixin.py +12 -8
- metadata/ingestion/source/database/unitycatalog/__init__.py +0 -0
- metadata/ingestion/source/database/unitycatalog/client.py +87 -0
- metadata/ingestion/source/database/unitycatalog/connection.py +97 -0
- metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/lineage.py +11 -11
- metadata/ingestion/source/database/{databricks/unity_catalog → unitycatalog}/metadata.py +42 -49
- metadata/ingestion/source/database/unitycatalog/query_parser.py +60 -0
- metadata/ingestion/source/database/unitycatalog/usage.py +31 -0
- metadata/ingestion/source/database/usage_source.py +3 -2
- metadata/ingestion/source/messaging/common_broker_source.py +6 -4
- metadata/ingestion/source/messaging/kinesis/metadata.py +6 -3
- metadata/ingestion/source/messaging/messaging_service.py +6 -2
- metadata/ingestion/source/metadata/amundsen/metadata.py +10 -7
- metadata/ingestion/source/metadata/atlas/metadata.py +5 -5
- metadata/ingestion/source/mlmodel/mlflow/metadata.py +5 -2
- metadata/ingestion/source/mlmodel/mlmodel_service.py +6 -2
- metadata/ingestion/source/pipeline/airflow/connection.py +0 -12
- metadata/ingestion/source/pipeline/airflow/lineage_parser.py +12 -6
- metadata/ingestion/source/pipeline/airflow/metadata.py +63 -34
- metadata/ingestion/source/pipeline/airflow/models.py +5 -4
- metadata/ingestion/source/pipeline/dagster/metadata.py +7 -4
- metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +12 -9
- metadata/ingestion/source/pipeline/domopipeline/metadata.py +7 -4
- metadata/ingestion/source/pipeline/gluepipeline/metadata.py +5 -2
- metadata/ingestion/source/pipeline/pipeline_service.py +6 -2
- metadata/ingestion/source/pipeline/spline/metadata.py +0 -1
- metadata/ingestion/source/search/elasticsearch/connection.py +4 -1
- metadata/ingestion/source/search/elasticsearch/metadata.py +1 -2
- metadata/ingestion/source/search/search_service.py +6 -2
- metadata/ingestion/source/storage/s3/metadata.py +22 -17
- metadata/ingestion/source/storage/storage_service.py +53 -11
- metadata/ingestion/stage/table_usage.py +9 -2
- metadata/mixins/sqalchemy/sqa_mixin.py +14 -7
- metadata/parsers/protobuf_parser.py +29 -11
- metadata/pii/processor.py +9 -2
- metadata/profiler/api/models.py +19 -1
- metadata/profiler/interface/pandas/profiler_interface.py +59 -18
- metadata/profiler/interface/profiler_interface.py +13 -2
- metadata/profiler/interface/profiler_interface_factory.py +49 -14
- metadata/profiler/interface/sqlalchemy/bigquery/profiler_interface.py +3 -0
- metadata/profiler/interface/sqlalchemy/databricks/profiler_interface.py +26 -0
- metadata/profiler/interface/sqlalchemy/db2/__init__.py +0 -0
- metadata/profiler/interface/sqlalchemy/db2/profiler_interface.py +38 -0
- metadata/profiler/interface/sqlalchemy/mariadb/profiler_interface.py +85 -0
- metadata/profiler/interface/sqlalchemy/profiler_interface.py +77 -34
- metadata/profiler/interface/sqlalchemy/single_store/profiler_interface.py +2 -2
- metadata/profiler/interface/sqlalchemy/snowflake/profiler_interface.py +7 -0
- metadata/profiler/interface/sqlalchemy/trino/profiler_interface.py +2 -2
- metadata/profiler/interface/sqlalchemy/unity_catalog/profiler_interface.py +33 -0
- metadata/profiler/metrics/composed/null_ratio.py +1 -1
- metadata/profiler/metrics/hybrid/histogram.py +1 -0
- metadata/profiler/metrics/static/max.py +4 -1
- metadata/profiler/metrics/static/min.py +4 -1
- metadata/profiler/metrics/system/queries/snowflake.py +89 -17
- metadata/profiler/metrics/system/system.py +62 -20
- metadata/profiler/orm/functions/length.py +1 -0
- metadata/profiler/orm/functions/median.py +7 -0
- metadata/profiler/orm/functions/table_metric_computer.py +462 -0
- metadata/profiler/orm/registry.py +1 -0
- metadata/profiler/processor/core.py +116 -52
- metadata/profiler/processor/default.py +14 -3
- metadata/profiler/processor/handle_partition.py +2 -2
- metadata/profiler/processor/processor.py +9 -4
- metadata/profiler/processor/sampler/sqlalchemy/bigquery/sampler.py +31 -3
- metadata/profiler/processor/sampler/sqlalchemy/sampler.py +29 -6
- metadata/profiler/processor/sampler/sqlalchemy/trino/sampler.py +10 -4
- metadata/profiler/source/base/profiler_source.py +5 -2
- metadata/profiler/source/bigquery/type_mapper.py +0 -1
- metadata/profiler/source/mariadb/functions/median.py +20 -0
- metadata/profiler/source/mariadb/metrics/window/first_quartile.py +10 -0
- metadata/profiler/source/mariadb/metrics/window/median.py +10 -0
- metadata/profiler/source/mariadb/metrics/window/third_quartile.py +10 -0
- metadata/profiler/source/metadata.py +43 -10
- metadata/profiler/source/metadata_ext.py +16 -50
- metadata/py.typed +0 -0
- metadata/readers/dataframe/json.py +5 -4
- metadata/readers/file/api_reader.py +0 -1
- metadata/utils/constants.py +5 -0
- metadata/utils/datalake/datalake_utils.py +363 -129
- metadata/utils/entity_link.py +26 -6
- metadata/utils/fqn.py +20 -0
- metadata/utils/helpers.py +55 -9
- metadata/utils/importer.py +2 -3
- metadata/utils/life_cycle_utils.py +4 -4
- metadata/utils/logger.py +13 -2
- metadata/utils/partition.py +10 -5
- metadata/utils/secrets/client/loader.py +0 -1
- metadata/utils/secrets/noop_secrets_manager.py +4 -3
- metadata/utils/secrets/secrets_manager_factory.py +3 -4
- metadata/utils/{source_hash_utils.py → source_hash.py} +10 -1
- metadata/utils/sqlalchemy_utils.py +21 -0
- metadata/utils/storage_metadata_config.py +42 -1
- metadata/utils/tag_utils.py +5 -2
- metadata/workflow/application.py +154 -0
- metadata/workflow/application_output_handler.py +34 -0
- metadata/workflow/base.py +84 -153
- metadata/workflow/data_insight.py +8 -7
- metadata/workflow/data_quality.py +3 -2
- metadata/workflow/ingestion.py +203 -0
- metadata/workflow/metadata.py +2 -3
- metadata/workflow/output_handler.py +204 -0
- metadata/workflow/profiler.py +2 -2
- metadata/workflow/usage.py +3 -4
- metadata/workflow/workflow_output_handler.py +15 -255
- metadata/workflow/workflow_status_mixin.py +44 -52
- openmetadata_ingestion-1.3.0.0.dist-info/METADATA +749 -0
- {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/RECORD +812 -725
- {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/WHEEL +1 -1
- metadata/generated/schema/entity/applications/appConfig.py +0 -21
- metadata/generated/schema/entity/applications/configuration/dataInsightsApp.py +0 -17
- metadata/generated/schema/entity/applications/configuration/externalAppIngestionConfig.py +0 -38
- metadata/generated/schema/entity/services/connections/database/mongoDB/mongoDBValues.py +0 -44
- metadata/generated/schema/events/dataInsightAlertConfig.py +0 -17
- metadata/generated/schema/events/entitySpelFilters.py +0 -19
- metadata/ingestion/models/es_documents.py +0 -339
- metadata/ingestion/ometa/mixins/glossary_mixin.py +0 -501
- metadata/ingestion/ometa/provider_registry.py +0 -144
- metadata/ingestion/source/database/databricks/legacy/lineage.py +0 -51
- metadata/ingestion/source/database/databricks/legacy/metadata.py +0 -339
- metadata/ingestion/source/metadata/metadata_elasticsearch/metadata.py +0 -144
- metadata/profiler/orm/functions/table_metric_construct.py +0 -365
- openmetadata_ingestion-1.2.5.1.dist-info/METADATA +0 -426
- /metadata/ingestion/source/{database/databricks/legacy → dashboard/mstr}/__init__.py +0 -0
- /metadata/ingestion/source/database/{databricks/unity_catalog → doris}/__init__.py +0 -0
- /metadata/ingestion/source/{metadata/metadata_elasticsearch → database/sas}/__init__.py +0 -0
- /metadata/ingestion/source/database/{databricks → unitycatalog}/models.py +0 -0
- {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/LICENSE +0 -0
- {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/entry_points.txt +0 -0
- {openmetadata_ingestion-1.2.5.1.dist-info → openmetadata_ingestion-1.3.0.0.dist-info}/top_level.txt +0 -0
|
@@ -45,10 +45,16 @@ from metadata.generated.schema.entity.services.pipelineService import (
|
|
|
45
45
|
)
|
|
46
46
|
from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
|
|
47
47
|
from metadata.generated.schema.type.entityReference import EntityReference
|
|
48
|
+
from metadata.ingestion.models.patch_request import (
|
|
49
|
+
ALLOWED_COMMON_PATCH_FIELDS,
|
|
50
|
+
RESTRICT_UPDATE_LIST,
|
|
51
|
+
)
|
|
48
52
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
49
53
|
from metadata.ingestion.source.pipeline.airflow.lineage_parser import XLets
|
|
54
|
+
from metadata.utils import fqn
|
|
50
55
|
from metadata.utils.constants import ENTITY_REFERENCE_TYPE_MAP
|
|
51
56
|
from metadata.utils.helpers import clean_uri, datetime_to_ts
|
|
57
|
+
from metadata.utils.source_hash import generate_source_hash
|
|
52
58
|
|
|
53
59
|
|
|
54
60
|
class SimpleEdge(BaseModel):
|
|
@@ -151,11 +157,24 @@ class AirflowLineageRunner:
|
|
|
151
157
|
for task in self.dag.tasks or []
|
|
152
158
|
]
|
|
153
159
|
|
|
154
|
-
def
|
|
160
|
+
def create_or_update_pipeline_entity(
|
|
161
|
+
self, pipeline_service: PipelineService
|
|
162
|
+
) -> Pipeline:
|
|
155
163
|
"""
|
|
156
|
-
Create the Pipeline Entity
|
|
164
|
+
Create the Pipeline Entity if it does not exist, or PATCH it
|
|
165
|
+
if there have been changes.
|
|
157
166
|
"""
|
|
158
|
-
self.
|
|
167
|
+
pipeline: Pipeline = self.metadata.get_by_name(
|
|
168
|
+
entity=Pipeline,
|
|
169
|
+
fqn=fqn.build(
|
|
170
|
+
self.metadata,
|
|
171
|
+
Pipeline,
|
|
172
|
+
service_name=self.service_name,
|
|
173
|
+
pipeline_name=self.dag.dag_id,
|
|
174
|
+
),
|
|
175
|
+
fields=["*"],
|
|
176
|
+
)
|
|
177
|
+
|
|
159
178
|
pipeline_request = CreatePipelineRequest(
|
|
160
179
|
name=self.dag.dag_id,
|
|
161
180
|
description=self.dag.description,
|
|
@@ -167,7 +186,27 @@ class AirflowLineageRunner:
|
|
|
167
186
|
service=pipeline_service.fullyQualifiedName,
|
|
168
187
|
)
|
|
169
188
|
|
|
170
|
-
|
|
189
|
+
create_entity_request_hash = generate_source_hash(
|
|
190
|
+
create_request=pipeline_request
|
|
191
|
+
)
|
|
192
|
+
pipeline_request.sourceHash = create_entity_request_hash
|
|
193
|
+
|
|
194
|
+
if pipeline is None:
|
|
195
|
+
self.dag.log.info("Creating Pipeline Entity from DAG...")
|
|
196
|
+
return self.metadata.create_or_update(pipeline_request)
|
|
197
|
+
|
|
198
|
+
if create_entity_request_hash != pipeline.sourceHash:
|
|
199
|
+
self.dag.log.info("Updating Pipeline Entity from DAG...")
|
|
200
|
+
return self.metadata.patch(
|
|
201
|
+
entity=Pipeline,
|
|
202
|
+
source=pipeline,
|
|
203
|
+
destination=pipeline.copy(update=pipeline_request.__dict__),
|
|
204
|
+
allowed_fields=ALLOWED_COMMON_PATCH_FIELDS,
|
|
205
|
+
restrict_update_fields=RESTRICT_UPDATE_LIST,
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
self.dag.log.info("DAG has not changed since last run")
|
|
209
|
+
return pipeline
|
|
171
210
|
|
|
172
211
|
def get_all_pipeline_status(self) -> List[PipelineStatus]:
|
|
173
212
|
"""
|
|
@@ -368,7 +407,7 @@ class AirflowLineageRunner:
|
|
|
368
407
|
"""
|
|
369
408
|
self.dag.log.info("Executing Airflow Lineage Runner...")
|
|
370
409
|
pipeline_service = self.get_or_create_pipeline_service()
|
|
371
|
-
pipeline = self.
|
|
410
|
+
pipeline = self.create_or_update_pipeline_entity(pipeline_service)
|
|
372
411
|
self.add_all_pipeline_status(pipeline)
|
|
373
412
|
|
|
374
413
|
self.dag.log.info(f"Processing XLet data {self.xlets}")
|
metadata/antlr/split_listener.py
CHANGED
|
@@ -35,7 +35,7 @@ class EntityLinkSplitListener(EntityLinkListener):
|
|
|
35
35
|
def __init__(self):
|
|
36
36
|
self._list = []
|
|
37
37
|
|
|
38
|
-
def
|
|
38
|
+
def enterNameOrFQN(self, ctx: EntityLinkParser.NameOrFQNContext):
|
|
39
39
|
self._list.append(ctx.getText())
|
|
40
40
|
|
|
41
41
|
def enterEntityType(self, ctx: EntityLinkParser.EntityTypeContext):
|
|
@@ -44,8 +44,5 @@ class EntityLinkSplitListener(EntityLinkListener):
|
|
|
44
44
|
def enterEntityField(self, ctx: EntityLinkParser.EntityFieldContext):
|
|
45
45
|
self._list.append(ctx.getText())
|
|
46
46
|
|
|
47
|
-
def enterEntityFqn(self, ctx: EntityLinkParser.EntityFqnContext):
|
|
48
|
-
self._list.append(ctx.getText())
|
|
49
|
-
|
|
50
47
|
def split(self):
|
|
51
48
|
return self._list
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
# Copyright 2021 Collate
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
7
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
8
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
9
|
+
# See the License for the specific language governing permissions and
|
|
10
|
+
# limitations under the License.
|
|
11
|
+
"""
|
|
12
|
+
PII application
|
|
13
|
+
"""
|
|
14
|
+
import traceback
|
|
15
|
+
from typing import Iterable, List, Optional
|
|
16
|
+
|
|
17
|
+
from metadata.generated.schema.entity.applications.configuration.external.autoTaggerAppConfig import (
|
|
18
|
+
AutoTaggerAppConfig,
|
|
19
|
+
)
|
|
20
|
+
from metadata.generated.schema.entity.data.table import Column, Table, TableData
|
|
21
|
+
from metadata.generated.schema.entity.services.ingestionPipelines.status import (
|
|
22
|
+
StackTraceError,
|
|
23
|
+
)
|
|
24
|
+
from metadata.generated.schema.metadataIngestion.application import (
|
|
25
|
+
OpenMetadataApplicationConfig,
|
|
26
|
+
)
|
|
27
|
+
from metadata.generated.schema.type.tagLabel import (
|
|
28
|
+
LabelType,
|
|
29
|
+
State,
|
|
30
|
+
TagFQN,
|
|
31
|
+
TagLabel,
|
|
32
|
+
TagSource,
|
|
33
|
+
)
|
|
34
|
+
from metadata.ingestion.models.table_metadata import ColumnTag
|
|
35
|
+
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
36
|
+
from metadata.pii.constants import PII
|
|
37
|
+
from metadata.pii.scanners.column_name_scanner import ColumnNameScanner
|
|
38
|
+
from metadata.pii.scanners.ner_scanner import NERScanner
|
|
39
|
+
from metadata.utils.logger import app_logger
|
|
40
|
+
from metadata.workflow.application import AppRunner, InvalidAppConfiguration
|
|
41
|
+
|
|
42
|
+
logger = app_logger()
|
|
43
|
+
|
|
44
|
+
DEFAULT_CONFIDENCE = 80
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class AutoTaggerApp(AppRunner):
|
|
48
|
+
"""
|
|
49
|
+
PII Application
|
|
50
|
+
You can execute it with `metadata app -c <path-to-yaml>`
|
|
51
|
+
with a YAML file like:
|
|
52
|
+
|
|
53
|
+
sourcePythonClass: metadata.applications.auto_tagger.AutoTaggerApp
|
|
54
|
+
appConfig:
|
|
55
|
+
type: AutoTagger
|
|
56
|
+
confidenceLevel: 80
|
|
57
|
+
workflowConfig:
|
|
58
|
+
loggerLevel: INFO
|
|
59
|
+
openMetadataServerConfig:
|
|
60
|
+
hostPort: http://localhost:8585/api
|
|
61
|
+
authProvider: openmetadata
|
|
62
|
+
securityConfig:
|
|
63
|
+
jwtToken: "..."
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
def __init__(self, config: OpenMetadataApplicationConfig, metadata: OpenMetadata):
|
|
67
|
+
super().__init__(config, metadata)
|
|
68
|
+
|
|
69
|
+
if not isinstance(self.app_config, AutoTaggerAppConfig):
|
|
70
|
+
raise InvalidAppConfiguration(
|
|
71
|
+
f"AutoTagger Runner expects an AutoTaggerAppConfig, we got [{config}]"
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
self._ner_scanner = None
|
|
75
|
+
self.confidence_threshold = (
|
|
76
|
+
self.app_config.confidenceLevel or DEFAULT_CONFIDENCE
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
@property
|
|
80
|
+
def name(self) -> str:
|
|
81
|
+
return "Auto Tagger"
|
|
82
|
+
|
|
83
|
+
@staticmethod
|
|
84
|
+
def build_column_tag(tag_fqn: str, column_fqn: str) -> ColumnTag:
|
|
85
|
+
"""
|
|
86
|
+
Build the tag and run the PATCH
|
|
87
|
+
"""
|
|
88
|
+
tag_label = TagLabel(
|
|
89
|
+
tagFQN=TagFQN(__root__=tag_fqn),
|
|
90
|
+
source=TagSource.Classification,
|
|
91
|
+
state=State.Suggested,
|
|
92
|
+
labelType=LabelType.Automated,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
return ColumnTag(column_fqn=column_fqn, tag_label=tag_label)
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def ner_scanner(self) -> NERScanner:
|
|
99
|
+
"""Load the NER Scanner only if called"""
|
|
100
|
+
if self._ner_scanner is None:
|
|
101
|
+
self._ner_scanner = NERScanner()
|
|
102
|
+
|
|
103
|
+
return self._ner_scanner
|
|
104
|
+
|
|
105
|
+
def process_column(
|
|
106
|
+
self,
|
|
107
|
+
idx: int,
|
|
108
|
+
column: Column,
|
|
109
|
+
table_data: Optional[TableData],
|
|
110
|
+
confidence_threshold: float,
|
|
111
|
+
) -> Optional[List[ColumnTag]]:
|
|
112
|
+
"""
|
|
113
|
+
Tag a column with PII if we find it using our scanners
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
# First, check if the column we are about to process
|
|
117
|
+
# already has PII tags or not
|
|
118
|
+
column_has_pii_tag = any(
|
|
119
|
+
(PII in tag.tagFQN.__root__ for tag in column.tags or [])
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
# If it has PII tags, we skip the processing
|
|
123
|
+
# for the column
|
|
124
|
+
if column_has_pii_tag is True:
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
# Scan by column name. If no results there, check the sample data, if any
|
|
128
|
+
tag_and_confidence = ColumnNameScanner.scan(column.name.__root__) or (
|
|
129
|
+
self.ner_scanner.scan([row[idx] for row in table_data.rows])
|
|
130
|
+
if table_data
|
|
131
|
+
else None
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
if (
|
|
135
|
+
tag_and_confidence
|
|
136
|
+
and tag_and_confidence.tag_fqn
|
|
137
|
+
and tag_and_confidence.confidence >= confidence_threshold / 100
|
|
138
|
+
):
|
|
139
|
+
# We support returning +1 tags for a single column in _run
|
|
140
|
+
return [
|
|
141
|
+
self.build_column_tag(
|
|
142
|
+
tag_fqn=tag_and_confidence.tag_fqn,
|
|
143
|
+
column_fqn=column.fullyQualifiedName.__root__,
|
|
144
|
+
)
|
|
145
|
+
]
|
|
146
|
+
|
|
147
|
+
return None
|
|
148
|
+
|
|
149
|
+
def process_table(self, table: Table) -> Optional[List[ColumnTag]]:
|
|
150
|
+
"""Run the patching of the table"""
|
|
151
|
+
column_tags = []
|
|
152
|
+
for idx, column in enumerate(table.columns):
|
|
153
|
+
try:
|
|
154
|
+
col_tags = self.process_column(
|
|
155
|
+
idx=idx,
|
|
156
|
+
column=column,
|
|
157
|
+
table_data=table.sampleData,
|
|
158
|
+
confidence_threshold=self.confidence_threshold,
|
|
159
|
+
)
|
|
160
|
+
if col_tags:
|
|
161
|
+
column_tags.extend(col_tags)
|
|
162
|
+
except Exception as err:
|
|
163
|
+
self.status.failed(
|
|
164
|
+
StackTraceError(
|
|
165
|
+
name=table.fullyQualifiedName.__root__,
|
|
166
|
+
error=f"Error computing PII tags for [{column}] - [{err}]",
|
|
167
|
+
stackTrace=traceback.format_exc(),
|
|
168
|
+
)
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
if column_tags:
|
|
172
|
+
return column_tags
|
|
173
|
+
|
|
174
|
+
return None
|
|
175
|
+
|
|
176
|
+
def patch_columns(self, table: Table, column_tags: List[ColumnTag]) -> None:
|
|
177
|
+
"""Patch columns with PII"""
|
|
178
|
+
patched = self.metadata.patch_column_tags(table=table, column_tags=column_tags)
|
|
179
|
+
if not patched:
|
|
180
|
+
self.status.failed(
|
|
181
|
+
StackTraceError(
|
|
182
|
+
name=table.fullyQualifiedName.__root__,
|
|
183
|
+
error="Error patching tags for table",
|
|
184
|
+
)
|
|
185
|
+
)
|
|
186
|
+
else:
|
|
187
|
+
self.status.scanned(table)
|
|
188
|
+
logger.debug(
|
|
189
|
+
f"Successfully patched tag {column_tags} for {table.fullyQualifiedName.__root__}"
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def run(self) -> None:
|
|
193
|
+
"""
|
|
194
|
+
The PII Application will:
|
|
195
|
+
1. List tables
|
|
196
|
+
2. Check their column names and sample data (if any)
|
|
197
|
+
3. PATCH PII tags when needed
|
|
198
|
+
"""
|
|
199
|
+
tables: Iterable[Table] = self.metadata.list_all_entities(
|
|
200
|
+
entity=Table, fields=["sampleData", "tags"]
|
|
201
|
+
)
|
|
202
|
+
for table in tables:
|
|
203
|
+
column_tags = self.process_table(table)
|
|
204
|
+
if column_tags:
|
|
205
|
+
self.patch_columns(table=table, column_tags=column_tags)
|
|
206
|
+
else:
|
|
207
|
+
self.status.filter(
|
|
208
|
+
key=table.fullyQualifiedName.__root__, reason="No PII found"
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
def close(self) -> None:
|
|
212
|
+
"""Nothing to close"""
|
metadata/cli/app.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Copyright 2021 Collate
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
7
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
8
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
9
|
+
# See the License for the specific language governing permissions and
|
|
10
|
+
# limitations under the License.
|
|
11
|
+
|
|
12
|
+
"""
|
|
13
|
+
Profiler utility for the metadata CLI
|
|
14
|
+
"""
|
|
15
|
+
import sys
|
|
16
|
+
import traceback
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
from metadata.config.common import load_config_file
|
|
20
|
+
from metadata.utils.logger import cli_logger
|
|
21
|
+
from metadata.workflow.application import ApplicationWorkflow
|
|
22
|
+
from metadata.workflow.application_output_handler import print_status
|
|
23
|
+
|
|
24
|
+
logger = cli_logger()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def run_app(config_path: Path) -> None:
|
|
28
|
+
"""
|
|
29
|
+
Run the application workflow from a config path
|
|
30
|
+
to a JSON or YAML file
|
|
31
|
+
:param config_path: Path to load JSON config
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
config_dict = None
|
|
35
|
+
try:
|
|
36
|
+
config_dict = load_config_file(config_path)
|
|
37
|
+
workflow = ApplicationWorkflow.create(config_dict)
|
|
38
|
+
logger.debug(f"Using config: {workflow.config}")
|
|
39
|
+
except Exception as exc:
|
|
40
|
+
logger.error(f"Error running the application {exc}")
|
|
41
|
+
logger.debug(traceback.format_exc())
|
|
42
|
+
sys.exit(1)
|
|
43
|
+
|
|
44
|
+
workflow.execute()
|
|
45
|
+
workflow.stop()
|
|
46
|
+
print_status(workflow)
|
|
47
|
+
workflow.raise_from_status()
|
metadata/cli/db_dump.py
CHANGED
|
@@ -123,9 +123,12 @@ def get_hash_column_name(engine: Engine, table_name: str) -> Optional[str]:
|
|
|
123
123
|
|
|
124
124
|
def run_query_iter(engine: Engine, query: str) -> Iterable[Row]:
|
|
125
125
|
"""Return a generator of rows, one row at a time, with a limit of 100 in-mem rows"""
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
126
|
+
with engine.connect() as conn:
|
|
127
|
+
result = conn.execution_options(
|
|
128
|
+
stream_results=True, max_row_buffer=100
|
|
129
|
+
).execute(text(query))
|
|
130
|
+
for row in result:
|
|
131
|
+
yield row
|
|
129
132
|
|
|
130
133
|
|
|
131
134
|
def dump_json(tables: List[str], engine: Engine, output: Path) -> None:
|
metadata/clients/aws_client.py
CHANGED
|
@@ -35,6 +35,7 @@ class AWSServices(Enum):
|
|
|
35
35
|
QUICKSIGHT = "quicksight"
|
|
36
36
|
ATHENA = "athena"
|
|
37
37
|
RDS = "rds"
|
|
38
|
+
LAKE_FORMATION = "lakeformation"
|
|
38
39
|
|
|
39
40
|
|
|
40
41
|
class AWSAssumeRoleException(Exception):
|
|
@@ -189,3 +190,6 @@ class AWSClient:
|
|
|
189
190
|
|
|
190
191
|
def get_athena_client(self):
|
|
191
192
|
return self.get_client(AWSServices.ATHENA.value)
|
|
193
|
+
|
|
194
|
+
def get_lake_formation_client(self):
|
|
195
|
+
return self.get_client(AWSServices.LAKE_FORMATION.value)
|
metadata/clients/domo_client.py
CHANGED
metadata/cmd.py
CHANGED
|
@@ -18,6 +18,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
|
18
18
|
from pathlib import Path
|
|
19
19
|
|
|
20
20
|
from metadata.__version__ import get_metadata_version
|
|
21
|
+
from metadata.cli.app import run_app
|
|
21
22
|
from metadata.cli.backup import UploadDestinationType, run_backup
|
|
22
23
|
from metadata.cli.dataquality import run_test
|
|
23
24
|
from metadata.cli.docker import BACKEND_DATABASES, DockerActions, run_docker
|
|
@@ -50,10 +51,22 @@ class MetadataCommands(Enum):
|
|
|
50
51
|
WEBHOOK = "webhook"
|
|
51
52
|
INSIGHT = "insight"
|
|
52
53
|
LINEAGE = "lineage"
|
|
54
|
+
APP = "app"
|
|
53
55
|
OPENMETADATA_IMPORTS_MIGRATION = "openmetadata_imports_migration"
|
|
54
56
|
OPENMETADATA_DAG_CONFIG_MIGRATION = "openmetadata_dag_config_migration"
|
|
55
57
|
|
|
56
58
|
|
|
59
|
+
RUN_PATH_METHODS = {
|
|
60
|
+
MetadataCommands.INGEST.value: run_ingest,
|
|
61
|
+
MetadataCommands.USAGE.value: run_usage,
|
|
62
|
+
MetadataCommands.LINEAGE.value: run_lineage,
|
|
63
|
+
MetadataCommands.INSIGHT.value: run_insight,
|
|
64
|
+
MetadataCommands.PROFILE.value: run_profiler,
|
|
65
|
+
MetadataCommands.TEST.value: run_test,
|
|
66
|
+
MetadataCommands.APP.value: run_app,
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
57
70
|
OM_IMPORTS_MIGRATION = """
|
|
58
71
|
Update DAG files generated after creating workflow in 0.11 and before.
|
|
59
72
|
In 0.12 the airflow managed API package name changed from `openmetadata` to
|
|
@@ -365,6 +378,12 @@ def get_parser(args=None):
|
|
|
365
378
|
MetadataCommands.TEST.value, help="Workflow for running test suites"
|
|
366
379
|
)
|
|
367
380
|
)
|
|
381
|
+
create_common_config_parser_args(
|
|
382
|
+
sub_parser.add_parser(
|
|
383
|
+
MetadataCommands.APP.value,
|
|
384
|
+
help="Workflow for running external applications",
|
|
385
|
+
)
|
|
386
|
+
)
|
|
368
387
|
create_openmetadata_imports_migration_args(
|
|
369
388
|
sub_parser.add_parser(
|
|
370
389
|
MetadataCommands.OPENMETADATA_IMPORTS_MIGRATION.value,
|
|
@@ -409,13 +428,14 @@ def get_parser(args=None):
|
|
|
409
428
|
return parser.parse_args(args)
|
|
410
429
|
|
|
411
430
|
|
|
412
|
-
def metadata(args=None):
|
|
431
|
+
def metadata(args=None):
|
|
413
432
|
"""
|
|
414
433
|
This method implements parsing of the arguments passed from CLI
|
|
415
434
|
"""
|
|
416
435
|
contains_args = vars(get_parser(args))
|
|
417
436
|
metadata_workflow = contains_args.get("command")
|
|
418
437
|
config_file = contains_args.get("config")
|
|
438
|
+
path = None
|
|
419
439
|
if config_file:
|
|
420
440
|
path = Path(config_file).expanduser()
|
|
421
441
|
if contains_args.get("debug"):
|
|
@@ -425,18 +445,9 @@ def metadata(args=None): # pylint: disable=too-many-branches
|
|
|
425
445
|
else:
|
|
426
446
|
set_loggers_level(logging.INFO)
|
|
427
447
|
|
|
428
|
-
if metadata_workflow
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
run_usage(config_path=path)
|
|
432
|
-
if metadata_workflow == MetadataCommands.LINEAGE.value:
|
|
433
|
-
run_lineage(config_path=path)
|
|
434
|
-
if metadata_workflow == MetadataCommands.INSIGHT.value:
|
|
435
|
-
run_insight(config_path=path)
|
|
436
|
-
if metadata_workflow == MetadataCommands.PROFILE.value:
|
|
437
|
-
run_profiler(config_path=path)
|
|
438
|
-
if metadata_workflow == MetadataCommands.TEST.value:
|
|
439
|
-
run_test(config_path=path)
|
|
448
|
+
if metadata_workflow in RUN_PATH_METHODS:
|
|
449
|
+
RUN_PATH_METHODS[metadata_workflow](path)
|
|
450
|
+
|
|
440
451
|
if metadata_workflow == MetadataCommands.BACKUP.value:
|
|
441
452
|
run_backup(
|
|
442
453
|
common_backup_obj_instance=BackupRestoreArgs(
|
|
@@ -38,10 +38,12 @@ from metadata.generated.schema.entity.data import (
|
|
|
38
38
|
table,
|
|
39
39
|
topic,
|
|
40
40
|
)
|
|
41
|
+
from metadata.generated.schema.entity.services.ingestionPipelines.status import (
|
|
42
|
+
StackTraceError,
|
|
43
|
+
)
|
|
41
44
|
from metadata.generated.schema.entity.teams.user import User
|
|
42
45
|
from metadata.generated.schema.type.entityReference import EntityReference
|
|
43
46
|
from metadata.generated.schema.type.entityReferenceList import EntityReferenceList
|
|
44
|
-
from metadata.ingestion.api.models import StackTraceError
|
|
45
47
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
46
48
|
from metadata.utils.helpers import get_entity_tier_from_tags
|
|
47
49
|
from metadata.utils.logger import data_insight_logger
|
|
@@ -75,6 +77,10 @@ class EntityReportDataProcessor(DataProcessor):
|
|
|
75
77
|
self._refined_data = defaultdict(lambda: defaultdict(lambda: defaultdict(dict)))
|
|
76
78
|
self.post_hook = self._post_hook_fn
|
|
77
79
|
|
|
80
|
+
@property
|
|
81
|
+
def name(self) -> str:
|
|
82
|
+
return "Entity Report Processor"
|
|
83
|
+
|
|
78
84
|
def _get_team(self, owner: EntityReference) -> Optional[str]:
|
|
79
85
|
"""Get the team from an entity. We'll use this info as well to
|
|
80
86
|
add info if an entity has an owner
|
|
@@ -191,7 +197,7 @@ class EntityReportDataProcessor(DataProcessor):
|
|
|
191
197
|
StackTraceError(
|
|
192
198
|
name=entity.name.__root__,
|
|
193
199
|
error="Error retrieving team",
|
|
194
|
-
|
|
200
|
+
stackTrace=traceback.format_exc(),
|
|
195
201
|
)
|
|
196
202
|
)
|
|
197
203
|
raise
|
|
@@ -78,6 +78,10 @@ class WebAnalyticEntityViewReportDataProcessor(DataProcessor):
|
|
|
78
78
|
super().__init__(metadata)
|
|
79
79
|
self.pre_hook = self._pre_hook_fn
|
|
80
80
|
|
|
81
|
+
@property
|
|
82
|
+
def name(self) -> str:
|
|
83
|
+
return "Web Analytics Processor"
|
|
84
|
+
|
|
81
85
|
def _pre_hook_fn(self):
|
|
82
86
|
"""Start our generator function"""
|
|
83
87
|
# pylint: disable=attribute-defined-outside-init
|
|
@@ -33,7 +33,10 @@ from metadata.data_insight.processor.reports.web_analytic_report_data_processor
|
|
|
33
33
|
)
|
|
34
34
|
from metadata.data_insight.producer.producer_factory import producer_factory
|
|
35
35
|
from metadata.generated.schema.analytics.reportData import ReportData, ReportDataType
|
|
36
|
-
from metadata.
|
|
36
|
+
from metadata.generated.schema.entity.services.ingestionPipelines.status import (
|
|
37
|
+
StackTraceError,
|
|
38
|
+
)
|
|
39
|
+
from metadata.ingestion.api.models import Either
|
|
37
40
|
from metadata.ingestion.api.step import Step
|
|
38
41
|
from metadata.ingestion.api.steps import Source
|
|
39
42
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
@@ -87,9 +90,13 @@ class DataInsightSource(Source):
|
|
|
87
90
|
_processors
|
|
88
91
|
) # make an immutable copy of the dict
|
|
89
92
|
|
|
93
|
+
@property
|
|
94
|
+
def name(self) -> str:
|
|
95
|
+
return "OpenMetadata Insights"
|
|
96
|
+
|
|
90
97
|
@property
|
|
91
98
|
def processors(self) -> Dict[str, Optional[DataProcessor]]:
|
|
92
|
-
"""
|
|
99
|
+
"""dictionary of processors"""
|
|
93
100
|
return self._processors
|
|
94
101
|
|
|
95
102
|
def _instantiate_processors(self) -> Dict[str, DataProcessor]:
|
|
@@ -136,7 +143,7 @@ class DataInsightSource(Source):
|
|
|
136
143
|
name=report_data_type.value,
|
|
137
144
|
error=f"Error retrieving processor with exception [{key_error}]."
|
|
138
145
|
"Available processors are {self.processors}",
|
|
139
|
-
|
|
146
|
+
stackTrace=traceback.format_exc(),
|
|
140
147
|
),
|
|
141
148
|
right=None,
|
|
142
149
|
)
|
|
@@ -145,7 +152,7 @@ class DataInsightSource(Source):
|
|
|
145
152
|
left=StackTraceError(
|
|
146
153
|
name=report_data_type.value,
|
|
147
154
|
error=f"Error listing data for report with exception [{exc}]",
|
|
148
|
-
|
|
155
|
+
stackTrace=traceback.format_exc(),
|
|
149
156
|
),
|
|
150
157
|
right=None,
|
|
151
158
|
)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Copyright 2021 Collate
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
7
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
8
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
9
|
+
# See the License for the specific language governing permissions and
|
|
10
|
+
# limitations under the License.
|
|
11
|
+
|
|
12
|
+
"""
|
|
13
|
+
Interfaces with database for all database engine
|
|
14
|
+
supporting sqlalchemy abstraction layer
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import (
|
|
19
|
+
SQATestSuiteInterface,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class DatabricksTestSuiteInterface(SQATestSuiteInterface):
|
|
24
|
+
def __init__(self, *args, **kwargs):
|
|
25
|
+
super().__init__(*args, **kwargs)
|
|
26
|
+
|
|
27
|
+
def create_session(self):
|
|
28
|
+
super().create_session()
|
|
29
|
+
self.set_catalog(self.session)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Copyright 2021 Collate
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
6
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
7
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
8
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
9
|
+
# See the License for the specific language governing permissions and
|
|
10
|
+
# limitations under the License.
|
|
11
|
+
|
|
12
|
+
"""
|
|
13
|
+
Interfaces with database for all database engine
|
|
14
|
+
supporting sqlalchemy abstraction layer
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
from metadata.data_quality.interface.sqlalchemy.sqa_test_suite_interface import (
|
|
19
|
+
SQATestSuiteInterface,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class SnowflakeTestSuiteInterface(SQATestSuiteInterface):
|
|
24
|
+
def __init__(self, *args, **kwargs):
|
|
25
|
+
super().__init__(*args, **kwargs)
|
|
26
|
+
|
|
27
|
+
def create_session(self):
|
|
28
|
+
super().create_session()
|
|
29
|
+
self.set_session_tag(self.session)
|
|
@@ -58,12 +58,7 @@ class SQATestSuiteInterface(SQAInterfaceMixin, TestSuiteInterface):
|
|
|
58
58
|
self.ometa_client = ometa_client
|
|
59
59
|
self.table_entity = table_entity
|
|
60
60
|
self.service_connection_config = service_connection_config
|
|
61
|
-
self.
|
|
62
|
-
get_connection(self.service_connection_config)
|
|
63
|
-
)
|
|
64
|
-
self.set_session_tag(self.session)
|
|
65
|
-
self.set_catalog(self.session)
|
|
66
|
-
|
|
61
|
+
self.create_session()
|
|
67
62
|
self._table = self._convert_table_to_orm_object(sqa_metadata)
|
|
68
63
|
|
|
69
64
|
(
|
|
@@ -75,6 +70,11 @@ class SQATestSuiteInterface(SQAInterfaceMixin, TestSuiteInterface):
|
|
|
75
70
|
self._sampler = self._create_sampler()
|
|
76
71
|
self._runner = self._create_runner()
|
|
77
72
|
|
|
73
|
+
def create_session(self):
|
|
74
|
+
self.session = create_and_bind_session(
|
|
75
|
+
get_connection(self.service_connection_config)
|
|
76
|
+
)
|
|
77
|
+
|
|
78
78
|
@property
|
|
79
79
|
def sample(self) -> Union[DeclarativeMeta, AliasedClass]:
|
|
80
80
|
"""_summary_
|