openmetadata-ingestion 1.5.7.1.dev2__py3-none-any.whl → 1.5.8.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openmetadata-ingestion might be problematic. Click here for more details.
- metadata/generated/schema/analytics/__init__.py +1 -1
- metadata/generated/schema/analytics/basic.py +1 -1
- metadata/generated/schema/analytics/reportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
- metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
- metadata/generated/schema/api/__init__.py +1 -1
- metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
- metadata/generated/schema/api/analytics/__init__.py +1 -1
- metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
- metadata/generated/schema/api/automations/__init__.py +1 -1
- metadata/generated/schema/api/automations/createWorkflow.py +1 -1
- metadata/generated/schema/api/bulkAssets.py +1 -1
- metadata/generated/schema/api/classification/__init__.py +1 -1
- metadata/generated/schema/api/classification/createClassification.py +1 -1
- metadata/generated/schema/api/classification/createTag.py +1 -1
- metadata/generated/schema/api/classification/loadTags.py +1 -1
- metadata/generated/schema/api/createBot.py +1 -1
- metadata/generated/schema/api/createEventPublisherJob.py +1 -1
- metadata/generated/schema/api/createType.py +1 -1
- metadata/generated/schema/api/data/__init__.py +1 -1
- metadata/generated/schema/api/data/createAPICollection.py +1 -1
- metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
- metadata/generated/schema/api/data/createChart.py +1 -1
- metadata/generated/schema/api/data/createContainer.py +1 -1
- metadata/generated/schema/api/data/createCustomProperty.py +1 -1
- metadata/generated/schema/api/data/createDashboard.py +1 -1
- metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
- metadata/generated/schema/api/data/createDatabase.py +1 -1
- metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
- metadata/generated/schema/api/data/createGlossary.py +1 -1
- metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
- metadata/generated/schema/api/data/createMlModel.py +1 -1
- metadata/generated/schema/api/data/createPipeline.py +1 -1
- metadata/generated/schema/api/data/createQuery.py +1 -1
- metadata/generated/schema/api/data/createSearchIndex.py +1 -1
- metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
- metadata/generated/schema/api/data/createTable.py +8 -1
- metadata/generated/schema/api/data/createTableProfile.py +1 -1
- metadata/generated/schema/api/data/createTopic.py +1 -1
- metadata/generated/schema/api/data/loadGlossary.py +1 -1
- metadata/generated/schema/api/data/restoreEntity.py +1 -1
- metadata/generated/schema/api/dataInsight/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
- metadata/generated/schema/api/docStore/__init__.py +1 -1
- metadata/generated/schema/api/docStore/createDocument.py +1 -1
- metadata/generated/schema/api/domains/__init__.py +1 -1
- metadata/generated/schema/api/domains/createDataProduct.py +1 -1
- metadata/generated/schema/api/domains/createDomain.py +1 -1
- metadata/generated/schema/api/feed/__init__.py +1 -1
- metadata/generated/schema/api/feed/closeTask.py +1 -1
- metadata/generated/schema/api/feed/createPost.py +1 -1
- metadata/generated/schema/api/feed/createSuggestion.py +1 -1
- metadata/generated/schema/api/feed/createThread.py +1 -1
- metadata/generated/schema/api/feed/resolveTask.py +1 -1
- metadata/generated/schema/api/feed/threadCount.py +1 -1
- metadata/generated/schema/api/lineage/__init__.py +1 -1
- metadata/generated/schema/api/lineage/addLineage.py +1 -1
- metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
- metadata/generated/schema/api/policies/__init__.py +1 -1
- metadata/generated/schema/api/policies/createPolicy.py +1 -1
- metadata/generated/schema/api/services/__init__.py +1 -1
- metadata/generated/schema/api/services/createApiService.py +1 -1
- metadata/generated/schema/api/services/createDashboardService.py +1 -1
- metadata/generated/schema/api/services/createDatabaseService.py +1 -1
- metadata/generated/schema/api/services/createMessagingService.py +1 -1
- metadata/generated/schema/api/services/createMetadataService.py +1 -1
- metadata/generated/schema/api/services/createMlModelService.py +1 -1
- metadata/generated/schema/api/services/createPipelineService.py +1 -1
- metadata/generated/schema/api/services/createSearchService.py +1 -1
- metadata/generated/schema/api/services/createStorageService.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
- metadata/generated/schema/api/setOwner.py +1 -1
- metadata/generated/schema/api/teams/__init__.py +1 -1
- metadata/generated/schema/api/teams/createPersona.py +1 -1
- metadata/generated/schema/api/teams/createRole.py +1 -1
- metadata/generated/schema/api/teams/createTeam.py +1 -1
- metadata/generated/schema/api/teams/createUser.py +1 -1
- metadata/generated/schema/api/tests/__init__.py +1 -1
- metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
- metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
- metadata/generated/schema/api/tests/createTestCase.py +1 -1
- metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
- metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
- metadata/generated/schema/api/tests/createTestSuite.py +1 -1
- metadata/generated/schema/api/voteRequest.py +1 -1
- metadata/generated/schema/auth/__init__.py +1 -1
- metadata/generated/schema/auth/basicAuth.py +1 -1
- metadata/generated/schema/auth/basicLoginRequest.py +1 -1
- metadata/generated/schema/auth/changePasswordRequest.py +1 -1
- metadata/generated/schema/auth/createPersonalToken.py +1 -1
- metadata/generated/schema/auth/emailRequest.py +1 -1
- metadata/generated/schema/auth/emailVerificationToken.py +1 -1
- metadata/generated/schema/auth/generateToken.py +1 -1
- metadata/generated/schema/auth/jwtAuth.py +1 -1
- metadata/generated/schema/auth/loginRequest.py +1 -1
- metadata/generated/schema/auth/logoutRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetToken.py +1 -1
- metadata/generated/schema/auth/personalAccessToken.py +1 -1
- metadata/generated/schema/auth/refreshToken.py +1 -1
- metadata/generated/schema/auth/registrationRequest.py +1 -1
- metadata/generated/schema/auth/revokePersonalToken.py +1 -1
- metadata/generated/schema/auth/revokeToken.py +1 -1
- metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
- metadata/generated/schema/auth/ssoAuth.py +1 -1
- metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
- metadata/generated/schema/configuration/__init__.py +1 -1
- metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
- metadata/generated/schema/configuration/authConfig.py +1 -1
- metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
- metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
- metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
- metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
- metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
- metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
- metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
- metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
- metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
- metadata/generated/schema/configuration/loginConfiguration.py +1 -1
- metadata/generated/schema/configuration/logoConfiguration.py +1 -1
- metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
- metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
- metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
- metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/themeConfiguration.py +1 -1
- metadata/generated/schema/configuration/uiThemePreference.py +1 -1
- metadata/generated/schema/dataInsight/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
- metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
- metadata/generated/schema/dataInsight/custom/lineChart.py +1 -1
- metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
- metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
- metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
- metadata/generated/schema/dataInsight/type/__init__.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
- metadata/generated/schema/email/__init__.py +1 -1
- metadata/generated/schema/email/emailRequest.py +1 -1
- metadata/generated/schema/email/emailTemplate.py +1 -1
- metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
- metadata/generated/schema/email/smtpSettings.py +1 -1
- metadata/generated/schema/email/templateValidationReponse.py +1 -1
- metadata/generated/schema/entity/__init__.py +1 -1
- metadata/generated/schema/entity/applications/__init__.py +1 -1
- metadata/generated/schema/entity/applications/app.py +1 -1
- metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
- metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/applicationConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/metaPilotAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/external/metaPilotAppPrivateConfig.py +1 -1
- metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
- metadata/generated/schema/entity/applications/jobStatus.py +1 -1
- metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +1 -1
- metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
- metadata/generated/schema/entity/automations/__init__.py +1 -1
- metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
- metadata/generated/schema/entity/automations/workflow.py +1 -1
- metadata/generated/schema/entity/bot.py +1 -1
- metadata/generated/schema/entity/classification/__init__.py +1 -1
- metadata/generated/schema/entity/classification/classification.py +1 -1
- metadata/generated/schema/entity/classification/tag.py +1 -1
- metadata/generated/schema/entity/data/__init__.py +1 -1
- metadata/generated/schema/entity/data/apiCollection.py +1 -1
- metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
- metadata/generated/schema/entity/data/chart.py +1 -1
- metadata/generated/schema/entity/data/container.py +1 -1
- metadata/generated/schema/entity/data/dashboard.py +1 -1
- metadata/generated/schema/entity/data/dashboardDataModel.py +2 -1
- metadata/generated/schema/entity/data/database.py +1 -1
- metadata/generated/schema/entity/data/databaseSchema.py +1 -1
- metadata/generated/schema/entity/data/glossary.py +1 -1
- metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
- metadata/generated/schema/entity/data/metrics.py +1 -1
- metadata/generated/schema/entity/data/mlmodel.py +1 -1
- metadata/generated/schema/entity/data/pipeline.py +1 -1
- metadata/generated/schema/entity/data/query.py +1 -1
- metadata/generated/schema/entity/data/report.py +1 -1
- metadata/generated/schema/entity/data/searchIndex.py +1 -1
- metadata/generated/schema/entity/data/storedProcedure.py +1 -1
- metadata/generated/schema/entity/data/table.py +8 -1
- metadata/generated/schema/entity/data/topic.py +1 -1
- metadata/generated/schema/entity/docStore/__init__.py +1 -1
- metadata/generated/schema/entity/docStore/document.py +1 -1
- metadata/generated/schema/entity/domains/__init__.py +1 -1
- metadata/generated/schema/entity/domains/dataProduct.py +1 -1
- metadata/generated/schema/entity/domains/domain.py +1 -1
- metadata/generated/schema/entity/events/__init__.py +1 -1
- metadata/generated/schema/entity/events/webhook.py +1 -1
- metadata/generated/schema/entity/feed/__init__.py +1 -1
- metadata/generated/schema/entity/feed/assets.py +1 -1
- metadata/generated/schema/entity/feed/customProperty.py +1 -1
- metadata/generated/schema/entity/feed/description.py +1 -1
- metadata/generated/schema/entity/feed/domain.py +1 -1
- metadata/generated/schema/entity/feed/entityInfo.py +1 -1
- metadata/generated/schema/entity/feed/owner.py +1 -1
- metadata/generated/schema/entity/feed/suggestion.py +1 -1
- metadata/generated/schema/entity/feed/tag.py +1 -1
- metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
- metadata/generated/schema/entity/feed/thread.py +1 -1
- metadata/generated/schema/entity/policies/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
- metadata/generated/schema/entity/policies/filters.py +1 -1
- metadata/generated/schema/entity/policies/policy.py +1 -1
- metadata/generated/schema/entity/services/__init__.py +1 -1
- metadata/generated/schema/entity/services/apiService.py +1 -1
- metadata/generated/schema/entity/services/connections/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/apiService/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/apiService/restConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +9 -1
- metadata/generated/schema/entity/services/connections/dashboard/mstrConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/athenaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/db2Connection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/druidConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/hiveConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/impalaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/oracleConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/postgresConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/teradataConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/trinoConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
- metadata/generated/schema/entity/services/dashboardService.py +1 -1
- metadata/generated/schema/entity/services/databaseService.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
- metadata/generated/schema/entity/services/messagingService.py +1 -1
- metadata/generated/schema/entity/services/metadataService.py +1 -1
- metadata/generated/schema/entity/services/mlmodelService.py +1 -1
- metadata/generated/schema/entity/services/pipelineService.py +1 -1
- metadata/generated/schema/entity/services/searchService.py +1 -1
- metadata/generated/schema/entity/services/serviceType.py +1 -1
- metadata/generated/schema/entity/services/storageService.py +1 -1
- metadata/generated/schema/entity/teams/__init__.py +1 -1
- metadata/generated/schema/entity/teams/persona.py +1 -1
- metadata/generated/schema/entity/teams/role.py +1 -1
- metadata/generated/schema/entity/teams/team.py +1 -1
- metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
- metadata/generated/schema/entity/teams/user.py +1 -1
- metadata/generated/schema/entity/type.py +1 -1
- metadata/generated/schema/entity/utils/__init__.py +1 -1
- metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
- metadata/generated/schema/entity/utils/servicesCount.py +1 -1
- metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
- metadata/generated/schema/events/__init__.py +1 -1
- metadata/generated/schema/events/alertMetrics.py +1 -1
- metadata/generated/schema/events/api/__init__.py +1 -1
- metadata/generated/schema/events/api/createEventSubscription.py +1 -1
- metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
- metadata/generated/schema/events/emailAlertConfig.py +1 -1
- metadata/generated/schema/events/eventFilterRule.py +1 -1
- metadata/generated/schema/events/eventSubscription.py +1 -1
- metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
- metadata/generated/schema/events/failedEvent.py +1 -1
- metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
- metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
- metadata/generated/schema/metadataIngestion/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/application.py +1 -1
- metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/workflow.py +1 -1
- metadata/generated/schema/monitoring/__init__.py +1 -1
- metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
- metadata/generated/schema/security/__init__.py +1 -1
- metadata/generated/schema/security/client/__init__.py +1 -1
- metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
- metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
- metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
- metadata/generated/schema/security/credentials/__init__.py +1 -1
- metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
- metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
- metadata/generated/schema/security/credentials/basicAuth.py +1 -1
- metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
- metadata/generated/schema/security/credentials/gcpValues.py +1 -1
- metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
- metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
- metadata/generated/schema/security/sasl/__init__.py +1 -1
- metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
- metadata/generated/schema/security/secrets/__init__.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
- metadata/generated/schema/security/securityConfiguration.py +1 -1
- metadata/generated/schema/security/ssl/__init__.py +1 -1
- metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
- metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
- metadata/generated/schema/settings/__init__.py +1 -1
- metadata/generated/schema/settings/settings.py +1 -1
- metadata/generated/schema/system/__init__.py +1 -1
- metadata/generated/schema/system/entityError.py +1 -1
- metadata/generated/schema/system/eventPublisherJob.py +1 -1
- metadata/generated/schema/system/indexingError.py +1 -1
- metadata/generated/schema/system/limitsResponse.py +1 -1
- metadata/generated/schema/system/ui/__init__.py +1 -1
- metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
- metadata/generated/schema/system/ui/page.py +1 -1
- metadata/generated/schema/system/validationResponse.py +1 -1
- metadata/generated/schema/tests/__init__.py +1 -1
- metadata/generated/schema/tests/assigned.py +1 -1
- metadata/generated/schema/tests/basic.py +1 -1
- metadata/generated/schema/tests/customMetric.py +1 -1
- metadata/generated/schema/tests/dataQualityReport.py +1 -1
- metadata/generated/schema/tests/resolved.py +1 -1
- metadata/generated/schema/tests/testCase.py +1 -1
- metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
- metadata/generated/schema/tests/testDefinition.py +1 -1
- metadata/generated/schema/tests/testSuite.py +1 -1
- metadata/generated/schema/type/__init__.py +1 -1
- metadata/generated/schema/type/apiSchema.py +1 -1
- metadata/generated/schema/type/auditLog.py +1 -1
- metadata/generated/schema/type/basic.py +1 -1
- metadata/generated/schema/type/bulkOperationResult.py +1 -1
- metadata/generated/schema/type/changeEvent.py +1 -1
- metadata/generated/schema/type/changeEventType.py +1 -1
- metadata/generated/schema/type/collectionDescriptor.py +1 -1
- metadata/generated/schema/type/csvDocumentation.py +1 -1
- metadata/generated/schema/type/csvErrorType.py +1 -1
- metadata/generated/schema/type/csvFile.py +1 -1
- metadata/generated/schema/type/csvImportResult.py +1 -1
- metadata/generated/schema/type/customProperties/__init__.py +1 -1
- metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
- metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
- metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
- metadata/generated/schema/type/customProperty.py +1 -1
- metadata/generated/schema/type/dailyCount.py +1 -1
- metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
- metadata/generated/schema/type/entityHierarchy.py +1 -1
- metadata/generated/schema/type/entityHistory.py +1 -1
- metadata/generated/schema/type/entityLineage.py +1 -1
- metadata/generated/schema/type/entityReference.py +1 -1
- metadata/generated/schema/type/entityReferenceList.py +1 -1
- metadata/generated/schema/type/entityRelationship.py +1 -1
- metadata/generated/schema/type/entityUsage.py +1 -1
- metadata/generated/schema/type/filterPattern.py +1 -1
- metadata/generated/schema/type/function.py +1 -1
- metadata/generated/schema/type/include.py +1 -1
- metadata/generated/schema/type/jdbcConnection.py +1 -1
- metadata/generated/schema/type/lifeCycle.py +1 -1
- metadata/generated/schema/type/paging.py +1 -1
- metadata/generated/schema/type/profile.py +1 -1
- metadata/generated/schema/type/queryParserData.py +1 -1
- metadata/generated/schema/type/reaction.py +1 -1
- metadata/generated/schema/type/schedule.py +1 -1
- metadata/generated/schema/type/schema.py +1 -1
- metadata/generated/schema/type/tableQuery.py +1 -1
- metadata/generated/schema/type/tableUsageCount.py +1 -1
- metadata/generated/schema/type/tagLabel.py +1 -1
- metadata/generated/schema/type/usageDetails.py +1 -1
- metadata/generated/schema/type/usageRequest.py +1 -1
- metadata/generated/schema/type/votes.py +1 -1
- metadata/ingestion/connections/test_connections.py +66 -40
- metadata/ingestion/source/dashboard/dashboard_service.py +7 -1
- metadata/ingestion/source/dashboard/domodashboard/connection.py +8 -3
- metadata/ingestion/source/dashboard/lightdash/connection.py +8 -3
- metadata/ingestion/source/dashboard/looker/connection.py +8 -2
- metadata/ingestion/source/dashboard/metabase/connection.py +8 -2
- metadata/ingestion/source/dashboard/mode/client.py +29 -2
- metadata/ingestion/source/dashboard/mode/connection.py +9 -3
- metadata/ingestion/source/dashboard/mode/metadata.py +4 -1
- metadata/ingestion/source/dashboard/mstr/client.py +2 -3
- metadata/ingestion/source/dashboard/mstr/connection.py +8 -2
- metadata/ingestion/source/dashboard/powerbi/connection.py +8 -2
- metadata/ingestion/source/dashboard/qlikcloud/connection.py +8 -2
- metadata/ingestion/source/dashboard/qliksense/connection.py +8 -2
- metadata/ingestion/source/dashboard/quicksight/connection.py +8 -2
- metadata/ingestion/source/dashboard/quicksight/metadata.py +363 -81
- metadata/ingestion/source/dashboard/quicksight/models.py +19 -1
- metadata/ingestion/source/dashboard/redash/connection.py +8 -2
- metadata/ingestion/source/dashboard/superset/connection.py +8 -2
- metadata/ingestion/source/dashboard/tableau/connection.py +8 -2
- metadata/ingestion/source/database/athena/connection.py +8 -2
- metadata/ingestion/source/database/azuresql/connection.py +8 -2
- metadata/ingestion/source/database/bigquery/connection.py +9 -3
- metadata/ingestion/source/database/bigtable/connection.py +8 -2
- metadata/ingestion/source/database/clickhouse/connection.py +8 -2
- metadata/ingestion/source/database/common_db_source.py +9 -0
- metadata/ingestion/source/database/common_nosql_source.py +4 -17
- metadata/ingestion/source/database/couchbase/connection.py +8 -2
- metadata/ingestion/source/database/couchbase/metadata.py +12 -128
- metadata/ingestion/source/database/couchbase/queries.py +1 -6
- metadata/ingestion/source/database/database_service.py +7 -1
- metadata/ingestion/source/database/databricks/connection.py +8 -3
- metadata/ingestion/source/database/databricks/metadata.py +19 -0
- metadata/ingestion/source/database/datalake/connection.py +8 -2
- metadata/ingestion/source/database/db2/connection.py +8 -2
- metadata/ingestion/source/database/deltalake/connection.py +8 -2
- metadata/ingestion/source/database/domodatabase/connection.py +8 -2
- metadata/ingestion/source/database/doris/connection.py +8 -2
- metadata/ingestion/source/database/druid/connection.py +8 -2
- metadata/ingestion/source/database/dynamodb/connection.py +8 -2
- metadata/ingestion/source/database/glue/connection.py +8 -2
- metadata/ingestion/source/database/glue/metadata.py +1 -11
- metadata/ingestion/source/database/glue/models.py +0 -1
- metadata/ingestion/source/database/greenplum/connection.py +8 -2
- metadata/ingestion/source/database/hive/connection.py +25 -4
- metadata/ingestion/source/database/iceberg/connection.py +8 -2
- metadata/ingestion/source/database/impala/connection.py +8 -2
- metadata/ingestion/source/database/mariadb/connection.py +8 -2
- metadata/ingestion/source/database/mongodb/connection.py +8 -2
- metadata/ingestion/source/database/mssql/connection.py +8 -2
- metadata/ingestion/source/database/mysql/connection.py +8 -2
- metadata/ingestion/source/database/oracle/connection.py +8 -2
- metadata/ingestion/source/database/pinotdb/connection.py +8 -2
- metadata/ingestion/source/database/postgres/connection.py +8 -2
- metadata/ingestion/source/database/presto/connection.py +8 -2
- metadata/ingestion/source/database/query_parser_source.py +5 -1
- metadata/ingestion/source/database/redshift/connection.py +10 -2
- metadata/ingestion/source/database/salesforce/connection.py +8 -2
- metadata/ingestion/source/database/salesforce/metadata.py +5 -1
- metadata/ingestion/source/database/saperp/connection.py +8 -2
- metadata/ingestion/source/database/saphana/connection.py +8 -2
- metadata/ingestion/source/database/saphana/lineage.py +5 -1
- metadata/ingestion/source/database/sas/connection.py +8 -2
- metadata/ingestion/source/database/sas/metadata.py +7 -1
- metadata/ingestion/source/database/singlestore/connection.py +8 -2
- metadata/ingestion/source/database/snowflake/connection.py +8 -2
- metadata/ingestion/source/database/sqlite/connection.py +8 -2
- metadata/ingestion/source/database/teradata/connection.py +8 -2
- metadata/ingestion/source/database/trino/connection.py +8 -2
- metadata/ingestion/source/database/unitycatalog/connection.py +8 -3
- metadata/ingestion/source/database/unitycatalog/lineage.py +7 -1
- metadata/ingestion/source/database/vertica/connection.py +8 -2
- metadata/ingestion/source/messaging/kafka/connection.py +8 -2
- metadata/ingestion/source/messaging/kinesis/connection.py +8 -2
- metadata/ingestion/source/messaging/messaging_service.py +7 -1
- metadata/ingestion/source/messaging/redpanda/connection.py +8 -2
- metadata/ingestion/source/metadata/alationsink/connection.py +8 -2
- metadata/ingestion/source/metadata/alationsink/metadata.py +5 -1
- metadata/ingestion/source/metadata/amundsen/connection.py +8 -2
- metadata/ingestion/source/metadata/amundsen/metadata.py +7 -1
- metadata/ingestion/source/metadata/atlas/connection.py +8 -2
- metadata/ingestion/source/metadata/atlas/metadata.py +7 -1
- metadata/ingestion/source/mlmodel/mlflow/connection.py +8 -2
- metadata/ingestion/source/mlmodel/mlmodel_service.py +7 -1
- metadata/ingestion/source/mlmodel/sagemaker/connection.py +8 -2
- metadata/ingestion/source/pipeline/airbyte/connection.py +8 -2
- metadata/ingestion/source/pipeline/airflow/connection.py +8 -2
- metadata/ingestion/source/pipeline/airflow/metadata.py +14 -2
- metadata/ingestion/source/pipeline/dagster/connection.py +8 -2
- metadata/ingestion/source/pipeline/databrickspipeline/connection.py +8 -2
- metadata/ingestion/source/pipeline/dbtcloud/connection.py +8 -2
- metadata/ingestion/source/pipeline/domopipeline/connection.py +8 -2
- metadata/ingestion/source/pipeline/fivetran/connection.py +8 -2
- metadata/ingestion/source/pipeline/flink/connection.py +8 -2
- metadata/ingestion/source/pipeline/gluepipeline/connection.py +8 -2
- metadata/ingestion/source/pipeline/kafkaconnect/connection.py +8 -2
- metadata/ingestion/source/pipeline/nifi/connection.py +8 -2
- metadata/ingestion/source/pipeline/openlineage/connection.py +8 -2
- metadata/ingestion/source/pipeline/pipeline_service.py +7 -1
- metadata/ingestion/source/pipeline/spline/connection.py +8 -2
- metadata/ingestion/source/search/elasticsearch/connection.py +8 -3
- metadata/ingestion/source/search/search_service.py +7 -1
- metadata/ingestion/source/storage/gcs/connection.py +8 -2
- metadata/ingestion/source/storage/s3/connection.py +8 -2
- metadata/ingestion/source/storage/storage_service.py +7 -1
- metadata/utils/constants.py +1 -0
- metadata/utils/timeout.py +6 -1
- metadata/workflow/profiler.py +6 -2
- {openmetadata_ingestion-1.5.7.1.dev2.dist-info → openmetadata_ingestion-1.5.8.0.dev0.dist-info}/METADATA +347 -347
- {openmetadata_ingestion-1.5.7.1.dev2.dist-info → openmetadata_ingestion-1.5.8.0.dev0.dist-info}/RECORD +695 -696
- metadata/ingestion/source/database/couchbase/models.py +0 -31
- {openmetadata_ingestion-1.5.7.1.dev2.dist-info → openmetadata_ingestion-1.5.8.0.dev0.dist-info}/LICENSE +0 -0
- {openmetadata_ingestion-1.5.7.1.dev2.dist-info → openmetadata_ingestion-1.5.8.0.dev0.dist-info}/WHEEL +0 -0
- {openmetadata_ingestion-1.5.7.1.dev2.dist-info → openmetadata_ingestion-1.5.8.0.dev0.dist-info}/entry_points.txt +0 -0
- {openmetadata_ingestion-1.5.7.1.dev2.dist-info → openmetadata_ingestion-1.5.8.0.dev0.dist-info}/top_level.txt +0 -0
|
@@ -27,6 +27,9 @@ from metadata.generated.schema.entity.automations.workflow import (
|
|
|
27
27
|
from metadata.generated.schema.entity.services.connections.database.athenaConnection import (
|
|
28
28
|
AthenaConnection,
|
|
29
29
|
)
|
|
30
|
+
from metadata.generated.schema.entity.services.connections.testConnectionResult import (
|
|
31
|
+
TestConnectionResult,
|
|
32
|
+
)
|
|
30
33
|
from metadata.ingestion.connections.builders import (
|
|
31
34
|
create_generic_db_connection,
|
|
32
35
|
get_connection_args_common,
|
|
@@ -37,6 +40,7 @@ from metadata.ingestion.connections.test_connections import (
|
|
|
37
40
|
test_connection_steps,
|
|
38
41
|
)
|
|
39
42
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
43
|
+
from metadata.utils.constants import THREE_MIN
|
|
40
44
|
|
|
41
45
|
|
|
42
46
|
def get_connection_url(connection: AthenaConnection) -> str:
|
|
@@ -94,7 +98,8 @@ def test_connection(
|
|
|
94
98
|
engine: Engine,
|
|
95
99
|
service_connection: AthenaConnection,
|
|
96
100
|
automation_workflow: Optional[AutomationWorkflow] = None,
|
|
97
|
-
|
|
101
|
+
timeout_seconds: Optional[int] = THREE_MIN,
|
|
102
|
+
) -> TestConnectionResult:
|
|
98
103
|
"""
|
|
99
104
|
Test connection. This can be executed either as part
|
|
100
105
|
of a metadata workflow or during an Automation Workflow
|
|
@@ -121,9 +126,10 @@ def test_connection(
|
|
|
121
126
|
"GetViews": custom_executor_for_view,
|
|
122
127
|
}
|
|
123
128
|
|
|
124
|
-
test_connection_steps(
|
|
129
|
+
return test_connection_steps(
|
|
125
130
|
metadata=metadata,
|
|
126
131
|
test_fn=test_fn,
|
|
127
132
|
service_type=service_connection.type.value,
|
|
128
133
|
automation_workflow=automation_workflow,
|
|
134
|
+
timeout_seconds=timeout_seconds,
|
|
129
135
|
)
|
|
@@ -28,6 +28,9 @@ from metadata.generated.schema.entity.services.connections.database.azureSQLConn
|
|
|
28
28
|
from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
|
|
29
29
|
MssqlConnection,
|
|
30
30
|
)
|
|
31
|
+
from metadata.generated.schema.entity.services.connections.testConnectionResult import (
|
|
32
|
+
TestConnectionResult,
|
|
33
|
+
)
|
|
31
34
|
from metadata.ingestion.connections.builders import (
|
|
32
35
|
create_generic_db_connection,
|
|
33
36
|
get_connection_args_common,
|
|
@@ -35,6 +38,7 @@ from metadata.ingestion.connections.builders import (
|
|
|
35
38
|
)
|
|
36
39
|
from metadata.ingestion.connections.test_connections import test_connection_db_common
|
|
37
40
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
41
|
+
from metadata.utils.constants import THREE_MIN
|
|
38
42
|
|
|
39
43
|
|
|
40
44
|
def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) -> str:
|
|
@@ -105,14 +109,16 @@ def test_connection(
|
|
|
105
109
|
engine: Engine,
|
|
106
110
|
service_connection: AzureSQLConnection,
|
|
107
111
|
automation_workflow: Optional[AutomationWorkflow] = None,
|
|
108
|
-
|
|
112
|
+
timeout_seconds: Optional[int] = THREE_MIN,
|
|
113
|
+
) -> TestConnectionResult:
|
|
109
114
|
"""
|
|
110
115
|
Test connection. This can be executed either as part
|
|
111
116
|
of a metadata workflow or during an Automation Workflow
|
|
112
117
|
"""
|
|
113
|
-
test_connection_db_common(
|
|
118
|
+
return test_connection_db_common(
|
|
114
119
|
metadata=metadata,
|
|
115
120
|
engine=engine,
|
|
116
121
|
service_connection=service_connection,
|
|
117
122
|
automation_workflow=automation_workflow,
|
|
123
|
+
timeout_seconds=timeout_seconds,
|
|
118
124
|
)
|
|
@@ -26,6 +26,9 @@ from metadata.generated.schema.entity.automations.workflow import (
|
|
|
26
26
|
from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
|
|
27
27
|
BigQueryConnection,
|
|
28
28
|
)
|
|
29
|
+
from metadata.generated.schema.entity.services.connections.testConnectionResult import (
|
|
30
|
+
TestConnectionResult,
|
|
31
|
+
)
|
|
29
32
|
from metadata.generated.schema.security.credentials.gcpCredentials import (
|
|
30
33
|
GcpCredentialsPath,
|
|
31
34
|
)
|
|
@@ -46,6 +49,7 @@ from metadata.ingestion.connections.test_connections import (
|
|
|
46
49
|
)
|
|
47
50
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
48
51
|
from metadata.ingestion.source.database.bigquery.queries import BIGQUERY_TEST_STATEMENT
|
|
52
|
+
from metadata.utils.constants import THREE_MIN
|
|
49
53
|
from metadata.utils.credentials import set_google_credentials
|
|
50
54
|
from metadata.utils.logger import ingestion_logger
|
|
51
55
|
|
|
@@ -108,7 +112,8 @@ def test_connection(
|
|
|
108
112
|
engine: Engine,
|
|
109
113
|
service_connection: BigQueryConnection,
|
|
110
114
|
automation_workflow: Optional[AutomationWorkflow] = None,
|
|
111
|
-
|
|
115
|
+
timeout_seconds: Optional[int] = THREE_MIN,
|
|
116
|
+
) -> TestConnectionResult:
|
|
112
117
|
"""
|
|
113
118
|
Test connection. This can be executed either as part
|
|
114
119
|
of a metadata workflow or during an Automation Workflow
|
|
@@ -162,11 +167,12 @@ def test_connection(
|
|
|
162
167
|
),
|
|
163
168
|
}
|
|
164
169
|
|
|
165
|
-
test_connection_steps(
|
|
170
|
+
return test_connection_steps(
|
|
166
171
|
metadata=metadata,
|
|
167
172
|
test_fn=test_fn,
|
|
168
173
|
service_type=service_connection.type.value,
|
|
169
174
|
automation_workflow=automation_workflow,
|
|
175
|
+
timeout_seconds=timeout_seconds,
|
|
170
176
|
)
|
|
171
177
|
|
|
172
|
-
test_connection_inner(engine)
|
|
178
|
+
return test_connection_inner(engine)
|
|
@@ -19,6 +19,9 @@ from metadata.generated.schema.entity.automations.workflow import (
|
|
|
19
19
|
from metadata.generated.schema.entity.services.connections.database.bigTableConnection import (
|
|
20
20
|
BigTableConnection,
|
|
21
21
|
)
|
|
22
|
+
from metadata.generated.schema.entity.services.connections.testConnectionResult import (
|
|
23
|
+
TestConnectionResult,
|
|
24
|
+
)
|
|
22
25
|
from metadata.generated.schema.security.credentials.gcpValues import (
|
|
23
26
|
GcpCredentialsValues,
|
|
24
27
|
SingleProjectId,
|
|
@@ -29,6 +32,7 @@ from metadata.ingestion.connections.test_connections import (
|
|
|
29
32
|
)
|
|
30
33
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
31
34
|
from metadata.ingestion.source.database.bigtable.client import MultiProjectClient
|
|
35
|
+
from metadata.utils.constants import THREE_MIN
|
|
32
36
|
from metadata.utils.credentials import set_google_credentials
|
|
33
37
|
from metadata.utils.logger import ingestion_logger
|
|
34
38
|
|
|
@@ -95,7 +99,8 @@ def test_connection(
|
|
|
95
99
|
client: MultiProjectClient,
|
|
96
100
|
service_connection: BigTableConnection,
|
|
97
101
|
automation_workflow: Optional[AutomationWorkflow] = None,
|
|
98
|
-
|
|
102
|
+
timeout_seconds: Optional[int] = THREE_MIN,
|
|
103
|
+
) -> TestConnectionResult:
|
|
99
104
|
"""
|
|
100
105
|
Test connection. This can be executed either as part
|
|
101
106
|
of a metadata workflow or during an Automation Workflow
|
|
@@ -108,9 +113,10 @@ def test_connection(
|
|
|
108
113
|
"GetRows": tester.get_row,
|
|
109
114
|
}
|
|
110
115
|
|
|
111
|
-
test_connection_steps(
|
|
116
|
+
return test_connection_steps(
|
|
112
117
|
metadata=metadata,
|
|
113
118
|
test_fn=test_fn,
|
|
114
119
|
service_type=service_connection.type.value,
|
|
115
120
|
automation_workflow=automation_workflow,
|
|
121
|
+
timeout_seconds=timeout_seconds,
|
|
116
122
|
)
|
|
@@ -23,6 +23,9 @@ from metadata.generated.schema.entity.automations.workflow import (
|
|
|
23
23
|
from metadata.generated.schema.entity.services.connections.database.clickhouseConnection import (
|
|
24
24
|
ClickhouseConnection,
|
|
25
25
|
)
|
|
26
|
+
from metadata.generated.schema.entity.services.connections.testConnectionResult import (
|
|
27
|
+
TestConnectionResult,
|
|
28
|
+
)
|
|
26
29
|
from metadata.ingestion.connections.builders import (
|
|
27
30
|
create_generic_db_connection,
|
|
28
31
|
get_connection_args_common,
|
|
@@ -35,6 +38,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
|
35
38
|
from metadata.ingestion.source.database.clickhouse.queries import (
|
|
36
39
|
CLICKHOUSE_SQL_STATEMENT_TEST,
|
|
37
40
|
)
|
|
41
|
+
from metadata.utils.constants import THREE_MIN
|
|
38
42
|
|
|
39
43
|
HTTPS_PROTOCOL = "https"
|
|
40
44
|
|
|
@@ -67,7 +71,8 @@ def test_connection(
|
|
|
67
71
|
engine: Engine,
|
|
68
72
|
service_connection: ClickhouseConnection,
|
|
69
73
|
automation_workflow: Optional[AutomationWorkflow] = None,
|
|
70
|
-
|
|
74
|
+
timeout_seconds: Optional[int] = THREE_MIN,
|
|
75
|
+
) -> TestConnectionResult:
|
|
71
76
|
"""
|
|
72
77
|
Test connection. This can be executed either as part
|
|
73
78
|
of a metadata workflow or during an Automation Workflow
|
|
@@ -75,10 +80,11 @@ def test_connection(
|
|
|
75
80
|
|
|
76
81
|
queries = {"GetQueries": CLICKHOUSE_SQL_STATEMENT_TEST}
|
|
77
82
|
|
|
78
|
-
test_connection_db_common(
|
|
83
|
+
return test_connection_db_common(
|
|
79
84
|
metadata=metadata,
|
|
80
85
|
engine=engine,
|
|
81
86
|
service_connection=service_connection,
|
|
82
87
|
automation_workflow=automation_workflow,
|
|
83
88
|
queries=queries,
|
|
89
|
+
timeout_seconds=timeout_seconds,
|
|
84
90
|
)
|
|
@@ -478,6 +478,12 @@ class CommonDbSourceService(
|
|
|
478
478
|
"""Not Implemented"""
|
|
479
479
|
yield from []
|
|
480
480
|
|
|
481
|
+
def get_location_path(self, table_name: str, schema_name: str) -> Optional[str]:
|
|
482
|
+
"""
|
|
483
|
+
Method to fetch the location path of the table
|
|
484
|
+
by default there will be no location path
|
|
485
|
+
"""
|
|
486
|
+
|
|
481
487
|
@calculate_execution_time_generator()
|
|
482
488
|
def yield_table(
|
|
483
489
|
self, table_name_and_type: Tuple[str, TableType]
|
|
@@ -553,6 +559,9 @@ class CommonDbSourceService(
|
|
|
553
559
|
table_type=table_type,
|
|
554
560
|
),
|
|
555
561
|
owners=self.get_owner_ref(table_name=table_name),
|
|
562
|
+
locationPath=self.get_location_path(
|
|
563
|
+
table_name=table_name, schema_name=schema_name
|
|
564
|
+
),
|
|
556
565
|
)
|
|
557
566
|
|
|
558
567
|
is_partitioned, partition_details = self.get_table_partition_details(
|
|
@@ -232,19 +232,9 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
|
|
|
232
232
|
schema_name = self.context.get().database_schema
|
|
233
233
|
try:
|
|
234
234
|
data = self.get_table_columns_dict(schema_name, table_name)
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
columns = column_parser.get_columns()
|
|
239
|
-
except Exception as exc:
|
|
240
|
-
logger.error(
|
|
241
|
-
f"Failed to parse columns for table "
|
|
242
|
-
f"[{self.context.get().database}.{schema_name}.{table_name}]: {exc}"
|
|
243
|
-
)
|
|
244
|
-
logger.error(traceback.format_exc())
|
|
245
|
-
logger.error(f"Type of data: {type(data)}")
|
|
246
|
-
logger.error(f"Data: {list(data)[:100]}")
|
|
247
|
-
columns = []
|
|
235
|
+
df = pd.DataFrame.from_records(list(data))
|
|
236
|
+
column_parser = DataFrameColumnParser.create(df)
|
|
237
|
+
columns = column_parser.get_columns()
|
|
248
238
|
table_request = CreateTableRequest(
|
|
249
239
|
name=EntityName(table_name),
|
|
250
240
|
tableType=table_type,
|
|
@@ -277,10 +267,7 @@ class CommonNoSQLSource(DatabaseServiceSource, ABC):
|
|
|
277
267
|
yield Either(
|
|
278
268
|
left=StackTraceError(
|
|
279
269
|
name=table_name,
|
|
280
|
-
error=
|
|
281
|
-
f"Unexpected exception to yield table"
|
|
282
|
-
f"[{self.context.get().database}.{schema_name}.{table_name}]: {exc}"
|
|
283
|
-
),
|
|
270
|
+
error=f"Unexpected exception to yield table [{table_name}]: {exc}",
|
|
284
271
|
stackTrace=traceback.format_exc(),
|
|
285
272
|
)
|
|
286
273
|
)
|
|
@@ -23,8 +23,12 @@ from metadata.generated.schema.entity.automations.workflow import (
|
|
|
23
23
|
from metadata.generated.schema.entity.services.connections.database.couchbaseConnection import (
|
|
24
24
|
CouchbaseConnection,
|
|
25
25
|
)
|
|
26
|
+
from metadata.generated.schema.entity.services.connections.testConnectionResult import (
|
|
27
|
+
TestConnectionResult,
|
|
28
|
+
)
|
|
26
29
|
from metadata.ingestion.connections.test_connections import test_connection_steps
|
|
27
30
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
31
|
+
from metadata.utils.constants import THREE_MIN
|
|
28
32
|
|
|
29
33
|
|
|
30
34
|
def get_connection(connection: CouchbaseConnection):
|
|
@@ -49,7 +53,8 @@ def test_connection(
|
|
|
49
53
|
client: Any,
|
|
50
54
|
service_connection: CouchbaseConnection,
|
|
51
55
|
automation_workflow: Optional[AutomationWorkflow] = None,
|
|
52
|
-
|
|
56
|
+
timeout_seconds: Optional[int] = THREE_MIN,
|
|
57
|
+
) -> TestConnectionResult:
|
|
53
58
|
"""
|
|
54
59
|
Test connection. This can be executed either as part
|
|
55
60
|
of a metadata workflow or during an Automation Workflow
|
|
@@ -80,9 +85,10 @@ def test_connection(
|
|
|
80
85
|
"GetCollections": partial(test_get_collections, client, holder),
|
|
81
86
|
}
|
|
82
87
|
|
|
83
|
-
test_connection_steps(
|
|
88
|
+
return test_connection_steps(
|
|
84
89
|
metadata=metadata,
|
|
85
90
|
test_fn=test_fn,
|
|
86
91
|
service_type=service_connection.type.value,
|
|
87
92
|
automation_workflow=automation_workflow,
|
|
93
|
+
timeout_seconds=timeout_seconds,
|
|
88
94
|
)
|
|
@@ -11,13 +11,10 @@
|
|
|
11
11
|
"""
|
|
12
12
|
Couchbase source methods.
|
|
13
13
|
"""
|
|
14
|
-
|
|
14
|
+
|
|
15
15
|
import traceback
|
|
16
16
|
from typing import Dict, Iterable, List, Optional
|
|
17
17
|
|
|
18
|
-
import requests
|
|
19
|
-
from requests.auth import HTTPBasicAuth
|
|
20
|
-
|
|
21
18
|
from metadata.generated.schema.entity.services.connections.database.couchbaseConnection import (
|
|
22
19
|
CouchbaseConnection,
|
|
23
20
|
)
|
|
@@ -26,17 +23,15 @@ from metadata.generated.schema.metadataIngestion.workflow import (
|
|
|
26
23
|
)
|
|
27
24
|
from metadata.ingestion.api.steps import InvalidSourceException
|
|
28
25
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
29
|
-
from metadata.ingestion.source.database.common_nosql_source import
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
COUCHBASE_GET_INDEX_KEYS,
|
|
26
|
+
from metadata.ingestion.source.database.common_nosql_source import (
|
|
27
|
+
SAMPLE_SIZE,
|
|
28
|
+
CommonNoSQLSource,
|
|
33
29
|
)
|
|
30
|
+
from metadata.ingestion.source.database.couchbase.queries import COUCHBASE_GET_DATA
|
|
34
31
|
from metadata.utils.logger import ingestion_logger
|
|
35
32
|
|
|
36
33
|
logger = ingestion_logger()
|
|
37
34
|
|
|
38
|
-
DEFAULT_SCHEMA_NAME = "_default"
|
|
39
|
-
|
|
40
35
|
|
|
41
36
|
class CouchbaseSource(CommonNoSQLSource):
|
|
42
37
|
"""
|
|
@@ -44,12 +39,9 @@ class CouchbaseSource(CommonNoSQLSource):
|
|
|
44
39
|
Database metadata from Dynamo Source
|
|
45
40
|
"""
|
|
46
41
|
|
|
47
|
-
service_connection: CouchbaseConnection
|
|
48
|
-
|
|
49
42
|
def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
|
|
50
43
|
super().__init__(config, metadata)
|
|
51
44
|
self.couchbase = self.connection_obj
|
|
52
|
-
self.index_condition_map = {}
|
|
53
45
|
|
|
54
46
|
@classmethod
|
|
55
47
|
def create(
|
|
@@ -66,12 +58,10 @@ class CouchbaseSource(CommonNoSQLSource):
|
|
|
66
58
|
def get_database_names(self) -> Iterable[str]:
|
|
67
59
|
try:
|
|
68
60
|
if self.service_connection.bucket:
|
|
69
|
-
self.index_condition_map.clear()
|
|
70
61
|
yield self.service_connection.__dict__.get("bucket")
|
|
71
62
|
else:
|
|
72
63
|
buckets = self.couchbase.buckets()
|
|
73
64
|
for bucket_name in buckets.get_all_buckets():
|
|
74
|
-
self.index_condition_map.clear()
|
|
75
65
|
yield bucket_name.name
|
|
76
66
|
except Exception as exp:
|
|
77
67
|
logger.debug(f"Failed to fetch bucket name: {exp}")
|
|
@@ -111,127 +101,21 @@ class CouchbaseSource(CommonNoSQLSource):
|
|
|
111
101
|
logger.debug(traceback.format_exc())
|
|
112
102
|
return []
|
|
113
103
|
|
|
114
|
-
def _is_valid_key(self, key: str) -> bool:
|
|
115
|
-
return bool(re.fullmatch(r"[a-zA-Z0-9._`\%]+", key))
|
|
116
|
-
|
|
117
|
-
def get_index_condition(self, schema_name: str) -> str:
|
|
118
|
-
"""
|
|
119
|
-
Method to prepare query condition based on index
|
|
120
|
-
"""
|
|
121
|
-
bucket_name = self.context.get().database
|
|
122
|
-
if self.index_condition_map.get((bucket_name, schema_name)):
|
|
123
|
-
return self.index_condition_map.get((bucket_name, schema_name))
|
|
124
|
-
|
|
125
|
-
index_condition = set()
|
|
126
|
-
|
|
127
|
-
if schema_name == DEFAULT_SCHEMA_NAME:
|
|
128
|
-
condition = f"keyspace_id = '{bucket_name}' "
|
|
129
|
-
else:
|
|
130
|
-
condition = f"bucket_id = '{bucket_name}' AND scope_id = '{schema_name}'"
|
|
131
|
-
|
|
132
|
-
query = COUCHBASE_GET_INDEX_KEYS.format(condition=condition)
|
|
133
|
-
result = self.couchbase.query(query)
|
|
134
|
-
for row in result.rows():
|
|
135
|
-
index_obj = Index(**dict(row))
|
|
136
|
-
if index_obj.indexes:
|
|
137
|
-
if index_obj.indexes.is_primary:
|
|
138
|
-
self.index_condition_map[(bucket_name, schema_name)] = ""
|
|
139
|
-
return ""
|
|
140
|
-
for key in index_obj.indexes.index_key or []:
|
|
141
|
-
if self._is_valid_key(key):
|
|
142
|
-
condition = ""
|
|
143
|
-
if index_obj.indexes.condition:
|
|
144
|
-
condition = f"AND {index_obj.indexes.condition}"
|
|
145
|
-
index_condition.add(f"({key} is not missing {condition})")
|
|
146
|
-
if index_condition:
|
|
147
|
-
self.index_condition_map[
|
|
148
|
-
(bucket_name, schema_name)
|
|
149
|
-
] = "WHERE " + " OR ".join(index_condition)
|
|
150
|
-
return self.index_condition_map[(bucket_name, schema_name)]
|
|
151
|
-
|
|
152
|
-
self.index_condition_map[(bucket_name, schema_name)] = ""
|
|
153
|
-
return ""
|
|
154
|
-
|
|
155
|
-
def _fetch_document_ids(self, schema_name: str, table_name: str) -> List[str]:
|
|
156
|
-
# Parameters for pagination
|
|
157
|
-
page_size = 100 # Number of documents to fetch per page
|
|
158
|
-
document_ids = []
|
|
159
|
-
|
|
160
|
-
try:
|
|
161
|
-
page = 0
|
|
162
|
-
while len(document_ids) < 1000:
|
|
163
|
-
# REST API endpoint with pagination
|
|
164
|
-
couchbase_port = 8091
|
|
165
|
-
database_name = self.context.get().database
|
|
166
|
-
connection_scheme = "http"
|
|
167
|
-
url = (
|
|
168
|
-
f"{connection_scheme}://{self.service_connection.hostport}:{couchbase_port}"
|
|
169
|
-
f"/pools/default/buckets/{database_name}/scopes/{schema_name}/collections/{table_name}/docs"
|
|
170
|
-
)
|
|
171
|
-
params = {"skip": page * page_size, "limit": page_size}
|
|
172
|
-
|
|
173
|
-
# Fetch the next page
|
|
174
|
-
response = requests.get(
|
|
175
|
-
url,
|
|
176
|
-
auth=HTTPBasicAuth(
|
|
177
|
-
self.service_connection.username,
|
|
178
|
-
self.service_connection.password.get_secret_value(),
|
|
179
|
-
),
|
|
180
|
-
params=params,
|
|
181
|
-
)
|
|
182
|
-
response.raise_for_status()
|
|
183
|
-
data = response.json()
|
|
184
|
-
|
|
185
|
-
# Add document IDs from the current page
|
|
186
|
-
ids = [doc["id"] for doc in data["rows"]]
|
|
187
|
-
document_ids.extend(ids)
|
|
188
|
-
|
|
189
|
-
# If fewer documents than page_size were returned, we’re at the end
|
|
190
|
-
if len(ids) < page_size:
|
|
191
|
-
break
|
|
192
|
-
|
|
193
|
-
# Increment the page number
|
|
194
|
-
page += 1
|
|
195
|
-
|
|
196
|
-
# Limit the list to the first 1,000 IDs
|
|
197
|
-
document_ids = document_ids[:1000]
|
|
198
|
-
|
|
199
|
-
return document_ids
|
|
200
|
-
|
|
201
|
-
except Exception as exc:
|
|
202
|
-
logger.debug(f"Error fetching document IDs: [{table_name}]: {exc}")
|
|
203
|
-
logger.debug(traceback.format_exc())
|
|
204
|
-
|
|
205
|
-
logger.info("Document IDs not found")
|
|
206
|
-
return []
|
|
207
|
-
|
|
208
104
|
def get_table_columns_dict(self, schema_name: str, table_name: str) -> List[Dict]:
|
|
209
105
|
"""
|
|
210
106
|
Method to get actual data available within table
|
|
211
107
|
need to be overridden by sources
|
|
212
108
|
"""
|
|
213
|
-
from couchbase.exceptions import QueryIndexNotFoundException
|
|
214
|
-
|
|
215
109
|
try:
|
|
216
110
|
database_name = self.context.get().database
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
111
|
+
query_coln = COUCHBASE_GET_DATA.format(
|
|
112
|
+
database_name=database_name,
|
|
113
|
+
schema_name=schema_name,
|
|
114
|
+
table_name=table_name,
|
|
115
|
+
sample_size=SAMPLE_SIZE,
|
|
222
116
|
)
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
lambda x: x.value if isinstance(x.value, dict) else {},
|
|
226
|
-
documents.results.values(),
|
|
227
|
-
)
|
|
228
|
-
)
|
|
229
|
-
except QueryIndexNotFoundException as exp:
|
|
230
|
-
logger.warning(
|
|
231
|
-
f"Fetching columns failed for [`{database_name}`.`{schema_name}`.`{table_name}`],"
|
|
232
|
-
" check if the index is created for the table or data exists in the table"
|
|
233
|
-
)
|
|
234
|
-
logger.debug(traceback.format_exc())
|
|
117
|
+
query_iter = self.couchbase.query(query_coln)
|
|
118
|
+
return list(query_iter.rows())
|
|
235
119
|
except Exception as exp:
|
|
236
120
|
logger.debug(f"Failed to list column names for table [{table_name}]: {exp}")
|
|
237
121
|
logger.debug(traceback.format_exc())
|
|
@@ -14,11 +14,6 @@ SQL Queries used during ingestion
|
|
|
14
14
|
|
|
15
15
|
import textwrap
|
|
16
16
|
|
|
17
|
-
COUCHBASE_GET_INDEX_KEYS = textwrap.dedent(
|
|
18
|
-
""" select * from system:indexes where {condition}"""
|
|
19
|
-
)
|
|
20
|
-
|
|
21
|
-
|
|
22
17
|
COUCHBASE_GET_DATA = textwrap.dedent(
|
|
23
|
-
""" select
|
|
18
|
+
""" select crc.* from `{database_name}`.`{schema_name}`.`{table_name}` crc limit {sample_size} """
|
|
24
19
|
)
|
|
@@ -58,6 +58,9 @@ from metadata.ingestion.api.delete import delete_entity_from_source
|
|
|
58
58
|
from metadata.ingestion.api.models import Either
|
|
59
59
|
from metadata.ingestion.api.steps import Source
|
|
60
60
|
from metadata.ingestion.api.topology_runner import TopologyRunnerMixin
|
|
61
|
+
from metadata.ingestion.connections.test_connections import (
|
|
62
|
+
raise_test_connection_exception,
|
|
63
|
+
)
|
|
61
64
|
from metadata.ingestion.models.life_cycle import OMetaLifeCycleData
|
|
62
65
|
from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
|
|
63
66
|
from metadata.ingestion.models.topology import (
|
|
@@ -609,4 +612,7 @@ class DatabaseServiceSource(
|
|
|
609
612
|
|
|
610
613
|
def test_connection(self) -> None:
|
|
611
614
|
test_connection_fn = get_test_connection_fn(self.service_connection)
|
|
612
|
-
test_connection_fn(
|
|
615
|
+
result = test_connection_fn(
|
|
616
|
+
self.metadata, self.connection_obj, self.service_connection
|
|
617
|
+
)
|
|
618
|
+
raise_test_connection_exception(result)
|
|
@@ -25,6 +25,9 @@ from metadata.generated.schema.entity.automations.workflow import (
|
|
|
25
25
|
from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
|
|
26
26
|
DatabricksConnection,
|
|
27
27
|
)
|
|
28
|
+
from metadata.generated.schema.entity.services.connections.testConnectionResult import (
|
|
29
|
+
TestConnectionResult,
|
|
30
|
+
)
|
|
28
31
|
from metadata.ingestion.connections.builders import (
|
|
29
32
|
create_generic_db_connection,
|
|
30
33
|
get_connection_args_common,
|
|
@@ -39,6 +42,7 @@ from metadata.ingestion.source.database.databricks.client import DatabricksClien
|
|
|
39
42
|
from metadata.ingestion.source.database.databricks.queries import (
|
|
40
43
|
DATABRICKS_GET_CATALOGS,
|
|
41
44
|
)
|
|
45
|
+
from metadata.utils.constants import THREE_MIN
|
|
42
46
|
from metadata.utils.logger import ingestion_logger
|
|
43
47
|
|
|
44
48
|
logger = ingestion_logger()
|
|
@@ -71,7 +75,8 @@ def test_connection(
|
|
|
71
75
|
connection: Engine,
|
|
72
76
|
service_connection: DatabricksConnection,
|
|
73
77
|
automation_workflow: Optional[AutomationWorkflow] = None,
|
|
74
|
-
|
|
78
|
+
timeout_seconds: Optional[int] = THREE_MIN,
|
|
79
|
+
) -> TestConnectionResult:
|
|
75
80
|
"""
|
|
76
81
|
Test connection. This can be executed either as part
|
|
77
82
|
of a metadata workflow or during an Automation Workflow
|
|
@@ -104,10 +109,10 @@ def test_connection(
|
|
|
104
109
|
"GetQueries": client.test_query_api_access,
|
|
105
110
|
}
|
|
106
111
|
|
|
107
|
-
test_connection_steps(
|
|
112
|
+
return test_connection_steps(
|
|
108
113
|
metadata=metadata,
|
|
109
114
|
test_fn=test_fn,
|
|
110
115
|
service_type=service_connection.type.value,
|
|
111
116
|
automation_workflow=automation_workflow,
|
|
112
|
-
timeout_seconds=service_connection.connectionTimeout,
|
|
117
|
+
timeout_seconds=service_connection.connectionTimeout or timeout_seconds,
|
|
113
118
|
)
|
|
@@ -684,11 +684,13 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
|
|
|
684
684
|
table_name=table_name,
|
|
685
685
|
)
|
|
686
686
|
)
|
|
687
|
+
found_location = False
|
|
687
688
|
for result in list(cursor):
|
|
688
689
|
data = result.values()
|
|
689
690
|
if data[0] and data[0].strip() == "Comment":
|
|
690
691
|
description = data[1] if data and data[1] else None
|
|
691
692
|
elif data[0] and data[0].strip() == "Location":
|
|
693
|
+
found_location = True
|
|
692
694
|
self.external_location_map[
|
|
693
695
|
(self.context.get().database, schema_name, table_name)
|
|
694
696
|
] = (
|
|
@@ -696,6 +698,10 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
|
|
|
696
698
|
if data and data[1] and not data[1].startswith("dbfs")
|
|
697
699
|
else None
|
|
698
700
|
)
|
|
701
|
+
if not found_location:
|
|
702
|
+
logger.debug(
|
|
703
|
+
f"Location not found for table {self.context.get().database}.{schema_name}.{table_name}"
|
|
704
|
+
)
|
|
699
705
|
|
|
700
706
|
# Catch any exception without breaking the ingestion
|
|
701
707
|
except Exception as exc: # pylint: disable=broad-except
|
|
@@ -705,6 +711,19 @@ class DatabricksSource(ExternalTableLineageMixin, CommonDbSourceService, MultiDB
|
|
|
705
711
|
)
|
|
706
712
|
return description
|
|
707
713
|
|
|
714
|
+
def get_location_path(self, table_name: str, schema_name: str) -> Optional[str]:
|
|
715
|
+
"""
|
|
716
|
+
Method to fetch the location path of the table
|
|
717
|
+
"""
|
|
718
|
+
location = self.external_location_map.get(
|
|
719
|
+
(self.context.get().database, schema_name, table_name)
|
|
720
|
+
)
|
|
721
|
+
if not location:
|
|
722
|
+
logger.debug(
|
|
723
|
+
f"Location not found in map for table {self.context.get().database}.{schema_name}.{table_name}"
|
|
724
|
+
)
|
|
725
|
+
return location
|
|
726
|
+
|
|
708
727
|
def _filter_owner_name(self, owner_name: str) -> str:
|
|
709
728
|
"""remove unnecessary keyword from name"""
|
|
710
729
|
pattern = r"\(Unknown\)"
|
|
@@ -31,6 +31,9 @@ from metadata.generated.schema.entity.services.connections.database.datalake.s3C
|
|
|
31
31
|
from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
|
|
32
32
|
DatalakeConnection,
|
|
33
33
|
)
|
|
34
|
+
from metadata.generated.schema.entity.services.connections.testConnectionResult import (
|
|
35
|
+
TestConnectionResult,
|
|
36
|
+
)
|
|
34
37
|
from metadata.ingestion.connections.test_connections import test_connection_steps
|
|
35
38
|
from metadata.ingestion.ometa.ometa_api import OpenMetadata
|
|
36
39
|
from metadata.ingestion.source.database.datalake.clients.azure_blob import (
|
|
@@ -38,6 +41,7 @@ from metadata.ingestion.source.database.datalake.clients.azure_blob import (
|
|
|
38
41
|
)
|
|
39
42
|
from metadata.ingestion.source.database.datalake.clients.gcs import DatalakeGcsClient
|
|
40
43
|
from metadata.ingestion.source.database.datalake.clients.s3 import DatalakeS3Client
|
|
44
|
+
from metadata.utils.constants import THREE_MIN
|
|
41
45
|
|
|
42
46
|
|
|
43
47
|
# Only import specific datalake dependencies if necessary
|
|
@@ -91,7 +95,8 @@ def test_connection(
|
|
|
91
95
|
connection: DatalakeClient,
|
|
92
96
|
service_connection: DatalakeConnection,
|
|
93
97
|
automation_workflow: Optional[AutomationWorkflow] = None,
|
|
94
|
-
|
|
98
|
+
timeout_seconds: Optional[int] = THREE_MIN,
|
|
99
|
+
) -> TestConnectionResult:
|
|
95
100
|
"""
|
|
96
101
|
Test connection. This can be executed either as part
|
|
97
102
|
of a metadata workflow or during an Automation Workflow
|
|
@@ -102,9 +107,10 @@ def test_connection(
|
|
|
102
107
|
),
|
|
103
108
|
}
|
|
104
109
|
|
|
105
|
-
test_connection_steps(
|
|
110
|
+
return test_connection_steps(
|
|
106
111
|
metadata=metadata,
|
|
107
112
|
test_fn=test_fn,
|
|
108
113
|
service_type=service_connection.type.value,
|
|
109
114
|
automation_workflow=automation_workflow,
|
|
115
|
+
timeout_seconds=timeout_seconds,
|
|
110
116
|
)
|