openmetadata-ingestion 1.10.1.0__py3-none-any.whl → 1.10.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openmetadata-ingestion might be problematic. Click here for more details.
- metadata/clients/aws_client.py +4 -0
- metadata/examples/workflows/confluent_cdc.yaml +70 -0
- metadata/examples/workflows/confluent_cdc_cloud.yaml +86 -0
- metadata/examples/workflows/confluent_cdc_local.yaml +120 -0
- metadata/examples/workflows/kinesisfirehose.yaml +99 -0
- metadata/examples/workflows/my_confluent_cloud.yaml +49 -0
- metadata/generated/schema/analytics/__init__.py +1 -1
- metadata/generated/schema/analytics/basic.py +1 -1
- metadata/generated/schema/analytics/reportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/__init__.py +1 -1
- metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/entityReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py +1 -1
- metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventData.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/__init__.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py +1 -1
- metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py +1 -1
- metadata/generated/schema/api/__init__.py +1 -1
- metadata/generated/schema/api/addGlossaryToAssetsRequest.py +1 -1
- metadata/generated/schema/api/addTagToAssetsRequest.py +1 -1
- metadata/generated/schema/api/analytics/__init__.py +1 -1
- metadata/generated/schema/api/analytics/createWebAnalyticEvent.py +1 -1
- metadata/generated/schema/api/automations/__init__.py +1 -1
- metadata/generated/schema/api/automations/createWorkflow.py +1 -1
- metadata/generated/schema/api/bulkAssets.py +1 -1
- metadata/generated/schema/api/classification/__init__.py +1 -1
- metadata/generated/schema/api/classification/createClassification.py +1 -1
- metadata/generated/schema/api/classification/createTag.py +1 -1
- metadata/generated/schema/api/classification/createTagWithRecognizers.py +1 -1
- metadata/generated/schema/api/classification/loadTags.py +1 -1
- metadata/generated/schema/api/configuration/__init__.py +1 -1
- metadata/generated/schema/api/configuration/rdfConfiguration.py +1 -1
- metadata/generated/schema/api/createBot.py +1 -1
- metadata/generated/schema/api/createEventPublisherJob.py +1 -1
- metadata/generated/schema/api/createType.py +1 -1
- metadata/generated/schema/api/data/__init__.py +1 -1
- metadata/generated/schema/api/data/createAPICollection.py +1 -1
- metadata/generated/schema/api/data/createAPIEndpoint.py +1 -1
- metadata/generated/schema/api/data/createChart.py +1 -1
- metadata/generated/schema/api/data/createContainer.py +1 -1
- metadata/generated/schema/api/data/createCustomProperty.py +1 -1
- metadata/generated/schema/api/data/createDashboard.py +1 -1
- metadata/generated/schema/api/data/createDashboardDataModel.py +1 -1
- metadata/generated/schema/api/data/createDataContract.py +1 -1
- metadata/generated/schema/api/data/createDatabase.py +1 -1
- metadata/generated/schema/api/data/createDatabaseSchema.py +1 -1
- metadata/generated/schema/api/data/createDirectory.py +1 -1
- metadata/generated/schema/api/data/createEntityProfile.py +1 -1
- metadata/generated/schema/api/data/createFile.py +1 -1
- metadata/generated/schema/api/data/createGlossary.py +1 -1
- metadata/generated/schema/api/data/createGlossaryTerm.py +1 -1
- metadata/generated/schema/api/data/createMetric.py +1 -1
- metadata/generated/schema/api/data/createMlModel.py +1 -1
- metadata/generated/schema/api/data/createPipeline.py +1 -1
- metadata/generated/schema/api/data/createQuery.py +1 -1
- metadata/generated/schema/api/data/createQueryCostRecord.py +1 -1
- metadata/generated/schema/api/data/createSearchIndex.py +1 -1
- metadata/generated/schema/api/data/createSpreadsheet.py +1 -1
- metadata/generated/schema/api/data/createStoredProcedure.py +1 -1
- metadata/generated/schema/api/data/createTable.py +1 -1
- metadata/generated/schema/api/data/createTableProfile.py +1 -1
- metadata/generated/schema/api/data/createTopic.py +1 -1
- metadata/generated/schema/api/data/createWorksheet.py +1 -1
- metadata/generated/schema/api/data/loadGlossary.py +1 -1
- metadata/generated/schema/api/data/restoreEntity.py +1 -1
- metadata/generated/schema/api/data/updateColumn.py +1 -1
- metadata/generated/schema/api/dataInsight/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/createDataInsightChart.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/custom/createDataInsightCustomChart.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/api/dataInsight/kpi/createKpiRequest.py +1 -1
- metadata/generated/schema/api/docStore/__init__.py +1 -1
- metadata/generated/schema/api/docStore/createDocument.py +1 -1
- metadata/generated/schema/api/domains/__init__.py +1 -1
- metadata/generated/schema/api/domains/createDataProduct.py +1 -1
- metadata/generated/schema/api/domains/createDomain.py +1 -1
- metadata/generated/schema/api/entityRelationship/__init__.py +1 -1
- metadata/generated/schema/api/entityRelationship/entityRelationshipDirection.py +1 -1
- metadata/generated/schema/api/entityRelationship/esEntityRelationshipData.py +1 -1
- metadata/generated/schema/api/entityRelationship/relationshipRef.py +1 -1
- metadata/generated/schema/api/entityRelationship/searchEntityRelationshipRequest.py +1 -1
- metadata/generated/schema/api/entityRelationship/searchEntityRelationshipResult.py +1 -1
- metadata/generated/schema/api/entityRelationship/searchSchemaEntityRelationshipResult.py +1 -1
- metadata/generated/schema/api/events/__init__.py +1 -1
- metadata/generated/schema/api/events/createNotificationTemplate.py +1 -1
- metadata/generated/schema/api/events/notificationTemplateValidationRequest.py +1 -1
- metadata/generated/schema/api/events/notificationTemplateValidationResponse.py +1 -1
- metadata/generated/schema/api/feed/__init__.py +1 -1
- metadata/generated/schema/api/feed/closeTask.py +1 -1
- metadata/generated/schema/api/feed/createPost.py +1 -1
- metadata/generated/schema/api/feed/createSuggestion.py +1 -1
- metadata/generated/schema/api/feed/createThread.py +1 -1
- metadata/generated/schema/api/feed/resolveTask.py +1 -1
- metadata/generated/schema/api/feed/threadCount.py +1 -1
- metadata/generated/schema/api/governance/__init__.py +1 -1
- metadata/generated/schema/api/governance/createWorkflowDefinition.py +1 -1
- metadata/generated/schema/api/governance/createWorkflowInstanceState.py +1 -1
- metadata/generated/schema/api/lineage/__init__.py +1 -1
- metadata/generated/schema/api/lineage/addLineage.py +1 -1
- metadata/generated/schema/api/lineage/entityCountLineageRequest.py +1 -1
- metadata/generated/schema/api/lineage/esLineageData.py +1 -1
- metadata/generated/schema/api/lineage/lineageDirection.py +1 -1
- metadata/generated/schema/api/lineage/lineagePaginationInfo.py +1 -1
- metadata/generated/schema/api/lineage/nodeInformation.py +1 -1
- metadata/generated/schema/api/lineage/searchLineageRequest.py +1 -1
- metadata/generated/schema/api/lineage/searchLineageResult.py +1 -1
- metadata/generated/schema/api/mcp/__init__.py +1 -1
- metadata/generated/schema/api/mcp/mcpSearchResponse.py +1 -1
- metadata/generated/schema/api/mcp/mcpToolDefinition.py +1 -1
- metadata/generated/schema/api/openMetadataServerVersion.py +1 -1
- metadata/generated/schema/api/policies/__init__.py +1 -1
- metadata/generated/schema/api/policies/createPolicy.py +1 -1
- metadata/generated/schema/api/rdf/__init__.py +1 -1
- metadata/generated/schema/api/rdf/sparqlQuery.py +1 -1
- metadata/generated/schema/api/rdf/sparqlResponse.py +1 -1
- metadata/generated/schema/api/scim/__init__.py +1 -1
- metadata/generated/schema/api/scim/scimGroup.py +1 -1
- metadata/generated/schema/api/scim/scimPatchOp.py +1 -1
- metadata/generated/schema/api/scim/scimUser.py +1 -1
- metadata/generated/schema/api/search/__init__.py +1 -1
- metadata/generated/schema/api/search/previewSearchRequest.py +1 -1
- metadata/generated/schema/api/services/__init__.py +1 -1
- metadata/generated/schema/api/services/createApiService.py +1 -1
- metadata/generated/schema/api/services/createDashboardService.py +1 -1
- metadata/generated/schema/api/services/createDatabaseService.py +1 -1
- metadata/generated/schema/api/services/createDriveService.py +1 -1
- metadata/generated/schema/api/services/createMessagingService.py +1 -1
- metadata/generated/schema/api/services/createMetadataService.py +1 -1
- metadata/generated/schema/api/services/createMlModelService.py +1 -1
- metadata/generated/schema/api/services/createPipelineService.py +1 -1
- metadata/generated/schema/api/services/createSearchService.py +1 -1
- metadata/generated/schema/api/services/createSecurityService.py +1 -1
- metadata/generated/schema/api/services/createStorageService.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/api/services/ingestionPipelines/createIngestionPipeline.py +1 -1
- metadata/generated/schema/api/setOwner.py +1 -1
- metadata/generated/schema/api/teams/__init__.py +1 -1
- metadata/generated/schema/api/teams/createPersona.py +1 -1
- metadata/generated/schema/api/teams/createRole.py +1 -1
- metadata/generated/schema/api/teams/createTeam.py +1 -1
- metadata/generated/schema/api/teams/createUser.py +1 -1
- metadata/generated/schema/api/tests/__init__.py +1 -1
- metadata/generated/schema/api/tests/createCustomMetric.py +1 -1
- metadata/generated/schema/api/tests/createLogicalTestCases.py +1 -1
- metadata/generated/schema/api/tests/createTestCase.py +1 -1
- metadata/generated/schema/api/tests/createTestCaseResolutionStatus.py +1 -1
- metadata/generated/schema/api/tests/createTestCaseResult.py +1 -1
- metadata/generated/schema/api/tests/createTestDefinition.py +1 -1
- metadata/generated/schema/api/tests/createTestSuite.py +1 -1
- metadata/generated/schema/api/tests/moveGlossaryTermRequest.py +1 -1
- metadata/generated/schema/api/validateGlossaryTagsRequest.py +1 -1
- metadata/generated/schema/api/voteRequest.py +1 -1
- metadata/generated/schema/auth/__init__.py +1 -1
- metadata/generated/schema/auth/basicAuth.py +1 -1
- metadata/generated/schema/auth/basicLoginRequest.py +1 -1
- metadata/generated/schema/auth/changePasswordRequest.py +1 -1
- metadata/generated/schema/auth/createPersonalToken.py +1 -1
- metadata/generated/schema/auth/emailRequest.py +1 -1
- metadata/generated/schema/auth/emailVerificationToken.py +1 -1
- metadata/generated/schema/auth/generateToken.py +1 -1
- metadata/generated/schema/auth/jwtAuth.py +1 -1
- metadata/generated/schema/auth/loginRequest.py +1 -1
- metadata/generated/schema/auth/logoutRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetRequest.py +1 -1
- metadata/generated/schema/auth/passwordResetToken.py +1 -1
- metadata/generated/schema/auth/personalAccessToken.py +1 -1
- metadata/generated/schema/auth/refreshToken.py +1 -1
- metadata/generated/schema/auth/registrationRequest.py +1 -1
- metadata/generated/schema/auth/revokePersonalToken.py +1 -1
- metadata/generated/schema/auth/revokeToken.py +1 -1
- metadata/generated/schema/auth/serviceTokenEnum.py +1 -1
- metadata/generated/schema/auth/ssoAuth.py +1 -1
- metadata/generated/schema/auth/supportToken.py +1 -1
- metadata/generated/schema/auth/tokenRefreshRequest.py +1 -1
- metadata/generated/schema/configuration/__init__.py +1 -1
- metadata/generated/schema/configuration/aiPlatformConfiguration.py +1 -1
- metadata/generated/schema/configuration/appsPrivateConfiguration.py +1 -1
- metadata/generated/schema/configuration/assetCertificationSettings.py +1 -1
- metadata/generated/schema/configuration/authConfig.py +1 -1
- metadata/generated/schema/configuration/authenticationConfiguration.py +1 -1
- metadata/generated/schema/configuration/authorizerConfiguration.py +1 -1
- metadata/generated/schema/configuration/cacheConfiguration.py +1 -1
- metadata/generated/schema/configuration/changeEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/dataQualityConfiguration.py +1 -1
- metadata/generated/schema/configuration/elasticSearchConfiguration.py +1 -1
- metadata/generated/schema/configuration/entityRulesSettings.py +1 -1
- metadata/generated/schema/configuration/eventHandlerConfiguration.py +1 -1
- metadata/generated/schema/configuration/fernetConfiguration.py +1 -1
- metadata/generated/schema/configuration/jwtTokenConfiguration.py +1 -1
- metadata/generated/schema/configuration/kafkaEventConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapConfiguration.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/__init__.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/customTrustManagerConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/hostNameConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/jvmDefaultConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/trustAllConfig.py +1 -1
- metadata/generated/schema/configuration/ldapTrustStoreConfig/truststoreConfig.py +1 -1
- metadata/generated/schema/configuration/limitsConfiguration.py +1 -1
- metadata/generated/schema/configuration/lineageSettings.py +1 -1
- metadata/generated/schema/configuration/logStorageConfiguration.py +1 -1
- metadata/generated/schema/configuration/loginConfiguration.py +1 -1
- metadata/generated/schema/configuration/logoConfiguration.py +1 -1
- metadata/generated/schema/configuration/openMetadataBaseUrlConfiguration.py +1 -1
- metadata/generated/schema/configuration/opertionalConfiguration.py +1 -1
- metadata/generated/schema/configuration/opsConfig.py +1 -1
- metadata/generated/schema/configuration/pipelineServiceClientConfiguration.py +1 -1
- metadata/generated/schema/configuration/profilerConfiguration.py +1 -1
- metadata/generated/schema/configuration/searchSettings.py +1 -1
- metadata/generated/schema/configuration/securityConfiguration.py +1 -1
- metadata/generated/schema/configuration/slackAppConfiguration.py +1 -1
- metadata/generated/schema/configuration/taskNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/testResultNotificationConfiguration.py +1 -1
- metadata/generated/schema/configuration/themeConfiguration.py +1 -1
- metadata/generated/schema/configuration/uiThemePreference.py +1 -1
- metadata/generated/schema/configuration/workflowSettings.py +1 -1
- metadata/generated/schema/dataInsight/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/__init__.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChart.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResult.py +1 -1
- metadata/generated/schema/dataInsight/custom/dataInsightCustomChartResultList.py +1 -1
- metadata/generated/schema/dataInsight/custom/formulaHolder.py +1 -1
- metadata/generated/schema/dataInsight/custom/lineChart.py +1 -1
- metadata/generated/schema/dataInsight/custom/summaryCard.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChart.py +1 -1
- metadata/generated/schema/dataInsight/dataInsightChartResult.py +1 -1
- metadata/generated/schema/dataInsight/kpi/__init__.py +1 -1
- metadata/generated/schema/dataInsight/kpi/basic.py +1 -1
- metadata/generated/schema/dataInsight/kpi/kpi.py +1 -1
- metadata/generated/schema/dataInsight/type/__init__.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsCount.py +1 -1
- metadata/generated/schema/dataInsight/type/aggregatedUsedVsUnusedAssetsSize.py +1 -1
- metadata/generated/schema/dataInsight/type/dailyActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostActiveUsers.py +1 -1
- metadata/generated/schema/dataInsight/type/mostViewedEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/pageViewsByEntities.py +1 -1
- metadata/generated/schema/dataInsight/type/unusedAssets.py +1 -1
- metadata/generated/schema/email/__init__.py +1 -1
- metadata/generated/schema/email/emailRequest.py +1 -1
- metadata/generated/schema/email/emailTemplate.py +1 -1
- metadata/generated/schema/email/emailTemplatePlaceholder.py +1 -1
- metadata/generated/schema/email/smtpSettings.py +1 -1
- metadata/generated/schema/email/templateValidationReponse.py +1 -1
- metadata/generated/schema/entity/__init__.py +1 -1
- metadata/generated/schema/entity/applications/__init__.py +1 -1
- metadata/generated/schema/entity/applications/app.py +1 -1
- metadata/generated/schema/entity/applications/appExtension.py +1 -1
- metadata/generated/schema/entity/applications/appRunRecord.py +1 -1
- metadata/generated/schema/entity/applications/configuration/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/applicationConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addCustomProperties.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDataProductAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTagsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTermsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTestCaseAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/addTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/lineagePropagationAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/mlTaggingAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/propagationStopConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeCustomPropertiesAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDataProductAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDescriptionAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeDomainAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeOwnerAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTagsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTermsAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTestCaseAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automator/removeTierAction.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/automatorAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/collateAIAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/bigQueryConnection.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/databricksConnection.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/redshiftConnection.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/metadataExporterConnectors/snowflakeConnection.py +1 -1
- metadata/generated/schema/entity/applications/configuration/external/slackAppTokenConfiguration.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/autoPilotAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/collateAITierAgentAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataInsightsReportAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/dataRetentionConfiguration.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/helloPipelinesConfiguration.py +1 -1
- metadata/generated/schema/entity/applications/configuration/internal/searchIndexingAppConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/external/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/external/collateAIAppPrivateConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/internal/__init__.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/internal/collateAITierAgentAppPrivateConfig.py +1 -1
- metadata/generated/schema/entity/applications/configuration/private/limits.py +1 -1
- metadata/generated/schema/entity/applications/createAppRequest.py +1 -1
- metadata/generated/schema/entity/applications/jobStatus.py +1 -1
- metadata/generated/schema/entity/applications/liveExecutionContext.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/__init__.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/appMarketPlaceDefinition.py +1 -1
- metadata/generated/schema/entity/applications/marketplace/createAppMarketPlaceDefinitionReq.py +1 -1
- metadata/generated/schema/entity/applications/scheduledExecutionContext.py +1 -1
- metadata/generated/schema/entity/automations/__init__.py +1 -1
- metadata/generated/schema/entity/automations/queryRunnerRequest.py +1 -1
- metadata/generated/schema/entity/automations/response/__init__.py +1 -1
- metadata/generated/schema/entity/automations/response/queryRunnerResponse.py +1 -1
- metadata/generated/schema/entity/automations/testServiceConnection.py +1 -1
- metadata/generated/schema/entity/automations/testSparkEngineConnection.py +1 -1
- metadata/generated/schema/entity/automations/workflow.py +1 -1
- metadata/generated/schema/entity/bot.py +1 -1
- metadata/generated/schema/entity/classification/__init__.py +1 -1
- metadata/generated/schema/entity/classification/classification.py +1 -1
- metadata/generated/schema/entity/classification/tag.py +1 -1
- metadata/generated/schema/entity/data/__init__.py +1 -1
- metadata/generated/schema/entity/data/apiCollection.py +1 -1
- metadata/generated/schema/entity/data/apiEndpoint.py +1 -1
- metadata/generated/schema/entity/data/chart.py +1 -1
- metadata/generated/schema/entity/data/container.py +1 -1
- metadata/generated/schema/entity/data/dashboard.py +1 -1
- metadata/generated/schema/entity/data/dashboardDataModel.py +1 -1
- metadata/generated/schema/entity/data/dataContract.py +1 -1
- metadata/generated/schema/entity/data/database.py +1 -1
- metadata/generated/schema/entity/data/databaseSchema.py +1 -1
- metadata/generated/schema/entity/data/directory.py +1 -1
- metadata/generated/schema/entity/data/file.py +1 -1
- metadata/generated/schema/entity/data/glossary.py +1 -1
- metadata/generated/schema/entity/data/glossaryTerm.py +1 -1
- metadata/generated/schema/entity/data/metric.py +1 -1
- metadata/generated/schema/entity/data/mlmodel.py +1 -1
- metadata/generated/schema/entity/data/pipeline.py +1 -1
- metadata/generated/schema/entity/data/query.py +1 -1
- metadata/generated/schema/entity/data/queryCostRecord.py +1 -1
- metadata/generated/schema/entity/data/queryCostSearchResult.py +1 -1
- metadata/generated/schema/entity/data/report.py +1 -1
- metadata/generated/schema/entity/data/searchIndex.py +1 -1
- metadata/generated/schema/entity/data/spreadsheet.py +1 -1
- metadata/generated/schema/entity/data/storedProcedure.py +1 -1
- metadata/generated/schema/entity/data/table.py +1 -1
- metadata/generated/schema/entity/data/topic.py +1 -1
- metadata/generated/schema/entity/data/worksheet.py +1 -1
- metadata/generated/schema/entity/datacontract/__init__.py +1 -1
- metadata/generated/schema/entity/datacontract/dataContractResult.py +1 -1
- metadata/generated/schema/entity/datacontract/qualityValidation.py +1 -1
- metadata/generated/schema/entity/datacontract/schemaValidation.py +1 -1
- metadata/generated/schema/entity/datacontract/semanticsValidation.py +1 -1
- metadata/generated/schema/entity/datacontract/slaValidation.py +1 -1
- metadata/generated/schema/entity/docStore/__init__.py +1 -1
- metadata/generated/schema/entity/docStore/document.py +1 -1
- metadata/generated/schema/entity/domains/__init__.py +1 -1
- metadata/generated/schema/entity/domains/dataProduct.py +1 -1
- metadata/generated/schema/entity/domains/domain.py +1 -1
- metadata/generated/schema/entity/events/__init__.py +1 -1
- metadata/generated/schema/entity/events/notificationTemplate.py +1 -1
- metadata/generated/schema/entity/events/webhook.py +1 -1
- metadata/generated/schema/entity/feed/__init__.py +1 -1
- metadata/generated/schema/entity/feed/assets.py +1 -1
- metadata/generated/schema/entity/feed/customProperty.py +1 -1
- metadata/generated/schema/entity/feed/description.py +1 -1
- metadata/generated/schema/entity/feed/domain.py +1 -1
- metadata/generated/schema/entity/feed/entityInfo.py +1 -1
- metadata/generated/schema/entity/feed/owner.py +1 -1
- metadata/generated/schema/entity/feed/suggestion.py +1 -1
- metadata/generated/schema/entity/feed/tag.py +1 -1
- metadata/generated/schema/entity/feed/testCaseResult.py +1 -1
- metadata/generated/schema/entity/feed/thread.py +1 -1
- metadata/generated/schema/entity/policies/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/__init__.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourceDescriptor.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/resourcePermission.py +1 -1
- metadata/generated/schema/entity/policies/accessControl/rule.py +1 -1
- metadata/generated/schema/entity/policies/filters.py +1 -1
- metadata/generated/schema/entity/policies/policy.py +1 -1
- metadata/generated/schema/entity/services/__init__.py +1 -1
- metadata/generated/schema/entity/services/apiService.py +1 -1
- metadata/generated/schema/entity/services/connections/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/api/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/api/restConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertPaths.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslCertValues.py +1 -1
- metadata/generated/schema/entity/services/connections/common/sslConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/connectionBasicType.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/customDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/domoDashboardConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/grafanaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/hexConnection.py +76 -0
- metadata/generated/schema/entity/services/connections/dashboard/lightdashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/lookerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/metabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/microStrategyConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/modeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerBIConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerBIReportServerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/bucketDetails.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/powerbi/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikCloudConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/qlikSenseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/quickSightConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/redashConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/sigmaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/supersetConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/tableauConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/dashboard/thoughtSpotConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/athenaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/azureSQLConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/bigQueryConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/bigTableConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/cassandra/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/cassandra/cloudConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/cassandraConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/clickhouseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/cockroachConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/iamAuthConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/jwtAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/common/noConfigAuthenticationTypes.py +1 -1
- metadata/generated/schema/entity/services/connections/database/couchbaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/customDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricks/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricks/azureAdSetup.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricks/databricksOAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricks/personalAccessToken.py +1 -1
- metadata/generated/schema/entity/services/connections/database/databricksConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/azureConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/gcsConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalake/s3Config.py +1 -1
- metadata/generated/schema/entity/services/connections/database/datalakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/db2Connection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltaLakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/metastoreConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/deltalake/storageConfig.py +1 -1
- metadata/generated/schema/entity/services/connections/database/domoDatabaseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dorisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/druidConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/dynamoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/epicConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/exasolConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/glueConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/greenplumConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/hiveConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/dynamoDbCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/glueCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/hiveCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergCatalog.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/icebergFileSystem.py +1 -1
- metadata/generated/schema/entity/services/connections/database/iceberg/restCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/icebergConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/impalaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mariaDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mongoDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mssqlConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/mysqlConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/oracleConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/pinotDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/postgresConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/prestoConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/redshiftConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/salesforceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapErpConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaHDBConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHana/sapHanaSQLConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sapHanaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/serviceNowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/singleStoreConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/snowflakeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/sqliteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/ssasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/synapseConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/teradataConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/timescaleConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/trinoConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/unityCatalogConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/database/verticaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/drive/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/drive/customDriveConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/drive/googleDriveConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/drive/sharePointConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/customMessagingConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kafkaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/kinesisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/pulsarConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/redpandaConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/messaging/saslMechanismType.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/alationSinkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/amundsenConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/atlasConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/metadataESConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/metadata/openMetadataConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/customMlModelConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/mlflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sageMakerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/sklearnConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/mlmodel/vertexaiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airbyteConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/airflowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/backendConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/customPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dagsterConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/databricksPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/datafactoryConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/dbtCloudConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/domoPipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/fivetranConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/flinkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/gluePipelineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/kafkaConnectConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/kinesisFirehoseConnection.py +58 -0
- metadata/generated/schema/entity/services/connections/pipeline/matillion/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/matillion/matillionETL.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/matillionConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifi/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifi/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifi/clientCertificateAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/nifiConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/openLineageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/snowplowConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/sparkConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/splineConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/ssisConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/stitchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/pipeline/wherescapeConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/customSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/apiAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearch/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/search/elasticSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/search/openSearchConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/security/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/security/ranger/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/security/ranger/basicAuth.py +1 -1
- metadata/generated/schema/entity/services/connections/security/rangerConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/serviceConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/__init__.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/adlsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/customStorageConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/gcsConnection.py +1 -1
- metadata/generated/schema/entity/services/connections/storage/s3Connection.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionDefinition.py +1 -1
- metadata/generated/schema/entity/services/connections/testConnectionResult.py +1 -1
- metadata/generated/schema/entity/services/dashboardService.py +4 -1
- metadata/generated/schema/entity/services/databaseService.py +1 -1
- metadata/generated/schema/entity/services/driveService.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/__init__.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/ingestionPipeline.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/pipelineServiceClientResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/reverseIngestionResponse.py +1 -1
- metadata/generated/schema/entity/services/ingestionPipelines/status.py +1 -1
- metadata/generated/schema/entity/services/messagingService.py +1 -1
- metadata/generated/schema/entity/services/metadataService.py +1 -1
- metadata/generated/schema/entity/services/mlmodelService.py +1 -1
- metadata/generated/schema/entity/services/pipelineService.py +4 -1
- metadata/generated/schema/entity/services/searchService.py +1 -1
- metadata/generated/schema/entity/services/securityService.py +1 -1
- metadata/generated/schema/entity/services/serviceType.py +1 -1
- metadata/generated/schema/entity/services/storageService.py +1 -1
- metadata/generated/schema/entity/teams/__init__.py +1 -1
- metadata/generated/schema/entity/teams/persona.py +1 -1
- metadata/generated/schema/entity/teams/role.py +1 -1
- metadata/generated/schema/entity/teams/team.py +1 -1
- metadata/generated/schema/entity/teams/teamHierarchy.py +1 -1
- metadata/generated/schema/entity/teams/user.py +1 -1
- metadata/generated/schema/entity/type.py +1 -1
- metadata/generated/schema/entity/utils/__init__.py +1 -1
- metadata/generated/schema/entity/utils/entitiesCount.py +1 -1
- metadata/generated/schema/entity/utils/servicesCount.py +1 -1
- metadata/generated/schema/entity/utils/supersetApiConnection.py +1 -1
- metadata/generated/schema/events/__init__.py +1 -1
- metadata/generated/schema/events/alertMetrics.py +1 -1
- metadata/generated/schema/events/api/__init__.py +1 -1
- metadata/generated/schema/events/api/createEventSubscription.py +1 -1
- metadata/generated/schema/events/api/eventSubscriptionDiagnosticInfo.py +1 -1
- metadata/generated/schema/events/api/eventsRecord.py +1 -1
- metadata/generated/schema/events/api/testEventSubscriptionDestination.py +1 -1
- metadata/generated/schema/events/api/typedEvent.py +1 -1
- metadata/generated/schema/events/emailAlertConfig.py +1 -1
- metadata/generated/schema/events/eventFilterRule.py +1 -1
- metadata/generated/schema/events/eventSubscription.py +1 -1
- metadata/generated/schema/events/eventSubscriptionOffset.py +1 -1
- metadata/generated/schema/events/failedEvent.py +1 -1
- metadata/generated/schema/events/failedEventResponse.py +1 -1
- metadata/generated/schema/events/filterResourceDescriptor.py +1 -1
- metadata/generated/schema/events/statusContext.py +1 -1
- metadata/generated/schema/events/subscriptionResourceDescriptor.py +1 -1
- metadata/generated/schema/events/subscriptionStatus.py +1 -1
- metadata/generated/schema/events/testDestinationStatus.py +1 -1
- metadata/generated/schema/governance/workflows/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/edge.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodeSubType.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodeType.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/checkEntityAttributesTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/createAndRunIngestionPipelineTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/runAppTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setEntityCertificationTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/automatedTask/setGlossaryTermStatusTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/endEvent/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/endEvent/endEvent.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/gateway/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/gateway/parallelGateway.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/startEvent/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/startEvent/startEvent.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/userTask/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/nodes/userTask/userApprovalTask.py +1 -1
- metadata/generated/schema/governance/workflows/elements/triggers/__init__.py +1 -1
- metadata/generated/schema/governance/workflows/elements/triggers/eventBasedEntityTrigger.py +1 -1
- metadata/generated/schema/governance/workflows/elements/triggers/noOpTrigger.py +1 -1
- metadata/generated/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.py +1 -1
- metadata/generated/schema/governance/workflows/workflowDefinition.py +1 -1
- metadata/generated/schema/governance/workflows/workflowInstance.py +1 -1
- metadata/generated/schema/governance/workflows/workflowInstanceState.py +1 -1
- metadata/generated/schema/jobs/__init__.py +1 -1
- metadata/generated/schema/jobs/backgroundJob.py +1 -1
- metadata/generated/schema/jobs/enumCleanupArgs.py +1 -1
- metadata/generated/schema/metadataIngestion/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/apiServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/application.py +1 -1
- metadata/generated/schema/metadataIngestion/applicationPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dashboardServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dataInsightPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceAutoClassificationPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceProfilerPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryLineagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/databaseServiceQueryUsagePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtAzureConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtCloudConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/dbtconfig/dbtS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/driveServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/engine/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/engine/nativeEngineConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/engine/sparkEngineConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/messagingServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/metadataToElasticSearchPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/mlmodelServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/pipelineServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/reverseIngestionPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/descriptionConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/ownerConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/reverseingestionconfig/tagsConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/searchServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/securityServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/__init__.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/containerMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/manifestMetadataConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageBucketDetails.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataADLSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataGCSConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataHttpConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataLocalConfig.py +1 -1
- metadata/generated/schema/metadataIngestion/storage/storageMetadataS3Config.py +1 -1
- metadata/generated/schema/metadataIngestion/storageServiceMetadataPipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/testSuitePipeline.py +1 -1
- metadata/generated/schema/metadataIngestion/workflow.py +1 -1
- metadata/generated/schema/monitoring/__init__.py +1 -1
- metadata/generated/schema/monitoring/eventMonitorProvider.py +1 -1
- metadata/generated/schema/scim/__init__.py +1 -1
- metadata/generated/schema/scim/scimConfiguration.py +1 -1
- metadata/generated/schema/search/__init__.py +1 -1
- metadata/generated/schema/search/aggregationRequest.py +1 -1
- metadata/generated/schema/search/searchRequest.py +1 -1
- metadata/generated/schema/security/__init__.py +1 -1
- metadata/generated/schema/security/client/__init__.py +1 -1
- metadata/generated/schema/security/client/auth0SSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/azureSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/customOidcSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/googleSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/oidcClientConfig.py +1 -1
- metadata/generated/schema/security/client/oktaSSOClientConfig.py +1 -1
- metadata/generated/schema/security/client/openMetadataJWTClientConfig.py +1 -1
- metadata/generated/schema/security/client/samlSSOClientConfig.py +1 -1
- metadata/generated/schema/security/credentials/__init__.py +1 -1
- metadata/generated/schema/security/credentials/accessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/apiAccessTokenAuth.py +1 -1
- metadata/generated/schema/security/credentials/awsCredentials.py +1 -1
- metadata/generated/schema/security/credentials/azureCredentials.py +1 -1
- metadata/generated/schema/security/credentials/basicAuth.py +1 -1
- metadata/generated/schema/security/credentials/bitbucketCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gcpExternalAccount.py +1 -1
- metadata/generated/schema/security/credentials/gcpValues.py +1 -1
- metadata/generated/schema/security/credentials/gitCredentials.py +1 -1
- metadata/generated/schema/security/credentials/githubCredentials.py +1 -1
- metadata/generated/schema/security/credentials/gitlabCredentials.py +1 -1
- metadata/generated/schema/security/credentials/kubernetesCredentials.py +1 -1
- metadata/generated/schema/security/sasl/__init__.py +1 -1
- metadata/generated/schema/security/sasl/saslClientConfig.py +1 -1
- metadata/generated/schema/security/secrets/__init__.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerClientLoader.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerConfiguration.py +1 -1
- metadata/generated/schema/security/secrets/secretsManagerProvider.py +1 -1
- metadata/generated/schema/security/securityConfiguration.py +1 -1
- metadata/generated/schema/security/ssl/__init__.py +1 -1
- metadata/generated/schema/security/ssl/validateSSLClientConfig.py +1 -1
- metadata/generated/schema/security/ssl/verifySSLConfig.py +1 -1
- metadata/generated/schema/settings/__init__.py +1 -1
- metadata/generated/schema/settings/settings.py +1 -1
- metadata/generated/schema/system/__init__.py +1 -1
- metadata/generated/schema/system/entityError.py +1 -1
- metadata/generated/schema/system/eventPublisherJob.py +1 -1
- metadata/generated/schema/system/indexingError.py +1 -1
- metadata/generated/schema/system/limitsResponse.py +1 -1
- metadata/generated/schema/system/securityValidationResponse.py +1 -1
- metadata/generated/schema/system/ui/__init__.py +1 -1
- metadata/generated/schema/system/ui/knowledgePanel.py +1 -1
- metadata/generated/schema/system/ui/navigationItem.py +1 -1
- metadata/generated/schema/system/ui/page.py +1 -1
- metadata/generated/schema/system/ui/tab.py +1 -1
- metadata/generated/schema/system/ui/uiCustomization.py +1 -1
- metadata/generated/schema/system/validationResponse.py +1 -1
- metadata/generated/schema/tests/__init__.py +1 -1
- metadata/generated/schema/tests/assigned.py +1 -1
- metadata/generated/schema/tests/basic.py +1 -1
- metadata/generated/schema/tests/customMetric.py +1 -1
- metadata/generated/schema/tests/dataQualityReport.py +1 -1
- metadata/generated/schema/tests/resolved.py +1 -1
- metadata/generated/schema/tests/testCase.py +1 -1
- metadata/generated/schema/tests/testCaseResolutionStatus.py +1 -1
- metadata/generated/schema/tests/testDefinition.py +1 -1
- metadata/generated/schema/tests/testSuite.py +1 -1
- metadata/generated/schema/type/__init__.py +1 -1
- metadata/generated/schema/type/apiSchema.py +1 -1
- metadata/generated/schema/type/assetCertification.py +1 -1
- metadata/generated/schema/type/auditLog.py +1 -1
- metadata/generated/schema/type/basic.py +1 -1
- metadata/generated/schema/type/bulkOperationResult.py +1 -1
- metadata/generated/schema/type/changeEvent.py +1 -1
- metadata/generated/schema/type/changeEventType.py +1 -1
- metadata/generated/schema/type/changeSummaryMap.py +1 -1
- metadata/generated/schema/type/collectionDescriptor.py +1 -1
- metadata/generated/schema/type/contextRecognizer.py +1 -1
- metadata/generated/schema/type/contractExecutionStatus.py +1 -1
- metadata/generated/schema/type/csvDocumentation.py +1 -1
- metadata/generated/schema/type/csvErrorType.py +1 -1
- metadata/generated/schema/type/csvFile.py +1 -1
- metadata/generated/schema/type/csvImportResult.py +1 -1
- metadata/generated/schema/type/customProperties/__init__.py +1 -1
- metadata/generated/schema/type/customProperties/complexTypes.py +1 -1
- metadata/generated/schema/type/customProperties/enumConfig.py +1 -1
- metadata/generated/schema/type/customProperties/tableConfig.py +1 -1
- metadata/generated/schema/type/customProperty.py +1 -1
- metadata/generated/schema/type/customRecognizer.py +1 -1
- metadata/generated/schema/type/dailyCount.py +1 -1
- metadata/generated/schema/type/databaseConnectionConfig.py +1 -1
- metadata/generated/schema/type/denyListRecognizer.py +1 -1
- metadata/generated/schema/type/entityHierarchy.py +1 -1
- metadata/generated/schema/type/entityHistory.py +1 -1
- metadata/generated/schema/type/entityLineage.py +1 -1
- metadata/generated/schema/type/entityProfile.py +1 -1
- metadata/generated/schema/type/entityReference.py +1 -1
- metadata/generated/schema/type/entityReferenceList.py +1 -1
- metadata/generated/schema/type/entityRelationship/__init__.py +1 -1
- metadata/generated/schema/type/entityRelationship/nodeInformation.py +1 -1
- metadata/generated/schema/type/entityUsage.py +1 -1
- metadata/generated/schema/type/filterPattern.py +1 -1
- metadata/generated/schema/type/function.py +1 -1
- metadata/generated/schema/type/include.py +1 -1
- metadata/generated/schema/type/jdbcConnection.py +1 -1
- metadata/generated/schema/type/layerPaging.py +1 -1
- metadata/generated/schema/type/lifeCycle.py +1 -1
- metadata/generated/schema/type/paging.py +1 -1
- metadata/generated/schema/type/patternRecognizer.py +1 -1
- metadata/generated/schema/type/personaPreferences.py +1 -1
- metadata/generated/schema/type/piiEntity.py +1 -1
- metadata/generated/schema/type/predefinedRecognizer.py +1 -1
- metadata/generated/schema/type/profile.py +1 -1
- metadata/generated/schema/type/queryParserData.py +1 -1
- metadata/generated/schema/type/reaction.py +1 -1
- metadata/generated/schema/type/recognizer.py +1 -1
- metadata/generated/schema/type/recognizerFeedback.py +1 -1
- metadata/generated/schema/type/recognizers/__init__.py +1 -1
- metadata/generated/schema/type/recognizers/patterns.py +1 -1
- metadata/generated/schema/type/recognizers/regexFlags.py +1 -1
- metadata/generated/schema/type/schedule.py +1 -1
- metadata/generated/schema/type/schema.py +1 -1
- metadata/generated/schema/type/status.py +1 -1
- metadata/generated/schema/type/tableQuery.py +1 -1
- metadata/generated/schema/type/tableUsageCount.py +1 -1
- metadata/generated/schema/type/tagLabel.py +1 -1
- metadata/generated/schema/type/usageDetails.py +1 -1
- metadata/generated/schema/type/usageRequest.py +1 -1
- metadata/generated/schema/type/votes.py +1 -1
- metadata/ingestion/ometa/mixins/es_mixin.py +5 -1
- metadata/ingestion/ometa/mixins/tests_mixin.py +1 -1
- metadata/ingestion/source/dashboard/grafana/models.py +1 -1
- metadata/ingestion/source/dashboard/hex/client.py +104 -0
- metadata/ingestion/source/dashboard/hex/connection.py +60 -0
- metadata/ingestion/source/dashboard/hex/metadata.py +363 -0
- metadata/ingestion/source/dashboard/hex/models.py +143 -0
- metadata/ingestion/source/dashboard/hex/query_fetcher.py +503 -0
- metadata/ingestion/source/dashboard/hex/service_spec.py +18 -0
- metadata/ingestion/source/dashboard/hex/warehouse_queries.py +225 -0
- metadata/ingestion/source/dashboard/powerbi/databricks_parser.py +102 -0
- metadata/ingestion/source/dashboard/powerbi/metadata.py +285 -90
- metadata/ingestion/source/dashboard/tableau/client.py +8 -4
- metadata/ingestion/source/database/bigquery/metadata.py +0 -8
- metadata/ingestion/source/database/databricks/client.py +120 -0
- metadata/ingestion/source/database/redshift/utils.py +2 -1
- metadata/ingestion/source/database/salesforce/metadata.py +7 -5
- metadata/ingestion/source/database/snowflake/lineage.py +44 -21
- metadata/ingestion/source/database/snowflake/queries.py +2 -0
- metadata/ingestion/source/database/snowflake/query_parser.py +12 -2
- metadata/ingestion/source/database/snowflake/usage.py +97 -0
- metadata/ingestion/source/database/unitycatalog/client.py +1 -1
- metadata/ingestion/source/database/unitycatalog/lineage.py +180 -5
- metadata/ingestion/source/database/unitycatalog/models.py +18 -2
- metadata/ingestion/source/pipeline/databrickspipeline/connection.py +1 -0
- metadata/ingestion/source/pipeline/databrickspipeline/kafka_parser.py +603 -0
- metadata/ingestion/source/pipeline/databrickspipeline/metadata.py +903 -37
- metadata/ingestion/source/pipeline/databrickspipeline/models.py +19 -3
- metadata/ingestion/source/pipeline/kafkaconnect/client.py +362 -30
- metadata/ingestion/source/pipeline/kafkaconnect/metadata.py +1666 -62
- metadata/ingestion/source/pipeline/kafkaconnect/models.py +35 -1
- metadata/ingestion/source/storage/s3/metadata.py +9 -2
- metadata/parsers/json_schema_parser.py +41 -1
- metadata/utils/fqn.py +59 -2
- {openmetadata_ingestion-1.10.1.0.dist-info → openmetadata_ingestion-1.10.3.0.dist-info}/METADATA +559 -560
- {openmetadata_ingestion-1.10.1.0.dist-info → openmetadata_ingestion-1.10.3.0.dist-info}/RECORD +855 -839
- {openmetadata_ingestion-1.10.1.0.dist-info → openmetadata_ingestion-1.10.3.0.dist-info}/LICENSE +0 -0
- {openmetadata_ingestion-1.10.1.0.dist-info → openmetadata_ingestion-1.10.3.0.dist-info}/WHEEL +0 -0
- {openmetadata_ingestion-1.10.1.0.dist-info → openmetadata_ingestion-1.10.3.0.dist-info}/entry_points.txt +0 -0
- {openmetadata_ingestion-1.10.1.0.dist-info → openmetadata_ingestion-1.10.3.0.dist-info}/top_level.txt +0 -0
metadata/clients/aws_client.py
CHANGED
|
@@ -36,6 +36,7 @@ class AWSServices(Enum):
|
|
|
36
36
|
GLUE = "glue"
|
|
37
37
|
SAGEMAKER = "sagemaker"
|
|
38
38
|
KINESIS = "kinesis"
|
|
39
|
+
FIREHOSE = "firehose"
|
|
39
40
|
QUICKSIGHT = "quicksight"
|
|
40
41
|
ATHENA = "athena"
|
|
41
42
|
RDS = "rds"
|
|
@@ -233,6 +234,9 @@ class AWSClient:
|
|
|
233
234
|
def get_kinesis_client(self):
|
|
234
235
|
return self.get_client(AWSServices.KINESIS.value)
|
|
235
236
|
|
|
237
|
+
def get_firehose_client(self):
|
|
238
|
+
return self.get_client(AWSServices.FIREHOSE.value)
|
|
239
|
+
|
|
236
240
|
def get_quicksight_client(self):
|
|
237
241
|
return self.get_client(AWSServices.QUICKSIGHT.value)
|
|
238
242
|
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
source:
|
|
2
|
+
type: kafkaconnect
|
|
3
|
+
serviceName: confluent_cdc_mysql_postgres
|
|
4
|
+
serviceConnection:
|
|
5
|
+
config:
|
|
6
|
+
type: KafkaConnect
|
|
7
|
+
hostPort: http://localhost:8083
|
|
8
|
+
# For Kafka Connect, choose one of the following authentication methods:
|
|
9
|
+
|
|
10
|
+
# Option 1: Username/Password Authentication (for self-hosted Kafka Connect)
|
|
11
|
+
KafkaConnectConfig:
|
|
12
|
+
username: admin
|
|
13
|
+
password: admin_password
|
|
14
|
+
|
|
15
|
+
# Option 2: API Key Authentication (for Confluent Cloud)
|
|
16
|
+
# KafkaConnectConfig:
|
|
17
|
+
# username: YOUR_CONFLUENT_CLOUD_API_KEY
|
|
18
|
+
# password: YOUR_CONFLUENT_CLOUD_API_SECRET
|
|
19
|
+
|
|
20
|
+
# Option 3: No Authentication
|
|
21
|
+
# KafkaConnectConfig: null
|
|
22
|
+
|
|
23
|
+
verifySSL: true
|
|
24
|
+
messagingServiceName: KafkaProd
|
|
25
|
+
|
|
26
|
+
# Optional: Filter CDC connectors/pipelines using regex patterns
|
|
27
|
+
# pipelineFilterPattern:
|
|
28
|
+
# includes:
|
|
29
|
+
# - "mysql-cdc-.*"
|
|
30
|
+
# - "postgres-sink-.*"
|
|
31
|
+
# excludes:
|
|
32
|
+
# - ".*test.*"
|
|
33
|
+
# - ".*dev.*"
|
|
34
|
+
|
|
35
|
+
sourceConfig:
|
|
36
|
+
config:
|
|
37
|
+
type: PipelineMetadata
|
|
38
|
+
lineageInformation:
|
|
39
|
+
dbServiceNames:
|
|
40
|
+
- "MysqlProd"
|
|
41
|
+
- "PostgresProd"
|
|
42
|
+
storageServiceNames: []
|
|
43
|
+
|
|
44
|
+
sink:
|
|
45
|
+
type: metadata-rest
|
|
46
|
+
config: {}
|
|
47
|
+
|
|
48
|
+
workflowConfig:
|
|
49
|
+
loggerLevel: INFO # DEBUG, INFO, WARN or ERROR
|
|
50
|
+
openMetadataServerConfig:
|
|
51
|
+
hostPort: http://localhost:8585/api
|
|
52
|
+
authProvider: openmetadata
|
|
53
|
+
securityConfig:
|
|
54
|
+
jwtToken: <your-jwt-token-here>
|
|
55
|
+
|
|
56
|
+
# To run this workflow:
|
|
57
|
+
# metadata ingest -c /path/to/confluent_cdc.yaml
|
|
58
|
+
|
|
59
|
+
# Prerequisites:
|
|
60
|
+
# 1. OpenMetadata server running at http://localhost:8585
|
|
61
|
+
# 2. MySQL database service already ingested into OpenMetadata
|
|
62
|
+
# 3. Postgres database service already ingested into OpenMetadata
|
|
63
|
+
# 4. Kafka messaging service already ingested into OpenMetadata
|
|
64
|
+
# 5. Kafka Connect connectors configured and running
|
|
65
|
+
#
|
|
66
|
+
# The connector will create:
|
|
67
|
+
# - Pipeline entities for each CDC connector
|
|
68
|
+
# - Table → Topic lineage with column-level mapping
|
|
69
|
+
# - Topic → Table lineage with column-level mapping
|
|
70
|
+
# - Complete end-to-end lineage: MySQL columns → Kafka topic fields → Postgres columns
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# Example: Confluent Cloud CDC Configuration
|
|
2
|
+
# This example shows how to connect to Confluent Cloud CDC connectors
|
|
3
|
+
# that replicate data from MySQL to PostgreSQL via Kafka
|
|
4
|
+
|
|
5
|
+
source:
|
|
6
|
+
type: kafkaconnect
|
|
7
|
+
serviceName: confluent_cloud_cdc_production
|
|
8
|
+
serviceConnection:
|
|
9
|
+
config:
|
|
10
|
+
type: KafkaConnect
|
|
11
|
+
# Confluent Cloud Kafka Connect REST API endpoint
|
|
12
|
+
hostPort: https://pkc-xxxxx.us-east-1.aws.confluent.cloud:443
|
|
13
|
+
|
|
14
|
+
# Confluent Cloud API Key authentication
|
|
15
|
+
KafkaConnectConfig:
|
|
16
|
+
username: YOUR_CONFLUENT_CLOUD_API_KEY
|
|
17
|
+
password: YOUR_CONFLUENT_CLOUD_API_SECRET
|
|
18
|
+
|
|
19
|
+
verifySSL: true
|
|
20
|
+
messagingServiceName: KafkaCloudProd
|
|
21
|
+
|
|
22
|
+
# Filter to include only production CDC connectors
|
|
23
|
+
pipelineFilterPattern:
|
|
24
|
+
includes:
|
|
25
|
+
- "prod-mysql-cdc-.*"
|
|
26
|
+
- "prod-postgres-sink-.*"
|
|
27
|
+
excludes:
|
|
28
|
+
- ".*staging.*"
|
|
29
|
+
- ".*test.*"
|
|
30
|
+
|
|
31
|
+
sourceConfig:
|
|
32
|
+
config:
|
|
33
|
+
type: PipelineMetadata
|
|
34
|
+
lineageInformation:
|
|
35
|
+
dbServiceNames:
|
|
36
|
+
- "MysqlProd"
|
|
37
|
+
- "PostgresProd"
|
|
38
|
+
storageServiceNames: []
|
|
39
|
+
|
|
40
|
+
sink:
|
|
41
|
+
type: metadata-rest
|
|
42
|
+
config: {}
|
|
43
|
+
|
|
44
|
+
workflowConfig:
|
|
45
|
+
loggerLevel: INFO
|
|
46
|
+
openMetadataServerConfig:
|
|
47
|
+
hostPort: https://your-openmetadata-server.com/api
|
|
48
|
+
authProvider: openmetadata
|
|
49
|
+
securityConfig:
|
|
50
|
+
jwtToken: <your-jwt-token>
|
|
51
|
+
|
|
52
|
+
# Example CDC Connector Configuration that this workflow will discover:
|
|
53
|
+
#
|
|
54
|
+
# MySQL CDC Source Connector:
|
|
55
|
+
# {
|
|
56
|
+
# "name": "prod-mysql-cdc-customers",
|
|
57
|
+
# "config": {
|
|
58
|
+
# "connector.class": "io.debezium.connector.mysql.MySqlConnector",
|
|
59
|
+
# "database.hostname": "mysql.example.com",
|
|
60
|
+
# "database.name": "ecommerce",
|
|
61
|
+
# "table.include.list": "ecommerce.customers,ecommerce.orders",
|
|
62
|
+
# "database.server.name": "prod-mysql",
|
|
63
|
+
# "topics": "prod-mysql.ecommerce.customers"
|
|
64
|
+
# }
|
|
65
|
+
# }
|
|
66
|
+
#
|
|
67
|
+
# Postgres Sink Connector:
|
|
68
|
+
# {
|
|
69
|
+
# "name": "prod-postgres-sink-customers",
|
|
70
|
+
# "config": {
|
|
71
|
+
# "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
|
|
72
|
+
# "connection.url": "jdbc:postgresql://postgres.example.com:5432/analytics",
|
|
73
|
+
# "topics": "prod-mysql.ecommerce.customers",
|
|
74
|
+
# "table.name.format": "customers",
|
|
75
|
+
# "schema": "public"
|
|
76
|
+
# }
|
|
77
|
+
# }
|
|
78
|
+
#
|
|
79
|
+
# This will create lineage:
|
|
80
|
+
# MySQL: ecommerce.customers (id, name, email, created_at)
|
|
81
|
+
# ↓
|
|
82
|
+
# Kafka: prod-mysql.ecommerce.customers (id, name, email, created_at) [topic schema fields]
|
|
83
|
+
# ↓
|
|
84
|
+
# Postgres: analytics.public.customers (id, name, email, created_at)
|
|
85
|
+
#
|
|
86
|
+
# With column-level lineage showing exact field mappings at each hop!
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# Example: Local Confluent CDC Setup for Development
|
|
2
|
+
# This example shows a typical local development setup with:
|
|
3
|
+
# - Local Kafka Connect cluster
|
|
4
|
+
# - MySQL source database
|
|
5
|
+
# - PostgreSQL target database
|
|
6
|
+
# - Local Kafka broker
|
|
7
|
+
|
|
8
|
+
source:
|
|
9
|
+
type: kafkaconnect
|
|
10
|
+
serviceName: local_cdc_dev
|
|
11
|
+
serviceConnection:
|
|
12
|
+
config:
|
|
13
|
+
type: KafkaConnect
|
|
14
|
+
# Local Kafka Connect REST API
|
|
15
|
+
hostPort: http://localhost:8083
|
|
16
|
+
|
|
17
|
+
# No authentication for local development
|
|
18
|
+
# KafkaConnectConfig: null
|
|
19
|
+
|
|
20
|
+
# Or use basic auth if configured
|
|
21
|
+
KafkaConnectConfig:
|
|
22
|
+
username: connect-user
|
|
23
|
+
password: connect-password
|
|
24
|
+
|
|
25
|
+
verifySSL: false # Typically disabled for local development
|
|
26
|
+
messagingServiceName: KafkaLocal
|
|
27
|
+
|
|
28
|
+
sourceConfig:
|
|
29
|
+
config:
|
|
30
|
+
type: PipelineMetadata
|
|
31
|
+
lineageInformation:
|
|
32
|
+
dbServiceNames:
|
|
33
|
+
- "MysqlLocal"
|
|
34
|
+
- "PostgresLocal"
|
|
35
|
+
storageServiceNames: []
|
|
36
|
+
|
|
37
|
+
sink:
|
|
38
|
+
type: metadata-rest
|
|
39
|
+
config: {}
|
|
40
|
+
|
|
41
|
+
workflowConfig:
|
|
42
|
+
loggerLevel: DEBUG # Use DEBUG for development to see detailed logs
|
|
43
|
+
openMetadataServerConfig:
|
|
44
|
+
hostPort: http://localhost:8585/api
|
|
45
|
+
authProvider: openmetadata
|
|
46
|
+
securityConfig:
|
|
47
|
+
jwtToken: eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg
|
|
48
|
+
|
|
49
|
+
# ============================================================================
|
|
50
|
+
# Local Development Setup Guide
|
|
51
|
+
# ============================================================================
|
|
52
|
+
#
|
|
53
|
+
# 1. Start local infrastructure:
|
|
54
|
+
# docker-compose up -d # Start MySQL, Postgres, Kafka, Kafka Connect
|
|
55
|
+
#
|
|
56
|
+
# 2. Ingest database metadata into OpenMetadata:
|
|
57
|
+
# # MySQL
|
|
58
|
+
# metadata ingest -c mysql_local.yaml
|
|
59
|
+
#
|
|
60
|
+
# # PostgreSQL
|
|
61
|
+
# metadata ingest -c postgres_local.yaml
|
|
62
|
+
#
|
|
63
|
+
# # Kafka
|
|
64
|
+
# metadata ingest -c kafka_local.yaml
|
|
65
|
+
#
|
|
66
|
+
# 3. Configure CDC connectors in Kafka Connect:
|
|
67
|
+
#
|
|
68
|
+
# # MySQL CDC Source Connector (Debezium)
|
|
69
|
+
# curl -X POST http://localhost:8083/connectors \
|
|
70
|
+
# -H "Content-Type: application/json" \
|
|
71
|
+
# -d '{
|
|
72
|
+
# "name": "mysql-source-customers",
|
|
73
|
+
# "config": {
|
|
74
|
+
# "connector.class": "io.debezium.connector.mysql.MySqlConnector",
|
|
75
|
+
# "database.hostname": "mysql",
|
|
76
|
+
# "database.port": "3306",
|
|
77
|
+
# "database.user": "debezium",
|
|
78
|
+
# "database.password": "dbz",
|
|
79
|
+
# "database.server.id": "184054",
|
|
80
|
+
# "database.server.name": "local-mysql",
|
|
81
|
+
# "database.name": "testdb",
|
|
82
|
+
# "table.include.list": "testdb.customers,testdb.orders",
|
|
83
|
+
# "database.history.kafka.bootstrap.servers": "kafka:9092",
|
|
84
|
+
# "database.history.kafka.topic": "schema-changes.testdb"
|
|
85
|
+
# }
|
|
86
|
+
# }'
|
|
87
|
+
#
|
|
88
|
+
# # PostgreSQL Sink Connector
|
|
89
|
+
# curl -X POST http://localhost:8083/connectors \
|
|
90
|
+
# -H "Content-Type: application/json" \
|
|
91
|
+
# -d '{
|
|
92
|
+
# "name": "postgres-sink-customers",
|
|
93
|
+
# "config": {
|
|
94
|
+
# "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
|
|
95
|
+
# "connection.url": "jdbc:postgresql://postgres:5432/warehouse",
|
|
96
|
+
# "connection.user": "postgres",
|
|
97
|
+
# "connection.password": "postgres",
|
|
98
|
+
# "topics": "local-mysql.testdb.customers",
|
|
99
|
+
# "table.name.format": "customers",
|
|
100
|
+
# "auto.create": "true",
|
|
101
|
+
# "auto.evolve": "true",
|
|
102
|
+
# "insert.mode": "upsert",
|
|
103
|
+
# "pk.mode": "record_key",
|
|
104
|
+
# "delete.enabled": "true"
|
|
105
|
+
# }
|
|
106
|
+
# }'
|
|
107
|
+
#
|
|
108
|
+
# 4. Run this CDC workflow:
|
|
109
|
+
# metadata ingest -c confluent_cdc_local.yaml
|
|
110
|
+
#
|
|
111
|
+
# 5. View lineage in OpenMetadata UI:
|
|
112
|
+
# http://localhost:8585
|
|
113
|
+
# Navigate to: Pipeline Services → local_cdc_dev → Pipelines
|
|
114
|
+
# Click on a pipeline to see column-level lineage
|
|
115
|
+
#
|
|
116
|
+
# Expected Lineage:
|
|
117
|
+
# testdb.customers.id → local-mysql.testdb.customers.id → warehouse.public.customers.id
|
|
118
|
+
# testdb.customers.name → local-mysql.testdb.customers.name → warehouse.public.customers.name
|
|
119
|
+
# testdb.customers.email → local-mysql.testdb.customers.email → warehouse.public.customers.email
|
|
120
|
+
# testdb.customers.created → local-mysql.testdb.customers.created → warehouse.public.customers.created
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# AWS Kinesis Firehose Workflow Configuration
|
|
2
|
+
#
|
|
3
|
+
# This workflow extracts metadata from AWS Kinesis Firehose delivery streams
|
|
4
|
+
# and creates lineage between sources (DynamoDB) and destinations
|
|
5
|
+
# (S3, Redshift, OpenSearch, Snowflake)
|
|
6
|
+
#
|
|
7
|
+
# Supported Features:
|
|
8
|
+
# - Pipeline metadata extraction
|
|
9
|
+
# - Entity-level lineage (DynamoDB → Firehose → Destination)
|
|
10
|
+
# - Column-level lineage (automatic column matching by name)
|
|
11
|
+
#
|
|
12
|
+
# Required IAM Permissions:
|
|
13
|
+
# - firehose:ListDeliveryStreams
|
|
14
|
+
# - firehose:DescribeDeliveryStream
|
|
15
|
+
|
|
16
|
+
source:
|
|
17
|
+
type: kinesisfirehose
|
|
18
|
+
serviceName: aws_firehose_production
|
|
19
|
+
serviceConnection:
|
|
20
|
+
config:
|
|
21
|
+
type: KinesisFirehose
|
|
22
|
+
awsConfig:
|
|
23
|
+
# AWS Region where Firehose delivery streams are located
|
|
24
|
+
awsRegion: us-east-1
|
|
25
|
+
|
|
26
|
+
# Authentication Option 1: Access Key & Secret (for development/testing)
|
|
27
|
+
awsAccessKeyId: AKIAIOSFODNN7EXAMPLE
|
|
28
|
+
awsSecretAccessKey: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
|
29
|
+
|
|
30
|
+
# Authentication Option 2: Temporary Session Token (for STS)
|
|
31
|
+
# awsSessionToken: "temporary-session-token"
|
|
32
|
+
|
|
33
|
+
# Authentication Option 3: AWS Profile (uses ~/.aws/credentials)
|
|
34
|
+
# profileName: "openmetadata-profile"
|
|
35
|
+
|
|
36
|
+
# Authentication Option 4: Assume Role (for cross-account access)
|
|
37
|
+
# assumeRoleArn: "arn:aws:iam::123456789012:role/OpenMetadataFirehoseRole"
|
|
38
|
+
# assumeRoleSessionName: "OpenMetadataFirehoseSession"
|
|
39
|
+
# assumeRoleSourceIdentity: "arn:aws:iam::987654321098:user/admin"
|
|
40
|
+
|
|
41
|
+
# Optional: Custom endpoint for LocalStack or VPC endpoints
|
|
42
|
+
# endPointURL: "https://firehose.us-east-1.amazonaws.com/"
|
|
43
|
+
|
|
44
|
+
sourceConfig:
|
|
45
|
+
config:
|
|
46
|
+
type: PipelineMetadata
|
|
47
|
+
|
|
48
|
+
# Optional: Filter delivery streams by name pattern
|
|
49
|
+
# pipelineFilterPattern:
|
|
50
|
+
# includes:
|
|
51
|
+
# - "prod-.*"
|
|
52
|
+
# - "cdc-.*"
|
|
53
|
+
# excludes:
|
|
54
|
+
# - "test-.*"
|
|
55
|
+
# - "dev-.*"
|
|
56
|
+
|
|
57
|
+
# Optional: Enable lineage extraction (default: true)
|
|
58
|
+
# includeLineage: true
|
|
59
|
+
|
|
60
|
+
# Optional: Mark deleted pipelines (delivery streams no longer in AWS)
|
|
61
|
+
# markDeletedPipelines: true
|
|
62
|
+
|
|
63
|
+
sink:
|
|
64
|
+
type: metadata-rest
|
|
65
|
+
config: {}
|
|
66
|
+
|
|
67
|
+
workflowConfig:
|
|
68
|
+
# Optional: Set log level (DEBUG, INFO, WARN, ERROR)
|
|
69
|
+
# loggerLevel: INFO
|
|
70
|
+
|
|
71
|
+
openMetadataServerConfig:
|
|
72
|
+
hostPort: "http://localhost:8585/api"
|
|
73
|
+
authProvider: openmetadata
|
|
74
|
+
securityConfig:
|
|
75
|
+
jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
|
|
76
|
+
|
|
77
|
+
# Lineage Support:
|
|
78
|
+
#
|
|
79
|
+
# This connector automatically creates lineage when it finds:
|
|
80
|
+
#
|
|
81
|
+
# 1. DynamoDB → Firehose → S3
|
|
82
|
+
# - Source: DynamoDB table (from KinesisStreamSourceConfiguration)
|
|
83
|
+
# - Destination: S3 container (from S3DestinationDescription)
|
|
84
|
+
#
|
|
85
|
+
# 2. DynamoDB → Firehose → Redshift
|
|
86
|
+
# - Source: DynamoDB table
|
|
87
|
+
# - Destination: Redshift table (from RedshiftDestinationDescription)
|
|
88
|
+
#
|
|
89
|
+
# 3. DynamoDB → Firehose → OpenSearch
|
|
90
|
+
# - Source: DynamoDB table
|
|
91
|
+
# - Destination: OpenSearch index (from AmazonopensearchserviceDestinationDescription)
|
|
92
|
+
#
|
|
93
|
+
# 4. DynamoDB → Firehose → Snowflake
|
|
94
|
+
# - Source: DynamoDB table
|
|
95
|
+
# - Destination: Snowflake table (from SnowflakeDestinationDescription)
|
|
96
|
+
#
|
|
97
|
+
# Note: Both source and destination entities must exist in OpenMetadata
|
|
98
|
+
# for lineage to be created. Column-level lineage is automatically
|
|
99
|
+
# generated by matching column names (case-insensitive).
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
source:
|
|
2
|
+
type: kafkaconnect
|
|
3
|
+
serviceName: confluent_cloud_cdc
|
|
4
|
+
serviceConnection:
|
|
5
|
+
config:
|
|
6
|
+
type: KafkaConnect
|
|
7
|
+
# IMPORTANT: Use the Kafka Connect API endpoint, NOT the Kafka broker endpoint
|
|
8
|
+
# Format: https://api.confluent.cloud/connect/v1/environments/{ENV_ID}/clusters/{CLUSTER_ID}
|
|
9
|
+
# Example: https://api.confluent.cloud/connect/v1/environments/env-abc123/clusters/lkc-xyz789
|
|
10
|
+
hostPort: https://api.confluent.cloud/connect/v1/environments/YOUR_ENV_ID/clusters/YOUR_CONNECT_CLUSTER_ID
|
|
11
|
+
|
|
12
|
+
# Use Kafka Connect API Key (NOT Kafka broker API key)
|
|
13
|
+
KafkaConnectConfig:
|
|
14
|
+
username: YOUR_KAFKA_CONNECT_API_KEY
|
|
15
|
+
password: YOUR_KAFKA_CONNECT_API_SECRET
|
|
16
|
+
|
|
17
|
+
verifySSL: true
|
|
18
|
+
|
|
19
|
+
# REQUIRED for Confluent Cloud: Specify your Kafka messaging service
|
|
20
|
+
# This should match the service name you used when ingesting Kafka topics
|
|
21
|
+
messagingServiceName: confluent_kafka_cloud
|
|
22
|
+
|
|
23
|
+
# Optional: Filter specific connectors
|
|
24
|
+
# pipelineFilterPattern:
|
|
25
|
+
# includes:
|
|
26
|
+
# - "mysql-cdc-.*"
|
|
27
|
+
# - "postgres-sink-.*"
|
|
28
|
+
|
|
29
|
+
sourceConfig:
|
|
30
|
+
config:
|
|
31
|
+
type: PipelineMetadata
|
|
32
|
+
# Specify the database services where your tables are ingested
|
|
33
|
+
lineageInformation:
|
|
34
|
+
dbServiceNames:
|
|
35
|
+
- local_mysql # Replace with your MySQL service name
|
|
36
|
+
- local_postgres # Replace with your Postgres service name
|
|
37
|
+
storageServiceNames: [] # Add S3/GCS service names if using sink connectors to storage
|
|
38
|
+
|
|
39
|
+
sink:
|
|
40
|
+
type: metadata-rest
|
|
41
|
+
config: {}
|
|
42
|
+
|
|
43
|
+
workflowConfig:
|
|
44
|
+
loggerLevel: DEBUG # Use DEBUG to see detailed connection info
|
|
45
|
+
openMetadataServerConfig:
|
|
46
|
+
hostPort: http://localhost:8585/api
|
|
47
|
+
authProvider: openmetadata
|
|
48
|
+
securityConfig:
|
|
49
|
+
jwtToken: eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg
|