openmetadata-ingestion 1.4.0.0rc1__py3-none-any.whl → 1.4.0.0rc3__py3-none-any.whl

metadata/__version__.py

@@ -16,6 +16,8 @@ import os
 import re
 import sys
 
+import pkg_resources
+
 try:
     from importlib.metadata import version
 except ImportError:
@@ -85,3 +87,14 @@ def get_major_minor_version() -> str:
     """
     major, minor, *_ = sys.version_info
     return f"{major}.{minor}"
+
+
+def match_versions(version1: str, version2: str) -> bool:
+    """Check if both versions match in minor and major"""
+    server_semver = pkg_resources.parse_version(version1)
+    client_semver = pkg_resources.parse_version(version2)
+
+    return (
+        server_semver.major == client_semver.major
+        and server_semver.minor == client_semver.minor
+    )

metadata/cli/app.py

@@ -31,7 +31,6 @@ def run_app(config_path: Path) -> None:
     :param config_path: Path to load JSON config
     """
 
-    config_dict = None
     try:
         config_dict = load_config_file(config_path)
         workflow = ApplicationWorkflow.create(config_dict)

metadata/cmd.py

@@ -19,15 +19,12 @@ from pathlib import Path
 
 from metadata.__version__ import get_metadata_version
 from metadata.cli.app import run_app
-from metadata.cli.backup import UploadDestinationType, run_backup
 from metadata.cli.dataquality import run_test
 from metadata.cli.ingest import run_ingest
 from metadata.cli.insight import run_insight
 from metadata.cli.lineage import run_lineage
 from metadata.cli.profile import run_profiler
-from metadata.cli.restore import run_restore
 from metadata.cli.usage import run_usage
-from metadata.utils.helpers import BackupRestoreArgs
 from metadata.utils.logger import cli_logger, set_loggers_level
 
 logger = cli_logger()
@@ -38,8 +35,6 @@ class MetadataCommands(Enum):
     USAGE = "usage"
     PROFILE = "profile"
     TEST = "test"
-    BACKUP = "backup"
-    RESTORE = "restore"
     WEBHOOK = "webhook"
     INSIGHT = "insight"
     LINEAGE = "lineage"
@@ -57,32 +52,6 @@ RUN_PATH_METHODS = {
 }
 
 
-BACKUP_HELP = """
-    Run a backup for the metadata DB. Uses a custom dump strategy for OpenMetadata tables.
-
-    We can pass as many connection options as required with `-o <opt1>, -o <opt2> [...]`
-    Same with connection arguments `-a <arg1>, -a <arg2> [...]`
-
-    To run the upload, provide the information as
-    `--upload endpoint bucket key` and properly configure the environment
-    variables AWS_ACCESS_KEY_ID & AWS_SECRET_ACCESS_KEY.
-
-    If `-s` or `--schema` is provided, we will trigger a Postgres backup instead
-    of a MySQL backup. This is the value of the schema containing the OpenMetadata
-    tables.
-    """
-RESTORE_HELP = """
-    Run a restore for the metadata DB.
-
-    We can pass as many connection options as required with `-o <opt1>, -o <opt2> [...]`
-    Same with connection arguments `-a <arg1>, -a <arg2> [...]`
-
-    If `-s` or `--schema` is provided, we will trigger a Postgres Restore instead
-    of a MySQL restore. This is the value of the schema containing the OpenMetadata
-    tables.
-    """
-
-
 def create_common_config_parser_args(parser: argparse.ArgumentParser):
     parser.add_argument(
         "-c",
@@ -103,125 +72,6 @@ def webhook_args(parser: argparse.ArgumentParser):
     parser.add_argument("-p", "--port", help="Webserver Port", type=int, default=8000)
 
 
-def backup_args(parser: argparse.ArgumentParser):
-    """
-    Additional Parser Arguments for Backup
-    """
-    parser.add_argument(
-        "-H", "--host", help="Host that runs the database", required=True
-    )
-    parser.add_argument(
-        "-u",
-        "--user",
-        help="User to run the backup",
-        required=True,
-    )
-    parser.add_argument(
-        "-p",
-        "--password",
-        help="Credentials for the user",
-        required=True,
-    )
-    parser.add_argument(
-        "-d",
-        "--database",
-        help="Database to backup",
-        required=True,
-    )
-    parser.add_argument(
-        "--port",
-        help="Database service port",
-        default="3306",
-    )
-    parser.add_argument(
-        "--output",
-        help="Local path to store the backup",
-        type=Path,
-        default=None,
-    )
-    parser.add_argument(
-        "--filename",
-        help="Filename to store the backup",
-        default=None,
-    )
-    parser.add_argument(
-        "--upload-destination-type",
-        help="AWS or AZURE",
-        choices=UploadDestinationType.__members__,
-        default=None,
-    )
-    parser.add_argument(
-        "--upload",
-        help="S3 endpoint, bucket & key to upload the backup file",
-        nargs=3,
-        default=None,
-    )
-    parser.add_argument("-o", "--options", default=None, action="append")
-    parser.add_argument("-a", "--arguments", default=None, action="append")
-    parser.add_argument(
-        "-s",
-        "--schema",
-        default=None,
-    )
-
-
-def restore_args(parser: argparse.ArgumentParser):
-    """
-    Additional Parser Arguments for Restore
-    """
-    parser.add_argument(
-        "-H",
-        "--host",
-        help="Host that runs the database",
-        required=True,
-    )
-    parser.add_argument(
-        "-u",
-        "--user",
-        help="User to run the restore backup",
-        required=True,
-    )
-
-    parser.add_argument(
-        "-p",
-        "--password",
-        help="Credentials for the user",
-        required=True,
-    )
-
-    parser.add_argument(
-        "-d",
-        "--database",
-        help="Database to restore",
-        required=True,
-    )
-
-    parser.add_argument(
-        "--port",
-        help="Database service port",
-        default="3306",
-        required=False,
-    )
-
-    parser.add_argument(
-        "--input",
-        help="Local backup file path for restore",
-        type=Path,
-        required=True,
-    )
-
-    parser.add_argument("-o", "--options", default=None, action="append")
-
-    parser.add_argument("-a", "--arguments", default=None, action="append")
-
-    parser.add_argument(
-        "-s",
-        "--schema",
-        default=None,
-        required=False,
-    )
-
-
 def add_metadata_args(parser: argparse.ArgumentParser):
     """
     Additional Parser Arguments for Metadata
@@ -274,18 +124,6 @@ def get_parser(args=None):
             help="Workflow for running external applications",
         )
     )
-    backup_args(
-        sub_parser.add_parser(
-            MetadataCommands.BACKUP.value,
-            help=BACKUP_HELP,
-        )
-    )
-    restore_args(
-        sub_parser.add_parser(
-            MetadataCommands.RESTORE.value,
-            help=RESTORE_HELP,
-        )
-    )
     webhook_args(
         sub_parser.add_parser(
             MetadataCommands.WEBHOOK.value,
@@ -323,37 +161,6 @@ def metadata(args=None):
     if metadata_workflow in RUN_PATH_METHODS:
         RUN_PATH_METHODS[metadata_workflow](path)
 
-    if metadata_workflow == MetadataCommands.BACKUP.value:
-        run_backup(
-            common_backup_obj_instance=BackupRestoreArgs(
-                host=contains_args.get("host"),
-                user=contains_args.get("user"),
-                password=contains_args.get("password"),
-                database=contains_args.get("database"),
-                port=contains_args.get("port"),
-                options=contains_args.get("options"),
-                arguments=contains_args.get("arguments"),
-                schema=contains_args.get("schema"),
-            ),
-            output=contains_args.get("output"),
-            filename=contains_args.get("filename"),
-            upload_destination_type=contains_args.get("upload_destination_type"),
-            upload=contains_args.get("upload"),
-        )
-    if metadata_workflow == MetadataCommands.RESTORE.value:
-        run_restore(
-            common_restore_obj_instance=BackupRestoreArgs(
-                host=contains_args.get("host"),
-                user=contains_args.get("user"),
-                password=contains_args.get("password"),
-                database=contains_args.get("database"),
-                port=contains_args.get("port"),
-                options=contains_args.get("options"),
-                arguments=contains_args.get("arguments"),
-                schema=contains_args.get("schema"),
-            ),
-            sql_file=contains_args.get("input"),
-        )
     if metadata_workflow == MetadataCommands.WEBHOOK.value:
 
         class WebhookHandler(BaseHTTPRequestHandler):

metadata/data_quality/source/test_suite.py

@@ -139,7 +139,13 @@ class TestSuiteSource(Source):
                 )
             )
 
-        if table.testSuite and not table.testSuite.executable:
+        test_suite: Optional[TestSuite] = None
+        if table.testSuite:
+            test_suite = self.metadata.get_by_id(
+                entity=TestSuite, entity_id=table.testSuite.id.__root__
+            )
+
+        if test_suite and not test_suite.executable:
             yield Either(
                 left=StackTraceError(
                     name="Non-executable Test Suite",
@@ -149,7 +155,7 @@ class TestSuiteSource(Source):
             )
 
         else:
-            test_suite_cases = self._get_test_cases_from_test_suite(table.testSuite)
+            test_suite_cases = self._get_test_cases_from_test_suite(test_suite)
 
             yield Either(
                 right=TableAndTests(

metadata/examples/workflows/kafka_connect.yaml

@@ -0,0 +1,30 @@
+source:
+  type: kafkaconnect
+  serviceName: kafka_connect_source
+  serviceConnection:
+    config:
+      type: KafkaConnect
+      # For KafkaConnect, choose one of noAuth or basic
+      # # For basic authentication
+      # authType:
+      #   username: username
+      #   password: password
+      hostPort: http://localhost:8083
+      verifySSL: true
+      messagingServiceName: local_kafka_source
+  sourceConfig:
+    config:
+      type: PipelineMetadata
+      lineageInformation:
+        dbServiceNames: []
+        storageServiceNames: []
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: INFO # DEBUG, INFO, WARN or ERROR
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg

metadata/generated/schema/analytics/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00

metadata/generated/schema/analytics/basic.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/basic.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportData.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00

metadata/generated/schema/analytics/reportDataType/aggregatedCostAnalysisReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/aggregatedCostAnalysisReportData.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/entityReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/entityReportData.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/rawCostAnalysisReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/rawCostAnalysisReportData.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/webAnalyticEntityViewReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/webAnalyticEntityViewReportData.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/reportDataType/webAnalyticUserActivityReportData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/reportDataType/webAnalyticUserActivityReportData.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/webAnalyticEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEvent.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/webAnalyticEventData.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEventData.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/webAnalyticEventType/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00

metadata/generated/schema/analytics/webAnalyticEventType/customEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEventType/customEvent.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/analytics/webAnalyticEventType/pageViewEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  analytics/webAnalyticEventType/pageViewEvent.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00

metadata/generated/schema/api/addGlossaryToAssetsRequest.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/addGlossaryToAssetsRequest.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/analytics/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00

metadata/generated/schema/api/analytics/createWebAnalyticEvent.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/analytics/createWebAnalyticEvent.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/automations/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00

metadata/generated/schema/api/automations/createWorkflow.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/automations/createWorkflow.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/bulkAssets.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/bulkAssets.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/classification/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00

metadata/generated/schema/api/classification/createClassification.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/classification/createClassification.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/classification/createTag.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/classification/createTag.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/classification/loadTags.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/classification/loadTags.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/createBot.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/createBot.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/createEventPublisherJob.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/createEventPublisherJob.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/createType.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/createType.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/data/__init__.py

@@ -1,3 +1,3 @@
 # generated by datamodel-codegen:
 #   filename:  schema
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00

metadata/generated/schema/api/data/createChart.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/data/createChart.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/data/createContainer.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/data/createContainer.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/data/createCustomProperty.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/data/createCustomProperty.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/data/createDashboard.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/data/createDashboard.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/data/createDashboardDataModel.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/data/createDashboardDataModel.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/data/createDatabase.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/data/createDatabase.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/data/createDatabaseSchema.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/data/createDatabaseSchema.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/data/createGlossary.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/data/createGlossary.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations
 

metadata/generated/schema/api/data/createGlossaryTerm.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api/data/createGlossaryTerm.json
-#   timestamp: 2024-05-02T07:41:57+00:00
+#   timestamp: 2024-05-17T18:24:48+00:00
 
 from __future__ import annotations