fivetran-connector-sdk 1.7.1__tar.gz → 1.7.3__tar.gz

{fivetran_connector_sdk-1.7.1 → fivetran_connector_sdk-1.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.7.1
+Version: 1.7.3
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-1.7.1 → fivetran_connector_sdk-1.7.3}/src/fivetran_connector_sdk/__init__.py

@@ -6,6 +6,9 @@ import shutil
 import argparse
 import traceback
 import requests as rq
+import threading
+import queue
+from types import GeneratorType
 from http import HTTPStatus
 from zipfile import ZipFile
 from concurrent import futures
@@ -19,7 +22,8 @@ from fivetran_connector_sdk.operations import Operations
 from fivetran_connector_sdk import constants
 from fivetran_connector_sdk.constants import (
     TESTER_VER, VERSION_FILENAME, UTF_8,
-    VALID_COMMANDS, DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS, TABLES
+    VALID_COMMANDS, DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS, TABLES,
+    CONNECTOR_SDK_NO_YIELD_LABEL
 )
 from fivetran_connector_sdk.helpers import (
     print_library_log, reset_local_file_directory, find_connector_object, suggest_correct_command,
@@ -39,7 +43,7 @@ from fivetran_connector_sdk.connector_helper import (
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
-__version__ = "1.7.1"
+__version__ = "1.7.3"
 TESTER_VERSION = TESTER_VER
 MAX_MESSAGE_LENGTH = 32 * 1024 * 1024 # 32MB
 
@@ -367,15 +371,47 @@ class Connector(connector_sdk_pb2_grpc.SourceConnectorServicer):
         """
         configuration = self.configuration if self.configuration else request.configuration
         state = self.state if self.state else json.loads(request.state_json)
+        exception_queue = queue.Queue()
 
         try:
             print_library_log("Initiating the 'update' method call...", Logging.Level.INFO)
-            for resp in self.update_method(configuration=configuration, state=state):
-                if isinstance(resp, list):
-                    for r in resp:
-                        yield r
-                else:
-                    yield resp
+
+            if os.environ.get(CONNECTOR_SDK_NO_YIELD_LABEL, "false").lower() == "true":
+                def run_update():
+                    try:
+                        result = self.update_method(configuration=configuration, state=state)
+                        # If the customer's update method returns a generator (i.e., uses yield),
+                        # exhaust the generator responses, they are None. From this point on, all operations
+                        # push update_response to a queue, and we yield from the queue instead.
+                        # We return None here intentionally.
+                        if isinstance(result, GeneratorType):
+                            for _ in result:
+                                pass
+                        # If the update method doesn't use yield, skip the response returned.
+                        else:
+                            pass
+                    except Exception as exc:
+                        exception_queue.put(exc)
+                    finally:
+                        Operations.operation_stream.mark_done()
+
+                thread = threading.Thread(target=run_update)
+                thread.start()
+
+                yield from Operations.operation_stream
+
+                thread.join()
+
+                # Check if any exception was raised during the update
+                if not exception_queue.empty():
+                    raise exception_queue.get()
+            else:
+                for update_response in self.update_method(configuration=configuration, state=state):
+                    if isinstance(update_response, list):
+                        for response in update_response:
+                            yield response
+                    else:
+                        yield update_response
 
         except TypeError as e:
             if str(e) != "'NoneType' object is not iterable":

{fivetran_connector_sdk-1.7.1 → fivetran_connector_sdk-1.7.3}/src/fivetran_connector_sdk/connector_helper.py

@@ -378,72 +378,61 @@ def validate_requirements_file(project_path: str, is_deploy: bool, version: str
     update_version_requirements = False
     update_missing_requirements = False
     update_unused_requirements = False
-    if len(tmp_requirements) > 0:
-        requirements = load_or_add_requirements_file(requirements_file_path)
-
-        version_mismatch_deps = {key: tmp_requirements[key] for key in
-                                 (requirements.keys() & tmp_requirements.keys())
-                                 if requirements[key] != tmp_requirements[key]}
-        if version_mismatch_deps:
-            print_library_log("We recommend using the current stable version for the following libraries:", Logging.Level.WARNING)
-            print(version_mismatch_deps)
-            if is_deploy and not force:
-                confirm = input(
-                        f"Would you like us to update {REQUIREMENTS_TXT} to the current stable versions of the dependent libraries? (y/N):")
-                if confirm.lower() == "y":
-                    update_version_requirements = True
-                    for requirement in version_mismatch_deps:
-                        requirements[requirement] = tmp_requirements[requirement]
+    requirements = load_or_add_requirements_file(requirements_file_path)
+
+    version_mismatch_deps = {key: tmp_requirements[key] for key in
+                             (requirements.keys() & tmp_requirements.keys())
+                             if requirements[key] != tmp_requirements[key]}
+    if version_mismatch_deps:
+        print_library_log("We recommend using the current stable version for the following libraries:", Logging.Level.WARNING)
+        print(version_mismatch_deps)
+        if is_deploy and not force:
+            confirm = input(
+                    f"Would you like us to update {REQUIREMENTS_TXT} to the current stable versions of the dependent libraries? (y/N):")
+            if confirm.lower() == "y":
+                update_version_requirements = True
+                for requirement in version_mismatch_deps:
+                    requirements[requirement] = tmp_requirements[requirement]
+                print_library_log(
+                    f"Successfully updated {REQUIREMENTS_TXT} to the current stable versions of the dependent libraries.")
+            elif confirm.lower() == "n":
+                print_library_log(f"Changes identified for libraries with version conflicts have been ignored. These changes have NOT been made to {REQUIREMENTS_TXT}.")
+
+    missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
+    if missing_deps:
+        handle_missing_deps(missing_deps)
+        if is_deploy and not force:
+            confirm = input(
+                    f"Would you like us to update {REQUIREMENTS_TXT} to add missing dependent libraries? (y/N):")
+            if confirm.lower() == "n":
+                print_library_log(f"Changes identified as missing dependencies for libraries have been ignored. These changes have NOT been made to {REQUIREMENTS_TXT}.")
+            elif confirm.lower() == "y":
+                update_missing_requirements = True
+                for requirement in missing_deps:
+                    requirements[requirement] = tmp_requirements[requirement]
+                print_library_log(f"Successfully added missing dependencies to {REQUIREMENTS_TXT}.")
+
+    unused_deps = list(requirements.keys() - tmp_requirements.keys())
+    if unused_deps:
+        handle_unused_deps(unused_deps, version)
+        if is_deploy and not force:
+            confirm = input(f"Would you like us to update {REQUIREMENTS_TXT} to remove the unused libraries? (y/N):")
+            if confirm.lower() == "n":
+                if 'fivetran_connector_sdk' in unused_deps or 'requests' in unused_deps:
                     print_library_log(
-                        f"Successfully updated {REQUIREMENTS_TXT} to the current stable versions of the dependent libraries.")
-                elif confirm.lower() == "n":
-                    print_library_log(f"Changes identified for libraries with version conflicts have been ignored. These changes have NOT been made to {REQUIREMENTS_TXT}.")
-
-        missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
-        if missing_deps:
-            handle_missing_deps(missing_deps)
-            if is_deploy and not force:
-                confirm = input(
-                        f"Would you like us to update {REQUIREMENTS_TXT} to add missing dependent libraries? (y/N):")
-                if confirm.lower() == "n":
-                    print_library_log(f"Changes identified as missing dependencies for libraries have been ignored. These changes have NOT been made to {REQUIREMENTS_TXT}.")
-                elif confirm.lower() == "y":
-                    update_missing_requirements = True
-                    for requirement in missing_deps:
-                        requirements[requirement] = tmp_requirements[requirement]
-                    print_library_log(f"Successfully added missing dependencies to {REQUIREMENTS_TXT}.")
-
-        unused_deps = list(requirements.keys() - tmp_requirements.keys())
-        if unused_deps:
-            handle_unused_deps(unused_deps, version)
-            if is_deploy and not force:
-                confirm = input(f"Would you like us to update {REQUIREMENTS_TXT} to remove the unused libraries? (y/N):")
-                if confirm.lower() == "n":
-                    if 'fivetran_connector_sdk' in unused_deps or 'requests' in unused_deps:
-                        print_library_log(
-                            f"Fix your {REQUIREMENTS_TXT} file by removing pre-installed dependencies [fivetran_connector_sdk, requests] to proceed with the deployment.")
-                        sys.exit(1)
-                    print_library_log(f"Changes identified for unused libraries have been ignored. These changes have NOT been made to {REQUIREMENTS_TXT}.")
-                elif confirm.lower() == "y":
-                    update_unused_requirements = True
-                    for requirement in unused_deps:
-                        del requirements[requirement]
-                    print_library_log(f"Successfully removed unused libraries from {REQUIREMENTS_TXT}.")
-
-
-        if update_version_requirements or update_missing_requirements or update_unused_requirements:
-            with open(requirements_file_path, "w", encoding=UTF_8) as file:
-                file.write("\n".join(requirements.values()))
-                print_library_log(f"`{REQUIREMENTS_TXT}` has been updated successfully.")
-
-    else:
-        if os.path.exists(requirements_file_path):
-            print_library_log(f"{REQUIREMENTS_TXT} is not required as no additional "
-                  "Python libraries are required or all required libraries for "
-                  "your code are pre-installed.", Logging.Level.WARNING)
-            with open(requirements_file_path, 'w') as file:
-                file.write("")
-
+                        f"Fix your {REQUIREMENTS_TXT} file by removing pre-installed dependencies [fivetran_connector_sdk, requests] to proceed with the deployment.")
+                    sys.exit(1)
+                print_library_log(f"Changes identified for unused libraries have been ignored. These changes have NOT been made to {REQUIREMENTS_TXT}.")
+            elif confirm.lower() == "y":
+                update_unused_requirements = True
+                for requirement in unused_deps:
+                    del requirements[requirement]
+                print_library_log(f"Successfully removed unused libraries from {REQUIREMENTS_TXT}.")
+
+    if update_version_requirements or update_missing_requirements or update_unused_requirements:
+        with open(requirements_file_path, "w", encoding=UTF_8) as file:
+            file.write("\n".join(requirements.values()))
+            print_library_log(f"`{REQUIREMENTS_TXT}` has been updated successfully.")
 
     if is_deploy: print_library_log(f"Validation of {REQUIREMENTS_TXT} completed.")
 
@@ -909,9 +898,9 @@ def _maybe_colorize_jar_output(line: str) -> str:
         return line
 
     if "SEVERE" in line or "ERROR" in line or "Exception" in line or "FAILED" in line:
-        return f"\033[91m{line}\033[0m"  # Red
+        return f"\033[196m{line}\033[0m"  # ANSI Red color #ff0000
     elif "WARN" in line or "WARNING" in line:
-        return f"\033[93m{line}\033[0m"  # Yellow
+        return f"\033[130m{line}\033[0m"  # ANSI Orange-like color #af5f00
     return line
 
 def process_tables(response, table_list):

{fivetran_connector_sdk-1.7.1 → fivetran_connector_sdk-1.7.3}/src/fivetran_connector_sdk/constants.py

@@ -39,6 +39,7 @@ LOGGING_PREFIX = "Fivetran-Connector-SDK"
 LOGGING_DELIMITER = ": "
 VIRTUAL_ENV_CONFIG = "pyvenv.cfg"
 ROOT_FILENAME = "connector.py"
+CONNECTOR_SDK_NO_YIELD_LABEL = "CONNECTOR_SDK_NO_YIELD_APPROACH"
 
 # Compile patterns used in the implementation
 WORD_DASH_DOT_PATTERN = re.compile(r'^[\w.-]*$')

{fivetran_connector_sdk-1.7.1 → fivetran_connector_sdk-1.7.3}/src/fivetran_connector_sdk/logger.py

@@ -41,9 +41,9 @@ class Logging:
     @staticmethod
     def get_color(level):
         if level == Logging.Level.WARNING:
-            return "\033[93m"  # Yellow
+            return "\033[130m"  # ANSI Orange-like color #af5f00
         elif level == Logging.Level.SEVERE:
-            return "\033[91m"  # Red
+            return "\033[196m"  # ANSI Red color #ff0000
         return ""
 
     @staticmethod

{fivetran_connector_sdk-1.7.1 → fivetran_connector_sdk-1.7.3}/src/fivetran_connector_sdk/operations.py

@@ -1,14 +1,14 @@
 import json
-import inspect
+import os
 import sys
+import queue
 
 from datetime import datetime
 from google.protobuf import timestamp_pb2
 
-from fivetran_connector_sdk import constants
 from fivetran_connector_sdk.constants import (
     JAVA_LONG_MAX_VALUE,
-    TABLES,
+    TABLES, CONNECTOR_SDK_NO_YIELD_LABEL,
 )
 from fivetran_connector_sdk.helpers import (
     get_renamed_table_name,
@@ -18,6 +18,65 @@ from fivetran_connector_sdk.helpers import (
 from fivetran_connector_sdk.logger import Logging
 from fivetran_connector_sdk.protos import connector_sdk_pb2, common_pb2
 
+class _OperationStream:
+    """
+    A simple iterator-based stream backed by a queue for producing and consuming operations.
+
+    This class allows adding data items into a queue and consuming them using standard iteration.
+    It uses a sentinel object to signal the end of the stream.
+
+    Example:
+        stream = _OperationStream()
+        stream.add("response1")
+        stream.mark_done()
+
+        for response in stream:
+            print(response)  # prints "response1"
+    """
+
+    def __init__(self):
+        """
+        Initializes the operation stream with a queue and a sentinel object.
+        """
+        self._queue = queue.Queue(maxsize=1)
+        self._sentinel = object()
+
+    def __iter__(self):
+        """
+        Returns the iterator instance itself.
+        """
+        return self
+
+    def add(self, data):
+        """
+        Adds an item to the stream.
+
+        Args:
+            data (object): The data item to add to the stream.
+        """
+        self._queue.put(data)
+
+    def mark_done(self):
+        """
+        Marks the end of the stream by putting a sentinel in the queue.
+        """
+        self._queue.put(self._sentinel)
+
+    def __next__(self):
+        """
+        Retrieves the next item from the stream. Raises StopIteration when the sentinel is encountered.
+
+        Returns:
+            object: The next item in the stream.
+
+        Raises:
+            StopIteration: If the sentinel object is encountered.
+        """
+        item = self._queue.get()
+        if item is self._sentinel:
+            raise StopIteration
+        return item
+
 _LOG_DATA_TYPE_INFERENCE = {
     "boolean": True,
     "binary": True,
@@ -25,8 +84,11 @@ _LOG_DATA_TYPE_INFERENCE = {
 }
 
 class Operations:
+    operation_stream = _OperationStream()
+    use_no_yield_approach = os.environ.get(CONNECTOR_SDK_NO_YIELD_LABEL, "false").lower() == "true"
+
     @staticmethod
-    def upsert(table: str, data: dict) -> list[connector_sdk_pb2.UpdateResponse]:
+    def upsert(table: str, data: dict):
         """Updates records with the same primary key if already present in the destination. Inserts new records if not already present in the destination.
 
         Args:
@@ -36,11 +98,6 @@ class Operations:
         Returns:
             list[connector_sdk_pb2.UpdateResponse]: A list of update responses.
         """
-        if constants.DEBUGGING:
-            _yield_check(inspect.stack())
-
-        responses = []
-
         table = get_renamed_table_name(table)
         columns = _get_columns(table)
         if not columns:
@@ -58,13 +115,17 @@ class Operations:
             type=common_pb2.RecordType.UPSERT,
             data=mapped_data
         )
+        update_response = connector_sdk_pb2.UpdateResponse(record=record)
 
-        responses.append(connector_sdk_pb2.UpdateResponse(record=record))
+        if Operations.use_no_yield_approach:
+            Operations.operation_stream.add(update_response)
+            return None
+        else:
+            return [update_response]
 
-        return responses
 
     @staticmethod
-    def update(table: str, modified: dict) -> connector_sdk_pb2.UpdateResponse:
+    def update(table: str, modified: dict):
         """Performs an update operation on the specified table with the given modified data.
 
         Args:
@@ -74,9 +135,6 @@ class Operations:
         Returns:
             connector_sdk_pb2.UpdateResponse: The update response.
         """
-        if constants.DEBUGGING:
-            _yield_check(inspect.stack())
-
         table = get_renamed_table_name(table)
         columns = _get_columns(table)
         mapped_data = _map_data_to_columns(modified, columns)
@@ -87,10 +145,16 @@ class Operations:
             data=mapped_data
         )
 
-        return connector_sdk_pb2.UpdateResponse(record=record)
+        update_response = connector_sdk_pb2.UpdateResponse(record=record)
+
+        if Operations.use_no_yield_approach:
+            Operations.operation_stream.add(update_response)
+            return None
+        else:
+            return update_response
 
     @staticmethod
-    def delete(table: str, keys: dict) -> connector_sdk_pb2.UpdateResponse:
+    def delete(table: str, keys: dict):
         """Performs a soft delete operation on the specified table with the given keys.
 
         Args:
@@ -100,9 +164,6 @@ class Operations:
         Returns:
             connector_sdk_pb2.UpdateResponse: The delete response.
         """
-        if constants.DEBUGGING:
-            _yield_check(inspect.stack())
-
         table = get_renamed_table_name(table)
         columns = _get_columns(table)
         mapped_data = _map_data_to_columns(keys, columns)
@@ -113,10 +174,16 @@ class Operations:
             data=mapped_data
         )
 
-        return connector_sdk_pb2.UpdateResponse(record=record)
+        update_response = connector_sdk_pb2.UpdateResponse(record=record)
+
+        if Operations.use_no_yield_approach:
+            Operations.operation_stream.add(update_response)
+            return None
+        else:
+            return update_response
 
     @staticmethod
-    def checkpoint(state: dict) -> connector_sdk_pb2.UpdateResponse:
+    def checkpoint(state: dict):
         """Checkpoint saves the connector's state. State is a dict which stores information to continue the
         sync from where it left off in the previous sync. For example, you may choose to have a field called
         "cursor" with a timestamp value to indicate up to when the data has been synced. This makes it possible
@@ -136,9 +203,14 @@ class Operations:
         Returns:
             connector_sdk_pb2.UpdateResponse: The checkpoint response.
         """
-        if constants.DEBUGGING:
-            _yield_check(inspect.stack())
-        return connector_sdk_pb2.UpdateResponse(checkpoint=connector_sdk_pb2.Checkpoint(state_json=json.dumps(state)))
+        update_response = connector_sdk_pb2.UpdateResponse(checkpoint=connector_sdk_pb2.Checkpoint(state_json=json.dumps(state)))
+
+        if Operations.use_no_yield_approach:
+            Operations.operation_stream.add(update_response)
+            return None
+        else:
+            return update_response
+
 
 def _get_columns(table: str) -> dict:
     """Retrieves the columns for the specified table.

{fivetran_connector_sdk-1.7.1 → fivetran_connector_sdk-1.7.3}/src/fivetran_connector_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.7.1
+Version: 1.7.3
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk