fivetran-connector-sdk 1.7.4__py3-none-any.whl → 1.8.0__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -22,8 +22,7 @@ from fivetran_connector_sdk.operations import Operations
 from fivetran_connector_sdk import constants
 from fivetran_connector_sdk.constants import (
     TESTER_VER, VERSION_FILENAME, UTF_8,
-    VALID_COMMANDS, DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS, TABLES,
-    CONNECTOR_SDK_NO_YIELD_LABEL
+    VALID_COMMANDS, DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS, TABLES
 )
 from fivetran_connector_sdk.helpers import (
     print_library_log, reset_local_file_directory, find_connector_object, suggest_correct_command,
@@ -43,7 +42,7 @@ from fivetran_connector_sdk.connector_helper import (
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
-__version__ = "1.7.4"
+__version__ = "1.8.0"
 TESTER_VERSION = TESTER_VER
 MAX_MESSAGE_LENGTH = 32 * 1024 * 1024 # 32MB
 
@@ -376,42 +375,43 @@ class Connector(connector_sdk_pb2_grpc.SourceConnectorServicer):
         try:
             print_library_log("Initiating the 'update' method call...", Logging.Level.INFO)
 
-            if os.environ.get(CONNECTOR_SDK_NO_YIELD_LABEL, "false").lower() == "true":
-                def run_update():
-                    try:
-                        result = self.update_method(configuration=configuration, state=state)
-                        # If the customer's update method returns a generator (i.e., uses yield),
-                        # exhaust the generator responses, they are None. From this point on, all operations
-                        # push update_response to a queue, and we yield from the queue instead.
-                        # We return None here intentionally.
-                        if isinstance(result, GeneratorType):
-                            for _ in result:
-                                pass
-                        # If the update method doesn't use yield, skip the response returned.
-                        else:
+            def run_update():
+                try:
+                    result = self.update_method(configuration=configuration, state=state)
+                    # If the customer's update method returns a generator (i.e., uses yield),
+                    # exhaust the generator responses, they are None. From this point on, all operations
+                    # push update_response to a queue, and we yield from the queue instead.
+                    # We return None here intentionally.
+                    if isinstance(result, GeneratorType):
+                        for _ in result:
                             pass
-                    except Exception as exc:
-                        exception_queue.put(exc)
-                    finally:
-                        Operations.operation_stream.mark_done()
-
-                thread = threading.Thread(target=run_update)
-                thread.start()
-
-                yield from Operations.operation_stream
+                    # If the update method doesn't use yield, skip the response returned.
+                    else:
+                        pass
+                except Exception as exc:
+                    exception_queue.put(exc)
+                finally:
+                    Operations.operation_stream.mark_done()
+
+            thread = threading.Thread(target=run_update)
+            thread.start()
+
+            # consumer - yield the operations in the operation_stream.
+            for response in Operations.operation_stream:
+                # checkpoint call always returns list of responses.
+                if isinstance(response, list):
+                    for res in response:
+                        yield res
+                    # checkpoint call blocks the queue (see _OperationStream.add method). unblock the queue after yielding all responses.
+                    Operations.operation_stream.unblock()
+                else:
+                    yield response
 
-                thread.join()
+            thread.join()
 
-                # Check if any exception was raised during the update
-                if not exception_queue.empty():
-                    raise exception_queue.get()
-            else:
-                for update_response in self.update_method(configuration=configuration, state=state):
-                    if isinstance(update_response, list):
-                        for response in update_response:
-                            yield response
-                    else:
-                        yield update_response
+            # Check if any exception was raised during the update
+            if not exception_queue.empty():
+                raise exception_queue.get()
 
         except TypeError as e:
             if str(e) != "'NoneType' object is not iterable":

fivetran_connector_sdk/connector_helper.py

@@ -57,27 +57,22 @@ def get_destination_group(args):
         ft_group = get_input_from_cli("Provide the destination name (as displayed in your dashboard destination list)", env_destination_name)
     return ft_group
 
-def get_connection_name(args):
+def get_connection_name(args, retrying=0):
     ft_connection = args.connection if args.connection else None
     env_connection_name = os.getenv('FIVETRAN_CONNECTION_NAME', None)
     if not ft_connection:
-        for retrying in range(MAX_RETRIES):
-            ft_connection = get_input_from_cli("Provide the connection name", env_connection_name)
-            if not is_connection_name_valid(ft_connection):
-                if retrying==MAX_RETRIES-1:
-                    sys.exit(1)
-                else:
-                    print_library_log(f"Connection name: {ft_connection} is invalid!\n The connection name should start with an "
-                                      f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
-                                      f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
-                    print_library_log("Please retry...", Logging.Level.INFO)
-            else:
-                break
+        ft_connection = get_input_from_cli("Provide the connection name", env_connection_name)
     if not is_connection_name_valid(ft_connection):
-        print_library_log(f"Connection name: {ft_connection} is invalid!\n The connection name should start with an "
-                          f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
-                          f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
-        sys.exit(1)
+        print_library_log(
+            f"Connection name: {ft_connection} is invalid!\n The connection name should start with an "
+            f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
+            f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
+        args.connection = None
+        if retrying >= MAX_RETRIES or args.force:
+            sys.exit(1)
+        else:
+            print_library_log("Please retry...", Logging.Level.INFO)
+            return get_connection_name(args, retrying + 1)
     return ft_connection
 
 def get_api_key(args):
@@ -113,32 +108,36 @@ def get_state(args):
     state = validate_and_load_state(args, state)
     return state
 
-def get_configuration(args):
+def get_configuration(args, retrying = 0):
     configuration = args.configuration if args.configuration else None
     env_configuration = os.getenv('FIVETRAN_CONFIGURATION', None)
-    if not configuration and not args.force and args.command.lower() == "deploy":
-        json_filepath = os.path.join(args.project_path, "configuration.json")
-        if os.path.exists(json_filepath):
-            print_library_log("configuration.json file detected in the project, "
-                              "but no configuration input provided via the command line", Logging.Level.WARNING)
-            env_configuration = env_configuration if env_configuration else "configuration.json"
-        confirm = input(f"Does this debug run/deploy need configuration (y/N):")
-        if confirm.lower()=='y':
-            for retrying in range(MAX_RETRIES):
-                try:
-                    configuration = get_input_from_cli("Provide the configuration file path", env_configuration)
-                    config_values = validate_and_load_configuration(args.project_path, configuration)
-                    return config_values, configuration
-                except ValueError as e:
-                    if retrying==MAX_RETRIES-1:
-                        print_library_log(f"{e}. Invalid Configuration, Exiting..", Logging.Level.WARNING)
-                        sys.exit(1)
-                    else:
-                        print_library_log(f"{e}. Please retry..", Logging.Level.INFO)
+    try:
+        if not configuration and not args.force and args.command.lower() == "deploy":
+            confirm = 'y'
+            if not retrying:
+                json_filepath = os.path.join(args.project_path, "configuration.json")
+                if os.path.exists(json_filepath):
+                    print_library_log("configuration.json file detected in the project, "
+                                      "but no configuration input provided via the command line", Logging.Level.WARNING)
+                    env_configuration = env_configuration if env_configuration else "configuration.json"
+                confirm = input(f"Does this debug run/deploy need configuration (y/N):")
+            if confirm.lower()=='y':
+                configuration = get_input_from_cli("Provide the configuration file path", env_configuration)
+                config_values = validate_and_load_configuration(args.project_path, configuration)
+                return config_values, configuration
+            else:
+                print_library_log("No input required for configuration. Continuing without configuration.", Logging.Level.INFO)
+                return {}, None
+        config_values = validate_and_load_configuration(args.project_path, configuration)
+        return config_values, configuration
+    except ValueError as e:
+        args.configuration = None
+        if retrying >= MAX_RETRIES or args.force:
+            print_library_log(f"{e}. Invalid Configuration, Exiting..", Logging.Level.WARNING)
+            sys.exit(1)
         else:
-            print_library_log("No input required for configuration. Continuing without configuration.", Logging.Level.INFO)
-    config_values = validate_and_load_configuration(args.project_path, configuration)
-    return config_values, configuration
+            print_library_log(f"{e}. Please retry..", Logging.Level.INFO)
+            return get_configuration(args, retrying + 1)
 
 
 def check_newer_version(version: str):
@@ -989,8 +988,6 @@ def process_data_type(column, type):
         column.type = common_pb2.DataType.FLOAT
     elif type.upper() == "DOUBLE":
         column.type = common_pb2.DataType.DOUBLE
-    elif type.upper() == "NAIVE_TIME":
-        column.type = common_pb2.DataType.NAIVE_TIME
     elif type.upper() == "NAIVE_DATE":
         column.type = common_pb2.DataType.NAIVE_DATE
     elif type.upper() == "NAIVE_DATETIME":

fivetran_connector_sdk/constants.py

@@ -1,6 +1,7 @@
+import os
 import re
 
-TESTER_VER = "2.25.0701.001"
+TESTER_VER = "2.25.0806.001"
 
 WIN_OS = "windows"
 ARM_64 = "arm64"
@@ -34,12 +35,16 @@ OUTPUT_FILES_DIR = "files"
 REQUIREMENTS_TXT = "requirements.txt"
 PYPI_PACKAGE_DETAILS_URL = "https://pypi.org/pypi/fivetran_connector_sdk/json"
 ONE_DAY_IN_SEC = 24 * 60 * 60
+CHECKPOINT_OP_TIMEOUT_IN_SEC = 30 # seconds
 MAX_RETRIES = 3
 LOGGING_PREFIX = "Fivetran-Connector-SDK"
 LOGGING_DELIMITER = ": "
 VIRTUAL_ENV_CONFIG = "pyvenv.cfg"
 ROOT_FILENAME = "connector.py"
-CONNECTOR_SDK_NO_YIELD_LABEL = "CONNECTOR_SDK_NO_YIELD_APPROACH"
+MAX_RECORDS_IN_BATCH = 100
+MAX_BATCH_SIZE_IN_BYTES = 100000 # 100 KB
+# The increased queue size shall be rolled out as Feature flag
+QUEUE_SIZE = int(os.environ.get("QUEUE_SIZE", 1))
 
 # Compile patterns used in the implementation
 WORD_DASH_DOT_PATTERN = re.compile(r'^[\w.-]*$')

fivetran_connector_sdk/operations.py

@@ -1,14 +1,18 @@
 import json
-import os
 import sys
 import queue
+import threading
 
 from datetime import datetime
 from google.protobuf import timestamp_pb2
 
 from fivetran_connector_sdk.constants import (
     JAVA_LONG_MAX_VALUE,
-    TABLES, CONNECTOR_SDK_NO_YIELD_LABEL,
+    TABLES,
+    MAX_RECORDS_IN_BATCH,
+    MAX_BATCH_SIZE_IN_BYTES,
+    QUEUE_SIZE,
+    CHECKPOINT_OP_TIMEOUT_IN_SEC
 )
 from fivetran_connector_sdk.helpers import (
     get_renamed_table_name,
@@ -38,8 +42,15 @@ class _OperationStream:
         """
         Initializes the operation stream with a queue and a sentinel object.
         """
-        self._queue = queue.Queue(maxsize=1)
+        self._queue = queue.Queue(maxsize=QUEUE_SIZE)
         self._sentinel = object()
+        self._is_done = False
+        self._buffer = []
+        self._buffer_record_count = 0
+        self._buffer_size_bytes = 0
+        self._checkpoint_lock = threading.Lock()
+        self._checkpoint_flush_signal = threading.Event()
+        self._checkpoint_flush_signal.set()
 
     def __iter__(self):
         """
@@ -47,14 +58,38 @@ class _OperationStream:
         """
         return self
 
-    def add(self, data):
+    def add(self, operation):
         """
-        Adds an item to the stream.
+        Adds an operation to the stream. Guarantees that operations within a single thread are processed in the order.
+
+        In multithreaded environment if a thread initiates a checkpoint, it's producer is blocked until the
+        checkpoint flush is complete. This block is localized, other threads
+        remain unblocked and can continue to perform other operations
+        (such as upserts, updates, deletes), but they are prevented from initiating a new checkpoint
+        until the existing one is finished.
 
         Args:
-            data (object): The data item to add to the stream.
+            operation (object): The data item to add to the stream.
+        """
+        if isinstance(operation, connector_sdk_pb2.Checkpoint):
+            # lock to ensure checkpoint operations are processed one at a time
+            with self._checkpoint_lock:
+                # clear the signal to indicate checkpoint operation is being processed.
+                self._checkpoint_flush_signal.clear()
+                self._queue.put(operation)
+                # wait until the consumer flushes the buffer and sets the flag.
+                if not self._checkpoint_flush_signal.wait(CHECKPOINT_OP_TIMEOUT_IN_SEC):
+                    raise TimeoutError(
+                        "Checkpoint flush timed out. Consumer may have failed to process checkpoint."
+                    )
+        else:
+            self._queue.put(operation)
+
+    def unblock(self):
+        """
+        Unblocks the queue, called by consumer after the checkpoint flush is completed.
         """
-        self._queue.put(data)
+        self._checkpoint_flush_signal.set()
 
     def mark_done(self):
         """
@@ -72,10 +107,76 @@ class _OperationStream:
         Raises:
             StopIteration: If the sentinel object is encountered.
         """
-        item = self._queue.get()
-        if item is self._sentinel:
+        # If stream is completed and buffer is empty, raise StopIteration. Else flush the buffer.
+        if self._is_done and not self._buffer:
             raise StopIteration
-        return item
+
+        if self._is_done:
+            return self._flush_buffer()
+
+        return self._build_next_batch()
+
+    def _build_next_batch(self):
+        """
+        Core logic to build the batch. The loop continues until the buffer is full,
+        but can be interrupted by a checkpoint or a sentinel from the producer.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse or list[connector_sdk_pb2.UpdateResponse]: Either a single response
+            containing records or checkpoint, or a list of responses when flushing data with a checkpoint.
+
+        """
+        while self._buffer_record_count < MAX_RECORDS_IN_BATCH and self._buffer_size_bytes < MAX_BATCH_SIZE_IN_BYTES:
+            operation = self._queue.get()
+
+            # Case 1: If operation is sentinel, mark the stream as done, flush the buffer.
+            if operation is self._sentinel:
+                self._is_done = True
+                if self._buffer:
+                    return self._flush_buffer()
+                else:
+                    raise StopIteration
+
+            # Case 2: if operation is a Checkpoint, flush the buffer and send the checkpoint.
+            elif isinstance(operation, connector_sdk_pb2.Checkpoint):
+                return self._flush_buffer_on_checkpoint(operation)
+
+            # it is record, buffer it to flush in batches
+            self._buffer_record_count += 1
+            self._buffer_size_bytes += len(operation.SerializeToString())
+            self._buffer.append(operation)
+
+        # Case 3: If buffer size limit is reached, flush the buffer and return the response.
+        return self._flush_buffer()
+
+    def _flush_buffer_on_checkpoint(self, checkpoint: connector_sdk_pb2.Checkpoint):
+        """
+        Creates the responses containing the checkpoint and buffered records.
+
+        Args:
+            checkpoint (object): Checkpoint operation to be added to the response.
+        """
+        responses = []
+
+        if self._buffer:
+            responses.append(self._flush_buffer())
+
+        responses.append(connector_sdk_pb2.UpdateResponse(checkpoint=checkpoint))
+        return responses
+
+    def _flush_buffer(self):
+        """
+        Flushes the current buffer and returns a response containing the buffered records.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: A response containing the buffered records.
+        """
+        batch_to_flush = self._buffer
+        self._buffer = []
+        self._buffer_record_count = 0
+        self._buffer_size_bytes = 0
+        return connector_sdk_pb2.UpdateResponse(records=connector_sdk_pb2.Records(records=batch_to_flush))
+
 
 _LOG_DATA_TYPE_INFERENCE = {
     "boolean": True,
@@ -85,7 +186,6 @@ _LOG_DATA_TYPE_INFERENCE = {
 
 class Operations:
     operation_stream = _OperationStream()
-    use_no_yield_approach = os.environ.get(CONNECTOR_SDK_NO_YIELD_LABEL, "false").lower() == "true"
 
     @staticmethod
     def upsert(table: str, data: dict):
@@ -115,14 +215,8 @@ class Operations:
             type=common_pb2.RecordType.UPSERT,
             data=mapped_data
         )
-        update_response = connector_sdk_pb2.UpdateResponse(record=record)
-
-        if Operations.use_no_yield_approach:
-            Operations.operation_stream.add(update_response)
-            return None
-        else:
-            return [update_response]
 
+        Operations.operation_stream.add(record)
 
     @staticmethod
     def update(table: str, modified: dict):
@@ -145,13 +239,7 @@ class Operations:
             data=mapped_data
         )
 
-        update_response = connector_sdk_pb2.UpdateResponse(record=record)
-
-        if Operations.use_no_yield_approach:
-            Operations.operation_stream.add(update_response)
-            return None
-        else:
-            return update_response
+        Operations.operation_stream.add(record)
 
     @staticmethod
     def delete(table: str, keys: dict):
@@ -174,13 +262,7 @@ class Operations:
             data=mapped_data
         )
 
-        update_response = connector_sdk_pb2.UpdateResponse(record=record)
-
-        if Operations.use_no_yield_approach:
-            Operations.operation_stream.add(update_response)
-            return None
-        else:
-            return update_response
+        Operations.operation_stream.add(record)
 
     @staticmethod
     def checkpoint(state: dict):
@@ -203,14 +285,9 @@ class Operations:
         Returns:
             connector_sdk_pb2.UpdateResponse: The checkpoint response.
         """
-        update_response = connector_sdk_pb2.UpdateResponse(checkpoint=connector_sdk_pb2.Checkpoint(state_json=json.dumps(state)))
-
-        if Operations.use_no_yield_approach:
-            Operations.operation_stream.add(update_response)
-            return None
-        else:
-            return update_response
+        checkpoint = connector_sdk_pb2.Checkpoint(state_json=json.dumps(state))
 
+        Operations.operation_stream.add(checkpoint)
 
 def _get_columns(table: str) -> dict:
     """Retrieves the columns for the specified table.
@@ -352,4 +429,3 @@ def _yield_check(stack):
         # This should never happen
         raise RuntimeError(
             f"The '{called_method}' function is missing in the connector calling code '{calling_code}'. Please ensure that the '{called_method}' function is properly defined in your code to proceed. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsmethods")
-

fivetran_connector_sdk/protos/connector_sdk_pb2.py

@@ -26,7 +26,7 @@ _sym_db = _symbol_database.Default()
 from fivetran_connector_sdk import common_pb2 as common__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x63onnector_sdk.proto\x12\x0f\x66ivetran_sdk.v2\x1a\x0c\x63ommon.proto\"\x8f\x01\n\rSchemaRequest\x12H\n\rconfiguration\x18\x01 \x03(\x0b\x32\x31.fivetran_sdk.v2.SchemaRequest.ConfigurationEntry\x1a\x34\n\x12\x43onfigurationEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xf1\x01\n\x0eSchemaResponse\x12\'\n\x1dschema_response_not_supported\x18\x01 \x01(\x08H\x00\x12\x32\n\x0bwith_schema\x18\x02 \x01(\x0b\x32\x1b.fivetran_sdk.v2.SchemaListH\x00\x12\x34\n\x0ewithout_schema\x18\x03 \x01(\x0b\x32\x1a.fivetran_sdk.v2.TableListH\x00\x12$\n\x17selection_not_supported\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\n\n\x08responseB\x1a\n\x18_selection_not_supported\"\xf9\x01\n\rUpdateRequest\x12H\n\rconfiguration\x18\x01 \x03(\x0b\x32\x31.fivetran_sdk.v2.UpdateRequest.ConfigurationEntry\x12\x32\n\tselection\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.SelectionH\x00\x88\x01\x01\x12\x17\n\nstate_json\x18\x03 \x01(\tH\x01\x88\x01\x01\x1a\x34\n\x12\x43onfigurationEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0c\n\n_selectionB\r\n\x0b_state_json\"\x91\x01\n\tSelection\x12=\n\x0ewithout_schema\x18\x01 \x01(\x0b\x32#.fivetran_sdk.v2.TablesWithNoSchemaH\x00\x12\x38\n\x0bwith_schema\x18\x02 \x01(\x0b\x32!.fivetran_sdk.v2.TablesWithSchemaH\x00\x42\x0b\n\tselection\"a\n\x12TablesWithNoSchema\x12/\n\x06tables\x18\x01 \x03(\x0b\x32\x1f.fivetran_sdk.v2.TableSelection\x12\x1a\n\x12include_new_tables\x18\x02 \x01(\x08\"b\n\x10TablesWithSchema\x12\x31\n\x07schemas\x18\x01 \x03(\x0b\x32 .fivetran_sdk.v2.SchemaSelection\x12\x1b\n\x13include_new_schemas\x18\x02 \x01(\x08\"\x85\x01\n\x0fSchemaSelection\x12\x10\n\x08included\x18\x01 \x01(\x08\x12\x13\n\x0bschema_name\x18\x02 \x01(\t\x12/\n\x06tables\x18\x03 \x03(\x0b\x32\x1f.fivetran_sdk.v2.TableSelection\x12\x1a\n\x12include_new_tables\x18\x04 \x01(\x08\"\xc2\x01\n\x0eTableSelection\x12\x10\n\x08included\x18\x01 \x01(\x08\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12=\n\x07\x63olumns\x18\x03 \x03(\x0b\x32,.fivetran_sdk.v2.TableSelection.ColumnsEntry\x12\x1b\n\x13include_new_columns\x18\x04 \x01(\x08\x1a.\n\x0c\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"\x87\x02\n\x0eUpdateResponse\x12)\n\x06record\x18\x01 \x01(\x0b\x32\x17.fivetran_sdk.v2.RecordH\x00\x12\x36\n\rschema_change\x18\x02 \x01(\x0b\x32\x1d.fivetran_sdk.v2.SchemaChangeH\x00\x12\x31\n\ncheckpoint\x18\x03 \x01(\x0b\x32\x1b.fivetran_sdk.v2.CheckpointH\x00\x12+\n\x07warning\x18\x04 \x01(\x0b\x32\x18.fivetran_sdk.v2.WarningH\x00\x12%\n\x04task\x18\x05 \x01(\x0b\x32\x15.fivetran_sdk.v2.TaskH\x00\x42\x0b\n\toperation\"\x82\x01\n\x0cSchemaChange\x12\x32\n\x0bwith_schema\x18\x01 \x01(\x0b\x32\x1b.fivetran_sdk.v2.SchemaListH\x00\x12\x34\n\x0ewithout_schema\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.TableListH\x00\x42\x08\n\x06\x63hange\"\xeb\x01\n\x06Record\x12\x18\n\x0bschema_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12)\n\x04type\x18\x03 \x01(\x0e\x32\x1b.fivetran_sdk.v2.RecordType\x12/\n\x04\x64\x61ta\x18\x04 \x03(\x0b\x32!.fivetran_sdk.v2.Record.DataEntry\x1aG\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.ValueType:\x02\x38\x01\x42\x0e\n\x0c_schema_name\" \n\nCheckpoint\x12\x12\n\nstate_json\x18\x01 \x01(\t2\xe2\x02\n\x0fSourceConnector\x12l\n\x11\x43onfigurationForm\x12).fivetran_sdk.v2.ConfigurationFormRequest\x1a*.fivetran_sdk.v2.ConfigurationFormResponse\"\x00\x12\x45\n\x04Test\x12\x1c.fivetran_sdk.v2.TestRequest\x1a\x1d.fivetran_sdk.v2.TestResponse\"\x00\x12K\n\x06Schema\x12\x1e.fivetran_sdk.v2.SchemaRequest\x1a\x1f.fivetran_sdk.v2.SchemaResponse\"\x00\x12M\n\x06Update\x12\x1e.fivetran_sdk.v2.UpdateRequest\x1a\x1f.fivetran_sdk.v2.UpdateResponse\"\x00\x30\x01\x42\"H\x01P\x01Z\x1c\x66ivetran.com/fivetran_sdk_v2b\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x63onnector_sdk.proto\x12\x0f\x66ivetran_sdk.v2\x1a\x0c\x63ommon.proto\"\x8f\x01\n\rSchemaRequest\x12H\n\rconfiguration\x18\x01 \x03(\x0b\x32\x31.fivetran_sdk.v2.SchemaRequest.ConfigurationEntry\x1a\x34\n\x12\x43onfigurationEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xf1\x01\n\x0eSchemaResponse\x12\'\n\x1dschema_response_not_supported\x18\x01 \x01(\x08H\x00\x12\x32\n\x0bwith_schema\x18\x02 \x01(\x0b\x32\x1b.fivetran_sdk.v2.SchemaListH\x00\x12\x34\n\x0ewithout_schema\x18\x03 \x01(\x0b\x32\x1a.fivetran_sdk.v2.TableListH\x00\x12$\n\x17selection_not_supported\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\n\n\x08responseB\x1a\n\x18_selection_not_supported\"\xf9\x01\n\rUpdateRequest\x12H\n\rconfiguration\x18\x01 \x03(\x0b\x32\x31.fivetran_sdk.v2.UpdateRequest.ConfigurationEntry\x12\x32\n\tselection\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.SelectionH\x00\x88\x01\x01\x12\x17\n\nstate_json\x18\x03 \x01(\tH\x01\x88\x01\x01\x1a\x34\n\x12\x43onfigurationEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0c\n\n_selectionB\r\n\x0b_state_json\"\x91\x01\n\tSelection\x12=\n\x0ewithout_schema\x18\x01 \x01(\x0b\x32#.fivetran_sdk.v2.TablesWithNoSchemaH\x00\x12\x38\n\x0bwith_schema\x18\x02 \x01(\x0b\x32!.fivetran_sdk.v2.TablesWithSchemaH\x00\x42\x0b\n\tselection\"a\n\x12TablesWithNoSchema\x12/\n\x06tables\x18\x01 \x03(\x0b\x32\x1f.fivetran_sdk.v2.TableSelection\x12\x1a\n\x12include_new_tables\x18\x02 \x01(\x08\"b\n\x10TablesWithSchema\x12\x31\n\x07schemas\x18\x01 \x03(\x0b\x32 .fivetran_sdk.v2.SchemaSelection\x12\x1b\n\x13include_new_schemas\x18\x02 \x01(\x08\"\x85\x01\n\x0fSchemaSelection\x12\x10\n\x08included\x18\x01 \x01(\x08\x12\x13\n\x0bschema_name\x18\x02 \x01(\t\x12/\n\x06tables\x18\x03 \x03(\x0b\x32\x1f.fivetran_sdk.v2.TableSelection\x12\x1a\n\x12include_new_tables\x18\x04 \x01(\x08\"\xc2\x01\n\x0eTableSelection\x12\x10\n\x08included\x18\x01 \x01(\x08\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12=\n\x07\x63olumns\x18\x03 \x03(\x0b\x32,.fivetran_sdk.v2.TableSelection.ColumnsEntry\x12\x1b\n\x13include_new_columns\x18\x04 \x01(\x08\x1a.\n\x0c\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"\xb4\x02\n\x0eUpdateResponse\x12)\n\x06record\x18\x01 \x01(\x0b\x32\x17.fivetran_sdk.v2.RecordH\x00\x12\x36\n\rschema_change\x18\x02 \x01(\x0b\x32\x1d.fivetran_sdk.v2.SchemaChangeH\x00\x12\x31\n\ncheckpoint\x18\x03 \x01(\x0b\x32\x1b.fivetran_sdk.v2.CheckpointH\x00\x12+\n\x07warning\x18\x04 \x01(\x0b\x32\x18.fivetran_sdk.v2.WarningH\x00\x12%\n\x04task\x18\x05 \x01(\x0b\x32\x15.fivetran_sdk.v2.TaskH\x00\x12+\n\x07records\x18\x06 \x01(\x0b\x32\x18.fivetran_sdk.v2.RecordsH\x00\x42\x0b\n\toperation\"\x82\x01\n\x0cSchemaChange\x12\x32\n\x0bwith_schema\x18\x01 \x01(\x0b\x32\x1b.fivetran_sdk.v2.SchemaListH\x00\x12\x34\n\x0ewithout_schema\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.TableListH\x00\x42\x08\n\x06\x63hange\"3\n\x07Records\x12(\n\x07records\x18\x01 \x03(\x0b\x32\x17.fivetran_sdk.v2.Record\"\xeb\x01\n\x06Record\x12\x18\n\x0bschema_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12)\n\x04type\x18\x03 \x01(\x0e\x32\x1b.fivetran_sdk.v2.RecordType\x12/\n\x04\x64\x61ta\x18\x04 \x03(\x0b\x32!.fivetran_sdk.v2.Record.DataEntry\x1aG\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.ValueType:\x02\x38\x01\x42\x0e\n\x0c_schema_name\" \n\nCheckpoint\x12\x12\n\nstate_json\x18\x01 \x01(\t2\xe2\x02\n\x0fSourceConnector\x12l\n\x11\x43onfigurationForm\x12).fivetran_sdk.v2.ConfigurationFormRequest\x1a*.fivetran_sdk.v2.ConfigurationFormResponse\"\x00\x12\x45\n\x04Test\x12\x1c.fivetran_sdk.v2.TestRequest\x1a\x1d.fivetran_sdk.v2.TestResponse\"\x00\x12K\n\x06Schema\x12\x1e.fivetran_sdk.v2.SchemaRequest\x1a\x1f.fivetran_sdk.v2.SchemaResponse\"\x00\x12M\n\x06Update\x12\x1e.fivetran_sdk.v2.UpdateRequest\x1a\x1f.fivetran_sdk.v2.UpdateResponse\"\x00\x30\x01\x42\"H\x01P\x01Z\x1c\x66ivetran.com/fivetran_sdk_v2b\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -65,15 +65,17 @@ if not _descriptor._USE_C_DESCRIPTORS:
   _globals['_TABLESELECTION_COLUMNSENTRY']._serialized_start=1328
   _globals['_TABLESELECTION_COLUMNSENTRY']._serialized_end=1374
   _globals['_UPDATERESPONSE']._serialized_start=1377
-  _globals['_UPDATERESPONSE']._serialized_end=1640
-  _globals['_SCHEMACHANGE']._serialized_start=1643
-  _globals['_SCHEMACHANGE']._serialized_end=1773
-  _globals['_RECORD']._serialized_start=1776
-  _globals['_RECORD']._serialized_end=2011
-  _globals['_RECORD_DATAENTRY']._serialized_start=1924
-  _globals['_RECORD_DATAENTRY']._serialized_end=1995
-  _globals['_CHECKPOINT']._serialized_start=2013
-  _globals['_CHECKPOINT']._serialized_end=2045
-  _globals['_SOURCECONNECTOR']._serialized_start=2048
-  _globals['_SOURCECONNECTOR']._serialized_end=2402
+  _globals['_UPDATERESPONSE']._serialized_end=1685
+  _globals['_SCHEMACHANGE']._serialized_start=1688
+  _globals['_SCHEMACHANGE']._serialized_end=1818
+  _globals['_RECORDS']._serialized_start=1820
+  _globals['_RECORDS']._serialized_end=1871
+  _globals['_RECORD']._serialized_start=1874
+  _globals['_RECORD']._serialized_end=2109
+  _globals['_RECORD_DATAENTRY']._serialized_start=2022
+  _globals['_RECORD_DATAENTRY']._serialized_end=2093
+  _globals['_CHECKPOINT']._serialized_start=2111
+  _globals['_CHECKPOINT']._serialized_end=2143
+  _globals['_SOURCECONNECTOR']._serialized_start=2146
+  _globals['_SOURCECONNECTOR']._serialized_end=2500
 # @@protoc_insertion_point(module_scope)

fivetran_connector_sdk/protos/connector_sdk_pb2.pyi

@@ -104,18 +104,20 @@ class TableSelection(_message.Message):
     def __init__(self, included: bool = ..., table_name: _Optional[str] = ..., columns: _Optional[_Mapping[str, bool]] = ..., include_new_columns: bool = ...) -> None: ...
 
 class UpdateResponse(_message.Message):
-    __slots__ = ("record", "schema_change", "checkpoint", "warning", "task")
+    __slots__ = ("record", "schema_change", "checkpoint", "warning", "task", "records")
     RECORD_FIELD_NUMBER: _ClassVar[int]
     SCHEMA_CHANGE_FIELD_NUMBER: _ClassVar[int]
     CHECKPOINT_FIELD_NUMBER: _ClassVar[int]
     WARNING_FIELD_NUMBER: _ClassVar[int]
     TASK_FIELD_NUMBER: _ClassVar[int]
+    RECORDS_FIELD_NUMBER: _ClassVar[int]
     record: Record
     schema_change: SchemaChange
     checkpoint: Checkpoint
     warning: _common_pb2.Warning
     task: _common_pb2.Task
-    def __init__(self, record: _Optional[_Union[Record, _Mapping]] = ..., schema_change: _Optional[_Union[SchemaChange, _Mapping]] = ..., checkpoint: _Optional[_Union[Checkpoint, _Mapping]] = ..., warning: _Optional[_Union[_common_pb2.Warning, _Mapping]] = ..., task: _Optional[_Union[_common_pb2.Task, _Mapping]] = ...) -> None: ...
+    records: Records
+    def __init__(self, record: _Optional[_Union[Record, _Mapping]] = ..., schema_change: _Optional[_Union[SchemaChange, _Mapping]] = ..., checkpoint: _Optional[_Union[Checkpoint, _Mapping]] = ..., warning: _Optional[_Union[_common_pb2.Warning, _Mapping]] = ..., task: _Optional[_Union[_common_pb2.Task, _Mapping]] = ..., records: _Optional[_Union[Records, _Mapping]] = ...) -> None: ...
 
 class SchemaChange(_message.Message):
     __slots__ = ("with_schema", "without_schema")
@@ -125,6 +127,12 @@ class SchemaChange(_message.Message):
     without_schema: _common_pb2.TableList
     def __init__(self, with_schema: _Optional[_Union[_common_pb2.SchemaList, _Mapping]] = ..., without_schema: _Optional[_Union[_common_pb2.TableList, _Mapping]] = ...) -> None: ...
 
+class Records(_message.Message):
+    __slots__ = ("records",)
+    RECORDS_FIELD_NUMBER: _ClassVar[int]
+    records: _containers.RepeatedCompositeFieldContainer[Record]
+    def __init__(self, records: _Optional[_Iterable[_Union[Record, _Mapping]]] = ...) -> None: ...
+
 class Record(_message.Message):
     __slots__ = ("schema_name", "table_name", "type", "data")
     class DataEntry(_message.Message):

{fivetran_connector_sdk-1.7.4.dist-info → fivetran_connector_sdk-1.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.7.4
+Version: 1.8.0
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

fivetran_connector_sdk-1.8.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+fivetran_connector_sdk/__init__.py,sha256=d_UKwca8fwy4nMGDLw655ISzjlC6KGV-z69gHZ_FiIU,22526
+fivetran_connector_sdk/connector_helper.py,sha256=hTEa_plE5xjPM0RFe9AziiJah5iGgO1oKGAGkibTyRo,43065
+fivetran_connector_sdk/constants.py,sha256=3vFpdoWGOSzcaw29P-GXQyJsSvKQgLcXU_0qdZrWQ0Y,2529
+fivetran_connector_sdk/helpers.py,sha256=k_iBaRacPN3YkOkZ8bLuflNYXkUrtuj6fYH_rV1M-RI,15224
+fivetran_connector_sdk/logger.py,sha256=ud8v8-mKx65OAPaZvxBqt2-CU0vjgBeiYwuiqsYh_hA,3063
+fivetran_connector_sdk/operations.py,sha256=9g-Eiy82qVHJGKfGHuIb1IZvVF9LbX_Kc1BldEs7BD0,16834
+fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fivetran_connector_sdk/protos/common_pb2.py,sha256=zkzs6Rd-lvsev6Nsq37xc4HLJZ_uNXPkotCLY7Y7i5U,8770
+fivetran_connector_sdk/protos/common_pb2.pyi,sha256=FdqlPKRqiXdUDT3e7adP5X42_Qzv_ItydUNJFKnJJIE,11478
+fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=qni6h6BoA1nwJXr2bNtznfTkrMokzzeQd8XQMZIiZUc,887
+fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=Inv87MlK5Q56GNvMNFQHyqIePDMKnkW9y_BrT9DgPck,7835
+fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=3AC-bK6ZM-Bmr_RETOB3y_0u4ATWlwcbHzqVanDuOB0,8115
+fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=bGlvc_vGwA9-FTqrj-BYlVcA-7jS8A9MSZ-XpZFytvY,8795
+fivetran_connector_sdk-1.8.0.dist-info/METADATA,sha256=_nmaaVX4rjtTGTlQL6XENjWm32oaZRK4HjP8PW0EgOA,3188
+fivetran_connector_sdk-1.8.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fivetran_connector_sdk-1.8.0.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-1.8.0.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-1.8.0.dist-info/RECORD,,

fivetran_connector_sdk-1.7.4.dist-info/RECORD

@@ -1,18 +0,0 @@
-fivetran_connector_sdk/__init__.py,sha256=2PC01krj_1210qy5n-_Gn7U6QZ35E9lbXpmf1CWlWMI,22573
-fivetran_connector_sdk/connector_helper.py,sha256=NDaaftRVsyEkdLFE8W59HFYpxfPsXZfIc1c0vTe03NI,43516
-fivetran_connector_sdk/constants.py,sha256=LBe_DOA1P8QLvqUDrMeNd-mdHWArTxKkbAhSp7zQD8w,2358
-fivetran_connector_sdk/helpers.py,sha256=k_iBaRacPN3YkOkZ8bLuflNYXkUrtuj6fYH_rV1M-RI,15224
-fivetran_connector_sdk/logger.py,sha256=ud8v8-mKx65OAPaZvxBqt2-CU0vjgBeiYwuiqsYh_hA,3063
-fivetran_connector_sdk/operations.py,sha256=yFhUqhMvbiHTZd3EPZXQuypwNGmyT-2yb0I8L6aDfkc,13288
-fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fivetran_connector_sdk/protos/common_pb2.py,sha256=zkzs6Rd-lvsev6Nsq37xc4HLJZ_uNXPkotCLY7Y7i5U,8770
-fivetran_connector_sdk/protos/common_pb2.pyi,sha256=FdqlPKRqiXdUDT3e7adP5X42_Qzv_ItydUNJFKnJJIE,11478
-fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=qni6h6BoA1nwJXr2bNtznfTkrMokzzeQd8XQMZIiZUc,887
-fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=qbce2wyScUg4cYRRjuYMgi5p0vb1zEA9Jf58polYhhs,7589
-fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=aE7DlQU3ZpuHK9aZrd1_cYs2_4Rl1lqSMw54BIXvYys,7721
-fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=bGlvc_vGwA9-FTqrj-BYlVcA-7jS8A9MSZ-XpZFytvY,8795
-fivetran_connector_sdk-1.7.4.dist-info/METADATA,sha256=YZNVl5viNASXjyCMbu4bZtEKw0pDJzKiwfc1OVM-Jws,3188
-fivetran_connector_sdk-1.7.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fivetran_connector_sdk-1.7.4.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-1.7.4.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-1.7.4.dist-info/RECORD,,