fivetran-connector-sdk 1.7.5__tar.gz → 1.8.1__tar.gz

{fivetran_connector_sdk-1.7.5 → fivetran_connector_sdk-1.8.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.7.5
+Version: 1.8.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-1.7.5 → fivetran_connector_sdk-1.8.1}/src/fivetran_connector_sdk/__init__.py

@@ -42,7 +42,7 @@ from fivetran_connector_sdk.connector_helper import (
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
-__version__ = "1.7.5"
+__version__ = "1.8.1"
 TESTER_VERSION = TESTER_VER
 MAX_MESSAGE_LENGTH = 32 * 1024 * 1024 # 32MB
 
@@ -396,7 +396,16 @@ class Connector(connector_sdk_pb2_grpc.SourceConnectorServicer):
             thread = threading.Thread(target=run_update)
             thread.start()
 
-            yield from Operations.operation_stream
+            # consumer - yield the operations in the operation_stream.
+            for response in Operations.operation_stream:
+                # checkpoint call always returns list of responses.
+                if isinstance(response, list):
+                    for res in response:
+                        yield res
+                    # checkpoint call blocks the queue (see _OperationStream.add method). unblock the queue after yielding all responses.
+                    Operations.operation_stream.unblock()
+                else:
+                    yield response
 
             thread.join()
 

{fivetran_connector_sdk-1.7.5 → fivetran_connector_sdk-1.8.1}/src/fivetran_connector_sdk/connector_helper.py

@@ -57,27 +57,22 @@ def get_destination_group(args):
         ft_group = get_input_from_cli("Provide the destination name (as displayed in your dashboard destination list)", env_destination_name)
     return ft_group
 
-def get_connection_name(args):
+def get_connection_name(args, retrying=0):
     ft_connection = args.connection if args.connection else None
     env_connection_name = os.getenv('FIVETRAN_CONNECTION_NAME', None)
     if not ft_connection:
-        for retrying in range(MAX_RETRIES):
-            ft_connection = get_input_from_cli("Provide the connection name", env_connection_name)
-            if not is_connection_name_valid(ft_connection):
-                if retrying==MAX_RETRIES-1:
-                    sys.exit(1)
-                else:
-                    print_library_log(f"Connection name: {ft_connection} is invalid!\n The connection name should start with an "
-                                      f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
-                                      f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
-                    print_library_log("Please retry...", Logging.Level.INFO)
-            else:
-                break
+        ft_connection = get_input_from_cli("Provide the connection name", env_connection_name)
     if not is_connection_name_valid(ft_connection):
-        print_library_log(f"Connection name: {ft_connection} is invalid!\n The connection name should start with an "
-                          f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
-                          f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
-        sys.exit(1)
+        print_library_log(
+            f"Connection name: {ft_connection} is invalid!\n The connection name should start with an "
+            f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
+            f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
+        args.connection = None
+        if retrying >= MAX_RETRIES or args.force:
+            sys.exit(1)
+        else:
+            print_library_log("Please retry...", Logging.Level.INFO)
+            return get_connection_name(args, retrying + 1)
     return ft_connection
 
 def get_api_key(args):
@@ -113,32 +108,36 @@ def get_state(args):
     state = validate_and_load_state(args, state)
     return state
 
-def get_configuration(args):
+def get_configuration(args, retrying = 0):
     configuration = args.configuration if args.configuration else None
     env_configuration = os.getenv('FIVETRAN_CONFIGURATION', None)
-    if not configuration and not args.force and args.command.lower() == "deploy":
-        json_filepath = os.path.join(args.project_path, "configuration.json")
-        if os.path.exists(json_filepath):
-            print_library_log("configuration.json file detected in the project, "
-                              "but no configuration input provided via the command line", Logging.Level.WARNING)
-            env_configuration = env_configuration if env_configuration else "configuration.json"
-        confirm = input(f"Does this debug run/deploy need configuration (y/N):")
-        if confirm.lower()=='y':
-            for retrying in range(MAX_RETRIES):
-                try:
-                    configuration = get_input_from_cli("Provide the configuration file path", env_configuration)
-                    config_values = validate_and_load_configuration(args.project_path, configuration)
-                    return config_values, configuration
-                except ValueError as e:
-                    if retrying==MAX_RETRIES-1:
-                        print_library_log(f"{e}. Invalid Configuration, Exiting..", Logging.Level.WARNING)
-                        sys.exit(1)
-                    else:
-                        print_library_log(f"{e}. Please retry..", Logging.Level.INFO)
+    try:
+        if not configuration and not args.force and args.command.lower() == "deploy":
+            confirm = 'y'
+            if not retrying:
+                json_filepath = os.path.join(args.project_path, "configuration.json")
+                if os.path.exists(json_filepath):
+                    print_library_log("configuration.json file detected in the project, "
+                                      "but no configuration input provided via the command line", Logging.Level.WARNING)
+                    env_configuration = env_configuration if env_configuration else "configuration.json"
+                confirm = input(f"Does this debug run/deploy need configuration (y/N):")
+            if confirm.lower()=='y':
+                configuration = get_input_from_cli("Provide the configuration file path", env_configuration)
+                config_values = validate_and_load_configuration(args.project_path, configuration)
+                return config_values, configuration
+            else:
+                print_library_log("No input required for configuration. Continuing without configuration.", Logging.Level.INFO)
+                return {}, None
+        config_values = validate_and_load_configuration(args.project_path, configuration)
+        return config_values, configuration
+    except ValueError as e:
+        args.configuration = None
+        if retrying >= MAX_RETRIES or args.force:
+            print_library_log(f"{e}. Invalid Configuration, Exiting..", Logging.Level.WARNING)
+            sys.exit(1)
         else:
-            print_library_log("No input required for configuration. Continuing without configuration.", Logging.Level.INFO)
-    config_values = validate_and_load_configuration(args.project_path, configuration)
-    return config_values, configuration
+            print_library_log(f"{e}. Please retry..", Logging.Level.INFO)
+            return get_configuration(args, retrying + 1)
 
 
 def check_newer_version(version: str):
@@ -989,8 +988,6 @@ def process_data_type(column, type):
         column.type = common_pb2.DataType.FLOAT
     elif type.upper() == "DOUBLE":
         column.type = common_pb2.DataType.DOUBLE
-    elif type.upper() == "NAIVE_TIME":
-        column.type = common_pb2.DataType.NAIVE_TIME
     elif type.upper() == "NAIVE_DATE":
         column.type = common_pb2.DataType.NAIVE_DATE
     elif type.upper() == "NAIVE_DATETIME":

{fivetran_connector_sdk-1.7.5 → fivetran_connector_sdk-1.8.1}/src/fivetran_connector_sdk/constants.py

@@ -1,6 +1,7 @@
+import os
 import re
 
-TESTER_VER = "2.25.0701.001"
+TESTER_VER = "2.25.0806.001"
 
 WIN_OS = "windows"
 ARM_64 = "arm64"
@@ -34,11 +35,16 @@ OUTPUT_FILES_DIR = "files"
 REQUIREMENTS_TXT = "requirements.txt"
 PYPI_PACKAGE_DETAILS_URL = "https://pypi.org/pypi/fivetran_connector_sdk/json"
 ONE_DAY_IN_SEC = 24 * 60 * 60
+CHECKPOINT_OP_TIMEOUT_IN_SEC = 30 # seconds
 MAX_RETRIES = 3
 LOGGING_PREFIX = "Fivetran-Connector-SDK"
 LOGGING_DELIMITER = ": "
 VIRTUAL_ENV_CONFIG = "pyvenv.cfg"
 ROOT_FILENAME = "connector.py"
+MAX_RECORDS_IN_BATCH = 100
+MAX_BATCH_SIZE_IN_BYTES = 100000 # 100 KB
+# The increased queue size shall be rolled out as Feature flag
+QUEUE_SIZE = int(os.environ.get("QUEUE_SIZE", 1))
 
 # Compile patterns used in the implementation
 WORD_DASH_DOT_PATTERN = re.compile(r'^[\w.-]*$')

{fivetran_connector_sdk-1.7.5 → fivetran_connector_sdk-1.8.1}/src/fivetran_connector_sdk/operations.py

@@ -1,7 +1,7 @@
 import json
-import os
 import sys
 import queue
+import threading
 
 from datetime import datetime
 from google.protobuf import timestamp_pb2
@@ -9,6 +9,10 @@ from google.protobuf import timestamp_pb2
 from fivetran_connector_sdk.constants import (
     JAVA_LONG_MAX_VALUE,
     TABLES,
+    MAX_RECORDS_IN_BATCH,
+    MAX_BATCH_SIZE_IN_BYTES,
+    QUEUE_SIZE,
+    CHECKPOINT_OP_TIMEOUT_IN_SEC
 )
 from fivetran_connector_sdk.helpers import (
     get_renamed_table_name,
@@ -38,8 +42,15 @@ class _OperationStream:
         """
         Initializes the operation stream with a queue and a sentinel object.
         """
-        self._queue = queue.Queue(maxsize=1)
+        self._queue = queue.Queue(maxsize=QUEUE_SIZE)
         self._sentinel = object()
+        self._is_done = False
+        self._buffer = []
+        self._buffer_record_count = 0
+        self._buffer_size_bytes = 0
+        self._checkpoint_lock = threading.Lock()
+        self._checkpoint_flush_signal = threading.Event()
+        self._checkpoint_flush_signal.set()
 
     def __iter__(self):
         """
@@ -47,14 +58,38 @@ class _OperationStream:
         """
         return self
 
-    def add(self, data):
+    def add(self, operation):
         """
-        Adds an item to the stream.
+        Adds an operation to the stream. Guarantees that operations within a single thread are processed in the order.
+
+        In multithreaded environment if a thread initiates a checkpoint, it's producer is blocked until the
+        checkpoint flush is complete. This block is localized, other threads
+        remain unblocked and can continue to perform other operations
+        (such as upserts, updates, deletes), but they are prevented from initiating a new checkpoint
+        until the existing one is finished.
 
         Args:
-            data (object): The data item to add to the stream.
+            operation (object): The data item to add to the stream.
+        """
+        if isinstance(operation, connector_sdk_pb2.Checkpoint):
+            # lock to ensure checkpoint operations are processed one at a time
+            with self._checkpoint_lock:
+                # clear the signal to indicate checkpoint operation is being processed.
+                self._checkpoint_flush_signal.clear()
+                self._queue.put(operation)
+                # wait until the consumer flushes the buffer and sets the flag.
+                if not self._checkpoint_flush_signal.wait(CHECKPOINT_OP_TIMEOUT_IN_SEC):
+                    raise TimeoutError(
+                        "Checkpoint flush timed out. Consumer may have failed to process checkpoint."
+                    )
+        else:
+            self._queue.put(operation)
+
+    def unblock(self):
+        """
+        Unblocks the queue, called by consumer after the checkpoint flush is completed.
         """
-        self._queue.put(data)
+        self._checkpoint_flush_signal.set()
 
     def mark_done(self):
         """
@@ -72,16 +107,78 @@ class _OperationStream:
         Raises:
             StopIteration: If the sentinel object is encountered.
         """
-        item = self._queue.get()
-        if item is self._sentinel:
+        # If stream is completed and buffer is empty, raise StopIteration. Else flush the buffer.
+        if self._is_done and not self._buffer:
             raise StopIteration
-        return item
 
-_LOG_DATA_TYPE_INFERENCE = {
-    "boolean": True,
-    "binary": True,
-    "json": True
-}
+        if self._is_done:
+            return self._flush_buffer()
+
+        return self._build_next_batch()
+
+    def _build_next_batch(self):
+        """
+        Core logic to build the batch. The loop continues until the buffer is full,
+        but can be interrupted by a checkpoint or a sentinel from the producer.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse or list[connector_sdk_pb2.UpdateResponse]: Either a single response
+            containing records or checkpoint, or a list of responses when flushing data with a checkpoint.
+
+        """
+        while self._buffer_record_count < MAX_RECORDS_IN_BATCH and self._buffer_size_bytes < MAX_BATCH_SIZE_IN_BYTES:
+            operation = self._queue.get()
+
+            # Case 1: If operation is sentinel, mark the stream as done, flush the buffer.
+            if operation is self._sentinel:
+                self._is_done = True
+                if self._buffer:
+                    return self._flush_buffer()
+                else:
+                    raise StopIteration
+
+            # Case 2: if operation is a Checkpoint, flush the buffer and send the checkpoint.
+            elif isinstance(operation, connector_sdk_pb2.Checkpoint):
+                return self._flush_buffer_on_checkpoint(operation)
+
+            # it is record, buffer it to flush in batches
+            self._buffer_record_count += 1
+            self._buffer_size_bytes += len(operation.SerializeToString())
+            self._buffer.append(operation)
+
+        # Case 3: If buffer size limit is reached, flush the buffer and return the response.
+        return self._flush_buffer()
+
+    def _flush_buffer_on_checkpoint(self, checkpoint: connector_sdk_pb2.Checkpoint):
+        """
+        Creates the responses containing the checkpoint and buffered records.
+
+        Args:
+            checkpoint (object): Checkpoint operation to be added to the response.
+        """
+        responses = []
+
+        if self._buffer:
+            responses.append(self._flush_buffer())
+
+        responses.append(connector_sdk_pb2.UpdateResponse(checkpoint=checkpoint))
+        return responses
+
+    def _flush_buffer(self):
+        """
+        Flushes the current buffer and returns a response containing the buffered records.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: A response containing the buffered records.
+        """
+        batch_to_flush = self._buffer
+        self._buffer = []
+        self._buffer_record_count = 0
+        self._buffer_size_bytes = 0
+        return connector_sdk_pb2.UpdateResponse(records=connector_sdk_pb2.Records(records=batch_to_flush))
+
+
+_LOG_DATA_TYPE_INFERENCE = {}
 
 class Operations:
     operation_stream = _OperationStream()
@@ -107,17 +204,15 @@ class Operations:
             new_table = common_pb2.Table(name=table, columns=columns.values())
             TABLES[table] = new_table
 
-        mapped_data = _map_data_to_columns(data, columns)
+        mapped_data = _map_data_to_columns(data, columns, table)
         record = connector_sdk_pb2.Record(
             schema_name=None,
             table_name=table,
             type=common_pb2.RecordType.UPSERT,
             data=mapped_data
         )
-        update_response = connector_sdk_pb2.UpdateResponse(record=record)
-
-        Operations.operation_stream.add(update_response)
 
+        Operations.operation_stream.add(record)
 
     @staticmethod
     def update(table: str, modified: dict):
@@ -132,7 +227,7 @@ class Operations:
         """
         table = get_renamed_table_name(table)
         columns = _get_columns(table)
-        mapped_data = _map_data_to_columns(modified, columns)
+        mapped_data = _map_data_to_columns(modified, columns, table)
         record = connector_sdk_pb2.Record(
             schema_name=None,
             table_name=table,
@@ -140,9 +235,7 @@ class Operations:
             data=mapped_data
         )
 
-        update_response = connector_sdk_pb2.UpdateResponse(record=record)
-
-        Operations.operation_stream.add(update_response)
+        Operations.operation_stream.add(record)
 
     @staticmethod
     def delete(table: str, keys: dict):
@@ -157,7 +250,7 @@ class Operations:
         """
         table = get_renamed_table_name(table)
         columns = _get_columns(table)
-        mapped_data = _map_data_to_columns(keys, columns)
+        mapped_data = _map_data_to_columns(keys, columns, table)
         record = connector_sdk_pb2.Record(
             schema_name=None,
             table_name=table,
@@ -165,9 +258,7 @@ class Operations:
             data=mapped_data
         )
 
-        update_response = connector_sdk_pb2.UpdateResponse(record=record)
-
-        Operations.operation_stream.add(update_response)
+        Operations.operation_stream.add(record)
 
     @staticmethod
     def checkpoint(state: dict):
@@ -190,9 +281,9 @@ class Operations:
         Returns:
             connector_sdk_pb2.UpdateResponse: The checkpoint response.
         """
-        update_response = connector_sdk_pb2.UpdateResponse(checkpoint=connector_sdk_pb2.Checkpoint(state_json=json.dumps(state)))
+        checkpoint = connector_sdk_pb2.Checkpoint(state_json=json.dumps(state))
 
-        Operations.operation_stream.add(update_response)
+        Operations.operation_stream.add(checkpoint)
 
 def _get_columns(table: str) -> dict:
     """Retrieves the columns for the specified table.
@@ -210,8 +301,24 @@ def _get_columns(table: str) -> dict:
 
     return columns
 
+def _get_table_pk(table: str) -> bool:
+    """Retrieves the columns for the specified table.
 
-def _map_data_to_columns(data: dict, columns: dict) -> dict:
+    Args:
+        table (str): The name of the table.
+
+    Returns:
+        dict: The columns for the table.
+    """
+    columns = {}
+    if table in TABLES:
+        for column in TABLES[table].columns:
+            if column.primary_key:
+                return True
+    return False
+
+
+def _map_data_to_columns(data: dict, columns: dict, table: str = "") -> dict:
     """Maps data to the specified columns.
 
     Args:
@@ -229,35 +336,31 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
         elif (key in columns) and columns[key].type != common_pb2.DataType.UNSPECIFIED:
             map_defined_data_type(columns[key].type, key, mapped_data, v)
         else:
-            map_inferred_data_type(key, mapped_data, v)
+            map_inferred_data_type(key, mapped_data, v, table)
     return mapped_data
 
-def map_inferred_data_type(k, mapped_data, v):
+def map_inferred_data_type(k, mapped_data, v, table=""):
     # We can infer type from the value
     if isinstance(v, int):
         if abs(v) > JAVA_LONG_MAX_VALUE:
             mapped_data[k] = common_pb2.ValueType(float=v)
         else:
             mapped_data[k] = common_pb2.ValueType(long=v)
-        if _LOG_DATA_TYPE_INFERENCE["boolean"] and isinstance(v, bool):
-            print_library_log("Fivetran: Boolean Datatype has been inferred", Logging.Level.INFO, True)
-            _LOG_DATA_TYPE_INFERENCE["boolean"] = False
+        if _LOG_DATA_TYPE_INFERENCE.get("boolean_" + table, True) and isinstance(v, bool):
+            print_library_log("Fivetran: Boolean Datatype has been inferred for " + table, Logging.Level.INFO, True)
+            if not _get_table_pk(table):
+                print_library_log("Fivetran: Boolean Datatype inference issue for " + table, Logging.Level.INFO, True)
+            _LOG_DATA_TYPE_INFERENCE["boolean_" + table] = False
     elif isinstance(v, float):
         mapped_data[k] = common_pb2.ValueType(float=v)
     elif isinstance(v, bool):
         mapped_data[k] = common_pb2.ValueType(bool=v)
     elif isinstance(v, bytes):
-        if _LOG_DATA_TYPE_INFERENCE["binary"]:
-            print_library_log("Fivetran: Binary Datatype has been inferred", Logging.Level.INFO, True)
-            _LOG_DATA_TYPE_INFERENCE["binary"] = False
         mapped_data[k] = common_pb2.ValueType(binary=v)
     elif isinstance(v, list):
         raise ValueError(
             "Values for the columns cannot be of type 'list'. Please ensure that all values are of a supported type. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#supporteddatatypes")
     elif isinstance(v, dict):
-        if _LOG_DATA_TYPE_INFERENCE["json"]:
-            print_library_log("Fivetran: JSON Datatype has been inferred", Logging.Level.INFO, True)
-            _LOG_DATA_TYPE_INFERENCE["json"] = False
         mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
     elif isinstance(v, str):
         mapped_data[k] = common_pb2.ValueType(string=v)
@@ -334,4 +437,3 @@ def _yield_check(stack):
         # This should never happen
         raise RuntimeError(
             f"The '{called_method}' function is missing in the connector calling code '{calling_code}'. Please ensure that the '{called_method}' function is properly defined in your code to proceed. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsmethods")
-

{fivetran_connector_sdk-1.7.5 → fivetran_connector_sdk-1.8.1}/src/fivetran_connector_sdk/protos/connector_sdk_pb2.py

@@ -26,7 +26,7 @@ _sym_db = _symbol_database.Default()
 from fivetran_connector_sdk import common_pb2 as common__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x63onnector_sdk.proto\x12\x0f\x66ivetran_sdk.v2\x1a\x0c\x63ommon.proto\"\x8f\x01\n\rSchemaRequest\x12H\n\rconfiguration\x18\x01 \x03(\x0b\x32\x31.fivetran_sdk.v2.SchemaRequest.ConfigurationEntry\x1a\x34\n\x12\x43onfigurationEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xf1\x01\n\x0eSchemaResponse\x12\'\n\x1dschema_response_not_supported\x18\x01 \x01(\x08H\x00\x12\x32\n\x0bwith_schema\x18\x02 \x01(\x0b\x32\x1b.fivetran_sdk.v2.SchemaListH\x00\x12\x34\n\x0ewithout_schema\x18\x03 \x01(\x0b\x32\x1a.fivetran_sdk.v2.TableListH\x00\x12$\n\x17selection_not_supported\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\n\n\x08responseB\x1a\n\x18_selection_not_supported\"\xf9\x01\n\rUpdateRequest\x12H\n\rconfiguration\x18\x01 \x03(\x0b\x32\x31.fivetran_sdk.v2.UpdateRequest.ConfigurationEntry\x12\x32\n\tselection\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.SelectionH\x00\x88\x01\x01\x12\x17\n\nstate_json\x18\x03 \x01(\tH\x01\x88\x01\x01\x1a\x34\n\x12\x43onfigurationEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0c\n\n_selectionB\r\n\x0b_state_json\"\x91\x01\n\tSelection\x12=\n\x0ewithout_schema\x18\x01 \x01(\x0b\x32#.fivetran_sdk.v2.TablesWithNoSchemaH\x00\x12\x38\n\x0bwith_schema\x18\x02 \x01(\x0b\x32!.fivetran_sdk.v2.TablesWithSchemaH\x00\x42\x0b\n\tselection\"a\n\x12TablesWithNoSchema\x12/\n\x06tables\x18\x01 \x03(\x0b\x32\x1f.fivetran_sdk.v2.TableSelection\x12\x1a\n\x12include_new_tables\x18\x02 \x01(\x08\"b\n\x10TablesWithSchema\x12\x31\n\x07schemas\x18\x01 \x03(\x0b\x32 .fivetran_sdk.v2.SchemaSelection\x12\x1b\n\x13include_new_schemas\x18\x02 \x01(\x08\"\x85\x01\n\x0fSchemaSelection\x12\x10\n\x08included\x18\x01 \x01(\x08\x12\x13\n\x0bschema_name\x18\x02 \x01(\t\x12/\n\x06tables\x18\x03 \x03(\x0b\x32\x1f.fivetran_sdk.v2.TableSelection\x12\x1a\n\x12include_new_tables\x18\x04 \x01(\x08\"\xc2\x01\n\x0eTableSelection\x12\x10\n\x08included\x18\x01 \x01(\x08\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12=\n\x07\x63olumns\x18\x03 \x03(\x0b\x32,.fivetran_sdk.v2.TableSelection.ColumnsEntry\x12\x1b\n\x13include_new_columns\x18\x04 \x01(\x08\x1a.\n\x0c\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"\x87\x02\n\x0eUpdateResponse\x12)\n\x06record\x18\x01 \x01(\x0b\x32\x17.fivetran_sdk.v2.RecordH\x00\x12\x36\n\rschema_change\x18\x02 \x01(\x0b\x32\x1d.fivetran_sdk.v2.SchemaChangeH\x00\x12\x31\n\ncheckpoint\x18\x03 \x01(\x0b\x32\x1b.fivetran_sdk.v2.CheckpointH\x00\x12+\n\x07warning\x18\x04 \x01(\x0b\x32\x18.fivetran_sdk.v2.WarningH\x00\x12%\n\x04task\x18\x05 \x01(\x0b\x32\x15.fivetran_sdk.v2.TaskH\x00\x42\x0b\n\toperation\"\x82\x01\n\x0cSchemaChange\x12\x32\n\x0bwith_schema\x18\x01 \x01(\x0b\x32\x1b.fivetran_sdk.v2.SchemaListH\x00\x12\x34\n\x0ewithout_schema\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.TableListH\x00\x42\x08\n\x06\x63hange\"\xeb\x01\n\x06Record\x12\x18\n\x0bschema_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12)\n\x04type\x18\x03 \x01(\x0e\x32\x1b.fivetran_sdk.v2.RecordType\x12/\n\x04\x64\x61ta\x18\x04 \x03(\x0b\x32!.fivetran_sdk.v2.Record.DataEntry\x1aG\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.ValueType:\x02\x38\x01\x42\x0e\n\x0c_schema_name\" \n\nCheckpoint\x12\x12\n\nstate_json\x18\x01 \x01(\t2\xe2\x02\n\x0fSourceConnector\x12l\n\x11\x43onfigurationForm\x12).fivetran_sdk.v2.ConfigurationFormRequest\x1a*.fivetran_sdk.v2.ConfigurationFormResponse\"\x00\x12\x45\n\x04Test\x12\x1c.fivetran_sdk.v2.TestRequest\x1a\x1d.fivetran_sdk.v2.TestResponse\"\x00\x12K\n\x06Schema\x12\x1e.fivetran_sdk.v2.SchemaRequest\x1a\x1f.fivetran_sdk.v2.SchemaResponse\"\x00\x12M\n\x06Update\x12\x1e.fivetran_sdk.v2.UpdateRequest\x1a\x1f.fivetran_sdk.v2.UpdateResponse\"\x00\x30\x01\x42\"H\x01P\x01Z\x1c\x66ivetran.com/fivetran_sdk_v2b\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13\x63onnector_sdk.proto\x12\x0f\x66ivetran_sdk.v2\x1a\x0c\x63ommon.proto\"\x8f\x01\n\rSchemaRequest\x12H\n\rconfiguration\x18\x01 \x03(\x0b\x32\x31.fivetran_sdk.v2.SchemaRequest.ConfigurationEntry\x1a\x34\n\x12\x43onfigurationEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xf1\x01\n\x0eSchemaResponse\x12\'\n\x1dschema_response_not_supported\x18\x01 \x01(\x08H\x00\x12\x32\n\x0bwith_schema\x18\x02 \x01(\x0b\x32\x1b.fivetran_sdk.v2.SchemaListH\x00\x12\x34\n\x0ewithout_schema\x18\x03 \x01(\x0b\x32\x1a.fivetran_sdk.v2.TableListH\x00\x12$\n\x17selection_not_supported\x18\x04 \x01(\x08H\x01\x88\x01\x01\x42\n\n\x08responseB\x1a\n\x18_selection_not_supported\"\xf9\x01\n\rUpdateRequest\x12H\n\rconfiguration\x18\x01 \x03(\x0b\x32\x31.fivetran_sdk.v2.UpdateRequest.ConfigurationEntry\x12\x32\n\tselection\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.SelectionH\x00\x88\x01\x01\x12\x17\n\nstate_json\x18\x03 \x01(\tH\x01\x88\x01\x01\x1a\x34\n\x12\x43onfigurationEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0c\n\n_selectionB\r\n\x0b_state_json\"\x91\x01\n\tSelection\x12=\n\x0ewithout_schema\x18\x01 \x01(\x0b\x32#.fivetran_sdk.v2.TablesWithNoSchemaH\x00\x12\x38\n\x0bwith_schema\x18\x02 \x01(\x0b\x32!.fivetran_sdk.v2.TablesWithSchemaH\x00\x42\x0b\n\tselection\"a\n\x12TablesWithNoSchema\x12/\n\x06tables\x18\x01 \x03(\x0b\x32\x1f.fivetran_sdk.v2.TableSelection\x12\x1a\n\x12include_new_tables\x18\x02 \x01(\x08\"b\n\x10TablesWithSchema\x12\x31\n\x07schemas\x18\x01 \x03(\x0b\x32 .fivetran_sdk.v2.SchemaSelection\x12\x1b\n\x13include_new_schemas\x18\x02 \x01(\x08\"\x85\x01\n\x0fSchemaSelection\x12\x10\n\x08included\x18\x01 \x01(\x08\x12\x13\n\x0bschema_name\x18\x02 \x01(\t\x12/\n\x06tables\x18\x03 \x03(\x0b\x32\x1f.fivetran_sdk.v2.TableSelection\x12\x1a\n\x12include_new_tables\x18\x04 \x01(\x08\"\xc2\x01\n\x0eTableSelection\x12\x10\n\x08included\x18\x01 \x01(\x08\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12=\n\x07\x63olumns\x18\x03 \x03(\x0b\x32,.fivetran_sdk.v2.TableSelection.ColumnsEntry\x12\x1b\n\x13include_new_columns\x18\x04 \x01(\x08\x1a.\n\x0c\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\"\xb4\x02\n\x0eUpdateResponse\x12)\n\x06record\x18\x01 \x01(\x0b\x32\x17.fivetran_sdk.v2.RecordH\x00\x12\x36\n\rschema_change\x18\x02 \x01(\x0b\x32\x1d.fivetran_sdk.v2.SchemaChangeH\x00\x12\x31\n\ncheckpoint\x18\x03 \x01(\x0b\x32\x1b.fivetran_sdk.v2.CheckpointH\x00\x12+\n\x07warning\x18\x04 \x01(\x0b\x32\x18.fivetran_sdk.v2.WarningH\x00\x12%\n\x04task\x18\x05 \x01(\x0b\x32\x15.fivetran_sdk.v2.TaskH\x00\x12+\n\x07records\x18\x06 \x01(\x0b\x32\x18.fivetran_sdk.v2.RecordsH\x00\x42\x0b\n\toperation\"\x82\x01\n\x0cSchemaChange\x12\x32\n\x0bwith_schema\x18\x01 \x01(\x0b\x32\x1b.fivetran_sdk.v2.SchemaListH\x00\x12\x34\n\x0ewithout_schema\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.TableListH\x00\x42\x08\n\x06\x63hange\"3\n\x07Records\x12(\n\x07records\x18\x01 \x03(\x0b\x32\x17.fivetran_sdk.v2.Record\"\xeb\x01\n\x06Record\x12\x18\n\x0bschema_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\ntable_name\x18\x02 \x01(\t\x12)\n\x04type\x18\x03 \x01(\x0e\x32\x1b.fivetran_sdk.v2.RecordType\x12/\n\x04\x64\x61ta\x18\x04 \x03(\x0b\x32!.fivetran_sdk.v2.Record.DataEntry\x1aG\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.fivetran_sdk.v2.ValueType:\x02\x38\x01\x42\x0e\n\x0c_schema_name\" \n\nCheckpoint\x12\x12\n\nstate_json\x18\x01 \x01(\t2\xe2\x02\n\x0fSourceConnector\x12l\n\x11\x43onfigurationForm\x12).fivetran_sdk.v2.ConfigurationFormRequest\x1a*.fivetran_sdk.v2.ConfigurationFormResponse\"\x00\x12\x45\n\x04Test\x12\x1c.fivetran_sdk.v2.TestRequest\x1a\x1d.fivetran_sdk.v2.TestResponse\"\x00\x12K\n\x06Schema\x12\x1e.fivetran_sdk.v2.SchemaRequest\x1a\x1f.fivetran_sdk.v2.SchemaResponse\"\x00\x12M\n\x06Update\x12\x1e.fivetran_sdk.v2.UpdateRequest\x1a\x1f.fivetran_sdk.v2.UpdateResponse\"\x00\x30\x01\x42\"H\x01P\x01Z\x1c\x66ivetran.com/fivetran_sdk_v2b\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -65,15 +65,17 @@ if not _descriptor._USE_C_DESCRIPTORS:
   _globals['_TABLESELECTION_COLUMNSENTRY']._serialized_start=1328
   _globals['_TABLESELECTION_COLUMNSENTRY']._serialized_end=1374
   _globals['_UPDATERESPONSE']._serialized_start=1377
-  _globals['_UPDATERESPONSE']._serialized_end=1640
-  _globals['_SCHEMACHANGE']._serialized_start=1643
-  _globals['_SCHEMACHANGE']._serialized_end=1773
-  _globals['_RECORD']._serialized_start=1776
-  _globals['_RECORD']._serialized_end=2011
-  _globals['_RECORD_DATAENTRY']._serialized_start=1924
-  _globals['_RECORD_DATAENTRY']._serialized_end=1995
-  _globals['_CHECKPOINT']._serialized_start=2013
-  _globals['_CHECKPOINT']._serialized_end=2045
-  _globals['_SOURCECONNECTOR']._serialized_start=2048
-  _globals['_SOURCECONNECTOR']._serialized_end=2402
+  _globals['_UPDATERESPONSE']._serialized_end=1685
+  _globals['_SCHEMACHANGE']._serialized_start=1688
+  _globals['_SCHEMACHANGE']._serialized_end=1818
+  _globals['_RECORDS']._serialized_start=1820
+  _globals['_RECORDS']._serialized_end=1871
+  _globals['_RECORD']._serialized_start=1874
+  _globals['_RECORD']._serialized_end=2109
+  _globals['_RECORD_DATAENTRY']._serialized_start=2022
+  _globals['_RECORD_DATAENTRY']._serialized_end=2093
+  _globals['_CHECKPOINT']._serialized_start=2111
+  _globals['_CHECKPOINT']._serialized_end=2143
+  _globals['_SOURCECONNECTOR']._serialized_start=2146
+  _globals['_SOURCECONNECTOR']._serialized_end=2500
 # @@protoc_insertion_point(module_scope)

{fivetran_connector_sdk-1.7.5 → fivetran_connector_sdk-1.8.1}/src/fivetran_connector_sdk/protos/connector_sdk_pb2.pyi

@@ -104,18 +104,20 @@ class TableSelection(_message.Message):
     def __init__(self, included: bool = ..., table_name: _Optional[str] = ..., columns: _Optional[_Mapping[str, bool]] = ..., include_new_columns: bool = ...) -> None: ...
 
 class UpdateResponse(_message.Message):
-    __slots__ = ("record", "schema_change", "checkpoint", "warning", "task")
+    __slots__ = ("record", "schema_change", "checkpoint", "warning", "task", "records")
     RECORD_FIELD_NUMBER: _ClassVar[int]
     SCHEMA_CHANGE_FIELD_NUMBER: _ClassVar[int]
     CHECKPOINT_FIELD_NUMBER: _ClassVar[int]
     WARNING_FIELD_NUMBER: _ClassVar[int]
     TASK_FIELD_NUMBER: _ClassVar[int]
+    RECORDS_FIELD_NUMBER: _ClassVar[int]
     record: Record
     schema_change: SchemaChange
     checkpoint: Checkpoint
     warning: _common_pb2.Warning
     task: _common_pb2.Task
-    def __init__(self, record: _Optional[_Union[Record, _Mapping]] = ..., schema_change: _Optional[_Union[SchemaChange, _Mapping]] = ..., checkpoint: _Optional[_Union[Checkpoint, _Mapping]] = ..., warning: _Optional[_Union[_common_pb2.Warning, _Mapping]] = ..., task: _Optional[_Union[_common_pb2.Task, _Mapping]] = ...) -> None: ...
+    records: Records
+    def __init__(self, record: _Optional[_Union[Record, _Mapping]] = ..., schema_change: _Optional[_Union[SchemaChange, _Mapping]] = ..., checkpoint: _Optional[_Union[Checkpoint, _Mapping]] = ..., warning: _Optional[_Union[_common_pb2.Warning, _Mapping]] = ..., task: _Optional[_Union[_common_pb2.Task, _Mapping]] = ..., records: _Optional[_Union[Records, _Mapping]] = ...) -> None: ...
 
 class SchemaChange(_message.Message):
     __slots__ = ("with_schema", "without_schema")
@@ -125,6 +127,12 @@ class SchemaChange(_message.Message):
     without_schema: _common_pb2.TableList
     def __init__(self, with_schema: _Optional[_Union[_common_pb2.SchemaList, _Mapping]] = ..., without_schema: _Optional[_Union[_common_pb2.TableList, _Mapping]] = ...) -> None: ...
 
+class Records(_message.Message):
+    __slots__ = ("records",)
+    RECORDS_FIELD_NUMBER: _ClassVar[int]
+    records: _containers.RepeatedCompositeFieldContainer[Record]
+    def __init__(self, records: _Optional[_Iterable[_Union[Record, _Mapping]]] = ...) -> None: ...
+
 class Record(_message.Message):
     __slots__ = ("schema_name", "table_name", "type", "data")
     class DataEntry(_message.Message):

{fivetran_connector_sdk-1.7.5 → fivetran_connector_sdk-1.8.1}/src/fivetran_connector_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.7.5
+Version: 1.8.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk