fivetran-connector-sdk 1.8.1__tar.gz → 2.0.1__tar.gz

{fivetran_connector_sdk-1.8.1 → fivetran_connector_sdk-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.8.1
+Version: 2.0.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -13,7 +13,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: grpcio==1.71.0
 Requires-Dist: grpcio-tools==1.71.0
-Requires-Dist: requests==2.32.4
+Requires-Dist: requests==2.32.5
 Requires-Dist: pipreqs==0.5.0
 Requires-Dist: prompt-toolkit==3.0.51
 Requires-Dist: unidecode==1.4.0

{fivetran_connector_sdk-1.8.1 → fivetran_connector_sdk-2.0.1}/pyproject.toml

@@ -15,7 +15,7 @@ classifiers = [
 dependencies = [
   "grpcio==1.71.0",
   "grpcio-tools==1.71.0",
-  "requests==2.32.4",
+  "requests==2.32.5",
   "pipreqs==0.5.0",
   "prompt-toolkit==3.0.51",
   "unidecode==1.4.0"

{fivetran_connector_sdk-1.8.1 → fivetran_connector_sdk-2.0.1}/src/fivetran_connector_sdk/__init__.py

@@ -41,8 +41,8 @@ from fivetran_connector_sdk.connector_helper import (
 )
 
 # Version format: <major_version>.<minor_version>.<patch_version>
-# (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
-__version__ = "1.8.1"
+# (where Major Version = 2, Minor Version is incremental MM from Aug 25 onwards, Patch Version is incremental within a month)
+__version__ = "2.0.1"
 TESTER_VERSION = TESTER_VER
 MAX_MESSAGE_LENGTH = 32 * 1024 * 1024 # 32MB
 
@@ -108,7 +108,11 @@ class Connector(connector_sdk_pb2_grpc.SourceConnectorServicer):
         if python_version:
             connection_config["python_version"] = python_version
 
-        validate_requirements_file(project_path, True, __version__, force)
+        if not force:
+            validate_requirements_file(project_path, True, __version__)
+        else:
+            print_library_log(
+                "Skipping requirements.txt validation as --force flag is set. Ensure that your code is structured accordingly and all dependencies are listed in `requirements.txt`")
 
         group_id, group_name = get_group_info(group, deploy_key)
         connection_id, service = get_connection_id(connection, group, group_id, deploy_key) or (None, None)

{fivetran_connector_sdk-1.8.1 → fivetran_connector_sdk-2.0.1}/src/fivetran_connector_sdk/connector_helper.py

@@ -314,7 +314,7 @@ def fetch_requirements_as_dict(file_path: str) -> dict:
             print_library_log(f"Invalid requirement format: '{requirement}'", Logging.Level.SEVERE)
     return requirements_dict
 
-def validate_requirements_file(project_path: str, is_deploy: bool, version: str , force: bool = False,):
+def validate_requirements_file(project_path: str, is_deploy: bool, version: str):
     """Validates the `requirements.txt` file against the project's actual dependencies.
 
     This method generates a temporary requirements file using `pipreqs`, compares
@@ -326,7 +326,6 @@ def validate_requirements_file(project_path: str, is_deploy: bool, version: str
         project_path (str): The path to the project directory containing the `requirements.txt`.
         is_deploy (bool): If `True`, the method will exit the process on critical errors.
         version (str): The current version of the connector.
-        force (bool): Force update an existing connection.
 
     """
     # Detect and exclude virtual environment directories
@@ -385,7 +384,7 @@ def validate_requirements_file(project_path: str, is_deploy: bool, version: str
     if version_mismatch_deps:
         print_library_log("We recommend using the current stable version for the following libraries:", Logging.Level.WARNING)
         print(version_mismatch_deps)
-        if is_deploy and not force:
+        if is_deploy:
             confirm = input(
                     f"Would you like us to update {REQUIREMENTS_TXT} to the current stable versions of the dependent libraries? (y/N):")
             if confirm.lower() == "y":
@@ -400,7 +399,7 @@ def validate_requirements_file(project_path: str, is_deploy: bool, version: str
     missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
     if missing_deps:
         handle_missing_deps(missing_deps)
-        if is_deploy and not force:
+        if is_deploy:
             confirm = input(
                     f"Would you like us to update {REQUIREMENTS_TXT} to add missing dependent libraries? (y/N):")
             if confirm.lower() == "n":
@@ -414,7 +413,7 @@ def validate_requirements_file(project_path: str, is_deploy: bool, version: str
     unused_deps = list(requirements.keys() - tmp_requirements.keys())
     if unused_deps:
         handle_unused_deps(unused_deps, version)
-        if is_deploy and not force:
+        if is_deploy:
             confirm = input(f"Would you like us to update {REQUIREMENTS_TXT} to remove the unused libraries? (y/N):")
             if confirm.lower() == "n":
                 if 'fivetran_connector_sdk' in unused_deps or 'requests' in unused_deps:
@@ -432,6 +431,8 @@ def validate_requirements_file(project_path: str, is_deploy: bool, version: str
         with open(requirements_file_path, "w", encoding=UTF_8) as file:
             file.write("\n".join(requirements.values()))
             print_library_log(f"`{REQUIREMENTS_TXT}` has been updated successfully.")
+    elif not requirements:
+        delete_file_if_exists(requirements_file_path)
 
     if is_deploy: print_library_log(f"Validation of {REQUIREMENTS_TXT} completed.")
 

{fivetran_connector_sdk-1.8.1 → fivetran_connector_sdk-2.0.1}/src/fivetran_connector_sdk/constants.py

@@ -43,8 +43,7 @@ VIRTUAL_ENV_CONFIG = "pyvenv.cfg"
 ROOT_FILENAME = "connector.py"
 MAX_RECORDS_IN_BATCH = 100
 MAX_BATCH_SIZE_IN_BYTES = 100000 # 100 KB
-# The increased queue size shall be rolled out as Feature flag
-QUEUE_SIZE = int(os.environ.get("QUEUE_SIZE", 1))
+QUEUE_SIZE = 100
 
 # Compile patterns used in the implementation
 WORD_DASH_DOT_PATTERN = re.compile(r'^[\w.-]*$')

fivetran_connector_sdk-2.0.1/src/fivetran_connector_sdk/operation_stream.py

@@ -0,0 +1,166 @@
+import queue
+import threading
+
+from fivetran_connector_sdk.constants import (
+    QUEUE_SIZE,
+    MAX_RECORDS_IN_BATCH,
+    MAX_BATCH_SIZE_IN_BYTES,
+    CHECKPOINT_OP_TIMEOUT_IN_SEC
+)
+from fivetran_connector_sdk.protos import connector_sdk_pb2
+
+
+class _OperationStream:
+    """
+    A simple iterator-based stream backed by a queue for producing and consuming operations.
+
+    This class allows adding data items into a queue and consuming them using standard iteration.
+    It uses a sentinel object to signal the end of the stream.
+
+    Example:
+        stream = _OperationStream()
+        stream.add("response1")
+        stream.mark_done()
+
+        for response in stream:
+            print(response)  # prints "response1"
+    """
+
+    def __init__(self):
+        """
+        Initializes the operation stream with a queue and a sentinel object.
+        """
+        self._queue = queue.Queue(maxsize=QUEUE_SIZE)
+        self._sentinel = object()
+        self._is_done = False
+        self._buffer = []
+        self._buffer_record_count = 0
+        self._buffer_size_bytes = 0
+        self._checkpoint_lock = threading.Lock()
+        self._checkpoint_flush_signal = threading.Event()
+        self._checkpoint_flush_signal.set()
+
+    def __iter__(self):
+        """
+        Returns the iterator instance itself.
+        """
+        return self
+
+    def add(self, operation):
+        """
+        Adds an operation to the stream. Guarantees that operations within a single thread are processed in the order.
+
+        In multithreaded environment if a thread initiates a checkpoint, it's producer is blocked until the
+        checkpoint flush is complete. This block is localized, other threads
+        remain unblocked and can continue to perform other operations
+        (such as upserts, updates, deletes), but they are prevented from initiating a new checkpoint
+        until the existing one is finished.
+
+        Args:
+            operation (object): The data item to add to the stream.
+        """
+        if isinstance(operation, connector_sdk_pb2.Checkpoint):
+            # lock to ensure checkpoint operations are processed one at a time
+            with self._checkpoint_lock:
+                # clear the signal to indicate checkpoint operation is being processed.
+                self._checkpoint_flush_signal.clear()
+                self._queue.put(operation)
+                # wait until the consumer flushes the buffer and sets the flag.
+                if not self._checkpoint_flush_signal.wait(CHECKPOINT_OP_TIMEOUT_IN_SEC):
+                    raise TimeoutError(
+                        "Checkpoint flush timed out. Consumer may have failed to process checkpoint."
+                    )
+        else:
+            self._queue.put(operation)
+
+    def unblock(self):
+        """
+        Unblocks the queue, called by consumer after the checkpoint flush is completed.
+        """
+        self._checkpoint_flush_signal.set()
+
+    def mark_done(self):
+        """
+        Marks the end of the stream by putting a sentinel in the queue.
+        """
+        self._queue.put(self._sentinel)
+
+    def __next__(self):
+        """
+        Retrieves the next item from the stream. Raises StopIteration when the sentinel is encountered.
+
+        Returns:
+            object: The next item in the stream.
+
+        Raises:
+            StopIteration: If the sentinel object is encountered.
+        """
+        # If stream is completed and buffer is empty, raise StopIteration. Else flush the buffer.
+        if self._is_done and not self._buffer:
+            raise StopIteration
+
+        if self._is_done:
+            return self._flush_buffer()
+
+        return self._build_next_batch()
+
+    def _build_next_batch(self):
+        """
+        Core logic to build the batch. The loop continues until the buffer is full,
+        but can be interrupted by a checkpoint or a sentinel from the producer.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse or list[connector_sdk_pb2.UpdateResponse]: Either a single response
+            containing records or checkpoint, or a list of responses when flushing data with a checkpoint.
+
+        """
+        while self._buffer_record_count < MAX_RECORDS_IN_BATCH and self._buffer_size_bytes < MAX_BATCH_SIZE_IN_BYTES:
+            operation = self._queue.get()
+
+            # Case 1: If operation is sentinel, mark the stream as done, flush the buffer.
+            if operation is self._sentinel:
+                self._is_done = True
+                if self._buffer:
+                    return self._flush_buffer()
+                else:
+                    raise StopIteration
+
+            # Case 2: if operation is a Checkpoint, flush the buffer and send the checkpoint.
+            elif isinstance(operation, connector_sdk_pb2.Checkpoint):
+                return self._flush_buffer_on_checkpoint(operation)
+
+            # it is record, buffer it to flush in batches
+            self._buffer_record_count += 1
+            self._buffer_size_bytes += len(operation.SerializeToString())
+            self._buffer.append(operation)
+
+        # Case 3: If buffer size limit is reached, flush the buffer and return the response.
+        return self._flush_buffer()
+
+    def _flush_buffer_on_checkpoint(self, checkpoint: connector_sdk_pb2.Checkpoint):
+        """
+        Creates the responses containing the checkpoint and buffered records.
+
+        Args:
+            checkpoint (object): Checkpoint operation to be added to the response.
+        """
+        responses = []
+
+        if self._buffer:
+            responses.append(self._flush_buffer())
+
+        responses.append(connector_sdk_pb2.UpdateResponse(checkpoint=checkpoint))
+        return responses
+
+    def _flush_buffer(self):
+        """
+        Flushes the current buffer and returns a response containing the buffered records.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: A response containing the buffered records.
+        """
+        batch_to_flush = self._buffer
+        self._buffer = []
+        self._buffer_record_count = 0
+        self._buffer_size_bytes = 0
+        return connector_sdk_pb2.UpdateResponse(records=connector_sdk_pb2.Records(records=batch_to_flush))

{fivetran_connector_sdk-1.8.1 → fivetran_connector_sdk-2.0.1}/src/fivetran_connector_sdk/operations.py

@@ -1,18 +1,12 @@
 import json
 import sys
-import queue
-import threading
 
-from datetime import datetime
+from datetime import datetime, date
 from google.protobuf import timestamp_pb2
 
 from fivetran_connector_sdk.constants import (
     JAVA_LONG_MAX_VALUE,
-    TABLES,
-    MAX_RECORDS_IN_BATCH,
-    MAX_BATCH_SIZE_IN_BYTES,
-    QUEUE_SIZE,
-    CHECKPOINT_OP_TIMEOUT_IN_SEC
+    TABLES
 )
 from fivetran_connector_sdk.helpers import (
     get_renamed_table_name,
@@ -21,161 +15,7 @@ from fivetran_connector_sdk.helpers import (
 )
 from fivetran_connector_sdk.logger import Logging
 from fivetran_connector_sdk.protos import connector_sdk_pb2, common_pb2
-
-class _OperationStream:
-    """
-    A simple iterator-based stream backed by a queue for producing and consuming operations.
-
-    This class allows adding data items into a queue and consuming them using standard iteration.
-    It uses a sentinel object to signal the end of the stream.
-
-    Example:
-        stream = _OperationStream()
-        stream.add("response1")
-        stream.mark_done()
-
-        for response in stream:
-            print(response)  # prints "response1"
-    """
-
-    def __init__(self):
-        """
-        Initializes the operation stream with a queue and a sentinel object.
-        """
-        self._queue = queue.Queue(maxsize=QUEUE_SIZE)
-        self._sentinel = object()
-        self._is_done = False
-        self._buffer = []
-        self._buffer_record_count = 0
-        self._buffer_size_bytes = 0
-        self._checkpoint_lock = threading.Lock()
-        self._checkpoint_flush_signal = threading.Event()
-        self._checkpoint_flush_signal.set()
-
-    def __iter__(self):
-        """
-        Returns the iterator instance itself.
-        """
-        return self
-
-    def add(self, operation):
-        """
-        Adds an operation to the stream. Guarantees that operations within a single thread are processed in the order.
-
-        In multithreaded environment if a thread initiates a checkpoint, it's producer is blocked until the
-        checkpoint flush is complete. This block is localized, other threads
-        remain unblocked and can continue to perform other operations
-        (such as upserts, updates, deletes), but they are prevented from initiating a new checkpoint
-        until the existing one is finished.
-
-        Args:
-            operation (object): The data item to add to the stream.
-        """
-        if isinstance(operation, connector_sdk_pb2.Checkpoint):
-            # lock to ensure checkpoint operations are processed one at a time
-            with self._checkpoint_lock:
-                # clear the signal to indicate checkpoint operation is being processed.
-                self._checkpoint_flush_signal.clear()
-                self._queue.put(operation)
-                # wait until the consumer flushes the buffer and sets the flag.
-                if not self._checkpoint_flush_signal.wait(CHECKPOINT_OP_TIMEOUT_IN_SEC):
-                    raise TimeoutError(
-                        "Checkpoint flush timed out. Consumer may have failed to process checkpoint."
-                    )
-        else:
-            self._queue.put(operation)
-
-    def unblock(self):
-        """
-        Unblocks the queue, called by consumer after the checkpoint flush is completed.
-        """
-        self._checkpoint_flush_signal.set()
-
-    def mark_done(self):
-        """
-        Marks the end of the stream by putting a sentinel in the queue.
-        """
-        self._queue.put(self._sentinel)
-
-    def __next__(self):
-        """
-        Retrieves the next item from the stream. Raises StopIteration when the sentinel is encountered.
-
-        Returns:
-            object: The next item in the stream.
-
-        Raises:
-            StopIteration: If the sentinel object is encountered.
-        """
-        # If stream is completed and buffer is empty, raise StopIteration. Else flush the buffer.
-        if self._is_done and not self._buffer:
-            raise StopIteration
-
-        if self._is_done:
-            return self._flush_buffer()
-
-        return self._build_next_batch()
-
-    def _build_next_batch(self):
-        """
-        Core logic to build the batch. The loop continues until the buffer is full,
-        but can be interrupted by a checkpoint or a sentinel from the producer.
-
-        Returns:
-            connector_sdk_pb2.UpdateResponse or list[connector_sdk_pb2.UpdateResponse]: Either a single response
-            containing records or checkpoint, or a list of responses when flushing data with a checkpoint.
-
-        """
-        while self._buffer_record_count < MAX_RECORDS_IN_BATCH and self._buffer_size_bytes < MAX_BATCH_SIZE_IN_BYTES:
-            operation = self._queue.get()
-
-            # Case 1: If operation is sentinel, mark the stream as done, flush the buffer.
-            if operation is self._sentinel:
-                self._is_done = True
-                if self._buffer:
-                    return self._flush_buffer()
-                else:
-                    raise StopIteration
-
-            # Case 2: if operation is a Checkpoint, flush the buffer and send the checkpoint.
-            elif isinstance(operation, connector_sdk_pb2.Checkpoint):
-                return self._flush_buffer_on_checkpoint(operation)
-
-            # it is record, buffer it to flush in batches
-            self._buffer_record_count += 1
-            self._buffer_size_bytes += len(operation.SerializeToString())
-            self._buffer.append(operation)
-
-        # Case 3: If buffer size limit is reached, flush the buffer and return the response.
-        return self._flush_buffer()
-
-    def _flush_buffer_on_checkpoint(self, checkpoint: connector_sdk_pb2.Checkpoint):
-        """
-        Creates the responses containing the checkpoint and buffered records.
-
-        Args:
-            checkpoint (object): Checkpoint operation to be added to the response.
-        """
-        responses = []
-
-        if self._buffer:
-            responses.append(self._flush_buffer())
-
-        responses.append(connector_sdk_pb2.UpdateResponse(checkpoint=checkpoint))
-        return responses
-
-    def _flush_buffer(self):
-        """
-        Flushes the current buffer and returns a response containing the buffered records.
-
-        Returns:
-            connector_sdk_pb2.UpdateResponse: A response containing the buffered records.
-        """
-        batch_to_flush = self._buffer
-        self._buffer = []
-        self._buffer_record_count = 0
-        self._buffer_size_bytes = 0
-        return connector_sdk_pb2.UpdateResponse(records=connector_sdk_pb2.Records(records=batch_to_flush))
+from fivetran_connector_sdk.operation_stream import _OperationStream
 
 
 _LOG_DATA_TYPE_INFERENCE = {}
@@ -393,23 +233,52 @@ def map_defined_data_type(data_type, k, mapped_data, v):
         raise ValueError(f"Unsupported data type encountered: {data_type}. Please use valid data types.")
 
 def _parse_utc_datetime_str(v):
+    """
+    Accepts a timezone-aware datetime object or a datetime string in one of the following formats:
+    - '%Y-%m-%dT%H:%M:%S.%f%z' (e.g., '2025-08-12T15:00:00.123456+00:00')
+    - '%Y-%m-%dT%H:%M:%S%z'    (e.g., '2025-08-12T15:00:00+00:00')
+    - Same as above formats but with a space instead of 'T' as the separator (e.g., '2025-08-12 15:00:00+00:00')
+
+    Returns a `google.protobuf.timestamp_pb2.Timestamp` object.
+    """
     timestamp = timestamp_pb2.Timestamp()
     dt = v
     if not isinstance(v, datetime):
+        dt = dt.strip().replace(' ', 'T', 1)  # Replace first space with 'T' for ISO format
         dt = datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f%z" if '.' in dt else "%Y-%m-%dT%H:%M:%S%z")
     timestamp.FromDatetime(dt)
     return timestamp
 
 def _parse_naive_datetime_str(v):
-    if '.' not in v: v = v + ".0"
+    """
+    Accepts a datetime.datetime object or naive datetime string in one of the following formats:
+    - '%Y-%m-%dT%H:%M:%S.%f'
+    - '%Y-%m-%d %H:%M:%S.%f'
+    Returns a `google.protobuf.timestamp_pb2.Timestamp` object.
+    """
     timestamp = timestamp_pb2.Timestamp()
-    dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f")
+    dt = v
+    if not isinstance(v, datetime):
+        dt = dt.strip().replace(' ', 'T', 1)  # Replace first space with 'T' for ISO format
+        dt = datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f" if '.' in dt else "%Y-%m-%dT%H:%M:%S")
+
     timestamp.FromDatetime(dt)
     return timestamp
 
 def _parse_naive_date_str(v):
+    """
+    Accepts a `datetime.date` object or a date string ('YYYY-MM-DD'),
+    Returns a `google.protobuf.timestamp_pb2.Timestamp` object.
+    """
     timestamp = timestamp_pb2.Timestamp()
-    dt = datetime.strptime(v, "%Y-%m-%d")
+
+    if isinstance(v, str):
+        dt = datetime.strptime(v, "%Y-%m-%d")
+    elif isinstance(v, (datetime, date)):
+        dt = datetime.combine(v if isinstance(v, date) else v.date(), datetime.min.time())
+    else:
+        raise TypeError(f"Expected str, datetime.date, or datetime.datetime, got {type(v)}")
+
     timestamp.FromDatetime(dt)
     return timestamp
 

{fivetran_connector_sdk-1.8.1 → fivetran_connector_sdk-2.0.1}/src/fivetran_connector_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.8.1
+Version: 2.0.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -13,7 +13,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: grpcio==1.71.0
 Requires-Dist: grpcio-tools==1.71.0
-Requires-Dist: requests==2.32.4
+Requires-Dist: requests==2.32.5
 Requires-Dist: pipreqs==0.5.0
 Requires-Dist: prompt-toolkit==3.0.51
 Requires-Dist: unidecode==1.4.0

{fivetran_connector_sdk-1.8.1 → fivetran_connector_sdk-2.0.1}/src/fivetran_connector_sdk.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ src/fivetran_connector_sdk/connector_helper.py
 src/fivetran_connector_sdk/constants.py
 src/fivetran_connector_sdk/helpers.py
 src/fivetran_connector_sdk/logger.py
+src/fivetran_connector_sdk/operation_stream.py
 src/fivetran_connector_sdk/operations.py
 src/fivetran_connector_sdk.egg-info/PKG-INFO
 src/fivetran_connector_sdk.egg-info/SOURCES.txt

{fivetran_connector_sdk-1.8.1 → fivetran_connector_sdk-2.0.1}/src/fivetran_connector_sdk.egg-info/requires.txt

@@ -1,6 +1,6 @@
 grpcio==1.71.0
 grpcio-tools==1.71.0
-requests==2.32.4
+requests==2.32.5
 pipreqs==0.5.0
 prompt-toolkit==3.0.51
 unidecode==1.4.0