fivetran-connector-sdk 1.5.0__py3-none-any.whl → 1.6.0__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -39,8 +39,9 @@ from fivetran_connector_sdk.connector_helper import (
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
-__version__ = "1.5.0"
+__version__ = "1.6.0"
 TESTER_VERSION = TESTER_VER
+MAX_MESSAGE_LENGTH = 32 * 1024 * 1024 # 32MB
 
 __all__ = [cls.__name__ for cls in [Logging, Operations]]
 
@@ -103,7 +104,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             if service != 'connector_sdk':
                 print_library_log(
                     f"The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.", Logging.Level.SEVERE)
-                os._exit(1)
+                sys.exit(1)
             else:
                 if args.force:
                     confirm = "y"
@@ -128,7 +129,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                         f"Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
                 else:
                     print_library_log("Update canceled. The process is now terminating.")
-                    os._exit(1)
+                    sys.exit(1)
         else:
             upload_project(args.project_path, deploy_key,
                                   group_id, group_name, connection)
@@ -151,7 +152,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                     f"Unable to create a new connection, failed with error: {response.json()['message']}", Logging.Level.SEVERE)
                 cleanup_uploaded_project(deploy_key,group_id, connection)
                 print_library_log("Please try again with the deploy command after resolving the issue!")
-                os._exit(1)
+                sys.exit(1)
 
     # Call this method to run the connector in production
     def run(self,
@@ -177,7 +178,13 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         if not constants.DEBUGGING:
             print_library_log(f"Running on fivetran_connector_sdk: {__version__}")
 
-        server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+        server = grpc.server(
+            futures.ThreadPoolExecutor(max_workers=10),
+            options=[
+                ('grpc.max_send_message_length', MAX_MESSAGE_LENGTH),
+                ('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH),
+            ]
+        )
         connector_sdk_pb2_grpc.add_ConnectorServicer_to_server(self, server)
         server.add_insecure_port("[::]:" + str(port))
         server.start()

fivetran_connector_sdk/connector_helper.py

@@ -2,6 +2,7 @@ import os
 import re
 import ast
 import json
+import sys
 import time
 import socket
 import platform
@@ -140,7 +141,7 @@ def check_dict(incoming: dict, string_only: bool = False) -> dict:
             if not isinstance(v, str):
                 print_library_log(
                     "All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.", Logging.Level.SEVERE)
-                os._exit(1)
+                sys.exit(1)
 
     return incoming
 
@@ -173,12 +174,12 @@ def validate_deploy_parameters(connection, deploy_key):
               "\t(Optional):\n"
               "\t\t--destination <DESTINATION_NAME> (Becomes required if there are multiple destinations)\n"
               "\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)", Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
     elif not is_connection_name_valid(connection):
         print_library_log(f"Connection name: {connection} is invalid!\n The connection name should start with an "
               f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
               f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
 
 
 def is_port_in_use(port: int):
@@ -346,7 +347,7 @@ def validate_requirements_file(project_path: str, is_deploy: bool, version: str
                     if 'fivetran_connector_sdk' in unused_deps or 'requests' in unused_deps:
                         print_library_log(
                             f"Please fix your {REQUIREMENTS_TXT} file by removing pre-installed dependencies [fivetran_connector_sdk, requests] to proceed with the deployment.")
-                        os._exit(1)
+                        sys.exit(1)
                     print_library_log(f"Changes identified for unused libraries have been ignored. These changes have NOT been made to {REQUIREMENTS_TXT}.")
                 elif confirm.lower() == "y":
                     update_unused_requirements = True
@@ -420,13 +421,13 @@ def upload_project(project_path: str, deploy_key: str, group_id: str, group_name
         upload_file_path, deploy_key, group_id, connection)
     delete_file_if_exists(upload_file_path)
     if not upload_result:
-        os._exit(1)
+        sys.exit(1)
 
 
 def cleanup_uploaded_project(deploy_key: str, group_id: str, connection: str):
     cleanup_result = cleanup_uploaded_code(deploy_key, group_id, connection)
     if not cleanup_result:
-        os._exit(1)
+        sys.exit(1)
 
 def update_connection(args: dict, id: str, name: str, group: str, config: dict, deploy_key: str, hd_agent_id: str):
     """Updates the connection with the given ID, name, group, configuration, and deployment key.
@@ -467,7 +468,7 @@ def update_connection(args: dict, id: str, name: str, group: str, config: dict,
         print_library_log(
             f"Unable to update Connection '{name}' in destination '{group}', failed with error: '{response.json()['message']}'.",
             Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
     return response
 
 def handle_failing_tests_message_and_exit(resp, log_message):
@@ -476,7 +477,7 @@ def handle_failing_tests_message_and_exit(resp, log_message):
     connection_id = resp.json().get('data', {}).get('id')
     print_library_log(f"Connection ID: {connection_id}")
     print_library_log("Please try again with the deploy command after resolving the issue!")
-    os._exit(1)
+    sys.exit(1)
 
 def are_setup_tests_failing(response) -> bool:
     """Checks for failed setup tests in the response and returns True if any test has failed, otherwise False."""
@@ -520,7 +521,7 @@ def get_connection_id(name: str, group: str, group_id: str, deploy_key: str) ->
     if not resp.ok:
         print_library_log(
             f"Unable to fetch connection list in destination '{group}'", Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
 
     if resp.json()['data']['items']:
         return resp.json()['data']['items'][0]['id'], resp.json()['data']['items'][0]['service']
@@ -599,11 +600,11 @@ def zip_folder(project_path: str) -> str:
         print_library_log(
             "The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.",
             Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
 
     if custom_drivers_exists and not custom_driver_installation_script_exists:
         print_library_log(INSTALLATION_SCRIPT_MISSING_MESSAGE, Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
 
     return upload_filepath
 
@@ -719,14 +720,14 @@ def get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
         print_library_log(
             f"The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.",
             Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
 
     data = resp.json().get("data", {})
     groups = data.get("items")
 
     if not groups:
         print_library_log("No destinations defined in the account", Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
 
     if not group:
         if len(groups) == 1:
@@ -735,7 +736,7 @@ def get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
             print_library_log(
                 "Destination name is required when there are multiple destinations in the account",
                 Logging.Level.SEVERE)
-            os._exit(1)
+            sys.exit(1)
     else:
         while True:
             for grp in groups:
@@ -753,7 +754,7 @@ def get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
 
     print_library_log(
         f"The specified destination '{group}' was not found in your account.", Logging.Level.SEVERE)
-    os._exit(1)
+    sys.exit(1)
 
 def java_exe_helper(location: str, os_arch_suffix: str) -> str:
     """Returns the path to the Java executable.

fivetran_connector_sdk/helpers.py

@@ -29,19 +29,23 @@ from fivetran_connector_sdk.constants import (
 RENAMED_TABLE_NAMES = {}
 RENAMED_COL_NAMES = {}
 
-def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
+def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO, dev_log: bool = False):
     """Logs a library message with the specified logging level.
 
     Args:
         level (Logging.Level): The logging level.
         message (str): The message to log.
+        dev_log (bool): Boolean value to check if it is dev log and shouldn't be visible to customer
     """
     if constants.DEBUGGING or constants.EXECUTED_VIA_CLI:
+        if dev_log:
+            return
         current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
         print(f"{Logging.get_color(level)}{current_time} {level.name} {LOGGING_PREFIX}: {message} {Logging.reset_color()}")
     else:
+        message_origin = "library_dev" if dev_log else "library"
         escaped_message = json.dumps(LOGGING_PREFIX + LOGGING_DELIMITER + message)
-        log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "library"}}'
+        log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "{message_origin}"}}'
         print(log_message)
 
 def is_special(c):

fivetran_connector_sdk/operations.py

@@ -1,6 +1,6 @@
-import os
 import json
 import inspect
+import sys
 
 from datetime import datetime
 from google.protobuf import timestamp_pb2
@@ -18,27 +18,10 @@ from fivetran_connector_sdk.helpers import (
 from fivetran_connector_sdk.logger import Logging
 from fivetran_connector_sdk.protos import connector_sdk_pb2, common_pb2
 
-_TYPE_HANDLERS = {
-    common_pb2.DataType.BOOLEAN: lambda val: common_pb2.ValueType(bool=val),
-    common_pb2.DataType.SHORT: lambda val: common_pb2.ValueType(short=val),
-    common_pb2.DataType.INT: lambda val: common_pb2.ValueType(int=val),
-    common_pb2.DataType.LONG: lambda val: common_pb2.ValueType(long=val),
-    common_pb2.DataType.DECIMAL: lambda val: common_pb2.ValueType(decimal=val),
-    common_pb2.DataType.FLOAT: lambda val: common_pb2.ValueType(float=val),
-    common_pb2.DataType.DOUBLE: lambda val: common_pb2.ValueType(double=val),
-    common_pb2.DataType.NAIVE_DATE: lambda val: common_pb2.ValueType(naive_date=
-                                                                     timestamp_pb2.Timestamp().FromDatetime(
-                                                                         datetime.strptime(val, "%Y-%m-%d")) or timestamp_pb2.Timestamp()),
-    common_pb2.DataType.NAIVE_DATETIME: lambda val: common_pb2.ValueType(naive_datetime=
-                                                                         timestamp_pb2.Timestamp().FromDatetime(
-                                                                             datetime.strptime(val + ".0" if '.' not in val else val, "%Y-%m-%dT%H:%M:%S.%f")) or timestamp_pb2.Timestamp()),
-    common_pb2.DataType.UTC_DATETIME: lambda val: common_pb2.ValueType(utc_datetime=
-                                                                       timestamp_pb2.Timestamp().FromDatetime(
-                                                                           val if isinstance(val, datetime) else _parse_datetime_str(val)) or timestamp_pb2.Timestamp()),
-    common_pb2.DataType.BINARY: lambda val: common_pb2.ValueType(binary=val),
-    common_pb2.DataType.XML: lambda val: common_pb2.ValueType(xml=val),
-    common_pb2.DataType.STRING: lambda val: common_pb2.ValueType(string=val if isinstance(val, str) else str(val)),
-    common_pb2.DataType.JSON: lambda val: common_pb2.ValueType(json=json.dumps(val))
+_LOG_DATA_TYPE_INFERENCE = {
+    "boolean": True,
+    "binary": True,
+    "json": True
 }
 
 class Operations:
@@ -197,7 +180,7 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
         if v is None:
             mapped_data[key] = common_pb2.ValueType(null=True)
         elif (key in columns) and columns[key].type != common_pb2.DataType.UNSPECIFIED:
-            map_defined_data_type(columns[key].type, key, mapped_data, v)
+            map_defined_data_type(columns, key, mapped_data, v)
         else:
             map_inferred_data_type(key, mapped_data, v)
     return mapped_data
@@ -212,13 +195,22 @@ def map_inferred_data_type(k, mapped_data, v):
     elif isinstance(v, float):
         mapped_data[k] = common_pb2.ValueType(float=v)
     elif isinstance(v, bool):
+        if _LOG_DATA_TYPE_INFERENCE["boolean"]:
+            print_library_log("Fivetran: Boolean Datatype has been inferred", Logging.Level.INFO, True)
+            _LOG_DATA_TYPE_INFERENCE["boolean"] = False
         mapped_data[k] = common_pb2.ValueType(bool=v)
     elif isinstance(v, bytes):
+        if _LOG_DATA_TYPE_INFERENCE["binary"]:
+            print_library_log("Fivetran: Binary Datatype has been inferred", Logging.Level.INFO, True)
+            _LOG_DATA_TYPE_INFERENCE["binary"] = False
         mapped_data[k] = common_pb2.ValueType(binary=v)
     elif isinstance(v, list):
         raise ValueError(
             "Values for the columns cannot be of type 'list'. Please ensure that all values are of a supported type. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#supporteddatatypes")
     elif isinstance(v, dict):
+        if _LOG_DATA_TYPE_INFERENCE["json"]:
+            print_library_log("Fivetran: JSON Datatype has been inferred", Logging.Level.INFO, True)
+            _LOG_DATA_TYPE_INFERENCE["json"] = False
         mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
     elif isinstance(v, str):
         mapped_data[k] = common_pb2.ValueType(string=v)
@@ -226,12 +218,49 @@ def map_inferred_data_type(k, mapped_data, v):
         # Convert arbitrary objects to string
         mapped_data[k] = common_pb2.ValueType(string=str(v))
 
-def map_defined_data_type(data_type, k, mapped_data, v):
-    handler = _TYPE_HANDLERS.get(data_type)
-    if handler:
-        mapped_data[k] = handler(v)
+
+def map_defined_data_type(columns, k, mapped_data, v):
+    if columns[k].type == common_pb2.DataType.BOOLEAN:
+        mapped_data[k] = common_pb2.ValueType(bool=v)
+    elif columns[k].type == common_pb2.DataType.SHORT:
+        mapped_data[k] = common_pb2.ValueType(short=v)
+    elif columns[k].type == common_pb2.DataType.INT:
+        mapped_data[k] = common_pb2.ValueType(int=v)
+    elif columns[k].type == common_pb2.DataType.LONG:
+        mapped_data[k] = common_pb2.ValueType(long=v)
+    elif columns[k].type == common_pb2.DataType.DECIMAL:
+        mapped_data[k] = common_pb2.ValueType(decimal=v)
+    elif columns[k].type == common_pb2.DataType.FLOAT:
+        mapped_data[k] = common_pb2.ValueType(float=v)
+    elif columns[k].type == common_pb2.DataType.DOUBLE:
+        mapped_data[k] = common_pb2.ValueType(double=v)
+    elif columns[k].type == common_pb2.DataType.NAIVE_DATE:
+        timestamp = timestamp_pb2.Timestamp()
+        dt = datetime.strptime(v, "%Y-%m-%d")
+        timestamp.FromDatetime(dt)
+        mapped_data[k] = common_pb2.ValueType(naive_date=timestamp)
+    elif columns[k].type == common_pb2.DataType.NAIVE_DATETIME:
+        if '.' not in v: v = v + ".0"
+        timestamp = timestamp_pb2.Timestamp()
+        dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f")
+        timestamp.FromDatetime(dt)
+        mapped_data[k] = common_pb2.ValueType(naive_datetime=timestamp)
+    elif columns[k].type == common_pb2.DataType.UTC_DATETIME:
+        timestamp = timestamp_pb2.Timestamp()
+        dt = v if isinstance(v, datetime) else _parse_datetime_str(v)
+        timestamp.FromDatetime(dt)
+        mapped_data[k] = common_pb2.ValueType(utc_datetime=timestamp)
+    elif columns[k].type == common_pb2.DataType.BINARY:
+        mapped_data[k] = common_pb2.ValueType(binary=v)
+    elif columns[k].type == common_pb2.DataType.XML:
+        mapped_data[k] = common_pb2.ValueType(xml=v)
+    elif columns[k].type == common_pb2.DataType.STRING:
+        incoming = v if isinstance(v, str) else str(v)
+        mapped_data[k] = common_pb2.ValueType(string=incoming)
+    elif columns[k].type == common_pb2.DataType.JSON:
+        mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
     else:
-        raise ValueError(f"Unsupported data type encountered: {data_type}. Please use valid data types.")
+        raise ValueError(f"Unsupported data type encountered: {columns[k].type}. Please use valid data types.")
 
 def _parse_datetime_str(dt):
     return datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f%z" if '.' in dt else "%Y-%m-%dT%H:%M:%S%z")
@@ -256,7 +285,7 @@ def _yield_check(stack):
         if 'yield' not in calling_code:
             print_library_log(
                 f"Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'", Logging.Level.SEVERE)
-            os._exit(1)
+            sys.exit(1)
     else:
         # This should never happen
         raise RuntimeError(

{fivetran_connector_sdk-1.5.0.dist-info → fivetran_connector_sdk-1.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.5.0
+Version: 1.6.0
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-1.5.0.dist-info → fivetran_connector_sdk-1.6.0.dist-info}/RECORD

@@ -1,9 +1,9 @@
-fivetran_connector_sdk/__init__.py,sha256=T0icwbaYQoj9hUiOY8zuJCOLU5i3t1Flc3woIq_IDRY,20768
-fivetran_connector_sdk/connector_helper.py,sha256=4NnHlr40dUPdta9gctN0WE8OYBKnZDEJKaurh8e45UE,38960
+fivetran_connector_sdk/__init__.py,sha256=C-VEwdIOtKmvF0km7I4cjzjxUZ7ZabQ4dAorggzY04c,21015
+fivetran_connector_sdk/connector_helper.py,sha256=aVZmy3_SuK62zEa91E1WOeCHyFek5j9zH3ct_epGdYc,38971
 fivetran_connector_sdk/constants.py,sha256=tY8-fwB7O11orifFcDIELxfvYNz9-bNUhh6f5cXXws0,2287
-fivetran_connector_sdk/helpers.py,sha256=5rPlqN4uYYLvmNLq0-awp3Xqv1LwyEUqp-TGfXsb9ts,12386
+fivetran_connector_sdk/helpers.py,sha256=OV7P79Mhju90m75ynG0AVbD_RCIvWfZimiTwrlPjmi8,12623
 fivetran_connector_sdk/logger.py,sha256=arI6eNUfm_AvE3EV_PmbSRzrpTVpZKJ8pZMtDn93TKU,3018
-fivetran_connector_sdk/operations.py,sha256=jQyjLNW3L8PfTL8J5VAw68s240euXbBVnGPz0QJ2_AY,11023
+fivetran_connector_sdk/operations.py,sha256=Ji6S9bH4ULf0QcrZX8M6RMhTc-IgJ7xXHmov5fbYPI0,11763
 fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
 fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -11,8 +11,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
 fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
 fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
 fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
-fivetran_connector_sdk-1.5.0.dist-info/METADATA,sha256=vQ1G788vqs7AdJy8JFyDWInLjoUntEZBlzR9LkipE7E,3150
-fivetran_connector_sdk-1.5.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fivetran_connector_sdk-1.5.0.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-1.5.0.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-1.5.0.dist-info/RECORD,,
+fivetran_connector_sdk-1.6.0.dist-info/METADATA,sha256=pBq3Y_LbaMhjBEqZeTxUK_DomwexOpbsyKhtd7QvnXw,3150
+fivetran_connector_sdk-1.6.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fivetran_connector_sdk-1.6.0.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-1.6.0.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-1.6.0.dist-info/RECORD,,