fivetran-connector-sdk 1.3.0__py3-none-any.whl → 1.3.2__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -17,6 +17,7 @@ import time
 import traceback
 import re
 import socket
+import ast
 
 from concurrent import futures
 from datetime import datetime
@@ -31,13 +32,13 @@ from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
-__version__ = "1.3.0"
+__version__ = "1.3.2"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
 LINUX_OS = "linux"
 
-TESTER_VERSION = "0.25.0403.001"
+TESTER_VERSION = "0.25.0415.001"
 TESTER_FILENAME = "run_sdk_tester.jar"
 VERSION_FILENAME = "version.txt"
 UPLOAD_FILENAME = "code.zip"
@@ -52,6 +53,7 @@ MAX_RETRIES = 3
 LOGGING_PREFIX = "Fivetran-Connector-SDK"
 LOGGING_DELIMITER = ": "
 VIRTUAL_ENV_CONFIG = "pyvenv.cfg"
+ROOT_FILENAME = "connector.py"
 
 # Compile patterns used in the implementation
 WORD_DASH_DOT_PATTERN = re.compile(r'^[\w.-]*$')
@@ -75,6 +77,8 @@ DEBUGGING = False
 EXECUTED_VIA_CLI = False
 PRODUCTION_BASE_URL = "https://api.fivetran.com"
 TABLES = {}
+RENAMED_TABLE_NAMES = {}
+RENAMED_COL_NAMES = {}
 INSTALLATION_SCRIPT_MISSING_MESSAGE = "The 'installation.sh' file is missing in the 'drivers' directory. Please ensure that 'installation.sh' is present to properly configure drivers."
 INSTALLATION_SCRIPT = "installation.sh"
 DRIVERS = "drivers"
@@ -142,15 +146,23 @@ class Logging:
             Logging.__log(Logging.Level.WARNING, message)
 
     @staticmethod
-    def severe(message: str):
+    def severe(message: str, exception: Exception = None):
         """Logs a severe-level message.
 
         Args:
             message (str): The message to log.
+            exception (Exception, optional): Exception to be logged if provided.
         """
         if Logging.LOG_LEVEL <= Logging.Level.SEVERE:
             Logging.__log(Logging.Level.SEVERE, message)
 
+            if exception:
+                exc_type, exc_value, exc_traceback = type(exception), exception, exception.__traceback__
+                tb_str = "".join(traceback.format_exception(exc_type, exc_value, exc_traceback, limit=1))
+
+                for error in tb_str.split("\n"):
+                    Logging.__log(Logging.Level.SEVERE, error)
+
 
 class Operations:
     @staticmethod
@@ -177,6 +189,8 @@ class Operations:
                 field_name = get_renamed_column_name(field)
                 columns[field_name] = common_pb2.Column(
                     name=field_name, type=common_pb2.DataType.UNSPECIFIED, primary_key=False)
+            new_table = common_pb2.Table(name=table, columns=columns.values())
+            TABLES[table] = new_table
 
         mapped_data = _map_data_to_columns(data, columns)
         record = connector_sdk_pb2.Record(
@@ -326,7 +340,6 @@ def _get_columns(table: str) -> dict:
     columns = {}
     if table in TABLES:
         for column in TABLES[table].columns:
-            column.name = get_renamed_column_name(column.name)
             columns[column.name] = column
 
     return columns
@@ -350,36 +363,10 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
         elif (key in columns) and columns[key].type != common_pb2.DataType.UNSPECIFIED:
             map_defined_data_type(columns, key, mapped_data, v)
         else:
-            map_inferred_data_type(key, mapped_data, v)
-
+            mapped_data[key] = common_pb2.ValueType(string=str(v))
     return mapped_data
 
 
-def map_inferred_data_type(k, mapped_data, v):
-    # We can infer type from the value
-    if isinstance(v, int):
-        if abs(v) > JAVA_LONG_MAX_VALUE:
-            mapped_data[k] = common_pb2.ValueType(float=v)
-        else:
-            mapped_data[k] = common_pb2.ValueType(long=v)
-    elif isinstance(v, float):
-        mapped_data[k] = common_pb2.ValueType(float=v)
-    elif isinstance(v, bool):
-        mapped_data[k] = common_pb2.ValueType(bool=v)
-    elif isinstance(v, bytes):
-        mapped_data[k] = common_pb2.ValueType(binary=v)
-    elif isinstance(v, list):
-        raise ValueError(
-            "Values for the columns cannot be of type 'list'. Please ensure that all values are of a supported type. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#supporteddatatypes")
-    elif isinstance(v, dict):
-        mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
-    elif isinstance(v, str):
-        mapped_data[k] = common_pb2.ValueType(string=v)
-    else:
-        # Convert arbitrary objects to string
-        mapped_data[k] = common_pb2.ValueType(string=str(v))
-
-
 def map_defined_data_type(columns, k, mapped_data, v):
     if columns[k].type == common_pb2.DataType.BOOLEAN:
         mapped_data[k] = common_pb2.ValueType(bool=v)
@@ -423,6 +410,36 @@ def map_defined_data_type(columns, k, mapped_data, v):
     else:
         raise ValueError(f"Unsupported data type encountered: {columns[k].type}. Please use valid data types.")
 
+def _warn_exit_usage(filename, line_no, func):
+    print_library_log(f"Avoid using {func} to exit from the Python code as this can cause the connector to become stuck. Throw a error if required " +
+                      f"at: {filename}:{line_no}. See the Technical Reference for details: https://fivetran.com/docs/connector-sdk/technical-reference#handlingexceptions",
+                      Logging.Level.WARNING)
+
+def _exit_check(project_path):
+    """Checks for the presence of 'exit()' in the calling code.
+    Args:
+        project_path: The absolute project_path to check exit in the connector.py file in the project.
+    """
+    # We expect the connector.py to catch errors or throw exceptions
+    # This is a warning shown to let the customer know that we expect either the yield call or error thrown
+    # exit() or sys.exit() in between some yields can cause the connector to be stuck without processing further upsert calls
+
+    filepath = os.path.join(project_path, ROOT_FILENAME)
+    with open(filepath, "r") as f:
+        try:
+            tree = ast.parse(f.read())
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Call):
+                    if isinstance(node.func, ast.Name) and node.func.id == "exit":
+                        _warn_exit_usage(ROOT_FILENAME, node.lineno, "exit()")
+                    elif isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name):
+                        if node.func.attr == "_exit" and node.func.value.id == "os":
+                            _warn_exit_usage(ROOT_FILENAME, node.lineno, "os._exit()")
+                        if node.func.attr == "exit" and node.func.value.id == "sys":
+                            _warn_exit_usage(ROOT_FILENAME, node.lineno, "sys.exit()")
+        except SyntaxError as e:
+            print_library_log(f"SyntaxError in {ROOT_FILENAME}: {e}", Logging.Level.SEVERE)
+
 
 def _parse_datetime_str(dt):
     return datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f%z" if '.' in dt else "%Y-%m-%dT%H:%M:%S%z")
@@ -459,7 +476,7 @@ def _check_dict(incoming: dict, string_only: bool = False) -> dict:
     Args:
         incoming (dict): The dictionary to validate.
         string_only (bool): Whether to allow only string values.
-    
+
     Returns:
         dict: The validated dictionary.
     """
@@ -661,14 +678,20 @@ def get_renamed_table_name(source_table):
     """
     Process a source table name to ensure it conforms to naming rules.
     """
-    return safe_drop_underscores(source_table)
+    if source_table not in RENAMED_TABLE_NAMES:
+        RENAMED_TABLE_NAMES[source_table] = safe_drop_underscores(source_table)
+
+    return RENAMED_TABLE_NAMES[source_table]
 
 
 def get_renamed_column_name(source_column):
     """
     Process a source column name to ensure it conforms to naming rules.
     """
-    return redshift_safe(source_column)
+    if source_column not in RENAMED_COL_NAMES:
+        RENAMED_COL_NAMES[source_column] = redshift_safe(source_column)
+
+    return RENAMED_COL_NAMES[source_column]
 
 class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
     def __init__(self, update, schema=None):
@@ -895,8 +918,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         connection_config = {
             "schema": connection,
             "secrets_list": secrets_list,
-            "sync_method": "DIRECT",
-            "custom_payloads": []
+            "sync_method": "DIRECT"
         }
 
         # args.python_version is already validated in validate_deploy_parameters - so its safe to add in connection_config
@@ -907,6 +929,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         self.validate_requirements_file(args.project_path, True, args.force)
 
         group_id, group_name = self.__get_group_info(group, deploy_key)
+        if hd_agent_id is None:
+            hd_agent_id = self.__get_hd_agent_id(group_id, deploy_key)
         connection_id, service = self.__get_connection_id(connection, group, group_id, deploy_key) or (None, None)
 
         if connection_id:
@@ -1248,6 +1272,31 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             return LINUX_OS
         raise ValueError(f"Unrecognized OS: {os_sysname}")
 
+
+    @staticmethod
+    def __get_hd_agent_id(destination_id: str, deploy_key: str) -> str:
+        """Retrieves the destination information for the specified destination ID and deployment key.
+
+        Args:
+            destination_id (str): The destination ID.
+            deploy_key (str): The deployment key.
+
+        Returns:
+            str: The hybrid_deployment_agent_id if HD enabled destination else returns None
+        """
+        destinations_url = f"{PRODUCTION_BASE_URL}/v1/destinations/{destination_id}"
+
+        headers = {"Authorization": f"Basic {deploy_key}"}
+        resp = rq.get(destinations_url, headers=headers)
+
+        if not resp.ok:
+            print_library_log(
+                f"The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.", Logging.Level.SEVERE)
+            os._exit(1)
+
+        data = resp.json().get("data", {})
+        return data.get("hybrid_deployment_agent_id")
+
     @staticmethod
     def __get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
         """Retrieves the group information for the specified group and deployment key.
@@ -1411,6 +1460,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         self.validate_requirements_file(project_path, False)
         print_library_log(f"Debugging connector at: {project_path}")
         available_port = get_available_port()
+        _exit_check(project_path)
 
         if available_port is None:
             raise RuntimeError("SEVERE: Unable to allocate a port in the range 50051-50061. "
@@ -1423,7 +1473,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
         try:
             print_library_log("Running connector tester...")
-            for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, available_port):
+            for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, available_port, json.dumps(self.state), json.dumps(self.configuration)):
                 print(log_msg, end="")
         except:
             print(traceback.format_exc())
@@ -1464,7 +1514,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 yield line
 
     @staticmethod
-    def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int):
+    def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int, state_json: str, configuration_json: str):
         """Runs the connector tester.
 
         Args:
@@ -1487,7 +1537,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                "--connector-sdk=true",
                f"--port={port}",
                f"--working-dir={working_dir}",
-               "--tester-type=source"]
+               "--tester-type=source",
+               f"--state={state_json}",
+               f"--configuration={configuration_json}"]
 
         popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
         for line in Connector.process_stream(popen.stderr):
@@ -1879,6 +1931,10 @@ def validate_and_load_configuration(args, configuration):
 def validate_and_load_state(args, state):
     if state:
         json_filepath = os.path.join(args.project_path, args.state)
+    else:
+        json_filepath = os.path.join(args.project_path, "files", "state.json")
+
+    if os.path.exists(json_filepath):
         if os.path.isfile(json_filepath):
             with open(json_filepath, 'r') as fi:
                 state = json.load(fi)

{fivetran_connector_sdk-1.3.0.dist-info → fivetran_connector_sdk-1.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.3.0
+Version: 1.3.2
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-1.3.0.dist-info → fivetran_connector_sdk-1.3.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fivetran_connector_sdk/__init__.py,sha256=ubzp6kUWV4j8A3hgC-GRqtcX4WIQV5vMk5gdVkOZw4s,80578
+fivetran_connector_sdk/__init__.py,sha256=NU9PygEtaibP-4ugblU8kieDGEJRBdLQcfSvlo9YYkY,83637
 fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
 fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -6,8 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
 fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
 fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
 fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
-fivetran_connector_sdk-1.3.0.dist-info/METADATA,sha256=ilFpUA6zFnHz4_tEpKpG7IZxORmHqdcMdW0wb0z2sME,2967
-fivetran_connector_sdk-1.3.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-fivetran_connector_sdk-1.3.0.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-1.3.0.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-1.3.0.dist-info/RECORD,,
+fivetran_connector_sdk-1.3.2.dist-info/METADATA,sha256=Z0t971CmSiseBUZcJ8RwFGdizkgGTO0CPKhNjqZGcvY,2967
+fivetran_connector_sdk-1.3.2.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+fivetran_connector_sdk-1.3.2.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-1.3.2.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-1.3.2.dist-info/RECORD,,

{fivetran_connector_sdk-1.3.0.dist-info → fivetran_connector_sdk-1.3.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any