fivetran-connector-sdk 1.3.1__py3-none-any.whl → 1.3.3__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -17,6 +17,7 @@ import time
 import traceback
 import re
 import socket
+import ast
 
 from concurrent import futures
 from datetime import datetime
@@ -31,13 +32,13 @@ from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
-__version__ = "1.3.1"
+__version__ = "1.3.3"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
 LINUX_OS = "linux"
 
-TESTER_VERSION = "0.25.0411.001"
+TESTER_VERSION = "0.25.0423.001"
 TESTER_FILENAME = "run_sdk_tester.jar"
 VERSION_FILENAME = "version.txt"
 UPLOAD_FILENAME = "code.zip"
@@ -52,6 +53,7 @@ MAX_RETRIES = 3
 LOGGING_PREFIX = "Fivetran-Connector-SDK"
 LOGGING_DELIMITER = ": "
 VIRTUAL_ENV_CONFIG = "pyvenv.cfg"
+ROOT_FILENAME = "connector.py"
 
 # Compile patterns used in the implementation
 WORD_DASH_DOT_PATTERN = re.compile(r'^[\w.-]*$')
@@ -85,6 +87,7 @@ MAX_CONFIG_FIELDS = 100
 SUPPORTED_PYTHON_VERSIONS = ["3.12.8", "3.11.11", "3.10.16", "3.9.21"]
 DEFAULT_PYTHON_VERSION = "3.12.8"
 FIVETRAN_HD_AGENT_ID = "FIVETRAN_HD_AGENT_ID"
+UTF_8 = "utf-8"
 
 
 class Logging:
@@ -144,15 +147,23 @@ class Logging:
             Logging.__log(Logging.Level.WARNING, message)
 
     @staticmethod
-    def severe(message: str):
+    def severe(message: str, exception: Exception = None):
         """Logs a severe-level message.
 
         Args:
             message (str): The message to log.
+            exception (Exception, optional): Exception to be logged if provided.
         """
         if Logging.LOG_LEVEL <= Logging.Level.SEVERE:
             Logging.__log(Logging.Level.SEVERE, message)
 
+            if exception:
+                exc_type, exc_value, exc_traceback = type(exception), exception, exception.__traceback__
+                tb_str = "".join(traceback.format_exception(exc_type, exc_value, exc_traceback, limit=1))
+
+                for error in tb_str.split("\n"):
+                    Logging.__log(Logging.Level.SEVERE, error)
+
 
 class Operations:
     @staticmethod
@@ -288,7 +299,7 @@ def check_newer_version():
 
     if os.path.isfile(last_check_file_path):
         # Is it time to check again?
-        with open(last_check_file_path, 'r') as f_in:
+        with open(last_check_file_path, 'r', encoding=UTF_8) as f_in:
             timestamp = int(f_in.read())
             if (int(time.time()) - timestamp) < ONE_DAY_IN_SEC:
                 return
@@ -304,7 +315,7 @@ def check_newer_version():
                 print_library_log(f"[notice] A new release of 'fivetran-connector-sdk' is available: {latest_version}\n" +
                       f"[notice] To update, run: pip install --upgrade fivetran-connector-sdk\n")
 
-            with open(last_check_file_path, 'w') as f_out:
+            with open(last_check_file_path, 'w', encoding=UTF_8) as f_out:
                 f_out.write(f"{int(time.time())}")
             break
         except Exception:
@@ -354,7 +365,6 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
             map_defined_data_type(columns, key, mapped_data, v)
         else:
             map_inferred_data_type(key, mapped_data, v)
-
     return mapped_data
 
 
@@ -426,6 +436,36 @@ def map_defined_data_type(columns, k, mapped_data, v):
     else:
         raise ValueError(f"Unsupported data type encountered: {columns[k].type}. Please use valid data types.")
 
+def _warn_exit_usage(filename, line_no, func):
+    print_library_log(f"Avoid using {func} to exit from the Python code as this can cause the connector to become stuck. Throw a error if required " +
+                      f"at: {filename}:{line_no}. See the Technical Reference for details: https://fivetran.com/docs/connector-sdk/technical-reference#handlingexceptions",
+                      Logging.Level.WARNING)
+
+def _exit_check(project_path):
+    """Checks for the presence of 'exit()' in the calling code.
+    Args:
+        project_path: The absolute project_path to check exit in the connector.py file in the project.
+    """
+    # We expect the connector.py to catch errors or throw exceptions
+    # This is a warning shown to let the customer know that we expect either the yield call or error thrown
+    # exit() or sys.exit() in between some yields can cause the connector to be stuck without processing further upsert calls
+
+    filepath = os.path.join(project_path, ROOT_FILENAME)
+    with open(filepath, "r", encoding=UTF_8) as f:
+        try:
+            tree = ast.parse(f.read())
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Call):
+                    if isinstance(node.func, ast.Name) and node.func.id == "exit":
+                        _warn_exit_usage(ROOT_FILENAME, node.lineno, "exit()")
+                    elif isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name):
+                        if node.func.attr == "_exit" and node.func.value.id == "os":
+                            _warn_exit_usage(ROOT_FILENAME, node.lineno, "os._exit()")
+                        if node.func.attr == "exit" and node.func.value.id == "sys":
+                            _warn_exit_usage(ROOT_FILENAME, node.lineno, "sys.exit()")
+        except SyntaxError as e:
+            print_library_log(f"SyntaxError in {ROOT_FILENAME}: {e}", Logging.Level.SEVERE)
+
 
 def _parse_datetime_str(dt):
     return datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f%z" if '.' in dt else "%Y-%m-%dT%H:%M:%S%z")
@@ -555,7 +595,7 @@ def get_available_port():
 def update_base_url_if_required():
     config_file_path = os.path.join(_tester_root_dir(), CONFIG_FILE)
     if os.path.isfile(config_file_path):
-        with open(config_file_path, 'r') as f:
+        with open(config_file_path, 'r', encoding=UTF_8) as f:
             data = json.load(f)
             base_url = data.get('production_base_url')
             if base_url is not None:
@@ -705,7 +745,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         Returns:
             list[str]: A list of dependencies as strings.
         """
-        with open(file_path, 'r') as f:
+        with open(file_path, 'r', encoding=UTF_8) as f:
             return f.read().splitlines()
 
     @staticmethod
@@ -836,7 +876,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 if force:
                     requirements = tmp_requirements
 
-                with open(REQUIREMENTS_TXT, "w") as file:
+                with open(REQUIREMENTS_TXT, "w", encoding=UTF_8) as file:
                     file.write("\n".join(requirements.values()))
                 print_library_log(f"`{REQUIREMENTS_TXT}` has been updated successfully")
 
@@ -869,7 +909,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         if os.path.exists(REQUIREMENTS_TXT):
             requirements = self.fetch_requirements_as_dict(self, os.path.join(project_path, 'requirements.txt'))
         else:
-            with open(REQUIREMENTS_TXT, 'w'):
+            with open(REQUIREMENTS_TXT, 'w', encoding=UTF_8):
                 pass
             requirements = {}
             print_library_log("Adding `requirements.txt` file to your project folder.", Logging.Level.WARNING)
@@ -904,8 +944,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         connection_config = {
             "schema": connection,
             "secrets_list": secrets_list,
-            "sync_method": "DIRECT",
-            "custom_payloads": []
+            "sync_method": "DIRECT"
         }
 
         # args.python_version is already validated in validate_deploy_parameters - so its safe to add in connection_config
@@ -1373,7 +1412,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         version_file = os.path.join(tester_root_dir, VERSION_FILENAME)
         if os.path.isfile(version_file):
             # Check version number & update if different
-            with open(version_file, 'r') as fi:
+            with open(version_file, 'r', encoding=UTF_8) as fi:
                 current_version = fi.readline()
 
             if current_version != TESTER_VERSION:
@@ -1420,6 +1459,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         self.validate_requirements_file(project_path, False)
         print_library_log(f"Debugging connector at: {project_path}")
         available_port = get_available_port()
+        _exit_check(project_path)
 
         if available_port is None:
             raise RuntimeError("SEVERE: Unable to allocate a port in the range 50051-50061. "
@@ -1871,7 +1911,7 @@ def validate_and_load_configuration(args, configuration):
     if configuration:
         json_filepath = os.path.join(args.project_path, args.configuration)
         if os.path.isfile(json_filepath):
-            with open(json_filepath, 'r') as fi:
+            with open(json_filepath, 'r', encoding=UTF_8) as fi:
                 configuration = json.load(fi)
             if len(configuration) > MAX_CONFIG_FIELDS:
                 raise ValueError(f"Configuration field count exceeds maximum of {MAX_CONFIG_FIELDS}. Reduce the field count.")
@@ -1890,8 +1930,12 @@ def validate_and_load_configuration(args, configuration):
 def validate_and_load_state(args, state):
     if state:
         json_filepath = os.path.join(args.project_path, args.state)
+    else:
+        json_filepath = os.path.join(args.project_path, "files", "state.json")
+
+    if os.path.exists(json_filepath):
         if os.path.isfile(json_filepath):
-            with open(json_filepath, 'r') as fi:
+            with open(json_filepath, 'r', encoding=UTF_8) as fi:
                 state = json.load(fi)
         elif state.lstrip().startswith("{"):
             state = json.loads(state)

{fivetran_connector_sdk-1.3.1.dist-info → fivetran_connector_sdk-1.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.3.1
+Version: 1.3.3
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-1.3.1.dist-info → fivetran_connector_sdk-1.3.3.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fivetran_connector_sdk/__init__.py,sha256=4ovEZ7G46t5JFEAUmyGNksa0RVs9x5eKsYfzmyBDuNs,81121
+fivetran_connector_sdk/__init__.py,sha256=sltQUs9T18U5aCkagxVdfT6-BO4d81FWWyaNEBj5NJ8,83830
 fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
 fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -6,8 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
 fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
 fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
 fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
-fivetran_connector_sdk-1.3.1.dist-info/METADATA,sha256=qEnx2X-ac7iQhdJ7bx0p9yKhliRUAz_cxhRRgX3P-Jk,2967
-fivetran_connector_sdk-1.3.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-fivetran_connector_sdk-1.3.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-1.3.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-1.3.1.dist-info/RECORD,,
+fivetran_connector_sdk-1.3.3.dist-info/METADATA,sha256=6T4trCMBQS0wnx2-iMOyX01xpU8yo3R9VqRI-d5uwhQ,2967
+fivetran_connector_sdk-1.3.3.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+fivetran_connector_sdk-1.3.3.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-1.3.3.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-1.3.3.dist-info/RECORD,,

{fivetran_connector_sdk-1.3.1.dist-info → fivetran_connector_sdk-1.3.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any