fivetran-connector-sdk 0.9.16.1__tar.gz → 0.10.7.1__tar.gz

{fivetran_connector_sdk-0.9.16.1 → fivetran_connector_sdk-0.10.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fivetran_connector_sdk
-Version: 0.9.16.1
+Version: 0.10.7.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-0.9.16.1 → fivetran_connector_sdk-0.10.7.1}/src/fivetran_connector_sdk/__init__.py

@@ -1,5 +1,5 @@
 import argparse
-from typing import Optional
+from typing import Optional, Tuple
 
 import grpc
 import importlib.util
@@ -25,7 +25,7 @@ from fivetran_connector_sdk.protos import common_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
-__version__ = "0.9.16.1"
+__version__ = "0.10.07.1"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
@@ -38,6 +38,7 @@ UPLOAD_FILENAME = "code.zip"
 LAST_VERSION_CHECK_FILE = "_last_version_check"
 ROOT_LOCATION = ".ft_sdk_connector_tester"
 OUTPUT_FILES_DIR = "files"
+REQUIREMENTS_TXT = "requirements.txt"
 ONE_DAY_IN_SEC = 24 * 60 * 60
 
 EXCLUDED_DIRS = ["__pycache__", "lib", "include", OUTPUT_FILES_DIR]
@@ -121,7 +122,7 @@ class Logging:
 class Operations:
     @staticmethod
     def upsert(table: str, data: dict) -> list[connector_sdk_pb2.UpdateResponse]:
-        """Performs an upsert operation on the specified table with the given data, deleting any existing value with the same primary key.
+        """Updates records with the same primary key if already present in the destination. Inserts new records if not already present in the destination.
 
         Args:
             table (str): The name of the table.
@@ -207,18 +208,29 @@ class Operations:
 
     @staticmethod
     def checkpoint(state: dict) -> connector_sdk_pb2.UpdateResponse:
-        """Tries to upload all rows to the data warehouse and save state.
+        """Checkpoint saves the connector's state. State is a dict which stores information to continue the
+        sync from where it left off in the previous sync. For example, you may choose to have a field called
+        "cursor" with a timestamp value to indicate up to when the data has been synced. This makes it possible
+        for the next sync to fetch data incrementally from that time forward. See below for a few example fields
+        which act as parameters for use by the connector code.\n
+        {
+            "initialSync": true,\n
+            "cursor": "1970-01-01T00:00:00.00Z",\n
+            "last_resync": "1970-01-01T00:00:00.00Z",\n
+            "thread_count": 5,\n
+            "api_quota_left": 5000000
+        }
 
         Args:
-            state (dict): The state to checkpoint.
+            state (dict): The state to checkpoint/save.
 
         Returns:
             connector_sdk_pb2.UpdateResponse: The checkpoint response.
         """
         _yield_check(inspect.stack())
         return connector_sdk_pb2.UpdateResponse(
-                 operation=connector_sdk_pb2.Operation(checkpoint=connector_sdk_pb2.Checkpoint(
-                     state_json=json.dumps(state))))
+            operation=connector_sdk_pb2.Operation(checkpoint=connector_sdk_pb2.Checkpoint(
+                state_json=json.dumps(state))))
 
 
 def check_newer_version():
@@ -284,76 +296,85 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
         if v is None:
             mapped_data[k] = common_pb2.ValueType(null=True)
         elif (k in columns) and columns[k].type != common_pb2.DataType.UNSPECIFIED:
-            if columns[k].type == common_pb2.DataType.BOOLEAN:
-                mapped_data[k] = common_pb2.ValueType(bool=v)
-            elif columns[k].type == common_pb2.DataType.SHORT:
-                mapped_data[k] = common_pb2.ValueType(short=v)
-            elif columns[k].type == common_pb2.DataType.INT:
-                mapped_data[k] = common_pb2.ValueType(int=v)
-            elif columns[k].type == common_pb2.DataType.LONG:
-                mapped_data[k] = common_pb2.ValueType(long=v)
-            elif columns[k].type == common_pb2.DataType.DECIMAL:
-                mapped_data[k] = common_pb2.ValueType(decimal=v)
-            elif columns[k].type == common_pb2.DataType.FLOAT:
-                mapped_data[k] = common_pb2.ValueType(float=v)
-            elif columns[k].type == common_pb2.DataType.DOUBLE:
-                mapped_data[k] = common_pb2.ValueType(double=v)
-            elif columns[k].type == common_pb2.DataType.NAIVE_DATE:
-                timestamp = timestamp_pb2.Timestamp()
-                dt = datetime.strptime(v, "%Y-%m-%d")
-                timestamp.FromDatetime(dt)
-                mapped_data[k] = common_pb2.ValueType(naive_date=timestamp)
-            elif columns[k].type == common_pb2.DataType.NAIVE_DATETIME:
-                if '.' not in v: v = v + ".0"
-                timestamp = timestamp_pb2.Timestamp()
-                dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f")
-                timestamp.FromDatetime(dt)
-                mapped_data[k] = common_pb2.ValueType(naive_datetime=timestamp)
-            elif columns[k].type == common_pb2.DataType.UTC_DATETIME:
-                timestamp = timestamp_pb2.Timestamp()
-                if '.' in v:
-                    dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f%z")
-                else:
-                    dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S%z")
-                timestamp.FromDatetime(dt)
-                mapped_data[k] = common_pb2.ValueType(utc_datetime=timestamp)
-            elif columns[k].type == common_pb2.DataType.BINARY:
-                mapped_data[k] = common_pb2.ValueType(binary=v)
-            elif columns[k].type == common_pb2.DataType.XML:
-                mapped_data[k] = common_pb2.ValueType(xml=v)
-            elif columns[k].type == common_pb2.DataType.STRING:
-                incoming = v if isinstance(v, str) else str(v)
-                mapped_data[k] = common_pb2.ValueType(string=incoming)
-            elif columns[k].type == common_pb2.DataType.JSON:
-                mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
-            else:
-                raise ValueError(f"Unsupported data type encountered: {columns[k].type}. Please use valid data types.")
+            map_defined_data_type(columns, k, mapped_data, v)
         else:
-            # We can infer type from the value
-            if isinstance(v, int):
-                if abs(v) > 2147483647:
-                    mapped_data[k] = common_pb2.ValueType(long=v)
-                else:
-                    mapped_data[k] = common_pb2.ValueType(int=v)
-            elif isinstance(v, float):
-                mapped_data[k] = common_pb2.ValueType(float=v)
-            elif isinstance(v, bool):
-                mapped_data[k] = common_pb2.ValueType(bool=v)
-            elif isinstance(v, bytes):
-                mapped_data[k] = common_pb2.ValueType(binary=v)
-            elif isinstance(v, list):
-                raise ValueError("Values for the columns cannot be of type 'list'. Please ensure that all values are of a supported type. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#supporteddatatypes")
-            elif isinstance(v, dict):
-                mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
-            elif isinstance(v, str):
-                mapped_data[k] = common_pb2.ValueType(string=v)
-            else:
-                # Convert arbitrary objects to string
-                mapped_data[k] = common_pb2.ValueType(string=str(v))
+            map_inferred_data_type(k, mapped_data, v)
 
     return mapped_data
 
 
+def map_inferred_data_type(k, mapped_data, v):
+    # We can infer type from the value
+    if isinstance(v, int):
+        if abs(v) > 2147483647:
+            mapped_data[k] = common_pb2.ValueType(long=v)
+        else:
+            mapped_data[k] = common_pb2.ValueType(int=v)
+    elif isinstance(v, float):
+        mapped_data[k] = common_pb2.ValueType(float=v)
+    elif isinstance(v, bool):
+        mapped_data[k] = common_pb2.ValueType(bool=v)
+    elif isinstance(v, bytes):
+        mapped_data[k] = common_pb2.ValueType(binary=v)
+    elif isinstance(v, list):
+        raise ValueError(
+            "Values for the columns cannot be of type 'list'. Please ensure that all values are of a supported type. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#supporteddatatypes")
+    elif isinstance(v, dict):
+        mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
+    elif isinstance(v, str):
+        mapped_data[k] = common_pb2.ValueType(string=v)
+    else:
+        # Convert arbitrary objects to string
+        mapped_data[k] = common_pb2.ValueType(string=str(v))
+
+
+def map_defined_data_type(columns, k, mapped_data, v):
+    if columns[k].type == common_pb2.DataType.BOOLEAN:
+        mapped_data[k] = common_pb2.ValueType(bool=v)
+    elif columns[k].type == common_pb2.DataType.SHORT:
+        mapped_data[k] = common_pb2.ValueType(short=v)
+    elif columns[k].type == common_pb2.DataType.INT:
+        mapped_data[k] = common_pb2.ValueType(int=v)
+    elif columns[k].type == common_pb2.DataType.LONG:
+        mapped_data[k] = common_pb2.ValueType(long=v)
+    elif columns[k].type == common_pb2.DataType.DECIMAL:
+        mapped_data[k] = common_pb2.ValueType(decimal=v)
+    elif columns[k].type == common_pb2.DataType.FLOAT:
+        mapped_data[k] = common_pb2.ValueType(float=v)
+    elif columns[k].type == common_pb2.DataType.DOUBLE:
+        mapped_data[k] = common_pb2.ValueType(double=v)
+    elif columns[k].type == common_pb2.DataType.NAIVE_DATE:
+        timestamp = timestamp_pb2.Timestamp()
+        dt = datetime.strptime(v, "%Y-%m-%d")
+        timestamp.FromDatetime(dt)
+        mapped_data[k] = common_pb2.ValueType(naive_date=timestamp)
+    elif columns[k].type == common_pb2.DataType.NAIVE_DATETIME:
+        if '.' not in v: v = v + ".0"
+        timestamp = timestamp_pb2.Timestamp()
+        dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f")
+        timestamp.FromDatetime(dt)
+        mapped_data[k] = common_pb2.ValueType(naive_datetime=timestamp)
+    elif columns[k].type == common_pb2.DataType.UTC_DATETIME:
+        timestamp = timestamp_pb2.Timestamp()
+        if '.' in v:
+            dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f%z")
+        else:
+            dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S%z")
+        timestamp.FromDatetime(dt)
+        mapped_data[k] = common_pb2.ValueType(utc_datetime=timestamp)
+    elif columns[k].type == common_pb2.DataType.BINARY:
+        mapped_data[k] = common_pb2.ValueType(binary=v)
+    elif columns[k].type == common_pb2.DataType.XML:
+        mapped_data[k] = common_pb2.ValueType(xml=v)
+    elif columns[k].type == common_pb2.DataType.STRING:
+        incoming = v if isinstance(v, str) else str(v)
+        mapped_data[k] = common_pb2.ValueType(string=incoming)
+    elif columns[k].type == common_pb2.DataType.JSON:
+        mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
+    else:
+        raise ValueError(f"Unsupported data type encountered: {columns[k].type}. Please use valid data types.")
+
+
 def _yield_check(stack):
     """Checks for the presence of 'yield' in the calling code.
     Args:
@@ -373,11 +394,13 @@ def _yield_check(stack):
     calling_code = stack[1].code_context[0]
     if f"{called_method}(" in calling_code:
         if 'yield' not in calling_code:
-            print(f"SEVERE: Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'")
+            print(
+                f"SEVERE: Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'")
             os._exit(1)
     else:
         # This should never happen
-        raise RuntimeError(f"The '{called_method}' function is missing in the connector. Please ensure that the '{called_method}' function is properly defined in your code to proceed. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsmethods")
+        raise RuntimeError(
+            f"The '{called_method}' function is missing in the connector. Please ensure that the '{called_method}' function is properly defined in your code to proceed. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsmethods")
 
 
 def _check_dict(incoming: dict, string_only: bool = False) -> dict:
@@ -394,12 +417,14 @@ def _check_dict(incoming: dict, string_only: bool = False) -> dict:
         return {}
 
     if not isinstance(incoming, dict):
-        raise ValueError("Configuration must be provided as a JSON dictionary. Please check your input. Reference: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
+        raise ValueError(
+            "Configuration must be provided as a JSON dictionary. Please check your input. Reference: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
 
     if string_only:
         for k, v in incoming.items():
             if not isinstance(v, str):
-                print("SEVERE: All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.")
+                print(
+                    "SEVERE: All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.")
                 os._exit(1)
 
     return incoming
@@ -425,6 +450,23 @@ def log_unused_deps_error(package_name: str, version: str):
     os._exit(1)
 
 
+def validate_deploy_parameters(connection, deploy_key):
+    if not deploy_key or not connection:
+        print("SEVERE: The deploy command needs the following parameters:"
+              "\n\tRequired:\n"
+              "\t\t--api-key <BASE64-ENCODED-FIVETRAN-API-KEY-FOR-DEPLOYMENT>\n"
+              "\t\t--connection <VALID-CONNECTOR-SCHEMA_NAME>\n"
+              "\t(Optional):\n"
+              "\t\t--destination <DESTINATION_NAME> (Becomes required if there are multiple destinations)\n"
+              "\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)")
+        os._exit(1)
+    if not is_connection_name_valid(connection):
+        print(f"SEVERE: Connection name: {connection} is invalid!\n The connection name should start with an "
+              f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
+              f"letters, or digits (0-9). Uppercase characters are not allowed.")
+        os._exit(1)
+
+
 class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
     def __init__(self, update, schema=None):
         """Initializes the Connector instance.
@@ -439,22 +481,6 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         self.configuration = None
         self.state = None
 
-    @staticmethod
-    def __unpause_connection(id: str, deploy_key: str) -> bool:
-        """Unpauses the connection with the given ID and deployment key.
-
-        Args:
-            id (str): The connection ID.
-            deploy_key (str): The deployment key.
-
-        Returns:
-            bool: True if the connection was successfully unpaused, False otherwise.
-        """
-        resp = rq.patch(f"https://api.fivetran.com/v1/connectors/{id}",
-                        headers={"Authorization": f"Basic {deploy_key}"},
-                        json={"force": True})
-        return resp.ok
-
     @staticmethod
     def fetch_requirements_from_file(file_path: str) -> list[str]:
         """Reads a requirements file and returns a list of dependencies.
@@ -515,13 +541,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         os.remove(tmp_requirements_file_path)
 
         if len(tmp_requirements) > 0:
-            if os.path.exists("requirements.txt"):
-                requirements = self.fetch_requirements_as_dict(self, os.path.join(project_path, 'requirements.txt'))
-            else:
-                with open("requirements.txt", 'w'):
-                    pass
-                requirements = {}
-                print("WARNING: Adding `requirements.txt` file to your project folder.")
+            requirements = self.load_or_add_requirements_file(project_path)
 
             version_mismatch_deps = {key: tmp_requirements[key] for key in
                                      (requirements.keys() & tmp_requirements.keys())
@@ -532,35 +552,51 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
             missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
             if missing_deps:
-                log_level = "ERROR" if is_deploy else "WARNING"
-                print(log_level +
-                      ":  Please include the following dependency libraries in requirements.txt, to be used by "
-                      "Fivetran production. "
-                      "For more information, please visit: "
-                      "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
-                      "#workingwithrequirementstxtfile")
-                print(*list(missing_deps.values()))
-                if is_deploy:
-                    os._exit(1)
+                self.handle_missing_deps(is_deploy, missing_deps)
 
             unused_deps = list(requirements.keys() - tmp_requirements.keys())
             if unused_deps:
-                if 'fivetran_connector_sdk' in unused_deps:
-                    log_unused_deps_error("fivetran_connector_sdk", __version__)
-                elif 'requests' in unused_deps:
-                    log_unused_deps_error("requests", "2.32.3")
-                else:
-                    print("INFO: The following dependencies are not needed, "
-                          "they are not used or already installed. Please remove them from requirements.txt:")
-                    print(*unused_deps)
+                self.handle_unused_deps(unused_deps)
         else:
-            if os.path.exists("requirements.txt"):
+            if os.path.exists(REQUIREMENTS_TXT):
                 print("WARNING: `requirements.txt` is not required as no additional "
                       "Python libraries are required or all required libraries for "
                       "your code are pre-installed.")
 
         if is_deploy: print("Successful validation of requirements.txt")
 
+    def handle_unused_deps(self, unused_deps):
+        if 'fivetran_connector_sdk' in unused_deps:
+            log_unused_deps_error("fivetran_connector_sdk", __version__)
+        elif 'requests' in unused_deps:
+            log_unused_deps_error("requests", "2.32.3")
+        else:
+            print("INFO: The following dependencies are not needed, "
+                  "they are not used or already installed. Please remove them from requirements.txt:")
+            print(*unused_deps)
+
+    def handle_missing_deps(self, is_deploy, missing_deps):
+        log_level = "ERROR" if is_deploy else "WARNING"
+        print(log_level +
+              ":  Please include the following dependency libraries in requirements.txt, to be used by "
+              "Fivetran production. "
+              "For more information, please visit: "
+              "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
+              "#workingwithrequirementstxtfile")
+        print(*list(missing_deps.values()))
+        if is_deploy:
+            os._exit(1)
+
+    def load_or_add_requirements_file(self, project_path):
+        if os.path.exists(REQUIREMENTS_TXT):
+            requirements = self.fetch_requirements_as_dict(self, os.path.join(project_path, 'requirements.txt'))
+        else:
+            with open(REQUIREMENTS_TXT, 'w'):
+                pass
+            requirements = {}
+            print("WARNING: Adding `requirements.txt` file to your project folder.")
+        return requirements
+
     # Call this method to deploy the connector to Fivetran platform
     def deploy(self, project_path: str, deploy_key: str, group: str, connection: str, configuration: dict = None):
         """Deploys the connector to the Fivetran platform.
@@ -572,21 +608,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             connection (str): The connection name.
             configuration (dict): The configuration dictionary.
         """
-        if not deploy_key or not connection:
-            print("SEVERE: The deploy command needs the following parameters:"
-                  "\n\tRequired:\n"
-                  "\t\t--api-key <BASE64-ENCODED-FIVETRAN-API-KEY-FOR-DEPLOYMENT>\n"
-                  "\t\t--connection <VALID-CONNECTOR-SCHEMA_NAME>\n"
-                  "\t(Optional):\n"
-                  "\t\t--destination <DESTINATION_NAME> (Becomes required if there are multiple destinations)\n"
-                  "\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)")
-            os._exit(1)
-
-        if not is_connection_name_valid(connection):
-            print(f"SEVERE: Connection name: {connection} is invalid!\n The connection name should start with an "
-                  f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
-                  f"letters, or digits (0-9). Uppercase characters are not allowed.")
-            os._exit(1)
+        validate_deploy_parameters(connection, deploy_key)
 
         _check_dict(configuration, True)
 
@@ -653,22 +675,6 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         if not upload_result:
             os._exit(1)
 
-    @staticmethod
-    def __force_sync(id: str, deploy_key: str) -> bool:
-        """Forces a sync operation on the connection with the given ID and deployment key.
-
-        Args:
-            id (str): The connection ID.
-            deploy_key (str): The deployment key.
-
-        Returns:
-            bool: True if the sync was successfully started, False otherwise.
-        """
-        resp = rq.post(f"https://api.fivetran.com/v1/connectors/{id}/sync",
-                       headers={"Authorization": f"Basic {deploy_key}"},
-                       json={"force": True})
-        return resp.ok
-
     @staticmethod
     def __update_connection(id: str, name: str, group: str, config: dict, deploy_key: str):
         """Updates the connection with the given ID, name, group, configuration, and deployment key.
@@ -686,16 +692,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         resp = rq.patch(f"https://api.fivetran.com/v1/connectors/{id}",
                         headers={"Authorization": f"Basic {deploy_key}"},
                         json={
-                                "config": config,
-                                "run_setup_tests": True
+                            "config": config,
+                            "run_setup_tests": True
                         })
 
         if not resp.ok:
-            print(f"SEVERE: Unable to update Connection '{name}' in destination '{group}', failed with error: '{resp.json()['message']}'.")
+            print(
+                f"SEVERE: Unable to update Connection '{name}' in destination '{group}', failed with error: '{resp.json()['message']}'.")
             os._exit(1)
 
     @staticmethod
-    def __get_connection_id(name: str, group: str, group_id: str, deploy_key: str) -> Optional[str]:
+    def __get_connection_id(name: str, group: str, group_id: str, deploy_key: str) -> Optional[Tuple[str, str]]:
         """Retrieves the connection ID for the specified connection schema name, group, and deployment key.
 
         Args:
@@ -718,7 +725,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         if resp.json()['data']['items']:
             return resp.json()['data']['items'][0]['id'], resp.json()['data']['items'][0]['service']
 
-        return None, None
+        return Optional[None, None]
 
     @staticmethod
     def __create_connection(deploy_key: str, group_id: str, config: dict) -> rq.Response:
@@ -735,12 +742,12 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         response = rq.post(f"https://api.fivetran.com/v1/connectors",
                            headers={"Authorization": f"Basic {deploy_key}"},
                            json={
-                                 "group_id": group_id,
-                                 "service": "connector_sdk",
-                                 "config": config,
-                                 "paused": True,
-                                 "run_setup_tests": True,
-                                 "sync_frequency": "360",
+                               "group_id": group_id,
+                               "service": "connector_sdk",
+                               "config": config,
+                               "paused": True,
+                               "run_setup_tests": True,
+                               "sync_frequency": "360",
                            })
         return response
 
@@ -780,7 +787,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                     zipf.write(file_path, arcname)
 
         if not connector_file_exists:
-            print("SEVERE: The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.")
+            print(
+                "SEVERE: The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.")
             os._exit(1)
         return upload_filepath
 
@@ -942,12 +950,14 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
               configuration: dict = None,
               state: dict = None,
               log_level: Logging.Level = Logging.Level.FINE) -> bool:
-        """Tests the connector code by running it with the connector tester.
+        """Tests the connector code by running it with the connector tester.\n
+        state.json docs: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithstatejsonfile\n
+        configuration.json docs: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile
 
         Args:
             project_path (str): The path to the project.
-            configuration (dict): The configuration dictionary.
-            state (dict): The state dictionary.
+            configuration (dict): The configuration dictionary, same as configuration.json if present.
+            state (dict): The state dictionary, same as state.json if present.
             log_level (Logging.Level): The logging level.
 
         Returns:
@@ -987,11 +997,12 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                     with open(download_filepath, 'wb') as fo:
                         fo.write(r.content)
                 else:
-                    print(f"\nSEVERE: Failed to download the connector tester. Please check your access permissions or try again later ( status code: {r.status_code}), url: {download_url}")
-                    os._exit(1)
-            except:
-                print(f"\nSEVERE: Failed to download the connector tester. Error details: {traceback.format_exc()}")
-                os._exit(1)
+                    raise RuntimeError(
+                        f"\nSEVERE: Failed to download the connector tester. Please check your access permissions or "
+                        f"try again later ( status code: {r.status_code}), url: {download_url}")
+            except RuntimeError:
+                raise RuntimeError(
+                    f"SEVERE: Failed to download the connector tester. Error details: {traceback.format_exc()}")
 
             try:
                 # unzip it
@@ -1005,9 +1016,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 os.chmod(java_exe, st.st_mode | stat.S_IEXEC)
                 print("✓")
             except:
-                print(f"\nSEVERE: Failed to install the connector tester. Error details: ", traceback.format_exc())
                 shutil.rmtree(tester_root_dir)
-                os._exit(1)
+                raise RuntimeError(f"\nSEVERE: Failed to install the connector tester. Error details: ",
+                                   traceback.format_exc())
 
         project_path = os.getcwd() if project_path is None else project_path
         self.validate_requirements_file(project_path, False)
@@ -1017,18 +1028,14 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         # Uncomment this to run the tester manually
         # server.wait_for_termination()
 
-        error = False
         try:
             print(f"INFO: Running connector tester...")
             for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, 50051):
                 print(log_msg, end="")
         except:
             print(traceback.format_exc())
-            error = True
-
         finally:
             server.stop(grace=2.0)
-            return error
 
     @staticmethod
     def __java_exe(location: str, os_name: str) -> str:
@@ -1147,80 +1154,90 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         else:
             configuration = self.configuration if self.configuration else request.configuration
             response = self.schema_method(configuration)
+            self.process_tables(response)
+            return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
 
-            for entry in response:
-                if 'table' not in entry:
-                    raise ValueError("Entry missing table name: " + entry)
-
-                table_name = entry['table']
-
-                if table_name in TABLES:
-                    raise ValueError("Table already defined: " + table_name)
-
-                table = common_pb2.Table(name=table_name)
-                columns = {}
-
-                if "primary_key" in entry:
-                    for pkey_name in entry["primary_key"]:
-                        column = columns[pkey_name] if pkey_name in columns else common_pb2.Column(name=pkey_name)
-                        column.primary_key = True
-                        columns[pkey_name] = column
-
-                if "columns" in entry:
-                    for name, type in entry["columns"].items():
-                        column = columns[name] if name in columns else common_pb2.Column(name=name)
-
-                        if isinstance(type, str):
-                            if type.upper() == "BOOLEAN":
-                                column.type = common_pb2.DataType.BOOLEAN
-                            elif type.upper() == "SHORT":
-                                column.type = common_pb2.DataType.SHORT
-                            elif type.upper() == "INT":
-                                column.type = common_pb2.DataType.SHORT
-                            elif type.upper() == "LONG":
-                                column.type = common_pb2.DataType.LONG
-                            elif type.upper() == "DECIMAL":
-                                raise ValueError("DECIMAL data type missing precision and scale")
-                            elif type.upper() == "FLOAT":
-                                column.type = common_pb2.DataType.FLOAT
-                            elif type.upper() == "DOUBLE":
-                                column.type = common_pb2.DataType.DOUBLE
-                            elif type.upper() == "NAIVE_DATE":
-                                column.type = common_pb2.DataType.NAIVE_DATE
-                            elif type.upper() == "NAIVE_DATETIME":
-                                column.type = common_pb2.DataType.NAIVE_DATETIME
-                            elif type.upper() == "UTC_DATETIME":
-                                column.type = common_pb2.DataType.UTC_DATETIME
-                            elif type.upper() == "BINARY":
-                                column.type = common_pb2.DataType.BINARY
-                            elif type.upper() == "XML":
-                                column.type = common_pb2.DataType.XML
-                            elif type.upper() == "STRING":
-                                column.type = common_pb2.DataType.STRING
-                            elif type.upper() == "JSON":
-                                column.type = common_pb2.DataType.JSON
-                            else:
-                                raise ValueError("Unrecognized column type encountered:: ", str(type))
-
-                        elif isinstance(type, dict):
-                            if type['type'].upper() != "DECIMAL":
-                                raise ValueError("Expecting DECIMAL data type")
-                            column.type = common_pb2.DataType.DECIMAL
-                            column.decimal.precision = type['precision']
-                            column.decimal.scale = type['scale']
-
-                        else:
-                            raise ValueError("Unrecognized column type: ", str(type))
-
-                        if "primary_key" in entry and name in entry["primary_key"]:
-                            column.primary_key = True
-
-                        columns[name] = column
-
-                table.columns.extend(columns.values())
-                TABLES[table_name] = table
+    def process_tables(self, response):
+        for entry in response:
+            if 'table' not in entry:
+                raise ValueError("Entry missing table name: " + entry)
 
-            return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
+            table_name = entry['table']
+
+            if table_name in TABLES:
+                raise ValueError("Table already defined: " + table_name)
+
+            table = common_pb2.Table(name=table_name)
+            columns = {}
+
+            if "primary_key" in entry:
+                self.process_primary_keys(columns, entry)
+
+            if "columns" in entry:
+                self.process_columns(columns, entry)
+
+            table.columns.extend(columns.values())
+            TABLES[table_name] = table
+
+    def process_primary_keys(self, columns, entry):
+        for pkey_name in entry["primary_key"]:
+            column = columns[pkey_name] if pkey_name in columns else common_pb2.Column(name=pkey_name)
+            column.primary_key = True
+            columns[pkey_name] = column
+
+    def process_columns(self, columns, entry):
+        for name, type in entry["columns"].items():
+            column = columns[name] if name in columns else common_pb2.Column(name=name)
+
+            if isinstance(type, str):
+                self.process_data_type(column, type)
+
+            elif isinstance(type, dict):
+                if type['type'].upper() != "DECIMAL":
+                    raise ValueError("Expecting DECIMAL data type")
+                column.type = common_pb2.DataType.DECIMAL
+                column.decimal.precision = type['precision']
+                column.decimal.scale = type['scale']
+
+            else:
+                raise ValueError("Unrecognized column type: ", str(type))
+
+            if "primary_key" in entry and name in entry["primary_key"]:
+                column.primary_key = True
+
+            columns[name] = column
+
+    def process_data_type(self, column, type):
+        if type.upper() == "BOOLEAN":
+            column.type = common_pb2.DataType.BOOLEAN
+        elif type.upper() == "SHORT":
+            column.type = common_pb2.DataType.SHORT
+        elif type.upper() == "INT":
+            column.type = common_pb2.DataType.SHORT
+        elif type.upper() == "LONG":
+            column.type = common_pb2.DataType.LONG
+        elif type.upper() == "DECIMAL":
+            raise ValueError("DECIMAL data type missing precision and scale")
+        elif type.upper() == "FLOAT":
+            column.type = common_pb2.DataType.FLOAT
+        elif type.upper() == "DOUBLE":
+            column.type = common_pb2.DataType.DOUBLE
+        elif type.upper() == "NAIVE_DATE":
+            column.type = common_pb2.DataType.NAIVE_DATE
+        elif type.upper() == "NAIVE_DATETIME":
+            column.type = common_pb2.DataType.NAIVE_DATETIME
+        elif type.upper() == "UTC_DATETIME":
+            column.type = common_pb2.DataType.UTC_DATETIME
+        elif type.upper() == "BINARY":
+            column.type = common_pb2.DataType.BINARY
+        elif type.upper() == "XML":
+            column.type = common_pb2.DataType.XML
+        elif type.upper() == "STRING":
+            column.type = common_pb2.DataType.STRING
+        elif type.upper() == "JSON":
+            column.type = common_pb2.DataType.JSON
+        else:
+            raise ValueError("Unrecognized column type encountered:: ", str(type))
 
     def Update(self, request, context):
         """Overrides the Update method from ConnectorServicer.
@@ -1269,14 +1286,16 @@ def find_connector_object(project_path) -> Connector:
             if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
                 return obj_attr
 
-    print("SEVERE: The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector")
+    print(
+        "SEVERE: The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector")
     sys.exit(1)
 
 
 def suggest_correct_command(input_command: str) -> bool:
     # for typos
     # calculate the edit distance of the input command (lowercased) with each of the valid commands
-    edit_distances_of_commands = sorted([(command, edit_distance(command, input_command.lower())) for command in VALID_COMMANDS], key=lambda x: x[1])
+    edit_distances_of_commands = sorted(
+        [(command, edit_distance(command, input_command.lower())) for command in VALID_COMMANDS], key=lambda x: x[1])
 
     if edit_distances_of_commands[0][1] <= MAX_ALLOWED_EDIT_DISTANCE_FROM_VALID_COMMAND:
         # if the closest command is within the max allowed edit distance, we suggest that command
@@ -1304,15 +1323,15 @@ def edit_distance(first_string: str, second_string: str) -> int:
     first_string_length: int = len(first_string)
     second_string_length: int = len(second_string)
 
-    # Initialize the previous row of distances (for the base case of an empty first string)
-    # 'previous_row[j]' holds the edit distance between an empty prefix of 'first_string' and the first 'j' characters of 'second_string'.
+    # Initialize the previous row of distances (for the base case of an empty first string) 'previous_row[j]' holds
+    # the edit distance between an empty prefix of 'first_string' and the first 'j' characters of 'second_string'.
     # The first row is filled with values [0, 1, 2, ..., second_string_length]
     previous_row: list[int] = list(range(second_string_length + 1))
 
     # Rest of the rows
     for first_string_index in range(1, first_string_length + 1):
         # Start the current row with the distance for an empty second string
-        current_row: list[int] = [first_string_index]  # j = 0
+        current_row: list[int] = [first_string_index]
 
         # Iterate over each character in the second string
         for second_string_index in range(1, second_string_length + 1):
@@ -1321,7 +1340,8 @@ def edit_distance(first_string: str, second_string: str) -> int:
                 current_row.append(previous_row[second_string_index - 1])
             else:
                 # Minimum cost of insertion, deletion, or substitution
-                current_row.append(1 + min(current_row[-1], previous_row[second_string_index], previous_row[second_string_index - 1]))
+                current_row.append(
+                    1 + min(current_row[-1], previous_row[second_string_index], previous_row[second_string_index - 1]))
 
         # Move to the next row
         previous_row = current_row
@@ -1350,6 +1370,10 @@ def main():
 
     args = parser.parse_args()
 
+    if args.command.lower() == "version":
+        print("fivetran_connector_sdk " + __version__)
+        return
+
     connector_object = find_connector_object(args.project_path)
 
     # Process optional args
@@ -1359,16 +1383,40 @@ def main():
     configuration = args.configuration if args.configuration else None
     state = args.state if args.state else os.getenv('FIVETRAN_STATE', None)
 
+    configuration = validate_and_load_configuration(args, configuration)
+    state = validate_and_load_state(args, state)
+
+    if args.command.lower() == "deploy":
+        if args.state:
+            print("WARNING: 'state' parameter is not used for 'deploy' command")
+        connector_object.deploy(args.project_path, ft_deploy_key, ft_group, ft_connection, configuration)
+
+    elif args.command.lower() == "debug":
+        connector_object.debug(args.project_path, configuration, state)
+
+    elif args.command.lower() == "reset":
+        reset_local_file_directory(args)
+    else:
+        if not suggest_correct_command(args.command):
+            raise NotImplementedError(f"Invalid command: {args.command}, see `fivetran --help`")
+
+
+def validate_and_load_configuration(args, configuration):
     if configuration:
         json_filepath = os.path.join(args.project_path, args.configuration)
         if os.path.isfile(json_filepath):
             with open(json_filepath, 'r') as fi:
                 configuration = json.load(fi)
         else:
-            raise ValueError("Configuration must be provided as a JSON file. Please check your input. Reference: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
+            raise ValueError(
+                "Configuration must be provided as a JSON file. Please check your input. Reference: "
+                "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
     else:
         configuration = {}
+    return configuration
+
 
+def validate_and_load_state(args, state):
     if state:
         json_filepath = os.path.join(args.project_path, args.state)
         if os.path.isfile(json_filepath):
@@ -1378,35 +1426,23 @@ def main():
             state = json.loads(state)
     else:
         state = {}
+    return state
 
-    if args.command.lower() == "deploy":
-        if args.state:
-            print("WARNING: 'state' parameter is not used for 'deploy' command")
-        connector_object.deploy(args.project_path, ft_deploy_key, ft_group, ft_connection, configuration)
-
-    elif args.command.lower() == "debug":
-        connector_object.debug(args.project_path, configuration, state)
 
-    elif args.command.lower() == "reset":
-        files_path = os.path.join(args.project_path, OUTPUT_FILES_DIR)
-        confirm = input("This will delete your current state and `warehouse.db` files. Do you want to continue? (Y/N): ")
-        if confirm.lower() != "y":
-            print("INFO: Reset canceled")
-        else:
-            try:
-                if os.path.exists(files_path) and os.path.isdir(files_path):
-                    shutil.rmtree(files_path)
-                print("INFO: Reset Successful")
-            except Exception as e:
-                print("ERROR: Reset Failed")
-                raise e
-
-    elif args.command.lower() == "version":
-        print("fivetran_connector_sdk " + __version__)
-        
+def reset_local_file_directory(args):
+    files_path = os.path.join(args.project_path, OUTPUT_FILES_DIR)
+    confirm = input(
+        "This will delete your current state and `warehouse.db` files. Do you want to continue? (Y/N): ")
+    if confirm.lower() != "y":
+        print("INFO: Reset canceled")
     else:
-        if not suggest_correct_command(args.command):
-            raise NotImplementedError(f"Invalid command: {args.command}, see `fivetran --help`")
+        try:
+            if os.path.exists(files_path) and os.path.isdir(files_path):
+                shutil.rmtree(files_path)
+            print("INFO: Reset Successful")
+        except Exception as e:
+            print("ERROR: Reset Failed")
+            raise e
 
 
 if __name__ == "__main__":

{fivetran_connector_sdk-0.9.16.1 → fivetran_connector_sdk-0.10.7.1}/src/fivetran_connector_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fivetran_connector_sdk
-Version: 0.9.16.1
+Version: 0.10.7.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk