fivetran-connector-sdk 0.7.24.3__py3-none-any.whl → 0.8.19.1__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -11,6 +11,7 @@ import subprocess
 import sys
 import time
 import traceback
+import re
 
 from concurrent import futures
 from datetime import datetime
@@ -22,14 +23,14 @@ from fivetran_connector_sdk.protos import common_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
-__version__ = "0.7.24.3"
+__version__ = "0.8.19.1"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
 LINUX_OS = "linux"
 
-TESTER_VERSION = "0.24.0711.001"
-TESTER_FILENAME = "sdk_connector_tester.jar"
+TESTER_VERSION = "0.24.0807.001"
+TESTER_FILENAME = "run_sdk_tester.jar"
 VERSION_FILENAME = "version.txt"
 UPLOAD_FILENAME = "code.zip"
 LAST_VERSION_CHECK_FILE = "_last_version_check"
@@ -38,6 +39,7 @@ OUTPUT_FILES_DIR = "files"
 ONE_DAY_IN_SEC = 24 * 60 * 60
 
 EXCLUDED_DIRS = ["__pycache__", "lib", "include", OUTPUT_FILES_DIR]
+EXCLUDED_PIPREQS_DIRS = ["bin,etc,include,lib,Lib,lib64,Scripts"]
 
 DEBUGGING = False
 TABLES = {}
@@ -54,28 +56,54 @@ class Logging:
 
     @staticmethod
     def __log(level: Level, message: str):
+        """Logs a message with the specified logging level.
+
+        Args:
+            level (Logging.Level): The logging level.
+            message (str): The message to log.
+        """
         if DEBUGGING:
             print(message)
         else:
-            print(f'{{"level":"{level}", "message": "{message}", "message-origin": "connector_sdk"}}')
+            print(f'{{"level":"{level.name}", "message": "{message}", "message-origin": "connector_sdk"}}')
 
     @staticmethod
     def fine(message: str):
+        """Logs a fine-level message.
+
+        Args:
+            message (str): The message to log.
+        """
         if DEBUGGING and Logging.LOG_LEVEL == Logging.Level.FINE:
             Logging.__log(Logging.Level.FINE, message)
 
     @staticmethod
     def info(message: str):
+        """Logs an info-level message.
+
+        Args:
+            message (str): The message to log.
+        """
         if Logging.LOG_LEVEL <= Logging.Level.INFO:
             Logging.__log(Logging.Level.INFO, message)
 
     @staticmethod
     def warning(message: str):
+        """Logs a warning-level message.
+
+        Args:
+            message (str): The message to log.
+        """
         if Logging.LOG_LEVEL <= Logging.Level.WARNING:
             Logging.__log(Logging.Level.WARNING, message)
 
     @staticmethod
     def severe(message: str):
+        """Logs a severe-level message.
+
+        Args:
+            message (str): The message to log.
+        """
         if Logging.LOG_LEVEL == Logging.Level.SEVERE:
             Logging.__log(Logging.Level.SEVERE, message)
 
@@ -83,6 +111,15 @@ class Logging:
 class Operations:
     @staticmethod
     def upsert(table: str, data: dict) -> list[connector_sdk_pb2.UpdateResponse]:
+        """Performs an upsert operation on the specified table with the given data, deleting any existing value with the same primary key.
+
+        Args:
+            table (str): The name of the table.
+            data (dict): The data to upsert.
+
+        Returns:
+            list[connector_sdk_pb2.UpdateResponse]: A list of update responses.
+        """
         _yield_check(inspect.stack())
 
         responses = []
@@ -117,6 +154,15 @@ class Operations:
 
     @staticmethod
     def update(table: str, modified: dict) -> connector_sdk_pb2.UpdateResponse:
+        """Performs an update operation on the specified table with the given modified data.
+
+        Args:
+            table (str): The name of the table.
+            modified (dict): The modified data.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: The update response.
+        """
         _yield_check(inspect.stack())
 
         columns = _get_columns(table)
@@ -133,6 +179,15 @@ class Operations:
 
     @staticmethod
     def delete(table: str, keys: dict) -> connector_sdk_pb2.UpdateResponse:
+        """Performs a soft delete operation on the specified table with the given keys.
+
+        Args:
+            table (str): The name of the table.
+            keys (dict): The keys to delete.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: The delete response.
+        """
         _yield_check(inspect.stack())
 
         columns = _get_columns(table)
@@ -147,9 +202,16 @@ class Operations:
         return connector_sdk_pb2.UpdateResponse(
             operation=connector_sdk_pb2.Operation(record=record))
 
-
     @staticmethod
     def checkpoint(state: dict) -> connector_sdk_pb2.UpdateResponse:
+        """Tries to upload all rows to the data warehouse and save state.
+
+        Args:
+            state (dict): The state to checkpoint.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: The checkpoint response.
+        """
         _yield_check(inspect.stack())
         return connector_sdk_pb2.UpdateResponse(
                  operation=connector_sdk_pb2.Operation(checkpoint=connector_sdk_pb2.Checkpoint(
@@ -157,6 +219,7 @@ class Operations:
 
 
 def check_newer_version():
+    """Periodically checks for a newer version of the SDK and notifies the user if one is available."""
     tester_root_dir = _tester_root_dir()
     last_check_file_path = os.path.join(tester_root_dir, LAST_VERSION_CHECK_FILE)
     if not os.path.isdir(tester_root_dir):
@@ -174,18 +237,27 @@ def check_newer_version():
     obtainer = GetPyPiLatestVersion()
     latest_version = obtainer('fivetran_connector_sdk')
     if __version__ < latest_version:
-        print(f"[notice] A new release of 'fivetran-connector-sdk' available: {latest_version}\n" +
+        print(f"[notice] A new release of 'fivetran-connector-sdk' is available: {latest_version}\n" +
               f"[notice] To update, run: pip install --upgrade fivetran-connector-sdk\n")
 
     with open(last_check_file_path, 'w') as f_out:
         f_out.write(f"{int(time.time())}")
 
 
-def _tester_root_dir():
+def _tester_root_dir() -> str:
+    """Returns the root directory for the tester."""
     return os.path.join(os.path.expanduser("~"), ROOT_LOCATION)
 
 
 def _get_columns(table: str) -> dict:
+    """Retrieves the columns for the specified table.
+
+    Args:
+        table (str): The name of the table.
+
+    Returns:
+        dict: The columns for the table.
+    """
     columns = {}
     if table in TABLES:
         for column in TABLES[table].columns:
@@ -195,39 +267,46 @@ def _get_columns(table: str) -> dict:
 
 
 def _map_data_to_columns(data: dict, columns: dict) -> dict:
+    """Maps data to the specified columns.
+
+    Args:
+        data (dict): The data to map.
+        columns (dict): The columns to map the data to.
+
+    Returns:
+        dict: The mapped data.
+    """
     mapped_data = {}
     for k, v in data.items():
         if v is None:
             mapped_data[k] = common_pb2.ValueType(null=True)
-        elif isinstance(v, list):
-            raise ValueError("Value type cannot be list")
         elif (k in columns) and columns[k].type != common_pb2.DataType.UNSPECIFIED:
             if columns[k].type == common_pb2.DataType.BOOLEAN:
                 mapped_data[k] = common_pb2.ValueType(bool=v)
             elif columns[k].type == common_pb2.DataType.SHORT:
                 mapped_data[k] = common_pb2.ValueType(short=v)
-            elif columns[k].type ==  common_pb2.DataType.INT:
+            elif columns[k].type == common_pb2.DataType.INT:
                 mapped_data[k] = common_pb2.ValueType(int=v)
-            elif columns[k].type ==  common_pb2.DataType.LONG:
+            elif columns[k].type == common_pb2.DataType.LONG:
                 mapped_data[k] = common_pb2.ValueType(long=v)
-            elif columns[k].type ==  common_pb2.DataType.DECIMAL:
+            elif columns[k].type == common_pb2.DataType.DECIMAL:
                 mapped_data[k] = common_pb2.ValueType(decimal=v)
-            elif columns[k].type ==  common_pb2.DataType.FLOAT:
+            elif columns[k].type == common_pb2.DataType.FLOAT:
                 mapped_data[k] = common_pb2.ValueType(float=v)
-            elif columns[k].type ==  common_pb2.DataType.DOUBLE:
+            elif columns[k].type == common_pb2.DataType.DOUBLE:
                 mapped_data[k] = common_pb2.ValueType(double=v)
-            elif columns[k].type ==  common_pb2.DataType.NAIVE_DATE:
+            elif columns[k].type == common_pb2.DataType.NAIVE_DATE:
                 timestamp = timestamp_pb2.Timestamp()
                 dt = datetime.strptime(v, "%Y-%m-%d")
                 timestamp.FromDatetime(dt)
                 mapped_data[k] = common_pb2.ValueType(naive_date=timestamp)
-            elif columns[k].type ==  common_pb2.DataType.NAIVE_DATETIME:
+            elif columns[k].type == common_pb2.DataType.NAIVE_DATETIME:
                 if '.' not in v: v = v + ".0"
                 timestamp = timestamp_pb2.Timestamp()
                 dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f")
                 timestamp.FromDatetime(dt)
                 mapped_data[k] = common_pb2.ValueType(naive_datetime=timestamp)
-            elif columns[k].type ==  common_pb2.DataType.UTC_DATETIME:
+            elif columns[k].type == common_pb2.DataType.UTC_DATETIME:
                 timestamp = timestamp_pb2.Timestamp()
                 if '.' in v:
                     dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f%z")
@@ -235,17 +314,17 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
                     dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S%z")
                 timestamp.FromDatetime(dt)
                 mapped_data[k] = common_pb2.ValueType(utc_datetime=timestamp)
-            elif columns[k].type ==  common_pb2.DataType.BINARY:
+            elif columns[k].type == common_pb2.DataType.BINARY:
                 mapped_data[k] = common_pb2.ValueType(binary=v)
-            elif columns[k].type ==  common_pb2.DataType.XML:
+            elif columns[k].type == common_pb2.DataType.XML:
                 mapped_data[k] = common_pb2.ValueType(xml=v)
-            elif columns[k].type ==  common_pb2.DataType.STRING:
+            elif columns[k].type == common_pb2.DataType.STRING:
                 incoming = v if isinstance(v, str) else str(v)
                 mapped_data[k] = common_pb2.ValueType(string=incoming)
-            elif columns[k].type ==  common_pb2.DataType.JSON:
+            elif columns[k].type == common_pb2.DataType.JSON:
                 mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
             else:
-                raise ValueError(f"Unknown data type: {columns[k].type}")
+                raise ValueError(f"Unsupported data type encountered: {columns[k].type}. Please use valid data types.")
         else:
             # We can infer type from the value
             if isinstance(v, int):
@@ -260,7 +339,7 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
             elif isinstance(v, bytes):
                 mapped_data[k] = common_pb2.ValueType(binary=v)
             elif isinstance(v, list):
-                raise ValueError("Value type cannot be list")
+                raise ValueError("Values for the columns cannot be of type 'list'. Please ensure that all values are of a supported type. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#supporteddatatypes")
             elif isinstance(v, dict):
                 mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
             elif isinstance(v, str):
@@ -273,6 +352,11 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
 
 
 def _yield_check(stack):
+    """Checks for the presence of 'yield' in the calling code.
+    Args:
+        stack: The stack frame to check.
+    """
+
     # Known issue with inspect.getmodule() and yield behavior in a frozen application.
     # When using inspect.getmodule() on stack frames obtained by inspect.stack(), it fails
     # to resolve the modules in a frozen application due to incompatible assumptions about
@@ -286,24 +370,33 @@ def _yield_check(stack):
     calling_code = stack[1].code_context[0]
     if f"{called_method}(" in calling_code:
         if 'yield' not in calling_code:
-            print(f"ERROR: Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'")
+            print(f"SEVERE: Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'")
             os._exit(1)
     else:
         # This should never happen
-        raise RuntimeError(f"Unable to find '{called_method}' function in stack")
+        raise RuntimeError(f"The '{called_method}' function is missing in the connector. Please ensure that the '{called_method}' function is properly defined in your code to proceed. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsmethods")
+
 
+def _check_dict(incoming: dict, string_only: bool = False) -> dict:
+    """Validates the incoming dictionary.
+    Args:
+        incoming (dict): The dictionary to validate.
+        string_only (bool): Whether to allow only string values.
+    
+    Returns:
+        dict: The validated dictionary.
+    """
 
-def _check_dict(incoming: dict, string_only: bool = False):
     if not incoming:
         return {}
 
     if not isinstance(incoming, dict):
-        raise ValueError("Configuration should be a dictionary")
+        raise ValueError("Configuration must be provided as a JSON dictionary. Please check your input. Reference: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
 
     if string_only:
         for k, v in incoming.items():
             if not isinstance(v, str):
-                print("ERROR: Use only string values in configuration")
+                print("SEVERE: All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.")
                 os._exit(1)
 
     return incoming
@@ -311,38 +404,134 @@ def _check_dict(incoming: dict, string_only: bool = False):
 
 class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
     def __init__(self, update, schema=None):
+        """Initializes the Connector instance.
+        Args:
+            update: The update method.
+            schema: The schema method.
+        """
+
         self.schema_method = schema
         self.update_method = update
 
         self.configuration = None
         self.state = None
 
-    # Call this method to unpause and start a connector
-    def start(self, deploy_key: str, group: str, connection: str):
-        if not deploy_key: print("ERROR: Missing deploy key"); os._exit(1)
-        if not connection: print("ERROR: Missing connection name"); os._exit(1)
-
-        group_id, group_name = self.__get_group_info(group, deploy_key)
-        connection_id = self.__get_connection_id(connection, group, group_id, deploy_key)
-        if not self.__unpause_connection():
-            print(f"WARNING: Unable to unpause connection '{connection}'")
-            os._exit(1)
-
-        if not self.__force_sync(connection_id, connection, deploy_key):
-            print(f"WARNING: Unable to start sync on connection '{connection}'")
-            os._exit(1)
-
     @staticmethod
     def __unpause_connection(id: str, deploy_key: str) -> bool:
+        """Unpauses the connection with the given ID and deployment key.
+
+        Args:
+            id (str): The connection ID.
+            deploy_key (str): The deployment key.
+
+        Returns:
+            bool: True if the connection was successfully unpaused, False otherwise.
+        """
         resp = rq.patch(f"https://api.fivetran.com/v1/connectors/{id}",
                         headers={"Authorization": f"Basic {deploy_key}"},
                         json={"force": True})
         return resp.ok
 
+    @staticmethod
+    def fetch_requirements_from_file(file_path: str) -> list[str]:
+        """Reads a requirements file and returns a list of dependencies.
+
+        Args:
+            file_path (str): The path to the requirements file.
+
+        Returns:
+            list[str]: A list of dependencies as strings.
+        """
+        with open(file_path, 'r') as f:
+            return f.read().splitlines()
+
+    @staticmethod
+    def fetch_requirements_as_dict(self, file_path: str) -> dict:
+        """Converts a list of dependencies from the requirements file into a dictionary.
+
+        Args:
+            file_path (str): The path to the requirements file.
+
+        Returns:
+            dict: A dictionary where keys are package names (lowercased) and
+            values are the full dependency strings.
+        """
+        return {item.split("==")[0].lower(): item.lower() for item in
+                self.fetch_requirements_from_file(file_path)}
+
+    def validate_requirements_file(self, project_path: str, is_deploy: bool):
+        """Validates the `requirements.txt` file against the project's actual dependencies.
+
+        This method generates a temporary requirements file using `pipreqs`, compares
+        it with the existing `requirements.txt`, and checks for version mismatches,
+        missing dependencies, and unused dependencies. It will issue warnings, errors,
+        or even terminate the process depending on whether it's being run for deployment.
+
+        Args:
+            project_path (str): The path to the project directory containing the `requirements.txt`.
+            is_deploy (bool): If `True`, the method will exit the process on critical errors.
+
+        """
+        subprocess.run(["pipreqs", "--savepath", "tmp_requirements.txt", "--ignore"] + EXCLUDED_PIPREQS_DIRS, text=True, check=True)
+        tmp_requirements_file_path = os.path.join(project_path, 'tmp_requirements.txt')
+
+        tmp_requirements = self.fetch_requirements_as_dict(self, tmp_requirements_file_path)
+        tmp_requirements.pop("fivetran_connector_sdk")
+        os.remove(tmp_requirements_file_path)
+
+        if len(tmp_requirements) > 0:
+            if os.path.exists("requirements.txt"):
+                requirements = self.fetch_requirements_as_dict(self, os.path.join(project_path, 'requirements.txt'))
+            else:
+                with open("requirements.txt", 'w'):
+                    pass
+                requirements = {}
+                print("WARNING: Adding `requirements.txt` file to your project folder.")
+
+            version_mismatch_deps = {key: tmp_requirements[key] for key in
+                                     (requirements.keys() & tmp_requirements.keys())
+                                     if requirements[key] != tmp_requirements[key]}
+            if version_mismatch_deps:
+                print("WARNING: We recommend using the current stable version for the following:")
+                print(version_mismatch_deps)
+
+            missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
+            if missing_deps:
+                log_level = "ERROR" if is_deploy else "WARNING"
+                print(log_level +
+                      ":  Please include the following dependency libraries in requirements.txt, to be used by "
+                      "Fivetran production. "
+                      "For more information, please visit: "
+                      "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
+                      "#workingwithrequirementstxtfile")
+                print(*list(missing_deps.values()))
+                if is_deploy:
+                    os._exit(1)
+
+            unused_deps = list(requirements.keys() - tmp_requirements.keys())
+            if unused_deps:
+                print("INFO: The following dependencies are not used in connector.py:")
+                print(*unused_deps)
+        else:
+            if os.path.exists("requirements.txt"):
+                print("WARNING: `requirements.txt` is not required as no additional "
+                      "Python libraries are required for your code.")
+
+        if is_deploy: print("Successful validation of requirements.txt")
+
     # Call this method to deploy the connector to Fivetran platform
     def deploy(self, project_path: str, deploy_key: str, group: str, connection: str, configuration: dict = None):
-        if not deploy_key: print("ERROR: Missing deploy key"); os._exit(1)
-        if not connection: print("ERROR: Missing connection name"); os._exit(1)
+        """Deploys the connector to the Fivetran platform.
+
+        Args:
+            project_path (str): The path to the connector project.
+            deploy_key (str): The deployment key.
+            group (str): The group name.
+            connection (str): The connection name.
+            configuration (dict): The configuration dictionary.
+        """
+        if not deploy_key: print("SEVERE: The Fivetran API key is missing. Please provide a valid Fivetran API key to create the connector."); os._exit(1)
+        if not connection: print("SEVERE: The connection name is missing. Please provide a valid connection name to create the connector."); os._exit(1)
         _check_dict(configuration)
 
         secrets_list = []
@@ -356,28 +545,40 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             "sync_method": "DIRECT",
             "custom_payloads": [],
         }
+
+        self.validate_requirements_file(project_path, True)
+
         group_id, group_name = self.__get_group_info(group, deploy_key)
-        print(f"Deploying '{project_path}' to '{group_name}/{connection}'")
+        print(f"INFO: Deploying '{project_path}' to connector '{connection}' in destination '{group_name}'.\n")
         upload_file_path = self.__create_upload_file(project_path)
-        upload_result = self.__upload(upload_file_path, deploy_key,group_id,connection)
+        upload_result = self.__upload(upload_file_path, deploy_key, group_id, connection)
         os.remove(upload_file_path)
         if not upload_result:
             os._exit(1)
         connection_id = self.__get_connection_id(connection, group, group_id, deploy_key)
         if connection_id:
-            print(f"Connection '{connection}' already exists in destination '{group}', updating .. ", end="", flush=True)
+            print(f"INFO: The connection '{connection}' already exists in destination '{group}', updating the existing connector... ", end="", flush=True)
             self.__update_connection(connection_id, connection, group_name, connection_config, deploy_key)
             print("✓")
         else:
             response = self.__create_connection(deploy_key, group_id, connection_config)
             if response.ok:
-                print(f"New connection with name '{connection}' created")
+                print(f"INFO: Connection named '{connection}' has been created successfully.\n")
             else:
-                print(f"ERROR: Failed to create new connection: {response.json()['message']}")
+                print(f"SEVERE: Unable to create a new Connection, failed with error: {response.json()['message']}")
                 os._exit(1)
 
     @staticmethod
     def __force_sync(id: str, deploy_key: str) -> bool:
+        """Forces a sync operation on the connection with the given ID and deployment key.
+
+        Args:
+            id (str): The connection ID.
+            deploy_key (str): The deployment key.
+
+        Returns:
+            bool: True if the sync was successfully started, False otherwise.
+        """
         resp = rq.post(f"https://api.fivetran.com/v1/connectors/{id}/sync",
                        headers={"Authorization": f"Basic {deploy_key}"},
                        json={"force": True})
@@ -385,6 +586,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     @staticmethod
     def __update_connection(id: str, name: str, group: str, config: dict, deploy_key: str):
+        """Updates the connection with the given ID, name, group, configuration, and deployment key.
+
+        Args:
+            id (str): The connection ID.
+            name (str): The connection name.
+            group (str): The group name.
+            config (dict): The configuration dictionary.
+            deploy_key (str): The deployment key.
+        """
         if not config["secrets_list"]:
             del config["secrets_list"]
 
@@ -396,16 +606,27 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                         })
 
         if not resp.ok:
-            print(f"ERROR: Unable to update connection '{name}' in group '{group}'")
+            print(f"SEVERE: Unable to update Connection '{name}' in destination '{group}', failed with error: '{response.json()['message']}'.")
             os._exit(1)
 
     @staticmethod
-    def __get_connection_id(name: str, group: str, group_id: str, deploy_key: str):
+    def __get_connection_id(name: str, group: str, group_id: str, deploy_key: str) -> str | None:
+        """Retrieves the connection ID for the specified connection schema name, group, and deployment key.
+
+        Args:
+            name (str): The connection name.
+            group (str): The group name.
+            group_id (str): The group ID.
+            deploy_key (str): The deployment key.
+
+        Returns:
+            str: The connection ID, or None
+        """
         resp = rq.get(f"https://api.fivetran.com/v1/groups/{group_id}/connectors",
                       headers={"Authorization": f"Basic {deploy_key}"},
                       params={"schema": name})
         if not resp.ok:
-            print(f"ERROR: Unable to fetch connection list in group '{group}'")
+            print(f"SEVERE: Unable to fetch connection list in destination '{group}'")
             os._exit(1)
 
         if resp.json()['data']['items']:
@@ -414,7 +635,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         return None
 
     @staticmethod
-    def __create_connection(deploy_key: str, group_id: str, config: dict):
+    def __create_connection(deploy_key: str, group_id: str, config: dict) -> rq.Response:
+        """Creates a new connection with the given deployment key, group ID, and configuration.
+
+        Args:
+            deploy_key (str): The deployment key.
+            group_id (str): The group ID.
+            config (dict): The configuration dictionary.
+
+        Returns:
+            rq.Response: The response object.
+        """
         response = rq.post(f"https://api.fivetran.com/v1/connectors",
                            headers={"Authorization": f"Basic {deploy_key}"},
                            json={
@@ -428,24 +659,54 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         return response
 
     def __create_upload_file(self, project_path: str) -> str:
-        print("Packaging project for upload ..")
+        """Creates an upload file for the given project path.
+
+        Args:
+            project_path (str): The path to the project.
+
+        Returns:
+            str: The path to the upload file.
+        """
+        print("INFO: Packaging your project for upload...")
         zip_file_path = self.__zip_folder(project_path)
         print("✓")
         return zip_file_path
 
     def __zip_folder(self, project_path: str) -> str:
+        """Zips the folder at the given project path.
+
+        Args:
+            project_path (str): The path to the project.
+
+        Returns:
+            str: The path to the zip file.
+        """
         upload_filepath = os.path.join(project_path, UPLOAD_FILENAME)
+        connector_file_exists = False
 
         with ZipFile(upload_filepath, 'w', ZIP_DEFLATED) as zipf:
             for root, files in self.__dir_walker(project_path):
                 for file in files:
+                    if file == "connector.py":
+                        connector_file_exists = True
                     file_path = os.path.join(root, file)
                     arcname = os.path.relpath(file_path, project_path)
                     zipf.write(file_path, arcname)
 
+        if not connector_file_exists:
+            print("SEVERE: The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.")
+            os._exit(1)
         return upload_filepath
 
     def __dir_walker(self, top):
+        """Walks the directory tree starting at the given top directory.
+
+        Args:
+            top (str): The top directory to start the walk.
+
+        Yields:
+            tuple: A tuple containing the current directory path and a list of files.
+        """
         dirs, files = [], []
         for name in os.listdir(top):
             path = os.path.join(top, name)
@@ -464,7 +725,18 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     @staticmethod
     def __upload(local_path: str, deploy_key: str, group_id: str, connection: str) -> bool:
-        print("Uploading project .. ", end="", flush=True)
+        """Uploads the local code file for the specified group and connection.
+
+        Args:
+            local_path (str): The local file path.
+            deploy_key (str): The deployment key.
+            group_id (str): The group ID.
+            connection (str): The connection name.
+
+        Returns:
+            bool: True if the upload was successful, False otherwise.
+        """
+        print("INFO: Uploading your project...", end="", flush=True)
         response = rq.post(f"https://api.fivetran.com/v2/deploy/{group_id}/{connection}",
                            files={'file': open(local_path, 'rb')},
                            headers={"Authorization": f"Basic {deploy_key}"})
@@ -472,11 +744,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             print("✓")
             return True
 
-        print("fail\nERROR: ", response.reason)
+        print("SEVERE: Unable to upload the project, failed with error: ", response.reason)
         return False
 
     @staticmethod
     def __get_os_name() -> str:
+        """Returns the name of the operating system.
+
+        Returns:
+            str: The name of the operating system.
+        """
         os_sysname = platform.system().lower()
         if os_sysname.startswith("darwin"):
             return MAC_OS
@@ -488,31 +765,40 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     @staticmethod
     def __get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
+        """Retrieves the group information for the specified group and deployment key.
+
+        Args:
+            group (str): The group name.
+            deploy_key (str): The deployment key.
+
+        Returns:
+            tuple[str, str]: A tuple containing the group ID and group name.
+        """
         resp = rq.get("https://api.fivetran.com/v1/groups",
                       headers={"Authorization": f"Basic {deploy_key}"})
 
         if not resp.ok:
-            print(f"ERROR: Unable to fetch list of destination names, status code = {resp.status_code}")
+            print(f"SEVERE: Unable to fetch list of destination names, status code = {resp.status_code}")
             os._exit(1)
 
         # TODO: Do we need to implement pagination?
         groups = resp.json()['data']['items']
         if not groups:
-            print("ERROR: No destinations defined in the account")
+            print("SEVERE: No destinations defined in the account")
             os._exit(1)
 
         if len(groups) == 1:
             return groups[0]['id'], groups[0]['name']
         else:
             if not group:
-                print("ERROR: Destination name is required when there are multiple destinations in the account")
+                print("SEVERE: Destination name is required when there are multiple destinations in the account")
                 os._exit(1)
 
             for grp in groups:
                 if grp['name'] == group:
                     return grp['id'], grp['name']
 
-        print(f"ERROR: Specified destination was not found in the account: {group}")
+        print(f"SEVERE: Specified destination was not found in the account: {group}")
         os._exit(1)
 
     # Call this method to run the connector in production
@@ -521,6 +807,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             configuration: dict = None,
             state: dict = None,
             log_level: Logging.Level = Logging.Level.INFO) -> grpc.Server:
+        """Runs the connector server.
+
+        Args:
+            port (int): The port number to listen for incoming requests.
+            configuration (dict): The configuration dictionary.
+            state (dict): The state dictionary.
+            log_level (Logging.Level): The logging level.
+
+        Returns:
+            grpc.Server: The gRPC server instance.
+        """
         self.configuration = _check_dict(configuration, True)
         self.state = _check_dict(state)
         Logging.LOG_LEVEL = log_level
@@ -529,7 +826,6 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         connector_sdk_pb2_grpc.add_ConnectorServicer_to_server(self, server)
         server.add_insecure_port("[::]:" + str(port))
         server.start()
-        print("Connector started, listening on " + str(port))
         if DEBUGGING:
             return server
         server.wait_for_termination()
@@ -541,6 +837,18 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
               configuration: dict = None,
               state: dict = None,
               log_level: Logging.Level = Logging.Level.FINE) -> bool:
+        """Tests the connector code by running it with the connector tester.
+
+        Args:
+            project_path (str): The path to the project.
+            port (int): The port number to listen for incoming requests.
+            configuration (dict): The configuration dictionary.
+            state (dict): The state dictionary.
+            log_level (Logging.Level): The logging level.
+
+        Returns:
+            bool: True if there was an error, False otherwise.
+        """
         global DEBUGGING
         DEBUGGING = True
 
@@ -568,17 +876,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             download_filename = f"sdk-connector-tester-{os_name}-{TESTER_VERSION}.zip"
             download_filepath = os.path.join(tester_root_dir, download_filename)
             try:
-                print(f"Downloading connector tester version {TESTER_VERSION} .. ", end="", flush=True)
+                print(f"INFO: Downloading connector tester version: {TESTER_VERSION} ", end="", flush=True)
                 download_url = f"https://github.com/fivetran/fivetran_sdk_tools/releases/download/{TESTER_VERSION}/{download_filename}"
                 r = rq.get(download_url)
                 if r.ok:
                     with open(download_filepath, 'wb') as fo:
                         fo.write(r.content)
                 else:
-                    print(f"\nDownload failed, status code: {r.status_code}, url: {download_url}")
+                    print(f"\nSEVERE: Failed to download the connector tester. Please check your access permissions or try again later ( status code: {r.status_code}), url: {download_url}")
                     os._exit(1)
             except:
-                print(f"\nSomething went wrong during download: {traceback.format_exc()}")
+                print(f"\nSEVERE: Failed to download the connector tester. Error details: {traceback.format_exc()}")
                 os._exit(1)
 
             try:
@@ -593,21 +901,22 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 os.chmod(java_exe, st.st_mode | stat.S_IEXEC)
                 print("✓")
             except:
-                print(f"\nSomething went wrong during install: ", traceback.format_exc())
+                print(f"\nSEVERE: Failed to install the connector tester. Error details: ", traceback.format_exc())
                 shutil.rmtree(tester_root_dir)
                 os._exit(1)
 
         project_path = os.getcwd() if project_path is None else project_path
-        print(f"Debugging connector at: {project_path}")
+        self.validate_requirements_file(project_path, False)
+        print(f"INFO: Debugging connector at: {project_path}")
         server = self.run(port, configuration, state, log_level=log_level)
 
         # Uncomment this to run the tester manually
-        #server.wait_for_termination()
+        # server.wait_for_termination()
 
         error = False
         try:
-            print(f"Starting connector tester..")
-            for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path):
+            print(f"INFO: Running connector tester...")
+            for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, port):
                 print(log_msg, end="")
         except:
             print(traceback.format_exc())
@@ -618,12 +927,51 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             return error
 
     @staticmethod
-    def __java_exe(location: str, os_name: str):
+    def __java_exe(location: str, os_name: str) -> str:
+        """Returns the path to the Java executable.
+
+        Args:
+            location (str): The location of the Java executable.
+            os_name (str): The name of the operating system.
+
+        Returns:
+            str: The path to the Java executable.
+        """
         java_exe_base = os.path.join(location, "bin", "java")
         return f"{java_exe_base}.exe" if os_name == WIN_OS else java_exe_base
 
     @staticmethod
-    def __run_tester(java_exe: str, root_dir: str, project_path: str):
+    def process_stream(stream):
+        """Processes a stream of text lines, replacing occurrences of a specified pattern.
+
+        This method reads each line from the provided stream, searches for occurrences of
+        a predefined pattern, and replaces them with a specified replacement string.
+
+        Args:
+            stream (iterable): An iterable stream of text lines, typically from a file or another input source.
+
+        Yields:
+            str: Each line from the stream after replacing the matched pattern with the replacement string.
+        """
+        pattern = re.compile(r'com\.fivetran\.fivetran_sdk\.tools\.testers\.\S+')
+        replacement = 'Fivetran SDK Tester'
+
+        for line in iter(stream.readline, ""):
+            modified_line = pattern.sub(replacement, line)
+            yield modified_line
+
+    @staticmethod
+    def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int = 50051):
+        """Runs the connector tester.
+
+        Args:
+            java_exe (str): The path to the Java executable.
+            root_dir (str): The root directory.
+            project_path (str): The path to the project.
+
+        Yields:
+            str: The log messages from the tester.
+        """
         working_dir = os.path.join(project_path, OUTPUT_FILES_DIR)
         try:
             os.mkdir(working_dir)
@@ -634,11 +982,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                "-jar",
                os.path.join(root_dir, TESTER_FILENAME),
                "--connector-sdk=true",
-               f"--working-dir={working_dir}"]
+               f"--port={port}",
+               f"--working-dir={working_dir}",
+               "--tester-type=source"]
+
+        popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+        for line in Connector.process_stream(popen.stderr):
+            yield line
 
-        popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True)
-        for stdout_line in iter(popen.stdout.readline, ""):
-            yield stdout_line
+        for line in Connector.process_stream(popen.stdout):
+            yield line
         popen.stdout.close()
         return_code = popen.wait()
         if return_code:
@@ -646,6 +999,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     # -- Methods below override ConnectorServicer methods
     def ConfigurationForm(self, request, context):
+        """Overrides the ConfigurationForm method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Returns:
+            common_pb2.ConfigurationFormResponse: An empty configuration form response.
+        """
         if not self.configuration:
             self.configuration = {}
 
@@ -653,9 +1015,27 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         return common_pb2.ConfigurationFormResponse()
 
     def Test(self, request, context):
+        """Overrides the Test method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Returns:
+            None: As this method is not implemented.
+        """
         return None
 
     def Schema(self, request, context):
+        """Overrides the Schema method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Returns:
+            connector_sdk_pb2.SchemaResponse: The schema response.
+        """
         global TABLES
 
         if not self.schema_method:
@@ -718,7 +1098,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                             elif type.upper() == "JSON":
                                 column.type = common_pb2.DataType.JSON
                             else:
-                                raise ValueError("Unrecognized column type: ", str(type))
+                                raise ValueError("Unrecognized column type encountered:: ", str(type))
 
                         elif isinstance(type, dict):
                             if type['type'].upper() != "DECIMAL":
@@ -741,6 +1121,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
 
     def Update(self, request, context):
+        """Overrides the Update method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Yields:
+            connector_sdk_pb2.UpdateResponse: The update response.
+        """
         configuration = self.configuration if self.configuration else request.configuration
         state = self.state if self.state else json.loads(request.state_json)
 
@@ -757,7 +1146,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 raise e
 
 
-def find_connector_object(project_path):
+def find_connector_object(project_path) -> Connector:
+    """Finds the connector object in the given project path.
+    Args:
+        project_path (str): The path to the project.
+    
+    Returns:
+        object: The connector object.
+    """
+
     module_name = "connector_connector_code"
     connector_py = os.path.join(project_path, "connector.py")
     spec = importlib.util.spec_from_file_location(module_name, connector_py)
@@ -770,15 +1167,19 @@ def find_connector_object(project_path):
             if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
                 return obj_attr
 
-    print("Unable to find connector object")
+    print("SEVERE: The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector")
     sys.exit(1)
 
 
 def main():
+    """The main entry point for the script.
+    Parses command line arguments and passes them to connector object methods
+    """
+
     parser = argparse.ArgumentParser(allow_abbrev=False)
 
     # Positional
-    parser.add_argument("command", help="debug|run|deploy|start")
+    parser.add_argument("command", help="debug|deploy")
     parser.add_argument("project_path", nargs='?', default=os.getcwd(), help="Path to connector project directory")
 
     # Optional (Not all of these are valid with every mutually exclusive option below)
@@ -806,7 +1207,7 @@ def main():
             with open(json_filepath, 'r') as fi:
                 configuration = json.load(fi)
         else:
-            raise ValueError("Configuration needs to be a JSON file")
+            raise ValueError("Configuration must be provided as a JSON file. Please check your input. Reference: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
     else:
         configuration = {}
 
@@ -827,25 +1228,10 @@ def main():
             print("WARNING: 'state' parameter is not used for 'deploy' command")
         connector_object.deploy(args.project_path, ft_deploy_key, ft_group, ft_connection, configuration)
 
-    elif args.command.lower() == "start":
-        if args.port:
-            print("WARNING: 'port' parameter is not used for 'deploy' command")
-        if args.state:
-            print("WARNING: 'state' parameter is not used for 'deploy' command")
-        connector_object.start(ft_deploy_key, ft_group, ft_connection)
-
     elif args.command.lower() == "debug":
         port = 50051 if not args.port else args.port
         connector_object.debug(args.project_path, port, configuration, state)
 
-    elif args.command.lower() == "run":
-        try:
-            port = 50051 if not args.port else args.port
-            connector_object.run(port, configuration, state)
-        except:
-            Logging.severe(traceback.format_exc())
-            os._exit(1)
-
     else:
         raise NotImplementedError("Invalid command: ", args.command)
 

{fivetran_connector_sdk-0.7.24.3.dist-info → fivetran_connector_sdk-0.8.19.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fivetran_connector_sdk
-Version: 0.7.24.3
+Version: 0.8.19.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -10,9 +10,9 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-License-File: LICENSE
 Requires-Dist: grpcio ==1.60.1
 Requires-Dist: grpcio-tools ==1.60.1
 Requires-Dist: requests ==2.31.0
 Requires-Dist: get-pypi-latest-version ==0.0.12
+Requires-Dist: pipreqs ==0.5.0
 

{fivetran_connector_sdk-0.7.24.3.dist-info → fivetran_connector_sdk-0.8.19.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fivetran_connector_sdk/__init__.py,sha256=4KHeHNuGIVs9EXVy5Kl-cqK6Me9FcaZrtuhM1zVOS-E,34638
+fivetran_connector_sdk/__init__.py,sha256=DRdHyxqKeBGsvwYcqe2aCmM3AlVCiqVjbLQvRkfFVs0,49915
 fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
 fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -6,9 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
 fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
 fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
 fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
-fivetran_connector_sdk-0.7.24.3.dist-info/LICENSE,sha256=Kutp3D0T7HmHuBifKmbw39OZLAL1ckaLRb8u9lyJxE8,1065
-fivetran_connector_sdk-0.7.24.3.dist-info/METADATA,sha256=ayOOpB6dr7_xlqYHXomietSYhbogxyraZ9mgQZNP59c,699
-fivetran_connector_sdk-0.7.24.3.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-fivetran_connector_sdk-0.7.24.3.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-0.7.24.3.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-0.7.24.3.dist-info/RECORD,,
+fivetran_connector_sdk-0.8.19.1.dist-info/METADATA,sha256=fgkWljNC3yuszXoF9mo6HGx2mw8KvKLAhf6G5uMuxHs,708
+fivetran_connector_sdk-0.8.19.1.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
+fivetran_connector_sdk-0.8.19.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-0.8.19.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-0.8.19.1.dist-info/RECORD,,

{fivetran_connector_sdk-0.7.24.3.dist-info → fivetran_connector_sdk-0.8.19.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (71.1.0)
+Generator: setuptools (72.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fivetran_connector_sdk-0.7.24.3.dist-info/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2023 Fivetran
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.