fivetran-connector-sdk 0.7.24.2__py3-none-any.whl → 0.8.12.1__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -11,6 +11,7 @@ import subprocess
 import sys
 import time
 import traceback
+import re
 
 from concurrent import futures
 from datetime import datetime
@@ -22,14 +23,14 @@ from fivetran_connector_sdk.protos import common_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
-__version__ = "0.7.24.2"
+__version__ = "0.8.12.1"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
 LINUX_OS = "linux"
 
-TESTER_VERSION = "0.24.0711.001"
-TESTER_FILENAME = "sdk_connector_tester.jar"
+TESTER_VERSION = "0.24.0807.001"
+TESTER_FILENAME = "run_sdk_tester.jar"
 VERSION_FILENAME = "version.txt"
 UPLOAD_FILENAME = "code.zip"
 LAST_VERSION_CHECK_FILE = "_last_version_check"
@@ -54,28 +55,54 @@ class Logging:
 
     @staticmethod
     def __log(level: Level, message: str):
+        """Logs a message with the specified logging level.
+
+        Args:
+            level (Logging.Level): The logging level.
+            message (str): The message to log.
+        """
         if DEBUGGING:
             print(message)
         else:
-            print(f'{{"level":"{level}", "message": "{message}", "message-origin": "connector_sdk"}}')
+            print(f'{{"level":"{level.name}", "message": "{message}", "message-origin": "connector_sdk"}}')
 
     @staticmethod
     def fine(message: str):
+        """Logs a fine-level message.
+
+        Args:
+            message (str): The message to log.
+        """
         if DEBUGGING and Logging.LOG_LEVEL == Logging.Level.FINE:
             Logging.__log(Logging.Level.FINE, message)
 
     @staticmethod
     def info(message: str):
+        """Logs an info-level message.
+
+        Args:
+            message (str): The message to log.
+        """
         if Logging.LOG_LEVEL <= Logging.Level.INFO:
             Logging.__log(Logging.Level.INFO, message)
 
     @staticmethod
     def warning(message: str):
+        """Logs a warning-level message.
+
+        Args:
+            message (str): The message to log.
+        """
         if Logging.LOG_LEVEL <= Logging.Level.WARNING:
             Logging.__log(Logging.Level.WARNING, message)
 
     @staticmethod
     def severe(message: str):
+        """Logs a severe-level message.
+
+        Args:
+            message (str): The message to log.
+        """
         if Logging.LOG_LEVEL == Logging.Level.SEVERE:
             Logging.__log(Logging.Level.SEVERE, message)
 
@@ -83,6 +110,15 @@ class Logging:
 class Operations:
     @staticmethod
     def upsert(table: str, data: dict) -> list[connector_sdk_pb2.UpdateResponse]:
+        """Performs an upsert operation on the specified table with the given data, deleting any existing value with the same primary key.
+
+        Args:
+            table (str): The name of the table.
+            data (dict): The data to upsert.
+
+        Returns:
+            list[connector_sdk_pb2.UpdateResponse]: A list of update responses.
+        """
         _yield_check(inspect.stack())
 
         responses = []
@@ -117,6 +153,15 @@ class Operations:
 
     @staticmethod
     def update(table: str, modified: dict) -> connector_sdk_pb2.UpdateResponse:
+        """Performs an update operation on the specified table with the given modified data.
+
+        Args:
+            table (str): The name of the table.
+            modified (dict): The modified data.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: The update response.
+        """
         _yield_check(inspect.stack())
 
         columns = _get_columns(table)
@@ -133,6 +178,15 @@ class Operations:
 
     @staticmethod
     def delete(table: str, keys: dict) -> connector_sdk_pb2.UpdateResponse:
+        """Performs a soft delete operation on the specified table with the given keys.
+
+        Args:
+            table (str): The name of the table.
+            keys (dict): The keys to delete.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: The delete response.
+        """
         _yield_check(inspect.stack())
 
         columns = _get_columns(table)
@@ -147,9 +201,16 @@ class Operations:
         return connector_sdk_pb2.UpdateResponse(
             operation=connector_sdk_pb2.Operation(record=record))
 
-
     @staticmethod
     def checkpoint(state: dict) -> connector_sdk_pb2.UpdateResponse:
+        """Tries to upload all rows to the data warehouse and save state.
+
+        Args:
+            state (dict): The state to checkpoint.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: The checkpoint response.
+        """
         _yield_check(inspect.stack())
         return connector_sdk_pb2.UpdateResponse(
                  operation=connector_sdk_pb2.Operation(checkpoint=connector_sdk_pb2.Checkpoint(
@@ -157,6 +218,7 @@ class Operations:
 
 
 def check_newer_version():
+    """Periodically checks for a newer version of the SDK and notifies the user if one is available."""
     tester_root_dir = _tester_root_dir()
     last_check_file_path = os.path.join(tester_root_dir, LAST_VERSION_CHECK_FILE)
     if not os.path.isdir(tester_root_dir):
@@ -181,11 +243,20 @@ def check_newer_version():
         f_out.write(f"{int(time.time())}")
 
 
-def _tester_root_dir():
+def _tester_root_dir() -> str:
+    """Returns the root directory for the tester."""
     return os.path.join(os.path.expanduser("~"), ROOT_LOCATION)
 
 
 def _get_columns(table: str) -> dict:
+    """Retrieves the columns for the specified table.
+
+    Args:
+        table (str): The name of the table.
+
+    Returns:
+        dict: The columns for the table.
+    """
     columns = {}
     if table in TABLES:
         for column in TABLES[table].columns:
@@ -195,6 +266,15 @@ def _get_columns(table: str) -> dict:
 
 
 def _map_data_to_columns(data: dict, columns: dict) -> dict:
+    """Maps data to the specified columns.
+
+    Args:
+        data (dict): The data to map.
+        columns (dict): The columns to map the data to.
+
+    Returns:
+        dict: The mapped data.
+    """
     mapped_data = {}
     for k, v in data.items():
         if v is None:
@@ -206,28 +286,28 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
                 mapped_data[k] = common_pb2.ValueType(bool=v)
             elif columns[k].type == common_pb2.DataType.SHORT:
                 mapped_data[k] = common_pb2.ValueType(short=v)
-            elif columns[k].type ==  common_pb2.DataType.INT:
+            elif columns[k].type == common_pb2.DataType.INT:
                 mapped_data[k] = common_pb2.ValueType(int=v)
-            elif columns[k].type ==  common_pb2.DataType.LONG:
+            elif columns[k].type == common_pb2.DataType.LONG:
                 mapped_data[k] = common_pb2.ValueType(long=v)
-            elif columns[k].type ==  common_pb2.DataType.DECIMAL:
+            elif columns[k].type == common_pb2.DataType.DECIMAL:
                 mapped_data[k] = common_pb2.ValueType(decimal=v)
-            elif columns[k].type ==  common_pb2.DataType.FLOAT:
+            elif columns[k].type == common_pb2.DataType.FLOAT:
                 mapped_data[k] = common_pb2.ValueType(float=v)
-            elif columns[k].type ==  common_pb2.DataType.DOUBLE:
+            elif columns[k].type == common_pb2.DataType.DOUBLE:
                 mapped_data[k] = common_pb2.ValueType(double=v)
-            elif columns[k].type ==  common_pb2.DataType.NAIVE_DATE:
+            elif columns[k].type == common_pb2.DataType.NAIVE_DATE:
                 timestamp = timestamp_pb2.Timestamp()
                 dt = datetime.strptime(v, "%Y-%m-%d")
                 timestamp.FromDatetime(dt)
                 mapped_data[k] = common_pb2.ValueType(naive_date=timestamp)
-            elif columns[k].type ==  common_pb2.DataType.NAIVE_DATETIME:
+            elif columns[k].type == common_pb2.DataType.NAIVE_DATETIME:
                 if '.' not in v: v = v + ".0"
                 timestamp = timestamp_pb2.Timestamp()
                 dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f")
                 timestamp.FromDatetime(dt)
                 mapped_data[k] = common_pb2.ValueType(naive_datetime=timestamp)
-            elif columns[k].type ==  common_pb2.DataType.UTC_DATETIME:
+            elif columns[k].type == common_pb2.DataType.UTC_DATETIME:
                 timestamp = timestamp_pb2.Timestamp()
                 if '.' in v:
                     dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f%z")
@@ -235,14 +315,14 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
                     dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S%z")
                 timestamp.FromDatetime(dt)
                 mapped_data[k] = common_pb2.ValueType(utc_datetime=timestamp)
-            elif columns[k].type ==  common_pb2.DataType.BINARY:
+            elif columns[k].type == common_pb2.DataType.BINARY:
                 mapped_data[k] = common_pb2.ValueType(binary=v)
-            elif columns[k].type ==  common_pb2.DataType.XML:
+            elif columns[k].type == common_pb2.DataType.XML:
                 mapped_data[k] = common_pb2.ValueType(xml=v)
-            elif columns[k].type ==  common_pb2.DataType.STRING:
+            elif columns[k].type == common_pb2.DataType.STRING:
                 incoming = v if isinstance(v, str) else str(v)
                 mapped_data[k] = common_pb2.ValueType(string=incoming)
-            elif columns[k].type ==  common_pb2.DataType.JSON:
+            elif columns[k].type == common_pb2.DataType.JSON:
                 mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
             else:
                 raise ValueError(f"Unknown data type: {columns[k].type}")
@@ -273,6 +353,11 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
 
 
 def _yield_check(stack):
+    """Checks for the presence of 'yield' in the calling code.
+    Args:
+        stack: The stack frame to check.
+    """
+
     # Known issue with inspect.getmodule() and yield behavior in a frozen application.
     # When using inspect.getmodule() on stack frames obtained by inspect.stack(), it fails
     # to resolve the modules in a frozen application due to incompatible assumptions about
@@ -293,7 +378,16 @@ def _yield_check(stack):
         raise RuntimeError(f"Unable to find '{called_method}' function in stack")
 
 
-def _check_dict(incoming: dict, string_only: bool = False):
+def _check_dict(incoming: dict, string_only: bool = False) -> dict:
+    """Validates the incoming dictionary.
+    Args:
+        incoming (dict): The dictionary to validate.
+        string_only (bool): Whether to allow only string values.
+    
+    Returns:
+        dict: The validated dictionary.
+    """
+
     if not incoming:
         return {}
 
@@ -311,6 +405,12 @@ def _check_dict(incoming: dict, string_only: bool = False):
 
 class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
     def __init__(self, update, schema=None):
+        """Initializes the Connector instance.
+        Args:
+            update: The update method.
+            schema: The schema method.
+        """
+
         self.schema_method = schema
         self.update_method = update
 
@@ -319,6 +419,13 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     # Call this method to unpause and start a connector
     def start(self, deploy_key: str, group: str, connection: str):
+        """Starts the connector with the given deployment key, group name, and connection schema name.
+
+        Args:
+            deploy_key (str): The deployment key.
+            group (str): The group name.
+            connection (str): The connection schema name.
+        """
         if not deploy_key: print("ERROR: Missing deploy key"); os._exit(1)
         if not connection: print("ERROR: Missing connection name"); os._exit(1)
 
@@ -334,13 +441,118 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     @staticmethod
     def __unpause_connection(id: str, deploy_key: str) -> bool:
+        """Unpauses the connection with the given ID and deployment key.
+
+        Args:
+            id (str): The connection ID.
+            deploy_key (str): The deployment key.
+
+        Returns:
+            bool: True if the connection was successfully unpaused, False otherwise.
+        """
         resp = rq.patch(f"https://api.fivetran.com/v1/connectors/{id}",
                         headers={"Authorization": f"Basic {deploy_key}"},
                         json={"force": True})
         return resp.ok
 
+    @staticmethod
+    def fetch_requirements_from_file(file_path: str) -> list[str]:
+        """Reads a requirements file and returns a list of dependencies.
+
+        Args:
+            file_path (str): The path to the requirements file.
+
+        Returns:
+            list[str]: A list of dependencies as strings.
+        """
+        with open(file_path, 'r') as f:
+            return f.read().splitlines()
+
+    @staticmethod
+    def fetch_requirements_as_dict(self, file_path: str) -> dict:
+        """Converts a list of dependencies from the requirements file into a dictionary.
+
+        Args:
+            file_path (str): The path to the requirements file.
+
+        Returns:
+            dict: A dictionary where keys are package names (lowercased) and
+            values are the full dependency strings.
+        """
+        return {item.split("==")[0].lower(): item.lower() for item in
+                self.fetch_requirements_from_file(file_path)}
+
+    def validate_requirements_file(self, project_path: str, is_deploy: bool):
+        """Validates the `requirements.txt` file against the project's actual dependencies.
+
+        This method generates a temporary requirements file using `pipreqs`, compares
+        it with the existing `requirements.txt`, and checks for version mismatches,
+        missing dependencies, and unused dependencies. It will issue warnings, errors,
+        or even terminate the process depending on whether it's being run for deployment.
+
+        Args:
+            project_path (str): The path to the project directory containing the `requirements.txt`.
+            is_deploy (bool): If `True`, the method will exit the process on critical errors.
+
+        """
+        subprocess.run(["pipreqs", "--savepath", "tmp_requirements.txt"], text=True, check=True)
+        tmp_requirements_file_path = os.path.join(project_path, 'tmp_requirements.txt')
+
+        tmp_requirements = self.fetch_requirements_as_dict(self, tmp_requirements_file_path)
+        tmp_requirements.pop("fivetran_connector_sdk")
+        os.remove(tmp_requirements_file_path)
+
+        if len(tmp_requirements) > 0:
+            if os.path.exists("requirements.txt"):
+                requirements = self.fetch_requirements_as_dict(self, os.path.join(project_path, 'requirements.txt'))
+            else:
+                with open("requirements.txt", 'w'):
+                    pass
+                requirements = {}
+                print("WARNING: Adding `requirements.txt` file to your project folder.")
+
+            version_mismatch_deps = {key: tmp_requirements[key] for key in
+                                     (requirements.keys() & tmp_requirements.keys())
+                                     if requirements[key] != tmp_requirements[key]}
+            if version_mismatch_deps:
+                print("WARNING: We recommend using the current stable version for the following:")
+                print(version_mismatch_deps)
+
+            missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
+            if missing_deps:
+                log_level = "ERROR" if is_deploy else "WARNING"
+                print(log_level +
+                      ":  Please include the following dependency libraries in requirements.txt, to be used by "
+                      "Fivetran production. "
+                      "For more information, please visit: "
+                      "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
+                      "#workingwithrequirementstxtfile")
+                print(*list(missing_deps.values()))
+                if is_deploy:
+                    os._exit(1)
+
+            unused_deps = list(requirements.keys() - tmp_requirements.keys())
+            if unused_deps:
+                print("INFO: The following dependencies are not used in connector.py:")
+                print(*unused_deps)
+        else:
+            if os.path.exists("requirements.txt"):
+                print("WARNING: `requirements.txt` is not required as no additional "
+                      "Python libraries are required for your code.")
+
+        if is_deploy: print("Successful validation of requirements.txt")
+
     # Call this method to deploy the connector to Fivetran platform
     def deploy(self, project_path: str, deploy_key: str, group: str, connection: str, configuration: dict = None):
+        """Deploys the connector to the Fivetran platform.
+
+        Args:
+            project_path (str): The path to the connector project.
+            deploy_key (str): The deployment key.
+            group (str): The group name.
+            connection (str): The connection name.
+            configuration (dict): The configuration dictionary.
+        """
         if not deploy_key: print("ERROR: Missing deploy key"); os._exit(1)
         if not connection: print("ERROR: Missing connection name"); os._exit(1)
         _check_dict(configuration)
@@ -356,10 +568,13 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             "sync_method": "DIRECT",
             "custom_payloads": [],
         }
+
+        self.validate_requirements_file(project_path, True)
+
         group_id, group_name = self.__get_group_info(group, deploy_key)
         print(f"Deploying '{project_path}' to '{group_name}/{connection}'")
         upload_file_path = self.__create_upload_file(project_path)
-        upload_result = self.__upload(upload_file_path, deploy_key,group_id,connection)
+        upload_result = self.__upload(upload_file_path, deploy_key, group_id, connection)
         os.remove(upload_file_path)
         if not upload_result:
             os._exit(1)
@@ -378,6 +593,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     @staticmethod
     def __force_sync(id: str, deploy_key: str) -> bool:
+        """Forces a sync operation on the connection with the given ID and deployment key.
+
+        Args:
+            id (str): The connection ID.
+            deploy_key (str): The deployment key.
+
+        Returns:
+            bool: True if the sync was successfully started, False otherwise.
+        """
         resp = rq.post(f"https://api.fivetran.com/v1/connectors/{id}/sync",
                        headers={"Authorization": f"Basic {deploy_key}"},
                        json={"force": True})
@@ -385,6 +609,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     @staticmethod
     def __update_connection(id: str, name: str, group: str, config: dict, deploy_key: str):
+        """Updates the connection with the given ID, name, group, configuration, and deployment key.
+
+        Args:
+            id (str): The connection ID.
+            name (str): The connection name.
+            group (str): The group name.
+            config (dict): The configuration dictionary.
+            deploy_key (str): The deployment key.
+        """
         if not config["secrets_list"]:
             del config["secrets_list"]
 
@@ -400,7 +633,18 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             os._exit(1)
 
     @staticmethod
-    def __get_connection_id(name: str, group: str, group_id: str, deploy_key: str):
+    def __get_connection_id(name: str, group: str, group_id: str, deploy_key: str) -> str | None:
+        """Retrieves the connection ID for the specified connection schema name, group, and deployment key.
+
+        Args:
+            name (str): The connection name.
+            group (str): The group name.
+            group_id (str): The group ID.
+            deploy_key (str): The deployment key.
+
+        Returns:
+            str: The connection ID, or None
+        """
         resp = rq.get(f"https://api.fivetran.com/v1/groups/{group_id}/connectors",
                       headers={"Authorization": f"Basic {deploy_key}"},
                       params={"schema": name})
@@ -414,7 +658,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         return None
 
     @staticmethod
-    def __create_connection(deploy_key: str, group_id: str, config: dict):
+    def __create_connection(deploy_key: str, group_id: str, config: dict) -> rq.Response:
+        """Creates a new connection with the given deployment key, group ID, and configuration.
+
+        Args:
+            deploy_key (str): The deployment key.
+            group_id (str): The group ID.
+            config (dict): The configuration dictionary.
+
+        Returns:
+            rq.Response: The response object.
+        """
         response = rq.post(f"https://api.fivetran.com/v1/connectors",
                            headers={"Authorization": f"Basic {deploy_key}"},
                            json={
@@ -428,29 +682,59 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         return response
 
     def __create_upload_file(self, project_path: str) -> str:
+        """Creates an upload file for the given project path.
+
+        Args:
+            project_path (str): The path to the project.
+
+        Returns:
+            str: The path to the upload file.
+        """
         print("Packaging project for upload ..")
         zip_file_path = self.__zip_folder(project_path)
         print("✓")
         return zip_file_path
 
     def __zip_folder(self, project_path: str) -> str:
+        """Zips the folder at the given project path.
+
+        Args:
+            project_path (str): The path to the project.
+
+        Returns:
+            str: The path to the zip file.
+        """
         upload_filepath = os.path.join(project_path, UPLOAD_FILENAME)
+        connector_file_exists = False
 
         with ZipFile(upload_filepath, 'w', ZIP_DEFLATED) as zipf:
             for root, files in self.__dir_walker(project_path):
                 for file in files:
+                    if file == "connector.py":
+                        connector_file_exists = True
                     file_path = os.path.join(root, file)
                     arcname = os.path.relpath(file_path, project_path)
                     zipf.write(file_path, arcname)
 
+        if not connector_file_exists:
+            print("ERROR: Missing connector.py file")
+            os._exit(1)
         return upload_filepath
 
     def __dir_walker(self, top):
+        """Walks the directory tree starting at the given top directory.
+
+        Args:
+            top (str): The top directory to start the walk.
+
+        Yields:
+            tuple: A tuple containing the current directory path and a list of files.
+        """
         dirs, files = [], []
         for name in os.listdir(top):
             path = os.path.join(top, name)
             if os.path.isdir(path):
-                if name not in EXCLUDED_DIRS:
+                if (name not in EXCLUDED_DIRS) and (not name.startswith(".")):
                     dirs.append(name)
             else:
                 if name.endswith(".py") or name == "requirements.txt":
@@ -464,6 +748,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     @staticmethod
     def __upload(local_path: str, deploy_key: str, group_id: str, connection: str) -> bool:
+        """Uploads the local code file for the specified group and connection.
+
+        Args:
+            local_path (str): The local file path.
+            deploy_key (str): The deployment key.
+            group_id (str): The group ID.
+            connection (str): The connection name.
+
+        Returns:
+            bool: True if the upload was successful, False otherwise.
+        """
         print("Uploading project .. ", end="", flush=True)
         response = rq.post(f"https://api.fivetran.com/v2/deploy/{group_id}/{connection}",
                            files={'file': open(local_path, 'rb')},
@@ -477,6 +772,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     @staticmethod
     def __get_os_name() -> str:
+        """Returns the name of the operating system.
+
+        Returns:
+            str: The name of the operating system.
+        """
         os_sysname = platform.system().lower()
         if os_sysname.startswith("darwin"):
             return MAC_OS
@@ -488,6 +788,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     @staticmethod
     def __get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
+        """Retrieves the group information for the specified group and deployment key.
+
+        Args:
+            group (str): The group name.
+            deploy_key (str): The deployment key.
+
+        Returns:
+            tuple[str, str]: A tuple containing the group ID and group name.
+        """
         resp = rq.get("https://api.fivetran.com/v1/groups",
                       headers={"Authorization": f"Basic {deploy_key}"})
 
@@ -521,6 +830,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             configuration: dict = None,
             state: dict = None,
             log_level: Logging.Level = Logging.Level.INFO) -> grpc.Server:
+        """Runs the connector server.
+
+        Args:
+            port (int): The port number to listen for incoming requests.
+            configuration (dict): The configuration dictionary.
+            state (dict): The state dictionary.
+            log_level (Logging.Level): The logging level.
+
+        Returns:
+            grpc.Server: The gRPC server instance.
+        """
         self.configuration = _check_dict(configuration, True)
         self.state = _check_dict(state)
         Logging.LOG_LEVEL = log_level
@@ -529,7 +849,6 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         connector_sdk_pb2_grpc.add_ConnectorServicer_to_server(self, server)
         server.add_insecure_port("[::]:" + str(port))
         server.start()
-        print("Connector started, listening on " + str(port))
         if DEBUGGING:
             return server
         server.wait_for_termination()
@@ -541,6 +860,18 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
               configuration: dict = None,
               state: dict = None,
               log_level: Logging.Level = Logging.Level.FINE) -> bool:
+        """Tests the connector code by running it with the connector tester.
+
+        Args:
+            project_path (str): The path to the project.
+            port (int): The port number to listen for incoming requests.
+            configuration (dict): The configuration dictionary.
+            state (dict): The state dictionary.
+            log_level (Logging.Level): The logging level.
+
+        Returns:
+            bool: True if there was an error, False otherwise.
+        """
         global DEBUGGING
         DEBUGGING = True
 
@@ -598,16 +929,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 os._exit(1)
 
         project_path = os.getcwd() if project_path is None else project_path
+        self.validate_requirements_file(project_path, False)
         print(f"Debugging connector at: {project_path}")
         server = self.run(port, configuration, state, log_level=log_level)
 
         # Uncomment this to run the tester manually
-        #server.wait_for_termination()
+        # server.wait_for_termination()
 
         error = False
         try:
             print(f"Starting connector tester..")
-            for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path):
+            for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, port):
                 print(log_msg, end="")
         except:
             print(traceback.format_exc())
@@ -618,12 +950,51 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             return error
 
     @staticmethod
-    def __java_exe(location: str, os_name: str):
+    def __java_exe(location: str, os_name: str) -> str:
+        """Returns the path to the Java executable.
+
+        Args:
+            location (str): The location of the Java executable.
+            os_name (str): The name of the operating system.
+
+        Returns:
+            str: The path to the Java executable.
+        """
         java_exe_base = os.path.join(location, "bin", "java")
         return f"{java_exe_base}.exe" if os_name == WIN_OS else java_exe_base
 
     @staticmethod
-    def __run_tester(java_exe: str, root_dir: str, project_path: str):
+    def process_stream(stream):
+        """Processes a stream of text lines, replacing occurrences of a specified pattern.
+
+        This method reads each line from the provided stream, searches for occurrences of
+        a predefined pattern, and replaces them with a specified replacement string.
+
+        Args:
+            stream (iterable): An iterable stream of text lines, typically from a file or another input source.
+
+        Yields:
+            str: Each line from the stream after replacing the matched pattern with the replacement string.
+        """
+        pattern = re.compile(r'com\.fivetran\.fivetran_sdk\.tools\.testers\.\S+')
+        replacement = 'Fivetran SDK Tester'
+
+        for line in iter(stream.readline, ""):
+            modified_line = pattern.sub(replacement, line)
+            yield modified_line
+
+    @staticmethod
+    def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int = 50051):
+        """Runs the connector tester.
+
+        Args:
+            java_exe (str): The path to the Java executable.
+            root_dir (str): The root directory.
+            project_path (str): The path to the project.
+
+        Yields:
+            str: The log messages from the tester.
+        """
         working_dir = os.path.join(project_path, OUTPUT_FILES_DIR)
         try:
             os.mkdir(working_dir)
@@ -634,11 +1005,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                "-jar",
                os.path.join(root_dir, TESTER_FILENAME),
                "--connector-sdk=true",
-               f"--working-dir={working_dir}"]
+               f"--port={port}",
+               f"--working-dir={working_dir}",
+               "--tester-type=source"]
 
-        popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True)
-        for stdout_line in iter(popen.stdout.readline, ""):
-            yield stdout_line
+        popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+        for line in Connector.process_stream(popen.stderr):
+            yield line
+
+        for line in Connector.process_stream(popen.stdout):
+            yield line
         popen.stdout.close()
         return_code = popen.wait()
         if return_code:
@@ -646,6 +1022,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     # -- Methods below override ConnectorServicer methods
     def ConfigurationForm(self, request, context):
+        """Overrides the ConfigurationForm method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Returns:
+            common_pb2.ConfigurationFormResponse: An empty configuration form response.
+        """
         if not self.configuration:
             self.configuration = {}
 
@@ -653,9 +1038,27 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         return common_pb2.ConfigurationFormResponse()
 
     def Test(self, request, context):
+        """Overrides the Test method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Returns:
+            None: As this method is not implemented.
+        """
         return None
 
     def Schema(self, request, context):
+        """Overrides the Schema method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Returns:
+            connector_sdk_pb2.SchemaResponse: The schema response.
+        """
         global TABLES
 
         if not self.schema_method:
@@ -741,6 +1144,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
 
     def Update(self, request, context):
+        """Overrides the Update method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Yields:
+            connector_sdk_pb2.UpdateResponse: The update response.
+        """
         configuration = self.configuration if self.configuration else request.configuration
         state = self.state if self.state else json.loads(request.state_json)
 
@@ -757,7 +1169,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 raise e
 
 
-def find_connector_object(project_path):
+def find_connector_object(project_path) -> Connector:
+    """Finds the connector object in the given project path.
+    Args:
+        project_path (str): The path to the project.
+    
+    Returns:
+        object: The connector object.
+    """
+
     module_name = "connector_connector_code"
     connector_py = os.path.join(project_path, "connector.py")
     spec = importlib.util.spec_from_file_location(module_name, connector_py)
@@ -775,6 +1195,10 @@ def find_connector_object(project_path):
 
 
 def main():
+    """The main entry point for the script.
+    Parses command line arguments and passes them to connector object methods
+    """
+
     parser = argparse.ArgumentParser(allow_abbrev=False)
 
     # Positional

{fivetran_connector_sdk-0.7.24.2.dist-info → fivetran_connector_sdk-0.8.12.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fivetran_connector_sdk
-Version: 0.7.24.2
+Version: 0.8.12.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -10,9 +10,9 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-License-File: LICENSE
 Requires-Dist: grpcio ==1.60.1
 Requires-Dist: grpcio-tools ==1.60.1
 Requires-Dist: requests ==2.31.0
 Requires-Dist: get-pypi-latest-version ==0.0.12
+Requires-Dist: pipreqs ==0.5.0
 

{fivetran_connector_sdk-0.7.24.2.dist-info → fivetran_connector_sdk-0.8.12.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fivetran_connector_sdk/__init__.py,sha256=sMNFVDkoFTiENMT4InkTy50TTr2ERQ5e-9SlbR21bLI,34605
+fivetran_connector_sdk/__init__.py,sha256=OaXaLM_1xxs7sF6sBAB3aBW4b9DTpc_mR6q23pGgQnw,49697
 fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
 fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -6,9 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
 fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
 fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
 fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
-fivetran_connector_sdk-0.7.24.2.dist-info/LICENSE,sha256=Kutp3D0T7HmHuBifKmbw39OZLAL1ckaLRb8u9lyJxE8,1065
-fivetran_connector_sdk-0.7.24.2.dist-info/METADATA,sha256=WweKTohlC8uqWLa5Atjtlnn_wI_iCszbCUSnY3joI0w,699
-fivetran_connector_sdk-0.7.24.2.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-fivetran_connector_sdk-0.7.24.2.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-0.7.24.2.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-0.7.24.2.dist-info/RECORD,,
+fivetran_connector_sdk-0.8.12.1.dist-info/METADATA,sha256=nZR69biAGw0z-EXCSUTcXXWcIpx1BkkNLQW3RukPAOY,708
+fivetran_connector_sdk-0.8.12.1.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+fivetran_connector_sdk-0.8.12.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-0.8.12.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-0.8.12.1.dist-info/RECORD,,

{fivetran_connector_sdk-0.7.24.2.dist-info → fivetran_connector_sdk-0.8.12.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (71.1.0)
+Generator: setuptools (72.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fivetran_connector_sdk-0.7.24.2.dist-info/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2023 Fivetran
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.