fivetran-connector-sdk 0.8.21.1__tar.gz → 0.8.26.1__tar.gz

{fivetran_connector_sdk-0.8.21.1 → fivetran_connector_sdk-0.8.26.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fivetran_connector_sdk
-Version: 0.8.21.1
+Version: 0.8.26.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -17,7 +17,7 @@ Requires-Dist: get_pypi_latest_version==0.0.12
 Requires-Dist: pipreqs==0.5.0
 
 # **fivetran-connector-sdk**
-The *fivetran-connector-sdk* is a Python module that enables you to write custom data connectors and deploy them as an extension of [Fivetran](https://www.fivetran.com/). Fivetran automatically manages running the connectors on your scheduled frequency and manages the required compute resources.
+The *fivetran-connector-sdk* SDK allows users to execute custom, self-written Python code within [Fivetran's](https://www.fivetran.com/) secure cloud environment. Fivetran automatically manages running the connectors on your scheduled frequency and manages the required compute resources.
 
 The Connector SDK service is the best fit for the following use cases:
 - Fivetran doesn't have a connector for your source and is unlikely to support it soon.

{fivetran_connector_sdk-0.8.21.1 → fivetran_connector_sdk-0.8.26.1}/README.md

@@ -1,5 +1,5 @@
 # **fivetran-connector-sdk**
-The *fivetran-connector-sdk* is a Python module that enables you to write custom data connectors and deploy them as an extension of [Fivetran](https://www.fivetran.com/). Fivetran automatically manages running the connectors on your scheduled frequency and manages the required compute resources.
+The *fivetran-connector-sdk* SDK allows users to execute custom, self-written Python code within [Fivetran's](https://www.fivetran.com/) secure cloud environment. Fivetran automatically manages running the connectors on your scheduled frequency and manages the required compute resources.
 
 The Connector SDK service is the best fit for the following use cases:
 - Fivetran doesn't have a connector for your source and is unlikely to support it soon.

{fivetran_connector_sdk-0.8.21.1 → fivetran_connector_sdk-0.8.26.1}/src/fivetran_connector_sdk/__init__.py

@@ -23,13 +23,13 @@ from fivetran_connector_sdk.protos import common_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
-__version__ = "0.8.21.1"
+__version__ = "0.8.26.1"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
 LINUX_OS = "linux"
 
-TESTER_VERSION = "0.24.0807.001"
+TESTER_VERSION = "0.24.0826.001"
 TESTER_FILENAME = "run_sdk_tester.jar"
 VERSION_FILENAME = "version.txt"
 UPLOAD_FILENAME = "code.zip"
@@ -39,8 +39,16 @@ OUTPUT_FILES_DIR = "files"
 ONE_DAY_IN_SEC = 24 * 60 * 60
 
 EXCLUDED_DIRS = ["__pycache__", "lib", "include", OUTPUT_FILES_DIR]
-EXCLUDED_PIPREQS_DIRS = ["bin,etc,include,lib,Lib,lib64,Scripts"]
-
+EXCLUDED_PIPREQS_DIRS = ["bin,etc,include,lib,Lib,lib64,Scripts,share"]
+VALID_COMMANDS = ["debug", "deploy", "reset", "version"]
+MAX_ALLOWED_EDIT_DISTANCE_FROM_VALID_COMMAND = 3
+COMMANDS_AND_SYNONYMS = {
+    "debug": {"test", "verify", "diagnose", "check"},
+    "deploy": {"upload", "ship", "launch", "release"},
+    "reset": {"reinitialize", "reinitialise", "re-initialize", "re-initialise", "restart", "restore"},
+}
+
+CONNECTION_SCHEMA_NAME_PATTERN = r'^[_a-z][_a-z0-9]*$'
 DEBUGGING = False
 TABLES = {}
 
@@ -130,12 +138,6 @@ class Operations:
             for field in data.keys():
                 columns[field] = common_pb2.Column(
                     name=field, type=common_pb2.DataType.UNSPECIFIED, primary_key=False)
-            new_table = common_pb2.Table(name=table, columns=columns.values())
-
-            responses.append(connector_sdk_pb2.UpdateResponse(
-                operation=connector_sdk_pb2.Operation(
-                    schema_change=connector_sdk_pb2.SchemaChange(
-                        without_schema=common_pb2.TableList(tables=[new_table])))))
 
         mapped_data = _map_data_to_columns(data, columns)
         record = connector_sdk_pb2.Record(
@@ -401,6 +403,19 @@ def _check_dict(incoming: dict, string_only: bool = False) -> dict:
     return incoming
 
 
+def is_connection_name_valid(connection: str):
+    """Validates if the incoming connection schema name is valid or not.
+    Args:
+        connection (str): The connection schema name being validated.
+
+    Returns:
+        bool: True if connection name is valid.
+    """
+
+    pattern = re.compile(CONNECTION_SCHEMA_NAME_PATTERN)
+    return pattern.match(connection)
+
+
 class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
     def __init__(self, update, schema=None):
         """Initializes the Connector instance.
@@ -455,8 +470,17 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             dict: A dictionary where keys are package names (lowercased) and
             values are the full dependency strings.
         """
-        return {item.split("==")[0].lower(): item.lower() for item in
-                self.fetch_requirements_from_file(file_path)}
+        requirements_dict = {}
+        for requirement in self.fetch_requirements_from_file(file_path):
+            requirement = requirement.strip()
+            if not requirement or requirement.startswith("#"):  # Skip empty lines and comments
+                continue
+            try:
+                key, _ = re.split(r"==|>=|<=|>|<", requirement)
+                requirements_dict[key.lower()] = requirement.lower()
+            except ValueError:
+                print(f"Error: Invalid requirement format: '{requirement}'")
+        return requirements_dict
 
     def validate_requirements_file(self, project_path: str, is_deploy: bool):
         """Validates the `requirements.txt` file against the project's actual dependencies.
@@ -471,7 +495,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             is_deploy (bool): If `True`, the method will exit the process on critical errors.
 
         """
-        subprocess.run(["pipreqs", "--savepath", "tmp_requirements.txt", "--ignore"] + EXCLUDED_PIPREQS_DIRS, text=True, check=True)
+        subprocess.check_call(["pipreqs", "--savepath", "tmp_requirements.txt", "--ignore"] + EXCLUDED_PIPREQS_DIRS,
+                              stderr=subprocess.PIPE)
         tmp_requirements_file_path = os.path.join(project_path, 'tmp_requirements.txt')
 
         tmp_requirements = self.fetch_requirements_as_dict(self, tmp_requirements_file_path)
@@ -509,6 +534,10 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
             unused_deps = list(requirements.keys() - tmp_requirements.keys())
             if unused_deps:
+                if 'fivetran_connector_sdk' in unused_deps:
+                    print("ERROR: Please remove fivetran_connector_sdk from requirements.txt. "
+                          "We always use the latest version of fivetran_connector_sdk when executing your code.")
+                    os._exit(1)
                 print("INFO: The following dependencies are not needed, "
                       "they are not used or already installed. Please remove them from requirements.txt:")
                 print(*unused_deps)
@@ -530,8 +559,22 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             connection (str): The connection name.
             configuration (dict): The configuration dictionary.
         """
-        if not deploy_key: print("SEVERE: The Fivetran API key is missing. Please provide a valid Fivetran API key to create the connector."); os._exit(1)
-        if not connection: print("SEVERE: The connection name is missing. Please provide a valid connection name to create the connector."); os._exit(1)
+        if not deploy_key or not connection:
+            print("SEVERE: The deploy command needs the following parameters:"
+                  "\n\tRequired:\n"
+                  "\t\t--api-key <BASE64-ENCODED-FIVETRAN-API-KEY-FOR-DEPLOYMENT>\n"
+                  "\t\t--connection <VALID-CONNECTOR-SCHEMA_NAME>\n"
+                  "\t(Optional):\n"
+                  "\t\t--destination <DESTINATION_NAME> (Becomes required if there are multiple destinations)\n"
+                  "\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)")
+            os._exit(1)
+
+        if not is_connection_name_valid(connection):
+            print(f"SEVERE: Connection name: {connection} is invalid!\n The connection name should start with an "
+                  f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
+                  f"letters, or digits (0-9). Uppercase characters are not allowed.")
+            os._exit(1)
+
         _check_dict(configuration, True)
 
         secrets_list = []
@@ -549,25 +592,54 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         self.validate_requirements_file(project_path, True)
 
         group_id, group_name = self.__get_group_info(group, deploy_key)
-        print(f"INFO: Deploying '{project_path}' to connector '{connection}' in destination '{group_name}'.\n")
-        upload_file_path = self.__create_upload_file(project_path)
-        upload_result = self.__upload(upload_file_path, deploy_key, group_id, connection)
-        os.remove(upload_file_path)
-        if not upload_result:
-            os._exit(1)
-        connection_id = self.__get_connection_id(connection, group, group_id, deploy_key)
+        connection_id, service = self.__get_connection_id(
+            connection, group, group_id, deploy_key)
+
         if connection_id:
-            print(f"INFO: The connection '{connection}' already exists in destination '{group}', updating the existing connector... ", end="", flush=True)
-            self.__update_connection(connection_id, connection, group_name, connection_config, deploy_key)
-            print("✓")
+            if service != 'connector_sdk':
+                print(
+                    f"SEVERE: The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.")
+                os._exit(1)
+            confirm = input(
+                f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code and configuration. Do you want to proceed with the update? (Y/N): ")
+            if confirm.lower() == "y":
+                print("INFO: Updating the connection...\n")
+                self.__upload_project(
+                    project_path, deploy_key, group_id, group_name, connection)
+                self.__update_connection(
+                    connection_id, connection, group_name, connection_config, deploy_key)
+                print("✓")
+                print(
+                    f"INFO: Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
+            else:
+                print("INFO: Update canceled. The process is now terminating.")
+                os._exit(1)
         else:
-            response = self.__create_connection(deploy_key, group_id, connection_config)
+            self.__upload_project(project_path, deploy_key,
+                                  group_id, group_name, connection)
+            response = self.__create_connection(
+                deploy_key, group_id, connection_config)
             if response.ok:
-                print(f"INFO: Connection named '{connection}' has been created successfully.\n")
+                print(
+                    f"INFO: The connection '{connection}' has been created successfully.\n")
+                connection_id = response.json()['data']['id']
+                print(
+                    f"INFO: Visit the Fivetran dashboard to start the initial sync: https://fivetran.com/dashboard/connectors/{connection_id}/status")
             else:
-                print(f"SEVERE: Unable to create a new Connection, failed with error: {response.json()['message']}")
+                print(
+                    f"SEVERE: Unable to create a new connection, failed with error: {response.json()['message']}")
                 os._exit(1)
 
+    def __upload_project(self, project_path: str, deploy_key: str, group_id: str, group_name: str, connection: str):
+        print(
+            f"INFO: Deploying '{project_path}' to connection '{connection}' in destination '{group_name}'.\n")
+        upload_file_path = self.__create_upload_file(project_path)
+        upload_result = self.__upload(
+            upload_file_path, deploy_key, group_id, connection)
+        os.remove(upload_file_path)
+        if not upload_result:
+            os._exit(1)
+
     @staticmethod
     def __force_sync(id: str, deploy_key: str) -> bool:
         """Forces a sync operation on the connection with the given ID and deployment key.
@@ -606,7 +678,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                         })
 
         if not resp.ok:
-            print(f"SEVERE: Unable to update Connection '{name}' in destination '{group}', failed with error: '{response.json()['message']}'.")
+            print(f"SEVERE: Unable to update Connection '{name}' in destination '{group}', failed with error: '{resp.json()['message']}'.")
             os._exit(1)
 
     @staticmethod
@@ -626,13 +698,14 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                       headers={"Authorization": f"Basic {deploy_key}"},
                       params={"schema": name})
         if not resp.ok:
-            print(f"SEVERE: Unable to fetch connection list in destination '{group}'")
+            print(
+                f"SEVERE: Unable to fetch connection list in destination '{group}'")
             os._exit(1)
 
         if resp.json()['data']['items']:
-            return resp.json()['data']['items'][0]['id']
+            return resp.json()['data']['items'][0]['id'], resp.json()['data']['items'][0]['service']
 
-        return None
+        return None, None
 
     @staticmethod
     def __create_connection(deploy_key: str, group_id: str, config: dict) -> rq.Response:
@@ -782,7 +855,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
         if not resp.ok:
             print(
-                f"SEVERE: Unable to fetch list of destination names, status code = {resp.status_code}")
+                f"SEVERE: Unable to retrieve destination details. The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.")
             os._exit(1)
 
         data = resp.json().get("data", {})
@@ -839,6 +912,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         self.state = _check_dict(state)
         Logging.LOG_LEVEL = log_level
 
+        if not DEBUGGING:
+            print(f"Running on fivetran_connector_sdk: {__version__}")
+
         server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
         connector_sdk_pb2_grpc.add_ConnectorServicer_to_server(self, server)
         server.add_insecure_port("[::]:" + str(port))
@@ -1186,6 +1262,63 @@ def find_connector_object(project_path) -> Connector:
     sys.exit(1)
 
 
+def suggest_correct_command(input_command: str) -> bool:
+    # for typos
+    # calculate the edit distance of the input command (lowercased) with each of the valid commands
+    edit_distances_of_commands = sorted([(command, edit_distance(command, input_command.lower())) for command in VALID_COMMANDS], key=lambda x: x[1])
+
+    if edit_distances_of_commands[0][1] <= MAX_ALLOWED_EDIT_DISTANCE_FROM_VALID_COMMAND:
+        # if the closest command is within the max allowed edit distance, we suggest that command
+        # threshold is kept to prevent suggesting a valid command for an obvious wrong command like `fivetran iknowthisisntacommandbuttryanyway`
+        print_suggested_command_message(edit_distances_of_commands[0][0], input_command)
+        return True
+
+    # for synonyms
+    for (command, synonyms) in COMMANDS_AND_SYNONYMS.items():
+        # check if the input command (lowercased) is a recognised synonym of any of the valid commands, if yes, suggest that command
+        if input_command.lower() in synonyms:
+            print_suggested_command_message(command, input_command)
+            return True
+
+    return False
+
+
+def print_suggested_command_message(valid_command: str, input_command: str) -> None:
+    print(f"`fivetran {input_command}` is not a valid command.")
+    print(f"Did you mean `fivetran {valid_command}`?")
+    print("Use `fivetran --help` for more details.")
+
+
+def edit_distance(first_string: str, second_string: str) -> int:
+    first_string_length: int = len(first_string)
+    second_string_length: int = len(second_string)
+
+    # Initialize the previous row of distances (for the base case of an empty first string)
+    # 'previous_row[j]' holds the edit distance between an empty prefix of 'first_string' and the first 'j' characters of 'second_string'.
+    # The first row is filled with values [0, 1, 2, ..., second_string_length]
+    previous_row: list[int] = list(range(second_string_length + 1))
+
+    # Rest of the rows
+    for first_string_index in range(1, first_string_length + 1):
+        # Start the current row with the distance for an empty second string
+        current_row: list[int] = [first_string_index]  # j = 0
+
+        # Iterate over each character in the second string
+        for second_string_index in range(1, second_string_length + 1):
+            if first_string[first_string_index - 1] == second_string[second_string_index - 1]:
+                # If characters match, no additional cost
+                current_row.append(previous_row[second_string_index - 1])
+            else:
+                # Minimum cost of insertion, deletion, or substitution
+                current_row.append(1 + min(current_row[-1], previous_row[second_string_index], previous_row[second_string_index - 1]))
+
+        # Move to the next row
+        previous_row = current_row
+
+    # The last value in the last row is the edit distance
+    return previous_row[second_string_length]
+
+
 def main():
     """The main entry point for the script.
     Parses command line arguments and passes them to connector object methods
@@ -1194,7 +1327,7 @@ def main():
     parser = argparse.ArgumentParser(allow_abbrev=False)
 
     # Positional
-    parser.add_argument("command", help="debug|deploy|reset")
+    parser.add_argument("command", help="|".join(VALID_COMMANDS))
     parser.add_argument("project_path", nargs='?', default=os.getcwd(), help="Path to connector project directory")
 
     # Optional (Not all of these are valid with every mutually exclusive option below)
@@ -1261,8 +1394,12 @@ def main():
                 print("ERROR: Reset Failed")
                 raise e
 
+    elif args.command.lower() == "version":
+        print("fivetran_connector_sdk " + __version__)
+        
     else:
-        raise NotImplementedError("Invalid command: ", args.command)
+        if not suggest_correct_command(args.command):
+            raise NotImplementedError(f"Invalid command: {args.command}, see `fivetran --help`")
 
 
 if __name__ == "__main__":

{fivetran_connector_sdk-0.8.21.1 → fivetran_connector_sdk-0.8.26.1}/src/fivetran_connector_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fivetran_connector_sdk
-Version: 0.8.21.1
+Version: 0.8.26.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -17,7 +17,7 @@ Requires-Dist: get_pypi_latest_version==0.0.12
 Requires-Dist: pipreqs==0.5.0
 
 # **fivetran-connector-sdk**
-The *fivetran-connector-sdk* is a Python module that enables you to write custom data connectors and deploy them as an extension of [Fivetran](https://www.fivetran.com/). Fivetran automatically manages running the connectors on your scheduled frequency and manages the required compute resources.
+The *fivetran-connector-sdk* SDK allows users to execute custom, self-written Python code within [Fivetran's](https://www.fivetran.com/) secure cloud environment. Fivetran automatically manages running the connectors on your scheduled frequency and manages the required compute resources.
 
 The Connector SDK service is the best fit for the following use cases:
 - Fivetran doesn't have a connector for your source and is unlikely to support it soon.