fivetran-connector-sdk 1.5.1__tar.gz → 1.7.0__tar.gz

{fivetran_connector_sdk-1.5.1 → fivetran_connector_sdk-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.5.1
+Version: 1.7.0
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -13,7 +13,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: grpcio==1.71.0
 Requires-Dist: grpcio-tools==1.71.0
-Requires-Dist: requests==2.32.3
+Requires-Dist: requests==2.32.4
 Requires-Dist: pipreqs==0.5.0
 Requires-Dist: unidecode==1.4.0
 

{fivetran_connector_sdk-1.5.1 → fivetran_connector_sdk-1.7.0}/pyproject.toml

@@ -15,7 +15,7 @@ classifiers = [
 dependencies = [
   "grpcio==1.71.0",
   "grpcio-tools==1.71.0",
-  "requests==2.32.3",
+  "requests==2.32.4",
   "pipreqs==0.5.0",
   "unidecode==1.4.0"
 ]

{fivetran_connector_sdk-1.5.1 → fivetran_connector_sdk-1.7.0}/src/fivetran_connector_sdk/__init__.py

@@ -19,27 +19,27 @@ from fivetran_connector_sdk.operations import Operations
 from fivetran_connector_sdk import constants
 from fivetran_connector_sdk.constants import (
     TESTER_VER, VERSION_FILENAME, UTF_8,
-    VALID_COMMANDS, DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS, FIVETRAN_HD_AGENT_ID, TABLES
+    VALID_COMMANDS, DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS, TABLES
 )
 from fivetran_connector_sdk.helpers import (
-    print_library_log, reset_local_file_directory, find_connector_object, validate_and_load_configuration,
-    validate_and_load_state, get_input_from_cli, suggest_correct_command,
+    print_library_log, reset_local_file_directory, find_connector_object, suggest_correct_command,
 )
 from fivetran_connector_sdk.connector_helper import (
     validate_requirements_file, upload_project,
-    update_connection, are_setup_tests_failing,
-    validate_deploy_parameters, get_connection_id,
+    update_connection, are_setup_tests_failing, get_connection_id,
     handle_failing_tests_message_and_exit, delete_file_if_exists,
     create_connection, get_os_arch_suffix, get_group_info,
     java_exe_helper, run_tester, process_tables,
     update_base_url_if_required, exit_check,
     get_available_port, tester_root_dir_helper,
     check_dict, check_newer_version, cleanup_uploaded_project,
+    get_destination_group, get_connection_name, get_api_key, get_state,
+    get_python_version, get_hd_agent_id, get_configuration
 )
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
-__version__ = "1.5.1"
+__version__ = "1.7.0"
 TESTER_VERSION = TESTER_VER
 MAX_MESSAGE_LENGTH = 32 * 1024 * 1024 # 32MB
 
@@ -62,7 +62,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         update_base_url_if_required()
 
     # Call this method to deploy the connector to Fivetran platform
-    def deploy(self, args: dict, deploy_key: str, group: str, connection: str, hd_agent_id: str, configuration: dict = None):
+    def deploy(self, project_path: str, deploy_key: str, group: str, connection: str, hd_agent_id: str,
+               configuration: dict = None, config_path = None, python_version: str = None, force: bool = False):
         """Deploys the connector to the Fivetran platform.
 
         Args:
@@ -77,8 +78,6 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
         print_library_log("We support only `.py` files and a `requirements.txt` file as part of the code upload. *No other code files* are supported or uploaded during the deployment process. Ensure that your code is structured accordingly and all dependencies are listed in `requirements.txt`")
 
-        validate_deploy_parameters(connection, deploy_key)
-
         check_dict(configuration, True)
 
         secrets_list = []
@@ -92,10 +91,10 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             "sync_method": "DIRECT"
         }
 
-        if args.python_version:
-            connection_config["python_version"] = args.python_version
+        if python_version:
+            connection_config["python_version"] = python_version
 
-        validate_requirements_file(args.project_path, True, __version__, args.force)
+        validate_requirements_file(project_path, True, __version__, force)
 
         group_id, group_name = get_group_info(group, deploy_key)
         connection_id, service = get_connection_id(connection, group, group_id, deploy_key) or (None, None)
@@ -104,23 +103,23 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             if service != 'connector_sdk':
                 print_library_log(
                     f"The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.", Logging.Level.SEVERE)
-                os._exit(1)
+                sys.exit(1)
             else:
-                if args.force:
+                if force:
                     confirm = "y"
-                    if args.configuration:
+                    if configuration:
                         confirm_config = "y"
                 else:
                     confirm = input(
-                        f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code. Do you want to proceed with the update? (Y/N): ")
-                    if confirm.lower() == "y" and args.configuration:
-                        confirm_config = input(f"Your deploy will overwrite the configuration using the values provided in '{args.configuration}': key-value pairs not present in the new configuration will be removed; existing keys' values set in the cofiguration file or in the dashboard will be overwritten with new (empty or non-empty) values; new key-value pairs will be added. Do you want to proceed with the update? (Y/N): ")
+                        f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code. Do you want to proceed with the update? (y/N): ")
+                    if confirm.lower() == "y" and configuration:
+                        confirm_config = input(f"Your deploy will overwrite the configuration using the values provided in '{config_path}': key-value pairs not present in the new configuration will be removed; existing keys' values set in the configuration file or in the dashboard will be overwritten with new (empty or non-empty) values; new key-value pairs will be added. Do you want to proceed with the update? (y/N): ")
                 if confirm.lower() == "y" and (not connection_config["secrets_list"] or (confirm_config.lower() == "y")):
                     print_library_log("Updating the connection...\n")
                     upload_project(
-                        args.project_path, deploy_key, group_id, group_name, connection)
+                        project_path, deploy_key, group_id, group_name, connection)
                     response = update_connection(
-                        args, connection_id, connection, group_name, connection_config, deploy_key, hd_agent_id)
+                        connection_id, connection, group_name, connection_config, deploy_key, hd_agent_id)
                     print("✓")
                     print_library_log(f"Python version {response.json()['data']['config']['python_version']} to be used at runtime.",
                                       Logging.Level.INFO)
@@ -129,9 +128,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                         f"Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
                 else:
                     print_library_log("Update canceled. The process is now terminating.")
-                    os._exit(1)
+                    sys.exit(1)
         else:
-            upload_project(args.project_path, deploy_key,
+            upload_project(project_path, deploy_key,
                                   group_id, group_name, connection)
             response = create_connection(
                 deploy_key, group_id, connection_config, hd_agent_id)
@@ -152,7 +151,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                     f"Unable to create a new connection, failed with error: {response.json()['message']}", Logging.Level.SEVERE)
                 cleanup_uploaded_project(deploy_key,group_id, connection)
                 print_library_log("Please try again with the deploy command after resolving the issue!")
-                os._exit(1)
+                sys.exit(1)
 
     # Call this method to run the connector in production
     def run(self,
@@ -377,6 +376,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             print_library_log(error_message, Logging.Level.SEVERE)
             raise RuntimeError(error_message) from e
 
+def print_version():
+    print_library_log("fivetran_connector_sdk " + __version__)
+    sys.exit(0)
 
 def main():
     """The main entry point for the script.
@@ -389,7 +391,8 @@ def main():
     parser._option_string_actions["-h"].help = "Show this help message and exit"
 
     # Positional
-    parser.add_argument("command", help="|".join(VALID_COMMANDS))
+    parser.add_argument("--version", action="store_true", help="Print the version of the fivetran_connector_sdk and exit")
+    parser.add_argument("command", nargs="?", help="|".join(VALID_COMMANDS))
     parser.add_argument("project_path", nargs='?', default=os.getcwd(), help="Path to connector project directory")
 
     # Optional (Not all of these are valid with every mutually exclusive option below)
@@ -401,52 +404,41 @@ def main():
     parser.add_argument("-f", "--force", action="store_true", help="Force update an existing connection")
     parser.add_argument("--python-version", "--python", type=str, help=f"Supported Python versions you can use: {SUPPORTED_PYTHON_VERSIONS}. Defaults to {DEFAULT_PYTHON_VERSION}")
     parser.add_argument("--hybrid-deployment-agent-id", type=str, help="The Hybrid Deployment agent within the Fivetran system. If nothing is passed, the default agent of the destination is used.")
-
     args = parser.parse_args()
 
+    if args.version:
+        print_version()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(1)
+
     if args.command.lower() == "version":
-        print_library_log("fivetran_connector_sdk " + __version__)
-        return
+        print_version()
     elif args.command.lower() == "reset":
         reset_local_file_directory(args)
-        return
+        sys.exit(0)
 
     connector_object = find_connector_object(args.project_path)
 
     if not connector_object:
         sys.exit(1)
 
-    # Process optional args
-    ft_group = args.destination if args.destination else None
-    ft_connection = args.connection if args.connection else None
-    ft_deploy_key = args.api_key if args.api_key else None
-    hd_agent_id = args.hybrid_deployment_agent_id if args.hybrid_deployment_agent_id else os.getenv(FIVETRAN_HD_AGENT_ID, None)
-    configuration = args.configuration if args.configuration else None
-    state = args.state if args.state else os.getenv('FIVETRAN_STATE', None)
-
-    configuration = validate_and_load_configuration(args, configuration)
-    state = validate_and_load_state(args, state)
-
-    FIVETRAN_BASE_64_ENCODED_API_KEY = os.getenv('FIVETRAN_BASE_64_ENCODED_API_KEY', None)
-    FIVETRAN_DESTINATION_NAME = os.getenv('FIVETRAN_DESTINATION_NAME', None)
-    FIVETRAN_CONNECTION_NAME = os.getenv('FIVETRAN_CONNECTION_NAME', None)
-
     if args.command.lower() == "deploy":
-        if args.state:
-            print_library_log("'state' parameter is not used for 'deploy' command", Logging.Level.WARNING)
-
-        if not ft_deploy_key:
-            ft_deploy_key = get_input_from_cli("Please provide the API Key", FIVETRAN_BASE_64_ENCODED_API_KEY)
-
-        if not ft_group:
-            ft_group = get_input_from_cli("Please provide the destination", FIVETRAN_DESTINATION_NAME)
-
-        if not ft_connection:
-            ft_connection = get_input_from_cli("Please provide the connection name",FIVETRAN_CONNECTION_NAME)
+        ft_group = get_destination_group(args)
+        ft_connection = get_connection_name(args)
+        ft_deploy_key = get_api_key(args)
+        python_version = get_python_version(args)
+        hd_agent_id = get_hd_agent_id(args)
+        configuration, config_path = get_configuration(args)
+        get_state(args)
 
-        connector_object.deploy(args, ft_deploy_key, ft_group, ft_connection, hd_agent_id, configuration)
+        connector_object.deploy(args.project_path, ft_deploy_key, ft_group, ft_connection, hd_agent_id,
+                                configuration, config_path, python_version, args.force)
 
     elif args.command.lower() == "debug":
+        configuration, config_path = get_configuration(args)
+        state = get_state(args)
         connector_object.debug(args.project_path, configuration, state)
     else:
         if not suggest_correct_command(args.command):

{fivetran_connector_sdk-1.5.1 → fivetran_connector_sdk-1.7.0}/src/fivetran_connector_sdk/connector_helper.py

@@ -2,6 +2,7 @@ import os
 import re
 import ast
 import json
+import sys
 import time
 import socket
 import platform
@@ -17,9 +18,10 @@ from fivetran_connector_sdk.protos import common_pb2
 from fivetran_connector_sdk import constants
 from fivetran_connector_sdk.logger import Logging
 from fivetran_connector_sdk.helpers import (
-    print_library_log,
-    get_renamed_table_name,
-    get_renamed_column_name
+    print_library_log, get_renamed_table_name,
+    get_renamed_column_name, get_input_from_cli,
+    validate_and_load_state,
+    validate_and_load_configuration
 )
 from fivetran_connector_sdk.constants import (
     OS_MAP,
@@ -48,6 +50,97 @@ from fivetran_connector_sdk.constants import (
     TABLES,
 )
 
+def get_destination_group(args):
+    ft_group = args.destination if args.destination else None
+    env_destination_name = os.getenv('FIVETRAN_DESTINATION_NAME', None)
+    if not ft_group:
+        ft_group = get_input_from_cli("Provide the destination name (as displayed in your dashboard destination list)", env_destination_name)
+    return ft_group
+
+def get_connection_name(args):
+    ft_connection = args.connection if args.connection else None
+    env_connection_name = os.getenv('FIVETRAN_CONNECTION_NAME', None)
+    if not ft_connection:
+        for retrying in range(MAX_RETRIES):
+            ft_connection = get_input_from_cli("Provide the connection name", env_connection_name)
+            if not is_connection_name_valid(ft_connection):
+                if retrying==MAX_RETRIES-1:
+                    sys.exit(1)
+                else:
+                    print_library_log(f"Connection name: {ft_connection} is invalid!\n The connection name should start with an "
+                                      f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
+                                      f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
+                    print_library_log("Please retry...", Logging.Level.INFO)
+            else:
+                break
+    if not is_connection_name_valid(ft_connection):
+        print_library_log(f"Connection name: {ft_connection} is invalid!\n The connection name should start with an "
+                          f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
+                          f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
+        sys.exit(1)
+    return ft_connection
+
+def get_api_key(args):
+    ft_deploy_key = args.api_key if args.api_key else None
+    env_api_key = os.getenv('FIVETRAN_API_KEY', None)
+    if not ft_deploy_key:
+        ft_deploy_key = get_input_from_cli("Provide your API Key (Base 64 Encoded)", env_api_key, True)
+    return ft_deploy_key
+
+def get_python_version(args):
+    python_version = args.python_version if args.python_version else None
+    env_python_version = os.getenv('FIVETRAN_PYTHON_VERSION', None)
+    if env_python_version and not python_version and not args.force:
+        python_version = get_input_from_cli("Provide your python version", env_python_version)
+    return python_version
+
+def get_hd_agent_id(args):
+    hd_agent_id = args.hybrid_deployment_agent_id if args.hybrid_deployment_agent_id else None
+    env_hd_agent_id = os.getenv('FIVETRAN_HD_AGENT_ID', None)
+
+    if env_hd_agent_id and not hd_agent_id and not args.force:
+        hd_agent_id = get_input_from_cli("Provide the Hybrid Deployment Agent ID", env_hd_agent_id)
+    return hd_agent_id
+
+def get_state(args):
+    if args.command.lower() == "deploy" and args.state:
+        print_library_log("Unrecognised argument for deploy: --state."
+                          "'state' is not set using the 'deploy' command. You can manage the state for deployed connections via "
+                          "Fivetran API, https://fivetran.com/docs/connector-sdk/working-with-connector-sdk#workingwithstatejsonfile",
+                          Logging.Level.WARNING)
+        sys.exit(1)
+    state = args.state if args.state else os.getenv('FIVETRAN_STATE', None)
+    state = validate_and_load_state(args, state)
+    return state
+
+def get_configuration(args):
+    configuration = args.configuration if args.configuration else None
+    env_configuration = os.getenv('FIVETRAN_CONFIGURATION', None)
+    if not configuration and not args.force and args.command.lower() == "deploy":
+        json_filepath = os.path.join(args.project_path, "configuration.json")
+        if os.path.exists(json_filepath):
+            print_library_log("configuration.json file detected in the project, "
+                              "but no configuration input provided via the command line", Logging.Level.WARNING)
+            env_configuration = env_configuration if env_configuration else "configuration.json"
+        confirm = input(f"Does this debug run/deploy need configuration (y/N):")
+        if confirm.lower()=='y':
+            for retrying in range(MAX_RETRIES):
+                try:
+                    configuration = get_input_from_cli("Provide the configuration file path", env_configuration)
+                    config_values = validate_and_load_configuration(args.project_path, configuration)
+                    return config_values, configuration
+                except ValueError as e:
+                    if retrying==MAX_RETRIES-1:
+                        print_library_log(f"{e}. Invalid Configuration, Exiting..", Logging.Level.WARNING)
+                        sys.exit(1)
+                    else:
+                        print_library_log(f"{e}. Please retry..", Logging.Level.INFO)
+        else:
+            print_library_log("No input required for configuration. Continuing without configuration.", Logging.Level.INFO)
+    config_values = validate_and_load_configuration(args.project_path, configuration)
+    return config_values, configuration
+
+
 def check_newer_version(version: str):
     """Periodically checks for a newer version of the SDK and notifies the user if one is available."""
     tester_root_dir = tester_root_dir_helper()
@@ -79,7 +172,7 @@ def check_newer_version(version: str):
         except Exception:
             retry_after = 2 ** index
             print_library_log(f"Unable to check if a newer version of `fivetran-connector-sdk` is available. "
-                              f"Retrying again after {retry_after} seconds", Logging.Level.WARNING)
+                              f"Retrying after {retry_after} seconds", Logging.Level.WARNING)
             time.sleep(retry_after)
 
 
@@ -88,7 +181,7 @@ def tester_root_dir_helper() -> str:
     return os.path.join(os.path.expanduser("~"), ROOT_LOCATION)
 
 def _warn_exit_usage(filename, line_no, func):
-    print_library_log(f"Avoid using {func} to exit from the Python code as this can cause the connector to become stuck. Throw a error if required " +
+    print_library_log(f"Avoid using {func} to exit from the Python code as this can cause the connector to become stuck. Throw an error if required " +
                       f"at: {filename}:{line_no}. See the Technical Reference for details: https://fivetran.com/docs/connector-sdk/technical-reference#handlingexceptions",
                       Logging.Level.WARNING)
 
@@ -133,14 +226,14 @@ def check_dict(incoming: dict, string_only: bool = False) -> dict:
 
     if not isinstance(incoming, dict):
         raise ValueError(
-            "Configuration must be provided as a JSON dictionary. Please check your input. Reference: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
+            "Configuration must be provided as a JSON dictionary. Check your input. Reference: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
 
     if string_only:
         for k, v in incoming.items():
             if not isinstance(v, str):
                 print_library_log(
-                    "All values in the configuration must be STRING. Please check your configuration and ensure that every value is a STRING.", Logging.Level.SEVERE)
-                os._exit(1)
+                    "All values in the configuration must be STRING. Check your configuration and ensure that every value is a STRING.", Logging.Level.SEVERE)
+                sys.exit(1)
 
     return incoming
 
@@ -159,28 +252,11 @@ def is_connection_name_valid(connection: str):
 
 
 def log_unused_deps_error(package_name: str, version: str):
-    print_library_log(f"Please remove `{package_name}` from requirements.txt."
+    print_library_log(f"Remove `{package_name}` from requirements.txt."
           f" The latest version of `{package_name}` is always available when executing your code."
           f" Current version: {version}", Logging.Level.SEVERE)
 
 
-def validate_deploy_parameters(connection, deploy_key):
-    if not deploy_key or not connection:
-        print_library_log("The deploy command needs the following parameters:"
-              "\n\tRequired:\n"
-              "\t\t--api-key <BASE64-ENCODED-FIVETRAN-API-KEY-FOR-DEPLOYMENT>\n"
-              "\t\t--connection <VALID-CONNECTOR-SCHEMA_NAME>\n"
-              "\t(Optional):\n"
-              "\t\t--destination <DESTINATION_NAME> (Becomes required if there are multiple destinations)\n"
-              "\t\t--configuration <CONFIGURATION_FILE> (Completely replaces the existing configuration)", Logging.Level.SEVERE)
-        os._exit(1)
-    elif not is_connection_name_valid(connection):
-        print_library_log(f"Connection name: {connection} is invalid!\n The connection name should start with an "
-              f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
-              f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
-        os._exit(1)
-
-
 def is_port_in_use(port: int):
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
         return s.connect_ex(('127.0.0.1', port)) == 0
@@ -204,7 +280,7 @@ def update_base_url_if_required():
                 print_library_log(f"Updating PRODUCTION_BASE_URL to: {base_url}")
 
 def fetch_requirements_from_file(file_path: str) -> list[str]:
-    """Reads a requirements file and returns a list of dependencies.
+    """Reads the requirements file and returns a list of dependencies.
 
     Args:
         file_path (str): The path to the requirements file.
@@ -222,7 +298,7 @@ def fetch_requirements_as_dict(file_path: str) -> dict:
         file_path (str): The path to the requirements file.
 
     Returns:
-        dict: A dictionary where keys are package names (lowercased) and
+        dict: A dictionary where keys are package names (lowercase) and
         values are the full dependency strings.
     """
     requirements_dict = {}
@@ -345,8 +421,8 @@ def validate_requirements_file(project_path: str, is_deploy: bool, version: str
                 if confirm.lower() == "n":
                     if 'fivetran_connector_sdk' in unused_deps or 'requests' in unused_deps:
                         print_library_log(
-                            f"Please fix your {REQUIREMENTS_TXT} file by removing pre-installed dependencies [fivetran_connector_sdk, requests] to proceed with the deployment.")
-                        os._exit(1)
+                            f"Fix your {REQUIREMENTS_TXT} file by removing pre-installed dependencies [fivetran_connector_sdk, requests] to proceed with the deployment.")
+                        sys.exit(1)
                     print_library_log(f"Changes identified for unused libraries have been ignored. These changes have NOT been made to {REQUIREMENTS_TXT}.")
                 elif confirm.lower() == "y":
                     update_unused_requirements = True
@@ -375,15 +451,15 @@ def handle_unused_deps(unused_deps, version):
     if 'fivetran_connector_sdk' in unused_deps:
         log_unused_deps_error("fivetran_connector_sdk", version)
     if 'requests' in unused_deps:
-        log_unused_deps_error("requests", "2.32.3")
+        log_unused_deps_error("requests", "2.32.4")
     print_library_log("The following dependencies are not needed, "
-          f"they are not used or already installed. Please remove them from {REQUIREMENTS_TXT}:", Logging.Level.WARNING)
+          f"they are already installed or not in use. Remove them from {REQUIREMENTS_TXT}:", Logging.Level.WARNING)
     print(*unused_deps)
 
 def handle_missing_deps(missing_deps):
-    print_library_log(f"Please include the following dependency libraries in {REQUIREMENTS_TXT}, to be used by "
+    print_library_log(f"Include the following dependency libraries in {REQUIREMENTS_TXT}, to be used by "
           "Fivetran production. "
-          "For more information, please visit: "
+          "For more information, see our docs: "
           "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
           "#workingwithrequirementstxtfile", Logging.Level.SEVERE)
     print(*list(missing_deps.values()))
@@ -420,15 +496,15 @@ def upload_project(project_path: str, deploy_key: str, group_id: str, group_name
         upload_file_path, deploy_key, group_id, connection)
     delete_file_if_exists(upload_file_path)
     if not upload_result:
-        os._exit(1)
+        sys.exit(1)
 
 
 def cleanup_uploaded_project(deploy_key: str, group_id: str, connection: str):
     cleanup_result = cleanup_uploaded_code(deploy_key, group_id, connection)
     if not cleanup_result:
-        os._exit(1)
+        sys.exit(1)
 
-def update_connection(args: dict, id: str, name: str, group: str, config: dict, deploy_key: str, hd_agent_id: str):
+def update_connection(id: str, name: str, group: str, config: dict, deploy_key: str, hd_agent_id: str):
     """Updates the connection with the given ID, name, group, configuration, and deployment key.
 
     Args:
@@ -440,7 +516,7 @@ def update_connection(args: dict, id: str, name: str, group: str, config: dict,
         deploy_key (str): The deployment key.
         hd_agent_id (str): The hybrid deployment agent ID within the Fivetran system.
     """
-    if not args.configuration:
+    if not config.get("secrets_list"):
         del config["secrets_list"]
 
     json_payload = {
@@ -467,7 +543,7 @@ def update_connection(args: dict, id: str, name: str, group: str, config: dict,
         print_library_log(
             f"Unable to update Connection '{name}' in destination '{group}', failed with error: '{response.json()['message']}'.",
             Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
     return response
 
 def handle_failing_tests_message_and_exit(resp, log_message):
@@ -475,8 +551,8 @@ def handle_failing_tests_message_and_exit(resp, log_message):
     print_failing_setup_tests(resp)
     connection_id = resp.json().get('data', {}).get('id')
     print_library_log(f"Connection ID: {connection_id}")
-    print_library_log("Please try again with the deploy command after resolving the issue!")
-    os._exit(1)
+    print_library_log("Try again with the deploy command after resolving the issue!")
+    sys.exit(1)
 
 def are_setup_tests_failing(response) -> bool:
     """Checks for failed setup tests in the response and returns True if any test has failed, otherwise False."""
@@ -496,7 +572,7 @@ def print_failing_setup_tests(response):
                     test.get("status") == "FAILED" or test.get("status") == "JOB_FAILED"]
 
     if failed_tests:
-        print_library_log("Following setup tests have failed!", Logging.Level.WARNING)
+        print_library_log("The following setup tests have failed!", Logging.Level.WARNING)
         for test in failed_tests:
             print_library_log(f"Test: {test.get('title')}", Logging.Level.WARNING)
             print_library_log(f"Status: {test.get('status')}", Logging.Level.WARNING)
@@ -520,7 +596,7 @@ def get_connection_id(name: str, group: str, group_id: str, deploy_key: str) ->
     if not resp.ok:
         print_library_log(
             f"Unable to fetch connection list in destination '{group}'", Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
 
     if resp.json()['data']['items']:
         return resp.json()['data']['items'][0]['id'], resp.json()['data']['items'][0]['service']
@@ -597,13 +673,13 @@ def zip_folder(project_path: str) -> str:
 
     if not connector_file_exists:
         print_library_log(
-            "The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.",
+            "The 'connector.py' file is missing. Ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase. All custom connectors require this file because Fivetran calls it to start a sync.",
             Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
 
     if custom_drivers_exists and not custom_driver_installation_script_exists:
         print_library_log(INSTALLATION_SCRIPT_MISSING_MESSAGE, Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
 
     return upload_filepath
 
@@ -717,16 +793,16 @@ def get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
 
     if not resp.ok:
         print_library_log(
-            f"The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.",
+            f"The request failed with status code: {resp.status_code}. Ensure you're using a valid base64-encoded API key and try again.",
             Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
 
     data = resp.json().get("data", {})
     groups = data.get("items")
 
     if not groups:
         print_library_log("No destinations defined in the account", Logging.Level.SEVERE)
-        os._exit(1)
+        sys.exit(1)
 
     if not group:
         if len(groups) == 1:
@@ -735,7 +811,7 @@ def get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
             print_library_log(
                 "Destination name is required when there are multiple destinations in the account",
                 Logging.Level.SEVERE)
-            os._exit(1)
+            sys.exit(1)
     else:
         while True:
             for grp in groups:
@@ -752,8 +828,8 @@ def get_group_info(group: str, deploy_key: str) -> tuple[str, str]:
             groups = data.get("items", [])
 
     print_library_log(
-        f"The specified destination '{group}' was not found in your account.", Logging.Level.SEVERE)
-    os._exit(1)
+        f"We couldn't find the specified destination '{group}' in your account.", Logging.Level.SEVERE)
+    sys.exit(1)
 
 def java_exe_helper(location: str, os_arch_suffix: str) -> str:
     """Returns the path to the Java executable.
@@ -878,13 +954,22 @@ def process_columns(columns, entry):
 
         elif isinstance(type, dict):
             if type['type'].upper() != "DECIMAL":
-                raise ValueError("Expecting DECIMAL data type")
+                error_message = (
+                    f"Expecting DECIMAL data type for dictionary column entry, but got: {type['type']} in entry: {entry} "
+                    f"for column: {column_name}. "
+                    "Dictionary type is only allowed for DECIMAL columns with 'precision' and 'scale' fields, "
+                    "as in: {'type': 'DECIMAL', 'precision': <int>, 'scale': <int>}. "
+                    "For all other data types, use a string as the column type."
+                )
+                raise ValueError(error_message)
             column.type = common_pb2.DataType.DECIMAL
             column.decimal.precision = type['precision']
             column.decimal.scale = type['scale']
 
         else:
-            raise ValueError("Unrecognized column type: ", str(type))
+            raise ValueError(
+                f"Unrecognized column type for column: {column_name} in entry: {entry}. Got: {str(type)}"
+            )
 
         if "primary_key" in entry and name in entry["primary_key"]:
             column.primary_key = True
@@ -901,7 +986,15 @@ def process_data_type(column, type):
     elif type.upper() == "LONG":
         column.type = common_pb2.DataType.LONG
     elif type.upper() == "DECIMAL":
-        raise ValueError("DECIMAL data type missing precision and scale")
+        raise ValueError(
+            "DECIMAL data type missing precision and scale. "
+            "Use a dictionary for DECIMAL column type like: "
+            '''"col_name": {  # Decimal data type with precision and scale.\n'''
+            '''    "type": "DECIMAL",\n'''
+            '''    "precision": 15,\n'''
+            '''    "scale": 2\n'''
+            '''}'''
+        )
     elif type.upper() == "FLOAT":
         column.type = common_pb2.DataType.FLOAT
     elif type.upper() == "DOUBLE":

{fivetran_connector_sdk-1.5.1 → fivetran_connector_sdk-1.7.0}/src/fivetran_connector_sdk/helpers.py

@@ -29,19 +29,23 @@ from fivetran_connector_sdk.constants import (
 RENAMED_TABLE_NAMES = {}
 RENAMED_COL_NAMES = {}
 
-def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
+def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO, dev_log: bool = False):
     """Logs a library message with the specified logging level.
 
     Args:
         level (Logging.Level): The logging level.
         message (str): The message to log.
+        dev_log (bool): Boolean value to check if it is dev log and shouldn't be visible to customer
     """
     if constants.DEBUGGING or constants.EXECUTED_VIA_CLI:
+        if dev_log:
+            return
         current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
         print(f"{Logging.get_color(level)}{current_time} {level.name} {LOGGING_PREFIX}: {message} {Logging.reset_color()}")
     else:
+        message_origin = "library_dev" if dev_log else "library"
         escaped_message = json.dumps(LOGGING_PREFIX + LOGGING_DELIMITER + message)
-        log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "library"}}'
+        log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "{message_origin}"}}'
         print(log_message)
 
 def is_special(c):
@@ -252,12 +256,16 @@ def edit_distance(first_string: str, second_string: str) -> int:
     return previous_row[second_string_length]
 
 
-def get_input_from_cli(prompt: str, default_value: str) -> str:
+def get_input_from_cli(prompt: str, default_value: str, hide_value = False) -> str:
     """
     Prompts the user for input.
     """
     if default_value:
-        value = input(f"{prompt} [Default : {default_value}]: ").strip() or default_value
+        if hide_value:
+            default_value_hidden = default_value[0:8] + "********"
+            value = input(f"{prompt} [Default : {default_value_hidden}]: ").strip() or default_value
+        else:
+            value = input(f"{prompt} [Default : {default_value}]: ").strip() or default_value
     else:
         value = input(f"{prompt}: ").strip()
 
@@ -265,29 +273,36 @@ def get_input_from_cli(prompt: str, default_value: str) -> str:
         raise ValueError("Missing required input: Expected a value but received None")
     return value
 
-def validate_and_load_configuration(args, configuration):
+def validate_and_load_configuration(project_path, configuration):
     if configuration:
-        json_filepath = os.path.join(args.project_path, args.configuration)
+        configuration = os.path.expanduser(configuration)
+        if os.path.isabs(configuration):
+            json_filepath = os.path.abspath(configuration)
+        else:
+            relative_path = os.path.join(project_path, configuration)
+            json_filepath = os.path.abspath(str(relative_path))
         if os.path.isfile(json_filepath):
             with open(json_filepath, 'r', encoding=UTF_8) as fi:
-                configuration = json.load(fi)
+                try:
+                    configuration = json.load(fi)
+                except:
+                    raise ValueError(
+                        "Configuration must be provided as a JSON file. Please check your input. Reference: "
+                        "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
             if len(configuration) > MAX_CONFIG_FIELDS:
                 raise ValueError(f"Configuration field count exceeds maximum of {MAX_CONFIG_FIELDS}. Reduce the field count.")
         else:
             raise ValueError(
-                "Configuration must be provided as a JSON file. Please check your input. Reference: "
-                "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
+                f"Configuration path is incorrect, cannot find file at the location {json_filepath}")
     else:
-        json_filepath = os.path.join(args.project_path, "configuration.json")
-        if os.path.exists(json_filepath):
-            print_library_log("Configuration file detected in the project, but no configuration input provided via the command line", Logging.Level.WARNING)
+        print_library_log("No configuration file passed.", Logging.Level.INFO)
         configuration = {}
     return configuration
 
 
 def validate_and_load_state(args, state):
     if state:
-        json_filepath = os.path.join(args.project_path, args.state)
+        json_filepath =  os.path.abspath(os.path.join(args.project_path, args.state))
     else:
         json_filepath = os.path.join(args.project_path, "files", "state.json")
 

{fivetran_connector_sdk-1.5.1 → fivetran_connector_sdk-1.7.0}/src/fivetran_connector_sdk/operations.py

@@ -1,6 +1,6 @@
-import os
 import json
 import inspect
+import sys
 
 from datetime import datetime
 from google.protobuf import timestamp_pb2
@@ -18,6 +18,12 @@ from fivetran_connector_sdk.helpers import (
 from fivetran_connector_sdk.logger import Logging
 from fivetran_connector_sdk.protos import connector_sdk_pb2, common_pb2
 
+_LOG_DATA_TYPE_INFERENCE = {
+    "boolean": True,
+    "binary": True,
+    "json": True
+}
+
 class Operations:
     @staticmethod
     def upsert(table: str, data: dict) -> list[connector_sdk_pb2.UpdateResponse]:
@@ -174,7 +180,7 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
         if v is None:
             mapped_data[key] = common_pb2.ValueType(null=True)
         elif (key in columns) and columns[key].type != common_pb2.DataType.UNSPECIFIED:
-            map_defined_data_type(columns, key, mapped_data, v)
+            map_defined_data_type(columns[key].type, key, mapped_data, v)
         else:
             map_inferred_data_type(key, mapped_data, v)
     return mapped_data
@@ -189,13 +195,22 @@ def map_inferred_data_type(k, mapped_data, v):
     elif isinstance(v, float):
         mapped_data[k] = common_pb2.ValueType(float=v)
     elif isinstance(v, bool):
+        if _LOG_DATA_TYPE_INFERENCE["boolean"]:
+            print_library_log("Fivetran: Boolean Datatype has been inferred", Logging.Level.INFO, True)
+            _LOG_DATA_TYPE_INFERENCE["boolean"] = False
         mapped_data[k] = common_pb2.ValueType(bool=v)
     elif isinstance(v, bytes):
+        if _LOG_DATA_TYPE_INFERENCE["binary"]:
+            print_library_log("Fivetran: Binary Datatype has been inferred", Logging.Level.INFO, True)
+            _LOG_DATA_TYPE_INFERENCE["binary"] = False
         mapped_data[k] = common_pb2.ValueType(binary=v)
     elif isinstance(v, list):
         raise ValueError(
             "Values for the columns cannot be of type 'list'. Please ensure that all values are of a supported type. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#supporteddatatypes")
     elif isinstance(v, dict):
+        if _LOG_DATA_TYPE_INFERENCE["json"]:
+            print_library_log("Fivetran: JSON Datatype has been inferred", Logging.Level.INFO, True)
+            _LOG_DATA_TYPE_INFERENCE["json"] = False
         mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
     elif isinstance(v, str):
         mapped_data[k] = common_pb2.ValueType(string=v)
@@ -203,53 +218,50 @@ def map_inferred_data_type(k, mapped_data, v):
         # Convert arbitrary objects to string
         mapped_data[k] = common_pb2.ValueType(string=str(v))
 
-
-def map_defined_data_type(columns, k, mapped_data, v):
-    if columns[k].type == common_pb2.DataType.BOOLEAN:
-        mapped_data[k] = common_pb2.ValueType(bool=v)
-    elif columns[k].type == common_pb2.DataType.SHORT:
-        mapped_data[k] = common_pb2.ValueType(short=v)
-    elif columns[k].type == common_pb2.DataType.INT:
-        mapped_data[k] = common_pb2.ValueType(int=v)
-    elif columns[k].type == common_pb2.DataType.LONG:
-        mapped_data[k] = common_pb2.ValueType(long=v)
-    elif columns[k].type == common_pb2.DataType.DECIMAL:
-        mapped_data[k] = common_pb2.ValueType(decimal=v)
-    elif columns[k].type == common_pb2.DataType.FLOAT:
-        mapped_data[k] = common_pb2.ValueType(float=v)
-    elif columns[k].type == common_pb2.DataType.DOUBLE:
-        mapped_data[k] = common_pb2.ValueType(double=v)
-    elif columns[k].type == common_pb2.DataType.NAIVE_DATE:
-        timestamp = timestamp_pb2.Timestamp()
-        dt = datetime.strptime(v, "%Y-%m-%d")
-        timestamp.FromDatetime(dt)
-        mapped_data[k] = common_pb2.ValueType(naive_date=timestamp)
-    elif columns[k].type == common_pb2.DataType.NAIVE_DATETIME:
-        if '.' not in v: v = v + ".0"
-        timestamp = timestamp_pb2.Timestamp()
-        dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f")
-        timestamp.FromDatetime(dt)
-        mapped_data[k] = common_pb2.ValueType(naive_datetime=timestamp)
-    elif columns[k].type == common_pb2.DataType.UTC_DATETIME:
-        timestamp = timestamp_pb2.Timestamp()
-        dt = v if isinstance(v, datetime) else _parse_datetime_str(v)
-        timestamp.FromDatetime(dt)
-        mapped_data[k] = common_pb2.ValueType(utc_datetime=timestamp)
-    elif columns[k].type == common_pb2.DataType.BINARY:
-        mapped_data[k] = common_pb2.ValueType(binary=v)
-    elif columns[k].type == common_pb2.DataType.XML:
-        mapped_data[k] = common_pb2.ValueType(xml=v)
-    elif columns[k].type == common_pb2.DataType.STRING:
-        incoming = v if isinstance(v, str) else str(v)
-        mapped_data[k] = common_pb2.ValueType(string=incoming)
-    elif columns[k].type == common_pb2.DataType.JSON:
-        mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
+_TYPE_HANDLERS = {
+    common_pb2.DataType.BOOLEAN: lambda val: common_pb2.ValueType(bool=val),
+    common_pb2.DataType.SHORT: lambda val: common_pb2.ValueType(short=val),
+    common_pb2.DataType.INT: lambda val: common_pb2.ValueType(int=val),
+    common_pb2.DataType.LONG: lambda val: common_pb2.ValueType(long=val),
+    common_pb2.DataType.DECIMAL: lambda val: common_pb2.ValueType(decimal=val),
+    common_pb2.DataType.FLOAT: lambda val: common_pb2.ValueType(float=val),
+    common_pb2.DataType.DOUBLE: lambda val: common_pb2.ValueType(double=val),
+    common_pb2.DataType.NAIVE_DATE: lambda val: common_pb2.ValueType(naive_date= _parse_naive_date_str(val)),
+    common_pb2.DataType.NAIVE_DATETIME: lambda val: common_pb2.ValueType(naive_datetime= _parse_naive_datetime_str(val)),
+    common_pb2.DataType.UTC_DATETIME: lambda val: common_pb2.ValueType(utc_datetime= _parse_utc_datetime_str(val)),
+    common_pb2.DataType.BINARY: lambda val: common_pb2.ValueType(binary=val),
+    common_pb2.DataType.XML: lambda val: common_pb2.ValueType(xml=val),
+    common_pb2.DataType.STRING: lambda val: common_pb2.ValueType(string=val if isinstance(val, str) else str(val)),
+    common_pb2.DataType.JSON: lambda val: common_pb2.ValueType(json=json.dumps(val))
+}
+
+def map_defined_data_type(data_type, k, mapped_data, v):
+    handler = _TYPE_HANDLERS.get(data_type)
+    if handler:
+        mapped_data[k] = handler(v)
     else:
-        raise ValueError(f"Unsupported data type encountered: {columns[k].type}. Please use valid data types.")
-
-def _parse_datetime_str(dt):
-    return datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f%z" if '.' in dt else "%Y-%m-%dT%H:%M:%S%z")
-
+        raise ValueError(f"Unsupported data type encountered: {data_type}. Please use valid data types.")
+
+def _parse_utc_datetime_str(v):
+    timestamp = timestamp_pb2.Timestamp()
+    dt = v
+    if not isinstance(v, datetime):
+        dt = datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f%z" if '.' in dt else "%Y-%m-%dT%H:%M:%S%z")
+    timestamp.FromDatetime(dt)
+    return timestamp
+
+def _parse_naive_datetime_str(v):
+    if '.' not in v: v = v + ".0"
+    timestamp = timestamp_pb2.Timestamp()
+    dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f")
+    timestamp.FromDatetime(dt)
+    return timestamp
+
+def _parse_naive_date_str(v):
+    timestamp = timestamp_pb2.Timestamp()
+    dt = datetime.strptime(v, "%Y-%m-%d")
+    timestamp.FromDatetime(dt)
+    return timestamp
 
 def _yield_check(stack):
     """Checks for the presence of 'yield' in the calling code.
@@ -270,7 +282,7 @@ def _yield_check(stack):
         if 'yield' not in calling_code:
             print_library_log(
                 f"Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'", Logging.Level.SEVERE)
-            os._exit(1)
+            sys.exit(1)
     else:
         # This should never happen
         raise RuntimeError(

{fivetran_connector_sdk-1.5.1 → fivetran_connector_sdk-1.7.0}/src/fivetran_connector_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.5.1
+Version: 1.7.0
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -13,7 +13,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: grpcio==1.71.0
 Requires-Dist: grpcio-tools==1.71.0
-Requires-Dist: requests==2.32.3
+Requires-Dist: requests==2.32.4
 Requires-Dist: pipreqs==0.5.0
 Requires-Dist: unidecode==1.4.0
 

{fivetran_connector_sdk-1.5.1 → fivetran_connector_sdk-1.7.0}/src/fivetran_connector_sdk.egg-info/requires.txt

@@ -1,5 +1,5 @@
 grpcio==1.71.0
 grpcio-tools==1.71.0
-requests==2.32.3
+requests==2.32.4
 pipreqs==0.5.0
 unidecode==1.4.0