fivetran-connector-sdk 0.13.17.1__tar.gz → 0.13.30.1__tar.gz

{fivetran_connector_sdk-0.13.17.1 → fivetran_connector_sdk-0.13.30.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: fivetran_connector_sdk
-Version: 0.13.17.1
+Version: 0.13.30.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -15,7 +15,7 @@ Requires-Dist: grpcio==1.60.1
 Requires-Dist: grpcio-tools==1.60.1
 Requires-Dist: requests==2.32.3
 Requires-Dist: pipreqs==0.5.0
-Requires-Dist: transliterate==1.10.2
+Requires-Dist: unidecode==1.3.8
 
 # **fivetran-connector-sdk**
 [![Downloads](https://static.pepy.tech/badge/fivetran-connector-sdk)](https://pepy.tech/project/fivetran-connector-sdk)

{fivetran_connector_sdk-0.13.17.1 → fivetran_connector_sdk-0.13.30.1}/pyproject.toml

@@ -17,7 +17,7 @@ dependencies = [
   "grpcio-tools==1.60.1",
   "requests==2.32.3",
   "pipreqs==0.5.0",
-  "transliterate==1.10.2"
+  "unidecode==1.3.8"
 ]
 
 [project.scripts]

{fivetran_connector_sdk-0.13.17.1 → fivetran_connector_sdk-0.13.30.1}/src/fivetran_connector_sdk/__init__.py

@@ -6,7 +6,8 @@ import importlib.util
 import inspect
 import json
 import os
-from transliterate import translit
+import unicodedata
+from unidecode import unidecode
 import platform
 import requests as rq
 import shutil
@@ -28,13 +29,13 @@ from fivetran_connector_sdk.protos import connector_sdk_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
 # Version format: <major_version>.<MM>.<DD>.<iteration> (where MM is incremental value from Jan 2024)
-__version__ = "0.13.17.1"
+__version__ = "0.13.30.1"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
 LINUX_OS = "linux"
 
-TESTER_VERSION = "0.25.0117.001"
+TESTER_VERSION = "0.25.0130.001"
 TESTER_FILENAME = "run_sdk_tester.jar"
 VERSION_FILENAME = "version.txt"
 UPLOAD_FILENAME = "code.zip"
@@ -54,11 +55,6 @@ WORD_OR_DOLLAR_PATTERN = re.compile(r'[\w$]')
 DROP_LEADING_UNDERSCORE = re.compile(r'_+([a-zA-Z]\w*)')
 WORD_PATTERN = re.compile(r'\w')
 
-# Transliteration rules
-TO_ASCII_RULES = (
-    "Han-Latin; Katakana-Latin; Arabic-Latin; NFD; [:Nonspacing Mark:] Remove; NFC"
-)
-
 EXCLUDED_DIRS = ["__pycache__", "lib", "include", OUTPUT_FILES_DIR]
 EXCLUDED_PIPREQS_DIRS = ["bin,etc,include,lib,Lib,lib64,Scripts,share"]
 VALID_COMMANDS = ["debug", "deploy", "reset", "version"]
@@ -79,6 +75,7 @@ INSTALLATION_SCRIPT = "installation.sh"
 DRIVERS = "drivers"
 JAVA_LONG_MAX_VALUE = 9223372036854775807
 MAX_CONFIG_FIELDS = 100
+SUPPORTED_PYTHON_VERSIONS = {"3.12.8", "3.11.11", "3.10.16", "3.9.21"}
 
 
 class Logging:
@@ -100,7 +97,7 @@ class Logging:
         """
         if DEBUGGING:
             current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
-            escaped_message = json.dumps(message) [1:-1]
+            escaped_message = json.dumps(message).strip('"')
             print(f"{current_time} {level.name}: {escaped_message}")
         else:
             escaped_message = json.dumps(message)
@@ -164,13 +161,14 @@ class Operations:
 
         responses = []
 
-        table = for_table(table)
+        table = get_renamed_table_name(table)
         columns = _get_columns(table)
         if not columns:
             global TABLES
             for field in data.keys():
-                columns[field] = common_pb2.Column(
-                    name=field, type=common_pb2.DataType.UNSPECIFIED, primary_key=False)
+                field_name = get_renamed_column_name(field)
+                columns[field_name] = common_pb2.Column(
+                    name=field_name, type=common_pb2.DataType.UNSPECIFIED, primary_key=False)
 
         mapped_data = _map_data_to_columns(data, columns)
         record = connector_sdk_pb2.Record(
@@ -199,7 +197,7 @@ class Operations:
         """
         _yield_check(inspect.stack())
 
-        table = for_table(table)
+        table = get_renamed_table_name(table)
         columns = _get_columns(table)
         mapped_data = _map_data_to_columns(modified, columns)
         record = connector_sdk_pb2.Record(
@@ -225,7 +223,7 @@ class Operations:
         """
         _yield_check(inspect.stack())
 
-        table = for_table(table)
+        table = get_renamed_table_name(table)
         columns = _get_columns(table)
         mapped_data = _map_data_to_columns(keys, columns)
         record = connector_sdk_pb2.Record(
@@ -316,6 +314,7 @@ def _get_columns(table: str) -> dict:
     columns = {}
     if table in TABLES:
         for column in TABLES[table].columns:
+            column.name = get_renamed_column_name(column.name)
             columns[column.name] = column
 
     return columns
@@ -333,12 +332,13 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
     """
     mapped_data = {}
     for k, v in data.items():
+        key = get_renamed_column_name(k)
         if v is None:
-            mapped_data[k] = common_pb2.ValueType(null=True)
-        elif (k in columns) and columns[k].type != common_pb2.DataType.UNSPECIFIED:
-            map_defined_data_type(columns, k, mapped_data, v)
+            mapped_data[key] = common_pb2.ValueType(null=True)
+        elif (key in columns) and columns[key].type != common_pb2.DataType.UNSPECIFIED:
+            map_defined_data_type(columns, key, mapped_data, v)
         else:
-            map_inferred_data_type(k, mapped_data, v)
+            map_inferred_data_type(key, mapped_data, v)
 
     return mapped_data
 
@@ -489,7 +489,7 @@ def log_unused_deps_error(package_name: str, version: str):
           f" Current version: {version}", Logging.Level.SEVERE)
 
 
-def validate_deploy_parameters(connection, deploy_key):
+def validate_deploy_parameters(connection, deploy_key, python_version):
     if not deploy_key or not connection:
         print_library_log("The deploy command needs the following parameters:"
               "\n\tRequired:\n"
@@ -504,6 +504,10 @@ def validate_deploy_parameters(connection, deploy_key):
               f"underscore or a lowercase letter (a-z), followed by any combination of underscores, lowercase "
               f"letters, or digits (0-9). Uppercase characters are not allowed.", Logging.Level.SEVERE)
         os._exit(1)
+    if python_version and python_version not in SUPPORTED_PYTHON_VERSIONS:
+        print_library_log(f"This Python version is not supported: {python_version}. "
+                          f"We only support the following python versions: {SUPPORTED_PYTHON_VERSIONS} ", Logging.Level.SEVERE)
+        os._exit(1)
 
 def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
     """Logs a library message with the specified logging level.
@@ -514,9 +518,12 @@ def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
     """
     if DEBUGGING or EXECUTED_VIA_CLI:
         current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
-        print(f'{current_time} {level.name}: {message}')
+        escaped_message = json.dumps(message).strip('"')
+        print(f"{current_time} {level.name}: {escaped_message}")
     else:
-        print(f'{{"level":"{level.name}", "message": "{message}", "message_origin": "library"}}')
+        escaped_message = json.dumps(message)
+        log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "library"}}'
+        print(log_message)
 
 
 def is_port_in_use(port: int):
@@ -605,7 +612,16 @@ def transliterate(name):
     """
     if contains_only_word_dash_dot(name):
         return name
-    return translit(name, 'en', reversed=True)
+    # Step 1: Normalize the name to NFD form (decomposed form)
+    normalized_name = unicodedata.normalize('NFD', name)
+    # Step 2: Remove combining characters (diacritics, accents, etc.)
+    normalized_name = ''.join(char for char in normalized_name if not unicodedata.combining(char))
+    # Step 3: Normalize back to NFC form (composed form)
+    normalized_name = unicodedata.normalize('NFC', normalized_name)
+    # Step 4: Convert the string to ASCII using `unidecode` (removes any remaining non-ASCII characters)
+    normalized_name = unidecode(normalized_name)
+    # Step 5: Return the normalized name
+    return normalized_name
 
 
 def redshift_safe(name):
@@ -630,13 +646,19 @@ def safe_drop_underscores(name):
     return safe_name
 
 
-def for_table(source_table):
+def get_renamed_table_name(source_table):
     """
     Process a source table name to ensure it conforms to naming rules.
     """
     return safe_drop_underscores(source_table)
 
 
+def get_renamed_column_name(source_column):
+    """
+    Process a source column name to ensure it conforms to naming rules.
+    """
+    return redshift_safe(source_column)
+
 class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
     def __init__(self, update, schema=None):
         """Initializes the Connector instance.
@@ -703,20 +725,31 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             force (bool): Force update an existing connection.
 
         """
-        # Run the pipreqs command and capture stderr
-        result = subprocess.run(
-            ["pipreqs", "--savepath", "tmp_requirements.txt", "--ignore"] + EXCLUDED_PIPREQS_DIRS,
-            stderr=subprocess.PIPE,
-            text=True  # Ensures output is in string format
-        )
-
-        if result.returncode != 0:
-            print_library_log("pipreqs failed with:", Logging.Level.SEVERE)
-            print(result.stderr)
-            sys.exit(1)
-
         # tmp_requirements is only generated when pipreqs command is successful
         tmp_requirements_file_path = os.path.join(project_path, 'tmp_requirements.txt')
+        # Run the pipreqs command and capture stderr
+        attempt = 0
+        while attempt < MAX_RETRIES:
+            attempt += 1
+            result = subprocess.run(
+                ["pipreqs", project_path, "--savepath", tmp_requirements_file_path, "--ignore"] + EXCLUDED_PIPREQS_DIRS,
+                stderr=subprocess.PIPE,
+                text=True  # Ensures output is in string format
+            )
+
+            if result.returncode == 0:
+                break
+
+            print_library_log(f"Attempt {attempt}: pipreqs check failed.", Logging.Level.WARNING)
+
+            if attempt < MAX_RETRIES:
+                retry_after = 3 ** attempt
+                print_library_log(f"Retrying in {retry_after} seconds...", Logging.Level.SEVERE)
+                time.sleep(retry_after)
+            else:
+                print_library_log(f"pipreqs failed after {MAX_RETRIES} attempts with:", Logging.Level.SEVERE)
+                print(result.stderr)
+                sys.exit(1)
 
         tmp_requirements = self.fetch_requirements_as_dict(self, tmp_requirements_file_path)
         tmp_requirements.pop("fivetran_connector_sdk")
@@ -798,12 +831,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         return requirements
 
     # Call this method to deploy the connector to Fivetran platform
-    def deploy(self, project_path: str, force: bool, deploy_key: str, group: str, connection: str, configuration: dict = None):
+    def deploy(self, args: dict, deploy_key: str, group: str, connection: str, configuration: dict = None):
         """Deploys the connector to the Fivetran platform.
 
         Args:
-            project_path (str): The path to the connector project.
-            force (bool): Force update an existing connection.
+            args (dict): The command arguments.
             deploy_key (str): The deployment key.
             group (str): The group name.
             connection (str): The connection name.
@@ -814,7 +846,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
         print_library_log("We support only `.py` files and a `requirements.txt` file as part of the code upload. *No other code files* are supported or uploaded during the deployment process. Ensure that your code is structured accordingly and all dependencies are listed in `requirements.txt`")
 
-        validate_deploy_parameters(connection, deploy_key)
+        validate_deploy_parameters(connection, deploy_key, args.python_version)
 
         _check_dict(configuration, True)
 
@@ -827,10 +859,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             "schema": connection,
             "secrets_list": secrets_list,
             "sync_method": "DIRECT",
-            "custom_payloads": [],
+            "custom_payloads": []
         }
 
-        self.validate_requirements_file(project_path, True, force)
+        # args.python_version is already validated in validate_deploy_parameters - so its safe to add in connection_config
+        if args.python_version:
+            connection_config["python_version"] = args.python_version
+            print_library_log(f"Python version {args.python_version} to be used at runtime.", Logging.Level.INFO)
+
+        self.validate_requirements_file(args.project_path, True, args.force)
 
         group_id, group_name = self.__get_group_info(group, deploy_key)
         connection_id, service = self.__get_connection_id(connection, group, group_id, deploy_key) or (None, None)
@@ -841,17 +878,19 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                     f"The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.", Logging.Level.SEVERE)
                 os._exit(1)
             else:
-                if force:
+                if args.force:
                     confirm = "y"
                 else:
                     confirm = input(
-                        f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code and configuration. Do you want to proceed with the update? (Y/N): ")
-                if confirm.lower() == "y":
+                        f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code. Do you want to proceed with the update? (Y/N): ")
+                    if confirm.lower() == "y" and args.configuration:
+                        confirm_config = input(f"Your deploy will overwrite the configuration using the values provided in '{args.configuration}': key-value pairs not present in the new configuration will be removed; existing keys' values set in the cofiguration file or in the dashboard will be overwritten with new (empty or non-empty) values; new key-value pairs will be added. Do you want to proceed with the update? (Y/N): ")
+                if confirm.lower() == "y" and (not connection_config["secrets_list"] or (confirm_config.lower() == "y")):
                     print_library_log("Updating the connection...\n")
                     self.__upload_project(
-                        project_path, deploy_key, group_id, group_name, connection)
+                        args.project_path, deploy_key, group_id, group_name, connection)
                     self.__update_connection(
-                        connection_id, connection, group_name, connection_config, deploy_key)
+                        args, connection_id, connection, group_name, connection_config, deploy_key)
                     print("✓")
                     print_library_log(f"Connector ID: {connection_id}")
                     print_library_log(
@@ -860,7 +899,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                     print_library_log("Update canceled. The process is now terminating.")
                     os._exit(1)
         else:
-            self.__upload_project(project_path, deploy_key,
+            self.__upload_project(args.project_path, deploy_key,
                                   group_id, group_name, connection)
             response = self.__create_connection(
                 deploy_key, group_id, connection_config)
@@ -894,17 +933,18 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             os._exit(1)
 
     @staticmethod
-    def __update_connection(id: str, name: str, group: str, config: dict, deploy_key: str):
+    def __update_connection(args: dict, id: str, name: str, group: str, config: dict, deploy_key: str):
         """Updates the connection with the given ID, name, group, configuration, and deployment key.
 
         Args:
+            args (dict): The command arguments.
             id (str): The connection ID.
             name (str): The connection name.
             group (str): The group name.
             config (dict): The configuration dictionary.
             deploy_key (str): The deployment key.
         """
-        if not config["secrets_list"]:
+        if not args.configuration:
             del config["secrets_list"]
 
         resp = rq.patch(f"{PRODUCTION_BASE_URL}/v1/connectors/{id}",
@@ -1130,7 +1170,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
         if not resp.ok:
             print_library_log(
-                f"Unable to retrieve destination details. The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.", Logging.Level.SEVERE)
+                f"The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.", Logging.Level.SEVERE)
             os._exit(1)
 
         data = resp.json().get("data", {})
@@ -1203,7 +1243,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
               project_path: str = None,
               configuration: dict = None,
               state: dict = None,
-              log_level: Logging.Level = Logging.Level.FINE) -> bool:
+              log_level: Logging.Level = Logging.Level.FINE):
         """Tests the connector code by running it with the connector tester.\n
         state.json docs: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithstatejsonfile\n
         configuration.json docs: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile
@@ -1213,9 +1253,6 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             configuration (dict): The configuration dictionary, same as configuration.json if present.
             state (dict): The state dictionary, same as state.json if present.
             log_level (Logging.Level): The logging level.
-
-        Returns:
-            bool: True if there was an error, False otherwise.
         """
         global DEBUGGING
         DEBUGGING = True
@@ -1412,7 +1449,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         else:
             try:
                 configuration = self.configuration if self.configuration else request.configuration
-                print_library_log("Initiating the 'schema' method call...", Logging.Level.WARNING)
+                print_library_log("Initiating the 'schema' method call...", Logging.Level.INFO)
                 response = self.schema_method(configuration)
                 self.process_tables(response)
                 return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
@@ -1428,7 +1465,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             if 'table' not in entry:
                 raise ValueError("Entry missing table name: " + entry)
 
-            table_name = for_table(entry['table'])
+            table_name = get_renamed_table_name(entry['table'])
 
             if table_name in TABLES:
                 raise ValueError("Table already defined: " + table_name)
@@ -1447,13 +1484,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     def process_primary_keys(self, columns, entry):
         for pkey_name in entry["primary_key"]:
-            column = columns[pkey_name] if pkey_name in columns else common_pb2.Column(name=pkey_name)
+            column_name = get_renamed_column_name(pkey_name)
+            column = columns[column_name] if column_name in columns else common_pb2.Column(name=column_name)
             column.primary_key = True
-            columns[pkey_name] = column
+            columns[column_name] = column
 
     def process_columns(self, columns, entry):
         for name, type in entry["columns"].items():
-            column = columns[name] if name in columns else common_pb2.Column(name=name)
+            column_name = get_renamed_column_name(name)
+            column = columns[column_name] if column_name in columns else common_pb2.Column(name=column_name)
 
             if isinstance(type, str):
                 self.process_data_type(column, type)
@@ -1471,7 +1510,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             if "primary_key" in entry and name in entry["primary_key"]:
                 column.primary_key = True
 
-            columns[name] = column
+            columns[column_name] = column
 
     def process_data_type(self, column, type):
         if type.upper() == "BOOLEAN":
@@ -1519,7 +1558,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         state = self.state if self.state else json.loads(request.state_json)
 
         try:
-            print_library_log("Initiating the 'update' method call...", Logging.Level.WARNING)
+            print_library_log("Initiating the 'update' method call...", Logging.Level.INFO)
             for resp in self.update_method(configuration=configuration, state=state):
                 if isinstance(resp, list):
                     for r in resp:
@@ -1636,7 +1675,8 @@ def main():
     global EXECUTED_VIA_CLI
     EXECUTED_VIA_CLI = True
 
-    parser = argparse.ArgumentParser(allow_abbrev=False)
+    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=True)
+    parser._option_string_actions["-h"].help = "Show this help message and exit"
 
     # Positional
     parser.add_argument("command", help="|".join(VALID_COMMANDS))
@@ -1645,10 +1685,11 @@ def main():
     # Optional (Not all of these are valid with every mutually exclusive option below)
     parser.add_argument("--state", type=str, default=None, help="Provide state as JSON string or file")
     parser.add_argument("--configuration", type=str, default=None, help="Provide secrets as JSON file")
-    parser.add_argument("--api-key", type=str, default=None, help="Provide api key for deployment to production")
+    parser.add_argument("--api-key", type=str, default=None, help="Provide your base64-encoded API key for deployment")
     parser.add_argument("--destination", type=str, default=None, help="Destination name (aka 'group name')")
     parser.add_argument("--connection", type=str, default=None, help="Connection name (aka 'destination schema')")
     parser.add_argument("-f", "--force", action="store_true", help="Force update an existing connection")
+    parser.add_argument("--python-version", "--python", type=str, help=f"Supported Python versions you can use: {SUPPORTED_PYTHON_VERSIONS}. Defaults to 3.12.8")
 
     args = parser.parse_args()
 
@@ -1677,7 +1718,7 @@ def main():
     if args.command.lower() == "deploy":
         if args.state:
             print_library_log("'state' parameter is not used for 'deploy' command", Logging.Level.WARNING)
-        connector_object.deploy(args.project_path, args.force, ft_deploy_key, ft_group, ft_connection, configuration)
+        connector_object.deploy(args, ft_deploy_key, ft_group, ft_connection, configuration)
 
     elif args.command.lower() == "debug":
         connector_object.debug(args.project_path, configuration, state)

{fivetran_connector_sdk-0.13.17.1 → fivetran_connector_sdk-0.13.30.1}/src/fivetran_connector_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: fivetran_connector_sdk
-Version: 0.13.17.1
+Version: 0.13.30.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -15,7 +15,7 @@ Requires-Dist: grpcio==1.60.1
 Requires-Dist: grpcio-tools==1.60.1
 Requires-Dist: requests==2.32.3
 Requires-Dist: pipreqs==0.5.0
-Requires-Dist: transliterate==1.10.2
+Requires-Dist: unidecode==1.3.8
 
 # **fivetran-connector-sdk**
 [![Downloads](https://static.pepy.tech/badge/fivetran-connector-sdk)](https://pepy.tech/project/fivetran-connector-sdk)

{fivetran_connector_sdk-0.13.17.1 → fivetran_connector_sdk-0.13.30.1}/src/fivetran_connector_sdk.egg-info/requires.txt

@@ -2,4 +2,4 @@ grpcio==1.60.1
 grpcio-tools==1.60.1
 requests==2.32.3
 pipreqs==0.5.0
-transliterate==1.10.2
+unidecode==1.3.8