fivetran-connector-sdk 0.12.17.1__py3-none-any.whl → 0.13.16.2__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -6,6 +6,7 @@ import importlib.util
 import inspect
 import json
 import os
+from transliterate import translit
 import platform
 import requests as rq
 import shutil
@@ -26,13 +27,14 @@ from fivetran_connector_sdk.protos import common_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
-__version__ = "0.12.17.1"
+# Version format: <major_version>.<MM>.<DD>.<iteration> (where MM is incremental value from Jan 2024)
+__version__ = "0.13.16.2"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
 LINUX_OS = "linux"
 
-TESTER_VERSION = "0.24.1209.001"
+TESTER_VERSION = "0.25.0116.001"
 TESTER_FILENAME = "run_sdk_tester.jar"
 VERSION_FILENAME = "version.txt"
 UPLOAD_FILENAME = "code.zip"
@@ -45,6 +47,18 @@ PYPI_PACKAGE_DETAILS_URL = "https://pypi.org/pypi/fivetran_connector_sdk/json"
 ONE_DAY_IN_SEC = 24 * 60 * 60
 MAX_RETRIES = 3
 
+# Compile patterns used in the implementation
+WORD_DASH_DOT_PATTERN = re.compile(r'^[\w.-]*$')
+NON_WORD_PATTERN = re.compile(r'\W')
+WORD_OR_DOLLAR_PATTERN = re.compile(r'[\w$]')
+DROP_LEADING_UNDERSCORE = re.compile(r'_+([a-zA-Z]\w*)')
+WORD_PATTERN = re.compile(r'\w')
+
+# Transliteration rules
+TO_ASCII_RULES = (
+    "Han-Latin; Katakana-Latin; Arabic-Latin; NFD; [:Nonspacing Mark:] Remove; NFC"
+)
+
 EXCLUDED_DIRS = ["__pycache__", "lib", "include", OUTPUT_FILES_DIR]
 EXCLUDED_PIPREQS_DIRS = ["bin,etc,include,lib,Lib,lib64,Scripts,share"]
 VALID_COMMANDS = ["debug", "deploy", "reset", "version"]
@@ -60,7 +74,9 @@ DEBUGGING = False
 EXECUTED_VIA_CLI = False
 PRODUCTION_BASE_URL = "https://api.fivetran.com"
 TABLES = {}
-
+INSTALLATION_SCRIPT_MISSING_MESSAGE = "The 'installation.sh' file is missing in the 'drivers' directory. Please ensure that 'installation.sh' is present to properly configure drivers."
+INSTALLATION_SCRIPT = "installation.sh"
+DRIVERS = "drivers"
 JAVA_LONG_MAX_VALUE = 9223372036854775807
 MAX_CONFIG_FIELDS = 100
 
@@ -84,9 +100,12 @@ class Logging:
         """
         if DEBUGGING:
             current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
-            print(f"{current_time} {level.name}: {message}")
+            escaped_message = json.dumps(message) [1:-1]
+            print(f"{current_time} {level.name}: {escaped_message}")
         else:
-            print(f'{{"level":"{level.name}", "message": "{message}", "message_origin": "connector_sdk"}}')
+            escaped_message = json.dumps(message)
+            log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "connector_sdk"}}'
+            print(log_message)
 
     @staticmethod
     def fine(message: str):
@@ -145,6 +164,7 @@ class Operations:
 
         responses = []
 
+        table = for_table(table)
         columns = _get_columns(table)
         if not columns:
             global TABLES
@@ -179,6 +199,7 @@ class Operations:
         """
         _yield_check(inspect.stack())
 
+        table = for_table(table)
         columns = _get_columns(table)
         mapped_data = _map_data_to_columns(modified, columns)
         record = connector_sdk_pb2.Record(
@@ -204,6 +225,7 @@ class Operations:
         """
         _yield_check(inspect.stack())
 
+        table = for_table(table)
         columns = _get_columns(table)
         mapped_data = _map_data_to_columns(keys, columns)
         record = connector_sdk_pb2.Record(
@@ -465,7 +487,6 @@ def log_unused_deps_error(package_name: str, version: str):
     print_library_log(f"Please remove `{package_name}` from requirements.txt."
           f" The latest version of `{package_name}` is always available when executing your code."
           f" Current version: {version}", Logging.Level.SEVERE)
-    os._exit(1)
 
 
 def validate_deploy_parameters(connection, deploy_key):
@@ -522,6 +543,100 @@ def update_base_url_if_required():
                 print_library_log(f"Updating PRODUCTION_BASE_URL to: {base_url}")
 
 
+def is_special(c):
+    """Check if the character is a special character."""
+    return not WORD_OR_DOLLAR_PATTERN.fullmatch(c)
+
+
+def starts_word(previous, current):
+    """
+    Check if the current character starts a new word based on the previous character.
+    """
+    return (previous and previous.islower() and current.isupper()) or (
+            previous and previous.isdigit() != current.isdigit()
+    )
+
+
+def underscore_invalid_leading_character(name, valid_leading_regex):
+    """
+    Ensure the name starts with a valid leading character.
+    """
+    if name and not valid_leading_regex.match(name[0]):
+        name = f'_{name}'
+    return name
+
+
+def single_underscore_case(name):
+    """
+    Convert the input name to single underscore case, replacing special characters and spaces.
+    """
+    acc = []
+    previous = None
+
+    for char_index, c in enumerate(name):
+        if char_index == 0 and c == '$':
+            acc.append('_')
+        elif is_special(c):
+            acc.append('_')
+        elif c == ' ':
+            acc.append('_')
+        elif starts_word(previous, c):
+            acc.append('_')
+            acc.append(c.lower())
+        else:
+            acc.append(c.lower())
+
+        previous = c
+
+    name = ''.join(acc)
+    return re.sub(r'_+', '_', name)
+
+
+def contains_only_word_dash_dot(name):
+    """
+    Check if the name contains only word characters, dashes, and dots.
+    """
+    return bool(WORD_DASH_DOT_PATTERN.fullmatch(name))
+
+
+def transliterate(name):
+    """
+    Transliterate the input name if it contains non-word, dash, or dot characters.
+    """
+    if contains_only_word_dash_dot(name):
+        return name
+    return translit(name, 'en', reversed=True)
+
+
+def redshift_safe(name):
+    """
+    Make the name safe for use in Redshift.
+    """
+    name = transliterate(name)
+    name = NON_WORD_PATTERN.sub('_', name)
+    name = single_underscore_case(name)
+    name = underscore_invalid_leading_character(name, WORD_PATTERN)
+    return name
+
+
+def safe_drop_underscores(name):
+    """
+    Drop leading underscores if the name starts with valid characters after sanitization.
+    """
+    safe_name = redshift_safe(name)
+    match = DROP_LEADING_UNDERSCORE.match(safe_name)
+    if match:
+        return match.group(1)
+    return safe_name
+
+
+def for_table(source_table):
+    """
+    Process a source table name to ensure it conforms to naming rules.
+    """
+    return safe_drop_underscores(source_table)
+
+
 class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
     def __init__(self, update, schema=None):
         """Initializes the Connector instance.
@@ -574,7 +689,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 print_library_log(f"Invalid requirement format: '{requirement}'", Logging.Level.SEVERE)
         return requirements_dict
 
-    def validate_requirements_file(self, project_path: str, is_deploy: bool):
+    def validate_requirements_file(self, project_path: str, is_deploy: bool, force: bool = False):
         """Validates the `requirements.txt` file against the project's actual dependencies.
 
         This method generates a temporary requirements file using `pipreqs`, compares
@@ -585,10 +700,22 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         Args:
             project_path (str): The path to the project directory containing the `requirements.txt`.
             is_deploy (bool): If `True`, the method will exit the process on critical errors.
+            force (bool): Force update an existing connection.
 
         """
-        subprocess.check_call(["pipreqs", "--savepath", "tmp_requirements.txt", "--ignore"] + EXCLUDED_PIPREQS_DIRS,
-                              stderr=subprocess.PIPE)
+        # Run the pipreqs command and capture stderr
+        result = subprocess.run(
+            ["pipreqs", "--savepath", "tmp_requirements.txt", "--ignore"] + EXCLUDED_PIPREQS_DIRS,
+            stderr=subprocess.PIPE,
+            text=True  # Ensures output is in string format
+        )
+
+        if result.returncode != 0:
+            print_library_log("pipreqs failed with:", Logging.Level.SEVERE)
+            print(result.stderr)
+            sys.exit(1)
+
+        # tmp_requirements is only generated when pipreqs command is successful
         tmp_requirements_file_path = os.path.join(project_path, 'tmp_requirements.txt')
 
         tmp_requirements = self.fetch_requirements_as_dict(self, tmp_requirements_file_path)
@@ -597,6 +724,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             tmp_requirements.pop("requests")
         os.remove(tmp_requirements_file_path)
 
+        # remove corrupt requirements listed by pipreqs
+        corrupt_requirements = [key for key in tmp_requirements if key.startswith("~")]
+        for requirement in corrupt_requirements:
+            del tmp_requirements[requirement]
+
         if len(tmp_requirements) > 0:
             requirements = self.load_or_add_requirements_file(project_path)
 
@@ -609,11 +741,26 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
             missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
             if missing_deps:
-                self.handle_missing_deps(is_deploy, missing_deps)
+                self.handle_missing_deps(missing_deps)
 
             unused_deps = list(requirements.keys() - tmp_requirements.keys())
             if unused_deps:
                 self.handle_unused_deps(unused_deps)
+
+            if is_deploy and (version_mismatch_deps or missing_deps or unused_deps):
+                if force:
+                    confirm = "y"
+                else:
+                    confirm = input("We detected issues in your requirements.txt. "
+                                    "Would you like us to update it to reflect the necessary changes? (Y/N):")
+                if confirm.lower() == "y":
+                    with open(REQUIREMENTS_TXT, "w") as file:
+                        file.write("\n".join(tmp_requirements.values()))
+                    print_library_log(f"`{REQUIREMENTS_TXT}` has been updated successfully")
+                else:
+                    if missing_deps or 'fivetran_connector_sdk' in unused_deps or 'requests' in unused_deps:
+                        print_library_log(f"Please fix your {REQUIREMENTS_TXT} file to proceed with deployment.")
+                        os._exit(1)
         else:
             if os.path.exists(REQUIREMENTS_TXT):
                 print_library_log("`requirements.txt` is not required as no additional "
@@ -632,15 +779,13 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                   "they are not used or already installed. Please remove them from requirements.txt:")
             print(*unused_deps)
 
-    def handle_missing_deps(self, is_deploy, missing_deps):
+    def handle_missing_deps(self, missing_deps):
         print_library_log("Please include the following dependency libraries in requirements.txt, to be used by "
               "Fivetran production. "
               "For more information, please visit: "
               "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
               "#workingwithrequirementstxtfile", Logging.Level.SEVERE)
         print(*list(missing_deps.values()))
-        if is_deploy:
-            os._exit(1)
 
     def load_or_add_requirements_file(self, project_path):
         if os.path.exists(REQUIREMENTS_TXT):
@@ -685,7 +830,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             "custom_payloads": [],
         }
 
-        self.validate_requirements_file(project_path, True)
+        self.validate_requirements_file(project_path, True, force)
 
         group_id, group_name = self.__get_group_info(group, deploy_key)
         connection_id, service = self.__get_connection_id(connection, group, group_id, deploy_key) or (None, None)
@@ -849,9 +994,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         """
         upload_filepath = os.path.join(project_path, UPLOAD_FILENAME)
         connector_file_exists = False
+        custom_drivers_exists = False
+        custom_driver_installation_script_exists = False
 
         with ZipFile(upload_filepath, 'w', ZIP_DEFLATED) as zipf:
             for root, files in self.__dir_walker(project_path):
+                if os.path.basename(root) == DRIVERS:
+                    custom_drivers_exists = True
+                if INSTALLATION_SCRIPT in files:
+                    custom_driver_installation_script_exists = True
                 for file in files:
                     if file == "connector.py":
                         connector_file_exists = True
@@ -863,6 +1014,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             print_library_log(
                 "The 'connector.py' file is missing. Please ensure that 'connector.py' is present in your project directory, and that the file name is in lowercase letters. All custom connectors require this file because Fivetran calls it to start a sync.", Logging.Level.SEVERE)
             os._exit(1)
+
+        if custom_drivers_exists and not custom_driver_installation_script_exists:
+            print_library_log(INSTALLATION_SCRIPT_MISSING_MESSAGE, Logging.Level.SEVERE)
+            os._exit(1)
+
         return upload_filepath
 
     def __dir_walker(self, top):
@@ -881,6 +1037,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 if (name not in EXCLUDED_DIRS) and (not name.startswith(".")):
                     dirs.append(name)
             else:
+                # Include all files if in `drivers` folder
+                if os.path.basename(top) == DRIVERS:
+                    files.append(name)
                 if name.endswith(".py") or name == "requirements.txt":
                     files.append(name)
 
@@ -1253,6 +1412,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         else:
             try:
                 configuration = self.configuration if self.configuration else request.configuration
+                print_library_log("Initiating the 'schema' method call...", Logging.Level.WARNING)
                 response = self.schema_method(configuration)
                 self.process_tables(response)
                 return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
@@ -1260,6 +1420,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             except Exception as e:
                 tb = traceback.format_exc()
                 error_message = f"Error: {str(e)}\n{tb}"
+                print_library_log(error_message, Logging.Level.SEVERE)
                 raise RuntimeError(error_message) from e
 
     def process_tables(self, response):
@@ -1267,7 +1428,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             if 'table' not in entry:
                 raise ValueError("Entry missing table name: " + entry)
 
-            table_name = entry['table']
+            table_name = for_table(entry['table'])
 
             if table_name in TABLES:
                 raise ValueError("Table already defined: " + table_name)
@@ -1358,6 +1519,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         state = self.state if self.state else json.loads(request.state_json)
 
         try:
+            print_library_log("Initiating the 'update' method call...", Logging.Level.WARNING)
             for resp in self.update_method(configuration=configuration, state=state):
                 if isinstance(resp, list):
                     for r in resp:
@@ -1372,6 +1534,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         except Exception as e:
             tb = traceback.format_exc()
             error_message = f"Error: {str(e)}\n{tb}"
+            print_library_log(error_message, Logging.Level.SEVERE)
             raise RuntimeError(error_message) from e
 
 
@@ -1387,19 +1550,24 @@ def find_connector_object(project_path) -> Connector:
     sys.path.append(project_path) # Allows python interpreter to search for modules in this path
     module_name = "connector_connector_code"
     connector_py = os.path.join(project_path, "connector.py")
-    spec = importlib.util.spec_from_file_location(module_name, connector_py)
-    module = importlib.util.module_from_spec(spec)
-    sys.modules[module_name] = module
-    spec.loader.exec_module(module)
-    for obj in dir(module):
-        if not obj.startswith('__'):  # Exclude built-in attributes
-            obj_attr = getattr(module, obj)
-            if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
-                return obj_attr
+    try:
+        spec = importlib.util.spec_from_file_location(module_name, connector_py)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+        spec.loader.exec_module(module)
+        for obj in dir(module):
+            if not obj.startswith('__'):  # Exclude built-in attributes
+                obj_attr = getattr(module, obj)
+                if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
+                    return obj_attr
+    except FileNotFoundError:
+        print_library_log(
+            "The connector object is missing in the current directory. Please ensure that you are running the command from correct directory or that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
+        return
 
     print_library_log(
         "The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
-    sys.exit(1)
+    return
 
 
 def suggest_correct_command(input_command: str) -> bool:
@@ -1487,9 +1655,15 @@ def main():
     if args.command.lower() == "version":
         print_library_log("fivetran_connector_sdk " + __version__)
         return
+    elif args.command.lower() == "reset":
+        reset_local_file_directory(args)
+        return
 
     connector_object = find_connector_object(args.project_path)
 
+    if not connector_object:
+        sys.exit(1)
+
     # Process optional args
     ft_group = args.destination if args.destination else os.getenv('FIVETRAN_DESTINATION_NAME', None)
     ft_connection = args.connection if args.connection else os.getenv('FIVETRAN_CONNECTION_NAME', None)
@@ -1508,8 +1682,6 @@ def main():
     elif args.command.lower() == "debug":
         connector_object.debug(args.project_path, configuration, state)
 
-    elif args.command.lower() == "reset":
-        reset_local_file_directory(args)
     else:
         if not suggest_correct_command(args.command):
             raise NotImplementedError(f"Invalid command: {args.command}, see `fivetran --help`")

{fivetran_connector_sdk-0.12.17.1.dist-info → fivetran_connector_sdk-0.13.16.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: fivetran_connector_sdk
-Version: 0.12.17.1
+Version: 0.13.16.2
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -15,6 +15,7 @@ Requires-Dist: grpcio==1.60.1
 Requires-Dist: grpcio-tools==1.60.1
 Requires-Dist: requests==2.32.3
 Requires-Dist: pipreqs==0.5.0
+Requires-Dist: transliterate==1.10.2
 
 # **fivetran-connector-sdk**
 [![Downloads](https://static.pepy.tech/badge/fivetran-connector-sdk)](https://pepy.tech/project/fivetran-connector-sdk)

{fivetran_connector_sdk-0.12.17.1.dist-info → fivetran_connector_sdk-0.13.16.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fivetran_connector_sdk/__init__.py,sha256=DFQiZdtyqblI_7npsJzaZ7jicU0U1B5LXA1SZY-WeYI,63949
+fivetran_connector_sdk/__init__.py,sha256=t6bJFY6EMwS7HqbumKVCJq7bpkVBExpjmTGRp-gBafA,70538
 fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
 fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -6,8 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
 fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
 fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
 fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
-fivetran_connector_sdk-0.12.17.1.dist-info/METADATA,sha256=_ORD9BvuGvj63YvicMhGKKi9N7m7QuXyN1v0dQ3GDIw,2939
-fivetran_connector_sdk-0.12.17.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-fivetran_connector_sdk-0.12.17.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-0.12.17.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-0.12.17.1.dist-info/RECORD,,
+fivetran_connector_sdk-0.13.16.2.dist-info/METADATA,sha256=PKR_2EXCjVTfC2LusmArJqVEq8aVpzum4J0eXfm4lhs,2976
+fivetran_connector_sdk-0.13.16.2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+fivetran_connector_sdk-0.13.16.2.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-0.13.16.2.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-0.13.16.2.dist-info/RECORD,,

{fivetran_connector_sdk-0.12.17.1.dist-info → fivetran_connector_sdk-0.13.16.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any