fivetran-connector-sdk 1.4.5__py3-none-any.whl → 1.5.0__py3-none-any.whl

fivetran_connector_sdk/constants.py

@@ -0,0 +1,70 @@
+import re
+
+TESTER_VER = "0.25.0521.001"
+
+WIN_OS = "windows"
+ARM_64 = "arm64"
+X64 = "x64"
+
+OS_MAP = {
+    "darwin": "mac",
+    "linux": "linux",
+    WIN_OS: WIN_OS
+}
+
+ARCH_MAP = {
+    "x86_64": X64,
+    "amd64": X64,
+    ARM_64: ARM_64,
+    "aarch64": ARM_64
+}
+
+# Global constants - use constants.<Global_variable> to access them as they can be overridden in the
+DEBUGGING = False
+EXECUTED_VIA_CLI = False
+TABLES = {}
+
+TESTER_FILENAME = "run_sdk_tester.jar"
+VERSION_FILENAME = "version.txt"
+UPLOAD_FILENAME = "code.zip"
+LAST_VERSION_CHECK_FILE = "_last_version_check"
+ROOT_LOCATION = ".ft_sdk_connector_tester"
+CONFIG_FILE = "_config.json"
+OUTPUT_FILES_DIR = "files"
+REQUIREMENTS_TXT = "requirements.txt"
+PYPI_PACKAGE_DETAILS_URL = "https://pypi.org/pypi/fivetran_connector_sdk/json"
+ONE_DAY_IN_SEC = 24 * 60 * 60
+MAX_RETRIES = 3
+LOGGING_PREFIX = "Fivetran-Connector-SDK"
+LOGGING_DELIMITER = ": "
+VIRTUAL_ENV_CONFIG = "pyvenv.cfg"
+ROOT_FILENAME = "connector.py"
+
+# Compile patterns used in the implementation
+WORD_DASH_DOT_PATTERN = re.compile(r'^[\w.-]*$')
+NON_WORD_PATTERN = re.compile(r'\W')
+WORD_OR_DOLLAR_PATTERN = re.compile(r'[\w$]')
+DROP_LEADING_UNDERSCORE = re.compile(r'_+([a-zA-Z]\w*)')
+WORD_PATTERN = re.compile(r'\w')
+
+EXCLUDED_DIRS = ["__pycache__", "lib", "include", OUTPUT_FILES_DIR]
+EXCLUDED_PIPREQS_DIRS = ["bin,etc,include,lib,Lib,lib64,Scripts,share"]
+VALID_COMMANDS = ["debug", "deploy", "reset", "version"]
+MAX_ALLOWED_EDIT_DISTANCE_FROM_VALID_COMMAND = 3
+COMMANDS_AND_SYNONYMS = {
+    "debug": {"test", "verify", "diagnose", "check"},
+    "deploy": {"upload", "ship", "launch", "release"},
+    "reset": {"reinitialize", "reinitialise", "re-initialize", "re-initialise", "restart", "restore"},
+}
+
+CONNECTION_SCHEMA_NAME_PATTERN = r'^[_a-z][_a-z0-9]*$'
+PRODUCTION_BASE_URL = "https://api.fivetran.com"
+INSTALLATION_SCRIPT_MISSING_MESSAGE = "The 'installation.sh' file is missing in the 'drivers' directory. Please ensure that 'installation.sh' is present to properly configure drivers."
+INSTALLATION_SCRIPT = "installation.sh"
+DRIVERS = "drivers"
+JAVA_LONG_MAX_VALUE = 9223372036854775807
+MAX_CONFIG_FIELDS = 100
+SUPPORTED_PYTHON_VERSIONS = ["3.12", "3.11", "3.10", "3.9"]
+DEFAULT_PYTHON_VERSION = "3.12"
+FIVETRAN_HD_AGENT_ID = "FIVETRAN_HD_AGENT_ID"
+UTF_8 = "utf-8"

fivetran_connector_sdk/helpers.py

@@ -0,0 +1,321 @@
+import re
+import os
+import shutil
+import sys
+import json
+import unicodedata
+import importlib.util
+from datetime import datetime
+from unidecode import unidecode
+
+from fivetran_connector_sdk.logger import Logging
+from fivetran_connector_sdk import constants
+from fivetran_connector_sdk.constants import (
+    LOGGING_PREFIX,
+    LOGGING_DELIMITER,
+    WORD_DASH_DOT_PATTERN,
+    NON_WORD_PATTERN,
+    WORD_OR_DOLLAR_PATTERN,
+    DROP_LEADING_UNDERSCORE,
+    WORD_PATTERN,
+    ROOT_FILENAME,
+    MAX_CONFIG_FIELDS,
+    MAX_ALLOWED_EDIT_DISTANCE_FROM_VALID_COMMAND,
+    COMMANDS_AND_SYNONYMS,
+    VALID_COMMANDS,
+    OUTPUT_FILES_DIR, UTF_8
+)
+
+RENAMED_TABLE_NAMES = {}
+RENAMED_COL_NAMES = {}
+
+def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
+    """Logs a library message with the specified logging level.
+
+    Args:
+        level (Logging.Level): The logging level.
+        message (str): The message to log.
+    """
+    if constants.DEBUGGING or constants.EXECUTED_VIA_CLI:
+        current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
+        print(f"{Logging.get_color(level)}{current_time} {level.name} {LOGGING_PREFIX}: {message} {Logging.reset_color()}")
+    else:
+        escaped_message = json.dumps(LOGGING_PREFIX + LOGGING_DELIMITER + message)
+        log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "library"}}'
+        print(log_message)
+
+def is_special(c):
+    """Check if the character is a special character."""
+    return not WORD_OR_DOLLAR_PATTERN.fullmatch(c)
+
+
+def starts_word(previous, current):
+    """
+    Check if the current character starts a new word based on the previous character.
+    """
+    return (previous and previous.islower() and current.isupper()) or (
+            previous and previous.isdigit() != current.isdigit()
+    )
+
+
+def underscore_invalid_leading_character(name, valid_leading_regex):
+    """
+    Ensure the name starts with a valid leading character.
+    """
+    if name and not valid_leading_regex.match(name[0]):
+        name = f'_{name}'
+    return name
+
+
+def single_underscore_case(name):
+    """
+    Convert the input name to single underscore case, replacing special characters and spaces.
+    """
+    acc = []
+    previous = None
+
+    for char_index, c in enumerate(name):
+        if char_index == 0 and c == '$':
+            acc.append('_')
+        elif is_special(c):
+            acc.append('_')
+        elif c == ' ':
+            acc.append('_')
+        elif starts_word(previous, c):
+            acc.append('_')
+            acc.append(c.lower())
+        else:
+            acc.append(c.lower())
+
+        previous = c
+
+    name = ''.join(acc)
+    return re.sub(r'_+', '_', name)
+
+
+def contains_only_word_dash_dot(name):
+    """
+    Check if the name contains only word characters, dashes, and dots.
+    """
+    return bool(WORD_DASH_DOT_PATTERN.fullmatch(name))
+
+
+def transliterate(name):
+    """
+    Transliterate the input name if it contains non-word, dash, or dot characters.
+    """
+    if contains_only_word_dash_dot(name):
+        return name
+    # Step 1: Normalize the name to NFD form (decomposed form)
+    normalized_name = unicodedata.normalize('NFD', name)
+    # Step 2: Remove combining characters (diacritics, accents, etc.)
+    normalized_name = ''.join(char for char in normalized_name if not unicodedata.combining(char))
+    # Step 3: Normalize back to NFC form (composed form)
+    normalized_name = unicodedata.normalize('NFC', normalized_name)
+    # Step 4: Convert the string to ASCII using `unidecode` (removes any remaining non-ASCII characters)
+    normalized_name = unidecode(normalized_name)
+    # Step 5: Return the normalized name
+    return normalized_name
+
+
+def redshift_safe(name):
+    """
+    Make the name safe for use in Redshift.
+    """
+    name = transliterate(name)
+    name = NON_WORD_PATTERN.sub('_', name)
+    name = single_underscore_case(name)
+    name = underscore_invalid_leading_character(name, WORD_PATTERN)
+    return name
+
+
+def safe_drop_underscores(name):
+    """
+    Drop leading underscores if the name starts with valid characters after sanitization.
+    """
+    safe_name = redshift_safe(name)
+    match = DROP_LEADING_UNDERSCORE.match(safe_name)
+    if match:
+        return match.group(1)
+    return safe_name
+
+
+def get_renamed_table_name(source_table):
+    """
+    Process a source table name to ensure it conforms to naming rules.
+    """
+    if source_table not in RENAMED_TABLE_NAMES:
+        RENAMED_TABLE_NAMES[source_table] = safe_drop_underscores(source_table)
+
+    return RENAMED_TABLE_NAMES[source_table]
+
+
+def get_renamed_column_name(source_column):
+    """
+    Process a source column name to ensure it conforms to naming rules.
+    """
+    if source_column not in RENAMED_COL_NAMES:
+        RENAMED_COL_NAMES[source_column] = redshift_safe(source_column)
+
+    return RENAMED_COL_NAMES[source_column]
+
+
+# Functions used by main method only
+
+def find_connector_object(project_path):
+    """Finds the connector object in the given project path.
+    Args:
+        project_path (str): The path to the project.
+    """
+
+    sys.path.append(project_path)  # Allows python interpreter to search for modules in this path
+    module_name = "connector_connector_code"
+    connector_py = os.path.join(project_path, ROOT_FILENAME)
+    try:
+        spec = importlib.util.spec_from_file_location(module_name, connector_py)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+        spec.loader.exec_module(module)
+        for obj in dir(module):
+            if not obj.startswith('__'):  # Exclude built-in attributes
+                obj_attr = getattr(module, obj)
+                if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
+                    return obj_attr
+    except FileNotFoundError:
+        print_library_log(
+            "The connector object is missing in the current directory. Please ensure that you are running the command from correct directory or that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector",
+            Logging.Level.SEVERE)
+        return None
+
+    print_library_log(
+        "The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector",
+        Logging.Level.SEVERE)
+    return None
+
+
+def suggest_correct_command(input_command: str) -> bool:
+    # for typos
+    # calculate the edit distance of the input command (lowercased) with each of the valid commands
+    edit_distances_of_commands = sorted(
+        [(command, edit_distance(command, input_command.lower())) for command in VALID_COMMANDS], key=lambda x: x[1])
+
+    if edit_distances_of_commands[0][1] <= MAX_ALLOWED_EDIT_DISTANCE_FROM_VALID_COMMAND:
+        # if the closest command is within the max allowed edit distance, we suggest that command
+        # threshold is kept to prevent suggesting a valid command for an obvious wrong command like `fivetran iknowthisisntacommandbuttryanyway`
+        print_suggested_command_message(edit_distances_of_commands[0][0], input_command)
+        return True
+
+    # for synonyms
+    for (command, synonyms) in COMMANDS_AND_SYNONYMS.items():
+        # check if the input command (lowercased) is a recognised synonym of the valid commands, if yes, suggest that command
+        if input_command.lower() in synonyms:
+            print_suggested_command_message(command, input_command)
+            return True
+
+    return False
+
+
+def print_suggested_command_message(valid_command: str, input_command: str) -> None:
+    print_library_log(f"`fivetran {input_command}` is not a valid command.", Logging.Level.SEVERE)
+    print_library_log(f"Did you mean `fivetran {valid_command}`?", Logging.Level.SEVERE)
+    print_library_log("Use `fivetran --help` for more details.", Logging.Level.SEVERE)
+
+
+def edit_distance(first_string: str, second_string: str) -> int:
+    first_string_length: int = len(first_string)
+    second_string_length: int = len(second_string)
+
+    # Initialize the previous row of distances (for the base case of an empty first string) 'previous_row[j]' holds
+    # the edit distance between an empty prefix of 'first_string' and the first 'j' characters of 'second_string'.
+    # The first row is filled with values [0, 1, 2, ..., second_string_length]
+    previous_row: list[int] = list(range(second_string_length + 1))
+
+    # Rest of the rows
+    for first_string_index in range(1, first_string_length + 1):
+        # Start the current row with the distance for an empty second string
+        current_row: list[int] = [first_string_index]
+
+        # Iterate over each character in the second string
+        for second_string_index in range(1, second_string_length + 1):
+            if first_string[first_string_index - 1] == second_string[second_string_index - 1]:
+                # If characters match, no additional cost
+                current_row.append(previous_row[second_string_index - 1])
+            else:
+                # Minimum cost of insertion, deletion, or substitution
+                current_row.append(
+                    1 + min(current_row[-1], previous_row[second_string_index], previous_row[second_string_index - 1]))
+
+        # Move to the next row
+        previous_row = current_row
+
+    # The last value in the last row is the edit distance
+    return previous_row[second_string_length]
+
+
+def get_input_from_cli(prompt: str, default_value: str) -> str:
+    """
+    Prompts the user for input.
+    """
+    if default_value:
+        value = input(f"{prompt} [Default : {default_value}]: ").strip() or default_value
+    else:
+        value = input(f"{prompt}: ").strip()
+
+    if not value:
+        raise ValueError("Missing required input: Expected a value but received None")
+    return value
+
+def validate_and_load_configuration(args, configuration):
+    if configuration:
+        json_filepath = os.path.join(args.project_path, args.configuration)
+        if os.path.isfile(json_filepath):
+            with open(json_filepath, 'r', encoding=UTF_8) as fi:
+                configuration = json.load(fi)
+            if len(configuration) > MAX_CONFIG_FIELDS:
+                raise ValueError(f"Configuration field count exceeds maximum of {MAX_CONFIG_FIELDS}. Reduce the field count.")
+        else:
+            raise ValueError(
+                "Configuration must be provided as a JSON file. Please check your input. Reference: "
+                "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile")
+    else:
+        json_filepath = os.path.join(args.project_path, "configuration.json")
+        if os.path.exists(json_filepath):
+            print_library_log("Configuration file detected in the project, but no configuration input provided via the command line", Logging.Level.WARNING)
+        configuration = {}
+    return configuration
+
+
+def validate_and_load_state(args, state):
+    if state:
+        json_filepath = os.path.join(args.project_path, args.state)
+    else:
+        json_filepath = os.path.join(args.project_path, "files", "state.json")
+
+    if os.path.exists(json_filepath):
+        if os.path.isfile(json_filepath):
+            with open(json_filepath, 'r', encoding=UTF_8) as fi:
+                state = json.load(fi)
+        elif state.lstrip().startswith("{"):
+            state = json.loads(state)
+    else:
+        state = {}
+    return state
+
+
+def reset_local_file_directory(args):
+    files_path = os.path.join(args.project_path, OUTPUT_FILES_DIR)
+    if args.force:
+        confirm = "y"
+    else:
+        confirm = input(
+            "This will delete your current state and `warehouse.db` files. Do you want to continue? (Y/N): ")
+    if confirm.lower() != "y":
+        print_library_log("Reset canceled")
+    else:
+        try:
+            if os.path.exists(files_path) and os.path.isdir(files_path):
+                shutil.rmtree(files_path)
+            print_library_log("Reset Successful")
+        except Exception as e:
+            print_library_log("Reset Failed", Logging.Level.SEVERE)
+            raise e

fivetran_connector_sdk/logger.py

@@ -0,0 +1,96 @@
+import json
+import traceback
+from enum import IntEnum
+from datetime import datetime
+
+from fivetran_connector_sdk import constants
+
+class Logging:
+    class Level(IntEnum):
+        FINE = 1
+        INFO = 2
+        WARNING = 3
+        SEVERE = 4
+
+    LOG_LEVEL = None
+
+    @staticmethod
+    def __log(level: Level, message: str):
+        """Logs a message with the specified logging level.
+
+        Args:
+            level (Logging.Level): The logging level.
+            message (str): The message to log.
+        """
+        if constants.DEBUGGING:
+            current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
+            prefix = f"{current_time} {level.name}: "
+            message = Logging.get_formatted_log(message, prefix)
+            print(f"{Logging.get_color(level)}{prefix}{message} {Logging.reset_color()}")
+        else:
+            escaped_message = json.dumps(message)
+            log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "connector_sdk"}}'
+            print(log_message)
+
+    @staticmethod
+    def get_formatted_log(message, prefix):
+        lines = message.split('\n')
+        padding = "\n" + " " * len(prefix)
+        return padding.join(lines)
+
+    @staticmethod
+    def get_color(level):
+        if level == Logging.Level.WARNING:
+            return "\033[93m"  # Yellow
+        elif level == Logging.Level.SEVERE:
+            return "\033[91m"  # Red
+        return ""
+
+    @staticmethod
+    def reset_color():
+        return "\033[0m"
+
+    @staticmethod
+    def fine(message: str):
+        """Logs a fine-level message.
+
+        Args:
+            message (str): The message to log.
+        """
+        if constants.DEBUGGING and Logging.LOG_LEVEL == Logging.Level.FINE:
+            Logging.__log(Logging.Level.FINE, message)
+
+    @staticmethod
+    def info(message: str):
+        """Logs an info-level message.
+
+        Args:
+            message (str): The message to log.
+        """
+        if Logging.LOG_LEVEL <= Logging.Level.INFO:
+            Logging.__log(Logging.Level.INFO, message)
+
+    @staticmethod
+    def warning(message: str):
+        """Logs a warning-level message.
+
+        Args:
+            message (str): The message to log.
+        """
+        if Logging.LOG_LEVEL <= Logging.Level.WARNING:
+            Logging.__log(Logging.Level.WARNING, message)
+
+    @staticmethod
+    def severe(message: str, exception: Exception = None):
+        """Logs a severe-level message.
+
+        Args:
+            message (str): The message to log.
+            exception (Exception, optional): Exception to be logged if provided.
+        """
+        if Logging.LOG_LEVEL <= Logging.Level.SEVERE:
+            if exception:
+                exc_type, exc_value, exc_traceback = type(exception), exception, exception.__traceback__
+                tb_str = "".join(traceback.format_exception(exc_type, exc_value, exc_traceback, limit=1))
+                message += "\n" + tb_str
+            Logging.__log(Logging.Level.SEVERE, message)