fivetran-connector-sdk 1.4.5__tar.gz → 1.4.6__tar.gz

{fivetran_connector_sdk-1.4.5 → fivetran_connector_sdk-1.4.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.4.5
+Version: 1.4.6
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

fivetran_connector_sdk-1.4.6/src/fivetran_connector_sdk/__init__.py

@@ -0,0 +1,450 @@
+import os
+import sys
+import grpc
+import json
+import shutil
+import argparse
+import traceback
+import requests as rq
+from http import HTTPStatus
+from zipfile import ZipFile
+from concurrent import futures
+
+from fivetran_connector_sdk.protos import common_pb2
+from fivetran_connector_sdk.protos import connector_sdk_pb2
+from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
+
+from fivetran_connector_sdk.logger import Logging
+from fivetran_connector_sdk.operations import Operations
+from fivetran_connector_sdk import constants
+from fivetran_connector_sdk.constants import (
+    TESTER_VER, VERSION_FILENAME, UTF_8,
+    VALID_COMMANDS, DEFAULT_PYTHON_VERSION, SUPPORTED_PYTHON_VERSIONS, FIVETRAN_HD_AGENT_ID, TABLES
+)
+from fivetran_connector_sdk.helpers import (
+    print_library_log, reset_local_file_directory, find_connector_object, validate_and_load_configuration,
+    validate_and_load_state, get_input_from_cli, suggest_correct_command,
+)
+from fivetran_connector_sdk.connector_helper import (
+    validate_requirements_file, upload_project,
+    update_connection, are_setup_tests_failing,
+    validate_deploy_parameters, get_connection_id,
+    handle_failing_tests_message_and_exit, delete_file_if_exists,
+    create_connection, get_os_arch_suffix, get_group_info,
+    java_exe_helper, run_tester, process_tables,
+    update_base_url_if_required, exit_check,
+    get_available_port, tester_root_dir_helper,
+    check_dict, check_newer_version, cleanup_uploaded_project,
+)
+
+# Version format: <major_version>.<minor_version>.<patch_version>
+# (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
+__version__ = "1.4.6"
+TESTER_VERSION = TESTER_VER
+
+__all__ = [cls.__name__ for cls in [Logging, Operations]]
+
+class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
+    def __init__(self, update, schema=None):
+        """Initializes the Connector instance.
+        Args:
+            update: The update method.
+            schema: The schema method.
+        """
+
+        self.schema_method = schema
+        self.update_method = update
+
+        self.configuration = None
+        self.state = None
+
+        update_base_url_if_required()
+
+    # Call this method to deploy the connector to Fivetran platform
+    def deploy(self, args: dict, deploy_key: str, group: str, connection: str, hd_agent_id: str, configuration: dict = None):
+        """Deploys the connector to the Fivetran platform.
+
+        Args:
+            args (dict): The command arguments.
+            deploy_key (str): The deployment key.
+            group (str): The group name.
+            connection (str): The connection name.
+            hd_agent_id (str): The hybrid deployment agent ID within the Fivetran system.
+            configuration (dict): The configuration dictionary.
+        """
+        constants.EXECUTED_VIA_CLI = True
+
+        print_library_log("We support only `.py` files and a `requirements.txt` file as part of the code upload. *No other code files* are supported or uploaded during the deployment process. Ensure that your code is structured accordingly and all dependencies are listed in `requirements.txt`")
+
+        validate_deploy_parameters(connection, deploy_key)
+
+        check_dict(configuration, True)
+
+        secrets_list = []
+        if configuration:
+            for k, v in configuration.items():
+                secrets_list.append({"key": k, "value": v})
+
+        connection_config = {
+            "schema": connection,
+            "secrets_list": secrets_list,
+            "sync_method": "DIRECT"
+        }
+
+        if args.python_version:
+            connection_config["python_version"] = args.python_version
+
+        validate_requirements_file(args.project_path, True, __version__, args.force)
+
+        group_id, group_name = get_group_info(group, deploy_key)
+        connection_id, service = get_connection_id(connection, group, group_id, deploy_key) or (None, None)
+
+        if connection_id:
+            if service != 'connector_sdk':
+                print_library_log(
+                    f"The connection '{connection}' already exists and does not use the 'Connector SDK' service. You cannot update this connection.", Logging.Level.SEVERE)
+                os._exit(1)
+            else:
+                if args.force:
+                    confirm = "y"
+                    if args.configuration:
+                        confirm_config = "y"
+                else:
+                    confirm = input(
+                        f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code. Do you want to proceed with the update? (Y/N): ")
+                    if confirm.lower() == "y" and args.configuration:
+                        confirm_config = input(f"Your deploy will overwrite the configuration using the values provided in '{args.configuration}': key-value pairs not present in the new configuration will be removed; existing keys' values set in the cofiguration file or in the dashboard will be overwritten with new (empty or non-empty) values; new key-value pairs will be added. Do you want to proceed with the update? (Y/N): ")
+                if confirm.lower() == "y" and (not connection_config["secrets_list"] or (confirm_config.lower() == "y")):
+                    print_library_log("Updating the connection...\n")
+                    upload_project(
+                        args.project_path, deploy_key, group_id, group_name, connection)
+                    response = update_connection(
+                        args, connection_id, connection, group_name, connection_config, deploy_key, hd_agent_id)
+                    print("✓")
+                    print_library_log(f"Python version {response.json()['data']['config']['python_version']} to be used at runtime.",
+                                      Logging.Level.INFO)
+                    print_library_log(f"Connection ID: {connection_id}")
+                    print_library_log(
+                        f"Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
+                else:
+                    print_library_log("Update canceled. The process is now terminating.")
+                    os._exit(1)
+        else:
+            upload_project(args.project_path, deploy_key,
+                                  group_id, group_name, connection)
+            response = create_connection(
+                deploy_key, group_id, connection_config, hd_agent_id)
+            if response.ok and response.status_code == HTTPStatus.CREATED:
+                if are_setup_tests_failing(response):
+                    handle_failing_tests_message_and_exit(response, "The connection was created, but setup tests failed!")
+                else:
+                    print_library_log(
+                        f"The connection '{connection}' has been created successfully.\n")
+                    connection_id = response.json()['data']['id']
+                    print_library_log(f"Python version {response.json()['data']['config']['python_version']} to be used at runtime.",
+                                      Logging.Level.INFO)
+                    print_library_log(f"Connection ID: {connection_id}")
+                    print_library_log(
+                        f"Visit the Fivetran dashboard to start the initial sync: https://fivetran.com/dashboard/connectors/{connection_id}/status")
+            else:
+                print_library_log(
+                    f"Unable to create a new connection, failed with error: {response.json()['message']}", Logging.Level.SEVERE)
+                cleanup_uploaded_project(deploy_key,group_id, connection)
+                print_library_log("Please try again with the deploy command after resolving the issue!")
+                os._exit(1)
+
+    # Call this method to run the connector in production
+    def run(self,
+            port: int = 50051,
+            configuration: dict = None,
+            state: dict = None,
+            log_level: Logging.Level = Logging.Level.INFO) -> grpc.Server:
+        """Runs the connector server.
+
+        Args:
+            port (int): The port number to listen for incoming requests.
+            configuration (dict): The configuration dictionary.
+            state (dict): The state dictionary.
+            log_level (Logging.Level): The logging level.
+
+        Returns:
+            grpc.Server: The gRPC server instance.
+        """
+        self.configuration = check_dict(configuration, True)
+        self.state = check_dict(state)
+        Logging.LOG_LEVEL = log_level
+
+        if not constants.DEBUGGING:
+            print_library_log(f"Running on fivetran_connector_sdk: {__version__}")
+
+        server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+        connector_sdk_pb2_grpc.add_ConnectorServicer_to_server(self, server)
+        server.add_insecure_port("[::]:" + str(port))
+        server.start()
+        if constants.DEBUGGING:
+            return server
+        server.wait_for_termination()
+
+    # This method starts both the server and the local testing environment
+    def debug(self,
+              project_path: str = None,
+              configuration: dict = None,
+              state: dict = None,
+              log_level: Logging.Level = Logging.Level.FINE):
+        """Tests the connector code by running it with the connector tester.\n
+        state.json docs: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithstatejsonfile\n
+        configuration.json docs: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile
+
+        Args:
+            project_path (str): The path to the project.
+            configuration (dict): The configuration dictionary, same as configuration.json if present.
+            state (dict): The state dictionary, same as state.json if present.
+            log_level (Logging.Level): The logging level.
+        """
+        constants.DEBUGGING = True
+
+        check_newer_version(__version__)
+
+        Logging.LOG_LEVEL = log_level
+        os_arch_suffix = get_os_arch_suffix()
+        tester_root_dir = tester_root_dir_helper()
+        java_exe = java_exe_helper(tester_root_dir, os_arch_suffix)
+        install_tester = False
+        version_file = os.path.join(tester_root_dir, VERSION_FILENAME)
+        if os.path.isfile(version_file):
+            # Check version number & update if different
+            with open(version_file, 'r', encoding=UTF_8) as fi:
+                current_version = fi.readline()
+
+            if current_version != TESTER_VERSION:
+                shutil.rmtree(tester_root_dir)
+                install_tester = True
+        else:
+            install_tester = True
+
+        if install_tester:
+            os.makedirs(tester_root_dir, exist_ok=True)
+            download_filename = f"sdk-connector-tester-{os_arch_suffix}-{TESTER_VERSION}.zip"
+            download_filepath = os.path.join(tester_root_dir, download_filename)
+            try:
+                print_library_log(f"Downloading connector tester version: {TESTER_VERSION} ")
+                download_url = f"https://github.com/fivetran/fivetran_sdk_tools/releases/download/{TESTER_VERSION}/{download_filename}"
+                r = rq.get(download_url)
+                if r.ok:
+                    with open(download_filepath, 'wb') as fo:
+                        fo.write(r.content)
+                else:
+                    raise RuntimeError(
+                        f"\nSEVERE: Failed to download the connector tester. Please check your access permissions or "
+                        f"try again later ( status code: {r.status_code}), url: {download_url}")
+            except RuntimeError:
+                raise RuntimeError(
+                    f"SEVERE: Failed to download the connector tester. Error details: {traceback.format_exc()}")
+
+            try:
+                # unzip it
+                with ZipFile(download_filepath, 'r') as z_object:
+                    z_object.extractall(path=tester_root_dir)
+                # delete zip file
+                delete_file_if_exists(download_filepath)
+                # make java binary executable
+                import stat
+                st = os.stat(java_exe)
+                os.chmod(java_exe, st.st_mode | stat.S_IEXEC)
+                print("✓")
+            except:
+                shutil.rmtree(tester_root_dir)
+                raise RuntimeError(f"\nSEVERE: Failed to install the connector tester. Error details: {traceback.format_exc()}")
+
+        project_path = os.getcwd() if project_path is None else project_path
+        validate_requirements_file(project_path, False, __version__)
+        print_library_log(f"Debugging connector at: {project_path}")
+        available_port = get_available_port()
+        exit_check(project_path)
+
+        if available_port is None:
+            raise RuntimeError("SEVERE: Unable to allocate a port in the range 50051-50061. "
+                               "Please ensure a port is available and try again")
+
+        server = self.run(available_port, configuration, state, log_level=log_level)
+
+        # Uncomment this to run the tester manually
+        # server.wait_for_termination()
+
+        try:
+            print_library_log("Running connector tester...")
+            for log_msg in run_tester(java_exe, tester_root_dir, project_path, available_port, json.dumps(self.state), json.dumps(self.configuration)):
+                print(log_msg, end="")
+        except:
+            print(traceback.format_exc())
+        finally:
+            server.stop(grace=2.0)
+
+    # -- Methods below override ConnectorServicer methods
+    def ConfigurationForm(self, request, context):
+        """Overrides the ConfigurationForm method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Returns:
+            common_pb2.ConfigurationFormResponse: An empty configuration form response.
+        """
+        if not self.configuration:
+            self.configuration = {}
+
+        # Not going to use the tester's configuration file
+        return common_pb2.ConfigurationFormResponse()
+
+    def Test(self, request, context):
+        """Overrides the Test method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Returns:
+            None: As this method is not implemented.
+        """
+        return None
+
+    def Schema(self, request, context):
+        """Overrides the Schema method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Returns:
+            connector_sdk_pb2.SchemaResponse: The schema response.
+        """
+
+        table_list = {}
+
+        if not self.schema_method:
+            return connector_sdk_pb2.SchemaResponse(schema_response_not_supported=True)
+        else:
+            try:
+                configuration = self.configuration if self.configuration else request.configuration
+                print_library_log("Initiating the 'schema' method call...", Logging.Level.INFO)
+                response = self.schema_method(configuration)
+                process_tables(response, table_list)
+                return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
+
+            except Exception as e:
+                tb = traceback.format_exc()
+                error_message = f"Error: {str(e)}\n{tb}"
+                print_library_log(error_message, Logging.Level.SEVERE)
+                raise RuntimeError(error_message) from e
+
+    def Update(self, request, context):
+        """Overrides the Update method from ConnectorServicer.
+
+        Args:
+            request: The gRPC request.
+            context: The gRPC context.
+
+        Yields:
+            connector_sdk_pb2.UpdateResponse: The update response.
+        """
+        configuration = self.configuration if self.configuration else request.configuration
+        state = self.state if self.state else json.loads(request.state_json)
+
+        try:
+            print_library_log("Initiating the 'update' method call...", Logging.Level.INFO)
+            for resp in self.update_method(configuration=configuration, state=state):
+                if isinstance(resp, list):
+                    for r in resp:
+                        yield r
+                else:
+                    yield resp
+
+        except TypeError as e:
+            if str(e) != "'NoneType' object is not iterable":
+                raise e
+
+        except Exception as e:
+            tb = traceback.format_exc()
+            error_message = f"Error: {str(e)}\n{tb}"
+            print_library_log(error_message, Logging.Level.SEVERE)
+            raise RuntimeError(error_message) from e
+
+
+def main():
+    """The main entry point for the script.
+    Parses command line arguments and passes them to connector object methods
+    """
+
+    constants.EXECUTED_VIA_CLI = True
+
+    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=True)
+    parser._option_string_actions["-h"].help = "Show this help message and exit"
+
+    # Positional
+    parser.add_argument("command", help="|".join(VALID_COMMANDS))
+    parser.add_argument("project_path", nargs='?', default=os.getcwd(), help="Path to connector project directory")
+
+    # Optional (Not all of these are valid with every mutually exclusive option below)
+    parser.add_argument("--state", type=str, default=None, help="Provide state as JSON string or file")
+    parser.add_argument("--configuration", type=str, default=None, help="Provide secrets as JSON file")
+    parser.add_argument("--api-key", type=str, default=None, help="Provide your base64-encoded API key for deployment")
+    parser.add_argument("--destination", type=str, default=None, help="Destination name (aka 'group name')")
+    parser.add_argument("--connection", type=str, default=None, help="Connection name (aka 'destination schema')")
+    parser.add_argument("-f", "--force", action="store_true", help="Force update an existing connection")
+    parser.add_argument("--python-version", "--python", type=str, help=f"Supported Python versions you can use: {SUPPORTED_PYTHON_VERSIONS}. Defaults to {DEFAULT_PYTHON_VERSION}")
+    parser.add_argument("--hybrid-deployment-agent-id", type=str, help="The Hybrid Deployment agent within the Fivetran system. If nothing is passed, the default agent of the destination is used.")
+
+    args = parser.parse_args()
+
+    if args.command.lower() == "version":
+        print_library_log("fivetran_connector_sdk " + __version__)
+        return
+    elif args.command.lower() == "reset":
+        reset_local_file_directory(args)
+        return
+
+    connector_object = find_connector_object(args.project_path)
+
+    if not connector_object:
+        sys.exit(1)
+
+    # Process optional args
+    ft_group = args.destination if args.destination else None
+    ft_connection = args.connection if args.connection else None
+    ft_deploy_key = args.api_key if args.api_key else None
+    hd_agent_id = args.hybrid_deployment_agent_id if args.hybrid_deployment_agent_id else os.getenv(FIVETRAN_HD_AGENT_ID, None)
+    configuration = args.configuration if args.configuration else None
+    state = args.state if args.state else os.getenv('FIVETRAN_STATE', None)
+
+    configuration = validate_and_load_configuration(args, configuration)
+    state = validate_and_load_state(args, state)
+
+    FIVETRAN_API_KEY = os.getenv('FIVETRAN_API_KEY', None)
+    FIVETRAN_DESTINATION_NAME = os.getenv('FIVETRAN_DESTINATION_NAME', None)
+    FIVETRAN_CONNECTION_NAME = os.getenv('FIVETRAN_CONNECTION_NAME', None)
+
+    if args.command.lower() == "deploy":
+        if args.state:
+            print_library_log("'state' parameter is not used for 'deploy' command", Logging.Level.WARNING)
+
+        if not ft_deploy_key:
+            ft_deploy_key = get_input_from_cli("Please provide the API Key", FIVETRAN_API_KEY)
+
+        if not ft_group:
+            ft_group = get_input_from_cli("Please provide the destination", FIVETRAN_DESTINATION_NAME)
+
+        if not ft_connection:
+            ft_connection = get_input_from_cli("Please provide the connection name",FIVETRAN_CONNECTION_NAME)
+
+        connector_object.deploy(args, ft_deploy_key, ft_group, ft_connection, hd_agent_id, configuration)
+
+    elif args.command.lower() == "debug":
+        connector_object.debug(args.project_path, configuration, state)
+    else:
+        if not suggest_correct_command(args.command):
+            raise NotImplementedError(f"Invalid command: {args.command}, see `fivetran --help`")
+
+
+if __name__ == "__main__":
+    main()