fivetran-connector-sdk 0.12.12.1__tar.gz → 0.13.10.1__tar.gz

{fivetran_connector_sdk-0.12.12.1 → fivetran_connector_sdk-0.13.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: fivetran_connector_sdk
-Version: 0.12.12.1
+Version: 0.13.10.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-0.12.12.1 → fivetran_connector_sdk-0.13.10.1}/src/fivetran_connector_sdk/__init__.py

@@ -26,7 +26,8 @@ from fivetran_connector_sdk.protos import common_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
-__version__ = "0.12.12.1"
+# Version format: <major_version>.<MM>.<DD>.<iteration> (where MM is incremental value from Jan 2024)
+__version__ = "0.13.10.1"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
@@ -38,6 +39,7 @@ VERSION_FILENAME = "version.txt"
 UPLOAD_FILENAME = "code.zip"
 LAST_VERSION_CHECK_FILE = "_last_version_check"
 ROOT_LOCATION = ".ft_sdk_connector_tester"
+CONFIG_FILE = "_config.json"
 OUTPUT_FILES_DIR = "files"
 REQUIREMENTS_TXT = "requirements.txt"
 PYPI_PACKAGE_DETAILS_URL = "https://pypi.org/pypi/fivetran_connector_sdk/json"
@@ -57,6 +59,7 @@ COMMANDS_AND_SYNONYMS = {
 CONNECTION_SCHEMA_NAME_PATTERN = r'^[_a-z][_a-z0-9]*$'
 DEBUGGING = False
 EXECUTED_VIA_CLI = False
+PRODUCTION_BASE_URL = "https://api.fivetran.com"
 TABLES = {}
 
 JAVA_LONG_MAX_VALUE = 9223372036854775807
@@ -463,7 +466,6 @@ def log_unused_deps_error(package_name: str, version: str):
     print_library_log(f"Please remove `{package_name}` from requirements.txt."
           f" The latest version of `{package_name}` is always available when executing your code."
           f" Current version: {version}", Logging.Level.SEVERE)
-    os._exit(1)
 
 
 def validate_deploy_parameters(connection, deploy_key):
@@ -508,6 +510,18 @@ def get_available_port():
     return None
 
 
+def update_base_url_if_required():
+    config_file_path = os.path.join(_tester_root_dir(), CONFIG_FILE)
+    if os.path.isfile(config_file_path):
+        with open(config_file_path, 'r') as f:
+            data = json.load(f)
+            base_url = data.get('production_base_url')
+            if base_url is not None:
+                global PRODUCTION_BASE_URL
+                PRODUCTION_BASE_URL = base_url
+                print_library_log(f"Updating PRODUCTION_BASE_URL to: {base_url}")
+
+
 class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
     def __init__(self, update, schema=None):
         """Initializes the Connector instance.
@@ -522,6 +536,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         self.configuration = None
         self.state = None
 
+        update_base_url_if_required()
+
     @staticmethod
     def fetch_requirements_from_file(file_path: str) -> list[str]:
         """Reads a requirements file and returns a list of dependencies.
@@ -558,7 +574,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 print_library_log(f"Invalid requirement format: '{requirement}'", Logging.Level.SEVERE)
         return requirements_dict
 
-    def validate_requirements_file(self, project_path: str, is_deploy: bool):
+    def validate_requirements_file(self, project_path: str, is_deploy: bool, force: bool = False):
         """Validates the `requirements.txt` file against the project's actual dependencies.
 
         This method generates a temporary requirements file using `pipreqs`, compares
@@ -569,10 +585,22 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         Args:
             project_path (str): The path to the project directory containing the `requirements.txt`.
             is_deploy (bool): If `True`, the method will exit the process on critical errors.
+            force (bool): Force update an existing connection.
 
         """
-        subprocess.check_call(["pipreqs", "--savepath", "tmp_requirements.txt", "--ignore"] + EXCLUDED_PIPREQS_DIRS,
-                              stderr=subprocess.PIPE)
+        # Run the pipreqs command and capture stderr
+        result = subprocess.run(
+            ["pipreqs", "--savepath", "tmp_requirements.txt", "--ignore"] + EXCLUDED_PIPREQS_DIRS,
+            stderr=subprocess.PIPE,
+            text=True  # Ensures output is in string format
+        )
+
+        if result.returncode != 0:
+            print_library_log("pipreqs failed with:", Logging.Level.SEVERE)
+            print(result.stderr)
+            sys.exit(1)
+
+        # tmp_requirements is only generated when pipreqs command is successful
         tmp_requirements_file_path = os.path.join(project_path, 'tmp_requirements.txt')
 
         tmp_requirements = self.fetch_requirements_as_dict(self, tmp_requirements_file_path)
@@ -581,6 +609,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             tmp_requirements.pop("requests")
         os.remove(tmp_requirements_file_path)
 
+        # remove corrupt requirements listed by pipreqs
+        corrupt_requirements = [key for key in tmp_requirements if key.startswith("~")]
+        for requirement in corrupt_requirements:
+            del tmp_requirements[requirement]
+
         if len(tmp_requirements) > 0:
             requirements = self.load_or_add_requirements_file(project_path)
 
@@ -593,11 +626,26 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
             missing_deps = {key: tmp_requirements[key] for key in (tmp_requirements.keys() - requirements.keys())}
             if missing_deps:
-                self.handle_missing_deps(is_deploy, missing_deps)
+                self.handle_missing_deps(missing_deps)
 
             unused_deps = list(requirements.keys() - tmp_requirements.keys())
             if unused_deps:
                 self.handle_unused_deps(unused_deps)
+
+            if is_deploy and (version_mismatch_deps or missing_deps or unused_deps):
+                if force:
+                    confirm = "y"
+                else:
+                    confirm = input("We detected issues in your requirements.txt. "
+                                    "Would you like us to update it to reflect the necessary changes? (Y/N):")
+                if confirm.lower() == "y":
+                    with open(REQUIREMENTS_TXT, "w") as file:
+                        file.write("\n".join(tmp_requirements.values()))
+                    print_library_log(f"`{REQUIREMENTS_TXT}` has been updated successfully")
+                else:
+                    if missing_deps or 'fivetran_connector_sdk' in unused_deps or 'requests' in unused_deps:
+                        print_library_log(f"Please fix your {REQUIREMENTS_TXT} file to proceed with deployment.")
+                        os._exit(1)
         else:
             if os.path.exists(REQUIREMENTS_TXT):
                 print_library_log("`requirements.txt` is not required as no additional "
@@ -616,15 +664,13 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                   "they are not used or already installed. Please remove them from requirements.txt:")
             print(*unused_deps)
 
-    def handle_missing_deps(self, is_deploy, missing_deps):
+    def handle_missing_deps(self, missing_deps):
         print_library_log("Please include the following dependency libraries in requirements.txt, to be used by "
               "Fivetran production. "
               "For more information, please visit: "
               "https://fivetran.com/docs/connectors/connector-sdk/detailed-guide"
               "#workingwithrequirementstxtfile", Logging.Level.SEVERE)
         print(*list(missing_deps.values()))
-        if is_deploy:
-            os._exit(1)
 
     def load_or_add_requirements_file(self, project_path):
         if os.path.exists(REQUIREMENTS_TXT):
@@ -669,7 +715,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             "custom_payloads": [],
         }
 
-        self.validate_requirements_file(project_path, True)
+        self.validate_requirements_file(project_path, True, force)
 
         group_id, group_name = self.__get_group_info(group, deploy_key)
         connection_id, service = self.__get_connection_id(connection, group, group_id, deploy_key) or (None, None)
@@ -746,7 +792,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         if not config["secrets_list"]:
             del config["secrets_list"]
 
-        resp = rq.patch(f"https://api.fivetran.com/v1/connectors/{id}",
+        resp = rq.patch(f"{PRODUCTION_BASE_URL}/v1/connectors/{id}",
                         headers={"Authorization": f"Basic {deploy_key}"},
                         json={
                             "config": config,
@@ -771,7 +817,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         Returns:
             str: The connection ID, or None
         """
-        resp = rq.get(f"https://api.fivetran.com/v1/groups/{group_id}/connectors",
+        resp = rq.get(f"{PRODUCTION_BASE_URL}/v1/groups/{group_id}/connectors",
                       headers={"Authorization": f"Basic {deploy_key}"},
                       params={"schema": name})
         if not resp.ok:
@@ -796,7 +842,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         Returns:
             rq.Response: The response object.
         """
-        response = rq.post(f"https://api.fivetran.com/v1/connectors",
+        response = rq.post(f"{PRODUCTION_BASE_URL}/v1/connectors",
                            headers={"Authorization": f"Basic {deploy_key}"},
                            json={
                                "group_id": group_id,
@@ -888,7 +934,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             bool: True if the upload was successful, False otherwise.
         """
         print_library_log("Uploading your project...")
-        response = rq.post(f"https://api.fivetran.com/v1/deploy/{group_id}/{connection}",
+        response = rq.post(f"{PRODUCTION_BASE_URL}/v1/deploy/{group_id}/{connection}",
                            files={'file': open(local_path, 'rb')},
                            headers={"Authorization": f"Basic {deploy_key}"})
         if response.ok:
@@ -911,7 +957,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             bool: True if the cleanup was successful, False otherwise.
         """
         print_library_log("INFO: Cleaning up your uploaded project ")
-        response = rq.post(f"https://api.fivetran.com/v1/cleanup_code/{group_id}/{connection}",
+        response = rq.post(f"{PRODUCTION_BASE_URL}/v1/cleanup_code/{group_id}/{connection}",
                            headers={"Authorization": f"Basic {deploy_key}"})
         if response.ok:
             print("✓")
@@ -947,7 +993,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         Returns:
             tuple[str, str]: A tuple containing the group ID and group name.
         """
-        groups_url = "https://api.fivetran.com/v1/groups"
+        groups_url = f"{PRODUCTION_BASE_URL}/v1/groups"
 
         params = {"limit": 500}
         headers = {"Authorization": f"Basic {deploy_key}"}
@@ -1096,8 +1142,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 print("✓")
             except:
                 shutil.rmtree(tester_root_dir)
-                raise RuntimeError(f"\nSEVERE: Failed to install the connector tester. Error details: ",
-                                   traceback.format_exc())
+                raise RuntimeError(f"\nSEVERE: Failed to install the connector tester. Error details: {traceback.format_exc()}")
 
         project_path = os.getcwd() if project_path is None else project_path
         self.validate_requirements_file(project_path, False)
@@ -1114,7 +1159,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         # server.wait_for_termination()
 
         try:
-            print_library_log(f"Running connector tester...")
+            print_library_log("Running connector tester...")
             for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, available_port):
                 print(log_msg, end="")
         except:
@@ -1238,6 +1283,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         else:
             try:
                 configuration = self.configuration if self.configuration else request.configuration
+                print_library_log("Initiating the 'schema' method call...", Logging.Level.WARNING)
                 response = self.schema_method(configuration)
                 self.process_tables(response)
                 return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
@@ -1245,6 +1291,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             except Exception as e:
                 tb = traceback.format_exc()
                 error_message = f"Error: {str(e)}\n{tb}"
+                print_library_log(error_message, Logging.Level.SEVERE)
                 raise RuntimeError(error_message) from e
 
     def process_tables(self, response):
@@ -1343,6 +1390,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         state = self.state if self.state else json.loads(request.state_json)
 
         try:
+            print_library_log("Initiating the 'update' method call...", Logging.Level.WARNING)
             for resp in self.update_method(configuration=configuration, state=state):
                 if isinstance(resp, list):
                     for r in resp:
@@ -1357,6 +1405,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         except Exception as e:
             tb = traceback.format_exc()
             error_message = f"Error: {str(e)}\n{tb}"
+            print_library_log(error_message, Logging.Level.SEVERE)
             raise RuntimeError(error_message) from e
 
 
@@ -1372,19 +1421,24 @@ def find_connector_object(project_path) -> Connector:
     sys.path.append(project_path) # Allows python interpreter to search for modules in this path
     module_name = "connector_connector_code"
     connector_py = os.path.join(project_path, "connector.py")
-    spec = importlib.util.spec_from_file_location(module_name, connector_py)
-    module = importlib.util.module_from_spec(spec)
-    sys.modules[module_name] = module
-    spec.loader.exec_module(module)
-    for obj in dir(module):
-        if not obj.startswith('__'):  # Exclude built-in attributes
-            obj_attr = getattr(module, obj)
-            if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
-                return obj_attr
+    try:
+        spec = importlib.util.spec_from_file_location(module_name, connector_py)
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+        spec.loader.exec_module(module)
+        for obj in dir(module):
+            if not obj.startswith('__'):  # Exclude built-in attributes
+                obj_attr = getattr(module, obj)
+                if '<fivetran_connector_sdk.Connector object at' in str(obj_attr):
+                    return obj_attr
+    except FileNotFoundError:
+        print_library_log(
+            "The connector object is missing in the current directory. Please ensure that you are running the command from correct directory or that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
+        return
 
     print_library_log(
         "The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
-    sys.exit(1)
+    return
 
 
 def suggest_correct_command(input_command: str) -> bool:
@@ -1475,6 +1529,9 @@ def main():
 
     connector_object = find_connector_object(args.project_path)
 
+    if not connector_object:
+        sys.exit(1)
+
     # Process optional args
     ft_group = args.destination if args.destination else os.getenv('FIVETRAN_DESTINATION_NAME', None)
     ft_connection = args.connection if args.connection else os.getenv('FIVETRAN_CONNECTION_NAME', None)

{fivetran_connector_sdk-0.12.12.1 → fivetran_connector_sdk-0.13.10.1}/src/fivetran_connector_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: fivetran_connector_sdk
-Version: 0.12.12.1
+Version: 0.13.10.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk