fivetran-connector-sdk 1.3.3__tar.gz → 1.4.0__tar.gz

{fivetran_connector_sdk-1.3.3 → fivetran_connector_sdk-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.3.3
+Version: 1.4.0
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-1.3.3 → fivetran_connector_sdk-1.4.0}/src/fivetran_connector_sdk/__init__.py

@@ -32,13 +32,13 @@ from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
-__version__ = "1.3.3"
+__version__ = "1.4.0"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
 LINUX_OS = "linux"
 
-TESTER_VERSION = "0.25.0423.001"
+TESTER_VERSION = "0.25.0424.001"
 TESTER_FILENAME = "run_sdk_tester.jar"
 VERSION_FILENAME = "version.txt"
 UPLOAD_FILENAME = "code.zip"
@@ -110,12 +110,24 @@ class Logging:
         if DEBUGGING:
             current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
             escaped_message = json.dumps(message).strip('"')
-            print(f"{current_time} {level.name}: {escaped_message}")
+            print(f"{Logging._get_color(level)}{current_time} {level.name}: {escaped_message} {Logging._reset_color()}")
         else:
             escaped_message = json.dumps(message)
             log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "connector_sdk"}}'
             print(log_message)
 
+    @staticmethod
+    def _get_color(level):
+        if level == Logging.Level.WARNING:
+            return "\033[93m"  # Yellow
+        elif level == Logging.Level.SEVERE:
+            return "\033[91m"  # Red
+        return ""
+
+    @staticmethod
+    def _reset_color():
+        return "\033[0m"
+
     @staticmethod
     def fine(message: str):
         """Logs a fine-level message.
@@ -313,7 +325,7 @@ def check_newer_version():
             latest_version = data["info"]["version"]
             if __version__ < latest_version:
                 print_library_log(f"[notice] A new release of 'fivetran-connector-sdk' is available: {latest_version}\n" +
-                      f"[notice] To update, run: pip install --upgrade fivetran-connector-sdk\n")
+                      "[notice] To update, run: pip install --upgrade fivetran-connector-sdk\n")
 
             with open(last_check_file_path, 'w', encoding=UTF_8) as f_out:
                 f_out.write(f"{int(time.time())}")
@@ -573,7 +585,7 @@ def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
     if DEBUGGING or EXECUTED_VIA_CLI:
         current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
         escaped_message = json.dumps(message).strip('"')
-        print(f"{current_time} {level.name} {LOGGING_PREFIX}: {escaped_message}")
+        print(f"{Logging._get_color(level)}{current_time} {level.name} {LOGGING_PREFIX}: {escaped_message} {Logging._reset_color()}")
     else:
         escaped_message = json.dumps(LOGGING_PREFIX + LOGGING_DELIMITER + message)
         log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "library"}}'
@@ -794,6 +806,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
         # tmp_requirements is only generated when pipreqs command is successful
         tmp_requirements_file_path = os.path.join(project_path, 'tmp_requirements.txt')
+        # copying packages of requirements file to tmp file to handle pipreqs fail use-case
+        self.copy_requirements_file_to_tmp_requirements_file(os.path.join(project_path, REQUIREMENTS_TXT), tmp_requirements_file_path)
         # Run the pipreqs command and capture stderr
         attempt = 0
         while attempt < MAX_RETRIES:
@@ -815,13 +829,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 time.sleep(retry_after)
             else:
                 print_library_log(f"pipreqs failed after {MAX_RETRIES} attempts with:", Logging.Level.SEVERE)
-                print(result.stderr)
-                sys.exit(1)
+                print_library_log(result.stderr, Logging.Level.SEVERE)
+                print_library_log(f"Skipping validation of requirements.txt due to error connecting to PyPI (Python Package Index) APIs. Continuing with {'deploy' if is_deploy else 'debug'}...", Logging.Level.SEVERE)
 
         tmp_requirements = self.fetch_requirements_as_dict(self, tmp_requirements_file_path)
-        tmp_requirements.pop("fivetran_connector_sdk")
-        if tmp_requirements.get('requests') is not None:
-            tmp_requirements.pop("requests")
+        self.remove_unwanted_packages(tmp_requirements)
         os.remove(tmp_requirements_file_path)
 
         # remove corrupt requirements listed by pipreqs
@@ -894,7 +906,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         if 'requests' in unused_deps:
             log_unused_deps_error("requests", "2.32.3")
         print_library_log("The following dependencies are not needed, "
-              f"they are not used or already installed. Please remove them from {REQUIREMENTS_TXT}:")
+              f"they are not used or already installed. Please remove them from {REQUIREMENTS_TXT}:", Logging.Level.WARNING)
         print(*unused_deps)
 
     def handle_missing_deps(self, missing_deps):
@@ -915,6 +927,19 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             print_library_log("Adding `requirements.txt` file to your project folder.", Logging.Level.WARNING)
         return requirements
 
+    def copy_requirements_file_to_tmp_requirements_file(self, requirements_file_path: str, tmp_requirements_file_path):
+        requirements_file_content = self.fetch_requirements_from_file(requirements_file_path)
+        with open(tmp_requirements_file_path, 'w') as file:
+            file.write("\n".join(requirements_file_content))
+
+    @staticmethod
+    def remove_unwanted_packages(requirements: dict):
+        # remove the `fivetran_connector_sdk` and `requests` packages from requirements as we already pre-installed them.
+        if requirements.get("fivetran_connector_sdk") is not None:
+            requirements.pop("fivetran_connector_sdk")
+        if requirements.get('requests') is not None:
+            requirements.pop("requests")
+
     # Call this method to deploy the connector to Fivetran platform
     def deploy(self, args: dict, deploy_key: str, group: str, connection: str, hd_agent_id: str, configuration: dict = None):
         """Deploys the connector to the Fivetran platform.
@@ -1188,7 +1213,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 if INSTALLATION_SCRIPT in files:
                     custom_driver_installation_script_exists = True
                 for file in files:
-                    if file == "connector.py":
+                    if file == ROOT_FILENAME:
                         connector_file_exists = True
                     file_path = os.path.join(root, file)
                     arcname = os.path.relpath(file_path, project_path)
@@ -1542,10 +1567,10 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
         popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
         for line in Connector.process_stream(popen.stderr):
-            yield line
+            yield Connector._maybe_colorize_jar_output(line)
 
         for line in Connector.process_stream(popen.stdout):
-            yield line
+            yield Connector._maybe_colorize_jar_output(line)
         popen.stdout.close()
         return_code = popen.wait()
         if return_code:
@@ -1726,19 +1751,30 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             print_library_log(error_message, Logging.Level.SEVERE)
             raise RuntimeError(error_message) from e
 
+    @staticmethod
+    def _maybe_colorize_jar_output(line: str) -> str:
+        if not DEBUGGING:
+            return line
+
+        if "SEVERE" in line or "ERROR" in line or "Exception" in line or "FAILED" in line:
+            return f"\033[91m{line}\033[0m"  # Red
+        elif "WARN" in line or "WARNING" in line:
+            return f"\033[93m{line}\033[0m"  # Yellow
+        return line
+
 
-def find_connector_object(project_path) -> Connector:
+def find_connector_object(project_path) -> Optional[Connector]:
     """Finds the connector object in the given project path.
     Args:
         project_path (str): The path to the project.
     
     Returns:
-        object: The connector object.
+        Optional[Connector]: The connector object or None if not found.
     """
 
     sys.path.append(project_path) # Allows python interpreter to search for modules in this path
     module_name = "connector_connector_code"
-    connector_py = os.path.join(project_path, "connector.py")
+    connector_py = os.path.join(project_path, ROOT_FILENAME)
     try:
         spec = importlib.util.spec_from_file_location(module_name, connector_py)
         module = importlib.util.module_from_spec(spec)
@@ -1752,11 +1788,11 @@ def find_connector_object(project_path) -> Connector:
     except FileNotFoundError:
         print_library_log(
             "The connector object is missing in the current directory. Please ensure that you are running the command from correct directory or that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
-        return
+        return None
 
     print_library_log(
         "The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
-    return
+    return None
 
 
 def suggest_correct_command(input_command: str) -> bool:
@@ -1782,9 +1818,9 @@ def suggest_correct_command(input_command: str) -> bool:
 
 
 def print_suggested_command_message(valid_command: str, input_command: str) -> None:
-    print(f"`fivetran {input_command}` is not a valid command.")
-    print(f"Did you mean `fivetran {valid_command}`?")
-    print("Use `fivetran --help` for more details.")
+    print_library_log(f"`fivetran {input_command}` is not a valid command.", Logging.Level.SEVERE)
+    print_library_log(f"Did you mean `fivetran {valid_command}`?", Logging.Level.SEVERE)
+    print_library_log("Use `fivetran --help` for more details.", Logging.Level.SEVERE)
 
 
 def edit_distance(first_string: str, second_string: str) -> int:
@@ -1829,7 +1865,7 @@ def get_input_from_cli(prompt : str, default_value: str) -> str:
 
     if not value:
         raise ValueError("Missing required input: Expected a value but received None")
-    return value or None
+    return value
 
 
 def main():
@@ -1854,7 +1890,7 @@ def main():
     parser.add_argument("--connection", type=str, default=None, help="Connection name (aka 'destination schema')")
     parser.add_argument("-f", "--force", action="store_true", help="Force update an existing connection")
     parser.add_argument("--python-version", "--python", type=str, help=f"Supported Python versions you can use: {SUPPORTED_PYTHON_VERSIONS}. Defaults to 3.12.8")
-    parser.add_argument("--hybrid-deployment-agent-id", type=str, help=argparse.SUPPRESS)
+    parser.add_argument("--hybrid-deployment-agent-id", type=str, help="The Hybrid Deployment agent within the Fivetran system. If nothing is passed, the default agent of the destination is used.")
 
     args = parser.parse_args()
 

{fivetran_connector_sdk-1.3.3 → fivetran_connector_sdk-1.4.0}/src/fivetran_connector_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.3.3
+Version: 1.4.0
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk