fivetran-connector-sdk 1.3.4__py3-none-any.whl → 1.4.1__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -32,7 +32,7 @@ from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
-__version__ = "1.3.4"
+__version__ = "1.4.1"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
@@ -110,12 +110,24 @@ class Logging:
         if DEBUGGING:
             current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
             escaped_message = json.dumps(message).strip('"')
-            print(f"{current_time} {level.name}: {escaped_message}")
+            print(f"{Logging._get_color(level)}{current_time} {level.name}: {escaped_message} {Logging._reset_color()}")
         else:
             escaped_message = json.dumps(message)
             log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "connector_sdk"}}'
             print(log_message)
 
+    @staticmethod
+    def _get_color(level):
+        if level == Logging.Level.WARNING:
+            return "\033[93m"  # Yellow
+        elif level == Logging.Level.SEVERE:
+            return "\033[91m"  # Red
+        return ""
+
+    @staticmethod
+    def _reset_color():
+        return "\033[0m"
+
     @staticmethod
     def fine(message: str):
         """Logs a fine-level message.
@@ -313,7 +325,7 @@ def check_newer_version():
             latest_version = data["info"]["version"]
             if __version__ < latest_version:
                 print_library_log(f"[notice] A new release of 'fivetran-connector-sdk' is available: {latest_version}\n" +
-                      f"[notice] To update, run: pip install --upgrade fivetran-connector-sdk\n")
+                      "[notice] To update, run: pip install --upgrade fivetran-connector-sdk\n")
 
             with open(last_check_file_path, 'w', encoding=UTF_8) as f_out:
                 f_out.write(f"{int(time.time())}")
@@ -573,7 +585,7 @@ def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
     if DEBUGGING or EXECUTED_VIA_CLI:
         current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
         escaped_message = json.dumps(message).strip('"')
-        print(f"{current_time} {level.name} {LOGGING_PREFIX}: {escaped_message}")
+        print(f"{Logging._get_color(level)}{current_time} {level.name} {LOGGING_PREFIX}: {escaped_message} {Logging._reset_color()}")
     else:
         escaped_message = json.dumps(LOGGING_PREFIX + LOGGING_DELIMITER + message)
         log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "library"}}'
@@ -760,6 +772,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             values are the full dependency strings.
         """
         requirements_dict = {}
+        if not os.path.exists(file_path):
+            return requirements_dict
         for requirement in self.fetch_requirements_from_file(file_path):
             requirement = requirement.strip()
             if not requirement or requirement.startswith("#"):  # Skip empty lines and comments
@@ -793,7 +807,10 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         ignored_dirs = EXCLUDED_PIPREQS_DIRS + venv_dirs if venv_dirs else EXCLUDED_PIPREQS_DIRS
 
         # tmp_requirements is only generated when pipreqs command is successful
+        requirements_file_path = os.path.join(project_path, REQUIREMENTS_TXT)
         tmp_requirements_file_path = os.path.join(project_path, 'tmp_requirements.txt')
+        # copying packages of requirements file to tmp file to handle pipreqs fail use-case
+        self.copy_requirements_file_to_tmp_requirements_file(os.path.join(project_path, REQUIREMENTS_TXT), tmp_requirements_file_path)
         # Run the pipreqs command and capture stderr
         attempt = 0
         while attempt < MAX_RETRIES:
@@ -815,13 +832,11 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 time.sleep(retry_after)
             else:
                 print_library_log(f"pipreqs failed after {MAX_RETRIES} attempts with:", Logging.Level.SEVERE)
-                print(result.stderr)
-                sys.exit(1)
+                print_library_log(result.stderr, Logging.Level.SEVERE)
+                print_library_log(f"Skipping validation of requirements.txt due to error connecting to PyPI (Python Package Index) APIs. Continuing with {'deploy' if is_deploy else 'debug'}...", Logging.Level.SEVERE)
 
         tmp_requirements = self.fetch_requirements_as_dict(self, tmp_requirements_file_path)
-        tmp_requirements.pop("fivetran_connector_sdk")
-        if tmp_requirements.get('requests') is not None:
-            tmp_requirements.pop("requests")
+        self.remove_unwanted_packages(tmp_requirements)
         os.remove(tmp_requirements_file_path)
 
         # remove corrupt requirements listed by pipreqs
@@ -830,7 +845,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             del tmp_requirements[requirement]
 
         if len(tmp_requirements) > 0:
-            requirements = self.load_or_add_requirements_file(project_path)
+            requirements = self.load_or_add_requirements_file(requirements_file_path)
 
             version_mismatch_deps = {key: tmp_requirements[key] for key in
                                      (requirements.keys() & tmp_requirements.keys())
@@ -876,15 +891,18 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 if force:
                     requirements = tmp_requirements
 
-                with open(REQUIREMENTS_TXT, "w", encoding=UTF_8) as file:
+                with open(requirements_file_path, "w", encoding=UTF_8) as file:
                     file.write("\n".join(requirements.values()))
                 print_library_log(f"`{REQUIREMENTS_TXT}` has been updated successfully")
 
         else:
-            if os.path.exists(REQUIREMENTS_TXT):
+            if os.path.exists(requirements_file_path):
                 print_library_log("`requirements.txt` is not required as no additional "
                       "Python libraries are required or all required libraries for "
                       "your code are pre-installed.", Logging.Level.WARNING)
+                with open(requirements_file_path, 'w') as file:
+                    file.write("")
+
 
         if is_deploy: print_library_log("Successful validation of requirements.txt")
 
@@ -894,7 +912,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         if 'requests' in unused_deps:
             log_unused_deps_error("requests", "2.32.3")
         print_library_log("The following dependencies are not needed, "
-              f"they are not used or already installed. Please remove them from {REQUIREMENTS_TXT}:")
+              f"they are not used or already installed. Please remove them from {REQUIREMENTS_TXT}:", Logging.Level.WARNING)
         print(*unused_deps)
 
     def handle_missing_deps(self, missing_deps):
@@ -905,16 +923,30 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
               "#workingwithrequirementstxtfile", Logging.Level.SEVERE)
         print(*list(missing_deps.values()))
 
-    def load_or_add_requirements_file(self, project_path):
-        if os.path.exists(REQUIREMENTS_TXT):
-            requirements = self.fetch_requirements_as_dict(self, os.path.join(project_path, 'requirements.txt'))
+    def load_or_add_requirements_file(self, requirements_file_path):
+        if os.path.exists(requirements_file_path):
+            requirements = self.fetch_requirements_as_dict(self, requirements_file_path)
         else:
-            with open(REQUIREMENTS_TXT, 'w', encoding=UTF_8):
+            with open(requirements_file_path, 'w', encoding=UTF_8):
                 pass
             requirements = {}
             print_library_log("Adding `requirements.txt` file to your project folder.", Logging.Level.WARNING)
         return requirements
 
+    def copy_requirements_file_to_tmp_requirements_file(self, requirements_file_path: str, tmp_requirements_file_path):
+        if os.path.exists(requirements_file_path):
+            requirements_file_content = self.fetch_requirements_from_file(requirements_file_path)
+            with open(tmp_requirements_file_path, 'w') as file:
+                file.write("\n".join(requirements_file_content))
+
+    @staticmethod
+    def remove_unwanted_packages(requirements: dict):
+        # remove the `fivetran_connector_sdk` and `requests` packages from requirements as we already pre-installed them.
+        if requirements.get("fivetran_connector_sdk") is not None:
+            requirements.pop("fivetran_connector_sdk")
+        if requirements.get('requests') is not None:
+            requirements.pop("requests")
+
     # Call this method to deploy the connector to Fivetran platform
     def deploy(self, args: dict, deploy_key: str, group: str, connection: str, hd_agent_id: str, configuration: dict = None):
         """Deploys the connector to the Fivetran platform.
@@ -1188,7 +1220,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 if INSTALLATION_SCRIPT in files:
                     custom_driver_installation_script_exists = True
                 for file in files:
-                    if file == "connector.py":
+                    if file == ROOT_FILENAME:
                         connector_file_exists = True
                     file_path = os.path.join(root, file)
                     arcname = os.path.relpath(file_path, project_path)
@@ -1506,7 +1538,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         Yields:
             str: Each line from the stream after replacing the matched pattern with the replacement string.
         """
-        pattern = r'com\.fivetran\.fivetran_sdk.*\.tools\.testers\.\S+'
+        pattern = r'com\.fivetran\.partner_sdk.*\.tools\.testers\.\S+'
 
         for line in iter(stream.readline, ""):
             if not re.search(pattern, line):
@@ -1542,10 +1574,10 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
         popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
         for line in Connector.process_stream(popen.stderr):
-            yield line
+            yield Connector._maybe_colorize_jar_output(line)
 
         for line in Connector.process_stream(popen.stdout):
-            yield line
+            yield Connector._maybe_colorize_jar_output(line)
         popen.stdout.close()
         return_code = popen.wait()
         if return_code:
@@ -1726,19 +1758,30 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             print_library_log(error_message, Logging.Level.SEVERE)
             raise RuntimeError(error_message) from e
 
+    @staticmethod
+    def _maybe_colorize_jar_output(line: str) -> str:
+        if not DEBUGGING:
+            return line
+
+        if "SEVERE" in line or "ERROR" in line or "Exception" in line or "FAILED" in line:
+            return f"\033[91m{line}\033[0m"  # Red
+        elif "WARN" in line or "WARNING" in line:
+            return f"\033[93m{line}\033[0m"  # Yellow
+        return line
+
 
-def find_connector_object(project_path) -> Connector:
+def find_connector_object(project_path) -> Optional[Connector]:
     """Finds the connector object in the given project path.
     Args:
         project_path (str): The path to the project.
     
     Returns:
-        object: The connector object.
+        Optional[Connector]: The connector object or None if not found.
     """
 
     sys.path.append(project_path) # Allows python interpreter to search for modules in this path
     module_name = "connector_connector_code"
-    connector_py = os.path.join(project_path, "connector.py")
+    connector_py = os.path.join(project_path, ROOT_FILENAME)
     try:
         spec = importlib.util.spec_from_file_location(module_name, connector_py)
         module = importlib.util.module_from_spec(spec)
@@ -1752,11 +1795,11 @@ def find_connector_object(project_path) -> Connector:
     except FileNotFoundError:
         print_library_log(
             "The connector object is missing in the current directory. Please ensure that you are running the command from correct directory or that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
-        return
+        return None
 
     print_library_log(
         "The connector object is missing. Please ensure that you have defined a connector object using the correct syntax in your `connector.py` file. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsrequiredobjectconnector", Logging.Level.SEVERE)
-    return
+    return None
 
 
 def suggest_correct_command(input_command: str) -> bool:
@@ -1782,9 +1825,9 @@ def suggest_correct_command(input_command: str) -> bool:
 
 
 def print_suggested_command_message(valid_command: str, input_command: str) -> None:
-    print(f"`fivetran {input_command}` is not a valid command.")
-    print(f"Did you mean `fivetran {valid_command}`?")
-    print("Use `fivetran --help` for more details.")
+    print_library_log(f"`fivetran {input_command}` is not a valid command.", Logging.Level.SEVERE)
+    print_library_log(f"Did you mean `fivetran {valid_command}`?", Logging.Level.SEVERE)
+    print_library_log("Use `fivetran --help` for more details.", Logging.Level.SEVERE)
 
 
 def edit_distance(first_string: str, second_string: str) -> int:
@@ -1829,7 +1872,7 @@ def get_input_from_cli(prompt : str, default_value: str) -> str:
 
     if not value:
         raise ValueError("Missing required input: Expected a value but received None")
-    return value or None
+    return value
 
 
 def main():

{fivetran_connector_sdk-1.3.4.dist-info → fivetran_connector_sdk-1.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.3.4
+Version: 1.4.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-1.3.4.dist-info → fivetran_connector_sdk-1.4.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fivetran_connector_sdk/__init__.py,sha256=GsNVJZJiFWlhdxH0mFzrdByWyHhsBT4tGWvb7RdAar0,83938
+fivetran_connector_sdk/__init__.py,sha256=22ATq-tnC_JYy2LUWh4yLWaTopg2v_gOah4k_10WoIQ,86407
 fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
 fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -6,8 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
 fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
 fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
 fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
-fivetran_connector_sdk-1.3.4.dist-info/METADATA,sha256=UYYWBmVMvJl5SWnE1Gh2yPWvWzo32Grr4NMb0cXgTcs,2967
-fivetran_connector_sdk-1.3.4.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
-fivetran_connector_sdk-1.3.4.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-1.3.4.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-1.3.4.dist-info/RECORD,,
+fivetran_connector_sdk-1.4.1.dist-info/METADATA,sha256=F5N5tcF6rl-23TOZohrKyk8uELa5kadAYPE8XUFy7s0,2967
+fivetran_connector_sdk-1.4.1.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+fivetran_connector_sdk-1.4.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-1.4.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-1.4.1.dist-info/RECORD,,

{fivetran_connector_sdk-1.3.4.dist-info → fivetran_connector_sdk-1.4.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.0.0)
+Generator: setuptools (80.3.1)
 Root-Is-Purelib: true
 Tag: py3-none-any