fivetran-connector-sdk 2.2.0__tar.gz → 2.3.0__tar.gz

{fivetran_connector_sdk-2.2.0 → fivetran_connector_sdk-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 2.2.0
+Version: 2.3.0
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-2.2.0 → fivetran_connector_sdk-2.3.0}/src/fivetran_connector_sdk/__init__.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import uuid
 import grpc
 import json
 import shutil
@@ -38,12 +39,12 @@ from fivetran_connector_sdk.connector_helper import (
     get_available_port, tester_root_dir_helper,
     check_dict, check_newer_version, cleanup_uploaded_project,
     get_destination_group, get_connection_name, get_api_key, get_state,
-    get_python_version, get_hd_agent_id, get_configuration
+    get_python_version, get_hd_agent_id, get_configuration, evaluate_project
 )
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 2, Minor Version is incremental MM from Aug 25 onwards, Patch Version is incremental within a month)
-__version__ = "2.2.0"
+__version__ = "2.3.0"
 TESTER_VERSION = TESTER_VER
 MAX_MESSAGE_LENGTH = 32 * 1024 * 1024 # 32MB
 
@@ -65,6 +66,9 @@ class Connector(connector_sdk_pb2_grpc.SourceConnectorServicer):
 
         update_base_url_if_required()
 
+    def evaluate(self, project_path: str, deploy_key: str):
+        evaluate_project(project_path, deploy_key)
+
     # Call this method to deploy the connector to Fivetran platform
     def deploy(self, project_path: str, deploy_key: str, group: str, connection: str, hd_agent_id: str,
                configuration: dict = None, config_path = None, python_version: str = None, force: bool = False):
@@ -476,6 +480,12 @@ def main():
     if not connector_object:
         sys.exit(1)
 
+    if args.command.lower() == "evaluate":
+        print_library_log("Evaluating the connector code...")
+        ft_deploy_key = get_api_key(args)
+        connector_object.evaluate(args.project_path, ft_deploy_key)
+        sys.exit(0)
+
     if args.command.lower() == "deploy":
         ft_group = get_destination_group(args)
         ft_connection = get_connection_name(args)
@@ -492,6 +502,8 @@ def main():
         configuration, config_path = get_configuration(args)
         state = get_state(args)
         try:
+            os.environ["FIVETRAN_CONNECTION_ID"] = "test_connection_id"
+            os.environ["FIVETRAN_DEPLOYMENT_MODEL"] = "local_debug"
             connector_object.debug(args.project_path, configuration, state)
         except subprocess.CalledProcessError as e:
             print_library_log(f"Connector tester failed with exit code: {e.returncode}", Logging.Level.SEVERE)
@@ -499,6 +511,9 @@ def main():
         except Exception as e:
             print_library_log(f"Debug command failed: {str(e)}", Logging.Level.SEVERE)
             sys.exit(1)
+        finally:
+            del os.environ["FIVETRAN_CONNECTION_ID"]
+            del os.environ["FIVETRAN_DEPLOYMENT_MODEL"]
     else:
         if not suggest_correct_command(args.command):
             raise NotImplementedError(f"Invalid command: {args.command}, see `fivetran --help`")

{fivetran_connector_sdk-2.2.0 → fivetran_connector_sdk-2.3.0}/src/fivetran_connector_sdk/connector_helper.py

@@ -1,3 +1,4 @@
+import io
 import os
 import re
 import ast
@@ -35,6 +36,7 @@ from fivetran_connector_sdk.constants import (
     CONFIG_FILE,
     OUTPUT_FILES_DIR,
     REQUIREMENTS_TXT,
+    CONFIGURATION_JSON,
     PYPI_PACKAGE_DETAILS_URL,
     ONE_DAY_IN_SEC,
     MAX_RETRIES,
@@ -48,6 +50,7 @@ from fivetran_connector_sdk.constants import (
     UTF_8,
     CONNECTION_SCHEMA_NAME_PATTERN,
     TABLES,
+    EVALUATION_MARKDOWN,
 )
 
 def get_destination_group(args):
@@ -113,21 +116,7 @@ def get_configuration(args, retrying = 0):
     env_configuration = os.getenv('FIVETRAN_CONFIGURATION', None)
     try:
         if not configuration and not args.force and args.command.lower() == "deploy":
-            confirm = 'y'
-            if not retrying:
-                json_filepath = os.path.join(args.project_path, "configuration.json")
-                if os.path.exists(json_filepath):
-                    print_library_log("configuration.json file detected in the project, "
-                                      "but no configuration input provided via the command line", Logging.Level.WARNING)
-                    env_configuration = env_configuration if env_configuration else "configuration.json"
-                confirm = input(f"Does this debug run/deploy need configuration (y/N):")
-            if confirm.lower()=='y':
-                configuration = get_input_from_cli("Provide the configuration file path", env_configuration)
-                config_values = validate_and_load_configuration(args.project_path, configuration)
-                return config_values, configuration
-            else:
-                print_library_log("No input required for configuration. Continuing without configuration.", Logging.Level.INFO)
-                return {}, None
+            return _deploy_config_flow(args, env_configuration, retrying)
         config_values = validate_and_load_configuration(args.project_path, configuration)
         return config_values, configuration
     except ValueError as e:
@@ -179,11 +168,43 @@ def tester_root_dir_helper() -> str:
     """Returns the root directory for the tester."""
     return os.path.join(os.path.expanduser("~"), ROOT_LOCATION)
 
+
+def _deploy_config_flow(args, env_configuration, retrying):
+    """Handles the configuration flow for the deploy command."""
+    confirm = 'y'
+    if not retrying:
+        json_filepath = os.path.join(args.project_path, CONFIGURATION_JSON)
+        if os.path.exists(json_filepath):
+            print_library_log("configuration.json file detected in the project, "
+                              "but no configuration input provided via the command line", Logging.Level.WARNING)
+            env_configuration = env_configuration if env_configuration else CONFIGURATION_JSON
+        confirm = input("Does this debug run/deploy need configuration (y/N):")
+    if confirm.lower() == 'y':
+        configuration = get_input_from_cli("Provide the configuration file path", env_configuration)
+        config_values = validate_and_load_configuration(args.project_path, configuration)
+        return config_values, configuration
+    else:
+        print_library_log("No input required for configuration. Continuing without configuration.", Logging.Level.INFO)
+        return {}, None
+
+
 def _warn_exit_usage(filename, line_no, func):
     print_library_log(f"Avoid using {func} to exit from the Python code as this can cause the connector to become stuck. Throw an error if required " +
                       f"at: {filename}:{line_no}. See the Technical Reference for details: https://fivetran.com/docs/connector-sdk/technical-reference#handlingexceptions",
                       Logging.Level.WARNING)
 
+
+def _check_and_warn_attribute_exit(node):
+    """Checks for the presence of 'exit()' in the AST node and warns if found."""
+    if isinstance(node.func, ast.Name) and node.func.id == "exit":
+        _warn_exit_usage(ROOT_FILENAME, node.lineno, "exit()")
+    elif isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name):
+        if node.func.attr == "_exit" and node.func.value.id == "os":
+            _warn_exit_usage(ROOT_FILENAME, node.lineno, "os._exit()")
+        if node.func.attr == "exit" and node.func.value.id == "sys":
+            _warn_exit_usage(ROOT_FILENAME, node.lineno, "sys.exit()")
+
+
 def exit_check(project_path):
     """Checks for the presence of 'exit()' in the calling code.
     Args:
@@ -199,13 +220,7 @@ def exit_check(project_path):
             tree = ast.parse(f.read())
             for node in ast.walk(tree):
                 if isinstance(node, ast.Call):
-                    if isinstance(node.func, ast.Name) and node.func.id == "exit":
-                        _warn_exit_usage(ROOT_FILENAME, node.lineno, "exit()")
-                    elif isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name):
-                        if node.func.attr == "_exit" and node.func.value.id == "os":
-                            _warn_exit_usage(ROOT_FILENAME, node.lineno, "os._exit()")
-                        if node.func.attr == "exit" and node.func.value.id == "sys":
-                            _warn_exit_usage(ROOT_FILENAME, node.lineno, "sys.exit()")
+                    _check_and_warn_attribute_exit(node)
         except SyntaxError as e:
             print_library_log(f"SyntaxError in {ROOT_FILENAME}: {e}", Logging.Level.SEVERE)
 
@@ -458,6 +473,7 @@ def load_or_add_requirements_file(requirements_file_path):
         requirements = fetch_requirements_as_dict(requirements_file_path)
     else:
         with open(requirements_file_path, 'w', encoding=UTF_8):
+            # Intentional empty block: Creating an empty requirements.txt file
             pass
         requirements = {}
         print_library_log("`requirements.txt` file not found in your project folder.", Logging.Level.WARNING)
@@ -476,6 +492,19 @@ def remove_unwanted_packages(requirements: dict):
     if requirements.get('requests') is not None:
         requirements.pop("requests")
 
+def evaluate_project(project_path: str, deploy_key: str):
+    print_library_log(f"Evaluating '{project_path}'...")
+    upload_file_path = create_upload_file(project_path)
+    try:
+        evaluation_report = evaluate(upload_file_path, project_path, deploy_key)
+        if not evaluation_report:
+            print_library_log(
+                "Project evaluation failed. No evaluation report was generated. Please check your project for errors and try again.",
+                Logging.Level.SEVERE
+            )
+            sys.exit(1)
+    finally:
+        delete_file_if_exists(upload_file_path)
 
 def upload_project(project_path: str, deploy_key: str, group_id: str, group_name: str, connection: str):
     print_library_log(
@@ -672,6 +701,15 @@ def zip_folder(project_path: str) -> str:
 
     return upload_filepath
 
+def _should_descend_into_dir(name, path):
+    """Determines whether to traverse into a subdirectory."""
+    if not os.path.isdir(path):
+        return False
+    if name in EXCLUDED_DIRS or name.startswith("."):
+        return False
+    if VIRTUAL_ENV_CONFIG in os.listdir(path):  # Check for virtual env indicator
+        return False
+    return True
 
 def dir_walker(top):
     """Walks the directory tree starting at the given top directory.
@@ -685,10 +723,8 @@ def dir_walker(top):
     dirs, files = [], []
     for name in os.listdir(top):
         path = os.path.join(top, name)
-        if os.path.isdir(path):
-            if (name not in EXCLUDED_DIRS) and (not name.startswith(".")):
-                if VIRTUAL_ENV_CONFIG not in os.listdir(path):  # Check for virtual env indicator
-                    dirs.append(name)
+        if _should_descend_into_dir(name, path):
+            dirs.append(name)
         else:
             # Include all files if in `drivers` folder
             if os.path.basename(top) == DRIVERS:
@@ -702,6 +738,126 @@ def dir_walker(top):
         for x in dir_walker(new_path):
             yield x
 
+
+def render_section(lines, title, items):
+    """
+    Renders a section of the markdown report.
+    Args:
+        lines (list): The list of lines to append to.
+        title (str): The title of the section.
+        items (list): The list of items in the section.
+    """
+    lines.extend([f"## {title}", ""])
+    if not items:
+        lines.extend(["_No recommendations._", ""])
+        return
+    for index, item in enumerate(items, 1):
+        issue = item.get("issue", "(missing issue)")
+        lines.append(f"### {index}. {issue}")
+        if recommendation := item.get("recommendation"):
+            lines.append(f"- **Recommendation:** {recommendation}")
+        if current_code := item.get("current_code"):
+            fenced_code = f"```python\n{current_code.rstrip()}\n```"
+            lines += ["\n**Current Code:**", fenced_code]
+        if fix := item.get("code_fix"):
+            fenced_code = f"```python\n{fix.rstrip()}\n```"
+            lines += ["\n**Code Fix:**", fenced_code]
+        lines.append("")
+
+
+def render_markdown(report):
+    """
+    Renders the evaluation report as markdown.
+    Args:
+        report (dict): The evaluation report.
+    """
+    lines = []
+    lines += ["# Evaluation Report", ""]
+    overall_score = report.get("score")
+    if overall_score:
+        lines += [f"**Overall Score:** {overall_score}", ""]
+
+    subscore = report.get("subscores") or {}
+    if subscore:
+        lines += ["## Subscores", "", "| Metric | Score |", "|---|---:|"]
+        lines += [f"| {key} | {value} |" for key, value in subscore.items()]
+        lines.append("")
+
+    render_section(lines, "Required", report.get("required"))
+    render_section(lines, "Good to Have", report.get("good_to_have"))
+
+    return "\n".join(lines)
+
+
+def save_evaluation_report(evaluation_report, project_path: str):
+    """Saves the evaluation report to a file in the project path.
+    Args:
+        evaluation_report: The evaluation data.
+        project_path: The path to the project.
+    """
+    # Save evaluation report
+    working_dir = os.path.join(project_path, OUTPUT_FILES_DIR)
+    os.makedirs(working_dir, exist_ok=True)
+    report_path = os.path.join(working_dir, EVALUATION_MARKDOWN)
+    markdown_content = render_markdown(evaluation_report)
+    try:
+        with open(report_path, 'w', encoding=UTF_8) as report_file:
+            report_file.write(markdown_content)
+        print_library_log(f"Evaluation report saved to {report_path}")
+    except Exception as e:
+        print_library_log(f"Failed to save evaluation report: {e}", Logging.Level.SEVERE)
+
+
+def evaluate(local_path: str, project_path:str, deploy_key: str):
+    """Uploads the local code file for evaluation and returns the evaluation report.
+
+    The server responds with a file containing JSON. This function streams the
+    file safely, parses the JSON, and prints it.
+
+    Args:
+        local_path (str): The local file path.
+        project_path (str): The path to the project.
+        deploy_key (str): The deployment key.
+
+    Returns:
+        dict | None: Parsed evaluation report if successful, None otherwise.
+    """
+    print_library_log("Uploading your project for evaluation...")
+
+    with open(local_path, 'rb') as f:
+        response = rq.post(
+            f"{constants.PRODUCTION_BASE_URL}{constants.EVALUATE_ENDPOINT}",
+            files={'file': f},
+            headers={"Authorization": f"Basic {deploy_key}"},
+            stream=True  # stream to handle file responses safely
+        )
+
+    if response.ok:
+        try:
+            buffer = io.BytesIO()
+            for chunk in response.iter_content(chunk_size=8192):
+                buffer.write(chunk)
+            buffer.seek(0)
+
+            evaluation_data = json.load(buffer)
+
+            print_library_log("✓ Evaluation Report:")
+            print(json.dumps(evaluation_data, indent=4))
+            save_evaluation_report(evaluation_data, project_path)
+
+            return evaluation_data
+
+        except json.JSONDecodeError as e:
+            print_library_log(f"Failed to parse evaluation report: {e}", Logging.Level.SEVERE)
+            return None
+
+    print_library_log(
+        f"Unable to upload the project for evaluation, failed with error: {response.text}",
+        Logging.Level.SEVERE
+    )
+    return None
+
+
 def upload(local_path: str, deploy_key: str, group_id: str, connection: str) -> bool:
     """Uploads the local code file for the specified group and connection.
 
@@ -722,7 +878,8 @@ def upload(local_path: str, deploy_key: str, group_id: str, connection: str) ->
         print("✓")
         return True
 
-    print_library_log(f"Unable to upload the project, failed with error: {response.reason}", Logging.Level.SEVERE)
+    error_message = response.reason + ": " + json.loads(response.text).get("message", "")
+    print_library_log(f"Unable to upload the project, failed with error. {error_message}", Logging.Level.SEVERE)
     return False
 
 def cleanup_uploaded_code(deploy_key: str, group_id: str, connection: str) -> bool:

{fivetran_connector_sdk-2.2.0 → fivetran_connector_sdk-2.3.0}/src/fivetran_connector_sdk/constants.py

@@ -1,7 +1,7 @@
 import os
 import re
 
-TESTER_VER = "2.25.0903.001"
+TESTER_VER = "2.25.0918.001"
 
 WIN_OS = "windows"
 ARM_64 = "arm64"
@@ -33,9 +33,11 @@ ROOT_LOCATION = ".ft_sdk_connector_tester"
 CONFIG_FILE = "_config.json"
 OUTPUT_FILES_DIR = "files"
 REQUIREMENTS_TXT = "requirements.txt"
+EVALUATION_MARKDOWN = "evaluation_report.md"
+CONFIGURATION_JSON = "configuration.json"
 PYPI_PACKAGE_DETAILS_URL = "https://pypi.org/pypi/fivetran_connector_sdk/json"
 ONE_DAY_IN_SEC = 24 * 60 * 60
-CHECKPOINT_OP_TIMEOUT_IN_SEC = 30 # seconds
+CHECKPOINT_OP_TIMEOUT_IN_SEC = 120 # seconds
 MAX_RETRIES = 3
 LOGGING_PREFIX = "Fivetran-Connector-SDK"
 LOGGING_DELIMITER = ": "
@@ -64,6 +66,7 @@ COMMANDS_AND_SYNONYMS = {
 
 CONNECTION_SCHEMA_NAME_PATTERN = r'^[_a-z][_a-z0-9]*$'
 PRODUCTION_BASE_URL = "https://api.fivetran.com"
+EVALUATE_ENDPOINT = "/v1/evaluate"
 INSTALLATION_SCRIPT_MISSING_MESSAGE = "The 'installation.sh' file is missing in the 'drivers' directory. Please ensure that 'installation.sh' is present to properly configure drivers."
 INSTALLATION_SCRIPT = "installation.sh"
 DRIVERS = "drivers"

{fivetran_connector_sdk-2.2.0 → fivetran_connector_sdk-2.3.0}/src/fivetran_connector_sdk/operations.py

@@ -150,7 +150,6 @@ def _get_table_pk(table: str) -> bool:
     Returns:
         dict: The columns for the table.
     """
-    columns = {}
     if table in TABLES:
         for column in TABLES[table].columns:
             if column.primary_key:
@@ -179,15 +178,20 @@ def _map_data_to_columns(data: dict, columns: dict, table: str = "") -> dict:
             map_inferred_data_type(key, mapped_data, v, table)
     return mapped_data
 
+def _log_boolean_inference_once(table):
+    """Log boolean inference once per table and mark it as logged."""
+    key = f"boolean_{table}"
+    if _LOG_DATA_TYPE_INFERENCE.get(key, True):
+        print_library_log("Fivetran: Boolean Datatype has been inferred for " + table, Logging.Level.INFO, True)
+        if not _get_table_pk(table):
+            print_library_log("Fivetran: Boolean Datatype inference issue for " + table, Logging.Level.INFO, True)
+        _LOG_DATA_TYPE_INFERENCE[key] = False
+
 def map_inferred_data_type(k, mapped_data, v, table=""):
     # We can infer type from the value
     if isinstance(v, bool):
         mapped_data[k] = common_pb2.ValueType(bool=v)
-        if _LOG_DATA_TYPE_INFERENCE.get("boolean_" + table, True):
-            print_library_log("Fivetran: Boolean Datatype has been inferred for " + table, Logging.Level.INFO, True)
-            if not _get_table_pk(table):
-                print_library_log("Fivetran: Boolean Datatype inference issue for " + table, Logging.Level.INFO, True)
-            _LOG_DATA_TYPE_INFERENCE["boolean_" + table] = False
+        _log_boolean_inference_once(table)
     elif isinstance(v, int):
         if abs(v) > JAVA_LONG_MAX_VALUE:
             mapped_data[k] = common_pb2.ValueType(float=v)

{fivetran_connector_sdk-2.2.0 → fivetran_connector_sdk-2.3.0}/src/fivetran_connector_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 2.2.0
+Version: 2.3.0
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk