fivetran-connector-sdk 2.2.1__py3-none-any.whl → 2.3.1__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -44,7 +44,7 @@ from fivetran_connector_sdk.connector_helper import (
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 2, Minor Version is incremental MM from Aug 25 onwards, Patch Version is incremental within a month)
-__version__ = "2.2.1"
+__version__ = "2.3.1"
 TESTER_VERSION = TESTER_VER
 MAX_MESSAGE_LENGTH = 32 * 1024 * 1024 # 32MB
 

fivetran_connector_sdk/connector_helper.py

@@ -36,6 +36,7 @@ from fivetran_connector_sdk.constants import (
     CONFIG_FILE,
     OUTPUT_FILES_DIR,
     REQUIREMENTS_TXT,
+    CONFIGURATION_JSON,
     PYPI_PACKAGE_DETAILS_URL,
     ONE_DAY_IN_SEC,
     MAX_RETRIES,
@@ -49,6 +50,7 @@ from fivetran_connector_sdk.constants import (
     UTF_8,
     CONNECTION_SCHEMA_NAME_PATTERN,
     TABLES,
+    EVALUATION_MARKDOWN,
 )
 
 def get_destination_group(args):
@@ -114,21 +116,7 @@ def get_configuration(args, retrying = 0):
     env_configuration = os.getenv('FIVETRAN_CONFIGURATION', None)
     try:
         if not configuration and not args.force and args.command.lower() == "deploy":
-            confirm = 'y'
-            if not retrying:
-                json_filepath = os.path.join(args.project_path, "configuration.json")
-                if os.path.exists(json_filepath):
-                    print_library_log("configuration.json file detected in the project, "
-                                      "but no configuration input provided via the command line", Logging.Level.WARNING)
-                    env_configuration = env_configuration if env_configuration else "configuration.json"
-                confirm = input(f"Does this debug run/deploy need configuration (y/N):")
-            if confirm.lower()=='y':
-                configuration = get_input_from_cli("Provide the configuration file path", env_configuration)
-                config_values = validate_and_load_configuration(args.project_path, configuration)
-                return config_values, configuration
-            else:
-                print_library_log("No input required for configuration. Continuing without configuration.", Logging.Level.INFO)
-                return {}, None
+            return _deploy_config_flow(args, env_configuration, retrying)
         config_values = validate_and_load_configuration(args.project_path, configuration)
         return config_values, configuration
     except ValueError as e:
@@ -180,11 +168,43 @@ def tester_root_dir_helper() -> str:
     """Returns the root directory for the tester."""
     return os.path.join(os.path.expanduser("~"), ROOT_LOCATION)
 
+
+def _deploy_config_flow(args, env_configuration, retrying):
+    """Handles the configuration flow for the deploy command."""
+    confirm = 'y'
+    if not retrying:
+        json_filepath = os.path.join(args.project_path, CONFIGURATION_JSON)
+        if os.path.exists(json_filepath):
+            print_library_log("configuration.json file detected in the project, "
+                              "but no configuration input provided via the command line", Logging.Level.WARNING)
+            env_configuration = env_configuration if env_configuration else CONFIGURATION_JSON
+        confirm = input("Does this debug run/deploy need configuration (y/N):")
+    if confirm.lower() == 'y':
+        configuration = get_input_from_cli("Provide the configuration file path", env_configuration)
+        config_values = validate_and_load_configuration(args.project_path, configuration)
+        return config_values, configuration
+    else:
+        print_library_log("No input required for configuration. Continuing without configuration.", Logging.Level.INFO)
+        return {}, None
+
+
 def _warn_exit_usage(filename, line_no, func):
     print_library_log(f"Avoid using {func} to exit from the Python code as this can cause the connector to become stuck. Throw an error if required " +
                       f"at: {filename}:{line_no}. See the Technical Reference for details: https://fivetran.com/docs/connector-sdk/technical-reference#handlingexceptions",
                       Logging.Level.WARNING)
 
+
+def _check_and_warn_attribute_exit(node):
+    """Checks for the presence of 'exit()' in the AST node and warns if found."""
+    if isinstance(node.func, ast.Name) and node.func.id == "exit":
+        _warn_exit_usage(ROOT_FILENAME, node.lineno, "exit()")
+    elif isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name):
+        if node.func.attr == "_exit" and node.func.value.id == "os":
+            _warn_exit_usage(ROOT_FILENAME, node.lineno, "os._exit()")
+        if node.func.attr == "exit" and node.func.value.id == "sys":
+            _warn_exit_usage(ROOT_FILENAME, node.lineno, "sys.exit()")
+
+
 def exit_check(project_path):
     """Checks for the presence of 'exit()' in the calling code.
     Args:
@@ -200,13 +220,7 @@ def exit_check(project_path):
             tree = ast.parse(f.read())
             for node in ast.walk(tree):
                 if isinstance(node, ast.Call):
-                    if isinstance(node.func, ast.Name) and node.func.id == "exit":
-                        _warn_exit_usage(ROOT_FILENAME, node.lineno, "exit()")
-                    elif isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name):
-                        if node.func.attr == "_exit" and node.func.value.id == "os":
-                            _warn_exit_usage(ROOT_FILENAME, node.lineno, "os._exit()")
-                        if node.func.attr == "exit" and node.func.value.id == "sys":
-                            _warn_exit_usage(ROOT_FILENAME, node.lineno, "sys.exit()")
+                    _check_and_warn_attribute_exit(node)
         except SyntaxError as e:
             print_library_log(f"SyntaxError in {ROOT_FILENAME}: {e}", Logging.Level.SEVERE)
 
@@ -459,6 +473,7 @@ def load_or_add_requirements_file(requirements_file_path):
         requirements = fetch_requirements_as_dict(requirements_file_path)
     else:
         with open(requirements_file_path, 'w', encoding=UTF_8):
+            # Intentional empty block: Creating an empty requirements.txt file
             pass
         requirements = {}
         print_library_log("`requirements.txt` file not found in your project folder.", Logging.Level.WARNING)
@@ -481,7 +496,7 @@ def evaluate_project(project_path: str, deploy_key: str):
     print_library_log(f"Evaluating '{project_path}'...")
     upload_file_path = create_upload_file(project_path)
     try:
-        evaluation_report = evaluate(upload_file_path, deploy_key)
+        evaluation_report = evaluate(upload_file_path, project_path, deploy_key)
         if not evaluation_report:
             print_library_log(
                 "Project evaluation failed. No evaluation report was generated. Please check your project for errors and try again.",
@@ -686,6 +701,15 @@ def zip_folder(project_path: str) -> str:
 
     return upload_filepath
 
+def _should_descend_into_dir(name, path):
+    """Determines whether to traverse into a subdirectory."""
+    if not os.path.isdir(path):
+        return False
+    if name in EXCLUDED_DIRS or name.startswith("."):
+        return False
+    if VIRTUAL_ENV_CONFIG in os.listdir(path):  # Check for virtual env indicator
+        return False
+    return True
 
 def dir_walker(top):
     """Walks the directory tree starting at the given top directory.
@@ -699,10 +723,8 @@ def dir_walker(top):
     dirs, files = [], []
     for name in os.listdir(top):
         path = os.path.join(top, name)
-        if os.path.isdir(path):
-            if (name not in EXCLUDED_DIRS) and (not name.startswith(".")):
-                if VIRTUAL_ENV_CONFIG not in os.listdir(path):  # Check for virtual env indicator
-                    dirs.append(name)
+        if _should_descend_into_dir(name, path):
+            dirs.append(name)
         else:
             # Include all files if in `drivers` folder
             if os.path.basename(top) == DRIVERS:
@@ -717,7 +739,76 @@ def dir_walker(top):
             yield x
 
 
-def evaluate(local_path: str, deploy_key: str):
+def render_section(lines, title, items):
+    """
+    Renders a section of the markdown report.
+    Args:
+        lines (list): The list of lines to append to.
+        title (str): The title of the section.
+        items (list): The list of items in the section.
+    """
+    lines.extend([f"## {title}", ""])
+    if not items:
+        lines.extend(["_No recommendations._", ""])
+        return
+    for index, item in enumerate(items, 1):
+        issue = item.get("issue", "(missing issue)")
+        lines.append(f"### {index}. {issue}")
+        if recommendation := item.get("recommendation"):
+            lines.append(f"- **Recommendation:** {recommendation}")
+        if current_code := item.get("current_code"):
+            fenced_code = f"```python\n{current_code.rstrip()}\n```"
+            lines += ["\n**Current Code:**", fenced_code]
+        if fix := item.get("code_fix"):
+            fenced_code = f"```python\n{fix.rstrip()}\n```"
+            lines += ["\n**Code Fix:**", fenced_code]
+        lines.append("")
+
+
+def render_markdown(report):
+    """
+    Renders the evaluation report as markdown.
+    Args:
+        report (dict): The evaluation report.
+    """
+    lines = []
+    lines += ["# Evaluation Report", ""]
+    overall_score = report.get("score")
+    if overall_score:
+        lines += [f"**Overall Score:** {overall_score}", ""]
+
+    subscore = report.get("subscores") or {}
+    if subscore:
+        lines += ["## Subscores", "", "| Metric | Score |", "|---|---:|"]
+        lines += [f"| {key} | {value} |" for key, value in subscore.items()]
+        lines.append("")
+
+    render_section(lines, "Required", report.get("required"))
+    render_section(lines, "Good to Have", report.get("good_to_have"))
+
+    return "\n".join(lines)
+
+
+def save_evaluation_report(evaluation_report, project_path: str):
+    """Saves the evaluation report to a file in the project path.
+    Args:
+        evaluation_report: The evaluation data.
+        project_path: The path to the project.
+    """
+    # Save evaluation report
+    working_dir = os.path.join(project_path, OUTPUT_FILES_DIR)
+    os.makedirs(working_dir, exist_ok=True)
+    report_path = os.path.join(working_dir, EVALUATION_MARKDOWN)
+    markdown_content = render_markdown(evaluation_report)
+    try:
+        with open(report_path, 'w', encoding=UTF_8) as report_file:
+            report_file.write(markdown_content)
+        print_library_log(f"Evaluation report saved to {report_path}")
+    except Exception as e:
+        print_library_log(f"Failed to save evaluation report: {e}", Logging.Level.SEVERE)
+
+
+def evaluate(local_path: str, project_path:str, deploy_key: str):
     """Uploads the local code file for evaluation and returns the evaluation report.
 
     The server responds with a file containing JSON. This function streams the
@@ -725,6 +816,7 @@ def evaluate(local_path: str, deploy_key: str):
 
     Args:
         local_path (str): The local file path.
+        project_path (str): The path to the project.
         deploy_key (str): The deployment key.
 
     Returns:
@@ -751,6 +843,7 @@ def evaluate(local_path: str, deploy_key: str):
 
             print_library_log("✓ Evaluation Report:")
             print(json.dumps(evaluation_data, indent=4))
+            save_evaluation_report(evaluation_data, project_path)
 
             return evaluation_data
 
@@ -785,7 +878,8 @@ def upload(local_path: str, deploy_key: str, group_id: str, connection: str) ->
         print("✓")
         return True
 
-    print_library_log(f"Unable to upload the project, failed with error: {response.reason}", Logging.Level.SEVERE)
+    error_message = response.reason + ": " + json.loads(response.text).get("message", "")
+    print_library_log(f"Unable to upload the project, failed with error. {error_message}", Logging.Level.SEVERE)
     return False
 
 def cleanup_uploaded_code(deploy_key: str, group_id: str, connection: str) -> bool:

fivetran_connector_sdk/constants.py

@@ -33,9 +33,11 @@ ROOT_LOCATION = ".ft_sdk_connector_tester"
 CONFIG_FILE = "_config.json"
 OUTPUT_FILES_DIR = "files"
 REQUIREMENTS_TXT = "requirements.txt"
+EVALUATION_MARKDOWN = "evaluation_report.md"
+CONFIGURATION_JSON = "configuration.json"
 PYPI_PACKAGE_DETAILS_URL = "https://pypi.org/pypi/fivetran_connector_sdk/json"
 ONE_DAY_IN_SEC = 24 * 60 * 60
-CHECKPOINT_OP_TIMEOUT_IN_SEC = 30 # seconds
+CHECKPOINT_OP_TIMEOUT_IN_SEC = 120 # seconds
 MAX_RETRIES = 3
 LOGGING_PREFIX = "Fivetran-Connector-SDK"
 LOGGING_DELIMITER = ": "

fivetran_connector_sdk/operations.py

@@ -150,7 +150,6 @@ def _get_table_pk(table: str) -> bool:
     Returns:
         dict: The columns for the table.
     """
-    columns = {}
     if table in TABLES:
         for column in TABLES[table].columns:
             if column.primary_key:
@@ -179,15 +178,32 @@ def _map_data_to_columns(data: dict, columns: dict, table: str = "") -> dict:
             map_inferred_data_type(key, mapped_data, v, table)
     return mapped_data
 
+def _log_boolean_inference_once(table):
+    """Log boolean inference once per table and mark it as logged."""
+    key = f"boolean_{table}"
+    if _LOG_DATA_TYPE_INFERENCE.get(key, True):
+        print_library_log("Fivetran: Boolean Datatype has been inferred for " + table, Logging.Level.INFO, True)
+        if not _get_table_pk(table):
+            print_library_log("Fivetran: Boolean Datatype inference issue for " + table, Logging.Level.INFO, True)
+        _LOG_DATA_TYPE_INFERENCE[key] = False
+
+_INFERENCE_TYPE_HANDLERS = {
+    int: lambda v: common_pb2.ValueType(float=v) if abs(v)>JAVA_LONG_MAX_VALUE else common_pb2.ValueType(long=v),
+    float: lambda v: common_pb2.ValueType(float=v),
+    bool: lambda v: common_pb2.ValueType(bool=v),
+    bytes: lambda v: common_pb2.ValueType(binary=v),
+    dict: lambda v: common_pb2.ValueType(json=json.dumps(v)),
+    str: lambda v: common_pb2.ValueType(string=v)
+}
+
 def map_inferred_data_type(k, mapped_data, v, table=""):
-    # We can infer type from the value
-    if isinstance(v, bool):
+    data_type = type(v)
+    handler = _INFERENCE_TYPE_HANDLERS.get(data_type)
+    if handler:
+        mapped_data[k] = handler(v)
+    elif isinstance(v, bool):
         mapped_data[k] = common_pb2.ValueType(bool=v)
-        if _LOG_DATA_TYPE_INFERENCE.get("boolean_" + table, True):
-            print_library_log("Fivetran: Boolean Datatype has been inferred for " + table, Logging.Level.INFO, True)
-            if not _get_table_pk(table):
-                print_library_log("Fivetran: Boolean Datatype inference issue for " + table, Logging.Level.INFO, True)
-            _LOG_DATA_TYPE_INFERENCE["boolean_" + table] = False
+        _log_boolean_inference_once(table)
     elif isinstance(v, int):
         if abs(v) > JAVA_LONG_MAX_VALUE:
             mapped_data[k] = common_pb2.ValueType(float=v)
@@ -197,10 +213,7 @@ def map_inferred_data_type(k, mapped_data, v, table=""):
         mapped_data[k] = common_pb2.ValueType(float=v)
     elif isinstance(v, bytes):
         mapped_data[k] = common_pb2.ValueType(binary=v)
-    elif isinstance(v, list):
-        raise ValueError(
-            "Values for the columns cannot be of type 'list'. Please ensure that all values are of a supported type. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#supporteddatatypes")
-    elif isinstance(v, dict):
+    elif isinstance(v, (list, dict)):
         mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
     elif isinstance(v, str):
         mapped_data[k] = common_pb2.ValueType(string=v)

{fivetran_connector_sdk-2.2.1.dist-info → fivetran_connector_sdk-2.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 2.2.1
+Version: 2.3.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-2.2.1.dist-info → fivetran_connector_sdk-2.3.1.dist-info}/RECORD

@@ -1,10 +1,10 @@
-fivetran_connector_sdk/__init__.py,sha256=TcTrOpMTlZbknXNfmkYXghZpKmhB5QORP7xiQEP-EMg,23809
-fivetran_connector_sdk/connector_helper.py,sha256=BtiFeLZCDqYBeQlO1IY2R94MbcBgR3-Omg0CHgCeZD0,45186
-fivetran_connector_sdk/constants.py,sha256=0btZyHUTmoMdrGuIYoWiS-9kLY8pOy5N1jlzxdobj2Q,2476
+fivetran_connector_sdk/__init__.py,sha256=zNRuNGV0ZD4jgnM5RO_XwXuRgcOr2a5LVIH2rZQbeaE,23809
+fivetran_connector_sdk/connector_helper.py,sha256=UMI743HxfVKJMkQhDD2QNFfqOqicqJ6BqyqL6UjcDuo,48462
+fivetran_connector_sdk/constants.py,sha256=4fVdH9BaJAPIHpEe1crYn3Cz1TFJ0GJIyfgHG8tw9UA,2564
 fivetran_connector_sdk/helpers.py,sha256=7YVB1JQ9T0hg90Z0pjJxFp0pQzeBfefrfvS4SYtrlv4,15254
 fivetran_connector_sdk/logger.py,sha256=ud8v8-mKx65OAPaZvxBqt2-CU0vjgBeiYwuiqsYh_hA,3063
 fivetran_connector_sdk/operation_stream.py,sha256=DXLDv961xZ_GVSEPUFLtZy0IEf_ayQSEXFpEJp-CAu4,6194
-fivetran_connector_sdk/operations.py,sha256=1NZjgP_DGLh6103aki2OHfeloPhXfLwtRaN7NIMRQbE,12300
+fivetran_connector_sdk/operations.py,sha256=vfEEyd0Rlnz5WS24llawUFtvckIaJydJadrj_gzQ4Q8,12654
 fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fivetran_connector_sdk/protos/common_pb2.py,sha256=zkzs6Rd-lvsev6Nsq37xc4HLJZ_uNXPkotCLY7Y7i5U,8770
 fivetran_connector_sdk/protos/common_pb2.pyi,sha256=FdqlPKRqiXdUDT3e7adP5X42_Qzv_ItydUNJFKnJJIE,11478
@@ -12,8 +12,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=qni6h6BoA1nwJXr2bNtznfTk
 fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=Inv87MlK5Q56GNvMNFQHyqIePDMKnkW9y_BrT9DgPck,7835
 fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=3AC-bK6ZM-Bmr_RETOB3y_0u4ATWlwcbHzqVanDuOB0,8115
 fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=bGlvc_vGwA9-FTqrj-BYlVcA-7jS8A9MSZ-XpZFytvY,8795
-fivetran_connector_sdk-2.2.1.dist-info/METADATA,sha256=2ryTaDn61Yvf1pGev8f2fA-fCGi4hmw_lm4cmdblEpU,3197
-fivetran_connector_sdk-2.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-fivetran_connector_sdk-2.2.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-2.2.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-2.2.1.dist-info/RECORD,,
+fivetran_connector_sdk-2.3.1.dist-info/METADATA,sha256=Vhr_Qy_jhcjDzGgSVTiL2eOgsP3Cyp3NU7imxz_0gx8,3197
+fivetran_connector_sdk-2.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fivetran_connector_sdk-2.3.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-2.3.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-2.3.1.dist-info/RECORD,,