fivetran-connector-sdk 1.2.1__py3-none-any.whl → 1.3.1__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -31,13 +31,13 @@ from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
 # Version format: <major_version>.<minor_version>.<patch_version>
 # (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
-__version__ = "1.2.1"
+__version__ = "1.3.1"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
 LINUX_OS = "linux"
 
-TESTER_VERSION = "0.25.0225.001"
+TESTER_VERSION = "0.25.0411.001"
 TESTER_FILENAME = "run_sdk_tester.jar"
 VERSION_FILENAME = "version.txt"
 UPLOAD_FILENAME = "code.zip"
@@ -49,6 +49,8 @@ REQUIREMENTS_TXT = "requirements.txt"
 PYPI_PACKAGE_DETAILS_URL = "https://pypi.org/pypi/fivetran_connector_sdk/json"
 ONE_DAY_IN_SEC = 24 * 60 * 60
 MAX_RETRIES = 3
+LOGGING_PREFIX = "Fivetran-Connector-SDK"
+LOGGING_DELIMITER = ": "
 VIRTUAL_ENV_CONFIG = "pyvenv.cfg"
 
 # Compile patterns used in the implementation
@@ -73,6 +75,8 @@ DEBUGGING = False
 EXECUTED_VIA_CLI = False
 PRODUCTION_BASE_URL = "https://api.fivetran.com"
 TABLES = {}
+RENAMED_TABLE_NAMES = {}
+RENAMED_COL_NAMES = {}
 INSTALLATION_SCRIPT_MISSING_MESSAGE = "The 'installation.sh' file is missing in the 'drivers' directory. Please ensure that 'installation.sh' is present to properly configure drivers."
 INSTALLATION_SCRIPT = "installation.sh"
 DRIVERS = "drivers"
@@ -175,6 +179,8 @@ class Operations:
                 field_name = get_renamed_column_name(field)
                 columns[field_name] = common_pb2.Column(
                     name=field_name, type=common_pb2.DataType.UNSPECIFIED, primary_key=False)
+            new_table = common_pb2.Table(name=table, columns=columns.values())
+            TABLES[table] = new_table
 
         mapped_data = _map_data_to_columns(data, columns)
         record = connector_sdk_pb2.Record(
@@ -324,7 +330,6 @@ def _get_columns(table: str) -> dict:
     columns = {}
     if table in TABLES:
         for column in TABLES[table].columns:
-            column.name = get_renamed_column_name(column.name)
             columns[column.name] = column
 
     return columns
@@ -406,10 +411,7 @@ def map_defined_data_type(columns, k, mapped_data, v):
         mapped_data[k] = common_pb2.ValueType(naive_datetime=timestamp)
     elif columns[k].type == common_pb2.DataType.UTC_DATETIME:
         timestamp = timestamp_pb2.Timestamp()
-        if '.' in v:
-            dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%f%z")
-        else:
-            dt = datetime.strptime(v, "%Y-%m-%dT%H:%M:%S%z")
+        dt = v if isinstance(v, datetime) else _parse_datetime_str(v)
         timestamp.FromDatetime(dt)
         mapped_data[k] = common_pb2.ValueType(utc_datetime=timestamp)
     elif columns[k].type == common_pb2.DataType.BINARY:
@@ -425,6 +427,10 @@ def map_defined_data_type(columns, k, mapped_data, v):
         raise ValueError(f"Unsupported data type encountered: {columns[k].type}. Please use valid data types.")
 
 
+def _parse_datetime_str(dt):
+    return datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f%z" if '.' in dt else "%Y-%m-%dT%H:%M:%S%z")
+
+
 def _yield_check(stack):
     """Checks for the presence of 'yield' in the calling code.
     Args:
@@ -456,7 +462,7 @@ def _check_dict(incoming: dict, string_only: bool = False) -> dict:
     Args:
         incoming (dict): The dictionary to validate.
         string_only (bool): Whether to allow only string values.
-    
+
     Returns:
         dict: The validated dictionary.
     """
@@ -527,9 +533,9 @@ def print_library_log(message: str, level: Logging.Level = Logging.Level.INFO):
     if DEBUGGING or EXECUTED_VIA_CLI:
         current_time = datetime.now().strftime("%b %d, %Y %I:%M:%S %p")
         escaped_message = json.dumps(message).strip('"')
-        print(f"{current_time} {level.name}: {escaped_message}")
+        print(f"{current_time} {level.name} {LOGGING_PREFIX}: {escaped_message}")
     else:
-        escaped_message = json.dumps(message)
+        escaped_message = json.dumps(LOGGING_PREFIX + LOGGING_DELIMITER + message)
         log_message = f'{{"level":"{level.name}", "message": {escaped_message}, "message_origin": "library"}}'
         print(log_message)
 
@@ -658,14 +664,20 @@ def get_renamed_table_name(source_table):
     """
     Process a source table name to ensure it conforms to naming rules.
     """
-    return safe_drop_underscores(source_table)
+    if source_table not in RENAMED_TABLE_NAMES:
+        RENAMED_TABLE_NAMES[source_table] = safe_drop_underscores(source_table)
+
+    return RENAMED_TABLE_NAMES[source_table]
 
 
 def get_renamed_column_name(source_column):
     """
     Process a source column name to ensure it conforms to naming rules.
     """
-    return redshift_safe(source_column)
+    if source_column not in RENAMED_COL_NAMES:
+        RENAMED_COL_NAMES[source_column] = redshift_safe(source_column)
+
+    return RENAMED_COL_NAMES[source_column]
 
 class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
     def __init__(self, update, schema=None):
@@ -928,7 +940,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                     self.__update_connection(
                         args, connection_id, connection, group_name, connection_config, deploy_key, hd_agent_id)
                     print("✓")
-                    print_library_log(f"Connector ID: {connection_id}")
+                    print_library_log(f"Connection ID: {connection_id}")
                     print_library_log(
                         f"Visit the Fivetran dashboard to manage the connection: https://fivetran.com/dashboard/connectors/{connection_id}/status")
                 else:
@@ -948,7 +960,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                     connection_id = response.json()['data']['id']
                     # Python Version is not passed into connection_config as default value will be picked up from ConnectorSdkCredentials.java class.
                     print_library_log(f"Python Version: {args.python_version if args.python_version else DEFAULT_PYTHON_VERSION}", Logging.Level.INFO)
-                    print_library_log(f"Connector ID: {connection_id}")
+                    print_library_log(f"Connection ID: {connection_id}")
                     print_library_log(
                         f"Visit the Fivetran dashboard to start the initial sync: https://fivetran.com/dashboard/connectors/{connection_id}/status")
             else:
@@ -989,13 +1001,19 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         if not args.configuration:
             del config["secrets_list"]
 
+        json_payload = {
+            "config": config,
+            "run_setup_tests": True
+        }
+
+        # hybrid_deployment_agent_id is optional when redeploying your connection.
+        # Customer can use it to change existing hybrid_deployment_agent_id.
+        if hd_agent_id:
+            json_payload["hybrid_deployment_agent_id"] = hd_agent_id
+
         response = rq.patch(f"{PRODUCTION_BASE_URL}/v1/connectors/{id}",
                         headers={"Authorization": f"Basic {deploy_key}"},
-                        json={
-                            "config": config,
-                            "hybrid_deployment_agent_id": hd_agent_id,
-                            "run_setup_tests": True
-                        })
+                        json=json_payload)
 
         if response.ok and response.status_code == HTTPStatus.OK:
             if Connector.__are_setup_tests_failing(response):
@@ -1013,7 +1031,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         print_library_log(log_message, Logging.Level.SEVERE)
         Connector.__print_failing_setup_tests(resp)
         connection_id = resp.json().get('data', {}).get('id')
-        print_library_log(f"Connector ID: {connection_id}")
+        print_library_log(f"Connection ID: {connection_id}")
         print_library_log("Please try again with the deploy command after resolving the issue!")
         os._exit(1)
 
@@ -1414,7 +1432,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
         try:
             print_library_log("Running connector tester...")
-            for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, available_port):
+            for log_msg in self.__run_tester(java_exe, tester_root_dir, project_path, available_port, json.dumps(self.state), json.dumps(self.configuration)):
                 print(log_msg, end="")
         except:
             print(traceback.format_exc())
@@ -1455,7 +1473,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 yield line
 
     @staticmethod
-    def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int):
+    def __run_tester(java_exe: str, root_dir: str, project_path: str, port: int, state_json: str, configuration_json: str):
         """Runs the connector tester.
 
         Args:
@@ -1478,7 +1496,9 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                "--connector-sdk=true",
                f"--port={port}",
                f"--working-dir={working_dir}",
-               "--tester-type=source"]
+               "--tester-type=source",
+               f"--state={state_json}",
+               f"--configuration={configuration_json}"]
 
         popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
         for line in Connector.process_stream(popen.stderr):

{fivetran_connector_sdk-1.2.1.dist-info → fivetran_connector_sdk-1.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.2.1
+Version: 1.3.1
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-1.2.1.dist-info → fivetran_connector_sdk-1.3.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fivetran_connector_sdk/__init__.py,sha256=3Kp8MaT8ZtvrOo_dNGFGt5mdumojUSPhYwNL7x6CBvE,80242
+fivetran_connector_sdk/__init__.py,sha256=4ovEZ7G46t5JFEAUmyGNksa0RVs9x5eKsYfzmyBDuNs,81121
 fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
 fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -6,8 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
 fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
 fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
 fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
-fivetran_connector_sdk-1.2.1.dist-info/METADATA,sha256=3h_7qvx3Wv4GvLUTsNnTx2CLWCZpHtVTi-H-vScF4wA,2967
-fivetran_connector_sdk-1.2.1.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-fivetran_connector_sdk-1.2.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-1.2.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-1.2.1.dist-info/RECORD,,
+fivetran_connector_sdk-1.3.1.dist-info/METADATA,sha256=qEnx2X-ac7iQhdJ7bx0p9yKhliRUAz_cxhRRgX3P-Jk,2967
+fivetran_connector_sdk-1.3.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+fivetran_connector_sdk-1.3.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-1.3.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-1.3.1.dist-info/RECORD,,

{fivetran_connector_sdk-1.2.1.dist-info → fivetran_connector_sdk-1.3.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.0.0)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any