fivetran-connector-sdk 0.13.22.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

fivetran_connector_sdk/__init__.py

@@ -28,14 +28,15 @@ from fivetran_connector_sdk.protos import common_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2
 from fivetran_connector_sdk.protos import connector_sdk_pb2_grpc
 
-# Version format: <major_version>.<MM>.<DD>.<iteration> (where MM is incremental value from Jan 2024)
-__version__ = "0.13.22.1"
+# Version format: <major_version>.<minor_version>.<patch_version>
+# (where Major Version = 1 for GA, Minor Version is incremental MM from Jan 25 onwards, Patch Version is incremental within a month)
+__version__ = "1.0.0"
 
 MAC_OS = "mac"
 WIN_OS = "windows"
 LINUX_OS = "linux"
 
-TESTER_VERSION = "0.25.0117.001"
+TESTER_VERSION = "0.25.0130.001"
 TESTER_FILENAME = "run_sdk_tester.jar"
 VERSION_FILENAME = "version.txt"
 UPLOAD_FILENAME = "code.zip"
@@ -76,6 +77,7 @@ DRIVERS = "drivers"
 JAVA_LONG_MAX_VALUE = 9223372036854775807
 MAX_CONFIG_FIELDS = 100
 SUPPORTED_PYTHON_VERSIONS = {"3.12.8", "3.11.11", "3.10.16", "3.9.21"}
+DEFAULT_PYTHON_VERSION = "3.12.8"
 
 
 class Logging:
@@ -161,13 +163,14 @@ class Operations:
 
         responses = []
 
-        table = for_table(table)
+        table = get_renamed_table_name(table)
         columns = _get_columns(table)
         if not columns:
             global TABLES
             for field in data.keys():
-                columns[field] = common_pb2.Column(
-                    name=field, type=common_pb2.DataType.UNSPECIFIED, primary_key=False)
+                field_name = get_renamed_column_name(field)
+                columns[field_name] = common_pb2.Column(
+                    name=field_name, type=common_pb2.DataType.UNSPECIFIED, primary_key=False)
 
         mapped_data = _map_data_to_columns(data, columns)
         record = connector_sdk_pb2.Record(
@@ -196,7 +199,7 @@ class Operations:
         """
         _yield_check(inspect.stack())
 
-        table = for_table(table)
+        table = get_renamed_table_name(table)
         columns = _get_columns(table)
         mapped_data = _map_data_to_columns(modified, columns)
         record = connector_sdk_pb2.Record(
@@ -222,7 +225,7 @@ class Operations:
         """
         _yield_check(inspect.stack())
 
-        table = for_table(table)
+        table = get_renamed_table_name(table)
         columns = _get_columns(table)
         mapped_data = _map_data_to_columns(keys, columns)
         record = connector_sdk_pb2.Record(
@@ -313,6 +316,7 @@ def _get_columns(table: str) -> dict:
     columns = {}
     if table in TABLES:
         for column in TABLES[table].columns:
+            column.name = get_renamed_column_name(column.name)
             columns[column.name] = column
 
     return columns
@@ -330,12 +334,13 @@ def _map_data_to_columns(data: dict, columns: dict) -> dict:
     """
     mapped_data = {}
     for k, v in data.items():
+        key = get_renamed_column_name(k)
         if v is None:
-            mapped_data[k] = common_pb2.ValueType(null=True)
-        elif (k in columns) and columns[k].type != common_pb2.DataType.UNSPECIFIED:
-            map_defined_data_type(columns, k, mapped_data, v)
+            mapped_data[key] = common_pb2.ValueType(null=True)
+        elif (key in columns) and columns[key].type != common_pb2.DataType.UNSPECIFIED:
+            map_defined_data_type(columns, key, mapped_data, v)
         else:
-            map_inferred_data_type(k, mapped_data, v)
+            map_inferred_data_type(key, mapped_data, v)
 
     return mapped_data
 
@@ -643,13 +648,19 @@ def safe_drop_underscores(name):
     return safe_name
 
 
-def for_table(source_table):
+def get_renamed_table_name(source_table):
     """
     Process a source table name to ensure it conforms to naming rules.
     """
     return safe_drop_underscores(source_table)
 
 
+def get_renamed_column_name(source_column):
+    """
+    Process a source column name to ensure it conforms to naming rules.
+    """
+    return redshift_safe(source_column)
+
 class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
     def __init__(self, update, schema=None):
         """Initializes the Connector instance.
@@ -716,14 +727,16 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             force (bool): Force update an existing connection.
 
         """
+        # tmp_requirements is only generated when pipreqs command is successful
+        tmp_requirements_file_path = os.path.join(project_path, 'tmp_requirements.txt')
         # Run the pipreqs command and capture stderr
         attempt = 0
         while attempt < MAX_RETRIES:
             attempt += 1
             result = subprocess.run(
-                ["pipreqs", "--savepath", "tmp_requirements.txt", "--ignore"] + EXCLUDED_PIPREQS_DIRS,
+                ["pipreqs", project_path, "--savepath", tmp_requirements_file_path, "--ignore"] + EXCLUDED_PIPREQS_DIRS,
                 stderr=subprocess.PIPE,
-                text=True # Ensures output is in string format
+                text=True  # Ensures output is in string format
             )
 
             if result.returncode == 0:
@@ -740,9 +753,6 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 print(result.stderr)
                 sys.exit(1)
 
-        # tmp_requirements is only generated when pipreqs command is successful
-        tmp_requirements_file_path = os.path.join(project_path, 'tmp_requirements.txt')
-
         tmp_requirements = self.fetch_requirements_as_dict(self, tmp_requirements_file_path)
         tmp_requirements.pop("fivetran_connector_sdk")
         if tmp_requirements.get('requests') is not None:
@@ -872,17 +882,19 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             else:
                 if args.force:
                     confirm = "y"
+                    if args.configuration:
+                        confirm_config = "y"
                 else:
                     confirm = input(
                         f"The connection '{connection}' already exists in the destination '{group}'. Updating it will overwrite the existing code. Do you want to proceed with the update? (Y/N): ")
-                    if confirm.lower() == "y" and connection_config["secrets_list"]:
+                    if confirm.lower() == "y" and args.configuration:
                         confirm_config = input(f"Your deploy will overwrite the configuration using the values provided in '{args.configuration}': key-value pairs not present in the new configuration will be removed; existing keys' values set in the cofiguration file or in the dashboard will be overwritten with new (empty or non-empty) values; new key-value pairs will be added. Do you want to proceed with the update? (Y/N): ")
                 if confirm.lower() == "y" and (not connection_config["secrets_list"] or (confirm_config.lower() == "y")):
                     print_library_log("Updating the connection...\n")
                     self.__upload_project(
                         args.project_path, deploy_key, group_id, group_name, connection)
                     self.__update_connection(
-                        connection_id, connection, group_name, connection_config, deploy_key)
+                        args, connection_id, connection, group_name, connection_config, deploy_key)
                     print("✓")
                     print_library_log(f"Connector ID: {connection_id}")
                     print_library_log(
@@ -899,6 +911,8 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
                 print_library_log(
                     f"The connection '{connection}' has been created successfully.\n")
                 connection_id = response.json()['data']['id']
+                # Python Version is not passed into connection_config as default value will be picked up from ConnectorSdkCredentials.java class.
+                print_library_log(f"Python Version: {args.python_version if args.python_version else DEFAULT_PYTHON_VERSION}", Logging.Level.INFO)
                 print_library_log(f"Connector ID: {connection_id}")
                 print_library_log(
                     f"Visit the Fivetran dashboard to start the initial sync: https://fivetran.com/dashboard/connectors/{connection_id}/status")
@@ -925,17 +939,18 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             os._exit(1)
 
     @staticmethod
-    def __update_connection(id: str, name: str, group: str, config: dict, deploy_key: str):
+    def __update_connection(args: dict, id: str, name: str, group: str, config: dict, deploy_key: str):
         """Updates the connection with the given ID, name, group, configuration, and deployment key.
 
         Args:
+            args (dict): The command arguments.
             id (str): The connection ID.
             name (str): The connection name.
             group (str): The group name.
             config (dict): The configuration dictionary.
             deploy_key (str): The deployment key.
         """
-        if not config["secrets_list"]:
+        if not args.configuration:
             del config["secrets_list"]
 
         resp = rq.patch(f"{PRODUCTION_BASE_URL}/v1/connectors/{id}",
@@ -1161,7 +1176,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
         if not resp.ok:
             print_library_log(
-                f"Unable to retrieve destination details. The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.", Logging.Level.SEVERE)
+                f"The request failed with status code: {resp.status_code}. Please ensure you're using a valid base64-encoded API key and try again.", Logging.Level.SEVERE)
             os._exit(1)
 
         data = resp.json().get("data", {})
@@ -1234,7 +1249,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
               project_path: str = None,
               configuration: dict = None,
               state: dict = None,
-              log_level: Logging.Level = Logging.Level.FINE) -> bool:
+              log_level: Logging.Level = Logging.Level.FINE):
         """Tests the connector code by running it with the connector tester.\n
         state.json docs: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithstatejsonfile\n
         configuration.json docs: https://fivetran.com/docs/connectors/connector-sdk/detailed-guide#workingwithconfigurationjsonfile
@@ -1244,9 +1259,6 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             configuration (dict): The configuration dictionary, same as configuration.json if present.
             state (dict): The state dictionary, same as state.json if present.
             log_level (Logging.Level): The logging level.
-
-        Returns:
-            bool: True if there was an error, False otherwise.
         """
         global DEBUGGING
         DEBUGGING = True
@@ -1443,7 +1455,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         else:
             try:
                 configuration = self.configuration if self.configuration else request.configuration
-                print_library_log("Initiating the 'schema' method call...", Logging.Level.WARNING)
+                print_library_log("Initiating the 'schema' method call...", Logging.Level.INFO)
                 response = self.schema_method(configuration)
                 self.process_tables(response)
                 return connector_sdk_pb2.SchemaResponse(without_schema=common_pb2.TableList(tables=TABLES.values()))
@@ -1459,7 +1471,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             if 'table' not in entry:
                 raise ValueError("Entry missing table name: " + entry)
 
-            table_name = for_table(entry['table'])
+            table_name = get_renamed_table_name(entry['table'])
 
             if table_name in TABLES:
                 raise ValueError("Table already defined: " + table_name)
@@ -1478,13 +1490,15 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
 
     def process_primary_keys(self, columns, entry):
         for pkey_name in entry["primary_key"]:
-            column = columns[pkey_name] if pkey_name in columns else common_pb2.Column(name=pkey_name)
+            column_name = get_renamed_column_name(pkey_name)
+            column = columns[column_name] if column_name in columns else common_pb2.Column(name=column_name)
             column.primary_key = True
-            columns[pkey_name] = column
+            columns[column_name] = column
 
     def process_columns(self, columns, entry):
         for name, type in entry["columns"].items():
-            column = columns[name] if name in columns else common_pb2.Column(name=name)
+            column_name = get_renamed_column_name(name)
+            column = columns[column_name] if column_name in columns else common_pb2.Column(name=column_name)
 
             if isinstance(type, str):
                 self.process_data_type(column, type)
@@ -1502,7 +1516,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
             if "primary_key" in entry and name in entry["primary_key"]:
                 column.primary_key = True
 
-            columns[name] = column
+            columns[column_name] = column
 
     def process_data_type(self, column, type):
         if type.upper() == "BOOLEAN":
@@ -1550,7 +1564,7 @@ class Connector(connector_sdk_pb2_grpc.ConnectorServicer):
         state = self.state if self.state else json.loads(request.state_json)
 
         try:
-            print_library_log("Initiating the 'update' method call...", Logging.Level.WARNING)
+            print_library_log("Initiating the 'update' method call...", Logging.Level.INFO)
             for resp in self.update_method(configuration=configuration, state=state):
                 if isinstance(resp, list):
                     for r in resp:
@@ -1667,7 +1681,8 @@ def main():
     global EXECUTED_VIA_CLI
     EXECUTED_VIA_CLI = True
 
-    parser = argparse.ArgumentParser(allow_abbrev=False)
+    parser = argparse.ArgumentParser(allow_abbrev=False, add_help=True)
+    parser._option_string_actions["-h"].help = "Show this help message and exit"
 
     # Positional
     parser.add_argument("command", help="|".join(VALID_COMMANDS))
@@ -1676,7 +1691,7 @@ def main():
     # Optional (Not all of these are valid with every mutually exclusive option below)
     parser.add_argument("--state", type=str, default=None, help="Provide state as JSON string or file")
     parser.add_argument("--configuration", type=str, default=None, help="Provide secrets as JSON file")
-    parser.add_argument("--api-key", type=str, default=None, help="Provide api key for deployment to production")
+    parser.add_argument("--api-key", type=str, default=None, help="Provide your base64-encoded API key for deployment")
     parser.add_argument("--destination", type=str, default=None, help="Destination name (aka 'group name')")
     parser.add_argument("--connection", type=str, default=None, help="Connection name (aka 'destination schema')")
     parser.add_argument("-f", "--force", action="store_true", help="Force update an existing connection")

{fivetran_connector_sdk-0.13.22.1.dist-info → fivetran_connector_sdk-1.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: fivetran_connector_sdk
-Version: 0.13.22.1
+Version: 1.0.0
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk

{fivetran_connector_sdk-0.13.22.1.dist-info → fivetran_connector_sdk-1.0.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-fivetran_connector_sdk/__init__.py,sha256=dUvOznhj7qZvJCMTnHMUiwTJBzmEGH10-IaG6ZD6gVo,73066
+fivetran_connector_sdk/__init__.py,sha256=Nx4fuu2Q901W2ecDr7f9-UgkRjsM4Oz0sYX0XVrChkA,74181
 fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
 fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
@@ -6,8 +6,8 @@ fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
 fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
 fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
 fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
-fivetran_connector_sdk-0.13.22.1.dist-info/METADATA,sha256=Oh5malMU3O9KjmJmsoLobGVq1Eq0_lC81Dd-QlIO8eM,2971
-fivetran_connector_sdk-0.13.22.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-fivetran_connector_sdk-0.13.22.1.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-0.13.22.1.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-0.13.22.1.dist-info/RECORD,,
+fivetran_connector_sdk-1.0.0.dist-info/METADATA,sha256=vcXCf52OxYYGX-uqK0gG52I382Iu_UNASzRAruUtshw,2967
+fivetran_connector_sdk-1.0.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+fivetran_connector_sdk-1.0.0.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-1.0.0.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-1.0.0.dist-info/RECORD,,