fivetran-connector-sdk 1.4.5__py3-none-any.whl → 1.5.0__py3-none-any.whl

fivetran_connector_sdk/operations.py

@@ -0,0 +1,264 @@
+import os
+import json
+import inspect
+
+from datetime import datetime
+from google.protobuf import timestamp_pb2
+
+from fivetran_connector_sdk import constants
+from fivetran_connector_sdk.constants import (
+    JAVA_LONG_MAX_VALUE,
+    TABLES,
+)
+from fivetran_connector_sdk.helpers import (
+    get_renamed_table_name,
+    get_renamed_column_name,
+    print_library_log,
+)
+from fivetran_connector_sdk.logger import Logging
+from fivetran_connector_sdk.protos import connector_sdk_pb2, common_pb2
+
+_TYPE_HANDLERS = {
+    common_pb2.DataType.BOOLEAN: lambda val: common_pb2.ValueType(bool=val),
+    common_pb2.DataType.SHORT: lambda val: common_pb2.ValueType(short=val),
+    common_pb2.DataType.INT: lambda val: common_pb2.ValueType(int=val),
+    common_pb2.DataType.LONG: lambda val: common_pb2.ValueType(long=val),
+    common_pb2.DataType.DECIMAL: lambda val: common_pb2.ValueType(decimal=val),
+    common_pb2.DataType.FLOAT: lambda val: common_pb2.ValueType(float=val),
+    common_pb2.DataType.DOUBLE: lambda val: common_pb2.ValueType(double=val),
+    common_pb2.DataType.NAIVE_DATE: lambda val: common_pb2.ValueType(naive_date=
+                                                                     timestamp_pb2.Timestamp().FromDatetime(
+                                                                         datetime.strptime(val, "%Y-%m-%d")) or timestamp_pb2.Timestamp()),
+    common_pb2.DataType.NAIVE_DATETIME: lambda val: common_pb2.ValueType(naive_datetime=
+                                                                         timestamp_pb2.Timestamp().FromDatetime(
+                                                                             datetime.strptime(val + ".0" if '.' not in val else val, "%Y-%m-%dT%H:%M:%S.%f")) or timestamp_pb2.Timestamp()),
+    common_pb2.DataType.UTC_DATETIME: lambda val: common_pb2.ValueType(utc_datetime=
+                                                                       timestamp_pb2.Timestamp().FromDatetime(
+                                                                           val if isinstance(val, datetime) else _parse_datetime_str(val)) or timestamp_pb2.Timestamp()),
+    common_pb2.DataType.BINARY: lambda val: common_pb2.ValueType(binary=val),
+    common_pb2.DataType.XML: lambda val: common_pb2.ValueType(xml=val),
+    common_pb2.DataType.STRING: lambda val: common_pb2.ValueType(string=val if isinstance(val, str) else str(val)),
+    common_pb2.DataType.JSON: lambda val: common_pb2.ValueType(json=json.dumps(val))
+}
+
+class Operations:
+    @staticmethod
+    def upsert(table: str, data: dict) -> list[connector_sdk_pb2.UpdateResponse]:
+        """Updates records with the same primary key if already present in the destination. Inserts new records if not already present in the destination.
+
+        Args:
+            table (str): The name of the table.
+            data (dict): The data to upsert.
+
+        Returns:
+            list[connector_sdk_pb2.UpdateResponse]: A list of update responses.
+        """
+        if constants.DEBUGGING:
+            _yield_check(inspect.stack())
+
+        responses = []
+
+        table = get_renamed_table_name(table)
+        columns = _get_columns(table)
+        if not columns:
+            for field in data.keys():
+                field_name = get_renamed_column_name(field)
+                columns[field_name] = common_pb2.Column(
+                    name=field_name, type=common_pb2.DataType.UNSPECIFIED, primary_key=False)
+            new_table = common_pb2.Table(name=table, columns=columns.values())
+            TABLES[table] = new_table
+
+        mapped_data = _map_data_to_columns(data, columns)
+        record = connector_sdk_pb2.Record(
+            schema_name=None,
+            table_name=table,
+            type=common_pb2.OpType.UPSERT,
+            data=mapped_data
+        )
+
+        responses.append(
+            connector_sdk_pb2.UpdateResponse(
+                operation=connector_sdk_pb2.Operation(record=record)))
+
+        return responses
+
+    @staticmethod
+    def update(table: str, modified: dict) -> connector_sdk_pb2.UpdateResponse:
+        """Performs an update operation on the specified table with the given modified data.
+
+        Args:
+            table (str): The name of the table.
+            modified (dict): The modified data.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: The update response.
+        """
+        if constants.DEBUGGING:
+            _yield_check(inspect.stack())
+
+        table = get_renamed_table_name(table)
+        columns = _get_columns(table)
+        mapped_data = _map_data_to_columns(modified, columns)
+        record = connector_sdk_pb2.Record(
+            schema_name=None,
+            table_name=table,
+            type=common_pb2.OpType.UPDATE,
+            data=mapped_data
+        )
+
+        return connector_sdk_pb2.UpdateResponse(
+            operation=connector_sdk_pb2.Operation(record=record))
+
+    @staticmethod
+    def delete(table: str, keys: dict) -> connector_sdk_pb2.UpdateResponse:
+        """Performs a soft delete operation on the specified table with the given keys.
+
+        Args:
+            table (str): The name of the table.
+            keys (dict): The keys to delete.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: The delete response.
+        """
+        if constants.DEBUGGING:
+            _yield_check(inspect.stack())
+
+        table = get_renamed_table_name(table)
+        columns = _get_columns(table)
+        mapped_data = _map_data_to_columns(keys, columns)
+        record = connector_sdk_pb2.Record(
+            schema_name=None,
+            table_name=table,
+            type=common_pb2.OpType.DELETE,
+            data=mapped_data
+        )
+
+        return connector_sdk_pb2.UpdateResponse(
+            operation=connector_sdk_pb2.Operation(record=record))
+
+    @staticmethod
+    def checkpoint(state: dict) -> connector_sdk_pb2.UpdateResponse:
+        """Checkpoint saves the connector's state. State is a dict which stores information to continue the
+        sync from where it left off in the previous sync. For example, you may choose to have a field called
+        "cursor" with a timestamp value to indicate up to when the data has been synced. This makes it possible
+        for the next sync to fetch data incrementally from that time forward. See below for a few example fields
+        which act as parameters for use by the connector code.\n
+        {
+            "initialSync": true,\n
+            "cursor": "1970-01-01T00:00:00.00Z",\n
+            "last_resync": "1970-01-01T00:00:00.00Z",\n
+            "thread_count": 5,\n
+            "api_quota_left": 5000000
+        }
+
+        Args:
+            state (dict): The state to checkpoint/save.
+
+        Returns:
+            connector_sdk_pb2.UpdateResponse: The checkpoint response.
+        """
+        if constants.DEBUGGING:
+            _yield_check(inspect.stack())
+
+        return connector_sdk_pb2.UpdateResponse(
+            operation=connector_sdk_pb2.Operation(checkpoint=connector_sdk_pb2.Checkpoint(
+                state_json=json.dumps(state))))
+
+def _get_columns(table: str) -> dict:
+    """Retrieves the columns for the specified table.
+
+    Args:
+        table (str): The name of the table.
+
+    Returns:
+        dict: The columns for the table.
+    """
+    columns = {}
+    if table in TABLES:
+        for column in TABLES[table].columns:
+            columns[column.name] = column
+
+    return columns
+
+
+def _map_data_to_columns(data: dict, columns: dict) -> dict:
+    """Maps data to the specified columns.
+
+    Args:
+        data (dict): The data to map.
+        columns (dict): The columns to map the data to.
+
+    Returns:
+        dict: The mapped data.
+    """
+    mapped_data = {}
+    for k, v in data.items():
+        key = get_renamed_column_name(k)
+        if v is None:
+            mapped_data[key] = common_pb2.ValueType(null=True)
+        elif (key in columns) and columns[key].type != common_pb2.DataType.UNSPECIFIED:
+            map_defined_data_type(columns[key].type, key, mapped_data, v)
+        else:
+            map_inferred_data_type(key, mapped_data, v)
+    return mapped_data
+
+def map_inferred_data_type(k, mapped_data, v):
+    # We can infer type from the value
+    if isinstance(v, int):
+        if abs(v) > JAVA_LONG_MAX_VALUE:
+            mapped_data[k] = common_pb2.ValueType(float=v)
+        else:
+            mapped_data[k] = common_pb2.ValueType(long=v)
+    elif isinstance(v, float):
+        mapped_data[k] = common_pb2.ValueType(float=v)
+    elif isinstance(v, bool):
+        mapped_data[k] = common_pb2.ValueType(bool=v)
+    elif isinstance(v, bytes):
+        mapped_data[k] = common_pb2.ValueType(binary=v)
+    elif isinstance(v, list):
+        raise ValueError(
+            "Values for the columns cannot be of type 'list'. Please ensure that all values are of a supported type. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#supporteddatatypes")
+    elif isinstance(v, dict):
+        mapped_data[k] = common_pb2.ValueType(json=json.dumps(v))
+    elif isinstance(v, str):
+        mapped_data[k] = common_pb2.ValueType(string=v)
+    else:
+        # Convert arbitrary objects to string
+        mapped_data[k] = common_pb2.ValueType(string=str(v))
+
+def map_defined_data_type(data_type, k, mapped_data, v):
+    handler = _TYPE_HANDLERS.get(data_type)
+    if handler:
+        mapped_data[k] = handler(v)
+    else:
+        raise ValueError(f"Unsupported data type encountered: {data_type}. Please use valid data types.")
+
+def _parse_datetime_str(dt):
+    return datetime.strptime(dt, "%Y-%m-%dT%H:%M:%S.%f%z" if '.' in dt else "%Y-%m-%dT%H:%M:%S%z")
+
+
+def _yield_check(stack):
+    """Checks for the presence of 'yield' in the calling code.
+    Args:
+        stack: The stack frame to check.
+    """
+
+    # Known issue with inspect.getmodule() and yield behavior in a frozen application.
+    # When using inspect.getmodule() on stack frames obtained by inspect.stack(), it fails
+    # to resolve the modules in a frozen application due to incompatible assumptions about
+    # the file paths. This can lead to unexpected behavior, such as yield returning None or
+    # the failure to retrieve the module inside a frozen app
+    # (Reference: https://github.com/pyinstaller/pyinstaller/issues/5963)
+
+    called_method = stack[0].function
+    calling_code = stack[1].code_context[0]
+    if f"{called_method}(" in calling_code:
+        if 'yield' not in calling_code:
+            print_library_log(
+                f"Please add 'yield' to '{called_method}' operation on line {stack[1].lineno} in file '{stack[1].filename}'", Logging.Level.SEVERE)
+            os._exit(1)
+    else:
+        # This should never happen
+        raise RuntimeError(
+            f"The '{called_method}' function is missing in the connector calling code '{calling_code}'. Please ensure that the '{called_method}' function is properly defined in your code to proceed. Reference: https://fivetran.com/docs/connectors/connector-sdk/technical-reference#technicaldetailsmethods")
+

{fivetran_connector_sdk-1.4.5.dist-info → fivetran_connector_sdk-1.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fivetran_connector_sdk
-Version: 1.4.5
+Version: 1.5.0
 Summary: Build custom connectors on Fivetran platform
 Author-email: Fivetran <developers@fivetran.com>
 Project-URL: Homepage, https://fivetran.com/docs/connectors/connector-sdk
@@ -11,11 +11,11 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: grpcio==1.60.1
-Requires-Dist: grpcio-tools==1.60.1
+Requires-Dist: grpcio==1.71.0
+Requires-Dist: grpcio-tools==1.71.0
 Requires-Dist: requests==2.32.3
 Requires-Dist: pipreqs==0.5.0
-Requires-Dist: unidecode==1.3.8
+Requires-Dist: unidecode==1.4.0
 
 # **fivetran-connector-sdk**
 [![Downloads](https://static.pepy.tech/badge/fivetran-connector-sdk)](https://pepy.tech/project/fivetran-connector-sdk)

fivetran_connector_sdk-1.5.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+fivetran_connector_sdk/__init__.py,sha256=T0icwbaYQoj9hUiOY8zuJCOLU5i3t1Flc3woIq_IDRY,20768
+fivetran_connector_sdk/connector_helper.py,sha256=4NnHlr40dUPdta9gctN0WE8OYBKnZDEJKaurh8e45UE,38960
+fivetran_connector_sdk/constants.py,sha256=tY8-fwB7O11orifFcDIELxfvYNz9-bNUhh6f5cXXws0,2287
+fivetran_connector_sdk/helpers.py,sha256=5rPlqN4uYYLvmNLq0-awp3Xqv1LwyEUqp-TGfXsb9ts,12386
+fivetran_connector_sdk/logger.py,sha256=arI6eNUfm_AvE3EV_PmbSRzrpTVpZKJ8pZMtDn93TKU,3018
+fivetran_connector_sdk/operations.py,sha256=jQyjLNW3L8PfTL8J5VAw68s240euXbBVnGPz0QJ2_AY,11023
+fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
+fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
+fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
+fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
+fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
+fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
+fivetran_connector_sdk-1.5.0.dist-info/METADATA,sha256=vQ1G788vqs7AdJy8JFyDWInLjoUntEZBlzR9LkipE7E,3150
+fivetran_connector_sdk-1.5.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fivetran_connector_sdk-1.5.0.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
+fivetran_connector_sdk-1.5.0.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
+fivetran_connector_sdk-1.5.0.dist-info/RECORD,,

{fivetran_connector_sdk-1.4.5.dist-info → fivetran_connector_sdk-1.5.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

fivetran_connector_sdk-1.4.5.dist-info/RECORD

@@ -1,13 +0,0 @@
-fivetran_connector_sdk/__init__.py,sha256=BF_eAT9h-UCVMX9d-ZDgC8jtOYMAp15gbKzEMPoLMos,86932
-fivetran_connector_sdk/protos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-fivetran_connector_sdk/protos/common_pb2.py,sha256=kUwVcyZHgLigNR-KnHZn7dHrlxaMnUXqzprsRx6T72M,6831
-fivetran_connector_sdk/protos/common_pb2.pyi,sha256=S0hdIzoXyyOKD5cjiGeDDLYpQ9J3LjAvu4rCj1JvJWE,9038
-fivetran_connector_sdk/protos/common_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
-fivetran_connector_sdk/protos/connector_sdk_pb2.py,sha256=9Ke_Ti1s0vAeXapfXT-EryrT2-TSGQb8mhs4gxTpUMk,7732
-fivetran_connector_sdk/protos/connector_sdk_pb2.pyi,sha256=FWYxRgshEF3QDYAE0TM_mv4N2gGvkxCH_uPpxnMc4oA,8406
-fivetran_connector_sdk/protos/connector_sdk_pb2_grpc.py,sha256=ZfJLp4DW7uP4pFOZ74s_wQ6tD3eIPi-08UfnLwe4tzo,7163
-fivetran_connector_sdk-1.4.5.dist-info/METADATA,sha256=oHKyhb91aaZ_obddY7yURX_SXNDqHYocbmi9lIDnlVs,3150
-fivetran_connector_sdk-1.4.5.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-fivetran_connector_sdk-1.4.5.dist-info/entry_points.txt,sha256=uQn0KPnFlQmXJfxlk0tifdNsSXWfVlnAFzNqjXZM_xM,57
-fivetran_connector_sdk-1.4.5.dist-info/top_level.txt,sha256=-_xk2MFY4psIh7jw1lJePMzFb5-vask8_ZtX-UzYWUI,23
-fivetran_connector_sdk-1.4.5.dist-info/RECORD,,