dc-python-sdk 1.5.43__tar.gz → 1.5.44__tar.gz

{dc_python_sdk-1.5.43/src/dc_python_sdk.egg-info → dc_python_sdk-1.5.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.43
+Version: 1.5.44
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.43 → dc_python_sdk-1.5.44}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dc-python-sdk"
-version = "1.5.43"
+version = "1.5.44"
 description = "Data Connector Python SDK"
 readme = "README.md"
 requires-python = ">=3.6"
@@ -45,4 +45,8 @@ ai = [
 dc-sdk = "dc_sdk.cli:main"
 
 [tool.setuptools.packages.find]
-where = ["src"]
+where = ["src"]
+
+[tool.pytest.ini_options]
+pythonpath = ["src", "tests"]
+testpaths = ["tests"]

{dc_python_sdk-1.5.43 → dc_python_sdk-1.5.44}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dc-python-sdk
-version = 1.5.43
+version = 1.5.44
 author = DataConnector
 author_email = josh@dataconnector.com
 description = A small example package

{dc_python_sdk-1.5.43 → dc_python_sdk-1.5.44/src/dc_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.43
+Version: 1.5.44
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.43 → dc_python_sdk-1.5.44}/src/dc_sdk/src/pipeline.py

@@ -1,4 +1,4 @@
-import json, io, math, re
+import json, io, math, re, inspect
 import time
 from .services.environment import PipelineEnvironment
 from .services.api import DataConnectorAPI
@@ -163,7 +163,7 @@ class PipelineConductor:
 
         if not keys:
             # Call load_data with empty data when there are no keys
-            loaded = self.connector.load_data([], self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, 0, 1)
+            loaded = self._call_connector_load_data([], self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, 0, 1)
             if not loaded:
                 raise errors.LoadDataError("Loading data failed.")
         else:
@@ -177,7 +177,7 @@ class PipelineConductor:
                 
                 data = json.load(file_object)
                 self.log(self.log_templates.LOAD_DATA_LOADED.format(len(data), self.pipeline_details.destination_object_id))
-                loaded = self.connector.load_data(data, self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, index, len(keys))
+                loaded = self._call_connector_load_data(data, self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, index, len(keys))
                 if loaded:
                     self.row_count += len(data)
                     if self.mode == "prod":
@@ -190,6 +190,45 @@ class PipelineConductor:
                     raise errors.LoadDataError("Loading data failed.")
         self.log(self.log_templates.LOAD_DATA_FINISHED.format(self.row_count, self.pipeline_details.destination_object_id))
 
+    def _call_connector_load_data(
+        self,
+        data,
+        object_id,
+        mapping,
+        update_method,
+        batch_number,
+        total_batches
+    ):
+        options = getattr(self.pipeline_details, "options", None) or {}
+
+        # Put primary key into options for upsert/incremental loads
+        primary_key_column_nm = getattr(self.pipeline_details, "primary_key_column_nm", None)
+        if primary_key_column_nm:
+            options["primary_key_column_nm"] = primary_key_column_nm
+
+        sig = inspect.signature(self.connector.load_data)
+
+        if "options" in sig.parameters:
+            return self.connector.load_data(
+                data,
+                object_id,
+                mapping,
+                update_method,
+                batch_number,
+                total_batches,
+                options=options
+            )
+
+        return self.connector.load_data(
+            data,
+            object_id,
+            mapping,
+            update_method,
+            batch_number,
+            total_batches
+        )
+
+
     def start_next_connector(self):
         self.internal_log(self.log_templates.INTERNAL_START_NEXT_CONNECTOR.format(self.pipeline_details.destination_connector_nm, self.pipeline_details.destination_object_id))
 
@@ -302,13 +341,30 @@ class PipelineConductor:
         return limit_reached
 
     def _get_credentials(self):
-        encyption_txt = self.pipeline_details.source_encryption_credential_txt if self.task == "SOURCE" else self.pipeline_details.destination_encryption_credential_txt
+        is_source = self.task == "SOURCE"
 
-        if not encyption_txt:
+        encryption_txt = (
+            self.pipeline_details.source_encryption_credential_txt
+            if is_source
+            else self.pipeline_details.destination_encryption_credential_txt
+        )
+
+        if not encryption_txt:
             return None
 
-        # Set Connector Credentials
-        return self.aws.decrypt_customer_data_object(encyption_txt, self.pipeline_details.customer_metadata_uuid)
+        credential_information = (
+            self.pipeline_details.source_credential_information
+            if is_source
+            else self.pipeline_details.destination_credential_information
+        ) or {}
+
+        return self.aws.decrypt_customer_data_object(
+            encrypted_data=encryption_txt,
+            customer_id=self.pipeline_details.customer_metadata_uuid,
+            encrypted_data_key_txt=credential_information.get("encrypted_data_key_txt"),
+            encryption_iv_txt=credential_information.get("encryption_iv_txt"),
+            encryption_auth_tag_txt=credential_information.get("encryption_auth_tag_txt"),
+        )
 
     def _get_pipeline_details(self):
         return self.api.get_pipeline_details(str(self.pipeline_id), self.task, str(self.pipeline_run_history_id), pipeline_mapping_id=self.pipeline_object_id)