dc-python-sdk 1.5.43__tar.gz → 1.5.45__tar.gz

{dc_python_sdk-1.5.43/src/dc_python_sdk.egg-info → dc_python_sdk-1.5.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.43
+Version: 1.5.45
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.43 → dc_python_sdk-1.5.45}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dc-python-sdk"
-version = "1.5.43"
+version = "1.5.45"
 description = "Data Connector Python SDK"
 readme = "README.md"
 requires-python = ">=3.6"
@@ -45,4 +45,8 @@ ai = [
 dc-sdk = "dc_sdk.cli:main"
 
 [tool.setuptools.packages.find]
-where = ["src"]
+where = ["src"]
+
+[tool.pytest.ini_options]
+pythonpath = ["src", "tests"]
+testpaths = ["tests"]

{dc_python_sdk-1.5.43 → dc_python_sdk-1.5.45}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dc-python-sdk
-version = 1.5.43
+version = 1.5.45
 author = DataConnector
 author_email = josh@dataconnector.com
 description = A small example package

{dc_python_sdk-1.5.43 → dc_python_sdk-1.5.45/src/dc_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.43
+Version: 1.5.45
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.43 → dc_python_sdk-1.5.45}/src/dc_sdk/src/models/pipeline_details.py

@@ -32,9 +32,11 @@ class PipelineDetails:
         self.destination_ecs_task_version_nbr = row_data['destination_ecs_task_version_nbr']
         self.options = json.loads(row_data['pipeline_object_options_json']) if 'pipeline_object_options_json' in row_data and row_data['pipeline_object_options_json'] else dict()
         self.max_allowed_retrieval = row_data.get('max_allowed_retrieval')
-        self.primary_key_column_nm = row_data.get('primary_key_column_nm')
         self.destination_credential_information = row_data.get('destination_credential_information')
         self.source_credential_information = row_data.get('source_credential_information')
+        self.primary_key_column_nm = row_data.get('primary_key_column_nm')
+        self.updated_date_column_nm = row_data.get('updated_date_column_nm')
+        self.sync_cursor_dsc = row_data.get('sync_cursor_dsc')
 
     def increment_stage(self):
         self.stage += 1

{dc_python_sdk-1.5.43 → dc_python_sdk-1.5.45}/src/dc_sdk/src/pipeline.py

@@ -1,4 +1,4 @@
-import json, io, math, re
+import json, io, math, re, inspect
 import time
 from .services.environment import PipelineEnvironment
 from .services.api import DataConnectorAPI
@@ -35,6 +35,7 @@ class PipelineConductor:
         self.mapping = kwargs.get("mapping")
         self.pipeline_object_id = kwargs.get("pipeline_mapping_id")
         self.successful_keys = []
+        self.extracted_sync_cursor = None
         self.config = PipelineEnvironment
         self.api = api or DataConnectorAPI()
         self.aws = aws or AwsService(PipelineEnvironment.aws_s3_bucket)
@@ -46,7 +47,7 @@ class PipelineConductor:
         
         # get connector credentials
         Connector = load_connector()
-        self.connector = Connector(self.credentials)
+        self.connector = self._create_connector(Connector, self.credentials)
 
         self.row_count = 0
         self.log_templates = self._get_log_messages()
@@ -131,6 +132,7 @@ class PipelineConductor:
 
         while "next_page" in results and results["next_page"] != None:
             if "data" in results and results["data"] != None and results["data"] != []:
+                self._update_extracted_sync_cursor(results.get("metadata"))
                 limit_reached = self._process_rows(results["data"], max_allowed)
                 if limit_reached:
                     break
@@ -145,9 +147,15 @@ class PipelineConductor:
                 results = self.connector.get_data(self.pipeline_details.source_object_id, self._get_field_ids(), n_rows=nrows, filters=self._get_filters(), options=self.pipeline_details.options, next_page=results["next_page"])
         
         if "data" in results and results["data"] != None and results["data"] != []:
+            self._update_extracted_sync_cursor(results.get("metadata"))
             self._process_rows(results["data"], max_allowed)
         elif results["data"] != []:
             self.internal_log(self.log_templates.INTERNAL_GET_DATA_FETCHED.format(0))
+        else:
+            self._update_extracted_sync_cursor(results.get("metadata"))
+
+        if self._should_track_sync_cursor():
+            self._upload_extracted_sync_cursor()
 
         self.log(self.log_templates.GET_DATA_FINISH.format(self.row_count, self.pipeline_details.source_object_id))
 
@@ -163,7 +171,7 @@ class PipelineConductor:
 
         if not keys:
             # Call load_data with empty data when there are no keys
-            loaded = self.connector.load_data([], self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, 0, 1)
+            loaded = self._call_connector_load_data([], self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, 0, 1)
             if not loaded:
                 raise errors.LoadDataError("Loading data failed.")
         else:
@@ -177,7 +185,7 @@ class PipelineConductor:
                 
                 data = json.load(file_object)
                 self.log(self.log_templates.LOAD_DATA_LOADED.format(len(data), self.pipeline_details.destination_object_id))
-                loaded = self.connector.load_data(data, self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, index, len(keys))
+                loaded = self._call_connector_load_data(data, self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, index, len(keys))
                 if loaded:
                     self.row_count += len(data)
                     if self.mode == "prod":
@@ -189,6 +197,159 @@ class PipelineConductor:
                 else:
                     raise errors.LoadDataError("Loading data failed.")
         self.log(self.log_templates.LOAD_DATA_FINISHED.format(self.row_count, self.pipeline_details.destination_object_id))
+        self._persist_sync_cursor_after_load()
+
+    def _create_connector(self, connector_cls, credentials):
+        init_params = inspect.signature(connector_cls.__init__).parameters
+        if "pipeline_context" in init_params:
+            return connector_cls(credentials, pipeline_context=self._get_pipeline_context())
+        return connector_cls(credentials)
+
+    def _normalize_update_method(self, update_method):
+        if update_method is None:
+            return None
+        if isinstance(update_method, int):
+            return update_method
+        if isinstance(update_method, str) and update_method.isdigit():
+            return int(update_method)
+        return update_method
+
+    def _get_sync_cursor(self):
+        return getattr(self.pipeline_details, "sync_cursor_dsc", None)
+
+    def _get_pipeline_context(self):
+        if self.task != "SOURCE":
+            return None
+
+        update_method = self._normalize_update_method(
+            getattr(self.pipeline_details, "update_method_cd", None)
+        )
+
+        return {
+            "update_method": update_method,
+            "update_date_column_nm": getattr(
+                self.pipeline_details, "updated_date_column_nm", None
+            ),
+            "sync_cursor": self._get_sync_cursor(),
+        }
+
+    def _should_track_sync_cursor(self):
+        if self.task != "SOURCE":
+            return False
+
+        update_method = self._normalize_update_method(
+            getattr(self.pipeline_details, "update_method_cd", None)
+        )
+        updated_date_column_nm = getattr(
+            self.pipeline_details, "updated_date_column_nm", None
+        )
+        return update_method == 3 and updated_date_column_nm
+
+    def _update_extracted_sync_cursor(self, metadata):
+        if not self._should_track_sync_cursor() or not metadata:
+            return
+
+        sync_cursor = metadata.get("sync_cursor")
+        if sync_cursor is None:
+            return
+
+        if self.extracted_sync_cursor is None or sync_cursor > self.extracted_sync_cursor:
+            self.extracted_sync_cursor = sync_cursor
+
+    def _get_sync_cursor_key_name(self):
+        return f"transfers/e{self.pipeline_run_history_id}-sync_cursor.json"
+
+    def _upload_extracted_sync_cursor(self):
+        if self.extracted_sync_cursor is None:
+            return
+
+        payload = json.dumps({"sync_cursor": self.extracted_sync_cursor})
+        json_buffer = io.StringIO(payload)
+        key_name = self._get_sync_cursor_key_name()
+
+        if PipelineEnvironment.platform == "aws":
+            self.aws.upload_object(key_name, json_buffer=json_buffer)
+        elif PipelineEnvironment.platform == "azure":
+            key_name = f"{PipelineEnvironment.app_env}/{key_name}"
+            self.azure.upload_object(key_name, json_buffer=json_buffer)
+
+        self.internal_log(f"Uploaded sync cursor sidecar to {key_name}")
+
+    def _load_extracted_sync_cursor(self):
+        key_name = self._get_sync_cursor_key_name()
+
+        try:
+            if PipelineEnvironment.platform == "aws":
+                file_object = self.aws.download_object(key_name)
+            elif PipelineEnvironment.platform == "azure":
+                file_object = self.azure.download_object(
+                    f"{PipelineEnvironment.app_env}/{key_name}"
+                )
+            else:
+                return None
+
+            payload = json.load(file_object)
+            return payload.get("sync_cursor")
+        except Exception as e:
+            self.internal_log(f"No sync cursor sidecar found: {e}")
+            return None
+
+    def _persist_sync_cursor_after_load(self):
+        if self.task != "DESTINATION":
+            return
+
+        sync_cursor = self._load_extracted_sync_cursor()
+        if sync_cursor is None:
+            return
+
+        if self.mode == "prod":
+            self.api.save_sync_cursor(self.pipeline_id, sync_cursor)
+
+        if PipelineEnvironment.platform == "aws":
+            self.aws.delete_object(self._get_sync_cursor_key_name())
+        elif PipelineEnvironment.platform == "azure":
+            self.azure.delete_object(
+                f"{PipelineEnvironment.app_env}/{self._get_sync_cursor_key_name()}"
+            )
+
+    def _call_connector_load_data(
+        self,
+        data,
+        object_id,
+        mapping,
+        update_method,
+        batch_number,
+        total_batches
+    ):
+        options = getattr(self.pipeline_details, "options", None) or {}
+
+        # Put primary key into options for upsert/incremental loads
+        primary_key_column_nm = getattr(self.pipeline_details, "primary_key_column_nm", None)
+        if primary_key_column_nm:
+            options["primary_key_column_nm"] = primary_key_column_nm
+
+        sig = inspect.signature(self.connector.load_data)
+
+        if "options" in sig.parameters:
+            return self.connector.load_data(
+                data,
+                object_id,
+                mapping,
+                update_method,
+                batch_number,
+                total_batches,
+                options=options
+            )
+
+        return self.connector.load_data(
+            data,
+            object_id,
+            mapping,
+            update_method,
+            batch_number,
+            total_batches
+        )
+
 
     def start_next_connector(self):
         self.internal_log(self.log_templates.INTERNAL_START_NEXT_CONNECTOR.format(self.pipeline_details.destination_connector_nm, self.pipeline_details.destination_object_id))
@@ -302,13 +463,30 @@ class PipelineConductor:
         return limit_reached
 
     def _get_credentials(self):
-        encyption_txt = self.pipeline_details.source_encryption_credential_txt if self.task == "SOURCE" else self.pipeline_details.destination_encryption_credential_txt
+        is_source = self.task == "SOURCE"
 
-        if not encyption_txt:
+        encryption_txt = (
+            self.pipeline_details.source_encryption_credential_txt
+            if is_source
+            else self.pipeline_details.destination_encryption_credential_txt
+        )
+
+        if not encryption_txt:
             return None
 
-        # Set Connector Credentials
-        return self.aws.decrypt_customer_data_object(encyption_txt, self.pipeline_details.customer_metadata_uuid)
+        credential_information = (
+            self.pipeline_details.source_credential_information
+            if is_source
+            else self.pipeline_details.destination_credential_information
+        ) or {}
+
+        return self.aws.decrypt_customer_data_object(
+            encrypted_data=encryption_txt,
+            customer_id=self.pipeline_details.customer_metadata_uuid,
+            encrypted_data_key_txt=credential_information.get("encrypted_data_key_txt"),
+            encryption_iv_txt=credential_information.get("encryption_iv_txt"),
+            encryption_auth_tag_txt=credential_information.get("encryption_auth_tag_txt"),
+        )
 
     def _get_pipeline_details(self):
         return self.api.get_pipeline_details(str(self.pipeline_id), self.task, str(self.pipeline_run_history_id), pipeline_mapping_id=self.pipeline_object_id)

{dc_python_sdk-1.5.43 → dc_python_sdk-1.5.45}/src/dc_sdk/src/services/api.py

@@ -113,6 +113,12 @@ class DataConnectorAPI:
 
         self.post(f"{pipeline_id}/mapping", body)
 
+    def save_sync_cursor(self, pipeline_id, sync_cursor):
+        self.put(
+            f"{pipeline_id}/sync-cursor",
+            {"sync_cursor_dsc": sync_cursor},
+        )
+
     def _request(self, method, endpoint, body=None, max_attempts=4):
         endpoint = endpoint.lstrip("/")
         url = f"{self.pipelines_base_url}/{endpoint}"

{dc_python_sdk-1.5.43 → dc_python_sdk-1.5.45}/src/dc_sdk/src/services/aws.py

@@ -164,9 +164,6 @@ class AwsService:
         )
 
 
-
-
-
 class EncryptionService:
     def decrypt_customer_data(
         self,