dc-python-sdk 1.5.44__tar.gz → 1.5.45__tar.gz

{dc_python_sdk-1.5.44/src/dc_python_sdk.egg-info → dc_python_sdk-1.5.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.44
+Version: 1.5.45
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.44 → dc_python_sdk-1.5.45}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dc-python-sdk"
-version = "1.5.44"
+version = "1.5.45"
 description = "Data Connector Python SDK"
 readme = "README.md"
 requires-python = ">=3.6"

{dc_python_sdk-1.5.44 → dc_python_sdk-1.5.45}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dc-python-sdk
-version = 1.5.44
+version = 1.5.45
 author = DataConnector
 author_email = josh@dataconnector.com
 description = A small example package

{dc_python_sdk-1.5.44 → dc_python_sdk-1.5.45/src/dc_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.44
+Version: 1.5.45
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.44 → dc_python_sdk-1.5.45}/src/dc_sdk/src/models/pipeline_details.py

@@ -32,9 +32,11 @@ class PipelineDetails:
         self.destination_ecs_task_version_nbr = row_data['destination_ecs_task_version_nbr']
         self.options = json.loads(row_data['pipeline_object_options_json']) if 'pipeline_object_options_json' in row_data and row_data['pipeline_object_options_json'] else dict()
         self.max_allowed_retrieval = row_data.get('max_allowed_retrieval')
-        self.primary_key_column_nm = row_data.get('primary_key_column_nm')
         self.destination_credential_information = row_data.get('destination_credential_information')
         self.source_credential_information = row_data.get('source_credential_information')
+        self.primary_key_column_nm = row_data.get('primary_key_column_nm')
+        self.updated_date_column_nm = row_data.get('updated_date_column_nm')
+        self.sync_cursor_dsc = row_data.get('sync_cursor_dsc')
 
     def increment_stage(self):
         self.stage += 1

{dc_python_sdk-1.5.44 → dc_python_sdk-1.5.45}/src/dc_sdk/src/pipeline.py

@@ -35,6 +35,7 @@ class PipelineConductor:
         self.mapping = kwargs.get("mapping")
         self.pipeline_object_id = kwargs.get("pipeline_mapping_id")
         self.successful_keys = []
+        self.extracted_sync_cursor = None
         self.config = PipelineEnvironment
         self.api = api or DataConnectorAPI()
         self.aws = aws or AwsService(PipelineEnvironment.aws_s3_bucket)
@@ -46,7 +47,7 @@ class PipelineConductor:
         
         # get connector credentials
         Connector = load_connector()
-        self.connector = Connector(self.credentials)
+        self.connector = self._create_connector(Connector, self.credentials)
 
         self.row_count = 0
         self.log_templates = self._get_log_messages()
@@ -131,6 +132,7 @@ class PipelineConductor:
 
         while "next_page" in results and results["next_page"] != None:
             if "data" in results and results["data"] != None and results["data"] != []:
+                self._update_extracted_sync_cursor(results.get("metadata"))
                 limit_reached = self._process_rows(results["data"], max_allowed)
                 if limit_reached:
                     break
@@ -145,9 +147,15 @@ class PipelineConductor:
                 results = self.connector.get_data(self.pipeline_details.source_object_id, self._get_field_ids(), n_rows=nrows, filters=self._get_filters(), options=self.pipeline_details.options, next_page=results["next_page"])
         
         if "data" in results and results["data"] != None and results["data"] != []:
+            self._update_extracted_sync_cursor(results.get("metadata"))
             self._process_rows(results["data"], max_allowed)
         elif results["data"] != []:
             self.internal_log(self.log_templates.INTERNAL_GET_DATA_FETCHED.format(0))
+        else:
+            self._update_extracted_sync_cursor(results.get("metadata"))
+
+        if self._should_track_sync_cursor():
+            self._upload_extracted_sync_cursor()
 
         self.log(self.log_templates.GET_DATA_FINISH.format(self.row_count, self.pipeline_details.source_object_id))
 
@@ -189,6 +197,120 @@ class PipelineConductor:
                 else:
                     raise errors.LoadDataError("Loading data failed.")
         self.log(self.log_templates.LOAD_DATA_FINISHED.format(self.row_count, self.pipeline_details.destination_object_id))
+        self._persist_sync_cursor_after_load()
+
+    def _create_connector(self, connector_cls, credentials):
+        init_params = inspect.signature(connector_cls.__init__).parameters
+        if "pipeline_context" in init_params:
+            return connector_cls(credentials, pipeline_context=self._get_pipeline_context())
+        return connector_cls(credentials)
+
+    def _normalize_update_method(self, update_method):
+        if update_method is None:
+            return None
+        if isinstance(update_method, int):
+            return update_method
+        if isinstance(update_method, str) and update_method.isdigit():
+            return int(update_method)
+        return update_method
+
+    def _get_sync_cursor(self):
+        return getattr(self.pipeline_details, "sync_cursor_dsc", None)
+
+    def _get_pipeline_context(self):
+        if self.task != "SOURCE":
+            return None
+
+        update_method = self._normalize_update_method(
+            getattr(self.pipeline_details, "update_method_cd", None)
+        )
+
+        return {
+            "update_method": update_method,
+            "update_date_column_nm": getattr(
+                self.pipeline_details, "updated_date_column_nm", None
+            ),
+            "sync_cursor": self._get_sync_cursor(),
+        }
+
+    def _should_track_sync_cursor(self):
+        if self.task != "SOURCE":
+            return False
+
+        update_method = self._normalize_update_method(
+            getattr(self.pipeline_details, "update_method_cd", None)
+        )
+        updated_date_column_nm = getattr(
+            self.pipeline_details, "updated_date_column_nm", None
+        )
+        return update_method == 3 and updated_date_column_nm
+
+    def _update_extracted_sync_cursor(self, metadata):
+        if not self._should_track_sync_cursor() or not metadata:
+            return
+
+        sync_cursor = metadata.get("sync_cursor")
+        if sync_cursor is None:
+            return
+
+        if self.extracted_sync_cursor is None or sync_cursor > self.extracted_sync_cursor:
+            self.extracted_sync_cursor = sync_cursor
+
+    def _get_sync_cursor_key_name(self):
+        return f"transfers/e{self.pipeline_run_history_id}-sync_cursor.json"
+
+    def _upload_extracted_sync_cursor(self):
+        if self.extracted_sync_cursor is None:
+            return
+
+        payload = json.dumps({"sync_cursor": self.extracted_sync_cursor})
+        json_buffer = io.StringIO(payload)
+        key_name = self._get_sync_cursor_key_name()
+
+        if PipelineEnvironment.platform == "aws":
+            self.aws.upload_object(key_name, json_buffer=json_buffer)
+        elif PipelineEnvironment.platform == "azure":
+            key_name = f"{PipelineEnvironment.app_env}/{key_name}"
+            self.azure.upload_object(key_name, json_buffer=json_buffer)
+
+        self.internal_log(f"Uploaded sync cursor sidecar to {key_name}")
+
+    def _load_extracted_sync_cursor(self):
+        key_name = self._get_sync_cursor_key_name()
+
+        try:
+            if PipelineEnvironment.platform == "aws":
+                file_object = self.aws.download_object(key_name)
+            elif PipelineEnvironment.platform == "azure":
+                file_object = self.azure.download_object(
+                    f"{PipelineEnvironment.app_env}/{key_name}"
+                )
+            else:
+                return None
+
+            payload = json.load(file_object)
+            return payload.get("sync_cursor")
+        except Exception as e:
+            self.internal_log(f"No sync cursor sidecar found: {e}")
+            return None
+
+    def _persist_sync_cursor_after_load(self):
+        if self.task != "DESTINATION":
+            return
+
+        sync_cursor = self._load_extracted_sync_cursor()
+        if sync_cursor is None:
+            return
+
+        if self.mode == "prod":
+            self.api.save_sync_cursor(self.pipeline_id, sync_cursor)
+
+        if PipelineEnvironment.platform == "aws":
+            self.aws.delete_object(self._get_sync_cursor_key_name())
+        elif PipelineEnvironment.platform == "azure":
+            self.azure.delete_object(
+                f"{PipelineEnvironment.app_env}/{self._get_sync_cursor_key_name()}"
+            )
 
     def _call_connector_load_data(
         self,

{dc_python_sdk-1.5.44 → dc_python_sdk-1.5.45}/src/dc_sdk/src/services/api.py

@@ -113,6 +113,12 @@ class DataConnectorAPI:
 
         self.post(f"{pipeline_id}/mapping", body)
 
+    def save_sync_cursor(self, pipeline_id, sync_cursor):
+        self.put(
+            f"{pipeline_id}/sync-cursor",
+            {"sync_cursor_dsc": sync_cursor},
+        )
+
     def _request(self, method, endpoint, body=None, max_attempts=4):
         endpoint = endpoint.lstrip("/")
         url = f"{self.pipelines_base_url}/{endpoint}"

{dc_python_sdk-1.5.44 → dc_python_sdk-1.5.45}/src/dc_sdk/src/services/aws.py

@@ -164,9 +164,6 @@ class AwsService:
         )
 
 
-
-
-
 class EncryptionService:
     def decrypt_customer_data(
         self,