dc-python-sdk 1.5.45__tar.gz → 1.5.46__tar.gz

{dc_python_sdk-1.5.45/src/dc_python_sdk.egg-info → dc_python_sdk-1.5.46}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.45
+Version: 1.5.46
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.46}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dc-python-sdk"
-version = "1.5.45"
+version = "1.5.46"
 description = "Data Connector Python SDK"
 readme = "README.md"
 requires-python = ">=3.6"

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.46}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dc-python-sdk
-version = 1.5.45
+version = 1.5.46
 author = DataConnector
 author_email = josh@dataconnector.com
 description = A small example package

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.46/src/dc_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.45
+Version: 1.5.46
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.46}/src/dc_sdk/src/models/pipeline_details.py

@@ -37,6 +37,7 @@ class PipelineDetails:
         self.primary_key_column_nm = row_data.get('primary_key_column_nm')
         self.updated_date_column_nm = row_data.get('updated_date_column_nm')
         self.sync_cursor_dsc = row_data.get('sync_cursor_dsc')
+        self.pending_sync_cursor_dsc = row_data.get('pending_sync_cursor_dsc')
 
     def increment_stage(self):
         self.stage += 1

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.46}/src/dc_sdk/src/pipeline.py

@@ -155,7 +155,7 @@ class PipelineConductor:
             self._update_extracted_sync_cursor(results.get("metadata"))
 
         if self._should_track_sync_cursor():
-            self._upload_extracted_sync_cursor()
+            self._save_pending_sync_cursor()
 
         self.log(self.log_templates.GET_DATA_FINISH.format(self.row_count, self.pipeline_details.source_object_id))
 
@@ -256,61 +256,19 @@ class PipelineConductor:
         if self.extracted_sync_cursor is None or sync_cursor > self.extracted_sync_cursor:
             self.extracted_sync_cursor = sync_cursor
 
-    def _get_sync_cursor_key_name(self):
-        return f"transfers/e{self.pipeline_run_history_id}-sync_cursor.json"
-
-    def _upload_extracted_sync_cursor(self):
+    def _save_pending_sync_cursor(self):
         if self.extracted_sync_cursor is None:
             return
 
-        payload = json.dumps({"sync_cursor": self.extracted_sync_cursor})
-        json_buffer = io.StringIO(payload)
-        key_name = self._get_sync_cursor_key_name()
-
-        if PipelineEnvironment.platform == "aws":
-            self.aws.upload_object(key_name, json_buffer=json_buffer)
-        elif PipelineEnvironment.platform == "azure":
-            key_name = f"{PipelineEnvironment.app_env}/{key_name}"
-            self.azure.upload_object(key_name, json_buffer=json_buffer)
-
-        self.internal_log(f"Uploaded sync cursor sidecar to {key_name}")
-
-    def _load_extracted_sync_cursor(self):
-        key_name = self._get_sync_cursor_key_name()
-
-        try:
-            if PipelineEnvironment.platform == "aws":
-                file_object = self.aws.download_object(key_name)
-            elif PipelineEnvironment.platform == "azure":
-                file_object = self.azure.download_object(
-                    f"{PipelineEnvironment.app_env}/{key_name}"
-                )
-            else:
-                return None
-
-            payload = json.load(file_object)
-            return payload.get("sync_cursor")
-        except Exception as e:
-            self.internal_log(f"No sync cursor sidecar found: {e}")
-            return None
+        if self.mode == "prod":
+            self.api.save_pending_sync_cursor(self.pipeline_id, self.extracted_sync_cursor)
 
     def _persist_sync_cursor_after_load(self):
         if self.task != "DESTINATION":
             return
 
-        sync_cursor = self._load_extracted_sync_cursor()
-        if sync_cursor is None:
-            return
-
         if self.mode == "prod":
-            self.api.save_sync_cursor(self.pipeline_id, sync_cursor)
-
-        if PipelineEnvironment.platform == "aws":
-            self.aws.delete_object(self._get_sync_cursor_key_name())
-        elif PipelineEnvironment.platform == "azure":
-            self.azure.delete_object(
-                f"{PipelineEnvironment.app_env}/{self._get_sync_cursor_key_name()}"
-            )
+            self.api.commit_sync_cursor(self.pipeline_id)
 
     def _call_connector_load_data(
         self,

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.46}/src/dc_sdk/src/services/api.py

@@ -113,6 +113,18 @@ class DataConnectorAPI:
 
         self.post(f"{pipeline_id}/mapping", body)
 
+    def save_pending_sync_cursor(self, pipeline_id, pending_sync_cursor):
+        self.put(
+            f"{pipeline_id}/pending-sync-cursor",
+            {"pending_sync_cursor_dsc": pending_sync_cursor},
+        )
+
+    def commit_sync_cursor(self, pipeline_id):
+        self.put(
+            f"{pipeline_id}/sync-cursor",
+            {"commit": True},
+        )
+
     def save_sync_cursor(self, pipeline_id, sync_cursor):
         self.put(
             f"{pipeline_id}/sync-cursor",