dc-python-sdk 1.5.45__tar.gz → 1.5.47__tar.gz

{dc_python_sdk-1.5.45/src/dc_python_sdk.egg-info → dc_python_sdk-1.5.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.45
+Version: 1.5.47
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.47}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dc-python-sdk"
-version = "1.5.45"
+version = "1.5.47"
 description = "Data Connector Python SDK"
 readme = "README.md"
 requires-python = ">=3.6"

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.47}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dc-python-sdk
-version = 1.5.45
+version = 1.5.47
 author = DataConnector
 author_email = josh@dataconnector.com
 description = A small example package

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.47/src/dc_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.45
+Version: 1.5.47
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.47}/src/dc_sdk/src/models/pipeline_details.py

@@ -37,6 +37,7 @@ class PipelineDetails:
         self.primary_key_column_nm = row_data.get('primary_key_column_nm')
         self.updated_date_column_nm = row_data.get('updated_date_column_nm')
         self.sync_cursor_dsc = row_data.get('sync_cursor_dsc')
+        self.pending_sync_cursor_dsc = row_data.get('pending_sync_cursor_dsc')
 
     def increment_stage(self):
         self.stage += 1

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.47}/src/dc_sdk/src/pipeline.py

@@ -155,7 +155,7 @@ class PipelineConductor:
             self._update_extracted_sync_cursor(results.get("metadata"))
 
         if self._should_track_sync_cursor():
-            self._upload_extracted_sync_cursor()
+            self._save_pending_sync_cursor()
 
         self.log(self.log_templates.GET_DATA_FINISH.format(self.row_count, self.pipeline_details.source_object_id))
 
@@ -256,60 +256,58 @@ class PipelineConductor:
         if self.extracted_sync_cursor is None or sync_cursor > self.extracted_sync_cursor:
             self.extracted_sync_cursor = sync_cursor
 
-    def _get_sync_cursor_key_name(self):
-        return f"transfers/e{self.pipeline_run_history_id}-sync_cursor.json"
+    def _save_pending_sync_cursor(self):
+        if not self._should_track_sync_cursor():
+            return
 
-    def _upload_extracted_sync_cursor(self):
         if self.extracted_sync_cursor is None:
+            self.internal_log(
+                "Stay in Sync: no sync cursor value was extracted from source data."
+            )
             return
 
-        payload = json.dumps({"sync_cursor": self.extracted_sync_cursor})
-        json_buffer = io.StringIO(payload)
-        key_name = self._get_sync_cursor_key_name()
-
-        if PipelineEnvironment.platform == "aws":
-            self.aws.upload_object(key_name, json_buffer=json_buffer)
-        elif PipelineEnvironment.platform == "azure":
-            key_name = f"{PipelineEnvironment.app_env}/{key_name}"
-            self.azure.upload_object(key_name, json_buffer=json_buffer)
-
-        self.internal_log(f"Uploaded sync cursor sidecar to {key_name}")
-
-    def _load_extracted_sync_cursor(self):
-        key_name = self._get_sync_cursor_key_name()
+        if self.mode != "prod":
+            self.internal_log(
+                f"Stay in Sync: extracted sync cursor {self.extracted_sync_cursor} (dev mode; not saved)."
+            )
+            return
 
         try:
-            if PipelineEnvironment.platform == "aws":
-                file_object = self.aws.download_object(key_name)
-            elif PipelineEnvironment.platform == "azure":
-                file_object = self.azure.download_object(
-                    f"{PipelineEnvironment.app_env}/{key_name}"
-                )
-            else:
-                return None
-
-            payload = json.load(file_object)
-            return payload.get("sync_cursor")
+            self.api.save_pending_sync_cursor(
+                self.pipeline_id, self.extracted_sync_cursor
+            )
+            self.log(
+                f"Stay in Sync: saved pending sync cursor {self.extracted_sync_cursor}."
+            )
         except Exception as e:
-            self.internal_log(f"No sync cursor sidecar found: {e}")
-            return None
+            self.log(
+                f"Stay in Sync: failed to save pending sync cursor {self.extracted_sync_cursor}. {e}"
+            )
 
     def _persist_sync_cursor_after_load(self):
         if self.task != "DESTINATION":
             return
 
-        sync_cursor = self._load_extracted_sync_cursor()
-        if sync_cursor is None:
+        if self.mode != "prod":
             return
 
-        if self.mode == "prod":
-            self.api.save_sync_cursor(self.pipeline_id, sync_cursor)
-
-        if PipelineEnvironment.platform == "aws":
-            self.aws.delete_object(self._get_sync_cursor_key_name())
-        elif PipelineEnvironment.platform == "azure":
-            self.azure.delete_object(
-                f"{PipelineEnvironment.app_env}/{self._get_sync_cursor_key_name()}"
+        try:
+            result = self.api.commit_sync_cursor(self.pipeline_id)
+            if result.get("committed"):
+                sync_cursor_dsc = result.get("sync_cursor_dsc", "")
+                self.log(
+                    f"Stay in Sync: committed sync cursor {sync_cursor_dsc}."
+                )
+            else:
+                self.log(
+                    "Stay in Sync: destination load finished but no pending sync cursor was committed."
+                )
+        except Exception as e:
+            self.log(
+                f"Stay in Sync: failed to commit sync cursor during destination load. {e}"
+            )
+            self.internal_log(
+                "Stay in Sync: commit will be retried when the pipeline run history is finalized."
             )
 
     def _call_connector_load_data(

{dc_python_sdk-1.5.45 → dc_python_sdk-1.5.47}/src/dc_sdk/src/services/api.py

@@ -113,6 +113,18 @@ class DataConnectorAPI:
 
         self.post(f"{pipeline_id}/mapping", body)
 
+    def save_pending_sync_cursor(self, pipeline_id, pending_sync_cursor):
+        self.put(
+            f"{pipeline_id}/pending-sync-cursor",
+            {"pending_sync_cursor_dsc": pending_sync_cursor},
+        )
+
+    def commit_sync_cursor(self, pipeline_id):
+        self.put(
+            f"{pipeline_id}/sync-cursor",
+            {"commit": True},
+        )
+
     def save_sync_cursor(self, pipeline_id, sync_cursor):
         self.put(
             f"{pipeline_id}/sync-cursor",