dc-python-sdk 1.5.45__py3-none-any.whl → 1.5.47__py3-none-any.whl

{dc_python_sdk-1.5.45.dist-info → dc_python_sdk-1.5.47.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.45
+Version: 1.5.47
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.45.dist-info → dc_python_sdk-1.5.47.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dc_python_sdk-1.5.45.dist-info/licenses/LICENSE,sha256=ACwmltkrXIz5VsEQcrqljq-fat6ZXAMepjXGoe40KtE,1069
+dc_python_sdk-1.5.47.dist-info/licenses/LICENSE,sha256=ACwmltkrXIz5VsEQcrqljq-fat6ZXAMepjXGoe40KtE,1069
 dc_sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dc_sdk/app.py,sha256=VeGe1q2F71XtZp9VWBUlNowGcF6J-gYsqOu7k2CaeuU,8320
 dc_sdk/cli.py,sha256=fu4ePzpurYO_URvxyaaUuiptOLqiWiwLoieSbKeni3c,891
@@ -9,22 +9,22 @@ dc_sdk/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dc_sdk/src/ai.py,sha256=aIKN6TiBKF8ZJYNpcphwWNqtI34Ni36UipxT-AEyQ1I,23321
 dc_sdk/src/ai_http.py,sha256=9c1X4Sr-2uutTrovHSZVmAacLLwcmFaQ8vA-tOKC0L0,24696
 dc_sdk/src/mapping.py,sha256=NuXrdE1MuRVRT7ILpc86RrVJq60t6Dl_lnlG1A7Ulks,3653
-dc_sdk/src/pipeline.py,sha256=nc7sXc-qNHisUQYdONNB7uosrN69ezNzv8fSiQRRNck,27262
+dc_sdk/src/pipeline.py,sha256=gjStYsUYbI_WOwfn9QpAbnoBL2yEwBsrfaDB61ix0MQ,27032
 dc_sdk/src/server.py,sha256=2ZhITJhVcj-jd4a-768lAmmmDjTWATGmgwKD-1OVOVI,3028
 dc_sdk/src/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dc_sdk/src/models/enums.py,sha256=UwUH7Bnu9SMyl_sgkfhrgFsuA7rM-eQV-b0PEjhTs_U,981
 dc_sdk/src/models/errors.py,sha256=uD7EDiQUFUZWjYfowJC4cx6Z0X53UoD1ckh7LD2hcWk,11413
 dc_sdk/src/models/log_templates.py,sha256=gNmU8c9V-fjiraCMSKi3GfojZ6KjJ4EHQst3-G_9OCE,3111
-dc_sdk/src/models/pipeline_details.py,sha256=mZPtPw0X-_dctARCV_i8PJRpHgjl4OMk_bplSNGOltg,2859
+dc_sdk/src/models/pipeline_details.py,sha256=025uh8jKWlfgdwYyrS6vXgl8SACXtD3UeoBzEWiaUSw,2938
 dc_sdk/src/services/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dc_sdk/src/services/api.py,sha256=mXX_ELmHWEUgWpYoEzR010LnXXCyB-RrbTmkMnw9o58,7302
+dc_sdk/src/services/api.py,sha256=lI6O4JmlCtNPnSVSwuUJqjIuWe1jRQCZNHCw8pXk3K8,7665
 dc_sdk/src/services/aws.py,sha256=zYAbTAjQ4bAO_YLfPYCVhdNz9SC_nCy4vAm8HdteHwY,9411
 dc_sdk/src/services/environment.py,sha256=lKOuDyzFbjdriPVeVy1GGeuZjutzhwUsZCQcF84XyHM,6447
 dc_sdk/src/services/loader.py,sha256=0zlQm4HOPKffN9Q6YlMv73fCMBsrwkEBnka5rEJTvqU,1175
 dc_sdk/src/services/logger.py,sha256=GrQGfgYfrzmPYFRHkctDSg54jvN7ObveETTPezYldXs,2055
 dc_sdk/src/services/session.py,sha256=SpaZ_qnxMbEZO5fxsbjw6WZXVP5JpncnWalgLqB3MYo,423
-dc_python_sdk-1.5.45.dist-info/METADATA,sha256=syZWEl2LAbT7SEs1eMhok9MhIGvle5FL8pWslPDAVA0,15980
-dc_python_sdk-1.5.45.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-dc_python_sdk-1.5.45.dist-info/entry_points.txt,sha256=HSIKPlg7qer7Atu8oOKVpptahi_ZxqfJE-Bh2bmVSss,43
-dc_python_sdk-1.5.45.dist-info/top_level.txt,sha256=gD_FZnF6mZfFU5EmdB2knYHDHKfH5kXJMtSgEZga3BI,7
-dc_python_sdk-1.5.45.dist-info/RECORD,,
+dc_python_sdk-1.5.47.dist-info/METADATA,sha256=jjnO75U_GV1ovT2iCGUde9PVZaOGAWzj3UpQhjRh1ZQ,15980
+dc_python_sdk-1.5.47.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+dc_python_sdk-1.5.47.dist-info/entry_points.txt,sha256=HSIKPlg7qer7Atu8oOKVpptahi_ZxqfJE-Bh2bmVSss,43
+dc_python_sdk-1.5.47.dist-info/top_level.txt,sha256=gD_FZnF6mZfFU5EmdB2knYHDHKfH5kXJMtSgEZga3BI,7
+dc_python_sdk-1.5.47.dist-info/RECORD,,

dc_sdk/src/models/pipeline_details.py

@@ -37,6 +37,7 @@ class PipelineDetails:
         self.primary_key_column_nm = row_data.get('primary_key_column_nm')
         self.updated_date_column_nm = row_data.get('updated_date_column_nm')
         self.sync_cursor_dsc = row_data.get('sync_cursor_dsc')
+        self.pending_sync_cursor_dsc = row_data.get('pending_sync_cursor_dsc')
 
     def increment_stage(self):
         self.stage += 1

dc_sdk/src/pipeline.py

@@ -155,7 +155,7 @@ class PipelineConductor:
             self._update_extracted_sync_cursor(results.get("metadata"))
 
         if self._should_track_sync_cursor():
-            self._upload_extracted_sync_cursor()
+            self._save_pending_sync_cursor()
 
         self.log(self.log_templates.GET_DATA_FINISH.format(self.row_count, self.pipeline_details.source_object_id))
 
@@ -256,60 +256,58 @@ class PipelineConductor:
         if self.extracted_sync_cursor is None or sync_cursor > self.extracted_sync_cursor:
             self.extracted_sync_cursor = sync_cursor
 
-    def _get_sync_cursor_key_name(self):
-        return f"transfers/e{self.pipeline_run_history_id}-sync_cursor.json"
+    def _save_pending_sync_cursor(self):
+        if not self._should_track_sync_cursor():
+            return
 
-    def _upload_extracted_sync_cursor(self):
         if self.extracted_sync_cursor is None:
+            self.internal_log(
+                "Stay in Sync: no sync cursor value was extracted from source data."
+            )
             return
 
-        payload = json.dumps({"sync_cursor": self.extracted_sync_cursor})
-        json_buffer = io.StringIO(payload)
-        key_name = self._get_sync_cursor_key_name()
-
-        if PipelineEnvironment.platform == "aws":
-            self.aws.upload_object(key_name, json_buffer=json_buffer)
-        elif PipelineEnvironment.platform == "azure":
-            key_name = f"{PipelineEnvironment.app_env}/{key_name}"
-            self.azure.upload_object(key_name, json_buffer=json_buffer)
-
-        self.internal_log(f"Uploaded sync cursor sidecar to {key_name}")
-
-    def _load_extracted_sync_cursor(self):
-        key_name = self._get_sync_cursor_key_name()
+        if self.mode != "prod":
+            self.internal_log(
+                f"Stay in Sync: extracted sync cursor {self.extracted_sync_cursor} (dev mode; not saved)."
+            )
+            return
 
         try:
-            if PipelineEnvironment.platform == "aws":
-                file_object = self.aws.download_object(key_name)
-            elif PipelineEnvironment.platform == "azure":
-                file_object = self.azure.download_object(
-                    f"{PipelineEnvironment.app_env}/{key_name}"
-                )
-            else:
-                return None
-
-            payload = json.load(file_object)
-            return payload.get("sync_cursor")
+            self.api.save_pending_sync_cursor(
+                self.pipeline_id, self.extracted_sync_cursor
+            )
+            self.log(
+                f"Stay in Sync: saved pending sync cursor {self.extracted_sync_cursor}."
+            )
         except Exception as e:
-            self.internal_log(f"No sync cursor sidecar found: {e}")
-            return None
+            self.log(
+                f"Stay in Sync: failed to save pending sync cursor {self.extracted_sync_cursor}. {e}"
+            )
 
     def _persist_sync_cursor_after_load(self):
         if self.task != "DESTINATION":
             return
 
-        sync_cursor = self._load_extracted_sync_cursor()
-        if sync_cursor is None:
+        if self.mode != "prod":
             return
 
-        if self.mode == "prod":
-            self.api.save_sync_cursor(self.pipeline_id, sync_cursor)
-
-        if PipelineEnvironment.platform == "aws":
-            self.aws.delete_object(self._get_sync_cursor_key_name())
-        elif PipelineEnvironment.platform == "azure":
-            self.azure.delete_object(
-                f"{PipelineEnvironment.app_env}/{self._get_sync_cursor_key_name()}"
+        try:
+            result = self.api.commit_sync_cursor(self.pipeline_id)
+            if result.get("committed"):
+                sync_cursor_dsc = result.get("sync_cursor_dsc", "")
+                self.log(
+                    f"Stay in Sync: committed sync cursor {sync_cursor_dsc}."
+                )
+            else:
+                self.log(
+                    "Stay in Sync: destination load finished but no pending sync cursor was committed."
+                )
+        except Exception as e:
+            self.log(
+                f"Stay in Sync: failed to commit sync cursor during destination load. {e}"
+            )
+            self.internal_log(
+                "Stay in Sync: commit will be retried when the pipeline run history is finalized."
             )
 
     def _call_connector_load_data(

dc_sdk/src/services/api.py

@@ -113,6 +113,18 @@ class DataConnectorAPI:
 
         self.post(f"{pipeline_id}/mapping", body)
 
+    def save_pending_sync_cursor(self, pipeline_id, pending_sync_cursor):
+        self.put(
+            f"{pipeline_id}/pending-sync-cursor",
+            {"pending_sync_cursor_dsc": pending_sync_cursor},
+        )
+
+    def commit_sync_cursor(self, pipeline_id):
+        self.put(
+            f"{pipeline_id}/sync-cursor",
+            {"commit": True},
+        )
+
     def save_sync_cursor(self, pipeline_id, sync_cursor):
         self.put(
             f"{pipeline_id}/sync-cursor",