airbyte-source-shopify 2.5.12.dev202412101009__tar.gz → 2.5.13__tar.gz

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 2.5.12.dev202412101009
+Version: 2.5.13
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2
@@ -11,7 +11,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=5,<6)
+Requires-Dist: airbyte-cdk (>=6,<7)
 Requires-Dist: graphql-query (>=1,<2)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/shopify

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.13}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.5.12.dev202412101009"
+version = "2.5.13"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [
@@ -22,7 +22,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.10,<3.12"
-airbyte-cdk = "^5"
+airbyte-cdk = "^6"
 sgqlc = "==16.3"
 graphql-query = "^1"
 

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.13}/source_shopify/shopify_graphql/bulk/exceptions.py

@@ -52,6 +52,11 @@ class ShopifyBulkExceptions:
 
         failure_type: FailureType = FailureType.transient_error
 
+    class BulkJobCheckpointCollisionError(BaseBulkException):
+        """Raised when an attempt to create a job using the `checkpointed cursor` value goes into inf.loop."""
+
+        failure_type: FailureType = FailureType.transient_error
+
     class BulkJobRedirectToOtherShopError(BaseBulkException):
         """Raised when the response contains another shop name"""
 

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.13}/source_shopify/shopify_graphql/bulk/job.py

@@ -53,7 +53,7 @@ class ShopifyBulkManager:
 
     # currents: _job_id, _job_state, _job_created_at, _job_self_canceled
     _job_id: Optional[str] = field(init=False, default=None)
-    _job_state: str = field(init=False, default=None)  # this string is based on ShopifyBulkJobStatus
+    _job_state: str | None = field(init=False, default=None)  # this string is based on ShopifyBulkJobStatus
     # completed and saved Bulk Job result filename
     _job_result_filename: Optional[str] = field(init=False, default=None)
     # date-time when the Bulk Job was created on the server
@@ -70,6 +70,8 @@ class ShopifyBulkManager:
     _job_last_rec_count: int = field(init=False, default=0)
     # the flag to adjust the next slice from the checkpointed cursor vaue
     _job_adjust_slice_from_checkpoint: bool = field(init=False, default=False)
+    # keeps the last checkpointed cursor value for supported streams
+    _job_last_checkpoint_cursor_value: str | None = field(init=False, default=None)
 
     # expand slice factor
     _job_size_expand_factor: int = field(init=False, default=2)
@@ -209,6 +211,27 @@ class ShopifyBulkManager:
         # reseting the checkpoint flag, if bulk job has completed normally
         self._job_adjust_slice_from_checkpoint = False
 
+    def _set_last_checkpoint_cursor_value(self, checkpointed_cursor: str) -> None:
+        """
+        Sets the last checkpoint cursor value.
+
+        Args:
+            checkpointed_cursor (str): The cursor value to set as the last checkpoint. Defaults to None.
+        """
+        self._job_last_checkpoint_cursor_value = checkpointed_cursor
+
+    def _checkpoint_cursor_has_collision(self, checkpointed_cursor: str) -> bool:
+        """
+        Checks if the provided checkpointed cursor collides with the last checkpointed cursor value.
+
+        Args:
+            checkpointed_cursor (str): The cursor value to check for collision. Defaults to None.
+
+        Returns:
+            bool: True if the provided cursor collides with the last checkpointed cursor value, False otherwise.
+        """
+        return self._job_last_checkpoint_cursor_value == checkpointed_cursor
+
     def _job_completed(self) -> bool:
         return self._job_state == ShopifyBulkJobStatus.COMPLETED.value
 
@@ -329,7 +352,7 @@ class ShopifyBulkManager:
     def _on_completed_job(self, response: Optional[requests.Response] = None) -> None:
         self._job_result_filename = self._job_get_result(response)
 
-    def _on_failed_job(self, response: requests.Response) -> AirbyteTracedException:
+    def _on_failed_job(self, response: requests.Response) -> AirbyteTracedException | None:
         if not self._supports_checkpointing:
             raise ShopifyBulkExceptions.BulkJobFailed(
                 f"The BULK Job: `{self._job_id}` exited with {self._job_state}, details: {response.text}",
@@ -485,7 +508,7 @@ class ShopifyBulkManager:
             self._job_state = ShopifyBulkJobStatus.CREATED.value
             LOGGER.info(f"Stream: `{self.http_client.name}`, the BULK Job: `{self._job_id}` is {ShopifyBulkJobStatus.CREATED.value}")
 
-    def job_size_normalize(self, start: datetime, end: datetime) -> datetime:
+    def job_size_normalize(self, start: datetime, end: datetime) -> None:
         # adjust slice size when it's bigger than the loop point when it should end,
         # to preserve correct job size adjustments when this is the only job we need to run, based on STATE provided
         requested_slice_size = (end - start).total_days()
@@ -498,8 +521,20 @@ class ShopifyBulkManager:
     def _adjust_slice_end(self, slice_end: datetime, checkpointed_cursor: Optional[str] = None) -> datetime:
         """
         Choose between the existing `slice_end` value or `checkpointed_cursor` value, if provided.
+
+        Optionally: raises the `transient` error if the checkpoint collision occurs.
         """
-        return pdm.parse(checkpointed_cursor) if checkpointed_cursor else slice_end
+
+        if checkpointed_cursor:
+            if self._checkpoint_cursor_has_collision(checkpointed_cursor):
+                raise ShopifyBulkExceptions.BulkJobCheckpointCollisionError(
+                    f"The stream: `{self.http_client.name}` checkpoint collision is detected. Try to increase the `BULK Job checkpoint (rows collected)` to the bigger value. The stream will be synced again during the next sync attempt."
+                )
+            # set the checkpointed cursor value
+            self._set_last_checkpoint_cursor_value(checkpointed_cursor)
+            return pdm.parse(checkpointed_cursor)
+
+        return slice_end
 
     def get_adjusted_job_end(self, slice_start: datetime, slice_end: datetime, checkpointed_cursor: Optional[str] = None) -> datetime:
         if self._job_adjust_slice_from_checkpoint:

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.13}/source_shopify/spec.json

@@ -115,10 +115,10 @@
       "job_checkpoint_interval": {
         "type": "integer",
         "title": "BULK Job checkpoint (rows collected)",
-        "description": "The threshold, after which the single BULK Job should be checkpointed.",
+        "description": "The threshold, after which the single BULK Job should be checkpointed (min: 15k, max: 1M)",
         "default": 100000,
         "minimum": 15000,
-        "maximum": 200000
+        "maximum": 1000000
       }
     }
   },