airbyte-source-shopify 2.5.12.dev202412101009__py3-none-any.whl → 2.5.13__py3-none-any.whl

{airbyte_source_shopify-2.5.12.dev202412101009.dist-info → airbyte_source_shopify-2.5.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 2.5.12.dev202412101009
+Version: 2.5.13
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2
@@ -11,7 +11,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=5,<6)
+Requires-Dist: airbyte-cdk (>=6,<7)
 Requires-Dist: graphql-query (>=1,<2)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/shopify

{airbyte_source_shopify-2.5.12.dev202412101009.dist-info → airbyte_source_shopify-2.5.13.dist-info}/RECORD

@@ -52,8 +52,8 @@ source_shopify/schemas/tender_transactions.json,sha256=U8fdT-eflycEPzYSpBDiB0lp9
 source_shopify/schemas/transactions.json,sha256=vbwscH3UcAtbSsC70mBka4oNaFR4S3S6IFBmzR7t37U,10226
 source_shopify/scopes.py,sha256=K1lH6ZsdOo4YkEnHaQm8d1RlK2E4PFNsLczomchc6jk,6533
 source_shopify/shopify_graphql/bulk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-source_shopify/shopify_graphql/bulk/exceptions.py,sha256=TUiRJETddeppOte3Prc9cuwFJCH1YWg_oMB5IodfRGY,2293
-source_shopify/shopify_graphql/bulk/job.py,sha256=ze4XJF391qO9z1fE60cJIu-FEklavvIpB6rhH6ZFMmg,25960
+source_shopify/shopify_graphql/bulk/exceptions.py,sha256=4dj7Za4xIfwL-zf8joT9svF_RSoGlE3GviMiIl1e1rs,2532
+source_shopify/shopify_graphql/bulk/job.py,sha256=jR0BGVtMWBZPfAL-_rNvY4qgYE3CjPV4i-uUNWITdjQ,27634
 source_shopify/shopify_graphql/bulk/query.py,sha256=tktkNYD3dCEPkod1OvXRckV3oEKF3X5ja32UTCow4KI,121669
 source_shopify/shopify_graphql/bulk/record.py,sha256=X6VGngugv7a_S8UEeDo121BkdCVLj5nWlHK76A21kyo,16898
 source_shopify/shopify_graphql/bulk/retry.py,sha256=4ICAWM74C4XHsbPNkztXA80NI48bMGD2Gsz89GmqBTU,2625
@@ -62,12 +62,12 @@ source_shopify/shopify_graphql/bulk/tools.py,sha256=g8-7NHHh1hlCWjUaLaxCxG2jR-WL
 source_shopify/shopify_graphql/graphql.py,sha256=wHzppRm2zbllOp1m4zCax6SY6xo3aYnwlwIScRVk10Q,1969
 source_shopify/shopify_graphql/schema.py,sha256=H3qj3AFjaKJ4Nsd2GlyNCQEOulDFrkQWvO00Ad3fqbY,1354903
 source_shopify/source.py,sha256=c_Z6ynQ8yrAjgvalD0nrDUqPkdBNDxtfjKYF6q_UGzk,8568
-source_shopify/spec.json,sha256=rK34rob4WrCHDD6fTyRNv42LpYta0ViD0D-9eQPsCQE,6124
+source_shopify/spec.json,sha256=ITYWiQ-NrI5VISk5qmUQhp9ChUE2FV18d8xzVzPwvAg,6144
 source_shopify/streams/base_streams.py,sha256=rreQGHrt_3CBdBmJ6Hq41GS10UDi1pWQMTZoeU-hTbk,40490
 source_shopify/streams/streams.py,sha256=MTsy1fGBWV6ZGYfjE-mx7rT-eXLOngyGSYJeUhiE6po,12056
 source_shopify/transform.py,sha256=mn0htL812_90zc_YszGQa0hHcIZQpYYdmk8IqpZm5TI,4685
 source_shopify/utils.py,sha256=l6BhCtzcZbAd9DMpeXHHHhkPnERNNx4rfS1DuH53w0k,14460
-airbyte_source_shopify-2.5.12.dev202412101009.dist-info/METADATA,sha256=BBGloZmUTiMa5gIxbY3CHjTc28hyEf6L_t6TPa-k3Qc,5273
-airbyte_source_shopify-2.5.12.dev202412101009.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_source_shopify-2.5.12.dev202412101009.dist-info/entry_points.txt,sha256=SyTwKSsPk9MCdPf01saWpnp8hcmZOgBssVcSIvMbBeQ,57
-airbyte_source_shopify-2.5.12.dev202412101009.dist-info/RECORD,,
+airbyte_source_shopify-2.5.13.dist-info/METADATA,sha256=qaxVwN0uxaVCyr-9Gv5sA1rOVDb5Ypq9cEPCkhjOL6g,5257
+airbyte_source_shopify-2.5.13.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_source_shopify-2.5.13.dist-info/entry_points.txt,sha256=SyTwKSsPk9MCdPf01saWpnp8hcmZOgBssVcSIvMbBeQ,57
+airbyte_source_shopify-2.5.13.dist-info/RECORD,,

source_shopify/shopify_graphql/bulk/exceptions.py

@@ -52,6 +52,11 @@ class ShopifyBulkExceptions:
 
         failure_type: FailureType = FailureType.transient_error
 
+    class BulkJobCheckpointCollisionError(BaseBulkException):
+        """Raised when an attempt to create a job using the `checkpointed cursor` value goes into inf.loop."""
+
+        failure_type: FailureType = FailureType.transient_error
+
     class BulkJobRedirectToOtherShopError(BaseBulkException):
         """Raised when the response contains another shop name"""
 

source_shopify/shopify_graphql/bulk/job.py

@@ -53,7 +53,7 @@ class ShopifyBulkManager:
 
     # currents: _job_id, _job_state, _job_created_at, _job_self_canceled
     _job_id: Optional[str] = field(init=False, default=None)
-    _job_state: str = field(init=False, default=None)  # this string is based on ShopifyBulkJobStatus
+    _job_state: str | None = field(init=False, default=None)  # this string is based on ShopifyBulkJobStatus
     # completed and saved Bulk Job result filename
     _job_result_filename: Optional[str] = field(init=False, default=None)
     # date-time when the Bulk Job was created on the server
@@ -70,6 +70,8 @@ class ShopifyBulkManager:
     _job_last_rec_count: int = field(init=False, default=0)
     # the flag to adjust the next slice from the checkpointed cursor vaue
     _job_adjust_slice_from_checkpoint: bool = field(init=False, default=False)
+    # keeps the last checkpointed cursor value for supported streams
+    _job_last_checkpoint_cursor_value: str | None = field(init=False, default=None)
 
     # expand slice factor
     _job_size_expand_factor: int = field(init=False, default=2)
@@ -209,6 +211,27 @@ class ShopifyBulkManager:
         # reseting the checkpoint flag, if bulk job has completed normally
         self._job_adjust_slice_from_checkpoint = False
 
+    def _set_last_checkpoint_cursor_value(self, checkpointed_cursor: str) -> None:
+        """
+        Sets the last checkpoint cursor value.
+
+        Args:
+            checkpointed_cursor (str): The cursor value to set as the last checkpoint. Defaults to None.
+        """
+        self._job_last_checkpoint_cursor_value = checkpointed_cursor
+
+    def _checkpoint_cursor_has_collision(self, checkpointed_cursor: str) -> bool:
+        """
+        Checks if the provided checkpointed cursor collides with the last checkpointed cursor value.
+
+        Args:
+            checkpointed_cursor (str): The cursor value to check for collision. Defaults to None.
+
+        Returns:
+            bool: True if the provided cursor collides with the last checkpointed cursor value, False otherwise.
+        """
+        return self._job_last_checkpoint_cursor_value == checkpointed_cursor
+
     def _job_completed(self) -> bool:
         return self._job_state == ShopifyBulkJobStatus.COMPLETED.value
 
@@ -329,7 +352,7 @@ class ShopifyBulkManager:
     def _on_completed_job(self, response: Optional[requests.Response] = None) -> None:
         self._job_result_filename = self._job_get_result(response)
 
-    def _on_failed_job(self, response: requests.Response) -> AirbyteTracedException:
+    def _on_failed_job(self, response: requests.Response) -> AirbyteTracedException | None:
         if not self._supports_checkpointing:
             raise ShopifyBulkExceptions.BulkJobFailed(
                 f"The BULK Job: `{self._job_id}` exited with {self._job_state}, details: {response.text}",
@@ -485,7 +508,7 @@ class ShopifyBulkManager:
             self._job_state = ShopifyBulkJobStatus.CREATED.value
             LOGGER.info(f"Stream: `{self.http_client.name}`, the BULK Job: `{self._job_id}` is {ShopifyBulkJobStatus.CREATED.value}")
 
-    def job_size_normalize(self, start: datetime, end: datetime) -> datetime:
+    def job_size_normalize(self, start: datetime, end: datetime) -> None:
         # adjust slice size when it's bigger than the loop point when it should end,
         # to preserve correct job size adjustments when this is the only job we need to run, based on STATE provided
         requested_slice_size = (end - start).total_days()
@@ -498,8 +521,20 @@ class ShopifyBulkManager:
     def _adjust_slice_end(self, slice_end: datetime, checkpointed_cursor: Optional[str] = None) -> datetime:
         """
         Choose between the existing `slice_end` value or `checkpointed_cursor` value, if provided.
+
+        Optionally: raises the `transient` error if the checkpoint collision occurs.
         """
-        return pdm.parse(checkpointed_cursor) if checkpointed_cursor else slice_end
+
+        if checkpointed_cursor:
+            if self._checkpoint_cursor_has_collision(checkpointed_cursor):
+                raise ShopifyBulkExceptions.BulkJobCheckpointCollisionError(
+                    f"The stream: `{self.http_client.name}` checkpoint collision is detected. Try to increase the `BULK Job checkpoint (rows collected)` to the bigger value. The stream will be synced again during the next sync attempt."
+                )
+            # set the checkpointed cursor value
+            self._set_last_checkpoint_cursor_value(checkpointed_cursor)
+            return pdm.parse(checkpointed_cursor)
+
+        return slice_end
 
     def get_adjusted_job_end(self, slice_start: datetime, slice_end: datetime, checkpointed_cursor: Optional[str] = None) -> datetime:
         if self._job_adjust_slice_from_checkpoint:

source_shopify/spec.json

@@ -115,10 +115,10 @@
       "job_checkpoint_interval": {
         "type": "integer",
         "title": "BULK Job checkpoint (rows collected)",
-        "description": "The threshold, after which the single BULK Job should be checkpointed.",
+        "description": "The threshold, after which the single BULK Job should be checkpointed (min: 15k, max: 1M)",
         "default": 100000,
         "minimum": 15000,
-        "maximum": 200000
+        "maximum": 1000000
       }
     }
   },