airbyte-source-shopify 4.2.0__tar.gz → 4.2.2__tar.gz

{airbyte_source_shopify-4.2.0 → airbyte_source_shopify-4.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 4.2.0
+Version: 4.2.2
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_shopify-4.2.0 → airbyte_source_shopify-4.2.2}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "4.2.0"
+version = "4.2.2"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [ "Airbyte <contact@airbyte.io>",]

{airbyte_source_shopify-4.2.0 → airbyte_source_shopify-4.2.2}/source_shopify/shopify_graphql/bulk/job.py

@@ -75,6 +75,8 @@ class ShopifyBulkManager:
     _job_created_at: Optional[str] = field(init=False, default=None)
     # indicated whether or not we manually force-cancel the current job
     _job_self_canceled: bool = field(init=False, default=False)
+    # indicates whether or not the current job was self-canceled to checkpoint (not because it was running too long)
+    _job_canceled_on_checkpoint: bool = field(init=False, default=False)
     # time between job status checks
     _job_check_interval: Final[int] = 3
 
@@ -87,8 +89,14 @@ class ShopifyBulkManager:
     _job_adjust_slice_from_checkpoint: bool = field(init=False, default=False)
     # keeps the last checkpointed cursor value for supported streams
     _job_last_checkpoint_cursor_value: str | None = field(init=False, default=None)
+    # the flag to re-run the current slice, because the checkpointed job returned no data
+    _job_retry_slice_without_checkpoint: bool = field(init=False, default=False)
+    # the flag to disable the checkpointing for the rest of the sync, once the checkpointed job returned no data
+    _job_checkpoint_disabled: bool = field(init=False, default=False)
     # indicates whether or not the checkpoint comes from the partial result of the FAILED job
     _job_checkpoint_from_failed_job: bool = field(init=False, default=False)
+    # indicates whether or not the checkpoint comes from the result of the job self-canceled on checkpointing
+    _job_checkpoint_from_self_cancel: bool = field(init=False, default=False)
 
     # expand slice factor
     _job_size_expand_factor: int = field(init=False, default=2)
@@ -179,7 +187,9 @@ class ShopifyBulkManager:
 
     @property
     def _job_should_checkpoint(self) -> bool:
-        return self._supports_checkpointing and self._job_last_rec_count >= self._job_checkpoint_interval
+        return (
+            self._supports_checkpointing and not self._job_checkpoint_disabled and self._job_last_rec_count >= self._job_checkpoint_interval
+        )
 
     @property
     def _job_any_lines_collected(self) -> bool:
@@ -216,6 +226,7 @@ class ShopifyBulkManager:
         self._job_result_filename = None
         # setting self-cancelation to default
         self._job_self_canceled = False
+        self._job_canceled_on_checkpoint = False
         # set the running job message counter to default
         self._log_job_msg_count = 0
         # set the running job object count to default
@@ -229,6 +240,7 @@ class ShopifyBulkManager:
         # reseting the checkpoint flag, if bulk job has completed normally
         self._job_adjust_slice_from_checkpoint = False
         self._job_checkpoint_from_failed_job = False
+        self._job_checkpoint_from_self_cancel = False
 
     def _set_last_checkpoint_cursor_value(self, checkpointed_cursor: str) -> None:
         """
@@ -315,10 +327,28 @@ class ShopifyBulkManager:
 
     def _job_get_checkpointed_result(self, response: Optional[requests.Response]) -> None:
         if self._job_any_lines_collected or self._job_should_checkpoint:
-            # set the flag to adjust the next slice from the checkpointed cursor value
-            self._set_checkpointing()
             # fetch the collected records from CANCELED Job on checkpointing
             self._job_result_filename = self._job_get_result(response)
+            if self._job_result_filename:
+                # set the flag to adjust the next slice from the checkpointed cursor value
+                self._set_checkpointing()
+
+    def _on_checkpoint_without_result(self) -> None:
+        """
+        The job was self-canceled on checkpointing, but the API returned no `url` / `partialDataUrl`.
+        The slice must not be treated as consumed: it is re-run, and the checkpointing is disabled
+        for the rest of the sync, since the API is unlikely to return the partial data for the next jobs either.
+        """
+        self._job_retry_slice_without_checkpoint = True
+        self._disable_checkpointing(f"after `{self._job_last_rec_count}` rows collected, but no result was returned by the API")
+
+    def _disable_checkpointing(self, reason: str) -> None:
+        if not self._job_checkpoint_disabled:
+            self._job_checkpoint_disabled = True
+            LOGGER.warning(
+                f"Stream: `{self.http_client.name}`, the BULK Job: `{self._job_id}` was canceled on checkpointing {reason}. "
+                f"The slice will be re-run and the checkpointing is disabled for the rest of this sync. Consider increasing the `BULK Job checkpoint (rows collected)` value to avoid this."
+            )
 
     def _job_update_state(self, response: Optional[requests.Response] = None) -> None:
         if response:
@@ -344,6 +374,12 @@ class ShopifyBulkManager:
             )
         else:
             self._job_get_checkpointed_result(response)
+            if self._job_canceled_on_checkpoint:
+                if self._job_result_filename:
+                    # the next slice resumes from the checkpointed cursor, see `get_adjusted_job_end`
+                    self._job_checkpoint_from_self_cancel = True
+                else:
+                    self._on_checkpoint_without_result()
 
     def _on_canceling_job(self, **kwargs) -> None:
         sleep(self._job_check_interval)
@@ -356,7 +392,7 @@ class ShopifyBulkManager:
 
     def _cancel_on_checkpointing(self) -> None:
         LOGGER.info(f"Stream: `{self.http_client.name}`, checkpointing after >= `{self._job_checkpoint_interval}` rows collected.")
-        # set the flag to adjust the next slice from the checkpointed cursor value
+        self._job_canceled_on_checkpoint = True
         self._job_cancel()
 
     def _on_running_job(self, **kwargs) -> None:
@@ -599,13 +635,23 @@ class ShopifyBulkManager:
         checkpointed_cursor: Optional[str] = None,
         filter_checkpointed_cursor: Optional[str] = None,
     ) -> datetime:
+        if self._job_retry_slice_without_checkpoint:
+            # re-run the same slice, the checkpointing stays disabled (see `_on_checkpoint_without_result`)
+            self._job_retry_slice_without_checkpoint = False
+            return slice_start
+
         if self._job_adjust_slice_from_checkpoint:
             checkpoint_from_failed_job = self._job_checkpoint_from_failed_job
+            checkpoint_from_self_cancel = self._job_checkpoint_from_self_cancel
             # set the checkpointing to default, before the next slice is emitted, to avoid inf.loop
             self._reset_checkpointing()
             if checkpoint_from_failed_job and not checkpointed_cursor:
                 # the partial result has no records to resume from, the rest of the slice must not be skipped
                 self._raise_job_without_result("exited with FAILED and its partial result has no records to checkpoint from")
+            if checkpoint_from_self_cancel and not checkpointed_cursor:
+                # the result has no records to resume from, the slice is re-run with the checkpointing disabled
+                self._disable_checkpointing("but its result has no records to checkpoint from")
+                return slice_start
             return self._adjust_slice_end(slice_end, checkpointed_cursor, filter_checkpointed_cursor)
 
         if self._is_long_running_job:

{airbyte_source_shopify-4.2.0 → airbyte_source_shopify-4.2.2}/source_shopify/streams/base_streams.py

@@ -831,7 +831,9 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream):
         slice_size_message = f"Slice size: `P{round(self.job_manager._job_size, 1)}D`"
         slice_message = f"Stream: `{self.name}` requesting BULK Job for period: {slice_start} -- {slice_end}. {slice_size_message}."
 
-        if self.job_manager._supports_checkpointing:
+        if self.job_manager._job_checkpoint_disabled:
+            checkpointing_message = f" The BULK checkpointing is disabled for the rest of this sync."
+        elif self.job_manager._supports_checkpointing:
             checkpointing_message = f" The BULK checkpoint after `{self.job_manager.job_checkpoint_interval}` lines."
         else:
             checkpointing_message = f" The BULK checkpointing is not supported."

{airbyte_source_shopify-4.2.0 → airbyte_source_shopify-4.2.2}/source_shopify/streams/streams.py

@@ -770,11 +770,14 @@ class Countries(HttpSubStream, FullRefreshShopifyGraphQlBulkStream):
     _page_cursor = None
     _sub_page_cursor = None
 
-    _synced_countries_ids = []
-
     query = DeliveryProfile
     response_field = "deliveryProfiles"
 
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        # per instance: a class-level list is shared by every stream built in the same process
+        self._synced_countries_ids = []
+
     def stream_slices(
         self,
         stream_state: Optional[Mapping[str, Any]] = None,