airbyte-source-shopify 3.0.8.dev202507101517__tar.gz → 3.0.8.dev202509081459__tar.gz

{airbyte_source_shopify-3.0.8.dev202507101517 → airbyte_source_shopify-3.0.8.dev202509081459}/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 3.0.8.dev202507101517
+Version: 3.0.8.dev202509081459
 Summary: Source CDK implementation for Shopify.
+Home-page: https://airbyte.com
 License: ELv2
 Author: Airbyte
 Author-email: contact@airbyte.io
@@ -15,7 +16,6 @@ Requires-Dist: graphql-query (>=1,<2)
 Requires-Dist: pendulum (>=2.1.2,<3.0.0)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/shopify
-Project-URL: Homepage, https://airbyte.com
 Project-URL: Repository, https://github.com/airbytehq/airbyte
 Description-Content-Type: text/markdown
 

{airbyte_source_shopify-3.0.8.dev202507101517 → airbyte_source_shopify-3.0.8.dev202509081459}/pyproject.toml

@@ -1,24 +1,19 @@
 [build-system]
-requires = [
-    "poetry-core>=1.0.0",
-]
+requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "3.0.8.dev202507101517"
+version = "3.0.8.dev.202509081459"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
-authors = [
-    "Airbyte <contact@airbyte.io>",
-]
+authors = [ "Airbyte <contact@airbyte.io>",]
 license = "ELv2"
 readme = "README.md"
 documentation = "https://docs.airbyte.com/integrations/sources/shopify"
 homepage = "https://airbyte.com"
 repository = "https://github.com/airbytehq/airbyte"
-packages = [
-    { include = "source_shopify" },
-]
+[[tool.poetry.packages]]
+include = "source_shopify"
 
 [tool.poetry.dependencies]
 python = "^3.10,<3.12"
@@ -38,5 +33,7 @@ freezegun = "^1.4.0"
 
 [tool.poe]
 include = [
+    # Shared tasks definition file(s) can be imported here.
+    # Run `poe` or `poe --help` to see the list of available tasks.
     "${POE_GIT_DIR}/poe-tasks/poetry-connector-tasks.toml",
 ]

{airbyte_source_shopify-3.0.8.dev202507101517 → airbyte_source_shopify-3.0.8.dev202509081459}/source_shopify/shopify_graphql/bulk/job.py

@@ -5,7 +5,6 @@
 from dataclasses import dataclass, field
 from datetime import datetime
 from enum import Enum
-from json import loads
 from time import sleep, time
 from typing import Any, Final, Iterable, List, Mapping, Optional
 
@@ -66,7 +65,7 @@ class ShopifyBulkManager:
 
     # currents: _job_id, _job_state, _job_created_at, _job_self_canceled
     _job_id: Optional[str] = field(init=False, default=None)
-    _job_state: Optional[str] = field(init=False, default=None)  # this string is based on ShopifyBulkJobStatus
+    _job_state: str | None = field(init=False, default=None)  # this string is based on ShopifyBulkJobStatus
     # completed and saved Bulk Job result filename
     _job_result_filename: Optional[str] = field(init=False, default=None)
     # date-time when the Bulk Job was created on the server
@@ -84,9 +83,7 @@ class ShopifyBulkManager:
     # the flag to adjust the next slice from the checkpointed cursor vaue
     _job_adjust_slice_from_checkpoint: bool = field(init=False, default=False)
     # keeps the last checkpointed cursor value for supported streams
-    _job_last_checkpoint_cursor_value: Optional[str] = field(init=False, default=None)
-    # stores extracted cursor from INTERNAL_SERVER_ERROR recovery (temporary storage)
-    _job_extracted_checkpoint_cursor: Optional[str] = field(init=False, default=None)
+    _job_last_checkpoint_cursor_value: str | None = field(init=False, default=None)
 
     # expand slice factor
     _job_size_expand_factor: int = field(init=False, default=2)
@@ -217,8 +214,6 @@ class ShopifyBulkManager:
         self._log_job_msg_count = 0
         # set the running job object count to default
         self._job_last_rec_count = 0
-        # clear any extracted cursor from INTERNAL_SERVER_ERROR recovery
-        self._job_extracted_checkpoint_cursor = None
 
     def _set_checkpointing(self) -> None:
         # set the flag to adjust the next slice from the checkpointed cursor value
@@ -318,24 +313,6 @@ class ShopifyBulkManager:
             # fetch the collected records from CANCELED Job on checkpointing
             self._job_result_filename = self._job_get_result(response)
 
-            # Special handling: For FAILED jobs with INTERNAL_SERVER_ERROR, extract the last processed cursor
-            if response:
-                parsed_response = response.json().get("data", {}).get("node", {}) if response else {}
-                error_code = parsed_response.get("errorCode")
-                if error_code == "INTERNAL_SERVER_ERROR":
-                    last_cursor = self._extract_last_cursor_from_partial_data(response)
-                    if last_cursor:
-                        # Check if this cursor would cause a collision before storing it
-                        if self._checkpoint_cursor_has_collision(last_cursor):
-                            # Skip cursor extraction to avoid collision
-                            pass
-                        else:
-                            # Store the extracted cursor for later use (don't set it yet to avoid collision)
-                            self._job_extracted_checkpoint_cursor = last_cursor
-        else:
-            # Not processing data due to insufficient records or checkpointing disabled
-            pass
-
     def _job_update_state(self, response: Optional[requests.Response] = None) -> None:
         if response:
             self._job_state = response.json().get("data", {}).get("node", {}).get("status")
@@ -386,26 +363,7 @@ class ShopifyBulkManager:
     def _on_completed_job(self, response: Optional[requests.Response] = None) -> None:
         self._job_result_filename = self._job_get_result(response)
 
-    def _on_failed_job(self, response: requests.Response) -> Optional[AirbyteTracedException]:
-        # Special handling for FAILED jobs with INTERNAL_SERVER_ERROR that support checkpointing
-        parsed_response = response.json().get("data", {}).get("node", {}) if response else {}
-        error_code = parsed_response.get("errorCode")
-
-        if error_code == "INTERNAL_SERVER_ERROR" and self._supports_checkpointing:
-            LOGGER.info(
-                f"Stream: `{self.http_client.name}`, BULK Job: `{self._job_id}` failed with INTERNAL_SERVER_ERROR. Waiting for partial data availability..."
-            )
-            # For INTERNAL_SERVER_ERROR specifically, wait and retry to check if partial data becomes available
-            partial_response = self._wait_for_partial_data_on_failure()
-            if partial_response:
-                # Use the updated response that may contain partialDataUrl
-                response = partial_response
-                # Update the job state with the new response to ensure _job_last_rec_count is set correctly
-                self._job_update_state(response)
-                # For INTERNAL_SERVER_ERROR with partial data, extract cursor and treat as checkpointable
-                self._job_get_checkpointed_result(response)
-                return None  # Don't raise exception, we recovered the data
-
+    def _on_failed_job(self, response: requests.Response) -> AirbyteTracedException | None:
         if not self._supports_checkpointing:
             raise ShopifyBulkExceptions.BulkJobFailed(
                 f"The BULK Job: `{self._job_id}` exited with {self._job_state}, details: {response.text}",
@@ -415,102 +373,6 @@ class ShopifyBulkManager:
             # we leverage the checkpointing in this case.
             self._job_get_checkpointed_result(response)
 
-    def _wait_for_partial_data_on_failure(self) -> Optional[requests.Response]:
-        """
-        Wait for partial data to become available when a BULK job fails with INTERNAL_SERVER_ERROR.
-
-        This method is specifically designed for INTERNAL_SERVER_ERROR cases where
-        Shopify's BULK API may make partial data available (via partialDataUrl)
-        after a short wait, even though the job initially failed.
-
-        Returns:
-            Optional[requests.Response]: Updated response with potential partialDataUrl, or None if no data
-        """
-        max_wait_attempts = 10  # Maximum number of wait attempts
-        wait_interval = 10  # Wait 10 seconds between checks
-
-        for attempt in range(max_wait_attempts):
-            sleep(wait_interval)
-
-            # Check job status again to see if partial data is now available
-            try:
-                _, response = self.http_client.send_request(
-                    http_method="POST",
-                    url=self.base_url,
-                    json={"query": ShopifyBulkTemplates.status(self._job_id)},
-                    request_kwargs={},
-                )
-
-                parsed_response = response.json().get("data", {}).get("node", {}) if response else {}
-                partial_data_url = parsed_response.get("partialDataUrl")
-                object_count = parsed_response.get("objectCount", "0")
-
-                # Only stop waiting if we actually have a partialDataUrl - objectCount alone is not sufficient
-                if partial_data_url and int(object_count) > 0:
-                    LOGGER.info(f"Stream: `{self.http_client.name}`, partial data available after wait. Object count: {object_count}")
-                    return response
-                elif int(object_count) > 0:
-                    # objectCount available but no partialDataUrl yet - continue waiting
-                    continue
-
-            except Exception as e:
-                # Error during partial data check - continue waiting
-                continue
-
-        LOGGER.warning(f"Stream: `{self.http_client.name}`, no partial data became available after {max_wait_attempts} attempts")
-        return None
-
-    def _extract_last_cursor_from_partial_data(self, response: Optional[requests.Response]) -> Optional[str]:
-        """
-        Extract the last processed cursor value from partial data for INTERNAL_SERVER_ERROR recovery.
-
-        This method retrieves partial data from a failed INTERNAL_SERVER_ERROR job and extracts
-        the updatedAt value of the last record, which can be used to resume processing from that point.
-        Only used in INTERNAL_SERVER_ERROR scenarios with checkpointing support.
-
-        Args:
-            response: The response containing partial data information
-
-        Returns:
-            Optional[str]: The cursor value of the last processed record, or None if unavailable
-        """
-        if not response:
-            return None
-
-        try:
-            parsed_response = response.json().get("data", {}).get("node", {})
-            partial_data_url = parsed_response.get("partialDataUrl")
-
-            if not partial_data_url:
-                return None
-
-            # Download the partial data
-            _, partial_response = self.http_client.send_request(http_method="GET", url=partial_data_url, request_kwargs={"stream": True})
-            partial_response.raise_for_status()
-
-            last_record = None
-            # Read through the JSONL data to find the last record
-            for line in partial_response.iter_lines(decode_unicode=True):
-                if line and line.strip() and line.strip() != END_OF_FILE:
-                    try:
-                        record = loads(line)
-                        # Look for the main record types (Order, Product, etc.)
-                        if record.get("__typename") in ["Order", "Product", "Customer", "FulfillmentOrder"]:
-                            last_record = record
-                    except Exception:
-                        continue
-
-            # Extract the updatedAt cursor from the last record
-            if last_record and "updatedAt" in last_record:
-                cursor_value = last_record["updatedAt"]
-                return cursor_value
-
-        except Exception as e:
-            # Failed to extract cursor from partial data
-            pass
-
-        return None
-
     def _on_timeout_job(self, **kwargs) -> AirbyteTracedException:
         raise ShopifyBulkExceptions.BulkJobTimout(
             f"The BULK Job: `{self._job_id}` exited with {self._job_state}, please reduce the `GraphQL BULK Date Range in Days` in SOURCES > Your Shopify Source > SETTINGS.",
@@ -665,7 +527,9 @@ class ShopifyBulkManager:
         step = self._job_size if self._job_size else self._job_size_min
         return slice_start.add(days=step)
 
-    def _adjust_slice_end(self, slice_end: datetime, checkpointed_cursor: Optional[str] = None) -> datetime:
+    def _adjust_slice_end(
+        self, slice_end: datetime, checkpointed_cursor: Optional[str] = None, filter_checkpointed_cursor: Optional[str] = None
+    ) -> datetime:
         """
         Choose between the existing `slice_end` value or `checkpointed_cursor` value, if provided.
 
@@ -673,34 +537,30 @@ class ShopifyBulkManager:
         """
 
         if checkpointed_cursor:
-            # Check for collision and provide more context in the error
             if self._checkpoint_cursor_has_collision(checkpointed_cursor):
-                # For INTERNAL_SERVER_ERROR recovery, if the cursor is the same, we might need to skip ahead slightly
-                # This can happen if the failure occurred right at the boundary of what was already processed
-                if hasattr(self, "_job_extracted_checkpoint_cursor") and self._job_extracted_checkpoint_cursor == checkpointed_cursor:
-                    pass  # Collision from INTERNAL_SERVER_ERROR recovery at boundary
-
                 raise ShopifyBulkExceptions.BulkJobCheckpointCollisionError(
-                    f"The stream: `{self.http_client.name}` checkpoint collision is detected. Current cursor: {self._job_last_checkpoint_cursor_value}, New cursor: {checkpointed_cursor}. Try to increase the `BULK Job checkpoint (rows collected)` to the bigger value. The stream will be synced again during the next sync attempt."
+                    f"The stream: `{self.http_client.name}` checkpoint collision is detected. Try to increase the `BULK Job checkpoint (rows collected)` to the bigger value. The stream will be synced again during the next sync attempt."
                 )
             # set the checkpointed cursor value
             self._set_last_checkpoint_cursor_value(checkpointed_cursor)
-            return pdm.parse(checkpointed_cursor)
+            if isinstance(checkpointed_cursor, str):
+                return pdm.parse(checkpointed_cursor)
+            if isinstance(checkpointed_cursor, int):
+                return pdm.parse(filter_checkpointed_cursor)
 
         return slice_end
 
-    def get_adjusted_job_end(self, slice_start: datetime, slice_end: datetime, checkpointed_cursor: Optional[str] = None) -> datetime:
+    def get_adjusted_job_end(
+        self,
+        slice_start: datetime,
+        slice_end: datetime,
+        checkpointed_cursor: Optional[str] = None,
+        filter_checkpointed_cursor: Optional[str] = None,
+    ) -> datetime:
         if self._job_adjust_slice_from_checkpoint:
             # set the checkpointing to default, before the next slice is emitted, to avoid inf.loop
             self._reset_checkpointing()
-            # Clear the extracted cursor after use to avoid reusing it
-            if self._job_extracted_checkpoint_cursor:
-                extracted_cursor = self._job_extracted_checkpoint_cursor
-                self._job_extracted_checkpoint_cursor = None
-                cursor_to_use = extracted_cursor
-            else:
-                cursor_to_use = checkpointed_cursor or self._job_last_checkpoint_cursor_value
-            return self._adjust_slice_end(slice_end, cursor_to_use)
+            return self._adjust_slice_end(slice_end, checkpointed_cursor, filter_checkpointed_cursor)
 
         if self._is_long_running_job:
             self._job_size_reduce_next()

{airbyte_source_shopify-3.0.8.dev202507101517 → airbyte_source_shopify-3.0.8.dev202509081459}/source_shopify/streams/base_streams.py

@@ -178,6 +178,11 @@ class IncrementalShopifyStream(ShopifyStream, ABC):
     # Setting the check point interval to the limit of the records output
     state_checkpoint_interval = 250
 
+    def __init__(self, config: Dict):
+        super().__init__(config)
+        # _filter_checkpointed_cursor used to checkpoint streams with cursor field - ID in job.get_adjusted_job_end
+        self._filter_checkpointed_cursor = None
+
     @property
     def filter_by_state_checkpoint(self) -> bool:
         """
@@ -216,7 +221,7 @@ class IncrementalShopifyStream(ShopifyStream, ABC):
                 params[self.filter_field] = stream_state.get(self.cursor_field)
         return params
 
-    def track_checkpoint_cursor(self, record_value: Union[str, int]) -> None:
+    def track_checkpoint_cursor(self, record_value: Union[str, int], filter_record_value: Optional[str] = None) -> None:
         if self.filter_by_state_checkpoint:
             # set checkpoint cursor
             if not self._checkpoint_cursor:
@@ -225,6 +230,12 @@ class IncrementalShopifyStream(ShopifyStream, ABC):
             if str(record_value) >= str(self._checkpoint_cursor):
                 self._checkpoint_cursor = record_value
 
+            if filter_record_value:
+                if not self._filter_checkpointed_cursor:
+                    self._filter_checkpointed_cursor = filter_record_value
+                if str(filter_record_value) >= str(self._filter_checkpointed_cursor):
+                    self._filter_checkpointed_cursor = filter_record_value
+
     def should_checkpoint(self, index: int) -> bool:
         return self.filter_by_state_checkpoint and index >= self.state_checkpoint_interval
 
@@ -242,7 +253,8 @@ class IncrementalShopifyStream(ShopifyStream, ABC):
             for index, record in enumerate(records_slice, 1):
                 if self.cursor_field in record:
                     record_value = record.get(self.cursor_field, self.default_state_comparison_value)
-                    self.track_checkpoint_cursor(record_value)
+                    filter_record_value = record.get(self.filter_field) if self.filter_field else None
+                    self.track_checkpoint_cursor(record_value, filter_record_value)
                     if record_value:
                         if record_value >= state_value:
                             yield record
@@ -815,7 +827,7 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream):
                 self.emit_slice_message(start, slice_end)
                 yield {"start": start.to_rfc3339_string(), "end": slice_end.to_rfc3339_string()}
                 # increment the end of the slice or reduce the next slice
-                start = self.job_manager.get_adjusted_job_end(start, slice_end, self._checkpoint_cursor)
+                start = self.job_manager.get_adjusted_job_end(start, slice_end, self._checkpoint_cursor, self._filter_checkpointed_cursor)
         else:
             # for the streams that don't support filtering
             yield {}