airbyte-source-shopify 4.2.2__tar.gz → 4.2.4__tar.gz

{airbyte_source_shopify-4.2.2 → airbyte_source_shopify-4.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 4.2.2
+Version: 4.2.4
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_shopify-4.2.2 → airbyte_source_shopify-4.2.4}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "4.2.2"
+version = "4.2.4"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [ "Airbyte <contact@airbyte.io>",]

{airbyte_source_shopify-4.2.2 → airbyte_source_shopify-4.2.4}/source_shopify/shopify_graphql/bulk/job.py

@@ -600,6 +600,20 @@ class ShopifyBulkManager:
         step = self._job_size if self._job_size else self._job_size_min
         return slice_start.add(days=step)
 
+    def _checkpoint_cursor_within_slice(
+        self,
+        slice_start: datetime,
+        checkpointed_cursor: Optional[str | int],
+        filter_checkpointed_cursor: Optional[str],
+    ) -> bool:
+        if not checkpointed_cursor:
+            return False
+        if isinstance(checkpointed_cursor, str):
+            return pdm.parse(checkpointed_cursor) > slice_start
+        if isinstance(checkpointed_cursor, int):
+            return bool(filter_checkpointed_cursor) and pdm.parse(filter_checkpointed_cursor) > slice_start
+        return False
+
     def _adjust_slice_end(
         self, slice_end: datetime, checkpointed_cursor: Optional[str] = None, filter_checkpointed_cursor: Optional[str] = None
     ) -> datetime:
@@ -645,12 +659,18 @@ class ShopifyBulkManager:
             checkpoint_from_self_cancel = self._job_checkpoint_from_self_cancel
             # set the checkpointing to default, before the next slice is emitted, to avoid inf.loop
             self._reset_checkpointing()
-            if checkpoint_from_failed_job and not checkpointed_cursor:
-                # the partial result has no records to resume from, the rest of the slice must not be skipped
-                self._raise_job_without_result("exited with FAILED and its partial result has no records to checkpoint from")
-            if checkpoint_from_self_cancel and not checkpointed_cursor:
-                # the result has no records to resume from, the slice is re-run with the checkpointing disabled
-                self._disable_checkpointing("but its result has no records to checkpoint from")
+            if checkpoint_from_failed_job and not self._checkpoint_cursor_within_slice(
+                slice_start, checkpointed_cursor, filter_checkpointed_cursor
+            ):
+                # the partial result has no records within the current slice to resume from, the rest of the slice must not be skipped
+                self._raise_job_without_result(
+                    "exited with FAILED and its partial result has no records within the slice to checkpoint from"
+                )
+            if checkpoint_from_self_cancel and not self._checkpoint_cursor_within_slice(
+                slice_start, checkpointed_cursor, filter_checkpointed_cursor
+            ):
+                # the result has no records within the current slice to resume from, the slice is re-run with the checkpointing disabled
+                self._disable_checkpointing("but its result has no records within the slice to checkpoint from")
                 return slice_start
             return self._adjust_slice_end(slice_end, checkpointed_cursor, filter_checkpointed_cursor)
 

{airbyte_source_shopify-4.2.2 → airbyte_source_shopify-4.2.4}/source_shopify/shopify_graphql/bulk/query.py

@@ -139,7 +139,7 @@ class ShopifyBulkQuery:
     def inject_parent_cursor_field(self, nodes: List[Field], key: str = "updatedAt", index: int = 2) -> List[Field]:
         if self.parent_stream_cursor_alias:
             # inject parent cursor key as alias to the `updatedAt` parent cursor field
-            nodes.insert(index, Field(name=key, alias=self.parent_stream_cursor_alias))
+            return [*nodes[:index], Field(name=key, alias=self.parent_stream_cursor_alias), *nodes[index:]]
         return nodes
 
     def get(self, filter_field: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None) -> str:

{airbyte_source_shopify-4.2.2 → airbyte_source_shopify-4.2.4}/source_shopify/streams/base_streams.py

@@ -18,7 +18,7 @@ from source_shopify.shopify_graphql.bulk.external_sort import DEFAULT_SORT_CHUNK
 from source_shopify.shopify_graphql.bulk.job import ShopifyBulkManager
 from source_shopify.shopify_graphql.bulk.query import DeliveryZoneList, ShopFeatures, ShopifyBulkQuery
 from source_shopify.transform import DataTypeEnforcer
-from source_shopify.utils import ApiTypeEnum, ShopifyGraphQlErrorHandler, ShopifyNonRetryableErrors, is_throttled_graphql_error
+from source_shopify.utils import ApiTypeEnum, ShopifyGraphQlErrorHandler, ShopifyNonRetryableErrors, is_retryable_graphql_error
 from source_shopify.utils import EagerlyCachedStreamState as stream_state_cache
 from source_shopify.utils import ShopifyRateLimiter as limiter
 
@@ -35,7 +35,7 @@ class ShopifyStream(HttpStream, ABC):
     logger = logging.getLogger("airbyte")
 
     # Latest Stable Release
-    api_version = "2026-07"
+    api_version = "2026-10"
     # Page size
     limit = 250
 
@@ -948,7 +948,7 @@ class FullRefreshShopifyGraphQlBulkStream(ShopifyStream):
     response_field: str
 
     def get_error_handler(self) -> Optional[ErrorHandler]:
-        # retries HTTP 200 pages that carry a THROTTLED GraphQL error, everything else follows the status-code mapping
+        # retries HTTP 200 pages that carry a transient GraphQL error, everything else follows the status-code mapping
         return ShopifyGraphQlErrorHandler(
             self.logger, max_retries=5, error_mapping=DEFAULT_ERROR_MAPPING | ShopifyNonRetryableErrors(self.name)
         )
@@ -1005,6 +1005,6 @@ class FullRefreshShopifyGraphQlBulkStream(ShopifyStream):
                 raise AirbyteTracedException(
                     message=f"Stream `{self.name}`: Shopify GraphQL request failed: "
                     + "; ".join(str(error.get("message", error)) for error in errors),
-                    failure_type=FailureType.transient_error if is_throttled_graphql_error(errors) else FailureType.system_error,
+                    failure_type=FailureType.transient_error if is_retryable_graphql_error(errors) else FailureType.system_error,
                 )
             yield from ((json_response.get("data") or {}).get(self.response_field) or {}).get("nodes") or []

{airbyte_source_shopify-4.2.2 → airbyte_source_shopify-4.2.4}/source_shopify/streams/streams.py

@@ -36,7 +36,13 @@ from source_shopify.shopify_graphql.bulk.query import (
     Transaction,
 )
 from source_shopify.shopify_graphql.bulk.tools import BulkTools
-from source_shopify.utils import LimitReducingErrorHandler, ShopifyNonRetryableErrors, ShopifyRateLimiter, is_throttled_graphql_error
+from source_shopify.utils import (
+    LimitReducingErrorHandler,
+    ShopifyGraphQlErrorHandler,
+    ShopifyNonRetryableErrors,
+    ShopifyRateLimiter,
+    is_retryable_graphql_error,
+)
 
 from airbyte_cdk import HttpSubStream
 from airbyte_cdk.models import FailureType
@@ -247,16 +253,29 @@ class DeletedProducts(IncrementalShopifyStream):
             "variables": variables,
         }
 
+    def get_error_handler(self) -> Optional[ErrorHandler]:
+        return ShopifyGraphQlErrorHandler(
+            self.logger, max_retries=5, error_mapping=DEFAULT_ERROR_MAPPING | ShopifyNonRetryableErrors(self.name)
+        )
+
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
         json_response = response.json()
-        page_info = json_response.get("data", {}).get("events", {}).get("pageInfo", {})
+        page_info = ((json_response.get("data") or {}).get("events") or {}).get("pageInfo", {})
         if page_info.get("hasNextPage"):
             return {"cursor": page_info.get("endCursor")}
         return None
 
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
         json_response = response.json()
-        events = json_response.get("data", {}).get("events", {}).get("nodes", [])
+        errors = json_response.get("errors")
+        if errors and response.ok:
+            # an HTTP 200 with GraphQL errors has no usable data, reading it as an empty page would end the sync as complete
+            raise AirbyteTracedException(
+                message=f"Stream `{self.name}`: Shopify GraphQL request failed: "
+                + "; ".join(str(error.get("message", error)) for error in errors),
+                failure_type=FailureType.transient_error if is_retryable_graphql_error(errors) else FailureType.system_error,
+            )
+        events = ((json_response.get("data") or {}).get("events") or {}).get("nodes", [])
         for event in events:
             if event.get("subjectType") == "PRODUCT" and event.get("subjectId"):
                 subject_id = event.get("subjectId", "")
@@ -607,18 +626,18 @@ class DiscountCodesSync(IncrementalShopifyStream):
             result = response.json()
             errors = result.get("errors")
             if errors:
-                if is_throttled_graphql_error(errors):
+                if is_retryable_graphql_error(errors):
                     if attempt < self._GRAPHQL_MAX_RETRIES:
                         ShopifyRateLimiter.wait_time(ShopifyRateLimiter.on_unknown_load)
                         continue
                     raise AirbyteTracedException(
-                        message="GraphQL query for stream `discount_codes_sync` exceeded max retries due to throttling."
+                        message="GraphQL query for stream `discount_codes_sync` exceeded max retries on transient errors."
                     )
                 error_messages = "; ".join(e.get("message", str(e)) for e in errors)
                 raise AirbyteTracedException(message=f"GraphQL query failed for stream `discount_codes_sync`: {error_messages}")
             ShopifyRateLimiter.wait_time(ShopifyRateLimiter.get_graphql_api_wait_time(response, threshold=0.9))
             return result
-        raise AirbyteTracedException(message="GraphQL query for stream `discount_codes_sync` exceeded max retries due to throttling.")
+        raise AirbyteTracedException(message="GraphQL query for stream `discount_codes_sync` exceeded max retries on transient errors.")
 
     @staticmethod
     def _extract_codes_connection(code_discount: Mapping[str, Any]) -> Mapping[str, Any]:
@@ -663,7 +682,11 @@ class DiscountCodesSync(IncrementalShopifyStream):
             if child_cursor:
                 variables["after"] = child_cursor
             result = self._graphql_request(self.CHILD_CODES_QUERY, variables)
-            code_discount = result.get("data", {}).get("codeDiscountNode", {}).get("codeDiscount") or {}
+            code_discount_node = (result.get("data") or {}).get("codeDiscountNode")
+            if not code_discount_node:
+                self.logger.warning(f"Stream `{self.name}`: discount `{parent_gid}` no longer exists, skipping its codes.")
+                return
+            code_discount = code_discount_node.get("codeDiscount") or {}
             codes_conn = self._extract_codes_connection(code_discount)
             for code_node in codes_conn.get("nodes", []):
                 record = self._build_child_record(code_node, parent_gid, parent_meta)

{airbyte_source_shopify-4.2.2 → airbyte_source_shopify-4.2.4}/source_shopify/utils.py

@@ -99,11 +99,13 @@ class ApiTypeEnum(enum.Enum):
 
 
 GRAPHQL_THROTTLE_ERROR_CODES: Final[set] = {"THROTTLED", "MAX_COST_EXCEEDED"}
+# Shopify returns INTERNAL_SERVER_ERROR with HTTP 200 instead of 500 and documents it as retryable
+GRAPHQL_RETRYABLE_ERROR_CODES: Final[set] = GRAPHQL_THROTTLE_ERROR_CODES | {"INTERNAL_SERVER_ERROR"}
 
 
-def is_throttled_graphql_error(errors: List[Mapping[str, Any]]) -> bool:
-    """Shopify GraphQL reports throttling as HTTP 200 with `errors[].extensions.code` in GRAPHQL_THROTTLE_ERROR_CODES."""
-    return any((error.get("extensions") or {}).get("code") in GRAPHQL_THROTTLE_ERROR_CODES for error in errors)
+def is_retryable_graphql_error(errors: List[Mapping[str, Any]]) -> bool:
+    """Shopify GraphQL reports transient errors as HTTP 200 with `errors[].extensions.code` in GRAPHQL_RETRYABLE_ERROR_CODES."""
+    return any((error.get("extensions") or {}).get("code") in GRAPHQL_RETRYABLE_ERROR_CODES for error in errors)
 
 
 class ShopifyRateLimiter:
@@ -373,9 +375,10 @@ class LimitReducingErrorHandler(HttpStatusErrorHandler):
 
 class ShopifyGraphQlErrorHandler(HttpStatusErrorHandler):
     """
-    Shopify GraphQL reports throttling as HTTP 200 with `errors[].extensions.code` THROTTLED / MAX_COST_EXCEEDED
-    (https://shopify.dev/docs/api/admin-graphql/2026-07#status-and-error-codes). Retry such pages instead of
-    treating the 200 as a success: the cost bucket refills at the shop's restore rate within seconds.
+    Shopify GraphQL reports throttling (THROTTLED / MAX_COST_EXCEEDED) and most internal errors (INTERNAL_SERVER_ERROR)
+    as HTTP 200 with `errors[].extensions.code` (https://shopify.dev/docs/api/admin-graphql/2026-10#status-and-error-codes).
+    Retry such pages instead of treating the 200 as a success: the cost bucket refills at the shop's restore rate
+    within seconds, and an internal error is retried like an HTTP 500.
     """
 
     def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
@@ -384,10 +387,10 @@ class ShopifyGraphQlErrorHandler(HttpStatusErrorHandler):
                 body = response_or_exception.json()
             except requests.exceptions.RequestException:
                 body = {}
-            if is_throttled_graphql_error(body.get("errors") or [] if isinstance(body, dict) else []):
+            if is_retryable_graphql_error(body.get("errors") or [] if isinstance(body, dict) else []):
                 return ErrorResolution(
                     response_action=ResponseAction.RETRY,
                     failure_type=FailureType.transient_error,
-                    error_message="Shopify GraphQL request throttled, retrying.",
+                    error_message="Shopify GraphQL request hit a transient error, retrying.",
                 )
         return super().interpret_response(response_or_exception)