airbyte-source-shopify 3.0.6.dev202505192339__py3-none-any.whl → 3.0.8.dev202507031652__py3-none-any.whl

{airbyte_source_shopify-3.0.6.dev202505192339.dist-info → airbyte_source_shopify-3.0.8.dev202507031652.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-source-shopify
-Version: 3.0.6.dev202505192339
+Version: 3.0.8.dev202507031652
 Summary: Source CDK implementation for Shopify.
 License: ELv2
 Author: Airbyte

{airbyte_source_shopify-3.0.6.dev202505192339.dist-info → airbyte_source_shopify-3.0.8.dev202507031652.dist-info}/RECORD

@@ -52,7 +52,7 @@ source_shopify/schemas/transactions.json,sha256=vbwscH3UcAtbSsC70mBka4oNaFR4S3S6
 source_shopify/scopes.py,sha256=N0njfMHn3Q1AQXuTj5VfjQOio10jaDarpC_oLYnWvqc,6490
 source_shopify/shopify_graphql/bulk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 source_shopify/shopify_graphql/bulk/exceptions.py,sha256=4dj7Za4xIfwL-zf8joT9svF_RSoGlE3GviMiIl1e1rs,2532
-source_shopify/shopify_graphql/bulk/job.py,sha256=0a3els2a5Fea-o30rkotq7yoPx56c481zwpZzxDajEw,27832
+source_shopify/shopify_graphql/bulk/job.py,sha256=Bk158tbTtX18FfBJFCvv-mzfhsSV3qHFOgvV7T4e5Qk,35914
 source_shopify/shopify_graphql/bulk/query.py,sha256=D8rnI1SDw50-Gt18lt7YwwNNdsbVMbBfxZa9xVJZbto,130981
 source_shopify/shopify_graphql/bulk/record.py,sha256=X6VGngugv7a_S8UEeDo121BkdCVLj5nWlHK76A21kyo,16898
 source_shopify/shopify_graphql/bulk/retry.py,sha256=R5rSJJE8D5zcj6mN-OmmNO2aFZEIdjAlWclDDVW5KPI,2626
@@ -60,11 +60,11 @@ source_shopify/shopify_graphql/bulk/status.py,sha256=RmuQ2XsYL3iRCpVGxea9F1wXGmb
 source_shopify/shopify_graphql/bulk/tools.py,sha256=nUQ2ZmPTKJNJdfLToR6KJtLKcJFCChSifkAOvwg0Vss,4065
 source_shopify/source.py,sha256=txb3wIm-3xXd8-5QLSeu2TeHBSnppwy5PEIOEl40mVw,8517
 source_shopify/spec.json,sha256=ITYWiQ-NrI5VISk5qmUQhp9ChUE2FV18d8xzVzPwvAg,6144
-source_shopify/streams/base_streams.py,sha256=Jhxe4rkm330g8Us1C8V-9-Zb7-4z1z5xrNtSI5QjQ28,42272
-source_shopify/streams/streams.py,sha256=ys0v66J2NTGEJr1L78FuxZBcFoOpLA-FJwf5OcbvwhM,14215
+source_shopify/streams/base_streams.py,sha256=FFIpHd5_-Z61W_jUucdr8D2MzUete1Y2E50bQDCLakE,41555
+source_shopify/streams/streams.py,sha256=D70Ik1vU75NKlmJMnS7W2-5gApA2ANq9eRnKligMTNw,14555
 source_shopify/transform.py,sha256=mn0htL812_90zc_YszGQa0hHcIZQpYYdmk8IqpZm5TI,4685
-source_shopify/utils.py,sha256=sjiBSh7Ygtg1RTsFvGY3oDjAhqEaodFiLmQd2aurgSk,16609
-airbyte_source_shopify-3.0.6.dev202505192339.dist-info/METADATA,sha256=AjuciXe4zunbOM2FEdv0qQFUhLeKIQlErtCpVoQiRO0,5325
-airbyte_source_shopify-3.0.6.dev202505192339.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-airbyte_source_shopify-3.0.6.dev202505192339.dist-info/entry_points.txt,sha256=SyTwKSsPk9MCdPf01saWpnp8hcmZOgBssVcSIvMbBeQ,57
-airbyte_source_shopify-3.0.6.dev202505192339.dist-info/RECORD,,
+source_shopify/utils.py,sha256=DSqEchu-MQJ7zust7CNfqOkGIv9OSR-5UUsuD-bsDa8,16224
+airbyte_source_shopify-3.0.8.dev202507031652.dist-info/METADATA,sha256=IWFlmQxyJV77pb_SwLFclF38YlEhIIF2H2IyWnwGzcs,5325
+airbyte_source_shopify-3.0.8.dev202507031652.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+airbyte_source_shopify-3.0.8.dev202507031652.dist-info/entry_points.txt,sha256=SyTwKSsPk9MCdPf01saWpnp8hcmZOgBssVcSIvMbBeQ,57
+airbyte_source_shopify-3.0.8.dev202507031652.dist-info/RECORD,,

source_shopify/shopify_graphql/bulk/job.py

@@ -5,6 +5,7 @@
 from dataclasses import dataclass, field
 from datetime import datetime
 from enum import Enum
+from json import loads
 from time import sleep, time
 from typing import Any, Final, Iterable, List, Mapping, Optional
 
@@ -65,7 +66,7 @@ class ShopifyBulkManager:
 
     # currents: _job_id, _job_state, _job_created_at, _job_self_canceled
     _job_id: Optional[str] = field(init=False, default=None)
-    _job_state: str | None = field(init=False, default=None)  # this string is based on ShopifyBulkJobStatus
+    _job_state: Optional[str] = field(init=False, default=None)  # this string is based on ShopifyBulkJobStatus
     # completed and saved Bulk Job result filename
     _job_result_filename: Optional[str] = field(init=False, default=None)
     # date-time when the Bulk Job was created on the server
@@ -83,7 +84,9 @@ class ShopifyBulkManager:
     # the flag to adjust the next slice from the checkpointed cursor vaue
     _job_adjust_slice_from_checkpoint: bool = field(init=False, default=False)
     # keeps the last checkpointed cursor value for supported streams
-    _job_last_checkpoint_cursor_value: str | None = field(init=False, default=None)
+    _job_last_checkpoint_cursor_value: Optional[str] = field(init=False, default=None)
+    # stores extracted cursor from INTERNAL_SERVER_ERROR recovery (temporary storage)
+    _job_extracted_checkpoint_cursor: Optional[str] = field(init=False, default=None)
 
     # expand slice factor
     _job_size_expand_factor: int = field(init=False, default=2)
@@ -214,6 +217,8 @@ class ShopifyBulkManager:
         self._log_job_msg_count = 0
         # set the running job object count to default
         self._job_last_rec_count = 0
+        # clear any extracted cursor from INTERNAL_SERVER_ERROR recovery
+        self._job_extracted_checkpoint_cursor = None
 
     def _set_checkpointing(self) -> None:
         # set the flag to adjust the next slice from the checkpointed cursor value
@@ -313,6 +318,24 @@ class ShopifyBulkManager:
             # fetch the collected records from CANCELED Job on checkpointing
             self._job_result_filename = self._job_get_result(response)
 
+            # Special handling: For FAILED jobs with INTERNAL_SERVER_ERROR, extract the last processed cursor
+            if response:
+                parsed_response = response.json().get("data", {}).get("node", {}) if response else {}
+                error_code = parsed_response.get("errorCode")
+                if error_code == "INTERNAL_SERVER_ERROR":
+                    last_cursor = self._extract_last_cursor_from_partial_data(response)
+                    if last_cursor:
+                        # Check if this cursor would cause a collision before storing it
+                        if self._checkpoint_cursor_has_collision(last_cursor):
+                            # Skip cursor extraction to avoid collision
+                            pass
+                        else:
+                            # Store the extracted cursor for later use (don't set it yet to avoid collision)
+                            self._job_extracted_checkpoint_cursor = last_cursor
+        else:
+            # Not processing data due to insufficient records or checkpointing disabled
+            pass
+
     def _job_update_state(self, response: Optional[requests.Response] = None) -> None:
         if response:
             self._job_state = response.json().get("data", {}).get("node", {}).get("status")
@@ -363,7 +386,26 @@ class ShopifyBulkManager:
     def _on_completed_job(self, response: Optional[requests.Response] = None) -> None:
         self._job_result_filename = self._job_get_result(response)
 
-    def _on_failed_job(self, response: requests.Response) -> AirbyteTracedException | None:
+    def _on_failed_job(self, response: requests.Response) -> Optional[AirbyteTracedException]:
+        # Special handling for FAILED jobs with INTERNAL_SERVER_ERROR that support checkpointing
+        parsed_response = response.json().get("data", {}).get("node", {}) if response else {}
+        error_code = parsed_response.get("errorCode")
+
+        if error_code == "INTERNAL_SERVER_ERROR" and self._supports_checkpointing:
+            LOGGER.info(
+                f"Stream: `{self.http_client.name}`, BULK Job: `{self._job_id}` failed with INTERNAL_SERVER_ERROR. Waiting for partial data availability..."
+            )
+            # For INTERNAL_SERVER_ERROR specifically, wait and retry to check if partial data becomes available
+            partial_response = self._wait_for_partial_data_on_failure()
+            if partial_response:
+                # Use the updated response that may contain partialDataUrl
+                response = partial_response
+                # Update the job state with the new response to ensure _job_last_rec_count is set correctly
+                self._job_update_state(response)
+                # For INTERNAL_SERVER_ERROR with partial data, extract cursor and treat as checkpointable
+                self._job_get_checkpointed_result(response)
+                return None  # Don't raise exception, we recovered the data
+
         if not self._supports_checkpointing:
             raise ShopifyBulkExceptions.BulkJobFailed(
                 f"The BULK Job: `{self._job_id}` exited with {self._job_state}, details: {response.text}",
@@ -373,6 +415,102 @@ class ShopifyBulkManager:
             # we leverage the checkpointing in this case.
             self._job_get_checkpointed_result(response)
 
+    def _wait_for_partial_data_on_failure(self) -> Optional[requests.Response]:
+        """
+        Wait for partial data to become available when a BULK job fails with INTERNAL_SERVER_ERROR.
+
+        This method is specifically designed for INTERNAL_SERVER_ERROR cases where
+        Shopify's BULK API may make partial data available (via partialDataUrl)
+        after a short wait, even though the job initially failed.
+
+        Returns:
+            Optional[requests.Response]: Updated response with potential partialDataUrl, or None if no data
+        """
+        max_wait_attempts = 10  # Maximum number of wait attempts
+        wait_interval = 10  # Wait 10 seconds between checks
+
+        for attempt in range(max_wait_attempts):
+            sleep(wait_interval)
+
+            # Check job status again to see if partial data is now available
+            try:
+                _, response = self.http_client.send_request(
+                    http_method="POST",
+                    url=self.base_url,
+                    json={"query": ShopifyBulkTemplates.status(self._job_id)},
+                    request_kwargs={},
+                )
+
+                parsed_response = response.json().get("data", {}).get("node", {}) if response else {}
+                partial_data_url = parsed_response.get("partialDataUrl")
+                object_count = parsed_response.get("objectCount", "0")
+
+                # Only stop waiting if we actually have a partialDataUrl - objectCount alone is not sufficient
+                if partial_data_url and int(object_count) > 0:
+                    LOGGER.info(f"Stream: `{self.http_client.name}`, partial data available after wait. Object count: {object_count}")
+                    return response
+                elif int(object_count) > 0:
+                    # objectCount available but no partialDataUrl yet - continue waiting
+                    continue
+
+            except Exception as e:
+                # Error during partial data check - continue waiting
+                continue
+
+        LOGGER.warning(f"Stream: `{self.http_client.name}`, no partial data became available after {max_wait_attempts} attempts")
+        return None
+
+    def _extract_last_cursor_from_partial_data(self, response: Optional[requests.Response]) -> Optional[str]:
+        """
+        Extract the last processed cursor value from partial data for INTERNAL_SERVER_ERROR recovery.
+
+        This method retrieves partial data from a failed INTERNAL_SERVER_ERROR job and extracts
+        the updatedAt value of the last record, which can be used to resume processing from that point.
+        Only used in INTERNAL_SERVER_ERROR scenarios with checkpointing support.
+
+        Args:
+            response: The response containing partial data information
+
+        Returns:
+            Optional[str]: The cursor value of the last processed record, or None if unavailable
+        """
+        if not response:
+            return None
+
+        try:
+            parsed_response = response.json().get("data", {}).get("node", {})
+            partial_data_url = parsed_response.get("partialDataUrl")
+
+            if not partial_data_url:
+                return None
+
+            # Download the partial data
+            _, partial_response = self.http_client.send_request(http_method="GET", url=partial_data_url, request_kwargs={"stream": True})
+            partial_response.raise_for_status()
+
+            last_record = None
+            # Read through the JSONL data to find the last record
+            for line in partial_response.iter_lines(decode_unicode=True):
+                if line and line.strip() and line.strip() != END_OF_FILE:
+                    try:
+                        record = loads(line)
+                        # Look for the main record types (Order, Product, etc.)
+                        if record.get("__typename") in ["Order", "Product", "Customer", "FulfillmentOrder"]:
+                            last_record = record
+                    except Exception:
+                        continue
+
+            # Extract the updatedAt cursor from the last record
+            if last_record and "updatedAt" in last_record:
+                cursor_value = last_record["updatedAt"]
+                return cursor_value
+
+        except Exception as e:
+            # Failed to extract cursor from partial data
+            pass
+
+        return None
+
     def _on_timeout_job(self, **kwargs) -> AirbyteTracedException:
         raise ShopifyBulkExceptions.BulkJobTimout(
             f"The BULK Job: `{self._job_id}` exited with {self._job_state}, please reduce the `GraphQL BULK Date Range in Days` in SOURCES > Your Shopify Source > SETTINGS.",
@@ -535,9 +673,15 @@ class ShopifyBulkManager:
         """
 
         if checkpointed_cursor:
+            # Check for collision and provide more context in the error
             if self._checkpoint_cursor_has_collision(checkpointed_cursor):
+                # For INTERNAL_SERVER_ERROR recovery, if the cursor is the same, we might need to skip ahead slightly
+                # This can happen if the failure occurred right at the boundary of what was already processed
+                if hasattr(self, "_job_extracted_checkpoint_cursor") and self._job_extracted_checkpoint_cursor == checkpointed_cursor:
+                    pass  # Collision from INTERNAL_SERVER_ERROR recovery at boundary
+
                 raise ShopifyBulkExceptions.BulkJobCheckpointCollisionError(
-                    f"The stream: `{self.http_client.name}` checkpoint collision is detected. Try to increase the `BULK Job checkpoint (rows collected)` to the bigger value. The stream will be synced again during the next sync attempt."
+                    f"The stream: `{self.http_client.name}` checkpoint collision is detected. Current cursor: {self._job_last_checkpoint_cursor_value}, New cursor: {checkpointed_cursor}. Try to increase the `BULK Job checkpoint (rows collected)` to the bigger value. The stream will be synced again during the next sync attempt."
                 )
             # set the checkpointed cursor value
             self._set_last_checkpoint_cursor_value(checkpointed_cursor)
@@ -549,7 +693,14 @@ class ShopifyBulkManager:
         if self._job_adjust_slice_from_checkpoint:
             # set the checkpointing to default, before the next slice is emitted, to avoid inf.loop
             self._reset_checkpointing()
-            return self._adjust_slice_end(slice_end, checkpointed_cursor)
+            # Clear the extracted cursor after use to avoid reusing it
+            if self._job_extracted_checkpoint_cursor:
+                extracted_cursor = self._job_extracted_checkpoint_cursor
+                self._job_extracted_checkpoint_cursor = None
+                cursor_to_use = extracted_cursor
+            else:
+                cursor_to_use = checkpointed_cursor or self._job_last_checkpoint_cursor_value
+            return self._adjust_slice_end(slice_end, cursor_to_use)
 
         if self._is_long_running_job:
             self._job_size_reduce_next()

source_shopify/streams/base_streams.py

@@ -45,7 +45,6 @@ class ShopifyStream(HttpStream, ABC):
         super().__init__(authenticator=config["authenticator"])
         self._transformer = DataTypeEnforcer(self.get_json_schema())
         self.config = config
-        self._current_limit = self.limit  # Dynamic limit initialized to default
 
     @property
     @abstractmethod
@@ -74,37 +73,24 @@ class ShopifyStream(HttpStream, ABC):
             return None
 
     def request_params(self, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs) -> MutableMapping[str, Any]:
-        params = {"limit": self._current_limit}
+        params = {"limit": self.limit}
         if next_page_token:
-            temp_params = dict(next_page_token)
-            temp_params.pop("limit", None)
-            params.update(temp_params)
+            params.update(**next_page_token)
         else:
             params["order"] = f"{self.order_field} asc"
-            stream_state = kwargs.get("stream_state")
-            if stream_state is not None:
-                params[self.filter_field] = stream_state.get(self.filter_field, self.default_filter_field_value)
-            else:
-                params[self.filter_field] = self.default_filter_field_value
+            params[self.filter_field] = self.default_filter_field_value
         return params
 
+    @limiter.balance_rate_limit()
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        if response.status_code == requests.codes.OK:
-            self._current_limit = self.limit
+        if response.status_code is requests.codes.OK:
             try:
                 json_response = response.json()
-                records = json_response.get(self.data_field, []) if self.data_field else json_response
+                records = json_response.get(self.data_field, []) if self.data_field is not None else json_response
                 yield from self.produce_records(records)
-            except requests.exceptions.JSONDecodeError as e:
-                error_msg = (
-                    f"Failed to decode JSON from response (status code: {response.status_code}, "
-                    f"content length: {len(response.content)}). JSONDecodeError at position {e.pos}: {e.msg}"
-                )
-                self.logger.warning(error_msg)
+            except RequestException as e:
+                self.logger.warning(f"Unexpected error in `parse_response`: {e}, the actual response data: {response.text}")
                 yield {}
-        else:
-            self.logger.warning(f"Non-OK response: {response.status_code}")
-            yield from []
 
     def produce_records(
         self, records: Optional[Union[Iterable[Mapping[str, Any]], Mapping[str, Any]]] = None

source_shopify/streams/streams.py

@@ -3,6 +3,7 @@
 #
 
 
+import logging
 from typing import Any, Iterable, Mapping, MutableMapping, Optional
 
 import requests
@@ -31,11 +32,12 @@ from source_shopify.shopify_graphql.bulk.query import (
     ProfileLocationGroups,
     Transaction,
 )
-from source_shopify.utils import LimitReducingErrorHandler
+from source_shopify.utils import LimitReducingErrorHandler, ShopifyNonRetryableErrors
 
 from airbyte_cdk import HttpSubStream
 from airbyte_cdk.sources.streams.core import package_name_from_class
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING
 from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
 
 from .base_streams import (
@@ -86,18 +88,24 @@ class MetafieldCustomers(IncrementalShopifyGraphQlBulkStream):
 class Orders(IncrementalShopifyStreamWithDeletedEvents):
     data_field = "orders"
     deleted_events_api_name = "Order"
+    initial_limit = 250
 
-    def request_params(
-        self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs
-    ) -> MutableMapping[str, Any]:
+    def __init__(self, config: Mapping[str, Any]):
+        self._error_handler = LimitReducingErrorHandler(
+            max_retries=5,
+            error_mapping=DEFAULT_ERROR_MAPPING | ShopifyNonRetryableErrors("orders"),
+        )
+        super().__init__(config)
+
+    def request_params(self, stream_state=None, next_page_token=None, **kwargs):
         params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs)
+        params["limit"] = self.initial_limit  # Always start with the default limit; error handler will mutate on retry
         if not next_page_token:
             params["status"] = "any"
         return params
 
-    def get_error_handler(self) -> Optional[ErrorHandler]:
-        default_handler = super().get_error_handler()
-        return LimitReducingErrorHandler(stream=self, default_handler=default_handler)
+    def get_error_handler(self):
+        return self._error_handler
 
 
 class Disputes(IncrementalShopifyStream):

source_shopify/utils.py

@@ -13,7 +13,7 @@ from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 import requests
 
 from airbyte_cdk.models import FailureType
-from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler
 from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction
 from airbyte_cdk.utils import AirbyteTracedException
 
@@ -327,49 +327,36 @@ class EagerlyCachedStreamState:
         return decorator
 
 
-class LimitReducingErrorHandler(ErrorHandler):
-    """Custom error handler that reduces the request limit (of items/page) on 500 errors and retries.
-    limit is halved on each retry until it reaches 1. We still exponentially backoff on retries in addition to this.
+class LimitReducingErrorHandler(HttpStatusErrorHandler):
+    """
+    Error handler that halves the page size (limit) on each 500 error, down to 1.
+    No stream instance required; operates directly on the request URL.
     """
 
-    def __init__(self, stream: "ShopifyStream", default_handler: ErrorHandler):
-        self.stream = stream
-        self.default_handler = default_handler
-
-    @property
-    def max_retries(self) -> Optional[int]:
-        return self.default_handler.max_retries
-
-    @property
-    def max_time(self) -> Optional[int]:
-        return self.default_handler.max_time
+    def __init__(self, max_retries: int, error_mapping: dict):
+        super().__init__(logger=None, max_retries=max_retries, error_mapping=error_mapping)
 
-    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]]) -> ErrorResolution:
+    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
         if isinstance(response_or_exception, requests.Response):
             response = response_or_exception
             if response.status_code == 500:
-                current_limit = self.stream._current_limit
+                # Extract current limit from the URL, default to 250 if not present
+                parsed = urlparse(response.request.url)
+                query = parse_qs(parsed.query)
+                current_limit = int(query.get("limit", ["250"])[0])
                 if current_limit > 1:
                     new_limit = max(1, current_limit // 2)
-                    self.stream._current_limit = new_limit
-                    new_url = self.update_limit_in_url(response.request.url, new_limit)
-                    response.request.url = new_url
+                    query["limit"] = [str(new_limit)]
+                    new_query = urlencode(query, doseq=True)
+                    response.request.url = urlunparse(parsed._replace(query=new_query))
                     return ErrorResolution(
                         response_action=ResponseAction.RETRY,
                         failure_type=FailureType.transient_error,
-                        error_message=f"Server error 500: Reduced limit to {new_limit}",
+                        error_message=f"Server error 500: Reduced limit to {new_limit} and updating request URL",
                     )
                 return ErrorResolution(
                     response_action=ResponseAction.FAIL,
                     failure_type=FailureType.transient_error,
                     error_message="Persistent 500 error after reducing limit to 1",
                 )
-        return self.default_handler.interpret_response(response_or_exception)
-
-    def update_limit_in_url(self, url: str, new_limit: int) -> str:
-        """Update the 'limit' parameter in the URL query string."""
-        parsed = urlparse(url)
-        query = parse_qs(parsed.query)
-        query["limit"] = [str(new_limit)]
-        new_query = urlencode(query, doseq=True)
-        return urlunparse(parsed._replace(query=new_query))
+        return super().interpret_response(response_or_exception)