airbyte-source-shopify 3.0.6.dev202505192339__tar.gz → 3.0.8__tar.gz

{airbyte_source_shopify-3.0.6.dev202505192339 → airbyte_source_shopify-3.0.8}/PKG-INFO

@@ -1,7 +1,8 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 3.0.6.dev202505192339
+Version: 3.0.8
 Summary: Source CDK implementation for Shopify.
+Home-page: https://airbyte.com
 License: ELv2
 Author: Airbyte
 Author-email: contact@airbyte.io
@@ -15,7 +16,6 @@ Requires-Dist: graphql-query (>=1,<2)
 Requires-Dist: pendulum (>=2.1.2,<3.0.0)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/shopify
-Project-URL: Homepage, https://airbyte.com
 Project-URL: Repository, https://github.com/airbytehq/airbyte
 Description-Content-Type: text/markdown
 

{airbyte_source_shopify-3.0.6.dev202505192339 → airbyte_source_shopify-3.0.8}/pyproject.toml

@@ -1,24 +1,19 @@
 [build-system]
-requires = [
-    "poetry-core>=1.0.0",
-]
+requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "3.0.6.dev202505192339"
+version = "3.0.8"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
-authors = [
-    "Airbyte <contact@airbyte.io>",
-]
+authors = [ "Airbyte <contact@airbyte.io>",]
 license = "ELv2"
 readme = "README.md"
 documentation = "https://docs.airbyte.com/integrations/sources/shopify"
 homepage = "https://airbyte.com"
 repository = "https://github.com/airbytehq/airbyte"
-packages = [
-    { include = "source_shopify" },
-]
+[[tool.poetry.packages]]
+include = "source_shopify"
 
 [tool.poetry.dependencies]
 python = "^3.10,<3.12"
@@ -38,5 +33,7 @@ freezegun = "^1.4.0"
 
 [tool.poe]
 include = [
+    # Shared tasks definition file(s) can be imported here.
+    # Run `poe` or `poe --help` to see the list of available tasks.
     "${POE_GIT_DIR}/poe-tasks/poetry-connector-tasks.toml",
 ]

{airbyte_source_shopify-3.0.6.dev202505192339 → airbyte_source_shopify-3.0.8}/source_shopify/streams/base_streams.py

@@ -45,7 +45,6 @@ class ShopifyStream(HttpStream, ABC):
         super().__init__(authenticator=config["authenticator"])
         self._transformer = DataTypeEnforcer(self.get_json_schema())
         self.config = config
-        self._current_limit = self.limit  # Dynamic limit initialized to default
 
     @property
     @abstractmethod
@@ -74,37 +73,24 @@ class ShopifyStream(HttpStream, ABC):
             return None
 
     def request_params(self, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs) -> MutableMapping[str, Any]:
-        params = {"limit": self._current_limit}
+        params = {"limit": self.limit}
         if next_page_token:
-            temp_params = dict(next_page_token)
-            temp_params.pop("limit", None)
-            params.update(temp_params)
+            params.update(**next_page_token)
         else:
             params["order"] = f"{self.order_field} asc"
-            stream_state = kwargs.get("stream_state")
-            if stream_state is not None:
-                params[self.filter_field] = stream_state.get(self.filter_field, self.default_filter_field_value)
-            else:
-                params[self.filter_field] = self.default_filter_field_value
+            params[self.filter_field] = self.default_filter_field_value
         return params
 
+    @limiter.balance_rate_limit()
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        if response.status_code == requests.codes.OK:
-            self._current_limit = self.limit
+        if response.status_code is requests.codes.OK:
             try:
                 json_response = response.json()
-                records = json_response.get(self.data_field, []) if self.data_field else json_response
+                records = json_response.get(self.data_field, []) if self.data_field is not None else json_response
                 yield from self.produce_records(records)
-            except requests.exceptions.JSONDecodeError as e:
-                error_msg = (
-                    f"Failed to decode JSON from response (status code: {response.status_code}, "
-                    f"content length: {len(response.content)}). JSONDecodeError at position {e.pos}: {e.msg}"
-                )
-                self.logger.warning(error_msg)
+            except RequestException as e:
+                self.logger.warning(f"Unexpected error in `parse_response`: {e}, the actual response data: {response.text}")
                 yield {}
-        else:
-            self.logger.warning(f"Non-OK response: {response.status_code}")
-            yield from []
 
     def produce_records(
         self, records: Optional[Union[Iterable[Mapping[str, Any]], Mapping[str, Any]]] = None

{airbyte_source_shopify-3.0.6.dev202505192339 → airbyte_source_shopify-3.0.8}/source_shopify/streams/streams.py

@@ -3,6 +3,8 @@
 #
 
 
+import logging
+import sys
 from typing import Any, Iterable, Mapping, MutableMapping, Optional
 
 import requests
@@ -31,11 +33,12 @@ from source_shopify.shopify_graphql.bulk.query import (
     ProfileLocationGroups,
     Transaction,
 )
-from source_shopify.utils import LimitReducingErrorHandler
+from source_shopify.utils import LimitReducingErrorHandler, ShopifyNonRetryableErrors
 
 from airbyte_cdk import HttpSubStream
 from airbyte_cdk.sources.streams.core import package_name_from_class
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING
 from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
 
 from .base_streams import (
@@ -86,18 +89,24 @@ class MetafieldCustomers(IncrementalShopifyGraphQlBulkStream):
 class Orders(IncrementalShopifyStreamWithDeletedEvents):
     data_field = "orders"
     deleted_events_api_name = "Order"
+    initial_limit = 250
 
-    def request_params(
-        self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs
-    ) -> MutableMapping[str, Any]:
+    def __init__(self, config: Mapping[str, Any]):
+        self._error_handler = LimitReducingErrorHandler(
+            max_retries=5,
+            error_mapping=DEFAULT_ERROR_MAPPING | ShopifyNonRetryableErrors("orders"),
+        )
+        super().__init__(config)
+
+    def request_params(self, stream_state=None, next_page_token=None, **kwargs):
         params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs)
+        params["limit"] = self.initial_limit  # Always start with the default limit; error handler will mutate on retry
         if not next_page_token:
             params["status"] = "any"
         return params
 
-    def get_error_handler(self) -> Optional[ErrorHandler]:
-        default_handler = super().get_error_handler()
-        return LimitReducingErrorHandler(stream=self, default_handler=default_handler)
+    def get_error_handler(self):
+        return self._error_handler
 
 
 class Disputes(IncrementalShopifyStream):
@@ -130,6 +139,12 @@ class MetafieldProducts(IncrementalShopifyGraphQlBulkStream):
     parent_stream_class = Products
     bulk_query: MetafieldProduct = MetafieldProduct
 
+    state_checkpoint_interval = sys.maxsize
+
+    @property
+    def filter_by_state_checkpoint(self) -> bool:
+        return True
+
 
 class ProductImages(IncrementalShopifyGraphQlBulkStream):
     parent_stream_class = Products

{airbyte_source_shopify-3.0.6.dev202505192339 → airbyte_source_shopify-3.0.8}/source_shopify/utils.py

@@ -13,7 +13,7 @@ from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 import requests
 
 from airbyte_cdk.models import FailureType
-from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler
 from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction
 from airbyte_cdk.utils import AirbyteTracedException
 
@@ -327,49 +327,36 @@ class EagerlyCachedStreamState:
         return decorator
 
 
-class LimitReducingErrorHandler(ErrorHandler):
-    """Custom error handler that reduces the request limit (of items/page) on 500 errors and retries.
-    limit is halved on each retry until it reaches 1. We still exponentially backoff on retries in addition to this.
+class LimitReducingErrorHandler(HttpStatusErrorHandler):
+    """
+    Error handler that halves the page size (limit) on each 500 error, down to 1.
+    No stream instance required; operates directly on the request URL.
     """
 
-    def __init__(self, stream: "ShopifyStream", default_handler: ErrorHandler):
-        self.stream = stream
-        self.default_handler = default_handler
-
-    @property
-    def max_retries(self) -> Optional[int]:
-        return self.default_handler.max_retries
-
-    @property
-    def max_time(self) -> Optional[int]:
-        return self.default_handler.max_time
+    def __init__(self, max_retries: int, error_mapping: dict):
+        super().__init__(logger=None, max_retries=max_retries, error_mapping=error_mapping)
 
-    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]]) -> ErrorResolution:
+    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
         if isinstance(response_or_exception, requests.Response):
             response = response_or_exception
             if response.status_code == 500:
-                current_limit = self.stream._current_limit
+                # Extract current limit from the URL, default to 250 if not present
+                parsed = urlparse(response.request.url)
+                query = parse_qs(parsed.query)
+                current_limit = int(query.get("limit", ["250"])[0])
                 if current_limit > 1:
                     new_limit = max(1, current_limit // 2)
-                    self.stream._current_limit = new_limit
-                    new_url = self.update_limit_in_url(response.request.url, new_limit)
-                    response.request.url = new_url
+                    query["limit"] = [str(new_limit)]
+                    new_query = urlencode(query, doseq=True)
+                    response.request.url = urlunparse(parsed._replace(query=new_query))
                     return ErrorResolution(
                         response_action=ResponseAction.RETRY,
                         failure_type=FailureType.transient_error,
-                        error_message=f"Server error 500: Reduced limit to {new_limit}",
+                        error_message=f"Server error 500: Reduced limit to {new_limit} and updating request URL",
                     )
                 return ErrorResolution(
                     response_action=ResponseAction.FAIL,
                     failure_type=FailureType.transient_error,
                     error_message="Persistent 500 error after reducing limit to 1",
                 )
-        return self.default_handler.interpret_response(response_or_exception)
-
-    def update_limit_in_url(self, url: str, new_limit: int) -> str:
-        """Update the 'limit' parameter in the URL query string."""
-        parsed = urlparse(url)
-        query = parse_qs(parsed.query)
-        query["limit"] = [str(new_limit)]
-        new_query = urlencode(query, doseq=True)
-        return urlunparse(parsed._replace(query=new_query))
+        return super().interpret_response(response_or_exception)