airbyte-source-shopify 3.0.5.dev202505152023__py3-none-any.whl → 3.0.6.dev202505192339__py3-none-any.whl

{airbyte_source_shopify-3.0.5.dev202505152023.dist-info → airbyte_source_shopify-3.0.6.dev202505192339.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: airbyte-source-shopify
-Version: 3.0.5.dev202505152023
+Version: 3.0.6.dev202505192339
 Summary: Source CDK implementation for Shopify.
 License: ELv2
 Author: Airbyte

{airbyte_source_shopify-3.0.5.dev202505152023.dist-info → airbyte_source_shopify-3.0.6.dev202505192339.dist-info}/RECORD

@@ -59,12 +59,12 @@ source_shopify/shopify_graphql/bulk/retry.py,sha256=R5rSJJE8D5zcj6mN-OmmNO2aFZEI
 source_shopify/shopify_graphql/bulk/status.py,sha256=RmuQ2XsYL3iRCpVGxea9F1wXGmbwasDCSXjaTyL4LMA,328
 source_shopify/shopify_graphql/bulk/tools.py,sha256=nUQ2ZmPTKJNJdfLToR6KJtLKcJFCChSifkAOvwg0Vss,4065
 source_shopify/source.py,sha256=txb3wIm-3xXd8-5QLSeu2TeHBSnppwy5PEIOEl40mVw,8517
-source_shopify/spec.json,sha256=ndDvDlMnjmWm_Ymb1l2skn1mjP-gCC1BscEopjVGkjM,6170
-source_shopify/streams/base_streams.py,sha256=FFIpHd5_-Z61W_jUucdr8D2MzUete1Y2E50bQDCLakE,41555
-source_shopify/streams/streams.py,sha256=8LkM-SRhbGX2MwfHsjcWY62Z6g0jKZ0QfcS4B-vKPoM,13882
+source_shopify/spec.json,sha256=ITYWiQ-NrI5VISk5qmUQhp9ChUE2FV18d8xzVzPwvAg,6144
+source_shopify/streams/base_streams.py,sha256=Jhxe4rkm330g8Us1C8V-9-Zb7-4z1z5xrNtSI5QjQ28,42272
+source_shopify/streams/streams.py,sha256=ys0v66J2NTGEJr1L78FuxZBcFoOpLA-FJwf5OcbvwhM,14215
 source_shopify/transform.py,sha256=mn0htL812_90zc_YszGQa0hHcIZQpYYdmk8IqpZm5TI,4685
-source_shopify/utils.py,sha256=CAEKcxIroBo6kRFCyvC1bfOyfbGy7Z7seqZB7Eekl44,14209
-airbyte_source_shopify-3.0.5.dev202505152023.dist-info/METADATA,sha256=4a7JyPg4YXoGCGurKvhiRIHwXd4nkQ3pnEUun6_xQ90,5325
-airbyte_source_shopify-3.0.5.dev202505152023.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-airbyte_source_shopify-3.0.5.dev202505152023.dist-info/entry_points.txt,sha256=SyTwKSsPk9MCdPf01saWpnp8hcmZOgBssVcSIvMbBeQ,57
-airbyte_source_shopify-3.0.5.dev202505152023.dist-info/RECORD,,
+source_shopify/utils.py,sha256=sjiBSh7Ygtg1RTsFvGY3oDjAhqEaodFiLmQd2aurgSk,16609
+airbyte_source_shopify-3.0.6.dev202505192339.dist-info/METADATA,sha256=AjuciXe4zunbOM2FEdv0qQFUhLeKIQlErtCpVoQiRO0,5325
+airbyte_source_shopify-3.0.6.dev202505192339.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+airbyte_source_shopify-3.0.6.dev202505192339.dist-info/entry_points.txt,sha256=SyTwKSsPk9MCdPf01saWpnp8hcmZOgBssVcSIvMbBeQ,57
+airbyte_source_shopify-3.0.6.dev202505192339.dist-info/RECORD,,

source_shopify/spec.json

@@ -4,7 +4,7 @@
     "$schema": "https://json-schema.org/draft-07/schema#",
     "title": "Shopify Source CDK Specifications",
     "type": "object",
-    "required": [],
+    "required": ["shop"],
     "additionalProperties": true,
     "properties": {
       "shop": {
@@ -13,8 +13,7 @@
         "description": "The name of your Shopify store found in the URL. For example, if your URL was https://NAME.myshopify.com, then the name would be 'NAME' or 'NAME.myshopify.com'.",
         "pattern": "^(?!https://)(?!https://).*",
         "examples": ["my-store", "my-store.myshopify.com"],
-        "order": 1,
-        "airbyte_hidden": true
+        "order": 1
       },
       "credentials": {
         "title": "Shopify Authorization Method",

source_shopify/streams/base_streams.py

@@ -45,6 +45,7 @@ class ShopifyStream(HttpStream, ABC):
         super().__init__(authenticator=config["authenticator"])
         self._transformer = DataTypeEnforcer(self.get_json_schema())
         self.config = config
+        self._current_limit = self.limit  # Dynamic limit initialized to default
 
     @property
     @abstractmethod
@@ -73,24 +74,37 @@ class ShopifyStream(HttpStream, ABC):
             return None
 
     def request_params(self, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs) -> MutableMapping[str, Any]:
-        params = {"limit": self.limit}
+        params = {"limit": self._current_limit}
         if next_page_token:
-            params.update(**next_page_token)
+            temp_params = dict(next_page_token)
+            temp_params.pop("limit", None)
+            params.update(temp_params)
         else:
             params["order"] = f"{self.order_field} asc"
-            params[self.filter_field] = self.default_filter_field_value
+            stream_state = kwargs.get("stream_state")
+            if stream_state is not None:
+                params[self.filter_field] = stream_state.get(self.filter_field, self.default_filter_field_value)
+            else:
+                params[self.filter_field] = self.default_filter_field_value
         return params
 
-    @limiter.balance_rate_limit()
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        if response.status_code is requests.codes.OK:
+        if response.status_code == requests.codes.OK:
+            self._current_limit = self.limit
             try:
                 json_response = response.json()
-                records = json_response.get(self.data_field, []) if self.data_field is not None else json_response
+                records = json_response.get(self.data_field, []) if self.data_field else json_response
                 yield from self.produce_records(records)
-            except RequestException as e:
-                self.logger.warning(f"Unexpected error in `parse_response`: {e}, the actual response data: {response.text}")
+            except requests.exceptions.JSONDecodeError as e:
+                error_msg = (
+                    f"Failed to decode JSON from response (status code: {response.status_code}, "
+                    f"content length: {len(response.content)}). JSONDecodeError at position {e.pos}: {e.msg}"
+                )
+                self.logger.warning(error_msg)
                 yield {}
+        else:
+            self.logger.warning(f"Non-OK response: {response.status_code}")
+            yield from []
 
     def produce_records(
         self, records: Optional[Union[Iterable[Mapping[str, Any]], Mapping[str, Any]]] = None

source_shopify/streams/streams.py

@@ -31,9 +31,11 @@ from source_shopify.shopify_graphql.bulk.query import (
     ProfileLocationGroups,
     Transaction,
 )
+from source_shopify.utils import LimitReducingErrorHandler
 
 from airbyte_cdk import HttpSubStream
 from airbyte_cdk.sources.streams.core import package_name_from_class
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
 from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
 
 from .base_streams import (
@@ -93,6 +95,10 @@ class Orders(IncrementalShopifyStreamWithDeletedEvents):
             params["status"] = "any"
         return params
 
+    def get_error_handler(self) -> Optional[ErrorHandler]:
+        default_handler = super().get_error_handler()
+        return LimitReducingErrorHandler(stream=self, default_handler=default_handler)
+
 
 class Disputes(IncrementalShopifyStream):
     data_field = "disputes"

source_shopify/utils.py

@@ -7,11 +7,13 @@ import enum
 import logging
 from functools import wraps
 from time import sleep
-from typing import Any, Callable, Dict, Final, List, Mapping, Optional
+from typing import Any, Callable, Dict, Final, List, Mapping, Optional, Union
+from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
 import requests
 
 from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
 from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction
 from airbyte_cdk.utils import AirbyteTracedException
 
@@ -323,3 +325,51 @@ class EagerlyCachedStreamState:
             return func(*args, **kwargs)
 
         return decorator
+
+
+class LimitReducingErrorHandler(ErrorHandler):
+    """Custom error handler that reduces the request limit (of items/page) on 500 errors and retries.
+    limit is halved on each retry until it reaches 1. We still exponentially backoff on retries in addition to this.
+    """
+
+    def __init__(self, stream: "ShopifyStream", default_handler: ErrorHandler):
+        self.stream = stream
+        self.default_handler = default_handler
+
+    @property
+    def max_retries(self) -> Optional[int]:
+        return self.default_handler.max_retries
+
+    @property
+    def max_time(self) -> Optional[int]:
+        return self.default_handler.max_time
+
+    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]]) -> ErrorResolution:
+        if isinstance(response_or_exception, requests.Response):
+            response = response_or_exception
+            if response.status_code == 500:
+                current_limit = self.stream._current_limit
+                if current_limit > 1:
+                    new_limit = max(1, current_limit // 2)
+                    self.stream._current_limit = new_limit
+                    new_url = self.update_limit_in_url(response.request.url, new_limit)
+                    response.request.url = new_url
+                    return ErrorResolution(
+                        response_action=ResponseAction.RETRY,
+                        failure_type=FailureType.transient_error,
+                        error_message=f"Server error 500: Reduced limit to {new_limit}",
+                    )
+                return ErrorResolution(
+                    response_action=ResponseAction.FAIL,
+                    failure_type=FailureType.transient_error,
+                    error_message="Persistent 500 error after reducing limit to 1",
+                )
+        return self.default_handler.interpret_response(response_or_exception)
+
+    def update_limit_in_url(self, url: str, new_limit: int) -> str:
+        """Update the 'limit' parameter in the URL query string."""
+        parsed = urlparse(url)
+        query = parse_qs(parsed.query)
+        query["limit"] = [str(new_limit)]
+        new_query = urlencode(query, doseq=True)
+        return urlunparse(parsed._replace(query=new_query))