airbyte-source-shopify 2.4.14.dev202407181247__py3-none-any.whl → 3.1.0__py3-none-any.whl

source_shopify/streams/streams.py

@@ -3,16 +3,16 @@
 #
 
 
+import logging
+import sys
 from typing import Any, Iterable, Mapping, MutableMapping, Optional
 
 import requests
-from airbyte_cdk.sources.streams.core import package_name_from_class
-from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
-from requests.exceptions import RequestException
 from source_shopify.shopify_graphql.bulk.query import (
     Collection,
     CustomerAddresses,
     CustomerJourney,
+    DeliveryProfile,
     DiscountCode,
     FulfillmentOrder,
     InventoryItem,
@@ -30,13 +30,19 @@ from source_shopify.shopify_graphql.bulk.query import (
     Product,
     ProductImage,
     ProductVariant,
+    ProfileLocationGroups,
     Transaction,
 )
-from source_shopify.shopify_graphql.graphql import get_query_products
-from source_shopify.utils import ApiTypeEnum
-from source_shopify.utils import ShopifyRateLimiter as limiter
+from source_shopify.utils import LimitReducingErrorHandler, ShopifyNonRetryableErrors
+
+from airbyte_cdk import HttpSubStream
+from airbyte_cdk.sources.streams.core import package_name_from_class
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING
+from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
 
 from .base_streams import (
+    FullRefreshShopifyGraphQlBulkStream,
     IncrementalShopifyGraphQlBulkStream,
     IncrementalShopifyNestedStream,
     IncrementalShopifyStream,
@@ -76,21 +82,32 @@ class Customers(IncrementalShopifyStream):
 
 
 class MetafieldCustomers(IncrementalShopifyGraphQlBulkStream):
+    parent_stream_class = Customers
     bulk_query: MetafieldCustomer = MetafieldCustomer
 
 
 class Orders(IncrementalShopifyStreamWithDeletedEvents):
     data_field = "orders"
     deleted_events_api_name = "Order"
+    initial_limit = 250
 
-    def request_params(
-        self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs
-    ) -> MutableMapping[str, Any]:
+    def __init__(self, config: Mapping[str, Any]):
+        self._error_handler = LimitReducingErrorHandler(
+            max_retries=5,
+            error_mapping=DEFAULT_ERROR_MAPPING | ShopifyNonRetryableErrors("orders"),
+        )
+        super().__init__(config)
+
+    def request_params(self, stream_state=None, next_page_token=None, **kwargs):
         params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs)
+        params["limit"] = self.initial_limit  # Always start with the default limit; error handler will mutate on retry
         if not next_page_token:
             params["status"] = "any"
         return params
 
+    def get_error_handler(self):
+        return self._error_handler
+
 
 class Disputes(IncrementalShopifyStream):
     data_field = "disputes"
@@ -116,69 +133,130 @@ class MetafieldDraftOrders(IncrementalShopifyGraphQlBulkStream):
 
 class Products(IncrementalShopifyGraphQlBulkStream):
     bulk_query: Product = Product
-    # pin the api version
 
 
-class ProductsGraphQl(IncrementalShopifyStream):
-    filter_field = "updatedAt"
-    cursor_field = "updatedAt"
+class DeletedProducts(IncrementalShopifyStream):
+    """
+    Stream for fetching deleted products using the Shopify GraphQL Events API.
+    This stream queries events with action:destroy and subject_type:Product to get deleted product records.
+    https://shopify.dev/docs/api/admin-graphql/latest/queries/events
+
+    Note: This stream extends IncrementalShopifyStream (REST base class) rather than IncrementalShopifyGraphQlBulkStream
+    because it uses Shopify's standard GraphQL Events API, NOT the Bulk Operations API (bulkOperationRunQuery).
+    The Events API has a fundamentally different architecture:
+    - Uses immediate queries with cursor pagination (not async job creation + file download)
+    - Returns results in response.data.events.nodes (not JSONL files)
+    - Requires different URL endpoint (/graphql.json), request format (POST with query in body), and pagination logic
+
+    Therefore, url_base, path, request_params, next_page_token, and parse_response are overridden to accommodate
+    the GraphQL request/response structure while maintaining incremental sync capabilities from the base class.
+    """
+
     data_field = "graphql"
+    cursor_field = "deleted_at"
     http_method = "POST"
-    # pin the old api_version before this stream is deprecated
-    api_version = "2023-07"
+    filter_field = None
+
+    _page_cursor: Optional[str] = None
+
+    EVENTS_QUERY = """
+    query GetDeletedProductEvents($first: Int!, $after: String, $query: String) {
+        events(first: $first, after: $after, query: $query, sortKey: CREATED_AT) {
+            pageInfo {
+                hasNextPage
+                endCursor
+            }
+            nodes {
+                ... on BasicEvent {
+                    id
+                    createdAt
+                    message
+                    subjectId
+                    subjectType
+                }
+            }
+        }
+    }
+    """
+
+    @property
+    def url_base(self) -> str:
+        return f"https://{self.config['shop']}.myshopify.com/admin/api/{self.api_version}/graphql.json"
+
+    def path(self, **kwargs) -> str:
+        return ""
 
     def request_params(
-        self,
-        stream_state: Optional[Mapping[str, Any]] = None,
-        next_page_token: Optional[Mapping[str, Any]] = None,
-        **kwargs,
+        self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs
     ) -> MutableMapping[str, Any]:
         return {}
 
     def request_body_json(
         self,
-        stream_state: Mapping[str, Any],
+        stream_state: Optional[Mapping[str, Any]] = None,
         stream_slice: Optional[Mapping[str, Any]] = None,
         next_page_token: Optional[Mapping[str, Any]] = None,
-    ) -> Optional[Mapping]:
-        state_value = stream_state.get(self.filter_field)
-        if state_value:
-            filter_value = state_value
-        else:
-            filter_value = self.default_filter_field_value
-        query = get_query_products(
-            first=self.limit, filter_field=self.filter_field, filter_value=filter_value, next_page_token=next_page_token
-        )
-        return {"query": query}
-
-    @staticmethod
-    def next_page_token(response: requests.Response) -> Optional[Mapping[str, Any]]:
-        page_info = response.json()["data"]["products"]["pageInfo"]
-        has_next_page = page_info["hasNextPage"]
-        if has_next_page:
-            return page_info["endCursor"]
-        else:
-            return None
+    ) -> Optional[Mapping[str, Any]]:
+        query_filter = "action:destroy AND subject_type:Product"
+        if stream_state and stream_state.get(self.cursor_field):
+            state_value = stream_state[self.cursor_field]
+            query_filter += f" AND created_at:>'{state_value}'"
+
+        variables = {
+            "first": 250,
+            "query": query_filter,
+        }
+
+        if next_page_token and next_page_token.get("cursor"):
+            variables["after"] = next_page_token["cursor"]
+
+        return {
+            "query": self.EVENTS_QUERY,
+            "variables": variables,
+        }
+
+    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+        json_response = response.json()
+        page_info = json_response.get("data", {}).get("events", {}).get("pageInfo", {})
+        if page_info.get("hasNextPage"):
+            return {"cursor": page_info.get("endCursor")}
+        return None
 
-    @limiter.balance_rate_limit(api_type=ApiTypeEnum.graphql.value)
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        if response.status_code is requests.codes.OK:
-            try:
-                json_response = response.json()["data"]["products"]["nodes"]
-                yield from self.produce_records(json_response)
-            except RequestException as e:
-                self.logger.warning(f"Unexpected error in `parse_ersponse`: {e}, the actual response data: {response.text}")
+        json_response = response.json()
+        events = json_response.get("data", {}).get("events", {}).get("nodes", [])
+        for event in events:
+            if event.get("subjectType") == "PRODUCT" and event.get("subjectId"):
+                subject_id = event.get("subjectId", "")
+                product_id = int(subject_id.split("/")[-1]) if "/" in subject_id else None
+                if product_id:
+                    yield {
+                        "id": product_id,
+                        "deleted_at": event.get("createdAt"),
+                        "deleted_message": event.get("message"),
+                        "deleted_description": None,
+                        "shop_url": self.config.get("shop"),
+                    }
 
 
 class MetafieldProducts(IncrementalShopifyGraphQlBulkStream):
+    parent_stream_class = Products
     bulk_query: MetafieldProduct = MetafieldProduct
 
+    state_checkpoint_interval = sys.maxsize
+
+    @property
+    def filter_by_state_checkpoint(self) -> bool:
+        return True
+
 
 class ProductImages(IncrementalShopifyGraphQlBulkStream):
+    parent_stream_class = Products
     bulk_query: ProductImage = ProductImage
 
 
 class MetafieldProductImages(IncrementalShopifyGraphQlBulkStream):
+    parent_stream_class = Products
     bulk_query: MetafieldProductImage = MetafieldProductImage
 
 
@@ -246,7 +324,6 @@ class MetafieldCollections(IncrementalShopifyGraphQlBulkStream):
 
 
 class BalanceTransactions(IncrementalShopifyStream):
-
     """
     PaymentsTransactions stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported:
     https://shopify.dev/api/admin-rest/2021-07/resources/transactions
@@ -274,7 +351,6 @@ class OrderRefunds(IncrementalShopifyNestedStream):
 
 class OrderRisks(IncrementalShopifyGraphQlBulkStream):
     bulk_query: OrderRisk = OrderRisk
-    # the updated stream works only with >= `2024-04` shopify api version
 
 
 class Transactions(IncrementalShopifySubstream):
@@ -375,19 +451,96 @@ class MetafieldShops(IncrementalShopifyStream):
     data_field = "metafields"
 
 
-class CustomerSavedSearch(IncrementalShopifyStream):
-    api_version = "2022-01"
-    cursor_field = "id"
-    order_field = "id"
-    data_field = "customer_saved_searches"
-    filter_field = "since_id"
-
-
 class CustomerAddress(IncrementalShopifyGraphQlBulkStream):
     parent_stream_class = Customers
     bulk_query: CustomerAddresses = CustomerAddresses
     cursor_field = "id"
 
 
-class Countries(ShopifyStream):
-    data_field = "countries"
+class ProfileLocationGroups(IncrementalShopifyGraphQlBulkStream):
+    bulk_query: ProfileLocationGroups = ProfileLocationGroups
+    filter_field = None
+
+
+class Countries(HttpSubStream, FullRefreshShopifyGraphQlBulkStream):
+    # https://shopify.dev/docs/api/admin-graphql/latest/queries/deliveryProfiles
+    _page_cursor = None
+    _sub_page_cursor = None
+
+    _synced_countries_ids = []
+
+    query = DeliveryProfile
+    response_field = "deliveryProfiles"
+
+    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+        json_response = response.json().get("data", {})
+        if not json_response:
+            return None
+
+        page_info = json_response.get("deliveryProfiles", {}).get("pageInfo", {})
+
+        sub_page_info = {"hasNextPage": False}
+        # only one top page in query
+        delivery_profiles_nodes = json_response.get("deliveryProfiles", {}).get("nodes")
+        if delivery_profiles_nodes:
+            profile_location_groups = delivery_profiles_nodes[0].get("profileLocationGroups")
+            if profile_location_groups:
+                sub_page_info = (
+                    # only first one
+                    profile_location_groups[0].get("locationGroupZones", {}).get("pageInfo", {})
+                )
+
+        if not sub_page_info["hasNextPage"] and not page_info["hasNextPage"]:
+            return None
+        if sub_page_info["hasNextPage"]:
+            # The cursor to retrieve nodes after in the connection. Typically, you should pass the endCursor of the previous page as after.
+            self._sub_page_cursor = sub_page_info["endCursor"]
+        if page_info["hasNextPage"] and not sub_page_info["hasNextPage"]:
+            # The cursor to retrieve nodes after in the connection. Typically, you should pass the endCursor of the previous page as after.
+            self._page_cursor = page_info["endCursor"]
+            self._sub_page_cursor = None
+
+        return {
+            "cursor": self._page_cursor,
+            "sub_cursor": self._sub_page_cursor,
+        }
+
+    def request_body_json(
+        self,
+        stream_state: Optional[Mapping[str, Any]],
+        stream_slice: Optional[Mapping[str, Any]] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Optional[Mapping[str, Any]]:
+        location_group_id = stream_slice["parent"]["profile_location_groups"][0]["locationGroup"]["id"]
+        return {
+            "query": self.query(location_group_id=location_group_id, location_group_zones_cursor=self._sub_page_cursor).get(
+                query_args={
+                    "cursor": self._page_cursor,
+                }
+            ),
+        }
+
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+        # TODO: to refactor this using functools + partial, see comment https://github.com/airbytehq/airbyte/pull/55823#discussion_r2016335672
+        for node in super().parse_response(response, **kwargs):
+            for location_group in node.get("profileLocationGroups", []):
+                for location_group_zone in location_group.get("locationGroupZones", {}).get("nodes", []):
+                    for country in location_group_zone.get("zone", {}).get("countries"):
+                        country = self._process_country(country)
+                        if country["id"] not in self._synced_countries_ids:
+                            self._synced_countries_ids.append(country["id"])
+                            yield country
+
+    def _process_country(self, country: Mapping[str, Any]) -> Mapping[str, Any]:
+        country["id"] = int(country["id"].split("/")[-1])
+
+        for province in country.get("provinces", []):
+            province["id"] = int(province["id"].split("/")[-1])
+            province["country_id"] = country["id"]
+
+        if country.get("code"):
+            country["rest_of_world"] = country["code"]["rest_of_world"] if country["code"].get("rest_of_world") is not None else "*"
+            country["code"] = country["code"]["country_code"] if country["code"].get("country_code") is not None else "*"
+
+        country["shop_url"] = self.config["shop"]
+        return country

source_shopify/utils.py

@@ -7,12 +7,19 @@ import enum
 import logging
 from functools import wraps
 from time import sleep
-from typing import Any, Callable, Dict, List, Mapping, Optional
+from typing import Any, Callable, Dict, Final, List, Mapping, Optional, Union
+from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
 import requests
+
+from airbyte_cdk.models import FailureType
+from airbyte_cdk.sources.streams.http.error_handlers import HttpStatusErrorHandler
 from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction
 from airbyte_cdk.utils import AirbyteTracedException
-from airbyte_protocol.models import FailureType
+
+
+# default logger instance
+LOGGER: Final[logging.Logger] = logging.getLogger("airbyte")
 
 
 class ShopifyNonRetryableErrors:
@@ -40,11 +47,6 @@ class ShopifyNonRetryableErrors:
                 failure_type=FailureType.config_error,
                 error_message=f"Stream `{stream}`. Not available or missing.",
             ),
-            500: ErrorResolution(
-                response_action=ResponseAction.IGNORE,
-                failure_type=FailureType.config_error,
-                error_message=f"Stream `{stream}`. Entity might not be available or missing.",
-            )
             # extend the mapping with more handable errors, if needed.
         }
 
@@ -112,8 +114,6 @@ class ShopifyRateLimiter:
     on_mid_load: float = 1.5
     on_high_load: float = 5.0
 
-    logger = logging.getLogger("airbyte")
-
     log_message_count = 0
     log_message_frequency = 3
 
@@ -124,7 +124,7 @@ class ShopifyRateLimiter:
         if ShopifyRateLimiter.log_message_count < ShopifyRateLimiter.log_message_frequency:
             ShopifyRateLimiter.log_message_count += 1
         else:
-            ShopifyRateLimiter.logger.info(message)
+            LOGGER.info(message)
             ShopifyRateLimiter.log_message_count = 0
 
     def get_response_from_args(*args) -> Optional[requests.Response]:
@@ -138,8 +138,8 @@ class ShopifyRateLimiter:
         Define wait_time based on load conditions.
 
         :: load - represents how close we are to being throttled
-                - 0.5 is half way through our allowance
-                - 1 indicates that all of the allowance is used and the api will start rejecting calls
+            - 0.5 is half way through our allowance
+            - 1 indicates that all of the allowance is used and the api will start rejecting calls
         :: threshold - is the % cutoff for the rate_limits/load
         :: wait_time - time to wait between each request in seconds
 
@@ -325,3 +325,38 @@ class EagerlyCachedStreamState:
             return func(*args, **kwargs)
 
         return decorator
+
+
+class LimitReducingErrorHandler(HttpStatusErrorHandler):
+    """
+    Error handler that halves the page size (limit) on each 500 error, down to 1.
+    No stream instance required; operates directly on the request URL.
+    """
+
+    def __init__(self, max_retries: int, error_mapping: dict):
+        super().__init__(logger=None, max_retries=max_retries, error_mapping=error_mapping)
+
+    def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
+        if isinstance(response_or_exception, requests.Response):
+            response = response_or_exception
+            if response.status_code == 500:
+                # Extract current limit from the URL, default to 250 if not present
+                parsed = urlparse(response.request.url)
+                query = parse_qs(parsed.query)
+                current_limit = int(query.get("limit", ["250"])[0])
+                if current_limit > 1:
+                    new_limit = max(1, current_limit // 2)
+                    query["limit"] = [str(new_limit)]
+                    new_query = urlencode(query, doseq=True)
+                    response.request.url = urlunparse(parsed._replace(query=new_query))
+                    return ErrorResolution(
+                        response_action=ResponseAction.RETRY,
+                        failure_type=FailureType.transient_error,
+                        error_message=f"Server error 500: Reduced limit to {new_limit} and updating request URL",
+                    )
+                return ErrorResolution(
+                    response_action=ResponseAction.FAIL,
+                    failure_type=FailureType.transient_error,
+                    error_message="Persistent 500 error after reducing limit to 1",
+                )
+        return super().interpret_response(response_or_exception)

source_shopify/schemas/customer_saved_search.json

@@ -1,32 +0,0 @@
-{
-  "type": ["null", "object"],
-  "additionalProperties": true,
-  "properties": {
-    "created_at": {
-      "description": "The date and time when the customer saved search was created.",
-      "type": ["null", "string"],
-      "format": "date-time"
-    },
-    "id": {
-      "description": "The unique identifier for the customer saved search.",
-      "type": ["null", "integer"]
-    },
-    "name": {
-      "description": "The name given to the customer saved search.",
-      "type": ["null", "string"]
-    },
-    "query": {
-      "description": "The search query string or parameters used for this saved search.",
-      "type": ["null", "string"]
-    },
-    "shop_url": {
-      "description": "The URL of the shop associated with this customer saved search.",
-      "type": ["null", "string"]
-    },
-    "updated_at": {
-      "description": "The date and time when the customer saved search was last updated.",
-      "type": ["null", "string"],
-      "format": "date-time"
-    }
-  }
-}

source_shopify/schemas/products_graph_ql.json

@@ -1,123 +0,0 @@
-{
-  "$schema": "http://json-schema.org/schema#",
-  "properties": {
-    "createdAt": {
-      "description": "The date and time when the product was created.",
-      "type": "string"
-    },
-    "description": {
-      "description": "The product's description.",
-      "type": "string"
-    },
-    "descriptionHtml": {
-      "description": "The product's description in HTML format.",
-      "type": "string"
-    },
-    "handle": {
-      "description": "The unique URL-friendly handle of the product.",
-      "type": "string"
-    },
-    "id": {
-      "description": "The unique identifier of the product.",
-      "type": "string"
-    },
-    "isGiftCard": {
-      "description": "Indicates whether the product is a gift card.",
-      "type": "boolean"
-    },
-    "legacyResourceId": {
-      "description": "The legacy resource ID of the product.",
-      "type": "string"
-    },
-    "mediaCount": {
-      "description": "The total count of media (images/videos) associated with the product.",
-      "type": "integer"
-    },
-    "onlineStorePreviewUrl": {
-      "description": "The URL for previewing the product on the online store.",
-      "type": "string"
-    },
-    "onlineStoreUrl": {
-      "description": "The URL of the product on the online store.",
-      "type": ["null", "string"]
-    },
-    "options": {
-      "description": "Represents various options available for the product",
-      "items": {
-        "properties": {
-          "id": {
-            "description": "The unique identifier of the option.",
-            "type": "string"
-          },
-          "name": {
-            "description": "The name of the option.",
-            "type": "string"
-          },
-          "position": {
-            "description": "The position of the option.",
-            "type": "integer"
-          },
-          "values": {
-            "description": "Contains the different values for the options",
-            "items": {
-              "description": "The possible values for the option.",
-              "type": "string"
-            },
-            "type": "array"
-          }
-        },
-        "type": "object"
-      },
-      "type": "array"
-    },
-    "productType": {
-      "description": "The type or category of the product.",
-      "type": "string"
-    },
-    "publishedAt": {
-      "description": "The date and time when the product was published.",
-      "type": ["null", "string"]
-    },
-    "shop_url": {
-      "description": "The URL of the shop where the product is listed.",
-      "type": "string"
-    },
-    "status": {
-      "description": "The status of the product.",
-      "type": "string"
-    },
-    "tags": {
-      "description": "Contains tags associated with the product",
-      "items": {
-        "description": "The tags associated with the product.",
-        "type": "string"
-      },
-      "type": "array"
-    },
-    "title": {
-      "description": "The title or name of the product.",
-      "type": "string"
-    },
-    "totalInventory": {
-      "description": "The total inventory count of the product.",
-      "type": "integer"
-    },
-    "totalVariants": {
-      "description": "The total number of variants available for the product.",
-      "type": "integer"
-    },
-    "tracksInventory": {
-      "description": "Indicates whether inventory tracking is enabled for the product.",
-      "type": "boolean"
-    },
-    "updatedAt": {
-      "description": "The date and time when the product was last updated.",
-      "type": "string"
-    },
-    "vendor": {
-      "description": "The vendor or manufacturer of the product.",
-      "type": "string"
-    }
-  },
-  "type": "object"
-}