airbyte-source-shopify 3.1.2.dev202601082220__py3-none-any.whl → 3.2.0.dev202602040050__py3-none-any.whl

{airbyte_source_shopify-3.1.2.dev202601082220.dist-info → airbyte_source_shopify-3.2.0.dev202602040050.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 3.1.2.dev202601082220
+Version: 3.2.0.dev202602040050
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_shopify-3.1.2.dev202601082220.dist-info → airbyte_source_shopify-3.2.0.dev202602040050.dist-info}/RECORD

@@ -7,6 +7,7 @@ source_shopify/schemas/abandoned_checkouts.json,sha256=nlYpb8VOQCO8wuHWM8gWhlzzj
 source_shopify/schemas/articles.json,sha256=tGnbM4lI2jA8joevDsrd-mQJksO2lLG3dIJGpxeuoYs,2548
 source_shopify/schemas/balance_transactions.json,sha256=RAU7duUHTWS7nI0pochhTZt575ksPwc51Qmcv4XL39o,2135
 source_shopify/schemas/blogs.json,sha256=ciBS_3eCf4UJUaB0DPCVadeJR4W6ndq7N0JwykXp0RY,2151
+source_shopify/schemas/collection_products.json,sha256=ycOpp5egXg3nEWpGy8IWFW6ekDXP4fgqdk_trKjvo7Q,1102
 source_shopify/schemas/collections.json,sha256=2iJMCyAn_yeMKsQVt7jGR3_u3N3CA8QQ6179QvRuwqY,1889
 source_shopify/schemas/collects.json,sha256=dOX0_O7meWELWHYQG_MWqGkWLelAoiIlPtDXuxz9ig8,1173
 source_shopify/schemas/countries.json,sha256=fdJPrd8tQEzzonkunm0hvMbqZeXixPaphHd3PSt8g58,1783
@@ -43,29 +44,29 @@ source_shopify/schemas/orders.json,sha256=SxX6fDtMnEkvm-58uNgaoF47Ep8DIwLpXK09C2
 source_shopify/schemas/pages.json,sha256=J7jxtk2nE_q-0oHmGz8orFq3EfSjJqUvF6qPJtiZCrg,2093
 source_shopify/schemas/price_rules.json,sha256=aZ-Re9Oc9g0ci8SPn0_ufvKPsGDBcTaXHVsq_9-Ygq4,7004
 source_shopify/schemas/product_images.json,sha256=0l3ACfOfnXz8N8oMHwvvfZUFKeeuZ2DlRRu5-FxUhaI,1760
-source_shopify/schemas/product_variants.json,sha256=hBtmB-OyVlmkQacY0s7Mpe-bMdYjx8mpwMqcGwN05kA,8121
+source_shopify/schemas/product_variants.json,sha256=TipJMwLLBi9ULVxCEya14W-de7Fr4qfLAfCPETTtdeM,8412
 source_shopify/schemas/products.json,sha256=n7m1ndEnAM7Pee5ca0JZWCl5Y0VffGmTZjqjHhiht1I,22609
 source_shopify/schemas/profile_location_groups.json,sha256=2_QdfMeM_WNDT-06fQfFUHXjN9D6ndKIGcviJxEl8kM,189
 source_shopify/schemas/shop.json,sha256=vEGiTvEYX7qnMq06MRVBycqih49h49xjTNC6gJuxTWs,8137
 source_shopify/schemas/smart_collections.json,sha256=kv7dINsvgzJ0RyKfFNKjU0apdNDXwQaHfnNZfQsshcU,2009
 source_shopify/schemas/tender_transactions.json,sha256=U8fdT-eflycEPzYSpBDiB0lp9wxmJHgioHTrICflh78,2006
 source_shopify/schemas/transactions.json,sha256=vbwscH3UcAtbSsC70mBka4oNaFR4S3S6IFBmzR7t37U,10226
-source_shopify/scopes.py,sha256=78f9QL3PJZ9UDx1gIWzNwx5fYJE9OB3vPi9RahB_kFw,6533
+source_shopify/scopes.py,sha256=R6f7t211n_nrh0T897u72ODo_hI6mzPC1NbN0vq-upI,6579
 source_shopify/shopify_graphql/bulk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 source_shopify/shopify_graphql/bulk/exceptions.py,sha256=4dj7Za4xIfwL-zf8joT9svF_RSoGlE3GviMiIl1e1rs,2532
 source_shopify/shopify_graphql/bulk/job.py,sha256=c3Cg70_Io9jTD-rU-5MvjHaPmJCtcpeqEYnRtFECGOo,28673
-source_shopify/shopify_graphql/bulk/query.py,sha256=D8rnI1SDw50-Gt18lt7YwwNNdsbVMbBfxZa9xVJZbto,130981
+source_shopify/shopify_graphql/bulk/query.py,sha256=A8-VJGxzJnkkVZm62TAw6OZlWcIs9VHzhO-rMaVHc0E,136522
 source_shopify/shopify_graphql/bulk/record.py,sha256=X6VGngugv7a_S8UEeDo121BkdCVLj5nWlHK76A21kyo,16898
 source_shopify/shopify_graphql/bulk/retry.py,sha256=R5rSJJE8D5zcj6mN-OmmNO2aFZEIdjAlWclDDVW5KPI,2626
 source_shopify/shopify_graphql/bulk/status.py,sha256=RmuQ2XsYL3iRCpVGxea9F1wXGmbwasDCSXjaTyL4LMA,328
 source_shopify/shopify_graphql/bulk/tools.py,sha256=nUQ2ZmPTKJNJdfLToR6KJtLKcJFCChSifkAOvwg0Vss,4065
-source_shopify/source.py,sha256=oikoM-VPNk62zlmeAQR59PMxfuXq2s42N7zaqLM6_lo,8575
-source_shopify/spec.json,sha256=vwEY5T3IryqSne0cRcJO53FaLnApuKOKRRS6yQJABpo,6667
-source_shopify/streams/base_streams.py,sha256=xYuyH6YjxZYl2x8CsdIYl3AqxBtz5xF_Oioatn5V-Rs,43979
-source_shopify/streams/streams.py,sha256=Ro0JXboCQHpfdI5VsLR4TseSL077GiFMZ-gOOnCCqtc,19356
+source_shopify/source.py,sha256=_eDPHsRDlpwPWdUzG1FPEyf6uDAHRm8Vt2n6VDfyAhE,8879
+source_shopify/spec.json,sha256=ITYWiQ-NrI5VISk5qmUQhp9ChUE2FV18d8xzVzPwvAg,6144
+source_shopify/streams/base_streams.py,sha256=k_4uLaLADLRTUcSmP8uA_830uuzRvnqUaCVGcb0Zpd8,42625
+source_shopify/streams/streams.py,sha256=I68Li0EPR9acj3Jhk7qd-zWhESkPHRzLA_b-LmbTjsE,20044
 source_shopify/transform.py,sha256=mn0htL812_90zc_YszGQa0hHcIZQpYYdmk8IqpZm5TI,4685
 source_shopify/utils.py,sha256=DSqEchu-MQJ7zust7CNfqOkGIv9OSR-5UUsuD-bsDa8,16224
-airbyte_source_shopify-3.1.2.dev202601082220.dist-info/METADATA,sha256=WtXZxV5MKHMrSSdUoChJ0kB6TrDYmET6oChVWm0LrIo,5313
-airbyte_source_shopify-3.1.2.dev202601082220.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_source_shopify-3.1.2.dev202601082220.dist-info/entry_points.txt,sha256=SyTwKSsPk9MCdPf01saWpnp8hcmZOgBssVcSIvMbBeQ,57
-airbyte_source_shopify-3.1.2.dev202601082220.dist-info/RECORD,,
+airbyte_source_shopify-3.2.0.dev202602040050.dist-info/METADATA,sha256=eo4mrM9Mova_yKHxI5QkOs2O4bZuqjoMt49z-vbkNFI,5313
+airbyte_source_shopify-3.2.0.dev202602040050.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_source_shopify-3.2.0.dev202602040050.dist-info/entry_points.txt,sha256=SyTwKSsPk9MCdPf01saWpnp8hcmZOgBssVcSIvMbBeQ,57
+airbyte_source_shopify-3.2.0.dev202602040050.dist-info/RECORD,,

source_shopify/schemas/collection_products.json

@@ -0,0 +1,35 @@
+{
+  "type": "object",
+  "additionalProperties": true,
+  "properties": {
+    "collection_id": {
+      "description": "The unique identifier for the collection.",
+      "type": ["null", "integer"]
+    },
+    "collection_admin_graphql_api_id": {
+      "description": "The Admin GraphQL API ID for the collection.",
+      "type": ["null", "string"]
+    },
+    "collection_handle": {
+      "description": "The handle (URL-friendly name) for the collection.",
+      "type": ["null", "string"]
+    },
+    "collection_updated_at": {
+      "description": "The date and time when the collection was last updated.",
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "product_id": {
+      "description": "The unique identifier for the product.",
+      "type": ["null", "integer"]
+    },
+    "product_admin_graphql_api_id": {
+      "description": "The Admin GraphQL API ID for the product.",
+      "type": ["null", "string"]
+    },
+    "shop_url": {
+      "description": "The URL of the shop associated with this collection-product association.",
+      "type": ["null", "string"]
+    }
+  }
+}

source_shopify/schemas/product_variants.json

@@ -52,12 +52,20 @@
       "items": {
         "type": ["null", "object"],
         "properties": {
+          "id": {
+            "description": "The unique identifier for the product option.",
+            "type": ["null", "integer"]
+          },
           "name": {
-            "description": "The product option’s name.",
+            "description": "The product option's name.",
             "type": ["null", "string"]
           },
+          "position": {
+            "description": "The display order of the product option.",
+            "type": ["null", "integer"]
+          },
           "value": {
-            "description": "The product option’s value.",
+            "description": "The product option's value.",
             "type": ["null", "string"]
           },
           "option_value": {

source_shopify/scopes.py

@@ -44,6 +44,7 @@ SCOPES_MAPPING: Mapping[str, set[str]] = {
     "MetafieldProductVariants": ("read_products",),
     "CustomCollections": ("read_products",),
     "Collects": ("read_products",),
+    "CollectionProducts": ("read_products",),
     "ProductVariants": ("read_products", "read_inventory"),
     "MetafieldCollections": ("read_products",),
     "SmartCollections": ("read_products",),

source_shopify/shopify_graphql/bulk/query.py

@@ -952,6 +952,114 @@ class Collection(ShopifyBulkQuery):
         yield record
 
 
+class CollectionProduct(ShopifyBulkQuery):
+    """
+    Returns the products associated with each collection, including both custom collections
+    and smart collections. This provides all product<>collection associations, not just
+    manually associated products (which is what the Collects REST API provides).
+
+    {
+        collections(query: "updated_at:>='2023-02-07T00:00:00+00:00' AND updated_at:<='2023-12-04T00:00:00+00:00'", sortKey: UPDATED_AT) {
+            edges {
+                node {
+                    __typename
+                    id
+                    handle
+                    updatedAt
+                    products {
+                        edges {
+                            node {
+                                __typename
+                                id
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    """
+
+    query_name = "collections"
+    sort_key = "UPDATED_AT"
+
+    products_fields: List[Field] = [
+        Field(
+            name="edges",
+            fields=[
+                Field(
+                    name="node",
+                    fields=[
+                        "__typename",
+                        "id",
+                    ],
+                )
+            ],
+        )
+    ]
+
+    query_nodes: List[Field] = [
+        "__typename",
+        "id",
+        Field(name="handle"),
+        Field(name="updatedAt"),
+        Field(name="products", fields=products_fields),
+    ]
+
+    record_composition = {
+        "new_record": "Collection",
+        "record_components": ["Product"],
+    }
+
+    def _process_product_components(self, products: List[dict]) -> List[dict]:
+        """
+        Process product components to resolve IDs from string to int and preserve the original ID.
+
+        Args:
+            products: List of product dictionaries with string IDs
+
+        Returns:
+            List of processed product dictionaries with both id (int) and admin_graphql_api_id (str)
+        """
+        for product in products:
+            # Save the original string ID before resolving
+            product["admin_graphql_api_id"] = product.get("id")
+            # Resolve the ID from string to int
+            product["id"] = self.tools.resolve_str_id(product.get("id"))
+        return products
+
+    def record_process_components(self, record: MutableMapping[str, Any]) -> Iterable[MutableMapping[str, Any]]:
+        """
+        Process collection records and yield one record per collection-product association.
+        """
+        record_components = record.get("record_components", {})
+        products = record_components.get("Product", [])
+
+        # Get collection info - id is already resolved to int, admin_graphql_api_id has the string version
+        collection_id = record.get("id")
+        collection_admin_graphql_api_id = record.get("admin_graphql_api_id")
+        collection_handle = record.get("handle")
+        collection_updated_at = self.tools.from_iso8601_to_rfc3339(record, "updatedAt")
+
+        if products:
+            # Process products to resolve their IDs
+            products = self._process_product_components(products)
+
+            for product in products:
+                product_id = product.get("id")
+                product_admin_graphql_api_id = product.get("admin_graphql_api_id")
+
+                yield {
+                    "collection_id": collection_id,
+                    "collection_admin_graphql_api_id": collection_admin_graphql_api_id,
+                    "collection_handle": collection_handle,
+                    "collection_updated_at": collection_updated_at,
+                    "product_id": product_id,
+                    "product_admin_graphql_api_id": product_admin_graphql_api_id,
+                    "shop_url": self.config.get("shop"),
+                }
+
+
 class CustomerAddresses(ShopifyBulkQuery):
     """
     {
@@ -2683,7 +2791,13 @@ class ProductVariant(ShopifyBulkQuery):
             Field(name="selectedOptions", alias="options", fields=option_fields),
             Field(name="image", fields=image_fields),
             Field(name="inventoryQuantity", alias="old_inventory_quantity"),
-            Field(name="product", fields=[Field(name="id", alias="product_id")]),
+            Field(
+                name="product",
+                fields=[
+                    Field(name="id", alias="product_id"),
+                    Field(name="options", alias="product_options", fields=["id", "name", "position"]),
+                ],
+            ),
             Field(name="inventoryItem", fields=inventory_item_fields),
         ] + presentment_prices
 
@@ -2730,6 +2844,34 @@ class ProductVariant(ShopifyBulkQuery):
         entity = record.get(from_property, {})
         return self.tools.resolve_str_id(entity.get(id_field)) if entity else None
 
+    def _enrich_options_with_product_options(self, record: MutableMapping[str, Any]) -> None:
+        """
+        Enriches the variant's options with id and position from the product's options.
+        Matches options by name and adds the corresponding ProductOption id and position.
+        """
+        options = record.get("options") or []
+        product = record.get("product") or {}
+        product_options = product.get("product_options") or []
+
+        # Build a lookup map from option name to ProductOption data
+        product_options_map = {}
+        for product_option in product_options:
+            if product_option:
+                name = product_option.get("name")
+                if name:
+                    product_options_map[name] = {
+                        "id": self.tools.resolve_str_id(product_option.get("id")),
+                        "position": product_option.get("position"),
+                    }
+
+        # Enrich each option with id and position from the matching ProductOption
+        for option in options:
+            if option:
+                option_name = option.get("name")
+                if option_name and option_name in product_options_map:
+                    option["id"] = product_options_map[option_name]["id"]
+                    option["position"] = product_options_map[option_name]["position"]
+
     def record_process_components(self, record: MutableMapping[str, Any]) -> Iterable[MutableMapping[str, Any]]:
         """
         Defines how to process collected components.
@@ -2742,6 +2884,9 @@ class ProductVariant(ShopifyBulkQuery):
             record["presentment_prices"] = self._process_presentment_prices(record_components.get("ProductVariantPricePair", []))
             record.pop("record_components")
 
+        # enrich options with id and position from product options (must be done before product is removed)
+        self._enrich_options_with_product_options(record)
+
         # unnest mandatory fields from their placeholders
         record["product_id"] = self._unnest_and_resolve_id(record, "product", "product_id")
         record["inventory_item_id"] = self._unnest_and_resolve_id(record, "inventoryItem", "inventory_item_id")

source_shopify/source.py

@@ -11,6 +11,7 @@ from requests.exceptions import ConnectionError, RequestException, SSLError
 from airbyte_cdk.models import FailureType, SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
+from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException
 from airbyte_cdk.utils import AirbyteTracedException
 
 from .auth import MissingAccessTokenError, ShopifyAuthenticator
@@ -20,6 +21,7 @@ from .streams.streams import (
     Articles,
     BalanceTransactions,
     Blogs,
+    CollectionProducts,
     Collections,
     Collects,
     Countries,
@@ -108,6 +110,8 @@ class ConnectionCheckTest:
             return False, self.describe_error("index_error", shop_name, response)
         except MissingAccessTokenError:
             return False, self.describe_error("missing_token_error")
+        except (BaseBackoffException, AirbyteTracedException) as error:
+            return False, self.describe_error("connection_error", shop_name) or str(error)
 
     def get_shop_id(self) -> str:
         """
@@ -178,6 +182,7 @@ class SourceShopify(AbstractSource):
             Articles(config),
             BalanceTransactions(config),
             Blogs(config),
+            CollectionProducts(config),
             Collections(config),
             Collects(config),
             CustomCollections(config),

source_shopify/spec.json

@@ -119,15 +119,6 @@
         "default": 100000,
         "minimum": 15000,
         "maximum": 1000000
-      },
-      "lookback_window_in_days": {
-        "type": "integer",
-        "title": "Lookback Window (in days)",
-        "description": "The number of days to look back for records that may have been missed due to race conditions or late-arriving data. This is useful for ensuring data completeness in incremental syncs. Setting this to 1 or more will re-fetch records from the specified number of days before the last sync state.",
-        "default": 0,
-        "minimum": 0,
-        "maximum": 30,
-        "order": 7
       }
     }
   },

source_shopify/streams/base_streams.py

@@ -209,24 +209,6 @@ class IncrementalShopifyStream(ShopifyStream, ABC):
         current_state_value = current_stream_state.get(self.cursor_field) or self.default_state_comparison_value
         return {self.cursor_field: max(last_record_value, current_state_value)}
 
-    def _apply_lookback_window(self, state_value: str) -> str:
-        """
-        Apply the lookback window to the state value by subtracting the configured number of days.
-        This helps capture records that may have been missed due to race conditions or late-arriving data.
-        """
-        lookback_days = self.config.get("lookback_window_in_days", 0)
-        if lookback_days > 0:
-            state_datetime = pdm.parse(state_value)
-            adjusted_datetime = state_datetime.subtract(days=lookback_days)
-            # Ensure we don't go before the configured start_date
-            start_date = self.config.get("start_date")
-            if start_date:
-                start_datetime = pdm.parse(start_date)
-                if adjusted_datetime < start_datetime:
-                    adjusted_datetime = start_datetime
-            return adjusted_datetime.to_rfc3339_string()
-        return state_value
-
     @stream_state_cache.cache_stream_state
     def request_params(
         self, stream_state: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs
@@ -236,11 +218,7 @@ class IncrementalShopifyStream(ShopifyStream, ABC):
         if not next_page_token:
             params["order"] = f"{self.order_field} asc"
             if stream_state:
-                state_value = stream_state.get(self.cursor_field)
-                # Apply lookback window to go back N days from the state
-                if state_value and self.filter_field != "since_id":
-                    state_value = self._apply_lookback_window(state_value)
-                params[self.filter_field] = state_value
+                params[self.filter_field] = stream_state.get(self.cursor_field)
         return params
 
     def track_checkpoint_cursor(self, record_value: Union[str, int], filter_record_value: Optional[str] = None) -> None:
@@ -844,9 +822,6 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream):
     def stream_slices(self, stream_state: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
         if self.filter_field:
             state = self._get_state_value(stream_state)
-            # Apply lookback window only when we have a stream state (not on first sync)
-            if stream_state:
-                state = self._apply_lookback_window(state)
             start = pdm.parse(state)
             end = pdm.now()
             while start < end:

source_shopify/streams/streams.py

@@ -10,6 +10,7 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional
 import requests
 from source_shopify.shopify_graphql.bulk.query import (
     Collection,
+    CollectionProduct,
     CustomerAddresses,
     CustomerJourney,
     DeliveryProfile,
@@ -323,6 +324,21 @@ class MetafieldCollections(IncrementalShopifyGraphQlBulkStream):
     bulk_query: MetafieldCollection = MetafieldCollection
 
 
+class CollectionProducts(IncrementalShopifyGraphQlBulkStream):
+    """
+    Stream that returns all products associated with each collection, including both
+    custom collections and smart collections. Unlike the Collects stream which only
+    returns manually associated products, this stream returns all products that belong
+    to a collection (including those matched by smart collection rules).
+
+    https://shopify.dev/docs/api/admin-graphql/latest/objects/Collection#field-Collection.fields.products
+    """
+
+    bulk_query: CollectionProduct = CollectionProduct
+    cursor_field = "collection_updated_at"
+    primary_key = ["collection_id", "product_id"]
+
+
 class BalanceTransactions(IncrementalShopifyStream):
     """
     PaymentsTransactions stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported: