airbyte-source-shopify 3.2.0.dev202602040050__tar.gz → 3.2.1.dev202601082335__tar.gz

{airbyte_source_shopify-3.2.0.dev202602040050 → airbyte_source_shopify-3.2.1.dev202601082335}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 3.2.0.dev202602040050
+Version: 3.2.1.dev202601082335
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_shopify-3.2.0.dev202602040050 → airbyte_source_shopify-3.2.1.dev202601082335}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "3.2.0.dev.202602040050"
+version = "3.2.1.dev.202601082335"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [ "Airbyte <contact@airbyte.io>",]

{airbyte_source_shopify-3.2.0.dev202602040050 → airbyte_source_shopify-3.2.1.dev202601082335}/source_shopify/schemas/product_variants.json

@@ -52,20 +52,12 @@
       "items": {
         "type": ["null", "object"],
         "properties": {
-          "id": {
-            "description": "The unique identifier for the product option.",
-            "type": ["null", "integer"]
-          },
           "name": {
-            "description": "The product option's name.",
+            "description": "The product option’s name.",
             "type": ["null", "string"]
           },
-          "position": {
-            "description": "The display order of the product option.",
-            "type": ["null", "integer"]
-          },
           "value": {
-            "description": "The product option's value.",
+            "description": "The product option’s value.",
             "type": ["null", "string"]
           },
           "option_value": {

{airbyte_source_shopify-3.2.0.dev202602040050 → airbyte_source_shopify-3.2.1.dev202601082335}/source_shopify/scopes.py

@@ -44,7 +44,6 @@ SCOPES_MAPPING: Mapping[str, set[str]] = {
     "MetafieldProductVariants": ("read_products",),
     "CustomCollections": ("read_products",),
     "Collects": ("read_products",),
-    "CollectionProducts": ("read_products",),
     "ProductVariants": ("read_products", "read_inventory"),
     "MetafieldCollections": ("read_products",),
     "SmartCollections": ("read_products",),

{airbyte_source_shopify-3.2.0.dev202602040050 → airbyte_source_shopify-3.2.1.dev202601082335}/source_shopify/shopify_graphql/bulk/query.py

@@ -952,114 +952,6 @@ class Collection(ShopifyBulkQuery):
         yield record
 
 
-class CollectionProduct(ShopifyBulkQuery):
-    """
-    Returns the products associated with each collection, including both custom collections
-    and smart collections. This provides all product<>collection associations, not just
-    manually associated products (which is what the Collects REST API provides).
-
-    {
-        collections(query: "updated_at:>='2023-02-07T00:00:00+00:00' AND updated_at:<='2023-12-04T00:00:00+00:00'", sortKey: UPDATED_AT) {
-            edges {
-                node {
-                    __typename
-                    id
-                    handle
-                    updatedAt
-                    products {
-                        edges {
-                            node {
-                                __typename
-                                id
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-    """
-
-    query_name = "collections"
-    sort_key = "UPDATED_AT"
-
-    products_fields: List[Field] = [
-        Field(
-            name="edges",
-            fields=[
-                Field(
-                    name="node",
-                    fields=[
-                        "__typename",
-                        "id",
-                    ],
-                )
-            ],
-        )
-    ]
-
-    query_nodes: List[Field] = [
-        "__typename",
-        "id",
-        Field(name="handle"),
-        Field(name="updatedAt"),
-        Field(name="products", fields=products_fields),
-    ]
-
-    record_composition = {
-        "new_record": "Collection",
-        "record_components": ["Product"],
-    }
-
-    def _process_product_components(self, products: List[dict]) -> List[dict]:
-        """
-        Process product components to resolve IDs from string to int and preserve the original ID.
-
-        Args:
-            products: List of product dictionaries with string IDs
-
-        Returns:
-            List of processed product dictionaries with both id (int) and admin_graphql_api_id (str)
-        """
-        for product in products:
-            # Save the original string ID before resolving
-            product["admin_graphql_api_id"] = product.get("id")
-            # Resolve the ID from string to int
-            product["id"] = self.tools.resolve_str_id(product.get("id"))
-        return products
-
-    def record_process_components(self, record: MutableMapping[str, Any]) -> Iterable[MutableMapping[str, Any]]:
-        """
-        Process collection records and yield one record per collection-product association.
-        """
-        record_components = record.get("record_components", {})
-        products = record_components.get("Product", [])
-
-        # Get collection info - id is already resolved to int, admin_graphql_api_id has the string version
-        collection_id = record.get("id")
-        collection_admin_graphql_api_id = record.get("admin_graphql_api_id")
-        collection_handle = record.get("handle")
-        collection_updated_at = self.tools.from_iso8601_to_rfc3339(record, "updatedAt")
-
-        if products:
-            # Process products to resolve their IDs
-            products = self._process_product_components(products)
-
-            for product in products:
-                product_id = product.get("id")
-                product_admin_graphql_api_id = product.get("admin_graphql_api_id")
-
-                yield {
-                    "collection_id": collection_id,
-                    "collection_admin_graphql_api_id": collection_admin_graphql_api_id,
-                    "collection_handle": collection_handle,
-                    "collection_updated_at": collection_updated_at,
-                    "product_id": product_id,
-                    "product_admin_graphql_api_id": product_admin_graphql_api_id,
-                    "shop_url": self.config.get("shop"),
-                }
-
-
 class CustomerAddresses(ShopifyBulkQuery):
     """
     {
@@ -2791,13 +2683,7 @@ class ProductVariant(ShopifyBulkQuery):
             Field(name="selectedOptions", alias="options", fields=option_fields),
             Field(name="image", fields=image_fields),
             Field(name="inventoryQuantity", alias="old_inventory_quantity"),
-            Field(
-                name="product",
-                fields=[
-                    Field(name="id", alias="product_id"),
-                    Field(name="options", alias="product_options", fields=["id", "name", "position"]),
-                ],
-            ),
+            Field(name="product", fields=[Field(name="id", alias="product_id")]),
             Field(name="inventoryItem", fields=inventory_item_fields),
         ] + presentment_prices
 
@@ -2844,34 +2730,6 @@ class ProductVariant(ShopifyBulkQuery):
         entity = record.get(from_property, {})
         return self.tools.resolve_str_id(entity.get(id_field)) if entity else None
 
-    def _enrich_options_with_product_options(self, record: MutableMapping[str, Any]) -> None:
-        """
-        Enriches the variant's options with id and position from the product's options.
-        Matches options by name and adds the corresponding ProductOption id and position.
-        """
-        options = record.get("options") or []
-        product = record.get("product") or {}
-        product_options = product.get("product_options") or []
-
-        # Build a lookup map from option name to ProductOption data
-        product_options_map = {}
-        for product_option in product_options:
-            if product_option:
-                name = product_option.get("name")
-                if name:
-                    product_options_map[name] = {
-                        "id": self.tools.resolve_str_id(product_option.get("id")),
-                        "position": product_option.get("position"),
-                    }
-
-        # Enrich each option with id and position from the matching ProductOption
-        for option in options:
-            if option:
-                option_name = option.get("name")
-                if option_name and option_name in product_options_map:
-                    option["id"] = product_options_map[option_name]["id"]
-                    option["position"] = product_options_map[option_name]["position"]
-
     def record_process_components(self, record: MutableMapping[str, Any]) -> Iterable[MutableMapping[str, Any]]:
         """
         Defines how to process collected components.
@@ -2884,9 +2742,6 @@ class ProductVariant(ShopifyBulkQuery):
             record["presentment_prices"] = self._process_presentment_prices(record_components.get("ProductVariantPricePair", []))
             record.pop("record_components")
 
-        # enrich options with id and position from product options (must be done before product is removed)
-        self._enrich_options_with_product_options(record)
-
         # unnest mandatory fields from their placeholders
         record["product_id"] = self._unnest_and_resolve_id(record, "product", "product_id")
         record["inventory_item_id"] = self._unnest_and_resolve_id(record, "inventoryItem", "inventory_item_id")

{airbyte_source_shopify-3.2.0.dev202602040050 → airbyte_source_shopify-3.2.1.dev202601082335}/source_shopify/source.py

@@ -11,7 +11,6 @@ from requests.exceptions import ConnectionError, RequestException, SSLError
 from airbyte_cdk.models import FailureType, SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
-from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException
 from airbyte_cdk.utils import AirbyteTracedException
 
 from .auth import MissingAccessTokenError, ShopifyAuthenticator
@@ -21,7 +20,6 @@ from .streams.streams import (
     Articles,
     BalanceTransactions,
     Blogs,
-    CollectionProducts,
     Collections,
     Collects,
     Countries,
@@ -110,8 +108,6 @@ class ConnectionCheckTest:
             return False, self.describe_error("index_error", shop_name, response)
         except MissingAccessTokenError:
             return False, self.describe_error("missing_token_error")
-        except (BaseBackoffException, AirbyteTracedException) as error:
-            return False, self.describe_error("connection_error", shop_name) or str(error)
 
     def get_shop_id(self) -> str:
         """
@@ -182,7 +178,6 @@ class SourceShopify(AbstractSource):
             Articles(config),
             BalanceTransactions(config),
             Blogs(config),
-            CollectionProducts(config),
             Collections(config),
             Collects(config),
             CustomCollections(config),

{airbyte_source_shopify-3.2.0.dev202602040050 → airbyte_source_shopify-3.2.1.dev202601082335}/source_shopify/streams/streams.py

@@ -10,7 +10,6 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional
 import requests
 from source_shopify.shopify_graphql.bulk.query import (
     Collection,
-    CollectionProduct,
     CustomerAddresses,
     CustomerJourney,
     DeliveryProfile,
@@ -111,14 +110,50 @@ class Orders(IncrementalShopifyStreamWithDeletedEvents):
 
 
 class Disputes(IncrementalShopifyStream):
+    """
+    Disputes stream for Shopify Payments API.
+
+    Note: Uses 'initiated_at' as cursor field to ensure dispute status updates are captured
+    during incremental syncs. Previously used 'id' with 'since_id' filtering, which only
+    captured new disputes but missed updates to existing disputes (e.g., status changes
+    from 'needs_response' to 'won' or 'lost').
+
+    The Shopify API supports 'initiated_at' as a filter parameter, but only for exact date
+    matching (e.g., initiated_at=2013-05-03), not for range queries needed for incremental
+    sync. An alternative approach would be to iterate through dates and make multiple API
+    calls for each date, but this would be complex and inefficient. Instead, we leverage
+    the existing datetime-based client-side filtering that the connector already provides,
+    fetching all disputes ordered by 'initiated_at' and filtering them client-side.
+
+    API Reference: https://shopify.dev/docs/api/admin-rest/latest/resources/dispute
+    """
+
     data_field = "disputes"
-    filter_field = "since_id"
-    cursor_field = "id"
-    order_field = "id"
+    cursor_field = "initiated_at"
+    order_field = "initiated_at"
 
     def path(self, **kwargs) -> str:
         return f"shopify_payments/{self.data_field}.json"
 
+    def request_params(
+        self, stream_state: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs
+    ) -> MutableMapping[str, Any]:
+        """
+        Override to exclude server-side filtering since the API only supports exact date matching,
+        not the range queries needed for incremental sync.
+        """
+        params = ShopifyStream.request_params(self, stream_state=stream_state, next_page_token=next_page_token, **kwargs)
+        if not next_page_token:
+            params["order"] = f"{self.order_field} asc"
+        return params
+
+    def read_records(self, stream_state: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]:
+        """
+        Override to apply client-side filtering based on initiated_at cursor field.
+        """
+        records = super().read_records(stream_state=stream_state, **kwargs)
+        yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=records)
+
 
 class MetafieldOrders(IncrementalShopifyGraphQlBulkStream):
     bulk_query: MetafieldOrder = MetafieldOrder
@@ -324,21 +359,6 @@ class MetafieldCollections(IncrementalShopifyGraphQlBulkStream):
     bulk_query: MetafieldCollection = MetafieldCollection
 
 
-class CollectionProducts(IncrementalShopifyGraphQlBulkStream):
-    """
-    Stream that returns all products associated with each collection, including both
-    custom collections and smart collections. Unlike the Collects stream which only
-    returns manually associated products, this stream returns all products that belong
-    to a collection (including those matched by smart collection rules).
-
-    https://shopify.dev/docs/api/admin-graphql/latest/objects/Collection#field-Collection.fields.products
-    """
-
-    bulk_query: CollectionProduct = CollectionProduct
-    cursor_field = "collection_updated_at"
-    primary_key = ["collection_id", "product_id"]
-
-
 class BalanceTransactions(IncrementalShopifyStream):
     """
     PaymentsTransactions stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported:

airbyte_source_shopify-3.2.0.dev202602040050/source_shopify/schemas/collection_products.json

@@ -1,35 +0,0 @@
-{
-  "type": "object",
-  "additionalProperties": true,
-  "properties": {
-    "collection_id": {
-      "description": "The unique identifier for the collection.",
-      "type": ["null", "integer"]
-    },
-    "collection_admin_graphql_api_id": {
-      "description": "The Admin GraphQL API ID for the collection.",
-      "type": ["null", "string"]
-    },
-    "collection_handle": {
-      "description": "The handle (URL-friendly name) for the collection.",
-      "type": ["null", "string"]
-    },
-    "collection_updated_at": {
-      "description": "The date and time when the collection was last updated.",
-      "type": ["null", "string"],
-      "format": "date-time"
-    },
-    "product_id": {
-      "description": "The unique identifier for the product.",
-      "type": ["null", "integer"]
-    },
-    "product_admin_graphql_api_id": {
-      "description": "The Admin GraphQL API ID for the product.",
-      "type": ["null", "string"]
-    },
-    "shop_url": {
-      "description": "The URL of the shop associated with this collection-product association.",
-      "type": ["null", "string"]
-    }
-  }
-}