airbyte-source-shopify 3.1.2__tar.gz → 3.2.0__tar.gz

{airbyte_source_shopify-3.1.2 → airbyte_source_shopify-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 3.1.2
+Version: 3.2.0
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_shopify-3.1.2 → airbyte_source_shopify-3.2.0}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "3.1.2"
+version = "3.2.0"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [ "Airbyte <contact@airbyte.io>",]

airbyte_source_shopify-3.2.0/source_shopify/schemas/collection_products.json

@@ -0,0 +1,35 @@
+{
+  "type": "object",
+  "additionalProperties": true,
+  "properties": {
+    "collection_id": {
+      "description": "The unique identifier for the collection.",
+      "type": ["null", "integer"]
+    },
+    "collection_admin_graphql_api_id": {
+      "description": "The Admin GraphQL API ID for the collection.",
+      "type": ["null", "string"]
+    },
+    "collection_handle": {
+      "description": "The handle (URL-friendly name) for the collection.",
+      "type": ["null", "string"]
+    },
+    "collection_updated_at": {
+      "description": "The date and time when the collection was last updated.",
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "product_id": {
+      "description": "The unique identifier for the product.",
+      "type": ["null", "integer"]
+    },
+    "product_admin_graphql_api_id": {
+      "description": "The Admin GraphQL API ID for the product.",
+      "type": ["null", "string"]
+    },
+    "shop_url": {
+      "description": "The URL of the shop associated with this collection-product association.",
+      "type": ["null", "string"]
+    }
+  }
+}

{airbyte_source_shopify-3.1.2 → airbyte_source_shopify-3.2.0}/source_shopify/scopes.py

@@ -44,6 +44,7 @@ SCOPES_MAPPING: Mapping[str, set[str]] = {
     "MetafieldProductVariants": ("read_products",),
     "CustomCollections": ("read_products",),
     "Collects": ("read_products",),
+    "CollectionProducts": ("read_products",),
     "ProductVariants": ("read_products", "read_inventory"),
     "MetafieldCollections": ("read_products",),
     "SmartCollections": ("read_products",),

{airbyte_source_shopify-3.1.2 → airbyte_source_shopify-3.2.0}/source_shopify/shopify_graphql/bulk/query.py

@@ -952,6 +952,114 @@ class Collection(ShopifyBulkQuery):
         yield record
 
 
+class CollectionProduct(ShopifyBulkQuery):
+    """
+    Returns the products associated with each collection, including both custom collections
+    and smart collections. This provides all product<>collection associations, not just
+    manually associated products (which is what the Collects REST API provides).
+
+    {
+        collections(query: "updated_at:>='2023-02-07T00:00:00+00:00' AND updated_at:<='2023-12-04T00:00:00+00:00'", sortKey: UPDATED_AT) {
+            edges {
+                node {
+                    __typename
+                    id
+                    handle
+                    updatedAt
+                    products {
+                        edges {
+                            node {
+                                __typename
+                                id
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    """
+
+    query_name = "collections"
+    sort_key = "UPDATED_AT"
+
+    products_fields: List[Field] = [
+        Field(
+            name="edges",
+            fields=[
+                Field(
+                    name="node",
+                    fields=[
+                        "__typename",
+                        "id",
+                    ],
+                )
+            ],
+        )
+    ]
+
+    query_nodes: List[Field] = [
+        "__typename",
+        "id",
+        Field(name="handle"),
+        Field(name="updatedAt"),
+        Field(name="products", fields=products_fields),
+    ]
+
+    record_composition = {
+        "new_record": "Collection",
+        "record_components": ["Product"],
+    }
+
+    def _process_product_components(self, products: List[dict]) -> List[dict]:
+        """
+        Process product components to resolve IDs from string to int and preserve the original ID.
+
+        Args:
+            products: List of product dictionaries with string IDs
+
+        Returns:
+            List of processed product dictionaries with both id (int) and admin_graphql_api_id (str)
+        """
+        for product in products:
+            # Save the original string ID before resolving
+            product["admin_graphql_api_id"] = product.get("id")
+            # Resolve the ID from string to int
+            product["id"] = self.tools.resolve_str_id(product.get("id"))
+        return products
+
+    def record_process_components(self, record: MutableMapping[str, Any]) -> Iterable[MutableMapping[str, Any]]:
+        """
+        Process collection records and yield one record per collection-product association.
+        """
+        record_components = record.get("record_components", {})
+        products = record_components.get("Product", [])
+
+        # Get collection info - id is already resolved to int, admin_graphql_api_id has the string version
+        collection_id = record.get("id")
+        collection_admin_graphql_api_id = record.get("admin_graphql_api_id")
+        collection_handle = record.get("handle")
+        collection_updated_at = self.tools.from_iso8601_to_rfc3339(record, "updatedAt")
+
+        if products:
+            # Process products to resolve their IDs
+            products = self._process_product_components(products)
+
+            for product in products:
+                product_id = product.get("id")
+                product_admin_graphql_api_id = product.get("admin_graphql_api_id")
+
+                yield {
+                    "collection_id": collection_id,
+                    "collection_admin_graphql_api_id": collection_admin_graphql_api_id,
+                    "collection_handle": collection_handle,
+                    "collection_updated_at": collection_updated_at,
+                    "product_id": product_id,
+                    "product_admin_graphql_api_id": product_admin_graphql_api_id,
+                    "shop_url": self.config.get("shop"),
+                }
+
+
 class CustomerAddresses(ShopifyBulkQuery):
     """
     {

{airbyte_source_shopify-3.1.2 → airbyte_source_shopify-3.2.0}/source_shopify/source.py

@@ -21,6 +21,7 @@ from .streams.streams import (
     Articles,
     BalanceTransactions,
     Blogs,
+    CollectionProducts,
     Collections,
     Collects,
     Countries,
@@ -181,6 +182,7 @@ class SourceShopify(AbstractSource):
             Articles(config),
             BalanceTransactions(config),
             Blogs(config),
+            CollectionProducts(config),
             Collections(config),
             Collects(config),
             CustomCollections(config),

{airbyte_source_shopify-3.1.2 → airbyte_source_shopify-3.2.0}/source_shopify/streams/streams.py

@@ -10,6 +10,7 @@ from typing import Any, Iterable, Mapping, MutableMapping, Optional
 import requests
 from source_shopify.shopify_graphql.bulk.query import (
     Collection,
+    CollectionProduct,
     CustomerAddresses,
     CustomerJourney,
     DeliveryProfile,
@@ -323,6 +324,21 @@ class MetafieldCollections(IncrementalShopifyGraphQlBulkStream):
     bulk_query: MetafieldCollection = MetafieldCollection
 
 
+class CollectionProducts(IncrementalShopifyGraphQlBulkStream):
+    """
+    Stream that returns all products associated with each collection, including both
+    custom collections and smart collections. Unlike the Collects stream which only
+    returns manually associated products, this stream returns all products that belong
+    to a collection (including those matched by smart collection rules).
+
+    https://shopify.dev/docs/api/admin-graphql/latest/objects/Collection#field-Collection.fields.products
+    """
+
+    bulk_query: CollectionProduct = CollectionProduct
+    cursor_field = "collection_updated_at"
+    primary_key = ["collection_id", "product_id"]
+
+
 class BalanceTransactions(IncrementalShopifyStream):
     """
     PaymentsTransactions stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported: