airbyte-source-shopify 3.1.0__tar.gz → 3.1.0.dev202512181928__tar.gz

{airbyte_source_shopify-3.1.0 → airbyte_source_shopify-3.1.0.dev202512181928}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 3.1.0
+Version: 3.1.0.dev202512181928
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_shopify-3.1.0 → airbyte_source_shopify-3.1.0.dev202512181928}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "3.1.0"
+version = "3.1.0.dev.202512181928"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [ "Airbyte <contact@airbyte.io>",]

{airbyte_source_shopify-3.1.0 → airbyte_source_shopify-3.1.0.dev202512181928}/source_shopify/scopes.py

@@ -37,7 +37,6 @@ SCOPES_MAPPING: Mapping[str, set[str]] = {
     "MetafieldDraftOrders": ("read_draft_orders",),
     # SCOPE: read_products
     "Products": ("read_products",),
-    "DeletedProducts": ("read_products",),
     "MetafieldProducts": ("read_products",),
     "ProductImages": ("read_products",),
     "MetafieldProductImages": ("read_products",),

{airbyte_source_shopify-3.1.0 → airbyte_source_shopify-3.1.0.dev202512181928}/source_shopify/source.py

@@ -27,7 +27,6 @@ from .streams.streams import (
     CustomerAddress,
     CustomerJourneySummary,
     Customers,
-    DeletedProducts,
     DiscountCodes,
     Disputes,
     DraftOrders,
@@ -212,7 +211,6 @@ class SourceShopify(AbstractSource):
             PriceRules(config),
             ProductImages(config),
             Products(config),
-            DeletedProducts(config),
             ProductVariants(config),
             Shop(config),
             SmartCollections(config),

{airbyte_source_shopify-3.1.0 → airbyte_source_shopify-3.1.0.dev202512181928}/source_shopify/streams/streams.py

@@ -135,110 +135,6 @@ class Products(IncrementalShopifyGraphQlBulkStream):
     bulk_query: Product = Product
 
 
-class DeletedProducts(IncrementalShopifyStream):
-    """
-    Stream for fetching deleted products using the Shopify GraphQL Events API.
-    This stream queries events with action:destroy and subject_type:Product to get deleted product records.
-    https://shopify.dev/docs/api/admin-graphql/latest/queries/events
-
-    Note: This stream extends IncrementalShopifyStream (REST base class) rather than IncrementalShopifyGraphQlBulkStream
-    because it uses Shopify's standard GraphQL Events API, NOT the Bulk Operations API (bulkOperationRunQuery).
-    The Events API has a fundamentally different architecture:
-    - Uses immediate queries with cursor pagination (not async job creation + file download)
-    - Returns results in response.data.events.nodes (not JSONL files)
-    - Requires different URL endpoint (/graphql.json), request format (POST with query in body), and pagination logic
-
-    Therefore, url_base, path, request_params, next_page_token, and parse_response are overridden to accommodate
-    the GraphQL request/response structure while maintaining incremental sync capabilities from the base class.
-    """
-
-    data_field = "graphql"
-    cursor_field = "deleted_at"
-    http_method = "POST"
-    filter_field = None
-
-    _page_cursor: Optional[str] = None
-
-    EVENTS_QUERY = """
-    query GetDeletedProductEvents($first: Int!, $after: String, $query: String) {
-        events(first: $first, after: $after, query: $query, sortKey: CREATED_AT) {
-            pageInfo {
-                hasNextPage
-                endCursor
-            }
-            nodes {
-                ... on BasicEvent {
-                    id
-                    createdAt
-                    message
-                    subjectId
-                    subjectType
-                }
-            }
-        }
-    }
-    """
-
-    @property
-    def url_base(self) -> str:
-        return f"https://{self.config['shop']}.myshopify.com/admin/api/{self.api_version}/graphql.json"
-
-    def path(self, **kwargs) -> str:
-        return ""
-
-    def request_params(
-        self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs
-    ) -> MutableMapping[str, Any]:
-        return {}
-
-    def request_body_json(
-        self,
-        stream_state: Optional[Mapping[str, Any]] = None,
-        stream_slice: Optional[Mapping[str, Any]] = None,
-        next_page_token: Optional[Mapping[str, Any]] = None,
-    ) -> Optional[Mapping[str, Any]]:
-        query_filter = "action:destroy AND subject_type:Product"
-        if stream_state and stream_state.get(self.cursor_field):
-            state_value = stream_state[self.cursor_field]
-            query_filter += f" AND created_at:>'{state_value}'"
-
-        variables = {
-            "first": 250,
-            "query": query_filter,
-        }
-
-        if next_page_token and next_page_token.get("cursor"):
-            variables["after"] = next_page_token["cursor"]
-
-        return {
-            "query": self.EVENTS_QUERY,
-            "variables": variables,
-        }
-
-    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
-        json_response = response.json()
-        page_info = json_response.get("data", {}).get("events", {}).get("pageInfo", {})
-        if page_info.get("hasNextPage"):
-            return {"cursor": page_info.get("endCursor")}
-        return None
-
-    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        json_response = response.json()
-        events = json_response.get("data", {}).get("events", {}).get("nodes", [])
-        for event in events:
-            if event.get("subjectType") == "PRODUCT" and event.get("subjectId"):
-                subject_id = event.get("subjectId", "")
-                product_id = int(subject_id.split("/")[-1]) if "/" in subject_id else None
-                if product_id:
-                    yield {
-                        "id": product_id,
-                        "deleted_at": event.get("createdAt"),
-                        "deleted_message": event.get("message"),
-                        "deleted_description": None,
-                        "shop_url": self.config.get("shop"),
-                    }
-
-
 class MetafieldProducts(IncrementalShopifyGraphQlBulkStream):
     parent_stream_class = Products
     bulk_query: MetafieldProduct = MetafieldProduct
@@ -301,18 +197,47 @@ class MetafieldSmartCollections(MetafieldShopifySubstream):
 
 class Collects(IncrementalShopifyStream):
     """
-    Collects stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported:
-    https://shopify.dev/docs/admin-api/rest/reference/products/collect
+    Collects stream uses client-side incremental sync because the Shopify REST API
+    does not support filtering by updated_at for this endpoint - only `since_id` is available.
+    https://shopify.dev/docs/api/admin-rest/latest/resources/collect
 
-    The Collect stream is the link between Products and Collections, if the Collection is created for Products,
-    the `collect` record is created, it's reasonable to Full Refresh all collects. As for Incremental refresh -
-    we would use the since_id specificaly for this stream.
+    The Collect stream is the link between Products and Collections. To capture both new records
+    AND updates to existing records, we fetch all records and filter client-side based on updated_at.
+
+    Breaking change from previous versions: cursor_field changed from `id` to `updated_at`.
     """
 
     data_field = "collects"
-    cursor_field = "id"
+    cursor_field = "updated_at"
     order_field = "id"
-    filter_field = "since_id"
+
+    def request_params(
+        self, stream_state: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs
+    ) -> MutableMapping[str, Any]:
+        """
+        Override to implement client-side incremental sync.
+        We fetch all records (no server-side filtering) and filter client-side using filter_records_newer_than_state.
+        The Shopify Collects API only supports since_id filtering, which cannot capture updates to existing records.
+        """
+        params = {"limit": self.limit}
+        if next_page_token:
+            params.update(**next_page_token)
+        else:
+            params["order"] = f"{self.order_field} asc"
+        return params
+
+    def read_records(
+        self,
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[Mapping[str, Any]] = None,
+        **kwargs,
+    ) -> Iterable[Mapping[str, Any]]:
+        """
+        Override to apply client-side filtering based on updated_at cursor.
+        This ensures we capture both new records and updates to existing records.
+        """
+        records = super().read_records(stream_state=stream_state, stream_slice=stream_slice, **kwargs)
+        yield from self.filter_records_newer_than_state(stream_state, records)
 
 
 class Collections(IncrementalShopifyGraphQlBulkStream):

airbyte_source_shopify-3.1.0/source_shopify/schemas/deleted_products.json

@@ -1,27 +0,0 @@
-{
-  "type": ["object", "null"],
-  "additionalProperties": true,
-  "properties": {
-    "id": {
-      "description": "The unique identifier of the deleted product.",
-      "type": ["null", "integer"]
-    },
-    "deleted_at": {
-      "description": "The date and time when the product was deleted.",
-      "type": ["null", "string"],
-      "format": "date-time"
-    },
-    "deleted_message": {
-      "description": "Message related to the deletion of the product.",
-      "type": ["null", "string"]
-    },
-    "deleted_description": {
-      "description": "Description of the reason for deletion.",
-      "type": ["null", "string"]
-    },
-    "shop_url": {
-      "description": "The URL of the shop where the product was listed.",
-      "type": ["null", "string"]
-    }
-  }
-}