airbyte-source-shopify 3.0.8.dev202509081459__py3-none-any.whl → 3.1.0__py3-none-any.whl

{airbyte_source_shopify-3.0.8.dev202509081459.dist-info → airbyte_source_shopify-3.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 3.0.8.dev202509081459
+Version: 3.1.0
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_shopify-3.0.8.dev202509081459.dist-info → airbyte_source_shopify-3.1.0.dist-info}/RECORD

@@ -14,6 +14,7 @@ source_shopify/schemas/custom_collections.json,sha256=ElDY1y_G_VFPOGr9ipU022AZDW
 source_shopify/schemas/customer_address.json,sha256=kLKylul_xK7eO_aduvDddj078k8aQLM4JtUEpa5Tp_8,2628
 source_shopify/schemas/customer_journey_summary.json,sha256=GlZmIqFeR1x84DwgwMo28HQ8QseImkhvYxLSNRkUmwg,12733
 source_shopify/schemas/customers.json,sha256=pywejBEtvMnRHvvnJQhfA8NlCBjELv7XuMlKdpS4cZI,10233
+source_shopify/schemas/deleted_products.json,sha256=7TbqrABNDg9ZFAeLJCPe2Z5jtk9QGbJDuoSKofGd1lg,773
 source_shopify/schemas/discount_codes.json,sha256=YxGptdpjXwM46fmzs5zoi3m96xAWKXrfllC_oIZW0vo,3730
 source_shopify/schemas/disputes.json,sha256=Uvzy1Gi954cLxij6H0Vbz0OWS6GS10YaJI4DJfY2x8I,1775
 source_shopify/schemas/draft_orders.json,sha256=3NaZt9Qk4mY8VO9pB5UBzw3GZibL8MvI0z_tlnMsdBI,24211
@@ -49,22 +50,22 @@ source_shopify/schemas/shop.json,sha256=vEGiTvEYX7qnMq06MRVBycqih49h49xjTNC6gJux
 source_shopify/schemas/smart_collections.json,sha256=kv7dINsvgzJ0RyKfFNKjU0apdNDXwQaHfnNZfQsshcU,2009
 source_shopify/schemas/tender_transactions.json,sha256=U8fdT-eflycEPzYSpBDiB0lp9wxmJHgioHTrICflh78,2006
 source_shopify/schemas/transactions.json,sha256=vbwscH3UcAtbSsC70mBka4oNaFR4S3S6IFBmzR7t37U,10226
-source_shopify/scopes.py,sha256=N0njfMHn3Q1AQXuTj5VfjQOio10jaDarpC_oLYnWvqc,6490
+source_shopify/scopes.py,sha256=78f9QL3PJZ9UDx1gIWzNwx5fYJE9OB3vPi9RahB_kFw,6533
 source_shopify/shopify_graphql/bulk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 source_shopify/shopify_graphql/bulk/exceptions.py,sha256=4dj7Za4xIfwL-zf8joT9svF_RSoGlE3GviMiIl1e1rs,2532
-source_shopify/shopify_graphql/bulk/job.py,sha256=qSd1EAVAjOazPRHc7G48OJJao3xqF9foKqxD0pIWick,28192
+source_shopify/shopify_graphql/bulk/job.py,sha256=c3Cg70_Io9jTD-rU-5MvjHaPmJCtcpeqEYnRtFECGOo,28673
 source_shopify/shopify_graphql/bulk/query.py,sha256=D8rnI1SDw50-Gt18lt7YwwNNdsbVMbBfxZa9xVJZbto,130981
 source_shopify/shopify_graphql/bulk/record.py,sha256=X6VGngugv7a_S8UEeDo121BkdCVLj5nWlHK76A21kyo,16898
 source_shopify/shopify_graphql/bulk/retry.py,sha256=R5rSJJE8D5zcj6mN-OmmNO2aFZEIdjAlWclDDVW5KPI,2626
 source_shopify/shopify_graphql/bulk/status.py,sha256=RmuQ2XsYL3iRCpVGxea9F1wXGmbwasDCSXjaTyL4LMA,328
 source_shopify/shopify_graphql/bulk/tools.py,sha256=nUQ2ZmPTKJNJdfLToR6KJtLKcJFCChSifkAOvwg0Vss,4065
-source_shopify/source.py,sha256=txb3wIm-3xXd8-5QLSeu2TeHBSnppwy5PEIOEl40mVw,8517
+source_shopify/source.py,sha256=oikoM-VPNk62zlmeAQR59PMxfuXq2s42N7zaqLM6_lo,8575
 source_shopify/spec.json,sha256=ITYWiQ-NrI5VISk5qmUQhp9ChUE2FV18d8xzVzPwvAg,6144
-source_shopify/streams/base_streams.py,sha256=j3x38czi13KgvDuP7oHmfTClDr9S6Jfh_r-J97BF9Iw,42322
-source_shopify/streams/streams.py,sha256=D70Ik1vU75NKlmJMnS7W2-5gApA2ANq9eRnKligMTNw,14555
+source_shopify/streams/base_streams.py,sha256=k_4uLaLADLRTUcSmP8uA_830uuzRvnqUaCVGcb0Zpd8,42625
+source_shopify/streams/streams.py,sha256=96ZzuhlKny2scejzRNhL7IHQ2FJ6e7Z4Qfot6nccfQg,18899
 source_shopify/transform.py,sha256=mn0htL812_90zc_YszGQa0hHcIZQpYYdmk8IqpZm5TI,4685
 source_shopify/utils.py,sha256=DSqEchu-MQJ7zust7CNfqOkGIv9OSR-5UUsuD-bsDa8,16224
-airbyte_source_shopify-3.0.8.dev202509081459.dist-info/METADATA,sha256=GUfH4CzDOyfH0Afvotv0IHmYcTGXCwoq44UpHficGHE,5313
-airbyte_source_shopify-3.0.8.dev202509081459.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_source_shopify-3.0.8.dev202509081459.dist-info/entry_points.txt,sha256=SyTwKSsPk9MCdPf01saWpnp8hcmZOgBssVcSIvMbBeQ,57
-airbyte_source_shopify-3.0.8.dev202509081459.dist-info/RECORD,,
+airbyte_source_shopify-3.1.0.dist-info/METADATA,sha256=zkgsUWjEe9nD_Mh6PfX0HgFTYBCShhc6I6oULF8ozrM,5297
+airbyte_source_shopify-3.1.0.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_source_shopify-3.1.0.dist-info/entry_points.txt,sha256=SyTwKSsPk9MCdPf01saWpnp8hcmZOgBssVcSIvMbBeQ,57
+airbyte_source_shopify-3.1.0.dist-info/RECORD,,

source_shopify/schemas/deleted_products.json

@@ -0,0 +1,27 @@
+{
+  "type": ["object", "null"],
+  "additionalProperties": true,
+  "properties": {
+    "id": {
+      "description": "The unique identifier of the deleted product.",
+      "type": ["null", "integer"]
+    },
+    "deleted_at": {
+      "description": "The date and time when the product was deleted.",
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "deleted_message": {
+      "description": "Message related to the deletion of the product.",
+      "type": ["null", "string"]
+    },
+    "deleted_description": {
+      "description": "Description of the reason for deletion.",
+      "type": ["null", "string"]
+    },
+    "shop_url": {
+      "description": "The URL of the shop where the product was listed.",
+      "type": ["null", "string"]
+    }
+  }
+}

source_shopify/scopes.py

@@ -37,6 +37,7 @@ SCOPES_MAPPING: Mapping[str, set[str]] = {
     "MetafieldDraftOrders": ("read_draft_orders",),
     # SCOPE: read_products
     "Products": ("read_products",),
+    "DeletedProducts": ("read_products",),
     "MetafieldProducts": ("read_products",),
     "ProductImages": ("read_products",),
     "MetafieldProductImages": ("read_products",),

source_shopify/shopify_graphql/bulk/job.py

@@ -531,9 +531,14 @@ class ShopifyBulkManager:
         self, slice_end: datetime, checkpointed_cursor: Optional[str] = None, filter_checkpointed_cursor: Optional[str] = None
     ) -> datetime:
         """
-        Choose between the existing `slice_end` value or `checkpointed_cursor` value, if provided.
+        Choose between the existing `slice_end` value or `checkpointed_cursor` value or `filter_checkpointed_cursor` value, if provided.
 
         Optionally: raises the `transient` error if the checkpoint collision occurs.
+
+        Note: filter_checkpointed_cursor is only used when cursor field is ID for streams like Customer Address etc.
+        This method should return a datetime from last checkpointed value to adjust slice end, when cursor value is ID (int type)
+        method gets end datetime from filter_checkpointed_cursor, which is value from filter field from last record.
+        See https://github.com/airbytehq/oncall/issues/9052 for more details.
         """
 
         if checkpointed_cursor:

source_shopify/source.py

@@ -27,6 +27,7 @@ from .streams.streams import (
     CustomerAddress,
     CustomerJourneySummary,
     Customers,
+    DeletedProducts,
     DiscountCodes,
     Disputes,
     DraftOrders,
@@ -211,6 +212,7 @@ class SourceShopify(AbstractSource):
             PriceRules(config),
             ProductImages(config),
             Products(config),
+            DeletedProducts(config),
             ProductVariants(config),
             Shop(config),
             SmartCollections(config),

source_shopify/streams/base_streams.py

@@ -222,6 +222,11 @@ class IncrementalShopifyStream(ShopifyStream, ABC):
         return params
 
     def track_checkpoint_cursor(self, record_value: Union[str, int], filter_record_value: Optional[str] = None) -> None:
+        """
+        Tracks _checkpoint_cursor value (values from cursor field) and _filter_checkpointed_cursor value (value from filter field).
+        _filter_checkpointed_cursor value is only used when cursor field is ID for streams like Customer Address etc.
+        When after canceled/failed job source tries to adjust stream slice (see ShopifyBulkManager._adjust_slice_end()).
+        """
         if self.filter_by_state_checkpoint:
             # set checkpoint cursor
             if not self._checkpoint_cursor:
@@ -231,9 +236,7 @@ class IncrementalShopifyStream(ShopifyStream, ABC):
                 self._checkpoint_cursor = record_value
 
             if filter_record_value:
-                if not self._filter_checkpointed_cursor:
-                    self._filter_checkpointed_cursor = filter_record_value
-                if str(filter_record_value) >= str(self._filter_checkpointed_cursor):
+                if not self._filter_checkpointed_cursor or str(filter_record_value) >= str(self._filter_checkpointed_cursor):
                     self._filter_checkpointed_cursor = filter_record_value
 
     def should_checkpoint(self, index: int) -> bool:

source_shopify/streams/streams.py

@@ -4,6 +4,7 @@
 
 
 import logging
+import sys
 from typing import Any, Iterable, Mapping, MutableMapping, Optional
 
 import requests
@@ -134,10 +135,120 @@ class Products(IncrementalShopifyGraphQlBulkStream):
     bulk_query: Product = Product
 
 
+class DeletedProducts(IncrementalShopifyStream):
+    """
+    Stream for fetching deleted products using the Shopify GraphQL Events API.
+    This stream queries events with action:destroy and subject_type:Product to get deleted product records.
+    https://shopify.dev/docs/api/admin-graphql/latest/queries/events
+
+    Note: This stream extends IncrementalShopifyStream (REST base class) rather than IncrementalShopifyGraphQlBulkStream
+    because it uses Shopify's standard GraphQL Events API, NOT the Bulk Operations API (bulkOperationRunQuery).
+    The Events API has a fundamentally different architecture:
+    - Uses immediate queries with cursor pagination (not async job creation + file download)
+    - Returns results in response.data.events.nodes (not JSONL files)
+    - Requires different URL endpoint (/graphql.json), request format (POST with query in body), and pagination logic
+
+    Therefore, url_base, path, request_params, next_page_token, and parse_response are overridden to accommodate
+    the GraphQL request/response structure while maintaining incremental sync capabilities from the base class.
+    """
+
+    data_field = "graphql"
+    cursor_field = "deleted_at"
+    http_method = "POST"
+    filter_field = None
+
+    _page_cursor: Optional[str] = None
+
+    EVENTS_QUERY = """
+    query GetDeletedProductEvents($first: Int!, $after: String, $query: String) {
+        events(first: $first, after: $after, query: $query, sortKey: CREATED_AT) {
+            pageInfo {
+                hasNextPage
+                endCursor
+            }
+            nodes {
+                ... on BasicEvent {
+                    id
+                    createdAt
+                    message
+                    subjectId
+                    subjectType
+                }
+            }
+        }
+    }
+    """
+
+    @property
+    def url_base(self) -> str:
+        return f"https://{self.config['shop']}.myshopify.com/admin/api/{self.api_version}/graphql.json"
+
+    def path(self, **kwargs) -> str:
+        return ""
+
+    def request_params(
+        self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs
+    ) -> MutableMapping[str, Any]:
+        return {}
+
+    def request_body_json(
+        self,
+        stream_state: Optional[Mapping[str, Any]] = None,
+        stream_slice: Optional[Mapping[str, Any]] = None,
+        next_page_token: Optional[Mapping[str, Any]] = None,
+    ) -> Optional[Mapping[str, Any]]:
+        query_filter = "action:destroy AND subject_type:Product"
+        if stream_state and stream_state.get(self.cursor_field):
+            state_value = stream_state[self.cursor_field]
+            query_filter += f" AND created_at:>'{state_value}'"
+
+        variables = {
+            "first": 250,
+            "query": query_filter,
+        }
+
+        if next_page_token and next_page_token.get("cursor"):
+            variables["after"] = next_page_token["cursor"]
+
+        return {
+            "query": self.EVENTS_QUERY,
+            "variables": variables,
+        }
+
+    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+        json_response = response.json()
+        page_info = json_response.get("data", {}).get("events", {}).get("pageInfo", {})
+        if page_info.get("hasNextPage"):
+            return {"cursor": page_info.get("endCursor")}
+        return None
+
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+        json_response = response.json()
+        events = json_response.get("data", {}).get("events", {}).get("nodes", [])
+        for event in events:
+            if event.get("subjectType") == "PRODUCT" and event.get("subjectId"):
+                subject_id = event.get("subjectId", "")
+                product_id = int(subject_id.split("/")[-1]) if "/" in subject_id else None
+                if product_id:
+                    yield {
+                        "id": product_id,
+                        "deleted_at": event.get("createdAt"),
+                        "deleted_message": event.get("message"),
+                        "deleted_description": None,
+                        "shop_url": self.config.get("shop"),
+                    }
+
+
 class MetafieldProducts(IncrementalShopifyGraphQlBulkStream):
     parent_stream_class = Products
     bulk_query: MetafieldProduct = MetafieldProduct
 
+    state_checkpoint_interval = sys.maxsize
+
+    @property
+    def filter_by_state_checkpoint(self) -> bool:
+        return True
+
 
 class ProductImages(IncrementalShopifyGraphQlBulkStream):
     parent_stream_class = Products