airbyte-source-shopify 3.1.2__tar.gz → 3.1.2.dev202601082220__tar.gz

{airbyte_source_shopify-3.1.2 → airbyte_source_shopify-3.1.2.dev202601082220}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 3.1.2
+Version: 3.1.2.dev202601082220
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_shopify-3.1.2 → airbyte_source_shopify-3.1.2.dev202601082220}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "3.1.2"
+version = "3.1.2.dev.202601082220"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [ "Airbyte <contact@airbyte.io>",]

{airbyte_source_shopify-3.1.2 → airbyte_source_shopify-3.1.2.dev202601082220}/source_shopify/source.py

@@ -11,7 +11,6 @@ from requests.exceptions import ConnectionError, RequestException, SSLError
 from airbyte_cdk.models import FailureType, SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
-from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException
 from airbyte_cdk.utils import AirbyteTracedException
 
 from .auth import MissingAccessTokenError, ShopifyAuthenticator
@@ -109,8 +108,6 @@ class ConnectionCheckTest:
             return False, self.describe_error("index_error", shop_name, response)
         except MissingAccessTokenError:
             return False, self.describe_error("missing_token_error")
-        except (BaseBackoffException, AirbyteTracedException) as error:
-            return False, self.describe_error("connection_error", shop_name) or str(error)
 
     def get_shop_id(self) -> str:
         """

{airbyte_source_shopify-3.1.2 → airbyte_source_shopify-3.1.2.dev202601082220}/source_shopify/spec.json

@@ -119,6 +119,15 @@
         "default": 100000,
         "minimum": 15000,
         "maximum": 1000000
+      },
+      "lookback_window_in_days": {
+        "type": "integer",
+        "title": "Lookback Window (in days)",
+        "description": "The number of days to look back for records that may have been missed due to race conditions or late-arriving data. This is useful for ensuring data completeness in incremental syncs. Setting this to 1 or more will re-fetch records from the specified number of days before the last sync state.",
+        "default": 0,
+        "minimum": 0,
+        "maximum": 30,
+        "order": 7
       }
     }
   },

{airbyte_source_shopify-3.1.2 → airbyte_source_shopify-3.1.2.dev202601082220}/source_shopify/streams/base_streams.py

@@ -209,6 +209,24 @@ class IncrementalShopifyStream(ShopifyStream, ABC):
         current_state_value = current_stream_state.get(self.cursor_field) or self.default_state_comparison_value
         return {self.cursor_field: max(last_record_value, current_state_value)}
 
+    def _apply_lookback_window(self, state_value: str) -> str:
+        """
+        Apply the lookback window to the state value by subtracting the configured number of days.
+        This helps capture records that may have been missed due to race conditions or late-arriving data.
+        """
+        lookback_days = self.config.get("lookback_window_in_days", 0)
+        if lookback_days > 0:
+            state_datetime = pdm.parse(state_value)
+            adjusted_datetime = state_datetime.subtract(days=lookback_days)
+            # Ensure we don't go before the configured start_date
+            start_date = self.config.get("start_date")
+            if start_date:
+                start_datetime = pdm.parse(start_date)
+                if adjusted_datetime < start_datetime:
+                    adjusted_datetime = start_datetime
+            return adjusted_datetime.to_rfc3339_string()
+        return state_value
+
     @stream_state_cache.cache_stream_state
     def request_params(
         self, stream_state: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs
@@ -218,7 +236,11 @@ class IncrementalShopifyStream(ShopifyStream, ABC):
         if not next_page_token:
             params["order"] = f"{self.order_field} asc"
             if stream_state:
-                params[self.filter_field] = stream_state.get(self.cursor_field)
+                state_value = stream_state.get(self.cursor_field)
+                # Apply lookback window to go back N days from the state
+                if state_value and self.filter_field != "since_id":
+                    state_value = self._apply_lookback_window(state_value)
+                params[self.filter_field] = state_value
         return params
 
     def track_checkpoint_cursor(self, record_value: Union[str, int], filter_record_value: Optional[str] = None) -> None:
@@ -822,6 +844,9 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream):
     def stream_slices(self, stream_state: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
         if self.filter_field:
             state = self._get_state_value(stream_state)
+            # Apply lookback window only when we have a stream state (not on first sync)
+            if stream_state:
+                state = self._apply_lookback_window(state)
             start = pdm.parse(state)
             end = pdm.now()
             while start < end: