airbyte-source-shopify 2.5.12.dev202412101009__tar.gz → 2.5.14__tar.gz

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 2.5.12.dev202412101009
+Version: 2.5.14
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2
@@ -11,7 +11,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=5,<6)
+Requires-Dist: airbyte-cdk (>=6,<7)
 Requires-Dist: graphql-query (>=1,<2)
 Requires-Dist: sgqlc (==16.3)
 Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/shopify

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.5.12.dev202412101009"
+version = "2.5.14"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [
@@ -22,7 +22,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.10,<3.12"
-airbyte-cdk = "^5"
+airbyte-cdk = "^6"
 sgqlc = "==16.3"
 graphql-query = "^1"
 

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/auth.py

@@ -25,7 +25,6 @@ class NotImplementedAuth(Exception):
 
 
 class ShopifyAuthenticator(TokenAuthenticator):
-
     """
     Making Authenticator to be able to accept Header-Based authentication.
     """

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/config_migrations.py

@@ -5,12 +5,13 @@
 
 from typing import Any, List, Mapping
 
+from orjson import orjson
+
 from airbyte_cdk.config_observation import create_connector_config_control_message
 from airbyte_cdk.entrypoint import AirbyteEntrypoint
 from airbyte_cdk.models import AirbyteMessageSerializer
 from airbyte_cdk.sources import Source
 from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository
-from orjson import orjson
 
 
 class MigrateConfig:

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/http_request.py

@@ -3,9 +3,11 @@
 from typing import Optional, Union
 
 import requests
+from requests import exceptions
+
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, ErrorResolution, ResponseAction
-from requests import exceptions
+
 
 RESPONSE_CONSUMPTION_EXCEPTIONS = (
     exceptions.ChunkedEncodingError,

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/scopes.py

@@ -7,12 +7,14 @@ import logging
 from typing import Any, Iterable, List, Mapping, Optional
 
 import requests
-from airbyte_cdk.sources.streams.http import HttpClient
 from requests.exceptions import InvalidURL, JSONDecodeError
 
+from airbyte_cdk.sources.streams.http import HttpClient
+
 from .http_request import ShopifyErrorHandler
 from .utils import ShopifyAccessScopesError, ShopifyBadJsonError, ShopifyWrongShopNameError
 
+
 SCOPES_MAPPING: Mapping[str, set[str]] = {
     # SCOPE: read_customers
     "Customers": ("read_customers",),
@@ -82,7 +84,6 @@ ALWAYS_PERMITTED_STREAMS: List[str] = [
 
 
 class ShopifyScopes:
-
     # define default logger
     logger = logging.getLogger("airbyte")
 

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/shopify_graphql/bulk/exceptions.py

@@ -52,6 +52,11 @@ class ShopifyBulkExceptions:
 
         failure_type: FailureType = FailureType.transient_error
 
+    class BulkJobCheckpointCollisionError(BaseBulkException):
+        """Raised when an attempt to create a job using the `checkpointed cursor` value goes into inf.loop."""
+
+        failure_type: FailureType = FailureType.transient_error
+
     class BulkJobRedirectToOtherShopError(BaseBulkException):
         """Raised when the response contains another shop name"""
 

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/shopify_graphql/bulk/job.py

@@ -9,11 +9,12 @@ from typing import Any, Final, Iterable, List, Mapping, Optional
 
 import pendulum as pdm
 import requests
-from airbyte_cdk.sources.streams.http import HttpClient
 from requests.exceptions import JSONDecodeError
 from source_shopify.utils import LOGGER, ApiTypeEnum
 from source_shopify.utils import ShopifyRateLimiter as limiter
 
+from airbyte_cdk.sources.streams.http import HttpClient
+
 from .exceptions import AirbyteTracedException, ShopifyBulkExceptions
 from .query import ShopifyBulkQuery, ShopifyBulkTemplates
 from .record import ShopifyBulkRecord
@@ -53,7 +54,7 @@ class ShopifyBulkManager:
 
     # currents: _job_id, _job_state, _job_created_at, _job_self_canceled
     _job_id: Optional[str] = field(init=False, default=None)
-    _job_state: str = field(init=False, default=None)  # this string is based on ShopifyBulkJobStatus
+    _job_state: str | None = field(init=False, default=None)  # this string is based on ShopifyBulkJobStatus
     # completed and saved Bulk Job result filename
     _job_result_filename: Optional[str] = field(init=False, default=None)
     # date-time when the Bulk Job was created on the server
@@ -70,6 +71,8 @@ class ShopifyBulkManager:
     _job_last_rec_count: int = field(init=False, default=0)
     # the flag to adjust the next slice from the checkpointed cursor vaue
     _job_adjust_slice_from_checkpoint: bool = field(init=False, default=False)
+    # keeps the last checkpointed cursor value for supported streams
+    _job_last_checkpoint_cursor_value: str | None = field(init=False, default=None)
 
     # expand slice factor
     _job_size_expand_factor: int = field(init=False, default=2)
@@ -209,6 +212,27 @@ class ShopifyBulkManager:
         # reseting the checkpoint flag, if bulk job has completed normally
         self._job_adjust_slice_from_checkpoint = False
 
+    def _set_last_checkpoint_cursor_value(self, checkpointed_cursor: str) -> None:
+        """
+        Sets the last checkpoint cursor value.
+
+        Args:
+            checkpointed_cursor (str): The cursor value to set as the last checkpoint. Defaults to None.
+        """
+        self._job_last_checkpoint_cursor_value = checkpointed_cursor
+
+    def _checkpoint_cursor_has_collision(self, checkpointed_cursor: str) -> bool:
+        """
+        Checks if the provided checkpointed cursor collides with the last checkpointed cursor value.
+
+        Args:
+            checkpointed_cursor (str): The cursor value to check for collision. Defaults to None.
+
+        Returns:
+            bool: True if the provided cursor collides with the last checkpointed cursor value, False otherwise.
+        """
+        return self._job_last_checkpoint_cursor_value == checkpointed_cursor
+
     def _job_completed(self) -> bool:
         return self._job_state == ShopifyBulkJobStatus.COMPLETED.value
 
@@ -329,7 +353,7 @@ class ShopifyBulkManager:
     def _on_completed_job(self, response: Optional[requests.Response] = None) -> None:
         self._job_result_filename = self._job_get_result(response)
 
-    def _on_failed_job(self, response: requests.Response) -> AirbyteTracedException:
+    def _on_failed_job(self, response: requests.Response) -> AirbyteTracedException | None:
         if not self._supports_checkpointing:
             raise ShopifyBulkExceptions.BulkJobFailed(
                 f"The BULK Job: `{self._job_id}` exited with {self._job_state}, details: {response.text}",
@@ -485,7 +509,7 @@ class ShopifyBulkManager:
             self._job_state = ShopifyBulkJobStatus.CREATED.value
             LOGGER.info(f"Stream: `{self.http_client.name}`, the BULK Job: `{self._job_id}` is {ShopifyBulkJobStatus.CREATED.value}")
 
-    def job_size_normalize(self, start: datetime, end: datetime) -> datetime:
+    def job_size_normalize(self, start: datetime, end: datetime) -> None:
         # adjust slice size when it's bigger than the loop point when it should end,
         # to preserve correct job size adjustments when this is the only job we need to run, based on STATE provided
         requested_slice_size = (end - start).total_days()
@@ -498,8 +522,20 @@ class ShopifyBulkManager:
     def _adjust_slice_end(self, slice_end: datetime, checkpointed_cursor: Optional[str] = None) -> datetime:
         """
         Choose between the existing `slice_end` value or `checkpointed_cursor` value, if provided.
+
+        Optionally: raises the `transient` error if the checkpoint collision occurs.
         """
-        return pdm.parse(checkpointed_cursor) if checkpointed_cursor else slice_end
+
+        if checkpointed_cursor:
+            if self._checkpoint_cursor_has_collision(checkpointed_cursor):
+                raise ShopifyBulkExceptions.BulkJobCheckpointCollisionError(
+                    f"The stream: `{self.http_client.name}` checkpoint collision is detected. Try to increase the `BULK Job checkpoint (rows collected)` to the bigger value. The stream will be synced again during the next sync attempt."
+                )
+            # set the checkpointed cursor value
+            self._set_last_checkpoint_cursor_value(checkpointed_cursor)
+            return pdm.parse(checkpointed_cursor)
+
+        return slice_end
 
     def get_adjusted_job_end(self, slice_start: datetime, slice_end: datetime, checkpointed_cursor: Optional[str] = None) -> datetime:
         if self._job_adjust_slice_from_checkpoint:

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/shopify_graphql/bulk/query.py

@@ -2562,7 +2562,6 @@ class ProductVariant(ShopifyBulkQuery):
 
     @property
     def query_nodes(self) -> Optional[Union[List[Field], List[str]]]:
-
         prices_fields: List[str] = ["amount", "currencyCode"]
         presentment_prices_fields: List[Field] = [
             Field(

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/shopify_graphql/bulk/retry.py

@@ -8,6 +8,7 @@ from source_shopify.utils import LOGGER
 
 from .exceptions import ShopifyBulkExceptions
 
+
 BULK_RETRY_ERRORS: Final[Tuple] = (
     ShopifyBulkExceptions.BulkJobBadResponse,
     ShopifyBulkExceptions.BulkJobError,

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/shopify_graphql/bulk/tools.py

@@ -11,6 +11,7 @@ import pendulum as pdm
 
 from .exceptions import ShopifyBulkExceptions
 
+
 # default end line tag
 END_OF_FILE: str = "<end_of_file>"
 BULK_PARENT_KEY: str = "__parentId"

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/shopify_graphql/graphql.py

@@ -9,6 +9,7 @@ import sgqlc.operation
 
 from . import schema
 
+
 _schema = schema
 _schema_root = _schema.shopify_schema
 

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/shopify_graphql/schema.py

@@ -7,6 +7,7 @@ import sgqlc.types
 import sgqlc.types.datetime
 import sgqlc.types.relay
 
+
 shopify_schema = sgqlc.types.Schema()
 
 

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/source.py

@@ -6,11 +6,12 @@
 import logging
 from typing import Any, List, Mapping, Tuple
 
+from requests.exceptions import ConnectionError, RequestException, SSLError
+
 from airbyte_cdk.models import FailureType, SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.utils import AirbyteTracedException
-from requests.exceptions import ConnectionError, RequestException, SSLError
 
 from .auth import MissingAccessTokenError, ShopifyAuthenticator
 from .scopes import ShopifyScopes

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/spec.json

@@ -115,10 +115,10 @@
       "job_checkpoint_interval": {
         "type": "integer",
         "title": "BULK Job checkpoint (rows collected)",
-        "description": "The threshold, after which the single BULK Job should be checkpointed.",
+        "description": "The threshold, after which the single BULK Job should be checkpointed (min: 15k, max: 1M)",
         "default": 100000,
         "minimum": 15000,
-        "maximum": 200000
+        "maximum": 1000000
       }
     }
   },

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/streams/base_streams.py

@@ -12,11 +12,6 @@ from urllib.parse import parse_qsl, urlparse
 
 import pendulum as pdm
 import requests
-from airbyte_cdk.models import SyncMode
-from airbyte_cdk.sources.streams.core import StreamData
-from airbyte_cdk.sources.streams.http import HttpClient, HttpStream
-from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, HttpStatusErrorHandler
-from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING
 from requests.exceptions import RequestException
 from source_shopify.http_request import ShopifyErrorHandler
 from source_shopify.shopify_graphql.bulk.job import ShopifyBulkManager
@@ -26,6 +21,12 @@ from source_shopify.utils import EagerlyCachedStreamState as stream_state_cache
 from source_shopify.utils import ShopifyNonRetryableErrors
 from source_shopify.utils import ShopifyRateLimiter as limiter
 
+from airbyte_cdk.models import SyncMode
+from airbyte_cdk.sources.streams.core import StreamData
+from airbyte_cdk.sources.streams.http import HttpClient, HttpStream
+from airbyte_cdk.sources.streams.http.error_handlers import ErrorHandler, HttpStatusErrorHandler
+from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING
+
 
 class ShopifyStream(HttpStream, ABC):
     # define default logger

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/utils.py

@@ -10,10 +10,12 @@ from time import sleep
 from typing import Any, Callable, Dict, Final, List, Mapping, Optional
 
 import requests
+
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction
 from airbyte_cdk.utils import AirbyteTracedException
 
+
 # default logger instance
 LOGGER: Final[logging.Logger] = logging.getLogger("airbyte")
 
@@ -47,7 +49,7 @@ class ShopifyNonRetryableErrors:
                 response_action=ResponseAction.IGNORE,
                 failure_type=FailureType.config_error,
                 error_message=f"Stream `{stream}`. Entity might not be available or missing.",
-            )
+            ),
             # extend the mapping with more handable errors, if needed.
         }
 

{airbyte_source_shopify-2.5.12.dev202412101009 → airbyte_source_shopify-2.5.14}/source_shopify/streams/streams.py

@@ -6,8 +6,6 @@
 from typing import Any, Iterable, Mapping, MutableMapping, Optional
 
 import requests
-from airbyte_cdk.sources.streams.core import package_name_from_class
-from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
 from requests.exceptions import RequestException
 from source_shopify.shopify_graphql.bulk.query import (
     Collection,
@@ -36,6 +34,9 @@ from source_shopify.shopify_graphql.graphql import get_query_products
 from source_shopify.utils import ApiTypeEnum
 from source_shopify.utils import ShopifyRateLimiter as limiter
 
+from airbyte_cdk.sources.streams.core import package_name_from_class
+from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader
+
 from .base_streams import (
     IncrementalShopifyGraphQlBulkStream,
     IncrementalShopifyNestedStream,
@@ -249,7 +250,6 @@ class MetafieldCollections(IncrementalShopifyGraphQlBulkStream):
 
 
 class BalanceTransactions(IncrementalShopifyStream):
-
     """
     PaymentsTransactions stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported:
     https://shopify.dev/api/admin-rest/2021-07/resources/transactions