airbyte-source-shopify 2.4.14.dev202407181247__py3-none-any.whl → 3.1.0__py3-none-any.whl

source_shopify/shopify_graphql/bulk/record.py

@@ -3,12 +3,14 @@
 #
 
 
-import logging
 from dataclasses import dataclass, field
+from functools import cached_property
 from io import TextIOWrapper
 from json import loads
 from os import remove
-from typing import Any, Callable, Final, Iterable, List, Mapping, MutableMapping, Optional, Union
+from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
+
+from source_shopify.utils import LOGGER
 
 from .exceptions import ShopifyBulkExceptions
 from .query import ShopifyBulkQuery
@@ -17,43 +19,194 @@ from .tools import END_OF_FILE, BulkTools
 
 @dataclass
 class ShopifyBulkRecord:
+    """
+    ShopifyBulkRecord is a class designed to handle the processing of bulk records from Shopify's GraphQL API.
+
+    Attributes:
+        query (ShopifyBulkQuery): The query object associated with the bulk record.
+        parent_stream_name (Optional[str]): The name of the parent stream, if any.
+        parent_stream_cursor (Optional[str]): The cursor of the parent stream, if any.
+        buffer (List[MutableMapping[str, Any]]): A buffer to store records before processing.
+        composition (Optional[Mapping[str, Any]]): The composition of the record, derived from the query.
+        record_process_components (Optional[Callable[[MutableMapping], MutableMapping]]): A callable to process record components.
+        components (List[str]): A list of components derived from the record composition.
+        _parent_stream_cursor_value (Optional[str | int]): The current value of the parent stream cursor.
+        record_composed (int): The count of records composed.
+
+    Methods:
+        __post_init__(): Initializes additional attributes after the object is created.
+        tools(): Returns an instance of BulkTools.
+        has_parent_stream(): Checks if the record has a parent stream.
+        parent_cursor_key(): Returns the key for the parent cursor if a parent stream exists.
+        check_type(record, types): Checks if the record's type matches the given type(s).
+        _parse_parent_state_value(value): Parses the parent state value and converts it to the appropriate format.
+        _set_parent_state_value(value): Sets the parent state value by parsing the provided value and updating the parent stream cursor value.
+        _track_parent_cursor(record): Tracks the cursor value from the parent stream if it exists and updates the parent state.
+        get_parent_stream_state(): Retrieves the state of the parent stream if it exists.
+        record_new(record): Processes a new record by preparing it, removing the "__typename" key, and appending it to the buffer.
+        record_new_component(record): Processes a new record by extracting its component type and adding it to the appropriate placeholder in the components list.
+        component_prepare(record): Prepares the given record by initializing a "record_components" dictionary.
+        buffer_flush(): Flushes the buffer by processing each record in the buffer.
+        record_compose(record): Processes a given record and yields buffered records if certain conditions are met.
+        process_line(jsonl_file): Processes a JSON Lines (jsonl) file and yields records.
+        record_resolve_id(record): Resolves and updates the 'id' field in the given record.
+        produce_records(filename): Reads the JSONL content saved from `job.job_retrieve_result()` line-by-line to avoid OOM.
+        read_file(filename, remove_file): Reads a file and produces records from it.
+    """
+
     query: ShopifyBulkQuery
+    parent_stream_name: Optional[str] = None
+    parent_stream_cursor: Optional[str] = None
 
     # default buffer
     buffer: List[MutableMapping[str, Any]] = field(init=False, default_factory=list)
 
-    # default logger
-    logger: Final[logging.Logger] = logging.getLogger("airbyte")
-
     def __post_init__(self) -> None:
         self.composition: Optional[Mapping[str, Any]] = self.query.record_composition
         self.record_process_components: Optional[Callable[[MutableMapping], MutableMapping]] = self.query.record_process_components
         self.components: List[str] = self.composition.get("record_components", []) if self.composition else []
+        # We track the parent state for BULK substreams outside of the main CDK methods,
+        # to be able to update the moving parent state when there are no substream records to emit.
+        self._parent_stream_cursor_value: Optional[str | int] = None
+        # how many records composed
+        self.record_composed: int = 0
 
-    @property
+    @cached_property
     def tools(self) -> BulkTools:
         return BulkTools()
 
+    @cached_property
+    def has_parent_stream(self) -> bool:
+        return True if self.parent_stream_name and self.parent_stream_cursor else False
+
+    @cached_property
+    def parent_cursor_key(self) -> Optional[str]:
+        if self.has_parent_stream:
+            return f"{self.parent_stream_name}_{self.parent_stream_cursor}"
+
     @staticmethod
     def check_type(record: Mapping[str, Any], types: Union[List[str], str]) -> bool:
+        """
+        Check if the record's type matches the given type(s).
+
+        Args:
+            record (Mapping[str, Any]): The record to check, expected to have a "__typename" key.
+            types (Union[List[str], str]): The type(s) to check against. Can be a single type (str) or a list of types (List[str]).
+
+        Returns:
+            bool: True if the record's type matches one of the given types, False otherwise.
+        """
+
         record_type = record.get("__typename")
         if isinstance(types, list):
             return any(record_type == t for t in types)
         else:
             return record_type == types
 
+    def _parse_parent_state_value(self, value: str | int) -> str | int:
+        """
+        Parses the parent state value and converts it to the appropriate format.
+
+        If the value is a string, it converts it to RFC 3339 datetime format using the `_datetime_str_to_rfc3339` method.
+        If the value is an integer, it returns the value as is.
+
+        Args:
+            value (str | int): The parent state value to be parsed.
+
+        Returns:
+            str | int: The parsed parent state value in the appropriate format.
+        """
+
+        if isinstance(value, str):
+            return self.tools._datetime_str_to_rfc3339(value)
+        elif isinstance(value, int):
+            return value
+
+    def _set_parent_state_value(self, value: str | int) -> None:
+        """
+        Sets the parent state value by parsing the provided value and updating the
+        parent stream cursor value. If the parent stream cursor value is already set,
+        it updates it to the maximum of the current and parsed values.
+
+        Args:
+            value (str | int): The value to be parsed and set as the parent state value.
+        """
+
+        parsed_value = self._parse_parent_state_value(value)
+        if not self._parent_stream_cursor_value:
+            self._parent_stream_cursor_value = parsed_value
+        else:
+            self._parent_stream_cursor_value = max(self._parent_stream_cursor_value, parsed_value)
+
+    def _track_parent_cursor(self, record: MutableMapping[str, Any]) -> None:
+        """
+        Tracks the cursor value from the parent stream if it exists and updates the parent state.
+
+        Args:
+            record (MutableMapping[str, Any]): The record from which to extract the parent cursor value.
+
+        Returns:
+            None
+        """
+
+        if self.has_parent_stream:
+            cursor_value: Optional[str | int] = record.get(self.parent_cursor_key, None)
+            if cursor_value:
+                self._set_parent_state_value(cursor_value)
+
+    def get_parent_stream_state(self) -> Optional[Union[str, Mapping[str, Any]]]:
+        """
+        Retrieve the state of the parent stream if it exists.
+
+        Returns:
+            Optional[Union[str, Mapping[str, Any]]]: A dictionary containing the parent stream cursor and its value
+            if the parent stream exists and has a cursor value, otherwise None.
+        """
+
+        if self.has_parent_stream and self._parent_stream_cursor_value:
+            return {self.parent_stream_cursor: self._parent_stream_cursor_value}
+
     def record_new(self, record: MutableMapping[str, Any]) -> None:
+        """
+        Processes a new record by preparing it, removing the "__typename" key, and appending it to the buffer.
+
+        Args:
+            record (MutableMapping[str, Any]): The record to be processed.
+        """
+
         record = self.component_prepare(record)
         record.pop("__typename")
         self.buffer.append(record)
 
     def record_new_component(self, record: MutableMapping[str, Any]) -> None:
+        """
+        Processes a new record by extracting its component type and adding it to the appropriate
+        placeholder in the components list.
+
+        Args:
+            record (MutableMapping[str, Any]): The record to be processed.
+            It is expected to contain a "__typename" key which indicates the component type.
+        """
+
         component = record.get("__typename")
         record.pop("__typename")
         # add component to its placeholder in the components list
         self.buffer[-1]["record_components"][component].append(record)
 
     def component_prepare(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+        """
+        Prepares the given record by initializing a "record_components" dictionary.
+
+        If the instance has components, this method will add a "record_components" key to the record,
+        with each component as a key and an empty list as its value.
+
+        Args:
+            record (MutableMapping[str, Any]): The record to be prepared.
+
+        Returns:
+            MutableMapping[str, Any]: The updated record with initialized "record_components".
+        """
+
         if self.components:
             record["record_components"] = {}
             for component in self.components:
@@ -61,8 +214,24 @@ class ShopifyBulkRecord:
         return record
 
     def buffer_flush(self) -> Iterable[Mapping[str, Any]]:
+        """
+        Flushes the buffer by processing each record in the buffer.
+
+        For each record in the buffer:
+        - Tracks the parent state using `_track_parent_cursor`.
+        - Resolves the record ID from `str` to `int` using `record_resolve_id`.
+        - Processes record components using `record_process_components`.
+
+        Yields:
+            Iterable[Mapping[str, Any]]: Processed records from the buffer.
+
+        After processing, the buffer is cleared.
+        """
+
         if len(self.buffer) > 0:
             for record in self.buffer:
+                # track the parent state
+                self._track_parent_cursor(record)
                 # resolve id from `str` to `int`
                 record = self.record_resolve_id(record)
                 # process record components
@@ -72,10 +241,25 @@ class ShopifyBulkRecord:
 
     def record_compose(self, record: Mapping[str, Any]) -> Optional[Iterable[MutableMapping[str, Any]]]:
         """
+        Processes a given record and yields buffered records if certain conditions are met.
+
+        Args:
+            record (Mapping[str, Any]): The record to be processed.
+
+        Returns:
+            Optional[Iterable[MutableMapping[str, Any]]]: An iterable of buffered records if conditions are met, otherwise None.
+
+        The method performs the following steps:
+        1. Checks if the record matches the type specified in the "new_record" composition.
+           - If it matches, it yields any buffered records from previous iterations and registers the new record.
+        2. Checks if the record matches any of the specified components.
+           - If it matches, it registers the new component record.
+
         Step 1: register the new record by it's `__typename`
         Step 2: check for `components` by their `__typename` and add to the placeholder
         Step 3: repeat until the `<END_OF_FILE>`.
         """
+
         if self.check_type(record, self.composition.get("new_record")):
             # emit from previous iteration, if present
             yield from self.buffer_flush()
@@ -86,9 +270,21 @@ class ShopifyBulkRecord:
             self.record_new_component(record)
 
     def process_line(self, jsonl_file: TextIOWrapper) -> Iterable[MutableMapping[str, Any]]:
-        # process the json lines
+        """
+        Processes a JSON Lines (jsonl) file and yields records.
+
+        Args:
+            jsonl_file (TextIOWrapper): A file-like object containing JSON Lines data.
+
+        Yields:
+            Iterable[MutableMapping[str, Any]]: An iterable of dictionaries representing the processed records.
+
+        The method reads each line from the provided jsonl_file. It exits the loop when it encounters the <end_of_file> marker.
+        For non-empty lines, it parses the JSON content and yields the resulting records. Finally, it emits any remaining
+        records in the buffer.
+        """
+
         for line in jsonl_file:
-            # we exit from the loop when receive <end_of_file> (file ends)
             if line == END_OF_FILE:
                 break
             elif line != "":
@@ -99,38 +295,76 @@ class ShopifyBulkRecord:
 
     def record_resolve_id(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
         """
-        The ids are fetched in the format of: " gid://shopify/Order/<Id> "
-        Input:
-            { "Id": "gid://shopify/Order/19435458986123"}
-        We need to extract the actual id from the string instead.
-        Output:
-            { "id": 19435458986123, "admin_graphql_api_id": "gid://shopify/Order/19435458986123"}
-        """
-        # save the actual api id to the `admin_graphql_api_id`
-        # while resolving the `id` in `record_resolve_id`,
-        # we re-assign the original id like `"gid://shopify/Order/19435458986123"`,
-        # into `admin_graphql_api_id` have the ability to identify the record oigin correctly in subsequent actions.
-        # IF NOT `id` field is provided by the query results, we should return composed record `as is`.
+        Resolves and updates the 'id' field in the given record.
+
+        This method extracts the 'id' from the record, checks if it is a string,
+        and if so, assigns it to 'admin_graphql_api_id' in the record. It then
+        resolves the string 'id' to an integer using the 'resolve_str_id' method
+        from the 'tools' attribute and updates the 'id' field in the record.
+
+        Args:
+            record (MutableMapping[str, Any]): The record containing the 'id' field to be resolved.
+            Example:
+                { "Id": "gid://shopify/Order/19435458986123"}
+
+        Returns:
+            MutableMapping[str, Any]: The updated record with the resolved 'id' field.
+            Example:
+                { "id": 19435458986123, "admin_graphql_api_id": "gid://shopify/Order/19435458986123"}
+        """
+
         id = record.get("id")
         if id and isinstance(id, str):
             record["admin_graphql_api_id"] = id
-            # extracting the int(id) and reassign
             record["id"] = self.tools.resolve_str_id(id)
         return record
 
     def produce_records(self, filename: str) -> Iterable[MutableMapping[str, Any]]:
         """
-        Read the JSONL content saved from `job.job_retrieve_result()` line-by-line to avoid OOM.
-        The filename example: `bulk-4039263649981.jsonl`,
-            where `4039263649981` is the `id` of the COMPLETED BULK Jobw with `result_url`.
-            Note: typically the `filename` is taken from the `result_url` string provided in the response.
+        Produce records from a JSON Lines (jsonl) file.
+
+        This method reads a JSON Lines file, processes each line, converts the field names to snake_case,
+        and yields each processed record. It also keeps track of the number of records processed.
+
+        Args:
+            filename (str): The path to the JSON Lines file.
+
+        Yields:
+            MutableMapping[str, Any]: A dictionary representing a processed record with field names in snake_case.
         """
 
         with open(filename, "r") as jsonl_file:
+            # reset the counter
+            self.record_composed = 0
+
             for record in self.process_line(jsonl_file):
                 yield self.tools.fields_names_to_snake_case(record)
+                self.record_composed += 1
 
     def read_file(self, filename: str, remove_file: Optional[bool] = True) -> Iterable[Mapping[str, Any]]:
+        """
+        Read the JSONL content saved from `job.job_retrieve_result()` line-by-line to avoid OOM.
+
+        Args:
+            filename (str): The name of the file to read.
+            remove_file (Optional[bool]): Flag indicating whether to remove the file after reading. Defaults to True.
+
+            Example:
+                Note: typically the `filename` is taken from the `result_url` string provided in the response.
+
+                `bulk-4039263649981.jsonl` :
+                    - the `4039263649981` is the `id` of the COMPLETED BULK Jobw with `result_url`
+
+        Yields:
+            Iterable[Mapping[str, Any]]: An iterable of records produced from the file.
+
+        Raises:
+            ShopifyBulkExceptions.BulkRecordProduceError: If an error occurs while producing records from the file.
+
+        Logs:
+            Logs an info message if the file removal fails.
+        """
+
         try:
             # produce records from saved result
             yield from self.produce_records(filename)
@@ -144,8 +378,5 @@ class ShopifyBulkRecord:
                 try:
                     remove(filename)
                 except Exception as e:
-                    self.logger.info(f"Failed to remove the `tmp job result` file, the file doen't exist. Details: {repr(e)}.")
-                    # we should pass here, if the file wasn't removed , it's either:
-                    # - doesn't exist
-                    # - will be dropped with the container shut down.
+                    LOGGER.info(f"Failed to remove the `tmp job result` file, the file doen't exist. Details: {repr(e)}.")
                     pass

source_shopify/shopify_graphql/bulk/retry.py

@@ -1,23 +1,24 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
-import logging
 from functools import wraps
 from time import sleep
 from typing import Any, Callable, Final, Optional, Tuple, Type
 
+from source_shopify.utils import LOGGER
+
 from .exceptions import ShopifyBulkExceptions
 
+
 BULK_RETRY_ERRORS: Final[Tuple] = (
     ShopifyBulkExceptions.BulkJobBadResponse,
     ShopifyBulkExceptions.BulkJobError,
 )
 
 
-def bulk_retry_on_exception(logger: logging.Logger, more_exceptions: Optional[Tuple[Type[Exception], ...]] = None) -> Callable:
+def bulk_retry_on_exception(more_exceptions: Optional[Tuple[Type[Exception], ...]] = None) -> Callable:
     """
     A decorator to retry a function when specified exceptions are raised.
 
-    :param logger: Number of times to retry.
     :param more_exceptions: A tuple of exception types to catch.
     """
 
@@ -31,29 +32,28 @@ def bulk_retry_on_exception(logger: logging.Logger, more_exceptions: Optional[Tu
                 except BULK_RETRY_ERRORS or more_exceptions as ex:
                     current_retries += 1
                     if current_retries > self._job_max_retries:
-                        logger.error("Exceeded retry limit. Giving up.")
+                        LOGGER.error("Exceeded retry limit. Giving up.")
                         raise
                     else:
-                        logger.warning(
-                            f"Stream `{self.http_client._name}`: {ex}. Retrying {current_retries}/{self._job_max_retries} after {self._job_backoff_time} seconds."
+                        LOGGER.warning(
+                            f"Stream `{self.http_client.name}`: {ex}. Retrying {current_retries}/{self._job_max_retries} after {self._job_backoff_time} seconds."
                         )
                         sleep(self._job_backoff_time)
                 except ShopifyBulkExceptions.BulkJobCreationFailedConcurrentError:
                     if self._concurrent_attempt == self._concurrent_max_retry:
                         message = f"The BULK Job couldn't be created at this time, since another job is running."
-                        logger.error(message)
+                        LOGGER.error(message)
                         raise ShopifyBulkExceptions.BulkJobConcurrentError(message)
 
                     self._concurrent_attempt += 1
-                    logger.warning(
-                        f"Stream: `{self.http_client._name}`, the BULK concurrency limit has reached. Waiting {self._concurrent_interval} sec before retry, attempt: {self._concurrent_attempt}.",
+                    LOGGER.warning(
+                        f"Stream: `{self.http_client.name}`, the BULK concurrency limit has reached. Waiting {self._concurrent_interval} sec before retry, attempt: {self._concurrent_attempt}.",
                     )
                     sleep(self._concurrent_interval)
                 except ShopifyBulkExceptions.BulkJobRedirectToOtherShopError:
-                    logger.warning(
-                        f"Stream: `{self.http_client._name}`, the `shop name` differs from the provided by the User: `{self.base_url}`. Switching to the `{self._new_base_url}`.",
+                    LOGGER.warning(
+                        f"Stream: `{self.http_client.name}`, the `shop name` differs from the provided in `input configuration`. Switching to the `{self._tools.shop_name_from_url(self.base_url)}`.",
                     )
-                    self._switch_base_url()
 
         return wrapper
 

source_shopify/shopify_graphql/bulk/tools.py

@@ -11,6 +11,7 @@ import pendulum as pdm
 
 from .exceptions import ShopifyBulkExceptions
 
+
 # default end line tag
 END_OF_FILE: str = "<end_of_file>"
 BULK_PARENT_KEY: str = "__parentId"
@@ -53,7 +54,21 @@ class BulkTools:
             )
 
     @staticmethod
-    def from_iso8601_to_rfc3339(record: Mapping[str, Any], field: str) -> Mapping[str, Any]:
+    def shop_name_from_url(url: str) -> str:
+        match = re.search(r"https://(.*?)(\.myshopify)", url)
+        if match:
+            return match.group(1)
+        else:
+            # safety net, if there is an error parsing url,
+            # on no match is found
+            return url
+
+    @staticmethod
+    def _datetime_str_to_rfc3339(value: str) -> str:
+        return pdm.parse(value).to_rfc3339_string()
+
+    @staticmethod
+    def from_iso8601_to_rfc3339(record: Mapping[str, Any], field: str) -> Optional[str]:
         """
         Converts date-time as follows:
             Input: "2023-01-01T15:00:00Z"
@@ -63,7 +78,7 @@ class BulkTools:
         # some fields that expected to be resolved as ids, might not be populated for the particular `RECORD`,
         # we should return `None` to make the field `null` in the output as the result of the transformation.
         target_value = record.get(field)
-        return pdm.parse(target_value).to_rfc3339_string() if target_value else record.get(field)
+        return BulkTools._datetime_str_to_rfc3339(target_value) if target_value else record.get(field)
 
     def fields_names_to_snake_case(self, dict_input: Optional[Mapping[str, Any]] = None) -> Optional[MutableMapping[str, Any]]:
         # transforming record field names from camel to snake case, leaving the `__parent_id` relation in place

source_shopify/source.py

@@ -6,11 +6,12 @@
 import logging
 from typing import Any, List, Mapping, Tuple
 
+from requests.exceptions import ConnectionError, RequestException, SSLError
+
 from airbyte_cdk.models import FailureType, SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
 from airbyte_cdk.utils import AirbyteTracedException
-from requests.exceptions import ConnectionError, RequestException, SSLError
 
 from .auth import MissingAccessTokenError, ShopifyAuthenticator
 from .scopes import ShopifyScopes
@@ -26,7 +27,7 @@ from .streams.streams import (
     CustomerAddress,
     CustomerJourneySummary,
     Customers,
-    CustomerSavedSearch,
+    DeletedProducts,
     DiscountCodes,
     Disputes,
     DraftOrders,
@@ -56,8 +57,8 @@ from .streams.streams import (
     PriceRules,
     ProductImages,
     Products,
-    ProductsGraphQl,
     ProductVariants,
+    ProfileLocationGroups,
     Shop,
     SmartCollections,
     TenderTransactions,
@@ -131,10 +132,6 @@ class SourceShopify(AbstractSource):
     def continue_sync_on_stream_failure(self) -> bool:
         return True
 
-    @property
-    def raise_exception_on_missing_stream(self) -> bool:
-        return False
-
     @staticmethod
     def get_shop_name(config) -> str:
         split_pattern = ".myshopify.com"
@@ -215,15 +212,14 @@ class SourceShopify(AbstractSource):
             PriceRules(config),
             ProductImages(config),
             Products(config),
-            ProductsGraphQl(config),
+            DeletedProducts(config),
             ProductVariants(config),
             Shop(config),
             SmartCollections(config),
             TenderTransactions(config),
             self.select_transactions_stream(config),
-            CustomerSavedSearch(config),
             CustomerAddress(config),
-            Countries(config),
+            Countries(config=config, parent=ProfileLocationGroups(config)),
         ]
 
         return [

source_shopify/spec.json

@@ -98,20 +98,26 @@
         "description": "Defines which API type (REST/BULK) to use to fetch `Transactions` data. If you are a `Shopify Plus` user, leave the default value to speed up the fetch.",
         "default": false
       },
+      "job_product_variants_include_pres_prices": {
+        "type": "boolean",
+        "title": "Add `Presentment prices` to Product Variants",
+        "description": "If enabled, the `Product Variants` stream attempts to include `Presentment prices` field (may affect the performance).",
+        "default": true
+      },
       "job_termination_threshold": {
         "type": "integer",
         "title": "BULK Job termination threshold",
         "description": "The max time in seconds, after which the single BULK Job should be `CANCELED` and retried. The bigger the value the longer the BULK Job is allowed to run.",
-        "default": 3600,
+        "default": 7200,
         "minimum": 3600,
         "maximum": 21600
       },
       "job_checkpoint_interval": {
         "type": "integer",
-        "title": "BULK Job checkpoint (lines collected)",
-        "description": "The threshold, after which the single BULK Job should be `CANCELED` and checkpointed.",
-        "default": 200000,
-        "minimum": 50000,
+        "title": "BULK Job checkpoint (rows collected)",
+        "description": "The threshold, after which the single BULK Job should be checkpointed (min: 15k, max: 1M)",
+        "default": 100000,
+        "minimum": 15000,
         "maximum": 1000000
       }
     }