airbyte-source-shopify 2.5.6__tar.gz → 2.5.8__tar.gz

{airbyte_source_shopify-2.5.6 → airbyte_source_shopify-2.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 2.5.6
+Version: 2.5.8
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_shopify-2.5.6 → airbyte_source_shopify-2.5.8}/pyproject.toml

@@ -5,7 +5,7 @@ requires = [
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.5.6"
+version = "2.5.8"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [

{airbyte_source_shopify-2.5.6 → airbyte_source_shopify-2.5.8}/source_shopify/shopify_graphql/bulk/job.py

@@ -2,7 +2,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-import logging
 from dataclasses import dataclass, field
 from datetime import datetime
 from time import sleep, time
@@ -12,7 +11,7 @@ import pendulum as pdm
 import requests
 from airbyte_cdk.sources.streams.http import HttpClient
 from requests.exceptions import JSONDecodeError
-from source_shopify.utils import ApiTypeEnum
+from source_shopify.utils import LOGGER, ApiTypeEnum
 from source_shopify.utils import ShopifyRateLimiter as limiter
 
 from .exceptions import AirbyteTracedException, ShopifyBulkExceptions
@@ -32,8 +31,8 @@ class ShopifyBulkManager:
     job_size: float
     job_checkpoint_interval: int
 
-    # default logger
-    logger: Final[logging.Logger] = logging.getLogger("airbyte")
+    parent_stream_name: Optional[str] = None
+    parent_stream_cursor: Optional[str] = None
 
     # 10Mb chunk size to save the file
     _retrieve_chunk_size: Final[int] = 1024 * 1024 * 10
@@ -94,7 +93,7 @@ class ShopifyBulkManager:
         # how many records should be collected before we use the checkpoining
         self._job_checkpoint_interval = self.job_checkpoint_interval
         # define Record Producer instance
-        self.record_producer: ShopifyBulkRecord = ShopifyBulkRecord(self.query)
+        self.record_producer: ShopifyBulkRecord = ShopifyBulkRecord(self.query, self.parent_stream_name, self.parent_stream_cursor)
 
     @property
     def _tools(self) -> BulkTools:
@@ -251,9 +250,9 @@ class ShopifyBulkManager:
     def _log_state(self, message: Optional[str] = None) -> None:
         pattern = f"Stream: `{self.http_client.name}`, the BULK Job: `{self._job_id}` is {self._job_state}"
         if message:
-            self.logger.info(f"{pattern}. {message}.")
+            LOGGER.info(f"{pattern}. {message}.")
         else:
-            self.logger.info(pattern)
+            LOGGER.info(pattern)
 
     def _job_get_result(self, response: Optional[requests.Response] = None) -> Optional[str]:
         parsed_response = response.json().get("data", {}).get("node", {}) if response else None
@@ -309,13 +308,13 @@ class ShopifyBulkManager:
         sleep(self._job_check_interval)
 
     def _cancel_on_long_running_job(self) -> None:
-        self.logger.info(
+        LOGGER.info(
             f"Stream: `{self.http_client.name}` the BULK Job: {self._job_id} runs longer than expected ({self._job_max_elapsed_time} sec). Retry with the reduced `Slice Size` after self-cancelation."
         )
         self._job_cancel()
 
     def _cancel_on_checkpointing(self) -> None:
-        self.logger.info(f"Stream: `{self.http_client.name}`, checkpointing after >= `{self._job_checkpoint_interval}` rows collected.")
+        LOGGER.info(f"Stream: `{self.http_client.name}`, checkpointing after >= `{self._job_checkpoint_interval}` rows collected.")
         # set the flag to adjust the next slice from the checkpointed cursor value
         self._job_cancel()
 
@@ -434,7 +433,7 @@ class ShopifyBulkManager:
             return True
         return False
 
-    @bulk_retry_on_exception(logger)
+    @bulk_retry_on_exception()
     def _job_check_state(self) -> None:
         while not self._job_completed():
             if self._job_canceled():
@@ -444,7 +443,7 @@ class ShopifyBulkManager:
             else:
                 self._job_track_running()
 
-    @bulk_retry_on_exception(logger)
+    @bulk_retry_on_exception()
     def create_job(self, stream_slice: Mapping[str, str], filter_field: str) -> None:
         if stream_slice:
             query = self.query.get(filter_field, stream_slice["start"], stream_slice["end"])
@@ -484,7 +483,7 @@ class ShopifyBulkManager:
             self._job_id = bulk_response.get("id")
             self._job_created_at = bulk_response.get("createdAt")
             self._job_state = ShopifyBulkJobStatus.CREATED.value
-            self.logger.info(f"Stream: `{self.http_client.name}`, the BULK Job: `{self._job_id}` is {ShopifyBulkJobStatus.CREATED.value}")
+            LOGGER.info(f"Stream: `{self.http_client.name}`, the BULK Job: `{self._job_id}` is {ShopifyBulkJobStatus.CREATED.value}")
 
     def job_size_normalize(self, start: datetime, end: datetime) -> datetime:
         # adjust slice size when it's bigger than the loop point when it should end,
@@ -522,7 +521,7 @@ class ShopifyBulkManager:
             final_message = final_message + lines_collected_message
 
         # emit final Bulk job status message
-        self.logger.info(f"{final_message}")
+        LOGGER.info(f"{final_message}")
 
     def _process_bulk_results(self) -> Iterable[Mapping[str, Any]]:
         if self._job_result_filename:

{airbyte_source_shopify-2.5.6 → airbyte_source_shopify-2.5.8}/source_shopify/shopify_graphql/bulk/query.py

@@ -80,17 +80,7 @@ class ShopifyBulkTemplates:
 @dataclass
 class ShopifyBulkQuery:
     config: Mapping[str, Any]
-    parent_stream_name: Optional[str] = None
-    parent_stream_cursor: Optional[str] = None
-
-    @property
-    def has_parent_stream(self) -> bool:
-        return True if self.parent_stream_name and self.parent_stream_cursor else False
-
-    @property
-    def parent_cursor_key(self) -> Optional[str]:
-        if self.has_parent_stream:
-            return f"{self.parent_stream_name}_{self.parent_stream_cursor}"
+    parent_stream_cursor_alias: Optional[str] = None
 
     @property
     def shop_id(self) -> int:
@@ -143,38 +133,12 @@ class ShopifyBulkQuery:
         """
         return ["__typename", "id"]
 
-    def _inject_parent_cursor_field(self, nodes: List[Field], key: str = "updatedAt", index: int = 2) -> List[Field]:
-        if self.has_parent_stream:
+    def inject_parent_cursor_field(self, nodes: List[Field], key: str = "updatedAt", index: int = 2) -> List[Field]:
+        if self.parent_stream_cursor_alias:
             # inject parent cursor key as alias to the `updatedAt` parent cursor field
-            nodes.insert(index, Field(name="updatedAt", alias=self.parent_cursor_key))
-
+            nodes.insert(index, Field(name=key, alias=self.parent_stream_cursor_alias))
         return nodes
 
-    def _add_parent_record_state(self, record: MutableMapping[str, Any], items: List[dict], to_rfc3339: bool = False) -> List[dict]:
-        """
-        Adds a parent cursor value to each item in the list.
-
-        This method iterates over a list of dictionaries and adds a new key-value pair to each dictionary.
-        The key is the value of `self.query_name`, and the value is another dictionary with a single key "updated_at"
-        and the provided `parent_cursor_value`.
-
-        Args:
-            items (List[dict]): A list of dictionaries to which the parent cursor value will be added.
-            parent_cursor_value (str): The value to be set for the "updated_at" key in the nested dictionary.
-
-        Returns:
-            List[dict]: The modified list of dictionaries with the added parent cursor values.
-        """
-
-        if self.has_parent_stream:
-            parent_cursor_value: Optional[str] = record.get(self.parent_cursor_key, None)
-            parent_state = self.tools._datetime_str_to_rfc3339(parent_cursor_value) if to_rfc3339 and parent_cursor_value else None
-
-            for item in items:
-                item[self.parent_stream_name] = {self.parent_stream_cursor: parent_state}
-
-        return items
-
     def get(self, filter_field: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None) -> str:
         # define filter query string, if passed
         filter_query = f"{filter_field}:>='{start}' AND {filter_field}:<='{end}'" if filter_field else None
@@ -339,7 +303,7 @@ class Metafield(ShopifyBulkQuery):
         elif isinstance(self.type.value, str):
             nodes = [*nodes, metafield_node]
 
-        nodes = self._inject_parent_cursor_field(nodes)
+        nodes = self.inject_parent_cursor_field(nodes)
 
         return nodes
 
@@ -372,9 +336,6 @@ class Metafield(ShopifyBulkQuery):
         else:
             metafields = record_components.get("Metafield", [])
             if len(metafields) > 0:
-                if self.has_parent_stream:
-                    # add parent state to each metafield
-                    metafields = self._add_parent_record_state(record, metafields, to_rfc3339=True)
                 yield from self._process_components(metafields)
 
 
@@ -637,7 +598,7 @@ class MetafieldProductImage(Metafield):
         media_node = self.get_edge_node("media", media_fields)
 
         fields: List[Field] = ["__typename", "id", media_node]
-        fields = self._inject_parent_cursor_field(fields)
+        fields = self.inject_parent_cursor_field(fields)
 
         return fields
 
@@ -2422,7 +2383,7 @@ class ProductImage(ShopifyBulkQuery):
 
     @property
     def query_nodes(self) -> List[Field]:
-        return self._inject_parent_cursor_field(self.nodes)
+        return self.inject_parent_cursor_field(self.nodes)
 
     def _process_component(self, entity: List[dict]) -> List[dict]:
         for item in entity:
@@ -2499,8 +2460,6 @@ class ProductImage(ShopifyBulkQuery):
 
             # add the product_id to each `Image`
             record["images"] = self._add_product_id(record.get("images", []), record.get("id"))
-            # add the product cursor to each `Image`
-            record["images"] = self._add_parent_record_state(record, record.get("images", []), to_rfc3339=True)
             record["images"] = self._merge_with_media(record_components)
             record.pop("record_components")
 

airbyte_source_shopify-2.5.8/source_shopify/shopify_graphql/bulk/record.py

@@ -0,0 +1,382 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from dataclasses import dataclass, field
+from functools import cached_property
+from io import TextIOWrapper
+from json import loads
+from os import remove
+from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Union
+
+from source_shopify.utils import LOGGER
+
+from .exceptions import ShopifyBulkExceptions
+from .query import ShopifyBulkQuery
+from .tools import END_OF_FILE, BulkTools
+
+
+@dataclass
+class ShopifyBulkRecord:
+    """
+    ShopifyBulkRecord is a class designed to handle the processing of bulk records from Shopify's GraphQL API.
+
+    Attributes:
+        query (ShopifyBulkQuery): The query object associated with the bulk record.
+        parent_stream_name (Optional[str]): The name of the parent stream, if any.
+        parent_stream_cursor (Optional[str]): The cursor of the parent stream, if any.
+        buffer (List[MutableMapping[str, Any]]): A buffer to store records before processing.
+        composition (Optional[Mapping[str, Any]]): The composition of the record, derived from the query.
+        record_process_components (Optional[Callable[[MutableMapping], MutableMapping]]): A callable to process record components.
+        components (List[str]): A list of components derived from the record composition.
+        _parent_stream_cursor_value (Optional[str | int]): The current value of the parent stream cursor.
+        record_composed (int): The count of records composed.
+
+    Methods:
+        __post_init__(): Initializes additional attributes after the object is created.
+        tools(): Returns an instance of BulkTools.
+        has_parent_stream(): Checks if the record has a parent stream.
+        parent_cursor_key(): Returns the key for the parent cursor if a parent stream exists.
+        check_type(record, types): Checks if the record's type matches the given type(s).
+        _parse_parent_state_value(value): Parses the parent state value and converts it to the appropriate format.
+        _set_parent_state_value(value): Sets the parent state value by parsing the provided value and updating the parent stream cursor value.
+        _track_parent_cursor(record): Tracks the cursor value from the parent stream if it exists and updates the parent state.
+        get_parent_stream_state(): Retrieves the state of the parent stream if it exists.
+        record_new(record): Processes a new record by preparing it, removing the "__typename" key, and appending it to the buffer.
+        record_new_component(record): Processes a new record by extracting its component type and adding it to the appropriate placeholder in the components list.
+        component_prepare(record): Prepares the given record by initializing a "record_components" dictionary.
+        buffer_flush(): Flushes the buffer by processing each record in the buffer.
+        record_compose(record): Processes a given record and yields buffered records if certain conditions are met.
+        process_line(jsonl_file): Processes a JSON Lines (jsonl) file and yields records.
+        record_resolve_id(record): Resolves and updates the 'id' field in the given record.
+        produce_records(filename): Reads the JSONL content saved from `job.job_retrieve_result()` line-by-line to avoid OOM.
+        read_file(filename, remove_file): Reads a file and produces records from it.
+    """
+
+    query: ShopifyBulkQuery
+    parent_stream_name: Optional[str] = None
+    parent_stream_cursor: Optional[str] = None
+
+    # default buffer
+    buffer: List[MutableMapping[str, Any]] = field(init=False, default_factory=list)
+
+    def __post_init__(self) -> None:
+        self.composition: Optional[Mapping[str, Any]] = self.query.record_composition
+        self.record_process_components: Optional[Callable[[MutableMapping], MutableMapping]] = self.query.record_process_components
+        self.components: List[str] = self.composition.get("record_components", []) if self.composition else []
+        # We track the parent state for BULK substreams outside of the main CDK methods,
+        # to be able to update the moving parent state when there are no substream records to emit.
+        self._parent_stream_cursor_value: Optional[str | int] = None
+        # how many records composed
+        self.record_composed: int = 0
+
+    @cached_property
+    def tools(self) -> BulkTools:
+        return BulkTools()
+
+    @cached_property
+    def has_parent_stream(self) -> bool:
+        return True if self.parent_stream_name and self.parent_stream_cursor else False
+
+    @cached_property
+    def parent_cursor_key(self) -> Optional[str]:
+        if self.has_parent_stream:
+            return f"{self.parent_stream_name}_{self.parent_stream_cursor}"
+
+    @staticmethod
+    def check_type(record: Mapping[str, Any], types: Union[List[str], str]) -> bool:
+        """
+        Check if the record's type matches the given type(s).
+
+        Args:
+            record (Mapping[str, Any]): The record to check, expected to have a "__typename" key.
+            types (Union[List[str], str]): The type(s) to check against. Can be a single type (str) or a list of types (List[str]).
+
+        Returns:
+            bool: True if the record's type matches one of the given types, False otherwise.
+        """
+
+        record_type = record.get("__typename")
+        if isinstance(types, list):
+            return any(record_type == t for t in types)
+        else:
+            return record_type == types
+
+    def _parse_parent_state_value(self, value: str | int) -> str | int:
+        """
+        Parses the parent state value and converts it to the appropriate format.
+
+        If the value is a string, it converts it to RFC 3339 datetime format using the `_datetime_str_to_rfc3339` method.
+        If the value is an integer, it returns the value as is.
+
+        Args:
+            value (str | int): The parent state value to be parsed.
+
+        Returns:
+            str | int: The parsed parent state value in the appropriate format.
+        """
+
+        if isinstance(value, str):
+            return self.tools._datetime_str_to_rfc3339(value)
+        elif isinstance(value, int):
+            return value
+
+    def _set_parent_state_value(self, value: str | int) -> None:
+        """
+        Sets the parent state value by parsing the provided value and updating the
+        parent stream cursor value. If the parent stream cursor value is already set,
+        it updates it to the maximum of the current and parsed values.
+
+        Args:
+            value (str | int): The value to be parsed and set as the parent state value.
+        """
+
+        parsed_value = self._parse_parent_state_value(value)
+        if not self._parent_stream_cursor_value:
+            self._parent_stream_cursor_value = parsed_value
+        else:
+            self._parent_stream_cursor_value = max(self._parent_stream_cursor_value, parsed_value)
+
+    def _track_parent_cursor(self, record: MutableMapping[str, Any]) -> None:
+        """
+        Tracks the cursor value from the parent stream if it exists and updates the parent state.
+
+        Args:
+            record (MutableMapping[str, Any]): The record from which to extract the parent cursor value.
+
+        Returns:
+            None
+        """
+
+        if self.has_parent_stream:
+            cursor_value: Optional[str | int] = record.get(self.parent_cursor_key, None)
+            if cursor_value:
+                self._set_parent_state_value(cursor_value)
+
+    def get_parent_stream_state(self) -> Optional[Union[str, Mapping[str, Any]]]:
+        """
+        Retrieve the state of the parent stream if it exists.
+
+        Returns:
+            Optional[Union[str, Mapping[str, Any]]]: A dictionary containing the parent stream cursor and its value
+            if the parent stream exists and has a cursor value, otherwise None.
+        """
+
+        if self.has_parent_stream and self._parent_stream_cursor_value:
+            return {self.parent_stream_cursor: self._parent_stream_cursor_value}
+
+    def record_new(self, record: MutableMapping[str, Any]) -> None:
+        """
+        Processes a new record by preparing it, removing the "__typename" key, and appending it to the buffer.
+
+        Args:
+            record (MutableMapping[str, Any]): The record to be processed.
+        """
+
+        record = self.component_prepare(record)
+        record.pop("__typename")
+        self.buffer.append(record)
+
+    def record_new_component(self, record: MutableMapping[str, Any]) -> None:
+        """
+        Processes a new record by extracting its component type and adding it to the appropriate
+        placeholder in the components list.
+
+        Args:
+            record (MutableMapping[str, Any]): The record to be processed.
+            It is expected to contain a "__typename" key which indicates the component type.
+        """
+
+        component = record.get("__typename")
+        record.pop("__typename")
+        # add component to its placeholder in the components list
+        self.buffer[-1]["record_components"][component].append(record)
+
+    def component_prepare(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+        """
+        Prepares the given record by initializing a "record_components" dictionary.
+
+        If the instance has components, this method will add a "record_components" key to the record,
+        with each component as a key and an empty list as its value.
+
+        Args:
+            record (MutableMapping[str, Any]): The record to be prepared.
+
+        Returns:
+            MutableMapping[str, Any]: The updated record with initialized "record_components".
+        """
+
+        if self.components:
+            record["record_components"] = {}
+            for component in self.components:
+                record["record_components"][component] = []
+        return record
+
+    def buffer_flush(self) -> Iterable[Mapping[str, Any]]:
+        """
+        Flushes the buffer by processing each record in the buffer.
+
+        For each record in the buffer:
+        - Tracks the parent state using `_track_parent_cursor`.
+        - Resolves the record ID from `str` to `int` using `record_resolve_id`.
+        - Processes record components using `record_process_components`.
+
+        Yields:
+            Iterable[Mapping[str, Any]]: Processed records from the buffer.
+
+        After processing, the buffer is cleared.
+        """
+
+        if len(self.buffer) > 0:
+            for record in self.buffer:
+                # track the parent state
+                self._track_parent_cursor(record)
+                # resolve id from `str` to `int`
+                record = self.record_resolve_id(record)
+                # process record components
+                yield from self.record_process_components(record)
+            # clean the buffer
+            self.buffer.clear()
+
+    def record_compose(self, record: Mapping[str, Any]) -> Optional[Iterable[MutableMapping[str, Any]]]:
+        """
+        Processes a given record and yields buffered records if certain conditions are met.
+
+        Args:
+            record (Mapping[str, Any]): The record to be processed.
+
+        Returns:
+            Optional[Iterable[MutableMapping[str, Any]]]: An iterable of buffered records if conditions are met, otherwise None.
+
+        The method performs the following steps:
+        1. Checks if the record matches the type specified in the "new_record" composition.
+           - If it matches, it yields any buffered records from previous iterations and registers the new record.
+        2. Checks if the record matches any of the specified components.
+           - If it matches, it registers the new component record.
+
+        Step 1: register the new record by it's `__typename`
+        Step 2: check for `components` by their `__typename` and add to the placeholder
+        Step 3: repeat until the `<END_OF_FILE>`.
+        """
+
+        if self.check_type(record, self.composition.get("new_record")):
+            # emit from previous iteration, if present
+            yield from self.buffer_flush()
+            # register the record
+            self.record_new(record)
+        # components check
+        elif self.check_type(record, self.components):
+            self.record_new_component(record)
+
+    def process_line(self, jsonl_file: TextIOWrapper) -> Iterable[MutableMapping[str, Any]]:
+        """
+        Processes a JSON Lines (jsonl) file and yields records.
+
+        Args:
+            jsonl_file (TextIOWrapper): A file-like object containing JSON Lines data.
+
+        Yields:
+            Iterable[MutableMapping[str, Any]]: An iterable of dictionaries representing the processed records.
+
+        The method reads each line from the provided jsonl_file. It exits the loop when it encounters the <end_of_file> marker.
+        For non-empty lines, it parses the JSON content and yields the resulting records. Finally, it emits any remaining
+        records in the buffer.
+        """
+
+        for line in jsonl_file:
+            if line == END_OF_FILE:
+                break
+            elif line != "":
+                yield from self.record_compose(loads(line))
+
+        # emit what's left in the buffer, typically last record
+        yield from self.buffer_flush()
+
+    def record_resolve_id(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+        """
+        Resolves and updates the 'id' field in the given record.
+
+        This method extracts the 'id' from the record, checks if it is a string,
+        and if so, assigns it to 'admin_graphql_api_id' in the record. It then
+        resolves the string 'id' to an integer using the 'resolve_str_id' method
+        from the 'tools' attribute and updates the 'id' field in the record.
+
+        Args:
+            record (MutableMapping[str, Any]): The record containing the 'id' field to be resolved.
+            Example:
+                { "Id": "gid://shopify/Order/19435458986123"}
+
+        Returns:
+            MutableMapping[str, Any]: The updated record with the resolved 'id' field.
+            Example:
+                { "id": 19435458986123, "admin_graphql_api_id": "gid://shopify/Order/19435458986123"}
+        """
+
+        id = record.get("id")
+        if id and isinstance(id, str):
+            record["admin_graphql_api_id"] = id
+            record["id"] = self.tools.resolve_str_id(id)
+        return record
+
+    def produce_records(self, filename: str) -> Iterable[MutableMapping[str, Any]]:
+        """
+        Produce records from a JSON Lines (jsonl) file.
+
+        This method reads a JSON Lines file, processes each line, converts the field names to snake_case,
+        and yields each processed record. It also keeps track of the number of records processed.
+
+        Args:
+            filename (str): The path to the JSON Lines file.
+
+        Yields:
+            MutableMapping[str, Any]: A dictionary representing a processed record with field names in snake_case.
+        """
+
+        with open(filename, "r") as jsonl_file:
+            # reset the counter
+            self.record_composed = 0
+
+            for record in self.process_line(jsonl_file):
+                yield self.tools.fields_names_to_snake_case(record)
+                self.record_composed += 1
+
+    def read_file(self, filename: str, remove_file: Optional[bool] = True) -> Iterable[Mapping[str, Any]]:
+        """
+        Read the JSONL content saved from `job.job_retrieve_result()` line-by-line to avoid OOM.
+
+        Args:
+            filename (str): The name of the file to read.
+            remove_file (Optional[bool]): Flag indicating whether to remove the file after reading. Defaults to True.
+
+            Example:
+                Note: typically the `filename` is taken from the `result_url` string provided in the response.
+
+                `bulk-4039263649981.jsonl` :
+                    - the `4039263649981` is the `id` of the COMPLETED BULK Jobw with `result_url`
+
+        Yields:
+            Iterable[Mapping[str, Any]]: An iterable of records produced from the file.
+
+        Raises:
+            ShopifyBulkExceptions.BulkRecordProduceError: If an error occurs while producing records from the file.
+
+        Logs:
+            Logs an info message if the file removal fails.
+        """
+
+        try:
+            # produce records from saved result
+            yield from self.produce_records(filename)
+        except Exception as e:
+            raise ShopifyBulkExceptions.BulkRecordProduceError(
+                f"An error occured while producing records from BULK Job result. Trace: {repr(e)}.",
+            )
+        finally:
+            # removing the tmp file, if requested
+            if remove_file and filename:
+                try:
+                    remove(filename)
+                except Exception as e:
+                    LOGGER.info(f"Failed to remove the `tmp job result` file, the file doen't exist. Details: {repr(e)}.")
+                    pass

{airbyte_source_shopify-2.5.6 → airbyte_source_shopify-2.5.8}/source_shopify/shopify_graphql/bulk/retry.py

@@ -1,10 +1,11 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 
-import logging
 from functools import wraps
 from time import sleep
 from typing import Any, Callable, Final, Optional, Tuple, Type
 
+from source_shopify.utils import LOGGER
+
 from .exceptions import ShopifyBulkExceptions
 
 BULK_RETRY_ERRORS: Final[Tuple] = (
@@ -13,11 +14,10 @@ BULK_RETRY_ERRORS: Final[Tuple] = (
 )
 
 
-def bulk_retry_on_exception(logger: logging.Logger, more_exceptions: Optional[Tuple[Type[Exception], ...]] = None) -> Callable:
+def bulk_retry_on_exception(more_exceptions: Optional[Tuple[Type[Exception], ...]] = None) -> Callable:
     """
     A decorator to retry a function when specified exceptions are raised.
 
-    :param logger: Number of times to retry.
     :param more_exceptions: A tuple of exception types to catch.
     """
 
@@ -31,26 +31,26 @@ def bulk_retry_on_exception(logger: logging.Logger, more_exceptions: Optional[Tu
                 except BULK_RETRY_ERRORS or more_exceptions as ex:
                     current_retries += 1
                     if current_retries > self._job_max_retries:
-                        logger.error("Exceeded retry limit. Giving up.")
+                        LOGGER.error("Exceeded retry limit. Giving up.")
                         raise
                     else:
-                        logger.warning(
+                        LOGGER.warning(
                             f"Stream `{self.http_client.name}`: {ex}. Retrying {current_retries}/{self._job_max_retries} after {self._job_backoff_time} seconds."
                         )
                         sleep(self._job_backoff_time)
                 except ShopifyBulkExceptions.BulkJobCreationFailedConcurrentError:
                     if self._concurrent_attempt == self._concurrent_max_retry:
                         message = f"The BULK Job couldn't be created at this time, since another job is running."
-                        logger.error(message)
+                        LOGGER.error(message)
                         raise ShopifyBulkExceptions.BulkJobConcurrentError(message)
 
                     self._concurrent_attempt += 1
-                    logger.warning(
+                    LOGGER.warning(
                         f"Stream: `{self.http_client.name}`, the BULK concurrency limit has reached. Waiting {self._concurrent_interval} sec before retry, attempt: {self._concurrent_attempt}.",
                     )
                     sleep(self._concurrent_interval)
                 except ShopifyBulkExceptions.BulkJobRedirectToOtherShopError:
-                    logger.warning(
+                    LOGGER.warning(
                         f"Stream: `{self.http_client.name}`, the `shop name` differs from the provided in `input configuration`. Switching to the `{self._tools.shop_name_from_url(self.base_url)}`.",
                     )
 

{airbyte_source_shopify-2.5.6 → airbyte_source_shopify-2.5.8}/source_shopify/streams/base_streams.py

@@ -644,12 +644,14 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream):
         self.job_manager: ShopifyBulkManager = ShopifyBulkManager(
             http_client=self.bulk_http_client,
             base_url=f"{self.url_base}{self.path()}",
-            query=self.bulk_query(config, self.parent_stream_name, self.parent_stream_cursor),
+            query=self.bulk_query(config, self.parent_stream_query_cursor_alias),
             job_termination_threshold=float(config.get("job_termination_threshold", 3600)),
             # overide the default job slice size, if provided (it's auto-adjusted, later on)
             job_size=config.get("bulk_window_in_days", 30.0),
             # provide the job checkpoint interval value, default value is 200k lines collected
             job_checkpoint_interval=config.get("job_checkpoint_interval", 200_000),
+            parent_stream_name=self.parent_stream_name,
+            parent_stream_cursor=self.parent_stream_cursor,
         )
 
     @property
@@ -670,20 +672,25 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream):
         """
         return self.parent_stream_class(self.config) if self.parent_stream_class else None
 
-    @property
+    @cached_property
     def parent_stream_name(self) -> Optional[str]:
         """
         Returns the parent stream name, if the substream has a `parent_stream_class` dependency.
         """
         return self.parent_stream.name if self.parent_stream_class else None
 
-    @property
+    @cached_property
     def parent_stream_cursor(self) -> Optional[str]:
         """
         Returns the parent stream cursor, if the substream has a `parent_stream_class` dependency.
         """
         return self.parent_stream.cursor_field if self.parent_stream_class else None
 
+    @cached_property
+    def parent_stream_query_cursor_alias(self) -> Optional[str]:
+        if self.parent_stream_name and self.parent_stream_cursor:
+            return f"{self.parent_stream_name}_{self.parent_stream_cursor}"
+
     @property
     @abstractmethod
     def bulk_query(self) -> ShopifyBulkQuery:
@@ -713,7 +720,9 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream):
         return None
 
     def get_updated_state(
-        self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]
+        self,
+        current_stream_state: MutableMapping[str, Any],
+        latest_record: Mapping[str, Any],
     ) -> MutableMapping[str, Any]:
         """UPDATING THE STATE OBJECT:
         Stream: CustomerAddress
@@ -728,29 +737,40 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream):
                 }
             }
         """
+
         updated_state = super().get_updated_state(current_stream_state, latest_record)
+
         if self.parent_stream_class:
-            parent_state = latest_record.get(self.parent_stream.name, {})
-            parent_state_value = (
-                parent_state.get(self.parent_stream.cursor_field) if parent_state else latest_record.get(self.parent_stream.cursor_field)
-            )
+            # the default way of getting the parent stream state is to use the value from the RecordProducer,
+            # since the parent record could be present but no substream's records are present to emit,
+            # the parent state is tracked for each parent record processed, thus updated regardless having substream records.
+            tracked_parent_state = self.job_manager.record_producer.get_parent_stream_state()
+            # fallback to the record level to search for the parent cursor or use the stream cursor value
+            parent_state = tracked_parent_state if tracked_parent_state else self._get_parent_state_from_record(latest_record)
             # add parent_stream_state to `updated_state`
-            updated_state[self.parent_stream.name] = {self.parent_stream.cursor_field: parent_state_value}
+            updated_state[self.parent_stream_name] = parent_state
+
         return updated_state
 
+    def _get_parent_state_from_record(self, latest_record: Mapping[str, Any]) -> MutableMapping[str, Any]:
+        parent_state = latest_record.get(self.parent_stream_name, {})
+        parent_state_value = parent_state.get(self.parent_stream_cursor) if parent_state else latest_record.get(self.parent_stream_cursor)
+        parent_state[self.parent_stream_cursor] = parent_state_value
+        return parent_state
+
     def _get_stream_cursor_value(self, stream_state: Optional[Mapping[str, Any]] = None) -> Optional[str]:
         if stream_state:
             return stream_state.get(self.cursor_field, self.default_state_comparison_value)
         else:
             return self.config.get("start_date")
 
-    def get_stream_state_value(self, stream_state: Optional[Mapping[str, Any]] = None) -> Optional[str]:
+    def _get_stream_state_value(self, stream_state: Optional[Mapping[str, Any]] = None) -> Optional[str]:
         if stream_state:
             if self.parent_stream_class:
                 # get parent stream state from the stream_state object.
-                parent_state = stream_state.get(self.parent_stream.name, {})
+                parent_state = stream_state.get(self.parent_stream_name, {})
                 if parent_state:
-                    return parent_state.get(self.parent_stream.cursor_field, self.default_state_comparison_value)
+                    return parent_state.get(self.parent_stream_cursor, self.default_state_comparison_value)
                 else:
                     # use the streams cursor value, if no parent state available
                     return self._get_stream_cursor_value(stream_state)
@@ -760,9 +780,9 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream):
         else:
             return self.config.get("start_date")
 
-    def get_state_value(self, stream_state: Optional[Mapping[str, Any]] = None) -> Optional[Union[str, int]]:
+    def _get_state_value(self, stream_state: Optional[Mapping[str, Any]] = None) -> Optional[Union[str, int]]:
         if stream_state:
-            return self.get_stream_state_value(stream_state)
+            return self._get_stream_state_value(stream_state)
         else:
             # for majority of cases we fallback to start_date, otherwise.
             return self.config.get("start_date")
@@ -785,7 +805,7 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream):
     @stream_state_cache.cache_stream_state
     def stream_slices(self, stream_state: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
         if self.filter_field:
-            state = self.get_state_value(stream_state)
+            state = self._get_state_value(stream_state)
             start = pdm.parse(state)
             end = pdm.now()
             while start < end:

{airbyte_source_shopify-2.5.6 → airbyte_source_shopify-2.5.8}/source_shopify/utils.py

@@ -7,13 +7,16 @@ import enum
 import logging
 from functools import wraps
 from time import sleep
-from typing import Any, Callable, Dict, List, Mapping, Optional
+from typing import Any, Callable, Dict, Final, List, Mapping, Optional
 
 import requests
 from airbyte_cdk.models import FailureType
 from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction
 from airbyte_cdk.utils import AirbyteTracedException
 
+# default logger instance
+LOGGER: Final[logging.Logger] = logging.getLogger("airbyte")
+
 
 class ShopifyNonRetryableErrors:
     """Holds the errors classification and messaging scenarios."""
@@ -112,8 +115,6 @@ class ShopifyRateLimiter:
     on_mid_load: float = 1.5
     on_high_load: float = 5.0
 
-    logger = logging.getLogger("airbyte")
-
     log_message_count = 0
     log_message_frequency = 3
 
@@ -124,7 +125,7 @@ class ShopifyRateLimiter:
         if ShopifyRateLimiter.log_message_count < ShopifyRateLimiter.log_message_frequency:
             ShopifyRateLimiter.log_message_count += 1
         else:
-            ShopifyRateLimiter.logger.info(message)
+            LOGGER.info(message)
             ShopifyRateLimiter.log_message_count = 0
 
     def get_response_from_args(*args) -> Optional[requests.Response]:
@@ -138,8 +139,8 @@ class ShopifyRateLimiter:
         Define wait_time based on load conditions.
 
         :: load - represents how close we are to being throttled
-                - 0.5 is half way through our allowance
-                - 1 indicates that all of the allowance is used and the api will start rejecting calls
+            - 0.5 is half way through our allowance
+            - 1 indicates that all of the allowance is used and the api will start rejecting calls
         :: threshold - is the % cutoff for the rate_limits/load
         :: wait_time - time to wait between each request in seconds
 

airbyte_source_shopify-2.5.6/source_shopify/shopify_graphql/bulk/record.py

@@ -1,157 +0,0 @@
-#
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-#
-
-
-import logging
-from dataclasses import dataclass, field
-from io import TextIOWrapper
-from json import loads
-from os import remove
-from typing import Any, Callable, Final, Iterable, List, Mapping, MutableMapping, Optional, Union
-
-from .exceptions import ShopifyBulkExceptions
-from .query import ShopifyBulkQuery
-from .tools import END_OF_FILE, BulkTools
-
-
-@dataclass
-class ShopifyBulkRecord:
-    query: ShopifyBulkQuery
-
-    # default buffer
-    buffer: List[MutableMapping[str, Any]] = field(init=False, default_factory=list)
-
-    # default logger
-    logger: Final[logging.Logger] = logging.getLogger("airbyte")
-
-    def __post_init__(self) -> None:
-        self.composition: Optional[Mapping[str, Any]] = self.query.record_composition
-        self.record_process_components: Optional[Callable[[MutableMapping], MutableMapping]] = self.query.record_process_components
-        self.components: List[str] = self.composition.get("record_components", []) if self.composition else []
-        # how many records composed
-        self.record_composed: int = 0
-
-    @property
-    def tools(self) -> BulkTools:
-        return BulkTools()
-
-    @staticmethod
-    def check_type(record: Mapping[str, Any], types: Union[List[str], str]) -> bool:
-        record_type = record.get("__typename")
-        if isinstance(types, list):
-            return any(record_type == t for t in types)
-        else:
-            return record_type == types
-
-    def record_new(self, record: MutableMapping[str, Any]) -> None:
-        record = self.component_prepare(record)
-        record.pop("__typename")
-        self.buffer.append(record)
-
-    def record_new_component(self, record: MutableMapping[str, Any]) -> None:
-        component = record.get("__typename")
-        record.pop("__typename")
-        # add component to its placeholder in the components list
-        self.buffer[-1]["record_components"][component].append(record)
-
-    def component_prepare(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
-        if self.components:
-            record["record_components"] = {}
-            for component in self.components:
-                record["record_components"][component] = []
-        return record
-
-    def buffer_flush(self) -> Iterable[Mapping[str, Any]]:
-        if len(self.buffer) > 0:
-            for record in self.buffer:
-                # resolve id from `str` to `int`
-                record = self.record_resolve_id(record)
-                # process record components
-                yield from self.record_process_components(record)
-            # clean the buffer
-            self.buffer.clear()
-
-    def record_compose(self, record: Mapping[str, Any]) -> Optional[Iterable[MutableMapping[str, Any]]]:
-        """
-        Step 1: register the new record by it's `__typename`
-        Step 2: check for `components` by their `__typename` and add to the placeholder
-        Step 3: repeat until the `<END_OF_FILE>`.
-        """
-        if self.check_type(record, self.composition.get("new_record")):
-            # emit from previous iteration, if present
-            yield from self.buffer_flush()
-            # register the record
-            self.record_new(record)
-        # components check
-        elif self.check_type(record, self.components):
-            self.record_new_component(record)
-
-    def process_line(self, jsonl_file: TextIOWrapper) -> Iterable[MutableMapping[str, Any]]:
-        # process the json lines
-        for line in jsonl_file:
-            # we exit from the loop when receive <end_of_file> (file ends)
-            if line == END_OF_FILE:
-                break
-            elif line != "":
-                yield from self.record_compose(loads(line))
-
-        # emit what's left in the buffer, typically last record
-        yield from self.buffer_flush()
-
-    def record_resolve_id(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
-        """
-        The ids are fetched in the format of: " gid://shopify/Order/<Id> "
-        Input:
-            { "Id": "gid://shopify/Order/19435458986123"}
-        We need to extract the actual id from the string instead.
-        Output:
-            { "id": 19435458986123, "admin_graphql_api_id": "gid://shopify/Order/19435458986123"}
-        """
-        # save the actual api id to the `admin_graphql_api_id`
-        # while resolving the `id` in `record_resolve_id`,
-        # we re-assign the original id like `"gid://shopify/Order/19435458986123"`,
-        # into `admin_graphql_api_id` have the ability to identify the record oigin correctly in subsequent actions.
-        # IF NOT `id` field is provided by the query results, we should return composed record `as is`.
-        id = record.get("id")
-        if id and isinstance(id, str):
-            record["admin_graphql_api_id"] = id
-            # extracting the int(id) and reassign
-            record["id"] = self.tools.resolve_str_id(id)
-        return record
-
-    def produce_records(self, filename: str) -> Iterable[MutableMapping[str, Any]]:
-        """
-        Read the JSONL content saved from `job.job_retrieve_result()` line-by-line to avoid OOM.
-        The filename example: `bulk-4039263649981.jsonl`,
-            where `4039263649981` is the `id` of the COMPLETED BULK Jobw with `result_url`.
-            Note: typically the `filename` is taken from the `result_url` string provided in the response.
-        """
-
-        with open(filename, "r") as jsonl_file:
-            # reset the counter
-            self.record_composed = 0
-
-            for record in self.process_line(jsonl_file):
-                yield self.tools.fields_names_to_snake_case(record)
-                self.record_composed += 1
-
-    def read_file(self, filename: str, remove_file: Optional[bool] = True) -> Iterable[Mapping[str, Any]]:
-        try:
-            # produce records from saved result
-            yield from self.produce_records(filename)
-        except Exception as e:
-            raise ShopifyBulkExceptions.BulkRecordProduceError(
-                f"An error occured while producing records from BULK Job result. Trace: {repr(e)}.",
-            )
-        finally:
-            # removing the tmp file, if requested
-            if remove_file and filename:
-                try:
-                    remove(filename)
-                except Exception as e:
-                    self.logger.info(f"Failed to remove the `tmp job result` file, the file doen't exist. Details: {repr(e)}.")
-                    # we should pass here, if the file wasn't removed , it's either:
-                    # - doesn't exist
-                    # - will be dropped with the container shut down.
-                    pass