airbyte-source-shopify 3.3.0__tar.gz → 3.3.2__tar.gz

{airbyte_source_shopify-3.3.0 → airbyte_source_shopify-3.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-shopify
-Version: 3.3.0
+Version: 3.3.2
 Summary: Source CDK implementation for Shopify.
 Home-page: https://airbyte.com
 License: ELv2

{airbyte_source_shopify-3.3.0 → airbyte_source_shopify-3.3.2}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "3.3.0"
+version = "3.3.2"
 name = "airbyte-source-shopify"
 description = "Source CDK implementation for Shopify."
 authors = [ "Airbyte <contact@airbyte.io>",]

airbyte_source_shopify-3.3.2/source_shopify/shopify_graphql/bulk/external_sort.py

@@ -0,0 +1,140 @@
+#
+# Copyright (c) 2026 Airbyte, Inc., all rights reserved.
+#
+
+"""Disk-backed external merge sort used by the Shopify bulk GraphQL streams.
+
+Large bulk GraphQL result sets (for example, the metafield streams) used to be
+globally sorted in memory with Python's builtin `sorted()` before emission,
+which materialized every record at once and caused out-of-memory failures on
+large slices. `external_stable_sort` replaces that in-memory sort with a
+stable, streaming, disk-backed sort: records are accumulated into fixed-size
+chunks, each chunk is sorted in memory, chunks that exceed the in-memory
+budget are spilled as sorted runs to temp files, and the final output is a
+k-way heap merge across those runs.
+
+The sort is **stable**: when two records have equal sort keys, the record that
+arrived first in the input iterable is emitted first in the output. That
+matches the prior `sorted()` behavior and preserves the existing checkpoint
+and state semantics for the Shopify bulk streams.
+
+When the entire input fits in a single chunk, the implementation skips the
+spill-and-merge path entirely and sorts in place, so small slices pay no disk
+cost.
+"""
+
+import heapq
+import json
+import logging
+import os
+import tempfile
+from typing import Any, Callable, Iterable, Iterator, List, Mapping, Optional
+
+
+logger = logging.getLogger("airbyte")
+
+
+DEFAULT_SORT_CHUNK_SIZE = 50_000
+
+
+def external_stable_sort(
+    records: Iterable[Mapping[str, Any]],
+    key_fn: Callable[[Mapping[str, Any]], Any],
+    chunk_size: int = DEFAULT_SORT_CHUNK_SIZE,
+    tmp_dir: Optional[str] = None,
+) -> Iterator[Mapping[str, Any]]:
+    """Stably sort `records` by `key_fn` using bounded memory.
+
+    Records are consumed lazily from the input iterable in chunks of at most
+    `chunk_size`. Each chunk is sorted in memory by `(key_fn(record), ordinal)`
+    where `ordinal` is a monotonically increasing insertion counter that
+    guarantees stability. Chunks beyond the first are spilled to temporary
+    JSON-lines files and then streamed back through a k-way heap merge.
+
+    The output iterator is generator-based and yields records one at a time,
+    so callers can pipe results into downstream filters without materializing
+    the full sorted output in memory.
+
+    Temp files are always cleaned up — on normal completion, on generator
+    close, and on exceptions raised from downstream consumers.
+    """
+    if chunk_size <= 0:
+        raise ValueError(f"chunk_size must be a positive integer, got {chunk_size}")
+
+    chunk: List[Any] = []
+    run_paths: List[str] = []
+    ordinal = 0
+
+    try:
+        for record in records:
+            chunk.append((key_fn(record), ordinal, record))
+            ordinal += 1
+            if len(chunk) >= chunk_size:
+                _spill_chunk(chunk, tmp_dir, run_paths)
+                chunk = []
+
+        if not run_paths:
+            # Fast path: entire input fits in one in-memory chunk, so we skip
+            # the spill-and-merge pipeline entirely.
+            chunk.sort(key=_sort_key)
+            for _, _, record in chunk:
+                yield record
+            return
+
+        if chunk:
+            _spill_chunk(chunk, tmp_dir, run_paths)
+
+        logger.info(f"External sort spilled {len(run_paths)} run file(s) covering {ordinal} record(s); merging.")
+        yield from _merge_runs(run_paths)
+    finally:
+        _cleanup_runs(run_paths)
+
+
+def _sort_key(item: Any) -> Any:
+    # item is (key, ordinal, record). Tuples compare lexicographically, so the
+    # ordinal acts as a stable secondary key for equal primary keys.
+    return (item[0], item[1])
+
+
+def _spill_chunk(chunk: List[Any], tmp_dir: Optional[str], run_paths: List[str]) -> None:
+    """Sort `chunk` in place and write it as a new sorted run file.
+
+    The newly created path is appended to `run_paths` **before** any write
+    begins, so a failure in `json.dumps` or `fh.write` still leaves the path
+    visible to the outer `finally` block for cleanup. Each line is a
+    JSON-encoded `[key, ordinal, record]` triple, deserialized during merge.
+    """
+    chunk.sort(key=_sort_key)
+    fd, path = tempfile.mkstemp(prefix="shopify_bulk_sort_", suffix=".jsonl", dir=tmp_dir)
+    run_paths.append(path)
+    with os.fdopen(fd, "w", encoding="utf-8") as fh:
+        for key, ord_value, record in chunk:
+            fh.write(json.dumps([key, ord_value, record], separators=(",", ":")) + "\n")
+
+
+def _iter_run(path: str) -> Iterator[Any]:
+    """Yield `(key, ordinal, record)` triples from a spilled run file."""
+    with open(path, "r", encoding="utf-8") as fh:
+        for line in fh:
+            key, ord_value, record = json.loads(line)
+            yield (key, ord_value, record)
+
+
+def _merge_runs(run_paths: List[str]) -> Iterator[Mapping[str, Any]]:
+    """Stream the merged output of all run files in sorted order."""
+    run_iters = [_iter_run(path) for path in run_paths]
+    for _, _, record in heapq.merge(*run_iters, key=_sort_key):
+        yield record
+
+
+def _cleanup_runs(run_paths: List[str]) -> None:
+    """Best-effort removal of spilled run files. Missing files are ignored."""
+    for path in run_paths:
+        try:
+            os.unlink(path)
+        except FileNotFoundError:
+            # Already removed (for example, by tempfile cleanup on container
+            # shutdown). Best-effort cleanup is intentionally silent here.
+            pass
+        except OSError as exc:
+            logger.warning(f"Failed to remove external-sort spill file `{path}`: {exc}")

{airbyte_source_shopify-3.3.0 → airbyte_source_shopify-3.3.2}/source_shopify/streams/base_streams.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+# Copyright (c) 2026 Airbyte, Inc., all rights reserved.
 #
 
 
@@ -14,6 +14,7 @@ import pendulum as pdm
 import requests
 from requests.exceptions import RequestException
 from source_shopify.http_request import ShopifyErrorHandler
+from source_shopify.shopify_graphql.bulk.external_sort import DEFAULT_SORT_CHUNK_SIZE, external_stable_sort
 from source_shopify.shopify_graphql.bulk.job import ShopifyBulkManager
 from source_shopify.shopify_graphql.bulk.query import DeliveryZoneList, ShopifyBulkQuery
 from source_shopify.transform import DataTypeEnforcer
@@ -835,22 +836,38 @@ class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream):
             # for the streams that don't support filtering
             yield {}
 
+    # Chunk size used by the disk-backed external sort in `sort_output_asc`.
+    # Exposed as a class attribute so tests and subclasses can tune it.
+    sort_chunk_size: int = DEFAULT_SORT_CHUNK_SIZE
+
     def sort_output_asc(self, non_sorted_records: Iterable[Mapping[str, Any]] = None) -> Iterable[Mapping[str, Any]]:
+        """Emit `non_sorted_records` in ascending `cursor_field` order.
+
+        The previous implementation relied on `sorted(...)` which fully
+        materialized the input iterable in memory. For large bulk GraphQL
+        slices (notably the metafield streams, where sorting is applied at the
+        parent-entity level but records are emitted at the child level) that
+        materialization triggered out-of-memory failures in production syncs.
+
+        This implementation preserves the prior ascending, stable ordering and
+        downstream checkpoint/state semantics while bounding peak memory via a
+        disk-backed external merge sort. Inputs that fit in a single in-memory
+        chunk take the fast path with no disk spill.
         """
-        Apply sorting for collected records, to guarantee the `ASC` output.
-        This handles the STATE and CHECKPOINTING correctly, for the `incremental` streams.
-        """
-        if non_sorted_records:
-            if not self.cursor_field:
-                yield from non_sorted_records
-            else:
-                yield from sorted(
-                    non_sorted_records,
-                    key=lambda x: x.get(self.cursor_field) if x.get(self.cursor_field) else self.default_state_comparison_value,
-                )
-        else:
+        if not non_sorted_records:
             # always return an empty iterable, if no records
             return []
+        if not self.cursor_field:
+            return non_sorted_records
+        return external_stable_sort(
+            non_sorted_records,
+            key_fn=self._sort_key_for_record,
+            chunk_size=self.sort_chunk_size,
+        )
+
+    def _sort_key_for_record(self, record: Mapping[str, Any]) -> Any:
+        value = record.get(self.cursor_field)
+        return value if value else self.default_state_comparison_value
 
     def read_records(
         self,

{airbyte_source_shopify-3.3.0 → airbyte_source_shopify-3.3.2}/source_shopify/utils.py

@@ -334,7 +334,7 @@ class LimitReducingErrorHandler(HttpStatusErrorHandler):
     """
 
     def __init__(self, max_retries: int, error_mapping: dict):
-        super().__init__(logger=None, max_retries=max_retries, error_mapping=error_mapping)
+        super().__init__(logger=LOGGER, max_retries=max_retries, error_mapping=error_mapping)
 
     def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
         if isinstance(response_or_exception, requests.Response):