airbyte-source-google-ads 4.1.0rc7.dev202510171521__tar.gz → 4.1.0rc8__tar.gz

{airbyte_source_google_ads-4.1.0rc7.dev202510171521 → airbyte_source_google_ads-4.1.0rc8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-google-ads
-Version: 4.1.0rc7.dev202510171521
+Version: 4.1.0rc8
 Summary: Source implementation for Google Ads.
 Home-page: https://airbyte.com
 License: Elv2
@@ -11,7 +11,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=7.0.5,<8.0.0)
+Requires-Dist: airbyte-cdk (>=7.4.1,<8.0.0)
 Requires-Dist: google-ads (==27.0.0)
 Requires-Dist: pendulum (<3.0.0)
 Requires-Dist: protobuf (==4.25.2)

{airbyte_source_google_ads-4.1.0rc7.dev202510171521 → airbyte_source_google_ads-4.1.0rc8}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "4.1.0-rc.7.dev.202510171521"
+version = "4.1.0-rc.8"
 name = "airbyte-source-google-ads"
 description = "Source implementation for Google Ads."
 authors = [ "Airbyte <contact@airbyte.io>",]
@@ -20,7 +20,7 @@ python = "^3.10,<3.12"
 google-ads = "==27.0.0"
 protobuf = "==4.25.2"
 pendulum = "<3.0.0"
-airbyte-cdk = "^7.0.5"
+airbyte-cdk = "^7.4.1"
 
 [tool.poetry.scripts]
 source-google-ads = "source_google_ads.run:run"

{airbyte_source_google_ads-4.1.0rc7.dev202510171521 → airbyte_source_google_ads-4.1.0rc8}/source_google_ads/components.py

@@ -2,18 +2,21 @@
 # Copyright (c) 2025 Airbyte, Inc., all rights reserved.
 #
 
+import io
 import json
 import logging
 import re
 import threading
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from itertools import groupby
-from typing import Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union
+from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union
 
 import anyascii
 import requests
 
 from airbyte_cdk import AirbyteTracedException, FailureType, InterpolatedString
+from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import JsonParser
+from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
 from airbyte_cdk.sources.declarative.migrations.state_migration import StateMigration
@@ -285,6 +288,10 @@ class GoogleAdsHttpRequester(HttpRequester):
 
     schema_loader: InlineSchemaLoader = None
 
+    def __post_init__(self, parameters: Mapping[str, Any]) -> None:
+        super().__post_init__(parameters)
+        self.stream_response = True
+
     def get_request_body_json(
         self,
         *,
@@ -408,54 +415,6 @@ class KeysToSnakeCaseGoogleAdsTransformation(RecordTransformation):
         return "_".join(token.lower() for token in tokens)
 
 
-@dataclass
-class ChangeStatusRetriever(SimpleRetriever):
-    """
-    Retrieves change status records from the Google Ads API.
-    ChangeStatus stream requires custom retriever because Google Ads API requires limit for this stream to be set to 10,000.
-    When the number of records exceeds this limit, we need to adjust the start date to the last record's cursor.
-    """
-
-    QUERY_LIMIT = 10000
-    cursor_field: str = "change_status.last_change_date_time"
-
-    def _read_pages(
-        self,
-        records_generator_fn: Callable[[Optional[Mapping]], Iterable[Record]],
-        stream_state: StreamState,
-        stream_slice: StreamSlice,
-    ) -> Iterable[Record]:
-        """
-        Since this stream doesn’t support “real” pagination, we treat each HTTP
-        call as a slice defined by a start_date / end_date. If we hit the
-        QUERY_LIMIT exactly, we assume there may be more data at the end of that
-        slice, so we bump start_date forward to the last-record cursor and retry.
-        """
-        while True:
-            record_count = 0
-            last_record = None
-            response = self._fetch_next_page(stream_state, stream_slice)
-
-            # Yield everything we got
-            for rec in records_generator_fn(response):
-                record_count += 1
-                last_record = rec
-                yield rec
-
-            if record_count < self.QUERY_LIMIT:
-                break
-
-            # Update the stream slice start time to the last record's cursor
-            last_cursor = last_record[self.cursor_field]
-            cursor_slice = stream_slice.cursor_slice
-            cursor_slice["start_time"] = last_cursor
-            stream_slice = StreamSlice(
-                partition=stream_slice.partition,
-                cursor_slice=cursor_slice,
-                extra_fields=stream_slice.extra_fields,
-            )
-
-
 @dataclass
 class ChangeStatusRequester(GoogleAdsHttpRequester):
     CURSOR_FIELD: str = "change_status.last_change_date_time"
@@ -501,7 +460,6 @@ class CriterionRetriever(SimpleRetriever):
     def _read_pages(
         self,
         records_generator_fn: Callable[[Optional[Mapping]], Iterable[Record]],
-        stream_state: StreamState,
         stream_slice: StreamSlice,
     ) -> Iterable[Record]:
         """
@@ -531,6 +489,7 @@ class CriterionRetriever(SimpleRetriever):
                             self.primary_key[0]: _id,
                             "deleted_at": ts,
                         },
+                        associated_slice=stream_slice,
                         stream_name=self.name,
                     )
                 else:
@@ -550,7 +509,7 @@ class CriterionRetriever(SimpleRetriever):
                 cursor_slice=stream_slice.cursor_slice,
                 extra_fields={"change_status.last_change_date_time": updated_times},
             )
-            response = self._fetch_next_page(stream_state, new_slice)
+            response = self._fetch_next_page(new_slice)
             for rec in records_generator_fn(response):
                 # attach timestamp from ChangeStatus
                 rec.data[self.cursor_field] = time_map.get(rec.data.get(self.primary_key[0]))
@@ -618,13 +577,26 @@ class GoogleAdsCriterionParentStateMigration(StateMigration):
     """
 
     def should_migrate(self, stream_state: Mapping[str, Any]) -> bool:
-        return stream_state and "parent_state" not in stream_state
+        return stream_state and not stream_state.get("parent_state")
 
     def migrate(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]:
         if not self.should_migrate(stream_state):
             return stream_state
 
-        return {"parent_state": stream_state}
+        return {"parent_state": {"change_status": stream_state}}
+
+
+class GoogleAdsGlobalStateMigration(StateMigration):
+    """
+    Migrates global state to include use_global_cursor key. Previously legacy GlobalSubstreamCursor was used.
+    """
+
+    def should_migrate(self, stream_state: Mapping[str, Any]) -> bool:
+        return stream_state and not stream_state.get("use_global_cursor")
+
+    def migrate(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]:
+        stream_state["use_global_cursor"] = True
+        return stream_state
 
 
 @dataclass(repr=False, eq=False, frozen=True)
@@ -889,3 +861,212 @@ class CustomGAQuerySchemaLoader(SchemaLoader):
                 internal_message=f"The provided query is invalid: {query}. Please refer to the Google Ads API documentation for the correct syntax: https://developers.google.com/google-ads/api/fields/v20/overview and test validate your query using the Google Ads Query Builder: https://developers.google.com/google-ads/api/fields/v20/query_validator",
                 message=f"The provided query is invalid: {query}. Please refer to the Google Ads API documentation for the correct syntax: https://developers.google.com/google-ads/api/fields/v20/overview and test validate your query using the Google Ads Query Builder: https://developers.google.com/google-ads/api/fields/v20/query_validator",
             )
+
+
+@dataclass
+class StringParseState:
+    inside_string: bool = False
+    escape_next_character: bool = False
+    collected_string_chars: List[str] = field(default_factory=list)
+    last_parsed_key: Optional[str] = None
+
+
+@dataclass
+class TopLevelObjectState:
+    depth: int = 0
+
+
+@dataclass
+class ResultsArrayState:
+    inside_results_array: bool = False
+    array_nesting_depth: int = 0
+    expecting_results_array_start: bool = False
+
+
+@dataclass
+class RecordParseState:
+    inside_record: bool = False
+    record_text_buffer: List[str] = field(default_factory=list)
+    record_nesting_depth: int = 0
+
+
+@dataclass
+class GoogleAdsStreamingDecoder(Decoder):
+    """
+    JSON streaming decoder optimized for Google Ads API responses.
+
+    Uses a fast JSON parse when the full payload fits within max_direct_decode_bytes;
+    otherwise streams records incrementally from the `results` array.
+    Ensures truncated or structurally invalid JSON is detected and reported.
+    """
+
+    chunk_size: int = 5 * 1024 * 1024  # 5 MB
+    # Fast-path threshold: if whole body < 20 MB, decode with json.loads
+    max_direct_decode_bytes: int = 20 * 1024 * 1024  # 20 MB
+
+    def __post_init__(self):
+        self.parser = JsonParser()
+
+    def is_stream_response(self) -> bool:
+        return True
+
+    def decode(self, response: requests.Response) -> Generator[MutableMapping[str, Any], None, None]:
+        data, complete = self._buffer_up_to_limit(response)
+        if complete:
+            yield from self.parser.parse(io.BytesIO(data))
+            return
+
+        records_batch: List[Dict[str, Any]] = []
+        for record in self._parse_records_from_stream(data):
+            records_batch.append(record)
+            if len(records_batch) >= 100:
+                yield {"results": records_batch}
+                records_batch = []
+
+        if records_batch:
+            yield {"results": records_batch}
+
+    def _buffer_up_to_limit(self, response: requests.Response) -> Tuple[Union[bytes, Iterable[bytes]], bool]:
+        buf = bytearray()
+        response_stream = response.iter_content(chunk_size=self.chunk_size)
+
+        while chunk := next(response_stream, None):
+            buf.extend(chunk)
+            if len(buf) >= self.max_direct_decode_bytes:
+                return (self._chain_prefix_and_stream(bytes(buf), response_stream), False)
+        return (bytes(buf), True)
+
+    @staticmethod
+    def _chain_prefix_and_stream(prefix: bytes, rest_stream: Iterable[bytes]) -> Iterable[bytes]:
+        yield prefix
+        yield from rest_stream
+
+    def _parse_records_from_stream(self, byte_iter: Iterable[bytes], encoding: str = "utf-8") -> Generator[Dict[str, Any], None, None]:
+        string_state = StringParseState()
+        results_state = ResultsArrayState()
+        record_state = RecordParseState()
+        top_level_state = TopLevelObjectState()
+
+        for chunk in byte_iter:
+            for char in chunk.decode(encoding, errors="replace"):
+                self._append_to_current_record_if_any(char, record_state)
+
+                if self._update_string_state(char, string_state):
+                    continue
+
+                # Track outer braces only outside results array
+                if not results_state.inside_results_array:
+                    if char == "{":
+                        top_level_state.depth += 1
+                    elif char == "}":
+                        top_level_state.depth = max(0, top_level_state.depth - 1)
+
+                if not results_state.inside_results_array:
+                    self._detect_results_array(char, string_state, results_state)
+                    continue
+
+                record = self._parse_record_structure(char, results_state, record_state)
+                if record is not None:
+                    yield record
+
+        # EOF validation
+        if (
+            string_state.inside_string
+            or record_state.inside_record
+            or record_state.record_nesting_depth != 0
+            or results_state.inside_results_array
+            or results_state.array_nesting_depth != 0
+            or top_level_state.depth != 0
+        ):
+            raise AirbyteTracedException(
+                message="Response JSON stream ended prematurely and is incomplete.",
+                internal_message=(
+                    "Detected truncated JSON stream: one or more structural elements were not fully closed before the response ended."
+                ),
+                failure_type=FailureType.system_error,
+            )
+
+    def _update_string_state(self, char: str, state: StringParseState) -> bool:
+        """Return True if char was handled as part of string parsing."""
+        if state.inside_string:
+            if state.escape_next_character:
+                state.escape_next_character = False
+                return True
+            if char == "\\":
+                state.escape_next_character = True
+                return True
+            if char == '"':
+                state.inside_string = False
+                state.last_parsed_key = "".join(state.collected_string_chars)
+                state.collected_string_chars.clear()
+                return True
+            state.collected_string_chars.append(char)
+            return True
+
+        if char == '"':
+            state.inside_string = True
+            state.collected_string_chars.clear()
+            return True
+
+        return False
+
+    def _detect_results_array(self, char: str, string_state: StringParseState, results_state: ResultsArrayState) -> None:
+        if char == ":" and string_state.last_parsed_key == "results":
+            results_state.expecting_results_array_start = True
+        elif char == "[" and results_state.expecting_results_array_start:
+            results_state.inside_results_array = True
+            results_state.array_nesting_depth = 1
+            results_state.expecting_results_array_start = False
+
+    def _parse_record_structure(
+        self, char: str, results_state: ResultsArrayState, record_state: RecordParseState
+    ) -> Optional[Dict[str, Any]]:
+        if char == "{":
+            if record_state.inside_record:
+                record_state.record_nesting_depth += 1
+            else:
+                self._start_record(record_state)
+            return None
+
+        if char == "}":
+            if record_state.inside_record:
+                record_state.record_nesting_depth -= 1
+                if record_state.record_nesting_depth == 0:
+                    return self._finish_record(record_state)
+            return None
+
+        if char == "[":
+            if record_state.inside_record:
+                record_state.record_nesting_depth += 1
+            else:
+                results_state.array_nesting_depth += 1
+            return None
+
+        if char == "]":
+            if record_state.inside_record:
+                record_state.record_nesting_depth -= 1
+            else:
+                results_state.array_nesting_depth -= 1
+                if results_state.array_nesting_depth == 0:
+                    results_state.inside_results_array = False
+
+        return None
+
+    @staticmethod
+    def _append_to_current_record_if_any(char: str, record_state: RecordParseState):
+        if record_state.inside_record:
+            record_state.record_text_buffer.append(char)
+
+    @staticmethod
+    def _start_record(record_state: RecordParseState):
+        record_state.inside_record = True
+        record_state.record_text_buffer = ["{"]
+        record_state.record_nesting_depth = 1
+
+    @staticmethod
+    def _finish_record(record_state: RecordParseState) -> Optional[Dict[str, Any]]:
+        text = "".join(record_state.record_text_buffer).strip()
+        record_state.inside_record = False
+        record_state.record_text_buffer.clear()
+        record_state.record_nesting_depth = 0
+        return json.loads(text) if text else None

{airbyte_source_google_ads-4.1.0rc7.dev202510171521 → airbyte_source_google_ads-4.1.0rc8}/source_google_ads/manifest.yaml

@@ -38,7 +38,7 @@ definitions:
         $ref: "#/schemas"
     authenticator:
       $ref: "#/definitions/authenticator"
-    url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:search"
+    url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:searchStream"
     http_method: POST
     error_handler:
       $ref: "#/definitions/base_error_handler"
@@ -56,7 +56,7 @@ definitions:
         action: IGNORE
         http_codes:
           - 403
-        error_message_contains: "The customer account can\\'t be accessed because it is not yet enabled or has been deactivated."
+  #        error_message_contains: "The customer account can\\'t be accessed because it is not yet enabled or has been deactivated."
 
   base_selector:
     type: RecordSelector
@@ -285,6 +285,8 @@ definitions:
       state_migrations:
         - type: CustomStateMigration
           class_name: source_google_ads.components.GoogleAdsCriterionParentStateMigration
+        - type: CustomStateMigration
+          class_name: source_google_ads.components.GoogleAdsGlobalStateMigration
 
   accessible_accounts:
     $ref: "#/definitions/stream_base"
@@ -401,7 +403,7 @@ definitions:
         class_name: "source_google_ads.components.CustomGAQueryHttpRequester"
         authenticator:
           $ref: "#/definitions/authenticator"
-        url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:search"
+        url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:searchStream"
         http_method: POST
         error_handler:
           $ref: "#/definitions/base_error_handler"
@@ -420,8 +422,11 @@ definitions:
             parent_key: "clientCustomer"
             partition_field: "customer_id"
             stream: "#/definitions/customer_client"
+      decoder:
+        type: CustomDecoder
+        class_name: "source_google_ads.components.GoogleAdsStreamingDecoder"
       paginator:
-        $ref: "#/definitions/cursor_paginator"
+        type: NoPagination
     transformations:
       - type: CustomTransformation
         class_name: "source_google_ads.components.KeysToSnakeCaseGoogleAdsTransformation"
@@ -482,6 +487,13 @@ definitions:
 
   ad_group_ad_stream:
     $ref: "#/definitions/incremental_stream_base"
+    retriever:
+      $ref: "#/definitions/incremental_stream_base/retriever"
+      paginator:
+        type: NoPagination
+      decoder:
+        type: CustomDecoder
+        class_name: "source_google_ads.components.GoogleAdsStreamingDecoder"
     name: ad_group_ad
     primary_key:
       - ad_group.id
@@ -833,8 +845,7 @@ definitions:
     $parameters:
       url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:search"
     retriever:
-      type: CustomRetriever
-      class_name: "source_google_ads.components.ChangeStatusRetriever"
+      type: SimpleRetriever
       requester:
         type: CustomRequester
         class_name: "source_google_ads.components.ChangeStatusRequester"
@@ -850,6 +861,12 @@ definitions:
         name: change_status
       paginator:
         $ref: "#/definitions/cursor_paginator"
+      pagination_reset:
+        type: PaginationReset
+        action: SPLIT_USING_CURSOR
+        limits:
+          type: PaginationResetLimits
+          number_of_records: 10000
       record_selector:
         type: RecordSelector
         $parameters: