airbyte-source-google-ads 4.1.3__py3-none-any.whl → 4.1.4__py3-none-any.whl

{airbyte_source_google_ads-4.1.3.dist-info → airbyte_source_google_ads-4.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-google-ads
-Version: 4.1.3
+Version: 4.1.4
 Summary: Source implementation for Google Ads.
 Home-page: https://airbyte.com
 License: Elv2
@@ -11,7 +11,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=7.4.1,<8.0.0)
+Requires-Dist: airbyte-cdk (>=7.5.1.post3.dev19705070276,<8.0.0)
 Requires-Dist: google-ads (==27.0.0)
 Requires-Dist: pendulum (<3.0.0)
 Requires-Dist: protobuf (==4.25.2)

{airbyte_source_google_ads-4.1.3.dist-info → airbyte_source_google_ads-4.1.4.dist-info}/RECORD

@@ -1,8 +1,8 @@
 source_google_ads/__init__.py,sha256=Nlo5H6LlaSgg7tx_LyqMIy3MXiAagfk3izZ9o44VvSE,1201
-source_google_ads/components.py,sha256=_Jku4_NHRg4OzYSmtk23H0LfYo0bbqfx1navUoUztug,40970
+source_google_ads/components.py,sha256=hzd6hgBuE0HvHVO71r3gdHsbM5sApQKpzW_QWKEAv8g,41479
 source_google_ads/config_migrations.py,sha256=oBi_qNqBpLS8GNCaIOBo0stNdYuyqVl6lkrhdXRwMX8,4405
 source_google_ads/google_ads.py,sha256=cxS18tz0fFJjmIhlhFQ3Zvu2K8bhDtmsl1kFeO7nNhk,11595
-source_google_ads/manifest.yaml,sha256=ICDxfH8-pT78PfQwU2h_mgCVkv-sSzuQBR4BdnVZEpU,219342
+source_google_ads/manifest.yaml,sha256=4LzKMj23yS8Vo8M6tfxv77vIp58pIIEN26x3lz3rm1M,223987
 source_google_ads/models.py,sha256=ZmdS3z_2roaEQgV2Mx1CDm33MztpQ66SfHDzP8XwZog,1658
 source_google_ads/run.py,sha256=ydIyq_vSNV5Z4mJYnsO5GyNDsLDd0qibBsq6wnvuFAo,2002
 source_google_ads/schemas/customer_client.json,sha256=oThcyUDO1yWpxtWPWdoAFqTXEIweF8N4q6mRI73Q6yU,984
@@ -11,7 +11,7 @@ source_google_ads/source.py,sha256=hz5ep6stMWHNvD73PIF_7bjnee49sY9YHHjaYNAPnOQ,1
 source_google_ads/spec.json,sha256=8hbc7smbaffIkYCkX2BYJLB9kgaH8vYKCg-H0y1FvUs,7810
 source_google_ads/streams.py,sha256=FB-DNJlXhjQADptT-wrv3iGWoliyRuvDuHGeqiN9HsY,13349
 source_google_ads/utils.py,sha256=-KpgGv2W8WueXvGRC3xbVreDl5-5-vU9OwzC5SZDKVc,21409
-airbyte_source_google_ads-4.1.3.dist-info/METADATA,sha256=r83LKIZKX9R2YF7o84QV4ut_pGm1AQpVG-lWUay9ZNE,5360
-airbyte_source_google_ads-4.1.3.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-airbyte_source_google_ads-4.1.3.dist-info/entry_points.txt,sha256=pP4Llir93XGkHFDZfXXxK7qOWo9_U1ssCJToyxEUB4w,63
-airbyte_source_google_ads-4.1.3.dist-info/RECORD,,
+airbyte_source_google_ads-4.1.4.dist-info/METADATA,sha256=K-SPkHaTXmmXpZa6kWRHhMwrK8wqBQG4LNvm3ldwLsI,5381
+airbyte_source_google_ads-4.1.4.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+airbyte_source_google_ads-4.1.4.dist-info/entry_points.txt,sha256=pP4Llir93XGkHFDZfXXxK7qOWo9_U1ssCJToyxEUB4w,63
+airbyte_source_google_ads-4.1.4.dist-info/RECORD,,

source_google_ads/components.py

@@ -697,6 +697,7 @@ class CustomGAQueryHttpRequester(HttpRequester):
     def __post_init__(self, parameters: Mapping[str, Any]):
         super().__post_init__(parameters=parameters)
         self.query = GAQL.parse(parameters.get("query"))
+        self.stream_response = True
 
     @staticmethod
     def is_metrics_in_custom_query(query: GAQL) -> bool:
@@ -761,6 +762,17 @@ class CustomGAQueryHttpRequester(HttpRequester):
         return self.query[from_index + 4 :].strip()
 
 
+class CustomGAQueryClickViewHttpRequester(CustomGAQueryHttpRequester):
+    @staticmethod
+    def _insert_segments_date_expr(query: GAQL, start_date: str, end_date: str) -> GAQL:
+        if "segments.date" not in query.fields:
+            query = query.append_field("segments.date")
+        condition = f"segments.date ='{start_date}'"
+        if query.where:
+            return query.set_where(query.where + " AND " + condition)
+        return query.set_where(condition)
+
+
 @dataclass()
 class CustomGAQuerySchemaLoader(SchemaLoader):
     """

source_google_ads/manifest.yaml

@@ -65,27 +65,15 @@ definitions:
       field_path:
         - results
 
-  cursor_paginator:
-    type: DefaultPaginator
-    pagination_strategy:
-      type: CursorPagination
-      cursor_value: "{{ response.get('nextPageToken', '') }}"
-      stop_condition: "{{ response.get('nextPageToken', '') is none }}"
-    page_token_option:
-      type: RequestOption
-      inject_into: body_json
-      field_path: ["page_token"]
-
   base_retriever:
     type: SimpleRetriever
     requester:
       $ref: "#/definitions/stream_requester"
     record_selector:
       $ref: "#/definitions/base_selector"
-    paginator:
-      $ref: "#/definitions/cursor_paginator"
     decoder:
-      type: JsonDecoder
+      type: CustomDecoder
+      class_name: "source_google_ads.components.GoogleAdsStreamingDecoder"
 
   stream_base:
     type: DeclarativeStream
@@ -199,7 +187,7 @@ definitions:
               $ref: "#/schemas"
           authenticator:
             $ref: "#/definitions/authenticator"
-          url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:search"
+          url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:searchStream"
           http_method: POST
           error_handler:
             $ref: "#/definitions/base_error_handler"
@@ -224,7 +212,7 @@ definitions:
     incremental_stream:
       $ref: "#/definitions/incremental_stream_base"
       $parameters:
-        url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['parent_slice']['customer_id'] }}/googleAds:search"
+        url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['parent_slice']['customer_id'] }}/googleAds:searchStream"
       retriever:
         type: CustomRetriever
         class_name: "source_google_ads.components.CriterionRetriever"
@@ -240,8 +228,6 @@ definitions:
           http_method: POST
           error_handler:
             $ref: "#/definitions/base_error_handler"
-        paginator:
-          $ref: "#/definitions/cursor_paginator"
         record_selector:
           type: RecordSelector
           extractor:
@@ -297,8 +283,6 @@ definitions:
         $ref: "#/definitions/base_requester"
         url_base: "https://googleads.googleapis.com/v20/customers:listAccessibleCustomers"
         http_method: GET
-      paginator:
-        type: NoPagination
       record_selector:
         extractor:
           type: CustomRecordExtractor
@@ -422,11 +406,6 @@ definitions:
             parent_key: "clientCustomer"
             partition_field: "customer_id"
             stream: "#/definitions/customer_client"
-      decoder:
-        type: CustomDecoder
-        class_name: "source_google_ads.components.GoogleAdsStreamingDecoder"
-      paginator:
-        type: NoPagination
     transformations:
       - type: CustomTransformation
         class_name: "source_google_ads.components.KeysToSnakeCaseGoogleAdsTransformation"
@@ -487,13 +466,6 @@ definitions:
 
   ad_group_ad_stream:
     $ref: "#/definitions/incremental_stream_base"
-    retriever:
-      $ref: "#/definitions/incremental_stream_base/retriever"
-      paginator:
-        type: NoPagination
-      decoder:
-        type: CustomDecoder
-        class_name: "source_google_ads.components.GoogleAdsStreamingDecoder"
     name: ad_group_ad
     primary_key:
       - ad_group.id
@@ -665,12 +637,10 @@ definitions:
             $ref: "#/schemas/click_view"
         authenticator:
           $ref: "#/definitions/authenticator"
-        url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:search"
+        url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:searchStream"
         http_method: POST
         error_handler:
           $ref: "#/definitions/base_error_handler"
-      paginator:
-        $ref: "#/definitions/cursor_paginator"
     incremental_sync:
       type: DatetimeBasedCursor
       cursor_field: segments.date
@@ -859,8 +829,6 @@ definitions:
         error_handler:
           $ref: "#/definitions/base_error_handler"
         name: change_status
-      paginator:
-        $ref: "#/definitions/cursor_paginator"
       pagination_reset:
         type: PaginationReset
         action: SPLIT_USING_CURSOR
@@ -1030,14 +998,6 @@ definitions:
               )
             }}
           create_or_update: true
-        - type: ComponentMappingDefinition
-          field_path:
-            - retriever
-            - requester
-            - $parameters
-            - query
-          value: "{{ components_values.get('query', None) }}"
-          create_or_update: true
         - type: ComponentMappingDefinition
           field_path:
             - retriever
@@ -1085,6 +1045,130 @@ definitions:
               )
             }}
           create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - incremental_sync
+            - step
+          value: "P1D"
+          condition: >-
+            {{
+              (
+                (
+                  components_values.get('query', '').count('segments.date') == 1
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                )
+                or
+                (
+                  components_values.get('query', '').count('segments.date') == 2
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                  and (components_values.get('query') | regex_search('(?i)(\\bORDER\\s+BY\\b[\\s\\S]*?segments\\.date)'))
+                )
+              ) and components_values.get('query', '') | regex_search('(?i)(\\bFROM\\s+click_view\\b)')
+            }}
+          create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - incremental_sync
+            - start_datetime
+          value: >-
+            {
+              "type": "MinMaxDatetime",
+              "datetime": "{{ max(config.get('start_date', day_delta(-90, format='%Y-%m-%d')), day_delta(-90, format='%Y-%m-%d')) }}",
+              "datetime_format": "%Y-%m-%d"
+            }
+          condition: >-
+            {{
+              (
+                (
+                  components_values.get('query', '').count('segments.date') == 1
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                )
+                or
+                (
+                  components_values.get('query', '').count('segments.date') == 2
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                  and (components_values.get('query') | regex_search('(?i)(\\bORDER\\s+BY\\b[\\s\\S]*?segments\\.date)'))
+                )
+              ) and components_values.get('query', '') | regex_search('(?i)(\\bFROM\\s+click_view\\b)')
+            }}
+          create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - incremental_sync
+            - end_datetime
+          value: >-
+            {
+              "type": "MinMaxDatetime",
+              "datetime": "{{ format_datetime((str_to_datetime(config.get('end_date')) if config.get('end_date') else now_utc()) + duration('P1D'), '%Y-%m-%d') }}",
+              "datetime_format": "%Y-%m-%d"
+            }
+          condition: >-
+            {{
+              (
+                (
+                  components_values.get('query', '').count('segments.date') == 1
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                )
+                or
+                (
+                  components_values.get('query', '').count('segments.date') == 2
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                  and (components_values.get('query') | regex_search('(?i)(\\bORDER\\s+BY\\b[\\s\\S]*?segments\\.date)'))
+                )
+              ) and components_values.get('query', '') | regex_search('(?i)(\\bFROM\\s+click_view\\b)')
+            }}
+          create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - incremental_sync
+            - cursor_granularity
+          value: P1D
+          condition: >-
+            {{
+              (
+                (
+                  components_values.get('query', '').count('segments.date') == 1
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                )
+                or
+                (
+                  components_values.get('query', '').count('segments.date') == 2
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                  and (components_values.get('query') | regex_search('(?i)(\\bORDER\\s+BY\\b[\\s\\S]*?segments\\.date)'))
+                )
+              ) and components_values.get('query', '') | regex_search('(?i)(\\bFROM\\s+click_view\\b)')
+            }}
+          create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - retriever
+            - requester
+            - class_name
+          value: "source_google_ads.components.CustomGAQueryClickViewHttpRequester"
+          condition: >-
+            {{
+              (
+                (
+                  components_values.get('query', '').count('segments.date') == 1
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                )
+                or
+                (
+                  components_values.get('query', '').count('segments.date') == 2
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                  and (components_values.get('query') | regex_search('(?i)(\\bORDER\\s+BY\\b[\\s\\S]*?segments\\.date)'))
+                )
+              ) and components_values.get('query', '') | regex_search('(?i)(\\bFROM\\s+click_view\\b)')
+            }}
+          create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - retriever
+            - requester
+            - $parameters
+            - query
+          value: "{{ components_values.get('query', None) }}"
+          create_or_update: true
         - type: ComponentMappingDefinition
           field_path:
             - retriever