airbyte-source-google-ads 4.1.3__tar.gz → 4.1.4__tar.gz

{airbyte_source_google_ads-4.1.3 → airbyte_source_google_ads-4.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: airbyte-source-google-ads
-Version: 4.1.3
+Version: 4.1.4
 Summary: Source implementation for Google Ads.
 Home-page: https://airbyte.com
 License: Elv2
@@ -11,7 +11,7 @@ Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: airbyte-cdk (>=7.4.1,<8.0.0)
+Requires-Dist: airbyte-cdk (>=7.5.1.post3.dev19705070276,<8.0.0)
 Requires-Dist: google-ads (==27.0.0)
 Requires-Dist: pendulum (<3.0.0)
 Requires-Dist: protobuf (==4.25.2)

{airbyte_source_google_ads-4.1.3 → airbyte_source_google_ads-4.1.4}/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "4.1.3"
+version = "4.1.4"
 name = "airbyte-source-google-ads"
 description = "Source implementation for Google Ads."
 authors = [ "Airbyte <contact@airbyte.io>",]
@@ -20,7 +20,7 @@ python = "^3.10,<3.12"
 google-ads = "==27.0.0"
 protobuf = "==4.25.2"
 pendulum = "<3.0.0"
-airbyte-cdk = "^7.4.1"
+airbyte-cdk = "^7.5.1.post3.dev19705070276"
 
 [tool.poetry.scripts]
 source-google-ads = "source_google_ads.run:run"

{airbyte_source_google_ads-4.1.3 → airbyte_source_google_ads-4.1.4}/source_google_ads/components.py

@@ -697,6 +697,7 @@ class CustomGAQueryHttpRequester(HttpRequester):
     def __post_init__(self, parameters: Mapping[str, Any]):
         super().__post_init__(parameters=parameters)
         self.query = GAQL.parse(parameters.get("query"))
+        self.stream_response = True
 
     @staticmethod
     def is_metrics_in_custom_query(query: GAQL) -> bool:
@@ -761,6 +762,17 @@ class CustomGAQueryHttpRequester(HttpRequester):
         return self.query[from_index + 4 :].strip()
 
 
+class CustomGAQueryClickViewHttpRequester(CustomGAQueryHttpRequester):
+    @staticmethod
+    def _insert_segments_date_expr(query: GAQL, start_date: str, end_date: str) -> GAQL:
+        if "segments.date" not in query.fields:
+            query = query.append_field("segments.date")
+        condition = f"segments.date ='{start_date}'"
+        if query.where:
+            return query.set_where(query.where + " AND " + condition)
+        return query.set_where(condition)
+
+
 @dataclass()
 class CustomGAQuerySchemaLoader(SchemaLoader):
     """

{airbyte_source_google_ads-4.1.3 → airbyte_source_google_ads-4.1.4}/source_google_ads/manifest.yaml

@@ -65,27 +65,15 @@ definitions:
       field_path:
         - results
 
-  cursor_paginator:
-    type: DefaultPaginator
-    pagination_strategy:
-      type: CursorPagination
-      cursor_value: "{{ response.get('nextPageToken', '') }}"
-      stop_condition: "{{ response.get('nextPageToken', '') is none }}"
-    page_token_option:
-      type: RequestOption
-      inject_into: body_json
-      field_path: ["page_token"]
-
   base_retriever:
     type: SimpleRetriever
     requester:
       $ref: "#/definitions/stream_requester"
     record_selector:
       $ref: "#/definitions/base_selector"
-    paginator:
-      $ref: "#/definitions/cursor_paginator"
     decoder:
-      type: JsonDecoder
+      type: CustomDecoder
+      class_name: "source_google_ads.components.GoogleAdsStreamingDecoder"
 
   stream_base:
     type: DeclarativeStream
@@ -199,7 +187,7 @@ definitions:
               $ref: "#/schemas"
           authenticator:
             $ref: "#/definitions/authenticator"
-          url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:search"
+          url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:searchStream"
           http_method: POST
           error_handler:
             $ref: "#/definitions/base_error_handler"
@@ -224,7 +212,7 @@ definitions:
     incremental_stream:
       $ref: "#/definitions/incremental_stream_base"
       $parameters:
-        url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['parent_slice']['customer_id'] }}/googleAds:search"
+        url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['parent_slice']['customer_id'] }}/googleAds:searchStream"
       retriever:
         type: CustomRetriever
         class_name: "source_google_ads.components.CriterionRetriever"
@@ -240,8 +228,6 @@ definitions:
           http_method: POST
           error_handler:
             $ref: "#/definitions/base_error_handler"
-        paginator:
-          $ref: "#/definitions/cursor_paginator"
         record_selector:
           type: RecordSelector
           extractor:
@@ -297,8 +283,6 @@ definitions:
         $ref: "#/definitions/base_requester"
         url_base: "https://googleads.googleapis.com/v20/customers:listAccessibleCustomers"
         http_method: GET
-      paginator:
-        type: NoPagination
       record_selector:
         extractor:
           type: CustomRecordExtractor
@@ -422,11 +406,6 @@ definitions:
             parent_key: "clientCustomer"
             partition_field: "customer_id"
             stream: "#/definitions/customer_client"
-      decoder:
-        type: CustomDecoder
-        class_name: "source_google_ads.components.GoogleAdsStreamingDecoder"
-      paginator:
-        type: NoPagination
     transformations:
       - type: CustomTransformation
         class_name: "source_google_ads.components.KeysToSnakeCaseGoogleAdsTransformation"
@@ -487,13 +466,6 @@ definitions:
 
   ad_group_ad_stream:
     $ref: "#/definitions/incremental_stream_base"
-    retriever:
-      $ref: "#/definitions/incremental_stream_base/retriever"
-      paginator:
-        type: NoPagination
-      decoder:
-        type: CustomDecoder
-        class_name: "source_google_ads.components.GoogleAdsStreamingDecoder"
     name: ad_group_ad
     primary_key:
       - ad_group.id
@@ -665,12 +637,10 @@ definitions:
             $ref: "#/schemas/click_view"
         authenticator:
           $ref: "#/definitions/authenticator"
-        url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:search"
+        url_base: "https://googleads.googleapis.com/v20/{{ stream_partition['customer_id'] }}/googleAds:searchStream"
         http_method: POST
         error_handler:
           $ref: "#/definitions/base_error_handler"
-      paginator:
-        $ref: "#/definitions/cursor_paginator"
     incremental_sync:
       type: DatetimeBasedCursor
       cursor_field: segments.date
@@ -859,8 +829,6 @@ definitions:
         error_handler:
           $ref: "#/definitions/base_error_handler"
         name: change_status
-      paginator:
-        $ref: "#/definitions/cursor_paginator"
       pagination_reset:
         type: PaginationReset
         action: SPLIT_USING_CURSOR
@@ -1030,14 +998,6 @@ definitions:
               )
             }}
           create_or_update: true
-        - type: ComponentMappingDefinition
-          field_path:
-            - retriever
-            - requester
-            - $parameters
-            - query
-          value: "{{ components_values.get('query', None) }}"
-          create_or_update: true
         - type: ComponentMappingDefinition
           field_path:
             - retriever
@@ -1085,6 +1045,130 @@ definitions:
               )
             }}
           create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - incremental_sync
+            - step
+          value: "P1D"
+          condition: >-
+            {{
+              (
+                (
+                  components_values.get('query', '').count('segments.date') == 1
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                )
+                or
+                (
+                  components_values.get('query', '').count('segments.date') == 2
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                  and (components_values.get('query') | regex_search('(?i)(\\bORDER\\s+BY\\b[\\s\\S]*?segments\\.date)'))
+                )
+              ) and components_values.get('query', '') | regex_search('(?i)(\\bFROM\\s+click_view\\b)')
+            }}
+          create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - incremental_sync
+            - start_datetime
+          value: >-
+            {
+              "type": "MinMaxDatetime",
+              "datetime": "{{ max(config.get('start_date', day_delta(-90, format='%Y-%m-%d')), day_delta(-90, format='%Y-%m-%d')) }}",
+              "datetime_format": "%Y-%m-%d"
+            }
+          condition: >-
+            {{
+              (
+                (
+                  components_values.get('query', '').count('segments.date') == 1
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                )
+                or
+                (
+                  components_values.get('query', '').count('segments.date') == 2
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                  and (components_values.get('query') | regex_search('(?i)(\\bORDER\\s+BY\\b[\\s\\S]*?segments\\.date)'))
+                )
+              ) and components_values.get('query', '') | regex_search('(?i)(\\bFROM\\s+click_view\\b)')
+            }}
+          create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - incremental_sync
+            - end_datetime
+          value: >-
+            {
+              "type": "MinMaxDatetime",
+              "datetime": "{{ format_datetime((str_to_datetime(config.get('end_date')) if config.get('end_date') else now_utc()) + duration('P1D'), '%Y-%m-%d') }}",
+              "datetime_format": "%Y-%m-%d"
+            }
+          condition: >-
+            {{
+              (
+                (
+                  components_values.get('query', '').count('segments.date') == 1
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                )
+                or
+                (
+                  components_values.get('query', '').count('segments.date') == 2
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                  and (components_values.get('query') | regex_search('(?i)(\\bORDER\\s+BY\\b[\\s\\S]*?segments\\.date)'))
+                )
+              ) and components_values.get('query', '') | regex_search('(?i)(\\bFROM\\s+click_view\\b)')
+            }}
+          create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - incremental_sync
+            - cursor_granularity
+          value: P1D
+          condition: >-
+            {{
+              (
+                (
+                  components_values.get('query', '').count('segments.date') == 1
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                )
+                or
+                (
+                  components_values.get('query', '').count('segments.date') == 2
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                  and (components_values.get('query') | regex_search('(?i)(\\bORDER\\s+BY\\b[\\s\\S]*?segments\\.date)'))
+                )
+              ) and components_values.get('query', '') | regex_search('(?i)(\\bFROM\\s+click_view\\b)')
+            }}
+          create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - retriever
+            - requester
+            - class_name
+          value: "source_google_ads.components.CustomGAQueryClickViewHttpRequester"
+          condition: >-
+            {{
+              (
+                (
+                  components_values.get('query', '').count('segments.date') == 1
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                )
+                or
+                (
+                  components_values.get('query', '').count('segments.date') == 2
+                  and (components_values.get('query') | regex_search('(?i)(\\bSELECT\\b[\\s\\S]*?segments\\.date[\\s\\S]*?\\bFROM\\b)'))
+                  and (components_values.get('query') | regex_search('(?i)(\\bORDER\\s+BY\\b[\\s\\S]*?segments\\.date)'))
+                )
+              ) and components_values.get('query', '') | regex_search('(?i)(\\bFROM\\s+click_view\\b)')
+            }}
+          create_or_update: true
+        - type: ComponentMappingDefinition
+          field_path:
+            - retriever
+            - requester
+            - $parameters
+            - query
+          value: "{{ components_values.get('query', None) }}"
+          create_or_update: true
         - type: ComponentMappingDefinition
           field_path:
             - retriever